From 77bc76193ce18a3cfc6ba1471ecc34ef34c36454 Mon Sep 17 00:00:00 2001 From: Austin Osagie Date: Thu, 27 Jun 2024 14:56:56 -0700 Subject: [PATCH 01/30] Implement IAMF audio decoder --- starboard/android/shared/BUILD.gn | 4 + .../shared/media_is_audio_supported.cc | 3 +- .../shared/platform_configuration/BUILD.gn | 5 + .../shared/player_components_factory.h | 8 + starboard/linux/shared/BUILD.gn | 4 + .../linux/shared/media_is_audio_supported.cc | 4 + .../linux/shared/player_components_factory.cc | 6 + .../shared/platform_configuration/BUILD.gn | 4 + .../shared/libiamf/iamf_audio_decoder.cc | 332 ++++++++++++++++++ starboard/shared/libiamf/iamf_audio_decoder.h | 85 +++++ .../shared/libiamf/iamf_config_reader.cc | 184 ++++++++++ starboard/shared/libiamf/iamf_config_reader.h | 75 ++++ 12 files changed, 713 insertions(+), 1 deletion(-) create mode 100644 starboard/shared/libiamf/iamf_audio_decoder.cc create mode 100644 starboard/shared/libiamf/iamf_audio_decoder.h create mode 100644 starboard/shared/libiamf/iamf_config_reader.cc create mode 100644 starboard/shared/libiamf/iamf_config_reader.h diff --git a/starboard/android/shared/BUILD.gn b/starboard/android/shared/BUILD.gn index fa80f7b325e6..10b1508d0b1c 100644 --- a/starboard/android/shared/BUILD.gn +++ b/starboard/android/shared/BUILD.gn @@ -75,6 +75,10 @@ static_library("starboard_platform") { "//starboard/shared/libevent/socket_waiter_wait.cc", "//starboard/shared/libevent/socket_waiter_wait_timed.cc", "//starboard/shared/libevent/socket_waiter_wake_up.cc", + "//starboard/shared/libiamf/iamf_audio_decoder.cc", + "//starboard/shared/libiamf/iamf_audio_decoder.h", + "//starboard/shared/libiamf/iamf_config_reader.cc", + "//starboard/shared/libiamf/iamf_config_reader.h", "//starboard/shared/linux/byte_swap.cc", "//starboard/shared/linux/cpu_features_get.cc", "//starboard/shared/linux/memory_get_stack_bounds.cc", diff --git a/starboard/android/shared/media_is_audio_supported.cc b/starboard/android/shared/media_is_audio_supported.cc index 023cbfc5f8fb..bb60f6124ab4 100644 --- a/starboard/android/shared/media_is_audio_supported.cc +++ b/starboard/android/shared/media_is_audio_supported.cc @@ -56,7 +56,8 @@ bool SbMediaIsAudioSupported(SbMediaAudioCodec audio_codec, // Android uses a libopus based opus decoder for clear content, or a platform // opus decoder for encrypted content, if available. - if (audio_codec == kSbMediaAudioCodecOpus) { + if (audio_codec == kSbMediaAudioCodecOpus || + audio_codec == kSbMediaAudioCodecIamf) { return true; } diff --git a/starboard/android/shared/platform_configuration/BUILD.gn b/starboard/android/shared/platform_configuration/BUILD.gn index f638914483e8..785a7ad48f78 100644 --- a/starboard/android/shared/platform_configuration/BUILD.gn +++ b/starboard/android/shared/platform_configuration/BUILD.gn @@ -175,6 +175,7 @@ config("platform_configuration") { "-Wl,--wrap=readdir_r", ] } + configs += [ ":libraries" ] } config("size") { @@ -217,3 +218,7 @@ config("pedantic_warnings") { "-Wno-unused-parameter", ] } + +config("libraries") { + libs = [ "//third_party/libiamf/code/android/libiamf.a" ] +} diff --git a/starboard/android/shared/player_components_factory.h b/starboard/android/shared/player_components_factory.h index d6060d8406e4..b3b766d7be1a 100644 --- a/starboard/android/shared/player_components_factory.h +++ b/starboard/android/shared/player_components_factory.h @@ -34,6 +34,7 @@ #include "starboard/common/media.h" #include "starboard/common/ref_counted.h" #include "starboard/media.h" +#include "starboard/shared/libiamf/iamf_audio_decoder.h" #include "starboard/shared/opus/opus_audio_decoder.h" #include "starboard/shared/starboard/media/media_util.h" #include "starboard/shared/starboard/media/mime_type.h" @@ -179,6 +180,7 @@ class PlayerComponentsPassthrough class PlayerComponentsFactory : public starboard::shared::starboard::player:: filter::PlayerComponents::Factory { typedef starboard::shared::starboard::media::MimeType MimeType; + typedef starboard::shared::libiamf::IamfAudioDecoder IamfAudioDecoder; typedef starboard::shared::opus::OpusAudioDecoder OpusAudioDecoder; typedef starboard::shared::starboard::player::filter::AdaptiveAudioDecoder AdaptiveAudioDecoder; @@ -444,6 +446,12 @@ class PlayerComponentsFactory : public starboard::shared::starboard::player:: return std::unique_ptr( std::move(audio_decoder_impl)); } + } else if (audio_stream_info.codec == kSbMediaAudioCodecIamf) { + scoped_ptr audio_decoder_impl( + new IamfAudioDecoder(audio_stream_info)); + if (audio_decoder_impl->is_valid()) { + return audio_decoder_impl.PassAs(); + } } else { SB_LOG(ERROR) << "Unsupported audio codec " << audio_stream_info.codec; diff --git a/starboard/linux/shared/BUILD.gn b/starboard/linux/shared/BUILD.gn index a5896b38ffb8..ee78adeee885 100644 --- a/starboard/linux/shared/BUILD.gn +++ b/starboard/linux/shared/BUILD.gn @@ -125,6 +125,10 @@ static_library("starboard_platform_sources") { "//starboard/shared/libfdkaac/fdk_aac_audio_decoder.h", "//starboard/shared/libfdkaac/libfdkaac_library_loader.cc", "//starboard/shared/libfdkaac/libfdkaac_library_loader.h", + "//starboard/shared/libiamf/iamf_audio_decoder.cc", + "//starboard/shared/libiamf/iamf_audio_decoder.h", + "//starboard/shared/libiamf/iamf_config_reader.cc", + "//starboard/shared/libiamf/iamf_config_reader.h", "//starboard/shared/libvpx/vpx_video_decoder.cc", "//starboard/shared/libvpx/vpx_video_decoder.h", "//starboard/shared/linux/byte_swap.cc", diff --git a/starboard/linux/shared/media_is_audio_supported.cc b/starboard/linux/shared/media_is_audio_supported.cc index c1b03d18d851..e0d17062e521 100644 --- a/starboard/linux/shared/media_is_audio_supported.cc +++ b/starboard/linux/shared/media_is_audio_supported.cc @@ -32,6 +32,10 @@ bool SbMediaIsAudioSupported(SbMediaAudioCodec audio_codec, return bitrate <= kSbMediaMaxAudioBitrateInBitsPerSecond; } + if (audio_codec == kSbMediaAudioCodecIamf) { + return bitrate <= kSbMediaMaxAudioBitrateInBitsPerSecond; + } + if (audio_codec == kSbMediaAudioCodecAc3 || audio_codec == kSbMediaAudioCodecEac3) { #if SB_API_VERSION < 15 diff --git a/starboard/linux/shared/player_components_factory.cc b/starboard/linux/shared/player_components_factory.cc index 67d98405b87d..4b82253d958b 100644 --- a/starboard/linux/shared/player_components_factory.cc +++ b/starboard/linux/shared/player_components_factory.cc @@ -25,6 +25,7 @@ #include "starboard/shared/libde265/de265_video_decoder.h" #include "starboard/shared/libfdkaac/fdk_aac_audio_decoder.h" #include "starboard/shared/libfdkaac/libfdkaac_library_loader.h" +#include "starboard/shared/libiamf/iamf_audio_decoder.h" #include "starboard/shared/libvpx/vpx_video_decoder.h" #include "starboard/shared/openh264/openh264_library_loader.h" #include "starboard/shared/openh264/openh264_video_decoder.h" @@ -68,6 +69,7 @@ class PlayerComponentsFactory : public PlayerComponents::Factory { SB_DCHECK(audio_renderer_sink); typedef ::starboard::shared::ffmpeg::AudioDecoder FfmpegAudioDecoder; + typedef ::starboard::shared::libiamf::IamfAudioDecoder IamfAudioDecoder; typedef ::starboard::shared::opus::OpusAudioDecoder OpusAudioDecoder; typedef ::starboard::shared::libfdkaac::FdkAacAudioDecoder FdkAacAudioDecoder; @@ -86,6 +88,10 @@ class PlayerComponentsFactory : public PlayerComponents::Factory { libfdkaac::LibfdkaacHandle::GetHandle()->IsLoaded()) { SB_LOG(INFO) << "Playing audio using FdkAacAudioDecoder."; return std::unique_ptr(new FdkAacAudioDecoder()); + } else if (audio_stream_info.codec == kSbMediaAudioCodecIamf) { + SB_LOG(INFO) << "Playing audio using IamfAudioDecoder."; + return std::unique_ptr( + new IamfAudioDecoder(audio_stream_info)); } else { std::unique_ptr audio_decoder_impl( FfmpegAudioDecoder::Create(audio_stream_info)); diff --git a/starboard/linux/x64x11/shared/platform_configuration/BUILD.gn b/starboard/linux/x64x11/shared/platform_configuration/BUILD.gn index 66eedfc23e3f..4874cf39e506 100644 --- a/starboard/linux/x64x11/shared/platform_configuration/BUILD.gn +++ b/starboard/linux/x64x11/shared/platform_configuration/BUILD.gn @@ -30,6 +30,10 @@ config("platform_configuration") { config("libraries") { configs = [ "//starboard/linux/shared/platform_configuration:libraries" ] + libs = [ + "//third_party/libiamf/code/libiamf.a", + "//third_party/libiamf/source/code/dep_codecs/lib/libfdk-aac.a", + ] } config("linker_flags") { diff --git a/starboard/shared/libiamf/iamf_audio_decoder.cc b/starboard/shared/libiamf/iamf_audio_decoder.cc new file mode 100644 index 000000000000..8b537f9d51cc --- /dev/null +++ b/starboard/shared/libiamf/iamf_audio_decoder.cc @@ -0,0 +1,332 @@ +// Copyright 2024 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "starboard/shared/libiamf/iamf_audio_decoder.h" + +#include + +#include "third_party/libiamf/source/code/include/IAMF_defines.h" + +namespace starboard { +namespace shared { +namespace libiamf { + +namespace { +using shared::starboard::player::DecodedAudio; +} // namespace + +IamfAudioDecoder::IamfAudioDecoder(const AudioStreamInfo& audio_stream_info) + : audio_stream_info_(audio_stream_info) {} + +IamfAudioDecoder::~IamfAudioDecoder() { + TeardownCodec(); +} + +bool IamfAudioDecoder::is_valid() const { + return decoder_ != NULL; +} + +void IamfAudioDecoder::Initialize(const OutputCB& output_cb, + const ErrorCB& error_cb) { + SB_DCHECK(BelongsToCurrentThread()); + SB_DCHECK(output_cb); + SB_DCHECK(!output_cb_); + SB_DCHECK(error_cb); + SB_DCHECK(!error_cb_); + + output_cb_ = output_cb; + error_cb_ = error_cb; +} + +void IamfAudioDecoder::Decode(const InputBuffers& input_buffers, + const ConsumedCB& consumed_cb) { + SB_DCHECK(BelongsToCurrentThread()); + SB_DCHECK(!input_buffers.empty()); + SB_DCHECK(pending_audio_buffers_.empty()); + SB_DCHECK(output_cb_); + + if (stream_ended_) { + SB_LOG(ERROR) << "Decode() is called after WriteEndOfStream() is called."; + return; + } + if (input_buffers.size() > kMinimumBuffersToDecode) { + std::copy(std::begin(input_buffers), std::end(input_buffers), + std::back_inserter(pending_audio_buffers_)); + consumed_cb_ = consumed_cb; + DecodePendingBuffers(); + } else { + for (const auto& input_buffer : input_buffers) { + if (!DecodeInternal(input_buffer)) { + return; + } + } + Schedule(consumed_cb); + } +} + +void IamfAudioDecoder::DecodePendingBuffers() { + SB_DCHECK(BelongsToCurrentThread()); + SB_DCHECK(!pending_audio_buffers_.empty()); + SB_DCHECK(consumed_cb_); + + for (int i = 0; i < kMinimumBuffersToDecode; ++i) { + if (!DecodeInternal(pending_audio_buffers_.front())) { + return; + } + pending_audio_buffers_.pop_front(); + if (pending_audio_buffers_.empty()) { + Schedule(consumed_cb_); + consumed_cb_ = nullptr; + if (stream_ended_) { + Schedule(std::bind(&IamfAudioDecoder::WriteEndOfStream, this)); + stream_ended_ = false; + } + return; + } + } + + SB_DCHECK(!pending_audio_buffers_.empty()); + Schedule(std::bind(&IamfAudioDecoder::DecodePendingBuffers, this)); +} + +bool IamfAudioDecoder::DecodeInternal( + const scoped_refptr& input_buffer) { + SB_DCHECK(BelongsToCurrentThread()); + SB_DCHECK(input_buffer); + SB_DCHECK(input_buffer->size() > 0); + SB_DCHECK(output_cb_); + SB_DCHECK(!stream_ended_ || !pending_audio_buffers_.empty()); + + SB_LOG(INFO) << "Start reading"; + reader_.Read(input_buffer); + SB_LOG(INFO) << "Back in iamf decoder"; + if (!decoder_) { + bool init = InitializeCodec(); + SB_LOG(INFO) << init; + if (!init) { + SB_LOG(INFO) << "Decoder init failure"; + error_cb_(kSbPlayerErrorDecode, "Failed to initialize IAMF decoder"); + return false; + } + } + + scoped_refptr decoded_audio = new DecodedAudio( + audio_stream_info_.number_of_channels, GetSampleType(), + kSbMediaAudioFrameStorageTypeInterleaved, input_buffer->timestamp(), + audio_stream_info_.number_of_channels * frames_per_au_ * + starboard::media::GetBytesPerSample(GetSampleType())); + uint32_t rsize = 0; + SB_LOG(INFO) << "Start decode"; + int ret = IAMF_decoder_decode(decoder_, reader_.data().data(), + reader_.data().size(), &rsize, + reinterpret_cast(decoded_audio->data())); + if (ret < 1) { + SB_LOG(INFO) << "IAMF_decoder_decode() error " << std::hex << ret; + error_cb_(kSbPlayerErrorDecode, "Failed to decode sample"); + return false; + } + + frames_per_au_ = ret; + decoded_audio->ShrinkTo(audio_stream_info_.number_of_channels * + frames_per_au_ * + starboard::media::GetBytesPerSample(GetSampleType())); + const auto& sample_info = input_buffer->audio_sample_info(); + decoded_audio->AdjustForDiscardedDurations( + audio_stream_info_.samples_per_second, + sample_info.discarded_duration_from_front, + sample_info.discarded_duration_from_back); + decoded_audios_.push(decoded_audio); + output_cb_(); + SB_LOG(INFO) << "Returning decoded audio"; + + return true; +} + +void IamfAudioDecoder::WriteEndOfStream() { + SB_DCHECK(BelongsToCurrentThread()); + SB_DCHECK(output_cb_); + + // Opus has no dependent frames so we needn't flush the decoder. Set the + // flag to ensure that Decode() is not called when the stream is ended. + stream_ended_ = true; + if (!pending_audio_buffers_.empty()) { + return; + } + + // Put EOS into the queue. + decoded_audios_.push(new DecodedAudio); + + Schedule(output_cb_); +} + +bool IamfAudioDecoder::InitializeCodec() { + int channels = audio_stream_info_.number_of_channels; + if (channels > 8 || channels < 1) { + SB_LOG(ERROR) << "Can't create decoder with " << channels << " channels"; + return false; + } + SB_LOG(INFO) << "1"; + + decoder_ = IAMF_decoder_open(); + if (!decoder_) { + SB_LOG(ERROR) << "Error creating libiamf decoder"; + return false; + } + SB_LOG(INFO) << "2"; + + int error = IAMF_decoder_set_bit_depth(decoder_, 32); + if (error != IAMF_OK) { + SB_LOG(ERROR) << "IAMF_decoder_set_bit_depth() fails with error " << error; + return false; + } + SB_LOG(INFO) << "3"; + + error = IAMF_decoder_set_sampling_rate(decoder_, 48000); + if (error != IAMF_OK) { + SB_LOG(ERROR) << "IAMF_decoder_set_sampling_rate() fails with error " + << error; + return false; + } + SB_LOG(INFO) << "4"; + + IAMF_SoundSystem sound_system = SOUND_SYSTEM_INVALID; + switch (channels) { + case 1: + sound_system = SOUND_SYSTEM_MONO; + break; + case 2: + sound_system = SOUND_SYSTEM_A; + break; + case 6: + sound_system = SOUND_SYSTEM_B; + break; + case 8: + sound_system = SOUND_SYSTEM_C; + break; + default: + SB_NOTREACHED(); + } + error = IAMF_decoder_output_layout_set_sound_system(decoder_, sound_system); + if (error != IAMF_OK) { + SB_LOG(ERROR) + << "IAMF_decoder_output_layout_set_sound_system() fails with error " + << error; + return false; + } + SB_LOG(INFO) << "5"; + + // TODO: Accurately set pts upon resume, if needed. + error = IAMF_decoder_set_pts(decoder_, 0, 90000); + if (error != IAMF_OK) { + SB_LOG(ERROR) << "IAMF_decoder_set_pts() fails with error " << error; + return false; + } + + SB_LOG(INFO) << "6"; + + if (reader_.has_mix_presentation_id()) { + error = IAMF_decoder_set_mix_presentation_id(decoder_, + reader_.mix_presentation_id()); + if (error != IAMF_OK) { + SB_LOG(ERROR) + << "IAMF_decoder_set_mix_presentation_id() fails with error " + << error; + return false; + } + SB_LOG(INFO) << "7"; + } + + error = IAMF_decoder_peak_limiter_enable(decoder_, 0); + if (error != IAMF_OK) { + SB_LOG(ERROR) << "IAMF_decoder_peak_limiter_enable() fails with error " + << error; + return false; + } + SB_LOG(INFO) << "8"; + + error = IAMF_decoder_set_normalization_loudness(decoder_, .0f); + if (error != IAMF_OK) { + SB_LOG(ERROR) + << "IAMF_decoder_set_normalization_loudness() fails with error " + << error; + return false; + } + SB_LOG(INFO) << "9"; + + uint32_t rsize = 0; + error = IAMF_decoder_configure(decoder_, reader_.config_obus().data(), + reader_.config_size(), &rsize); + if (error != IAMF_OK) { + SB_LOG(ERROR) << "IAMF_decoder_configure() fails with error " << error + << ", rsize " << rsize; + return false; + } + SB_LOG(INFO) << "rsize is " << rsize; + return true; +} + +void IamfAudioDecoder::TeardownCodec() { + if (is_valid()) { + IAMF_decoder_close(decoder_); + decoder_ = NULL; + } +} + +scoped_refptr IamfAudioDecoder::Read( + int* samples_per_second) { + SB_DCHECK(BelongsToCurrentThread()); + SB_DCHECK(output_cb_); + SB_DCHECK(!decoded_audios_.empty()); + + scoped_refptr result; + if (!decoded_audios_.empty()) { + result = decoded_audios_.front(); + decoded_audios_.pop(); + } + *samples_per_second = 48000; + return result; +} + +void IamfAudioDecoder::Reset() { + SB_DCHECK(BelongsToCurrentThread()); + + if (is_valid()) { + // If fail to reset opus decoder, re-create it. + TeardownCodec(); + InitializeCodec(); + } + + frames_per_au_ = kMaxOpusFramesPerAU; + stream_ended_ = false; + while (!decoded_audios_.empty()) { + decoded_audios_.pop(); + } + pending_audio_buffers_.clear(); + consumed_cb_ = nullptr; + + CancelPendingJobs(); +} + +SbMediaAudioSampleType IamfAudioDecoder::GetSampleType() const { + SB_DCHECK(BelongsToCurrentThread()); +#if SB_API_VERSION <= 15 && SB_HAS_QUIRK(SUPPORT_INT16_AUDIO_SAMPLES) + return kSbMediaAudioSampleTypeInt16; +#else // SB_API_VERSION <= 15 && SB_HAS_QUIRK(SUPPORT_INT16_AUDIO_SAMPLES) + return kSbMediaAudioSampleTypeFloat32; +#endif // SB_API_VERSION <= 15 && SB_HAS_QUIRK(SUPPORT_INT16_AUDIO_SAMPLES) +} + +} // namespace libiamf +} // namespace shared +} // namespace starboard diff --git a/starboard/shared/libiamf/iamf_audio_decoder.h b/starboard/shared/libiamf/iamf_audio_decoder.h new file mode 100644 index 000000000000..47279a0cf986 --- /dev/null +++ b/starboard/shared/libiamf/iamf_audio_decoder.h @@ -0,0 +1,85 @@ +// Copyright 2024 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef STARBOARD_SHARED_LIBIAMF_IAMF_AUDIO_DECODER_H_ +#define STARBOARD_SHARED_LIBIAMF_IAMF_AUDIO_DECODER_H_ + +#include +#include +#include + +#include "starboard/common/ref_counted.h" +#include "starboard/media.h" +#include "starboard/shared/internal_only.h" +#include "starboard/shared/libiamf/iamf_config_reader.h" +#include "starboard/shared/starboard/media/media_util.h" +#include "starboard/shared/starboard/player/decoded_audio_internal.h" +#include "starboard/shared/starboard/player/filter/audio_decoder_internal.h" +#include "starboard/shared/starboard/player/job_queue.h" +#include "third_party/libiamf/source/code/include/IAMF_decoder.h" + +namespace starboard { +namespace shared { +namespace libiamf { + +class IamfAudioDecoder + : public ::starboard::shared::starboard::player::filter::AudioDecoder, + private starboard::player::JobQueue::JobOwner { + public: + typedef starboard::media::AudioStreamInfo AudioStreamInfo; + + explicit IamfAudioDecoder(const AudioStreamInfo& audio_stream_info); + ~IamfAudioDecoder() override; + + bool is_valid() const; + + // AudioDecoder functions + void Initialize(const OutputCB& output_cb, const ErrorCB& error_cb) override; + void Decode(const InputBuffers& input_buffers, + const ConsumedCB& consumed_cb) override; + void WriteEndOfStream() override; + scoped_refptr Read(int* samples_per_second) override; + void Reset() override; + + private: + static constexpr int kMinimumBuffersToDecode = 2; + + bool InitializeCodec(); + void TeardownCodec(); + void DecodePendingBuffers(); + bool DecodeInternal(const scoped_refptr& input_buffer); + static const int kMaxOpusFramesPerAU = 9600; + + SbMediaAudioSampleType GetSampleType() const; + + OutputCB output_cb_; + ErrorCB error_cb_; + + IAMF_Decoder* decoder_ = NULL; + bool stream_ended_ = false; + std::queue> decoded_audios_; + AudioStreamInfo audio_stream_info_; + int frames_per_au_ = kMaxOpusFramesPerAU; + + std::deque> pending_audio_buffers_; + ConsumedCB consumed_cb_; + + IamfConfigReader reader_; +}; + +} // namespace libiamf +} // namespace shared +} // namespace starboard + +#endif // STARBOARD_SHARED_LIBIAMF_IAMF_AUDIO_DECODER_H_ diff --git a/starboard/shared/libiamf/iamf_config_reader.cc b/starboard/shared/libiamf/iamf_config_reader.cc new file mode 100644 index 000000000000..bc797937d0d3 --- /dev/null +++ b/starboard/shared/libiamf/iamf_config_reader.cc @@ -0,0 +1,184 @@ +// Copyright 2024 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "starboard/shared/libiamf/iamf_config_reader.h" + +#include "starboard/common/string.h" + +namespace starboard { +namespace shared { +namespace libiamf { + +namespace { +// From +// https://aomediacodec.github.io/iamf/v1.0.0-errata.html#obu-header-syntax. +constexpr int kObuTypeCodecConfig = 0; +constexpr int kObuTypeAudioElement = 1; +constexpr int kObuTypeMixPresentation = 2; +constexpr int kObuTypeSequenceHeader = 31; + +// Decodes an Leb128 value and stores it in |value|. Returns the number of bytes +// read. Returns -1 on error. +int ReadLeb128Value(const uint8_t* buf, uint32_t* value) { + SB_DCHECK(buf); + SB_DCHECK(value); + *value = 0; + bool error = true; + size_t i = 0; + for (; i < sizeof(uint32_t); ++i) { + uint8_t byte = buf[i]; + *value |= ((byte & 0x7f) << (i * 7)); + if (!(byte & 0x80)) { + error = false; + break; + } + } + + if (error) { + return -1; + } + return i + 1; +} +} // namespace + +bool IamfConfigReader::Read(scoped_refptr input_buffer) { + buffer_head_ = 0; + has_mix_presentation_id_ = false; + SB_LOG(INFO) << "Input buffer size is " << input_buffer->size(); + const uint8_t* buf = input_buffer->data(); + SB_DCHECK(buf); + + bool completed_parsing = false; + while (!completed_parsing && buffer_head_ < input_buffer->size()) { + if (!ReadOBU(buf, completed_parsing)) { + SB_LOG(INFO) << "Error parsing config OBUs"; + return false; + } + } + + SB_CHECK(completed_parsing); + + SB_LOG(INFO) << "End OBU loop"; + + data_size_ = input_buffer->size() - config_size_; + config_obus_.assign(buf, buf + config_size_); + data_.assign(buf + config_size_, buf + input_buffer->size()); + SB_LOG(INFO) << ::starboard::HexEncode(config_obus_.data(), + config_obus_.size()); + SB_LOG(INFO) << "Return read"; + + return true; +} + +bool IamfConfigReader::ReadOBU(const uint8_t* buf, bool& completed_parsing) { + uint8_t obu_type = 0; + uint32_t obu_size = 0; + uint32_t header_size = 0; + if (!ReadOBUHeader(buf, &obu_type, &obu_size, &header_size)) { + SB_LOG(ERROR) << "Error reading OBU header"; + return false; + } + SB_LOG(INFO) << "OBU size is " << obu_size << " with current buffer head " + << buffer_head_; + + // const uint8_t* last_byte = buf + buffer_head_ + obu_size; + int next_obu_pos = buffer_head_ + obu_size; + int bytes_read = 0; + + switch (static_cast(obu_type)) { + case kObuTypeCodecConfig: + SB_LOG(INFO) << "Reading codec config OBU"; + break; + case kObuTypeAudioElement: + SB_LOG(INFO) << "Reading audio element OBU"; + break; + case kObuTypeSequenceHeader: + SB_LOG(INFO) << "Reading sequence header OBU"; + break; + case kObuTypeMixPresentation: + SB_LOG(INFO) << "Reading mix presentation OBU"; + has_mix_presentation_id_ = true; + bytes_read = ReadLeb128Value(&buf[buffer_head_], &mix_presentation_id_); + if (bytes_read < 0) { + return false; + } + buffer_head_ += bytes_read; + break; + default: + SB_LOG(INFO) << "OBU type " << static_cast(obu_type); + completed_parsing = true; + break; + } + + if (!completed_parsing) { + // Skip to next OBU + buffer_head_ = next_obu_pos; + config_size_ += obu_size + header_size; + } + return true; +} + +bool IamfConfigReader::ReadOBUHeader(const uint8_t* buf, + uint8_t* obu_type, + uint32_t* obu_size, + uint32_t* header_size) { + SB_LOG(INFO) << "buffer head is " << buffer_head_; + uint8_t header_flags = buf[buffer_head_]; + *obu_type = (header_flags >> 3) & 0x1f; + buffer_head_++; + + const bool obu_redundant_copy = (header_flags >> 2) & 1; + const bool obu_trimming_status_flag = (header_flags >> 1) & 1; + const bool obu_extension_flag = header_flags & 1; + + SB_LOG(INFO) << static_cast(*obu_type); + + *header_size = 1; + + // redundant_copy |= obu_redundant_copy; + + *obu_size = 0; + int bytes_read = ReadLeb128Value(&buf[buffer_head_], obu_size); + if (bytes_read < 0) { + SB_LOG(INFO) << "Error reading OBU size"; + return false; + } + buffer_head_ += bytes_read; + *header_size += bytes_read; + + if (obu_trimming_status_flag) { + uint32_t value; + bytes_read = ReadLeb128Value(&buf[buffer_head_], &value); + buffer_head_ += bytes_read; + *header_size += bytes_read; + *obu_size -= bytes_read; + bytes_read = ReadLeb128Value(&buf[buffer_head_], &value); + buffer_head_ += bytes_read; + *header_size += bytes_read; + *obu_size -= bytes_read; + } + + if (obu_extension_flag) { + uint32_t extension_header_size; + bytes_read = ReadLeb128Value(&buf[buffer_head_], &extension_header_size); + buffer_head_ += extension_header_size; + *obu_size -= extension_header_size; + *header_size += bytes_read; + } + return true; +} + +} // namespace libiamf +} // namespace shared +} // namespace starboard diff --git a/starboard/shared/libiamf/iamf_config_reader.h b/starboard/shared/libiamf/iamf_config_reader.h new file mode 100644 index 000000000000..d1be99c50218 --- /dev/null +++ b/starboard/shared/libiamf/iamf_config_reader.h @@ -0,0 +1,75 @@ +// Copyright 2024 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef STARBOARD_SHARED_LIBIAMF_IAMF_CONFIG_READER_H_ +#define STARBOARD_SHARED_LIBIAMF_IAMF_CONFIG_READER_H_ + +#include + +#include "starboard/common/ref_counted.h" +#include "starboard/shared/internal_only.h" +#include "starboard/shared/starboard/player/input_buffer_internal.h" + +namespace starboard { +namespace shared { +namespace libiamf { + +class IamfConfigReader { + public: + typedef ::starboard::shared::starboard::player::InputBuffer InputBuffer; + + IamfConfigReader() = default; + + bool Read(scoped_refptr input_buffer); + + int sample_rate() { return sample_rate_; } + bool config_changed() { return false; } + uint32_t config_size() { + SB_LOG(INFO) << "Config size is " << config_size_; + SB_LOG(INFO) << "Buffer size is " << config_obus_.size(); + return config_size_; + } + bool has_mix_presentation_id() { return has_mix_presentation_id_; } + uint32_t mix_presentation_id() { return mix_presentation_id_; } + uint32_t data_size() { return data_size_; } + + std::vector config_obus() { return config_obus_; } + std::vector data() { return data_; } + + private: + bool ReadOBU(const uint8_t* buf, bool& completed_parsing); + bool ReadOBUHeader(const uint8_t* buf, + uint8_t* obu_type, + uint32_t* obu_size, + uint32_t* header_size); + + int buffer_head_ = 0; + int sample_rate_ = 0; + int samples_per_buffer_ = 0; + uint32_t config_size_ = 0; + uint32_t data_size_ = 0; + + bool has_mix_presentation_id_ = false; + uint32_t mix_presentation_id_ = 0; + + bool has_valid_config_ = false; + std::vector config_obus_; + std::vector data_; +}; + +} // namespace libiamf +} // namespace shared +} // namespace starboard + +#endif // STARBOARD_SHARED_LIBIAMF_IAMF_CONFIG_READER_H_ From 4c71f32d7512f2da4f8102d8ac4cbde69c77df36 Mon Sep 17 00:00:00 2001 From: Austin Osagie Date: Tue, 9 Jul 2024 14:43:08 -0700 Subject: [PATCH 02/30] Update config parsing --- .../shared/libiamf/iamf_audio_decoder.cc | 56 ++++++++----------- starboard/shared/libiamf/iamf_audio_decoder.h | 4 +- .../shared/libiamf/iamf_config_reader.cc | 54 +++++++++++++++++- 3 files changed, 78 insertions(+), 36 deletions(-) diff --git a/starboard/shared/libiamf/iamf_audio_decoder.cc b/starboard/shared/libiamf/iamf_audio_decoder.cc index 8b537f9d51cc..aa80f3d02e9d 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.cc +++ b/starboard/shared/libiamf/iamf_audio_decoder.cc @@ -124,28 +124,30 @@ bool IamfAudioDecoder::DecodeInternal( scoped_refptr decoded_audio = new DecodedAudio( audio_stream_info_.number_of_channels, GetSampleType(), kSbMediaAudioFrameStorageTypeInterleaved, input_buffer->timestamp(), - audio_stream_info_.number_of_channels * frames_per_au_ * + audio_stream_info_.number_of_channels * kMaxOpusFramesPerAU * starboard::media::GetBytesPerSample(GetSampleType())); - uint32_t rsize = 0; - SB_LOG(INFO) << "Start decode"; + SB_LOG(INFO) << "Start decode with AU size " << reader_.data().size(); int ret = IAMF_decoder_decode(decoder_, reader_.data().data(), - reader_.data().size(), &rsize, + reader_.data().size(), nullptr, reinterpret_cast(decoded_audio->data())); if (ret < 1) { SB_LOG(INFO) << "IAMF_decoder_decode() error " << std::hex << ret; error_cb_(kSbPlayerErrorDecode, "Failed to decode sample"); return false; + } else { + SB_LOG(INFO) << "Decoded " << ret << " samples"; + SB_DCHECK(ret <= kMaxOpusFramesPerAU); } frames_per_au_ = ret; decoded_audio->ShrinkTo(audio_stream_info_.number_of_channels * frames_per_au_ * starboard::media::GetBytesPerSample(GetSampleType())); - const auto& sample_info = input_buffer->audio_sample_info(); - decoded_audio->AdjustForDiscardedDurations( - audio_stream_info_.samples_per_second, - sample_info.discarded_duration_from_front, - sample_info.discarded_duration_from_back); + // const auto& sample_info = input_buffer->audio_sample_info(); + // decoded_audio->AdjustForDiscardedDurations( + // audio_stream_info_.samples_per_second, + // sample_info.discarded_duration_from_front, + // sample_info.discarded_duration_from_back); decoded_audios_.push(decoded_audio); output_cb_(); SB_LOG(INFO) << "Returning decoded audio"; @@ -176,47 +178,48 @@ bool IamfAudioDecoder::InitializeCodec() { SB_LOG(ERROR) << "Can't create decoder with " << channels << " channels"; return false; } - SB_LOG(INFO) << "1"; decoder_ = IAMF_decoder_open(); if (!decoder_) { SB_LOG(ERROR) << "Error creating libiamf decoder"; return false; } - SB_LOG(INFO) << "2"; - int error = IAMF_decoder_set_bit_depth(decoder_, 32); + int error = IAMF_decoder_set_bit_depth(decoder_, kOutputBitDepth); if (error != IAMF_OK) { SB_LOG(ERROR) << "IAMF_decoder_set_bit_depth() fails with error " << error; return false; } - SB_LOG(INFO) << "3"; - error = IAMF_decoder_set_sampling_rate(decoder_, 48000); + error = IAMF_decoder_set_sampling_rate(decoder_, kOutputSamplesPerSecond); if (error != IAMF_OK) { SB_LOG(ERROR) << "IAMF_decoder_set_sampling_rate() fails with error " << error; return false; } - SB_LOG(INFO) << "4"; IAMF_SoundSystem sound_system = SOUND_SYSTEM_INVALID; switch (channels) { case 1: sound_system = SOUND_SYSTEM_MONO; + SB_LOG(INFO) << "Configuring IamfAudioDecoder for mono output"; break; case 2: sound_system = SOUND_SYSTEM_A; + SB_LOG(INFO) << "Configuring IamfAudioDecoder for stereo output"; break; case 6: sound_system = SOUND_SYSTEM_B; + SB_LOG(INFO) << "Configuring IamfAudioDecoder for 5.1 output"; break; case 8: sound_system = SOUND_SYSTEM_C; + SB_LOG(INFO) << "Configuring IamfAudioDecoder for 7.1 output"; break; default: SB_NOTREACHED(); } + error = IAMF_decoder_output_layout_set_sound_system(decoder_, sound_system); if (error != IAMF_OK) { SB_LOG(ERROR) @@ -224,7 +227,8 @@ bool IamfAudioDecoder::InitializeCodec() { << error; return false; } - SB_LOG(INFO) << "5"; + SB_LOG(INFO) << "num channels: " + << IAMF_layout_sound_system_channels_count(sound_system); // TODO: Accurately set pts upon resume, if needed. error = IAMF_decoder_set_pts(decoder_, 0, 90000); @@ -233,8 +237,6 @@ bool IamfAudioDecoder::InitializeCodec() { return false; } - SB_LOG(INFO) << "6"; - if (reader_.has_mix_presentation_id()) { error = IAMF_decoder_set_mix_presentation_id(decoder_, reader_.mix_presentation_id()); @@ -244,7 +246,6 @@ bool IamfAudioDecoder::InitializeCodec() { << error; return false; } - SB_LOG(INFO) << "7"; } error = IAMF_decoder_peak_limiter_enable(decoder_, 0); @@ -253,7 +254,6 @@ bool IamfAudioDecoder::InitializeCodec() { << error; return false; } - SB_LOG(INFO) << "8"; error = IAMF_decoder_set_normalization_loudness(decoder_, .0f); if (error != IAMF_OK) { @@ -262,17 +262,14 @@ bool IamfAudioDecoder::InitializeCodec() { << error; return false; } - SB_LOG(INFO) << "9"; - uint32_t rsize = 0; error = IAMF_decoder_configure(decoder_, reader_.config_obus().data(), - reader_.config_size(), &rsize); + reader_.config_size(), nullptr); if (error != IAMF_OK) { - SB_LOG(ERROR) << "IAMF_decoder_configure() fails with error " << error - << ", rsize " << rsize; + SB_LOG(ERROR) << "IAMF_decoder_configure() fails with error " << error; return false; } - SB_LOG(INFO) << "rsize is " << rsize; + return true; } @@ -294,7 +291,7 @@ scoped_refptr IamfAudioDecoder::Read( result = decoded_audios_.front(); decoded_audios_.pop(); } - *samples_per_second = 48000; + *samples_per_second = kOutputSamplesPerSecond; return result; } @@ -302,7 +299,6 @@ void IamfAudioDecoder::Reset() { SB_DCHECK(BelongsToCurrentThread()); if (is_valid()) { - // If fail to reset opus decoder, re-create it. TeardownCodec(); InitializeCodec(); } @@ -320,11 +316,7 @@ void IamfAudioDecoder::Reset() { SbMediaAudioSampleType IamfAudioDecoder::GetSampleType() const { SB_DCHECK(BelongsToCurrentThread()); -#if SB_API_VERSION <= 15 && SB_HAS_QUIRK(SUPPORT_INT16_AUDIO_SAMPLES) - return kSbMediaAudioSampleTypeInt16; -#else // SB_API_VERSION <= 15 && SB_HAS_QUIRK(SUPPORT_INT16_AUDIO_SAMPLES) return kSbMediaAudioSampleTypeFloat32; -#endif // SB_API_VERSION <= 15 && SB_HAS_QUIRK(SUPPORT_INT16_AUDIO_SAMPLES) } } // namespace libiamf diff --git a/starboard/shared/libiamf/iamf_audio_decoder.h b/starboard/shared/libiamf/iamf_audio_decoder.h index 47279a0cf986..3dd29023e65b 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.h +++ b/starboard/shared/libiamf/iamf_audio_decoder.h @@ -59,7 +59,9 @@ class IamfAudioDecoder void TeardownCodec(); void DecodePendingBuffers(); bool DecodeInternal(const scoped_refptr& input_buffer); - static const int kMaxOpusFramesPerAU = 9600; + static const int kMaxOpusFramesPerAU = 960; + static const int kOutputSamplesPerSecond = 48000; + static const int kOutputBitDepth = 32; SbMediaAudioSampleType GetSampleType() const; diff --git a/starboard/shared/libiamf/iamf_config_reader.cc b/starboard/shared/libiamf/iamf_config_reader.cc index bc797937d0d3..1391ae7132ff 100644 --- a/starboard/shared/libiamf/iamf_config_reader.cc +++ b/starboard/shared/libiamf/iamf_config_reader.cc @@ -14,6 +14,8 @@ #include "starboard/shared/libiamf/iamf_config_reader.h" +#include + #include "starboard/common/string.h" namespace starboard { @@ -50,11 +52,26 @@ int ReadLeb128Value(const uint8_t* buf, uint32_t* value) { } return i + 1; } + +int ReadString(const uint8_t* buf, std::string& value) { + SB_DCHECK(buf); + value = ""; + value.reserve(128); + + int bytes_read = 0; + while (bytes_read < 128 && buf[bytes_read] != '\0') { + value.push_back(static_cast(buf[bytes_read])); + bytes_read++; + } + + return bytes_read; +} } // namespace bool IamfConfigReader::Read(scoped_refptr input_buffer) { buffer_head_ = 0; has_mix_presentation_id_ = false; + config_size_ = 0; SB_LOG(INFO) << "Input buffer size is " << input_buffer->size(); const uint8_t* buf = input_buffer->data(); SB_DCHECK(buf); @@ -72,10 +89,11 @@ bool IamfConfigReader::Read(scoped_refptr input_buffer) { SB_LOG(INFO) << "End OBU loop"; data_size_ = input_buffer->size() - config_size_; + SB_LOG(INFO) << "Input size: " << input_buffer->size() + << ", Data size: " << input_buffer->size() - config_size_ + << ", config size: " << config_size_; config_obus_.assign(buf, buf + config_size_); data_.assign(buf + config_size_, buf + input_buffer->size()); - SB_LOG(INFO) << ::starboard::HexEncode(config_obus_.data(), - config_obus_.size()); SB_LOG(INFO) << "Return read"; return true; @@ -96,6 +114,10 @@ bool IamfConfigReader::ReadOBU(const uint8_t* buf, bool& completed_parsing) { int next_obu_pos = buffer_head_ + obu_size; int bytes_read = 0; + uint32_t count_label = 0; + std::string str; + uint32_t num_sub_mixes = 0; + switch (static_cast(obu_type)) { case kObuTypeCodecConfig: SB_LOG(INFO) << "Reading codec config OBU"; @@ -106,7 +128,7 @@ bool IamfConfigReader::ReadOBU(const uint8_t* buf, bool& completed_parsing) { case kObuTypeSequenceHeader: SB_LOG(INFO) << "Reading sequence header OBU"; break; - case kObuTypeMixPresentation: + case kObuTypeMixPresentation: { SB_LOG(INFO) << "Reading mix presentation OBU"; has_mix_presentation_id_ = true; bytes_read = ReadLeb128Value(&buf[buffer_head_], &mix_presentation_id_); @@ -114,7 +136,33 @@ bool IamfConfigReader::ReadOBU(const uint8_t* buf, bool& completed_parsing) { return false; } buffer_head_ += bytes_read; + + // count_label + bytes_read = ReadLeb128Value(&buf[buffer_head_], &count_label); + if (bytes_read < 0) { + return false; + } + buffer_head_ += bytes_read; + + // language_label + for (int i = 0; i < count_label; ++i) { + buffer_head_ += ReadString(&buf[buffer_head_], str); + } + + // MixPresentationAnnotations + for (int i = 0; i < count_label; ++i) { + buffer_head_ += ReadString(&buf[buffer_head_], str); + } + + // num_sub_mixes + bytes_read = ReadLeb128Value(&buf[buffer_head_], &num_sub_mixes); + if (bytes_read < 0) { + return false; + } + buffer_head_ += bytes_read; + break; + } default: SB_LOG(INFO) << "OBU type " << static_cast(obu_type); completed_parsing = true; From 416f7b49cd18969e22216021ff221b9ef6f655f7 Mon Sep 17 00:00:00 2001 From: Austin Osagie Date: Wed, 10 Jul 2024 12:57:46 -0700 Subject: [PATCH 03/30] Decode samples to int16 --- .../shared/media_is_audio_supported.cc | 9 +- .../linux/shared/media_is_audio_supported.cc | 2 + .../shared/libiamf/iamf_audio_decoder.cc | 79 ++++++------ starboard/shared/libiamf/iamf_audio_decoder.h | 8 +- .../shared/libiamf/iamf_config_reader.cc | 116 ++++++++++++++---- starboard/shared/libiamf/iamf_config_reader.h | 18 ++- 6 files changed, 156 insertions(+), 76 deletions(-) diff --git a/starboard/android/shared/media_is_audio_supported.cc b/starboard/android/shared/media_is_audio_supported.cc index bb60f6124ab4..84b288614b51 100644 --- a/starboard/android/shared/media_is_audio_supported.cc +++ b/starboard/android/shared/media_is_audio_supported.cc @@ -56,11 +56,16 @@ bool SbMediaIsAudioSupported(SbMediaAudioCodec audio_codec, // Android uses a libopus based opus decoder for clear content, or a platform // opus decoder for encrypted content, if available. - if (audio_codec == kSbMediaAudioCodecOpus || - audio_codec == kSbMediaAudioCodecIamf) { + if (audio_codec == kSbMediaAudioCodecOpus) { return true; } +#if SB_API_VERSION >= 15 + if (audio_codec == kSbMediaAudioCodecIamf) { + return bitrate <= kSbMediaMaxAudioBitrateInBitsPerSecond; + } +#endif // SB_API_VERSION >= 15 + bool media_codec_supported = MediaCapabilitiesCache::GetInstance()->HasAudioDecoderFor(mime, bitrate); diff --git a/starboard/linux/shared/media_is_audio_supported.cc b/starboard/linux/shared/media_is_audio_supported.cc index e0d17062e521..6f686eb7737f 100644 --- a/starboard/linux/shared/media_is_audio_supported.cc +++ b/starboard/linux/shared/media_is_audio_supported.cc @@ -32,9 +32,11 @@ bool SbMediaIsAudioSupported(SbMediaAudioCodec audio_codec, return bitrate <= kSbMediaMaxAudioBitrateInBitsPerSecond; } +#if SB_API_VERSION >= 15 if (audio_codec == kSbMediaAudioCodecIamf) { return bitrate <= kSbMediaMaxAudioBitrateInBitsPerSecond; } +#endif // SB_API_VERSION >= 15 if (audio_codec == kSbMediaAudioCodecAc3 || audio_codec == kSbMediaAudioCodecEac3) { diff --git a/starboard/shared/libiamf/iamf_audio_decoder.cc b/starboard/shared/libiamf/iamf_audio_decoder.cc index aa80f3d02e9d..e380cb0af537 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.cc +++ b/starboard/shared/libiamf/iamf_audio_decoder.cc @@ -108,14 +108,10 @@ bool IamfAudioDecoder::DecodeInternal( SB_DCHECK(output_cb_); SB_DCHECK(!stream_ended_ || !pending_audio_buffers_.empty()); - SB_LOG(INFO) << "Start reading"; reader_.Read(input_buffer); - SB_LOG(INFO) << "Back in iamf decoder"; if (!decoder_) { - bool init = InitializeCodec(); - SB_LOG(INFO) << init; - if (!init) { - SB_LOG(INFO) << "Decoder init failure"; + bool decoder_initialized = InitializeCodec(); + if (!decoder_initialized) { error_cb_(kSbPlayerErrorDecode, "Failed to initialize IAMF decoder"); return false; } @@ -124,33 +120,36 @@ bool IamfAudioDecoder::DecodeInternal( scoped_refptr decoded_audio = new DecodedAudio( audio_stream_info_.number_of_channels, GetSampleType(), kSbMediaAudioFrameStorageTypeInterleaved, input_buffer->timestamp(), - audio_stream_info_.number_of_channels * kMaxOpusFramesPerAU * + audio_stream_info_.number_of_channels * kMaxIamfFramesPerAU * starboard::media::GetBytesPerSample(GetSampleType())); - SB_LOG(INFO) << "Start decode with AU size " << reader_.data().size(); - int ret = IAMF_decoder_decode(decoder_, reader_.data().data(), - reader_.data().size(), nullptr, - reinterpret_cast(decoded_audio->data())); - if (ret < 1) { - SB_LOG(INFO) << "IAMF_decoder_decode() error " << std::hex << ret; - error_cb_(kSbPlayerErrorDecode, "Failed to decode sample"); + int samples_decoded = IAMF_decoder_decode( + decoder_, reader_.data().data(), reader_.data().size(), nullptr, + reinterpret_cast(decoded_audio->data())); + if (samples_decoded < 1) { + SB_LOG(INFO) << "IAMF_decoder_decode() error " << std::hex + << samples_decoded; + error_cb_(kSbPlayerErrorDecode, "Failed to decode IAMF sample"); return false; - } else { - SB_LOG(INFO) << "Decoded " << ret << " samples"; - SB_DCHECK(ret <= kMaxOpusFramesPerAU); } - frames_per_au_ = ret; + SB_DCHECK(samples_decoded <= kMaxIamfFramesPerAU); + SB_LOG(INFO) << "Decoded " << samples_decoded << " samples"; + + frames_per_au_ = samples_decoded; decoded_audio->ShrinkTo(audio_stream_info_.number_of_channels * frames_per_au_ * starboard::media::GetBytesPerSample(GetSampleType())); + + // TODO: Enable once float32 pcm output is fixed. // const auto& sample_info = input_buffer->audio_sample_info(); // decoded_audio->AdjustForDiscardedDurations( // audio_stream_info_.samples_per_second, // sample_info.discarded_duration_from_front, // sample_info.discarded_duration_from_back); + decoded_audios_.push(decoded_audio); + output_cb_(); - SB_LOG(INFO) << "Returning decoded audio"; return true; } @@ -159,8 +158,6 @@ void IamfAudioDecoder::WriteEndOfStream() { SB_DCHECK(BelongsToCurrentThread()); SB_DCHECK(output_cb_); - // Opus has no dependent frames so we needn't flush the decoder. Set the - // flag to ensure that Decode() is not called when the stream is ended. stream_ended_ = true; if (!pending_audio_buffers_.empty()) { return; @@ -175,23 +172,25 @@ void IamfAudioDecoder::WriteEndOfStream() { bool IamfAudioDecoder::InitializeCodec() { int channels = audio_stream_info_.number_of_channels; if (channels > 8 || channels < 1) { - SB_LOG(ERROR) << "Can't create decoder with " << channels << " channels"; + SB_DLOG(ERROR) << "Can't create decoder with " << channels << " channels"; return false; } decoder_ = IAMF_decoder_open(); if (!decoder_) { - SB_LOG(ERROR) << "Error creating libiamf decoder"; + SB_DLOG(ERROR) << "Error creating libiamf decoder"; return false; } - int error = IAMF_decoder_set_bit_depth(decoder_, kOutputBitDepth); + // TODO: libiamf has an issue outputting 32 bit float samples, set to 16 bit + // for now. + int error = IAMF_decoder_set_bit_depth(decoder_, 16); if (error != IAMF_OK) { - SB_LOG(ERROR) << "IAMF_decoder_set_bit_depth() fails with error " << error; + SB_DLOG(ERROR) << "IAMF_decoder_set_bit_depth() fails with error " << error; return false; } - error = IAMF_decoder_set_sampling_rate(decoder_, kOutputSamplesPerSecond); + error = IAMF_decoder_set_sampling_rate(decoder_, reader_.sample_rate()); if (error != IAMF_OK) { SB_LOG(ERROR) << "IAMF_decoder_set_sampling_rate() fails with error " << error; @@ -202,17 +201,19 @@ bool IamfAudioDecoder::InitializeCodec() { switch (channels) { case 1: sound_system = SOUND_SYSTEM_MONO; - SB_LOG(INFO) << "Configuring IamfAudioDecoder for mono output"; break; case 2: + // Stereo output. sound_system = SOUND_SYSTEM_A; SB_LOG(INFO) << "Configuring IamfAudioDecoder for stereo output"; break; case 6: + // 5.1 output. sound_system = SOUND_SYSTEM_B; SB_LOG(INFO) << "Configuring IamfAudioDecoder for 5.1 output"; break; case 8: + // 7.1 output. sound_system = SOUND_SYSTEM_C; SB_LOG(INFO) << "Configuring IamfAudioDecoder for 7.1 output"; break; @@ -237,15 +238,12 @@ bool IamfAudioDecoder::InitializeCodec() { return false; } - if (reader_.has_mix_presentation_id()) { - error = IAMF_decoder_set_mix_presentation_id(decoder_, - reader_.mix_presentation_id()); - if (error != IAMF_OK) { - SB_LOG(ERROR) - << "IAMF_decoder_set_mix_presentation_id() fails with error " - << error; - return false; - } + error = IAMF_decoder_set_mix_presentation_id(decoder_, + reader_.mix_presentation_id()); + if (error != IAMF_OK) { + SB_LOG(ERROR) << "IAMF_decoder_set_mix_presentation_id() fails with error " + << error; + return false; } error = IAMF_decoder_peak_limiter_enable(decoder_, 0); @@ -291,7 +289,7 @@ scoped_refptr IamfAudioDecoder::Read( result = decoded_audios_.front(); decoded_audios_.pop(); } - *samples_per_second = kOutputSamplesPerSecond; + *samples_per_second = reader_.sample_rate(); return result; } @@ -303,7 +301,7 @@ void IamfAudioDecoder::Reset() { InitializeCodec(); } - frames_per_au_ = kMaxOpusFramesPerAU; + frames_per_au_ = kMaxIamfFramesPerAU; stream_ended_ = false; while (!decoded_audios_.empty()) { decoded_audios_.pop(); @@ -316,7 +314,10 @@ void IamfAudioDecoder::Reset() { SbMediaAudioSampleType IamfAudioDecoder::GetSampleType() const { SB_DCHECK(BelongsToCurrentThread()); - return kSbMediaAudioSampleTypeFloat32; + // if (audio_stream_info_.bits_per_sample == 32) { + // return kSbMediaAudioSampleTypeFloat32; + // } + return kSbMediaAudioSampleTypeInt16Deprecated; } } // namespace libiamf diff --git a/starboard/shared/libiamf/iamf_audio_decoder.h b/starboard/shared/libiamf/iamf_audio_decoder.h index 3dd29023e65b..362dc4b27c72 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.h +++ b/starboard/shared/libiamf/iamf_audio_decoder.h @@ -17,7 +17,6 @@ #include #include -#include #include "starboard/common/ref_counted.h" #include "starboard/media.h" @@ -54,14 +53,13 @@ class IamfAudioDecoder private: static constexpr int kMinimumBuffersToDecode = 2; + static constexpr int kMaxIamfFramesPerAU = 2048; + static constexpr int kMaxOpusFramesPerAU = 960; bool InitializeCodec(); void TeardownCodec(); void DecodePendingBuffers(); bool DecodeInternal(const scoped_refptr& input_buffer); - static const int kMaxOpusFramesPerAU = 960; - static const int kOutputSamplesPerSecond = 48000; - static const int kOutputBitDepth = 32; SbMediaAudioSampleType GetSampleType() const; @@ -72,7 +70,7 @@ class IamfAudioDecoder bool stream_ended_ = false; std::queue> decoded_audios_; AudioStreamInfo audio_stream_info_; - int frames_per_au_ = kMaxOpusFramesPerAU; + int frames_per_au_ = kMaxIamfFramesPerAU; std::deque> pending_audio_buffers_; ConsumedCB consumed_cb_; diff --git a/starboard/shared/libiamf/iamf_config_reader.cc b/starboard/shared/libiamf/iamf_config_reader.cc index 1391ae7132ff..e3ec533a1478 100644 --- a/starboard/shared/libiamf/iamf_config_reader.cc +++ b/starboard/shared/libiamf/iamf_config_reader.cc @@ -30,6 +30,21 @@ constexpr int kObuTypeAudioElement = 1; constexpr int kObuTypeMixPresentation = 2; constexpr int kObuTypeSequenceHeader = 31; +// From +// https://aomediacodec.github.io/iamf/v1.0.0-errata.html#obu-codecconfig. +constexpr int kFourccOpus = 0x4f707573; +constexpr int kFourccMp4a = 0x6d703461; +constexpr int kFourccFlac = 0x664c6143; +constexpr int kFourccIpcm = 0x6970636d; + +inline uint32_t ByteSwap(uint32_t x) { +#if defined(COMPILER_MSVC) + return _byteswap_ulong(x); +#else + return __builtin_bswap32(x); +#endif +} + // Decodes an Leb128 value and stores it in |value|. Returns the number of bytes // read. Returns -1 on error. int ReadLeb128Value(const uint8_t* buf, uint32_t* value) { @@ -69,10 +84,8 @@ int ReadString(const uint8_t* buf, std::string& value) { } // namespace bool IamfConfigReader::Read(scoped_refptr input_buffer) { - buffer_head_ = 0; - has_mix_presentation_id_ = false; - config_size_ = 0; - SB_LOG(INFO) << "Input buffer size is " << input_buffer->size(); + Reset(); + const uint8_t* buf = input_buffer->data(); SB_DCHECK(buf); @@ -86,19 +99,24 @@ bool IamfConfigReader::Read(scoped_refptr input_buffer) { SB_CHECK(completed_parsing); - SB_LOG(INFO) << "End OBU loop"; - data_size_ = input_buffer->size() - config_size_; - SB_LOG(INFO) << "Input size: " << input_buffer->size() - << ", Data size: " << input_buffer->size() - config_size_ - << ", config size: " << config_size_; config_obus_.assign(buf, buf + config_size_); data_.assign(buf + config_size_, buf + input_buffer->size()); - SB_LOG(INFO) << "Return read"; return true; } +void IamfConfigReader::Reset() { + buffer_head_ = 0; + has_mix_presentation_id_ = false; + mix_presentation_id_ = 0; + config_size_ = 0; + data_size_ = 0; + sample_rate_ = 0; + samples_per_buffer_ = 0; + sample_size_ = 0; +} + bool IamfConfigReader::ReadOBU(const uint8_t* buf, bool& completed_parsing) { uint8_t obu_type = 0; uint32_t obu_size = 0; @@ -107,10 +125,7 @@ bool IamfConfigReader::ReadOBU(const uint8_t* buf, bool& completed_parsing) { SB_LOG(ERROR) << "Error reading OBU header"; return false; } - SB_LOG(INFO) << "OBU size is " << obu_size << " with current buffer head " - << buffer_head_; - // const uint8_t* last_byte = buf + buffer_head_ + obu_size; int next_obu_pos = buffer_head_ + obu_size; int bytes_read = 0; @@ -119,17 +134,73 @@ bool IamfConfigReader::ReadOBU(const uint8_t* buf, bool& completed_parsing) { uint32_t num_sub_mixes = 0; switch (static_cast(obu_type)) { - case kObuTypeCodecConfig: - SB_LOG(INFO) << "Reading codec config OBU"; + case kObuTypeCodecConfig: { + sample_rate_ = 0; + uint32_t codec_config_id; + bytes_read = ReadLeb128Value(&buf[buffer_head_], &codec_config_id); + if (bytes_read < 0) { + return false; + } + buffer_head_ += bytes_read; + + uint32_t codec_id = 0; + std::memcpy(&codec_id, &buf[buffer_head_], sizeof(uint32_t)); + // Mp4 is in big-endian + codec_id = ByteSwap(codec_id); + buffer_head_ += 4; + SB_LOG(INFO) << std::hex << codec_id; + + bytes_read = ReadLeb128Value(&buf[buffer_head_], &samples_per_buffer_); + if (bytes_read < 0) { + return false; + } + buffer_head_ += bytes_read; + + // audio_roll_distance + buffer_head_ += 2; + + switch (codec_id) { + case kFourccOpus: + sample_rate_ = 48000; + break; + case kFourccMp4a: + // TODO: Properly parse for the sample rate. 48000 is the assumed + // sample rate. + sample_rate_ = 48000; + break; + case kFourccFlac: { + // Skip METADATA_BLOCK_HEADER. + buffer_head_ += 4; + // Skip first 10 bytes of METADATA_BLOCK_STREAMINFO. + buffer_head_ += 10; + + std::memcpy(&sample_rate_, &buf[buffer_head_], sizeof(uint32_t)); + sample_rate_ = ByteSwap(sample_rate_); + sample_rate_ = sample_rate_ >> 12; + break; + } + case kFourccIpcm: { + // sample_format_flags + buffer_head_++; + uint8_t sample_size_byte = buf[buffer_head_]; + sample_size_ = static_cast(sample_size_byte); + buffer_head_++; + + std::memcpy(&sample_rate_, &buf[buffer_head_], sizeof(uint32_t)); + sample_rate_ = ByteSwap(sample_rate_); + break; + } + default: + SB_NOTREACHED(); + } + break; + } case kObuTypeAudioElement: - SB_LOG(INFO) << "Reading audio element OBU"; - break; case kObuTypeSequenceHeader: - SB_LOG(INFO) << "Reading sequence header OBU"; break; case kObuTypeMixPresentation: { - SB_LOG(INFO) << "Reading mix presentation OBU"; + // TODO: Complete Mix Presentation OBU parsing has_mix_presentation_id_ = true; bytes_read = ReadLeb128Value(&buf[buffer_head_], &mix_presentation_id_); if (bytes_read < 0) { @@ -164,7 +235,8 @@ bool IamfConfigReader::ReadOBU(const uint8_t* buf, bool& completed_parsing) { break; } default: - SB_LOG(INFO) << "OBU type " << static_cast(obu_type); + // Once an OBU is read that is not a descriptor, descriptor parsing is + // assumed to be complete. completed_parsing = true; break; } @@ -181,7 +253,6 @@ bool IamfConfigReader::ReadOBUHeader(const uint8_t* buf, uint8_t* obu_type, uint32_t* obu_size, uint32_t* header_size) { - SB_LOG(INFO) << "buffer head is " << buffer_head_; uint8_t header_flags = buf[buffer_head_]; *obu_type = (header_flags >> 3) & 0x1f; buffer_head_++; @@ -194,12 +265,9 @@ bool IamfConfigReader::ReadOBUHeader(const uint8_t* buf, *header_size = 1; - // redundant_copy |= obu_redundant_copy; - *obu_size = 0; int bytes_read = ReadLeb128Value(&buf[buffer_head_], obu_size); if (bytes_read < 0) { - SB_LOG(INFO) << "Error reading OBU size"; return false; } buffer_head_ += bytes_read; diff --git a/starboard/shared/libiamf/iamf_config_reader.h b/starboard/shared/libiamf/iamf_config_reader.h index d1be99c50218..04a6d76d98ae 100644 --- a/starboard/shared/libiamf/iamf_config_reader.h +++ b/starboard/shared/libiamf/iamf_config_reader.h @@ -25,6 +25,7 @@ namespace starboard { namespace shared { namespace libiamf { +// TODO: Add handling for non-redundant OBUS class IamfConfigReader { public: typedef ::starboard::shared::starboard::player::InputBuffer InputBuffer; @@ -33,13 +34,17 @@ class IamfConfigReader { bool Read(scoped_refptr input_buffer); + void Reset(); + + bool is_valid() { + return data_size_ > 0 && config_size_ > 0 && has_mix_presentation_id_ && + sample_rate_ > 0; + } int sample_rate() { return sample_rate_; } + uint32_t samples_per_buffer() { return samples_per_buffer_; } bool config_changed() { return false; } - uint32_t config_size() { - SB_LOG(INFO) << "Config size is " << config_size_; - SB_LOG(INFO) << "Buffer size is " << config_obus_.size(); - return config_size_; - } + uint32_t config_size() { return config_size_; } + // TODO: Allow for selection of multiple mix presentation IDs. bool has_mix_presentation_id() { return has_mix_presentation_id_; } uint32_t mix_presentation_id() { return mix_presentation_id_; } uint32_t data_size() { return data_size_; } @@ -56,7 +61,8 @@ class IamfConfigReader { int buffer_head_ = 0; int sample_rate_ = 0; - int samples_per_buffer_ = 0; + int sample_size_ = 0; + uint32_t samples_per_buffer_ = 0; uint32_t config_size_ = 0; uint32_t data_size_ = 0; From f0c1c573969e1605b2c758b9d184eba6b50a3060 Mon Sep 17 00:00:00 2001 From: Austin Osagie Date: Wed, 10 Jul 2024 17:43:47 -0700 Subject: [PATCH 04/30] Enable Android IAMF decode support --- starboard/android/shared/media_common.h | 5 ++++ .../shared/media_is_audio_supported.cc | 2 +- .../shared/player_components_factory.h | 6 +++-- starboard/android/shared/player_create.cc | 6 ++++- .../shared/libiamf/iamf_audio_decoder.cc | 24 ++++++++++++------- starboard/shared/libiamf/iamf_audio_decoder.h | 2 ++ .../shared/libiamf/iamf_config_reader.cc | 3 --- 7 files changed, 32 insertions(+), 16 deletions(-) diff --git a/starboard/android/shared/media_common.h b/starboard/android/shared/media_common.h index dcbd0b36d108..4a2b6b86731c 100644 --- a/starboard/android/shared/media_common.h +++ b/starboard/android/shared/media_common.h @@ -61,6 +61,11 @@ inline const char* SupportedAudioCodecToMimeType( if (audio_codec == kSbMediaAudioCodecOpus) { return "audio/opus"; } +#if SB_API_VERSION >= 15 + if (audio_codec == kSbMediaAudioCodecIamf) { + return "audio/iamf"; + } +#endif // SB_API_VERSION >= 15 return nullptr; } diff --git a/starboard/android/shared/media_is_audio_supported.cc b/starboard/android/shared/media_is_audio_supported.cc index 84b288614b51..6074dce486f0 100644 --- a/starboard/android/shared/media_is_audio_supported.cc +++ b/starboard/android/shared/media_is_audio_supported.cc @@ -62,7 +62,7 @@ bool SbMediaIsAudioSupported(SbMediaAudioCodec audio_codec, #if SB_API_VERSION >= 15 if (audio_codec == kSbMediaAudioCodecIamf) { - return bitrate <= kSbMediaMaxAudioBitrateInBitsPerSecond; + return true; } #endif // SB_API_VERSION >= 15 diff --git a/starboard/android/shared/player_components_factory.h b/starboard/android/shared/player_components_factory.h index b3b766d7be1a..98a4d8700075 100644 --- a/starboard/android/shared/player_components_factory.h +++ b/starboard/android/shared/player_components_factory.h @@ -447,10 +447,12 @@ class PlayerComponentsFactory : public starboard::shared::starboard::player:: std::move(audio_decoder_impl)); } } else if (audio_stream_info.codec == kSbMediaAudioCodecIamf) { - scoped_ptr audio_decoder_impl( + SB_LOG(INFO) << "Creating IAMF audio decoder"; + std::unique_ptr audio_decoder_impl( new IamfAudioDecoder(audio_stream_info)); if (audio_decoder_impl->is_valid()) { - return audio_decoder_impl.PassAs(); + return std::unique_ptr( + std::move(audio_decoder_impl)); } } else { SB_LOG(ERROR) << "Unsupported audio codec " diff --git a/starboard/android/shared/player_create.cc b/starboard/android/shared/player_create.cc index 78b4ba46f252..2b9a88903b09 100644 --- a/starboard/android/shared/player_create.cc +++ b/starboard/android/shared/player_create.cc @@ -127,7 +127,11 @@ SbPlayer SbPlayerCreate(SbWindow window, audio_codec != kSbMediaAudioCodecAac && audio_codec != kSbMediaAudioCodecAc3 && audio_codec != kSbMediaAudioCodecEac3 && - audio_codec != kSbMediaAudioCodecOpus) { + audio_codec != kSbMediaAudioCodecOpus +#if SB_API_VERSION >= 15 + && audio_codec != kSbMediaAudioCodecIamf +#endif // SB_API_VERSION >= 15 + ) { SB_LOG(ERROR) << "Unsupported audio codec: " << starboard::GetMediaAudioCodecName(audio_codec) << "."; player_error_func( diff --git a/starboard/shared/libiamf/iamf_audio_decoder.cc b/starboard/shared/libiamf/iamf_audio_decoder.cc index e380cb0af537..075aa44a2d07 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.cc +++ b/starboard/shared/libiamf/iamf_audio_decoder.cc @@ -27,7 +27,12 @@ using shared::starboard::player::DecodedAudio; } // namespace IamfAudioDecoder::IamfAudioDecoder(const AudioStreamInfo& audio_stream_info) - : audio_stream_info_(audio_stream_info) {} + : audio_stream_info_(audio_stream_info) { + decoder_ = IAMF_decoder_open(); + if (!decoder_) { + SB_DLOG(ERROR) << "Error creating libiamf decoder"; + } +} IamfAudioDecoder::~IamfAudioDecoder() { TeardownCodec(); @@ -107,11 +112,13 @@ bool IamfAudioDecoder::DecodeInternal( SB_DCHECK(input_buffer->size() > 0); SB_DCHECK(output_cb_); SB_DCHECK(!stream_ended_ || !pending_audio_buffers_.empty()); + SB_DCHECK(is_valid()); reader_.Read(input_buffer); - if (!decoder_) { + if (!decoder_is_configured_) { bool decoder_initialized = InitializeCodec(); if (!decoder_initialized) { + SB_LOG(INFO) << "Failed to initialize IAMF decoder"; error_cb_(kSbPlayerErrorDecode, "Failed to initialize IAMF decoder"); return false; } @@ -170,18 +177,13 @@ void IamfAudioDecoder::WriteEndOfStream() { } bool IamfAudioDecoder::InitializeCodec() { + SB_DCHECK(is_valid()); int channels = audio_stream_info_.number_of_channels; if (channels > 8 || channels < 1) { SB_DLOG(ERROR) << "Can't create decoder with " << channels << " channels"; return false; } - decoder_ = IAMF_decoder_open(); - if (!decoder_) { - SB_DLOG(ERROR) << "Error creating libiamf decoder"; - return false; - } - // TODO: libiamf has an issue outputting 32 bit float samples, set to 16 bit // for now. int error = IAMF_decoder_set_bit_depth(decoder_, 16); @@ -268,6 +270,8 @@ bool IamfAudioDecoder::InitializeCodec() { return false; } + decoder_is_configured_ = true; + return true; } @@ -298,9 +302,11 @@ void IamfAudioDecoder::Reset() { if (is_valid()) { TeardownCodec(); - InitializeCodec(); + decoder_ = IAMF_decoder_open(); } + decoder_is_configured_ = false; + frames_per_au_ = kMaxIamfFramesPerAU; stream_ended_ = false; while (!decoded_audios_.empty()) { diff --git a/starboard/shared/libiamf/iamf_audio_decoder.h b/starboard/shared/libiamf/iamf_audio_decoder.h index 362dc4b27c72..a166f60b5a86 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.h +++ b/starboard/shared/libiamf/iamf_audio_decoder.h @@ -72,6 +72,8 @@ class IamfAudioDecoder AudioStreamInfo audio_stream_info_; int frames_per_au_ = kMaxIamfFramesPerAU; + bool decoder_is_configured_ = false; + std::deque> pending_audio_buffers_; ConsumedCB consumed_cb_; diff --git a/starboard/shared/libiamf/iamf_config_reader.cc b/starboard/shared/libiamf/iamf_config_reader.cc index e3ec533a1478..a9fd377d0f97 100644 --- a/starboard/shared/libiamf/iamf_config_reader.cc +++ b/starboard/shared/libiamf/iamf_config_reader.cc @@ -148,7 +148,6 @@ bool IamfConfigReader::ReadOBU(const uint8_t* buf, bool& completed_parsing) { // Mp4 is in big-endian codec_id = ByteSwap(codec_id); buffer_head_ += 4; - SB_LOG(INFO) << std::hex << codec_id; bytes_read = ReadLeb128Value(&buf[buffer_head_], &samples_per_buffer_); if (bytes_read < 0) { @@ -261,8 +260,6 @@ bool IamfConfigReader::ReadOBUHeader(const uint8_t* buf, const bool obu_trimming_status_flag = (header_flags >> 1) & 1; const bool obu_extension_flag = header_flags & 1; - SB_LOG(INFO) << static_cast(*obu_type); - *header_size = 1; *obu_size = 0; From 7bffaf1b40b0dc9a1cfe1581e9f8766e809cc461 Mon Sep 17 00:00:00 2001 From: Austin Osagie Date: Wed, 17 Jul 2024 16:10:32 -0700 Subject: [PATCH 05/30] Add build args Also clean up logs and reformat decoder. --- starboard/android/shared/BUILD.gn | 16 +- .../shared/media_is_audio_supported.cc | 2 +- .../shared/platform_configuration/BUILD.gn | 10 +- .../shared/player_components_factory.h | 14 +- starboard/build/config/BUILDCONFIG.gn | 2 + starboard/linux/shared/BUILD.gn | 15 +- .../linux/shared/media_is_audio_supported.cc | 2 +- .../linux/shared/player_components_factory.cc | 11 +- starboard/linux/x64x11/BUILD.gn | 1 + .../shared/platform_configuration/BUILD.gn | 10 +- .../shared/libiamf/iamf_audio_decoder.cc | 197 +++++++++++------- starboard/shared/libiamf/iamf_audio_decoder.h | 8 +- .../shared/libiamf/iamf_config_reader.cc | 70 +++---- starboard/shared/libiamf/iamf_config_reader.h | 26 ++- 14 files changed, 233 insertions(+), 151 deletions(-) diff --git a/starboard/android/shared/BUILD.gn b/starboard/android/shared/BUILD.gn index 10b1508d0b1c..5a05de935791 100644 --- a/starboard/android/shared/BUILD.gn +++ b/starboard/android/shared/BUILD.gn @@ -75,10 +75,6 @@ static_library("starboard_platform") { "//starboard/shared/libevent/socket_waiter_wait.cc", "//starboard/shared/libevent/socket_waiter_wait_timed.cc", "//starboard/shared/libevent/socket_waiter_wake_up.cc", - "//starboard/shared/libiamf/iamf_audio_decoder.cc", - "//starboard/shared/libiamf/iamf_audio_decoder.h", - "//starboard/shared/libiamf/iamf_config_reader.cc", - "//starboard/shared/libiamf/iamf_config_reader.h", "//starboard/shared/linux/byte_swap.cc", "//starboard/shared/linux/cpu_features_get.cc", "//starboard/shared/linux/memory_get_stack_bounds.cc", @@ -497,6 +493,18 @@ static_library("starboard_platform") { if (sb_evergreen_compatible_use_libunwind) { deps += [ "//third_party/llvm-project/libunwind:unwind_starboard" ] } + + defines = [] + if (enable_iamf_decode) { + sources += [ + "//starboard/shared/libiamf/iamf_audio_decoder.cc", + "//starboard/shared/libiamf/iamf_audio_decoder.h", + "//starboard/shared/libiamf/iamf_config_reader.cc", + "//starboard/shared/libiamf/iamf_config_reader.h", + ] + + defines += [ "ENABLE_IAMF_DECODE" ] + } } static_library("starboard_base_symbolize") { diff --git a/starboard/android/shared/media_is_audio_supported.cc b/starboard/android/shared/media_is_audio_supported.cc index 6074dce486f0..b2c9d0b9c62f 100644 --- a/starboard/android/shared/media_is_audio_supported.cc +++ b/starboard/android/shared/media_is_audio_supported.cc @@ -60,7 +60,7 @@ bool SbMediaIsAudioSupported(SbMediaAudioCodec audio_codec, return true; } -#if SB_API_VERSION >= 15 +#if SB_API_VERSION >= 15 && ENABLE_IAMF_DECODE if (audio_codec == kSbMediaAudioCodecIamf) { return true; } diff --git a/starboard/android/shared/platform_configuration/BUILD.gn b/starboard/android/shared/platform_configuration/BUILD.gn index 785a7ad48f78..dfcdc08ab6d9 100644 --- a/starboard/android/shared/platform_configuration/BUILD.gn +++ b/starboard/android/shared/platform_configuration/BUILD.gn @@ -175,7 +175,9 @@ config("platform_configuration") { "-Wl,--wrap=readdir_r", ] } - configs += [ ":libraries" ] + if (enable_iamf_decode) { + configs += [ ":libiamf_config" ] + } } config("size") { @@ -219,6 +221,8 @@ config("pedantic_warnings") { ] } -config("libraries") { - libs = [ "//third_party/libiamf/code/android/libiamf.a" ] +if (enable_iamf_decode) { + config("libiamf_config") { + libs = [ "//third_party/libiamf/code/android/libiamf.a" ] + } } diff --git a/starboard/android/shared/player_components_factory.h b/starboard/android/shared/player_components_factory.h index 98a4d8700075..83cb08c37651 100644 --- a/starboard/android/shared/player_components_factory.h +++ b/starboard/android/shared/player_components_factory.h @@ -34,7 +34,6 @@ #include "starboard/common/media.h" #include "starboard/common/ref_counted.h" #include "starboard/media.h" -#include "starboard/shared/libiamf/iamf_audio_decoder.h" #include "starboard/shared/opus/opus_audio_decoder.h" #include "starboard/shared/starboard/media/media_util.h" #include "starboard/shared/starboard/media/mime_type.h" @@ -49,6 +48,10 @@ #include "starboard/shared/starboard/player/filter/video_renderer_internal_impl.h" #include "starboard/shared/starboard/player/filter/video_renderer_sink.h" +#if ENABLE_IAMF_DECODE +#include "starboard/shared/libiamf/iamf_audio_decoder.h" +#endif // ENABLE_IAMF_DECODE + namespace starboard { namespace android { namespace shared { @@ -180,7 +183,6 @@ class PlayerComponentsPassthrough class PlayerComponentsFactory : public starboard::shared::starboard::player:: filter::PlayerComponents::Factory { typedef starboard::shared::starboard::media::MimeType MimeType; - typedef starboard::shared::libiamf::IamfAudioDecoder IamfAudioDecoder; typedef starboard::shared::opus::OpusAudioDecoder OpusAudioDecoder; typedef starboard::shared::starboard::player::filter::AdaptiveAudioDecoder AdaptiveAudioDecoder; @@ -446,14 +448,18 @@ class PlayerComponentsFactory : public starboard::shared::starboard::player:: return std::unique_ptr( std::move(audio_decoder_impl)); } +#if SB_API_VERSION >= 15 && ENABLE_IAMF_DECODE } else if (audio_stream_info.codec == kSbMediaAudioCodecIamf) { SB_LOG(INFO) << "Creating IAMF audio decoder"; - std::unique_ptr audio_decoder_impl( - new IamfAudioDecoder(audio_stream_info)); + std::unique_ptr + audio_decoder_impl( + new starboard::shared::libiamf::IamfAudioDecoder( + audio_stream_info, /* prefer_binarual_audio */ false)); if (audio_decoder_impl->is_valid()) { return std::unique_ptr( std::move(audio_decoder_impl)); } +#endif // SB_API_VERSION >= 15 && ENABLE_IAMF_DECODE } else { SB_LOG(ERROR) << "Unsupported audio codec " << audio_stream_info.codec; diff --git a/starboard/build/config/BUILDCONFIG.gn b/starboard/build/config/BUILDCONFIG.gn index 77d7034b160e..ad514f23c8de 100644 --- a/starboard/build/config/BUILDCONFIG.gn +++ b/starboard/build/config/BUILDCONFIG.gn @@ -40,6 +40,8 @@ declare_args() { build_with_separate_cobalt_toolchain = false use_xcode_clang = false + + enable_iamf_decode = false } _is_on_pythonpath = exec_script("//starboard/build/is_on_path.py", [], "json") diff --git a/starboard/linux/shared/BUILD.gn b/starboard/linux/shared/BUILD.gn index ee78adeee885..bea587c63a10 100644 --- a/starboard/linux/shared/BUILD.gn +++ b/starboard/linux/shared/BUILD.gn @@ -125,10 +125,6 @@ static_library("starboard_platform_sources") { "//starboard/shared/libfdkaac/fdk_aac_audio_decoder.h", "//starboard/shared/libfdkaac/libfdkaac_library_loader.cc", "//starboard/shared/libfdkaac/libfdkaac_library_loader.h", - "//starboard/shared/libiamf/iamf_audio_decoder.cc", - "//starboard/shared/libiamf/iamf_audio_decoder.h", - "//starboard/shared/libiamf/iamf_config_reader.cc", - "//starboard/shared/libiamf/iamf_config_reader.h", "//starboard/shared/libvpx/vpx_video_decoder.cc", "//starboard/shared/libvpx/vpx_video_decoder.h", "//starboard/shared/linux/byte_swap.cc", @@ -444,6 +440,17 @@ static_library("starboard_platform_sources") { if (is_debug || is_devel) { defines += [ "SB_PLAYER_ENABLE_VIDEO_DUMPER" ] } + + if (enable_iamf_decode) { + sources += [ + "//starboard/shared/libiamf/iamf_audio_decoder.cc", + "//starboard/shared/libiamf/iamf_audio_decoder.h", + "//starboard/shared/libiamf/iamf_config_reader.cc", + "//starboard/shared/libiamf/iamf_config_reader.h", + ] + + defines += [ "ENABLE_IAMF_DECODE" ] + } } if (current_toolchain == starboard_toolchain) { diff --git a/starboard/linux/shared/media_is_audio_supported.cc b/starboard/linux/shared/media_is_audio_supported.cc index 6f686eb7737f..8541cd76c55b 100644 --- a/starboard/linux/shared/media_is_audio_supported.cc +++ b/starboard/linux/shared/media_is_audio_supported.cc @@ -32,7 +32,7 @@ bool SbMediaIsAudioSupported(SbMediaAudioCodec audio_codec, return bitrate <= kSbMediaMaxAudioBitrateInBitsPerSecond; } -#if SB_API_VERSION >= 15 +#if SB_API_VERSION >= 15 && ENABLE_IAMF_DECODE if (audio_codec == kSbMediaAudioCodecIamf) { return bitrate <= kSbMediaMaxAudioBitrateInBitsPerSecond; } diff --git a/starboard/linux/shared/player_components_factory.cc b/starboard/linux/shared/player_components_factory.cc index 4b82253d958b..a49a33f934a6 100644 --- a/starboard/linux/shared/player_components_factory.cc +++ b/starboard/linux/shared/player_components_factory.cc @@ -25,7 +25,6 @@ #include "starboard/shared/libde265/de265_video_decoder.h" #include "starboard/shared/libfdkaac/fdk_aac_audio_decoder.h" #include "starboard/shared/libfdkaac/libfdkaac_library_loader.h" -#include "starboard/shared/libiamf/iamf_audio_decoder.h" #include "starboard/shared/libvpx/vpx_video_decoder.h" #include "starboard/shared/openh264/openh264_library_loader.h" #include "starboard/shared/openh264/openh264_video_decoder.h" @@ -42,6 +41,10 @@ #include "starboard/shared/starboard/player/filter/video_render_algorithm_impl.h" #include "starboard/shared/starboard/player/filter/video_renderer_sink.h" +#if ENABLE_IAMF_DECODE +#include "starboard/shared/libiamf/iamf_audio_decoder.h" +#endif // ENABLE_IAMF_DECODE + namespace starboard { namespace shared { namespace starboard { @@ -69,7 +72,6 @@ class PlayerComponentsFactory : public PlayerComponents::Factory { SB_DCHECK(audio_renderer_sink); typedef ::starboard::shared::ffmpeg::AudioDecoder FfmpegAudioDecoder; - typedef ::starboard::shared::libiamf::IamfAudioDecoder IamfAudioDecoder; typedef ::starboard::shared::opus::OpusAudioDecoder OpusAudioDecoder; typedef ::starboard::shared::libfdkaac::FdkAacAudioDecoder FdkAacAudioDecoder; @@ -88,10 +90,13 @@ class PlayerComponentsFactory : public PlayerComponents::Factory { libfdkaac::LibfdkaacHandle::GetHandle()->IsLoaded()) { SB_LOG(INFO) << "Playing audio using FdkAacAudioDecoder."; return std::unique_ptr(new FdkAacAudioDecoder()); +#if SB_API_VERSION >= 15 && ENABLE_IAMF_DECODE } else if (audio_stream_info.codec == kSbMediaAudioCodecIamf) { SB_LOG(INFO) << "Playing audio using IamfAudioDecoder."; return std::unique_ptr( - new IamfAudioDecoder(audio_stream_info)); + new ::starboard::shared::libiamf::IamfAudioDecoder( + audio_stream_info, /* prefer_binarual_audio */ false)); +#endif // SB_API_VERSION >= 15 && ENABLE_IAMF_DECODE } else { std::unique_ptr audio_decoder_impl( FfmpegAudioDecoder::Create(audio_stream_info)); diff --git a/starboard/linux/x64x11/BUILD.gn b/starboard/linux/x64x11/BUILD.gn index 3666f36cb196..fa50bb0cff0c 100644 --- a/starboard/linux/x64x11/BUILD.gn +++ b/starboard/linux/x64x11/BUILD.gn @@ -34,5 +34,6 @@ if (sb_is_modular && !sb_is_evergreen) { sources = [ "//starboard/linux/x64x11/main.cc" ] configs += [ "//starboard/build/config:starboard_implementation" ] public_deps = [ ":starboard_platform" ] + deps = [ "//third_party/flac" ] } } diff --git a/starboard/linux/x64x11/shared/platform_configuration/BUILD.gn b/starboard/linux/x64x11/shared/platform_configuration/BUILD.gn index 4874cf39e506..5ebd592449ed 100644 --- a/starboard/linux/x64x11/shared/platform_configuration/BUILD.gn +++ b/starboard/linux/x64x11/shared/platform_configuration/BUILD.gn @@ -30,10 +30,12 @@ config("platform_configuration") { config("libraries") { configs = [ "//starboard/linux/shared/platform_configuration:libraries" ] - libs = [ - "//third_party/libiamf/code/libiamf.a", - "//third_party/libiamf/source/code/dep_codecs/lib/libfdk-aac.a", - ] + if (enable_iamf_decode) { + libs = [ + "//third_party/libiamf/code/libiamf.a", + "//third_party/libiamf/source/code/dep_codecs/lib/libfdk-aac.a", + ] + } } config("linker_flags") { diff --git a/starboard/shared/libiamf/iamf_audio_decoder.cc b/starboard/shared/libiamf/iamf_audio_decoder.cc index 075aa44a2d07..acc2f395cfb0 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.cc +++ b/starboard/shared/libiamf/iamf_audio_decoder.cc @@ -15,6 +15,7 @@ #include "starboard/shared/libiamf/iamf_audio_decoder.h" #include +#include #include "third_party/libiamf/source/code/include/IAMF_defines.h" @@ -24,10 +25,39 @@ namespace libiamf { namespace { using shared::starboard::player::DecodedAudio; + +constexpr int kForceBinauralAudio = false; +// Keep disabled as surround audio may require changes further up the SbPlayer. +constexpr int kEnableSurroundAudio = false; + +std::string ErrorCodeToString(int code) { + switch (code) { + case IAMF_OK: + return "IAMF_OK"; + case IAMF_ERR_BAD_ARG: + return "IAMF_ERR_BAD_ARG"; + case IAMF_ERR_BUFFER_TOO_SMALL: + return "IAMF_ERR_BUFFER_TOO_SMALL"; + case IAMF_ERR_INTERNAL: + return "IAMF_ERR_INTERNAL"; + case IAMF_ERR_INVALID_PACKET: + return "IAMF_ERR_INVALID_PACKET"; + case IAMF_ERR_INVALID_STATE: + return "IAMF_ERR_INVALID_STATE"; + case IAMF_ERR_UNIMPLEMENTED: + return "IAMF_ERR_UNIMPLEMENTED"; + case IAMF_ERR_ALLOC_FAIL: + return "IAMF_ERR_ALLOC_FAIL"; + default: + return "Unknown IAMF error code " + std::to_string(code); + } +} } // namespace -IamfAudioDecoder::IamfAudioDecoder(const AudioStreamInfo& audio_stream_info) - : audio_stream_info_(audio_stream_info) { +IamfAudioDecoder::IamfAudioDecoder(const AudioStreamInfo& audio_stream_info, + bool prefer_binaural_audio) + : audio_stream_info_(audio_stream_info), + prefer_binarual_audio_(prefer_binaural_audio) { decoder_ = IAMF_decoder_open(); if (!decoder_) { SB_DLOG(ERROR) << "Error creating libiamf decoder"; @@ -62,9 +92,10 @@ void IamfAudioDecoder::Decode(const InputBuffers& input_buffers, SB_DCHECK(output_cb_); if (stream_ended_) { - SB_LOG(ERROR) << "Decode() is called after WriteEndOfStream() is called."; + SB_DLOG(ERROR) << "Decode() is called after WriteEndOfStream() is called."; return; } + if (input_buffers.size() > kMinimumBuffersToDecode) { std::copy(std::begin(input_buffers), std::end(input_buffers), std::back_inserter(pending_audio_buffers_)); @@ -114,11 +145,15 @@ bool IamfAudioDecoder::DecodeInternal( SB_DCHECK(!stream_ended_ || !pending_audio_buffers_.empty()); SB_DCHECK(is_valid()); - reader_.Read(input_buffer); + reader_.ResetAndRead(input_buffer); + if (!reader_.is_valid()) { + SB_DLOG(INFO) << "Failed to parse IA Descriptors"; + error_cb_(kSbPlayerErrorDecode, "Failed to parse IA Descriptors"); + return false; + } if (!decoder_is_configured_) { - bool decoder_initialized = InitializeCodec(); - if (!decoder_initialized) { - SB_LOG(INFO) << "Failed to initialize IAMF decoder"; + if (!InitializeCodec()) { + SB_DLOG(INFO) << "Failed to initialize IAMF decoder"; error_cb_(kSbPlayerErrorDecode, "Failed to initialize IAMF decoder"); return false; } @@ -127,32 +162,31 @@ bool IamfAudioDecoder::DecodeInternal( scoped_refptr decoded_audio = new DecodedAudio( audio_stream_info_.number_of_channels, GetSampleType(), kSbMediaAudioFrameStorageTypeInterleaved, input_buffer->timestamp(), - audio_stream_info_.number_of_channels * kMaxIamfFramesPerAU * + audio_stream_info_.number_of_channels * reader_.samples_per_buffer() * starboard::media::GetBytesPerSample(GetSampleType())); int samples_decoded = IAMF_decoder_decode( decoder_, reader_.data().data(), reader_.data().size(), nullptr, reinterpret_cast(decoded_audio->data())); if (samples_decoded < 1) { - SB_LOG(INFO) << "IAMF_decoder_decode() error " << std::hex - << samples_decoded; - error_cb_(kSbPlayerErrorDecode, "Failed to decode IAMF sample"); + SB_DLOG(INFO) << "IAMF_decoder_decode() error " + << ErrorCodeToString(samples_decoded); + error_cb_(kSbPlayerErrorDecode, "Failed to decode IAMF sample, error " + + ErrorCodeToString(samples_decoded)); return false; } - SB_DCHECK(samples_decoded <= kMaxIamfFramesPerAU); - SB_LOG(INFO) << "Decoded " << samples_decoded << " samples"; + SB_DCHECK(samples_decoded <= reader_.samples_per_buffer()); - frames_per_au_ = samples_decoded; decoded_audio->ShrinkTo(audio_stream_info_.number_of_channels * - frames_per_au_ * + reader_.samples_per_buffer() * starboard::media::GetBytesPerSample(GetSampleType())); - // TODO: Enable once float32 pcm output is fixed. - // const auto& sample_info = input_buffer->audio_sample_info(); - // decoded_audio->AdjustForDiscardedDurations( - // audio_stream_info_.samples_per_second, - // sample_info.discarded_duration_from_front, - // sample_info.discarded_duration_from_back); + // TODO: Enable partial audio once float32 pcm output is available. + const auto& sample_info = input_buffer->audio_sample_info(); + decoded_audio->AdjustForDiscardedDurations( + audio_stream_info_.samples_per_second, + sample_info.discarded_duration_from_front, + sample_info.discarded_duration_from_back); decoded_audios_.push(decoded_audio); @@ -178,95 +212,116 @@ void IamfAudioDecoder::WriteEndOfStream() { bool IamfAudioDecoder::InitializeCodec() { SB_DCHECK(is_valid()); - int channels = audio_stream_info_.number_of_channels; - if (channels > 8 || channels < 1) { - SB_DLOG(ERROR) << "Can't create decoder with " << channels << " channels"; - return false; - } + SB_DCHECK(!decoder_is_configured_); // TODO: libiamf has an issue outputting 32 bit float samples, set to 16 bit // for now. int error = IAMF_decoder_set_bit_depth(decoder_, 16); if (error != IAMF_OK) { - SB_DLOG(ERROR) << "IAMF_decoder_set_bit_depth() fails with error " << error; + SB_DLOG(ERROR) << "IAMF_decoder_set_bit_depth() fails with error " + << ErrorCodeToString(error); return false; } error = IAMF_decoder_set_sampling_rate(decoder_, reader_.sample_rate()); if (error != IAMF_OK) { - SB_LOG(ERROR) << "IAMF_decoder_set_sampling_rate() fails with error " - << error; + SB_DLOG(ERROR) << "IAMF_decoder_set_sampling_rate() fails with error " + << ErrorCodeToString(error); return false; } - IAMF_SoundSystem sound_system = SOUND_SYSTEM_INVALID; - switch (channels) { - case 1: - sound_system = SOUND_SYSTEM_MONO; - break; - case 2: - // Stereo output. - sound_system = SOUND_SYSTEM_A; - SB_LOG(INFO) << "Configuring IamfAudioDecoder for stereo output"; - break; - case 6: - // 5.1 output. - sound_system = SOUND_SYSTEM_B; - SB_LOG(INFO) << "Configuring IamfAudioDecoder for 5.1 output"; - break; - case 8: - // 7.1 output. - sound_system = SOUND_SYSTEM_C; - SB_LOG(INFO) << "Configuring IamfAudioDecoder for 7.1 output"; - break; - default: - SB_NOTREACHED(); - } + if (kForceBinauralAudio || prefer_binarual_audio_) { + SB_DLOG(INFO) << "Configuring IamfAudioDecoder for binaural output"; + error = IAMF_decoder_output_layout_set_binaural(decoder_); + if (error != IAMF_OK) { + SB_DLOG(ERROR) + << "IAMF_decoder_output_layout_set_binaural() fails with error " + << ErrorCodeToString(error); + return false; + } + } else { + // Default to stereo output. If kEnableSurroundAudio is true, set to a sound + // system matching, the platform's audio configuration, if available. + IAMF_SoundSystem sound_system = SOUND_SYSTEM_A; + if (kEnableSurroundAudio) { + SbMediaAudioConfiguration out_config; + SbMediaGetAudioConfiguration(0, &out_config); + int channels = std::max(out_config.number_of_channels, 2); + if (channels > 8 || channels < 1) { + SB_DLOG(ERROR) << "Can't create decoder with " << channels + << " channels"; + return false; + } + switch (channels) { + case 1: + sound_system = SOUND_SYSTEM_MONO; + break; + case 2: + // Stereo output. + sound_system = SOUND_SYSTEM_A; + SB_DLOG(INFO) << "Configuring IamfAudioDecoder for stereo output"; + break; + case 6: + // 5.1 output. + sound_system = SOUND_SYSTEM_B; + SB_DLOG(INFO) << "Configuring IamfAudioDecoder for 5.1 output"; + break; + case 8: + // 7.1 output. + sound_system = SOUND_SYSTEM_C; + SB_DLOG(INFO) << "Configuring IamfAudioDecoder for 7.1 output"; + break; + default: + SB_NOTREACHED(); + } + } else { + SB_DLOG(INFO) << "Defaulting to stereo output."; + } - error = IAMF_decoder_output_layout_set_sound_system(decoder_, sound_system); - if (error != IAMF_OK) { - SB_LOG(ERROR) - << "IAMF_decoder_output_layout_set_sound_system() fails with error " - << error; - return false; + error = IAMF_decoder_output_layout_set_sound_system(decoder_, sound_system); + if (error != IAMF_OK) { + SB_DLOG(ERROR) + << "IAMF_decoder_output_layout_set_sound_system() fails with error " + << ErrorCodeToString(error); + return false; + } } - SB_LOG(INFO) << "num channels: " - << IAMF_layout_sound_system_channels_count(sound_system); - // TODO: Accurately set pts upon resume, if needed. error = IAMF_decoder_set_pts(decoder_, 0, 90000); if (error != IAMF_OK) { - SB_LOG(ERROR) << "IAMF_decoder_set_pts() fails with error " << error; + SB_DLOG(ERROR) << "IAMF_decoder_set_pts() fails with error " + << ErrorCodeToString(error); return false; } error = IAMF_decoder_set_mix_presentation_id(decoder_, reader_.mix_presentation_id()); if (error != IAMF_OK) { - SB_LOG(ERROR) << "IAMF_decoder_set_mix_presentation_id() fails with error " - << error; + SB_DLOG(ERROR) << "IAMF_decoder_set_mix_presentation_id() fails with error " + << ErrorCodeToString(error); return false; } error = IAMF_decoder_peak_limiter_enable(decoder_, 0); if (error != IAMF_OK) { - SB_LOG(ERROR) << "IAMF_decoder_peak_limiter_enable() fails with error " - << error; + SB_DLOG(ERROR) << "IAMF_decoder_peak_limiter_enable() fails with error " + << ErrorCodeToString(error); return false; } error = IAMF_decoder_set_normalization_loudness(decoder_, .0f); if (error != IAMF_OK) { - SB_LOG(ERROR) + SB_DLOG(ERROR) << "IAMF_decoder_set_normalization_loudness() fails with error " - << error; + << ErrorCodeToString(error); return false; } error = IAMF_decoder_configure(decoder_, reader_.config_obus().data(), reader_.config_size(), nullptr); if (error != IAMF_OK) { - SB_LOG(ERROR) << "IAMF_decoder_configure() fails with error " << error; + SB_DLOG(ERROR) << "IAMF_decoder_configure() fails with error " + << ErrorCodeToString(error); return false; } @@ -307,7 +362,6 @@ void IamfAudioDecoder::Reset() { decoder_is_configured_ = false; - frames_per_au_ = kMaxIamfFramesPerAU; stream_ended_ = false; while (!decoded_audios_.empty()) { decoded_audios_.pop(); @@ -320,9 +374,6 @@ void IamfAudioDecoder::Reset() { SbMediaAudioSampleType IamfAudioDecoder::GetSampleType() const { SB_DCHECK(BelongsToCurrentThread()); - // if (audio_stream_info_.bits_per_sample == 32) { - // return kSbMediaAudioSampleTypeFloat32; - // } return kSbMediaAudioSampleTypeInt16Deprecated; } diff --git a/starboard/shared/libiamf/iamf_audio_decoder.h b/starboard/shared/libiamf/iamf_audio_decoder.h index a166f60b5a86..55568dc8de8d 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.h +++ b/starboard/shared/libiamf/iamf_audio_decoder.h @@ -38,7 +38,8 @@ class IamfAudioDecoder public: typedef starboard::media::AudioStreamInfo AudioStreamInfo; - explicit IamfAudioDecoder(const AudioStreamInfo& audio_stream_info); + explicit IamfAudioDecoder(const AudioStreamInfo& audio_stream_info, + bool prefer_binaural_audio); ~IamfAudioDecoder() override; bool is_valid() const; @@ -53,8 +54,6 @@ class IamfAudioDecoder private: static constexpr int kMinimumBuffersToDecode = 2; - static constexpr int kMaxIamfFramesPerAU = 2048; - static constexpr int kMaxOpusFramesPerAU = 960; bool InitializeCodec(); void TeardownCodec(); @@ -70,7 +69,6 @@ class IamfAudioDecoder bool stream_ended_ = false; std::queue> decoded_audios_; AudioStreamInfo audio_stream_info_; - int frames_per_au_ = kMaxIamfFramesPerAU; bool decoder_is_configured_ = false; @@ -78,6 +76,8 @@ class IamfAudioDecoder ConsumedCB consumed_cb_; IamfConfigReader reader_; + + const bool prefer_binarual_audio_; }; } // namespace libiamf diff --git a/starboard/shared/libiamf/iamf_config_reader.cc b/starboard/shared/libiamf/iamf_config_reader.cc index a9fd377d0f97..2643e336a92f 100644 --- a/starboard/shared/libiamf/iamf_config_reader.cc +++ b/starboard/shared/libiamf/iamf_config_reader.cc @@ -37,6 +37,10 @@ constexpr int kFourccMp4a = 0x6d703461; constexpr int kFourccFlac = 0x664c6143; constexpr int kFourccIpcm = 0x6970636d; +constexpr int kLayoutTypeReserved = 1; +constexpr int kLayoutTypeSpeaker = 2; +constexpr int kLayoutTypeBinaural = 3; + inline uint32_t ByteSwap(uint32_t x) { #if defined(COMPILER_MSVC) return _byteswap_ulong(x); @@ -74,25 +78,44 @@ int ReadString(const uint8_t* buf, std::string& value) { value.reserve(128); int bytes_read = 0; + bool error = true; while (bytes_read < 128 && buf[bytes_read] != '\0') { value.push_back(static_cast(buf[bytes_read])); bytes_read++; + error = false; + } + + if (error) { + return -1; } return bytes_read; } } // namespace -bool IamfConfigReader::Read(scoped_refptr input_buffer) { +bool IamfConfigReader::ResetAndRead(scoped_refptr input_buffer) { Reset(); + return Read(input_buffer); +} +void IamfConfigReader::Reset() { + buffer_head_ = 0; + mix_presentation_id_ = std::optional(); + config_size_ = 0; + data_size_ = 0; + sample_rate_ = 0; + samples_per_buffer_ = 0; + sample_size_ = 0; +} + +bool IamfConfigReader::Read(scoped_refptr input_buffer) { const uint8_t* buf = input_buffer->data(); SB_DCHECK(buf); bool completed_parsing = false; while (!completed_parsing && buffer_head_ < input_buffer->size()) { if (!ReadOBU(buf, completed_parsing)) { - SB_LOG(INFO) << "Error parsing config OBUs"; + SB_DLOG(INFO) << "Error parsing config OBUs"; return false; } } @@ -106,23 +129,12 @@ bool IamfConfigReader::Read(scoped_refptr input_buffer) { return true; } -void IamfConfigReader::Reset() { - buffer_head_ = 0; - has_mix_presentation_id_ = false; - mix_presentation_id_ = 0; - config_size_ = 0; - data_size_ = 0; - sample_rate_ = 0; - samples_per_buffer_ = 0; - sample_size_ = 0; -} - bool IamfConfigReader::ReadOBU(const uint8_t* buf, bool& completed_parsing) { uint8_t obu_type = 0; uint32_t obu_size = 0; uint32_t header_size = 0; if (!ReadOBUHeader(buf, &obu_type, &obu_size, &header_size)) { - SB_LOG(ERROR) << "Error reading OBU header"; + SB_DLOG(ERROR) << "Error reading OBU header"; return false; } @@ -200,36 +212,14 @@ bool IamfConfigReader::ReadOBU(const uint8_t* buf, bool& completed_parsing) { break; case kObuTypeMixPresentation: { // TODO: Complete Mix Presentation OBU parsing - has_mix_presentation_id_ = true; - bytes_read = ReadLeb128Value(&buf[buffer_head_], &mix_presentation_id_); - if (bytes_read < 0) { - return false; - } - buffer_head_ += bytes_read; - - // count_label - bytes_read = ReadLeb128Value(&buf[buffer_head_], &count_label); + uint32_t value; + bytes_read = ReadLeb128Value(&buf[buffer_head_], &value); if (bytes_read < 0) { return false; } - buffer_head_ += bytes_read; - - // language_label - for (int i = 0; i < count_label; ++i) { - buffer_head_ += ReadString(&buf[buffer_head_], str); - } - - // MixPresentationAnnotations - for (int i = 0; i < count_label; ++i) { - buffer_head_ += ReadString(&buf[buffer_head_], str); - } - - // num_sub_mixes - bytes_read = ReadLeb128Value(&buf[buffer_head_], &num_sub_mixes); - if (bytes_read < 0) { - return false; + if (!mix_presentation_id_.has_value()) { + mix_presentation_id_ = value; } - buffer_head_ += bytes_read; break; } diff --git a/starboard/shared/libiamf/iamf_config_reader.h b/starboard/shared/libiamf/iamf_config_reader.h index 04a6d76d98ae..aa2e7bc529ab 100644 --- a/starboard/shared/libiamf/iamf_config_reader.h +++ b/starboard/shared/libiamf/iamf_config_reader.h @@ -15,8 +15,10 @@ #ifndef STARBOARD_SHARED_LIBIAMF_IAMF_CONFIG_READER_H_ #define STARBOARD_SHARED_LIBIAMF_IAMF_CONFIG_READER_H_ +#include #include +#include "starboard/common/log.h" #include "starboard/common/ref_counted.h" #include "starboard/shared/internal_only.h" #include "starboard/shared/starboard/player/input_buffer_internal.h" @@ -26,33 +28,37 @@ namespace shared { namespace libiamf { // TODO: Add handling for non-redundant OBUS +// TODO: Implement or depend on a buffer reader to simplify the implementation. class IamfConfigReader { public: typedef ::starboard::shared::starboard::player::InputBuffer InputBuffer; IamfConfigReader() = default; - bool Read(scoped_refptr input_buffer); + bool ResetAndRead(scoped_refptr input_buffer); void Reset(); bool is_valid() { - return data_size_ > 0 && config_size_ > 0 && has_mix_presentation_id_ && - sample_rate_ > 0; + return data_size_ > 0 && config_size_ > 0 && has_mix_presentation_id() && + sample_rate_ > 0 && samples_per_buffer_ > 0; } int sample_rate() { return sample_rate_; } uint32_t samples_per_buffer() { return samples_per_buffer_; } - bool config_changed() { return false; } uint32_t config_size() { return config_size_; } - // TODO: Allow for selection of multiple mix presentation IDs. - bool has_mix_presentation_id() { return has_mix_presentation_id_; } - uint32_t mix_presentation_id() { return mix_presentation_id_; } - uint32_t data_size() { return data_size_; } + // TODO: Allow for selection of multiple mix presentation IDs. Currently, + // only the first mix presentation parsed is selected. + bool has_mix_presentation_id() { return mix_presentation_id_.has_value(); } + uint32_t mix_presentation_id() { + SB_DCHECK(mix_presentation_id_.has_value()); + return mix_presentation_id_.value(); + } std::vector config_obus() { return config_obus_; } std::vector data() { return data_; } private: + bool Read(scoped_refptr input_buffer); bool ReadOBU(const uint8_t* buf, bool& completed_parsing); bool ReadOBUHeader(const uint8_t* buf, uint8_t* obu_type, @@ -66,10 +72,10 @@ class IamfConfigReader { uint32_t config_size_ = 0; uint32_t data_size_ = 0; - bool has_mix_presentation_id_ = false; - uint32_t mix_presentation_id_ = 0; + std::optional mix_presentation_id_; bool has_valid_config_ = false; + bool binaural_mix_presentation_id_ = -1; std::vector config_obus_; std::vector data_; }; From 819d65b35b78e4b9453b03b5cd546da8bd4658f3 Mon Sep 17 00:00:00 2001 From: Austin Osagie Date: Thu, 18 Jul 2024 15:53:53 -0700 Subject: [PATCH 06/30] Remove flac dep --- starboard/linux/x64x11/BUILD.gn | 1 - 1 file changed, 1 deletion(-) diff --git a/starboard/linux/x64x11/BUILD.gn b/starboard/linux/x64x11/BUILD.gn index fa50bb0cff0c..3666f36cb196 100644 --- a/starboard/linux/x64x11/BUILD.gn +++ b/starboard/linux/x64x11/BUILD.gn @@ -34,6 +34,5 @@ if (sb_is_modular && !sb_is_evergreen) { sources = [ "//starboard/linux/x64x11/main.cc" ] configs += [ "//starboard/build/config:starboard_implementation" ] public_deps = [ ":starboard_platform" ] - deps = [ "//third_party/flac" ] } } From 8a1aa26bdd9ce4671204402fd8a259dd8084aaeb Mon Sep 17 00:00:00 2001 From: Austin Osagie Date: Mon, 5 Aug 2024 12:20:22 -0700 Subject: [PATCH 07/30] Move location of libiamf binaries --- starboard/android/shared/platform_configuration/BUILD.gn | 2 +- starboard/linux/x64x11/shared/platform_configuration/BUILD.gn | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/starboard/android/shared/platform_configuration/BUILD.gn b/starboard/android/shared/platform_configuration/BUILD.gn index dfcdc08ab6d9..2885fccd10f5 100644 --- a/starboard/android/shared/platform_configuration/BUILD.gn +++ b/starboard/android/shared/platform_configuration/BUILD.gn @@ -223,6 +223,6 @@ config("pedantic_warnings") { if (enable_iamf_decode) { config("libiamf_config") { - libs = [ "//third_party/libiamf/code/android/libiamf.a" ] + libs = [ "//third_party/libiamf/platforms/android/libiamf.a" ] } } diff --git a/starboard/linux/x64x11/shared/platform_configuration/BUILD.gn b/starboard/linux/x64x11/shared/platform_configuration/BUILD.gn index 5ebd592449ed..21c6d95015f0 100644 --- a/starboard/linux/x64x11/shared/platform_configuration/BUILD.gn +++ b/starboard/linux/x64x11/shared/platform_configuration/BUILD.gn @@ -32,7 +32,7 @@ config("libraries") { configs = [ "//starboard/linux/shared/platform_configuration:libraries" ] if (enable_iamf_decode) { libs = [ - "//third_party/libiamf/code/libiamf.a", + "//third_party/libiamf/platforms/linux/libiamf.a", "//third_party/libiamf/source/code/dep_codecs/lib/libfdk-aac.a", ] } From 4b6608af781f9803a405e10de3cbc405f9bc8d63 Mon Sep 17 00:00:00 2001 From: Austin Osagie Date: Sun, 1 Sep 2024 03:12:51 -0700 Subject: [PATCH 08/30] Max IamfConfigReader actually servicable --- .../shared/platform_configuration/BUILD.gn | 5 +- .../shared/libiamf/iamf_audio_decoder.cc | 7 +- starboard/shared/libiamf/iamf_audio_decoder.h | 2 +- .../shared/libiamf/iamf_config_reader.cc | 514 +++++++++++++----- starboard/shared/libiamf/iamf_config_reader.h | 100 +++- 5 files changed, 467 insertions(+), 161 deletions(-) diff --git a/starboard/linux/x64x11/shared/platform_configuration/BUILD.gn b/starboard/linux/x64x11/shared/platform_configuration/BUILD.gn index 21c6d95015f0..39fed6b2bce2 100644 --- a/starboard/linux/x64x11/shared/platform_configuration/BUILD.gn +++ b/starboard/linux/x64x11/shared/platform_configuration/BUILD.gn @@ -31,10 +31,7 @@ config("platform_configuration") { config("libraries") { configs = [ "//starboard/linux/shared/platform_configuration:libraries" ] if (enable_iamf_decode) { - libs = [ - "//third_party/libiamf/platforms/linux/libiamf.a", - "//third_party/libiamf/source/code/dep_codecs/lib/libfdk-aac.a", - ] + libs = [ "//third_party/libiamf/platforms/linux/libiamf.a" ] } } diff --git a/starboard/shared/libiamf/iamf_audio_decoder.cc b/starboard/shared/libiamf/iamf_audio_decoder.cc index acc2f395cfb0..9f88f5a83e3c 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.cc +++ b/starboard/shared/libiamf/iamf_audio_decoder.cc @@ -57,7 +57,8 @@ std::string ErrorCodeToString(int code) { IamfAudioDecoder::IamfAudioDecoder(const AudioStreamInfo& audio_stream_info, bool prefer_binaural_audio) : audio_stream_info_(audio_stream_info), - prefer_binarual_audio_(prefer_binaural_audio) { + prefer_binarual_audio_(prefer_binaural_audio), + reader_(false, false) { decoder_ = IAMF_decoder_open(); if (!decoder_) { SB_DLOG(ERROR) << "Error creating libiamf decoder"; @@ -223,7 +224,7 @@ bool IamfAudioDecoder::InitializeCodec() { return false; } - error = IAMF_decoder_set_sampling_rate(decoder_, reader_.sample_rate()); + error = IAMF_decoder_set_sampling_rate(decoder_, 48000); if (error != IAMF_OK) { SB_DLOG(ERROR) << "IAMF_decoder_set_sampling_rate() fails with error " << ErrorCodeToString(error); @@ -348,7 +349,7 @@ scoped_refptr IamfAudioDecoder::Read( result = decoded_audios_.front(); decoded_audios_.pop(); } - *samples_per_second = reader_.sample_rate(); + *samples_per_second = 48000; return result; } diff --git a/starboard/shared/libiamf/iamf_audio_decoder.h b/starboard/shared/libiamf/iamf_audio_decoder.h index 55568dc8de8d..d89958c1fdde 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.h +++ b/starboard/shared/libiamf/iamf_audio_decoder.h @@ -65,7 +65,7 @@ class IamfAudioDecoder OutputCB output_cb_; ErrorCB error_cb_; - IAMF_Decoder* decoder_ = NULL; + IAMF_Decoder* decoder_ = nullptr; bool stream_ended_ = false; std::queue> decoded_audios_; AudioStreamInfo audio_stream_info_; diff --git a/starboard/shared/libiamf/iamf_config_reader.cc b/starboard/shared/libiamf/iamf_config_reader.cc index 2643e336a92f..d1a4e904d1b6 100644 --- a/starboard/shared/libiamf/iamf_config_reader.cc +++ b/starboard/shared/libiamf/iamf_config_reader.cc @@ -14,15 +14,27 @@ #include "starboard/shared/libiamf/iamf_config_reader.h" +#include #include #include "starboard/common/string.h" +#include "third_party/libiamf/source/code/include/IAMF_defines.h" namespace starboard { namespace shared { namespace libiamf { namespace { + +// From //media/formats/mp4/rcheck.h. +#define RCHECK(condition) \ + do { \ + if (!(condition)) { \ + SB_DLOG(ERROR) << "Failure while parsing IAMF config: " #condition; \ + return false; \ + } \ + } while (0) + // From // https://aomediacodec.github.io/iamf/v1.0.0-errata.html#obu-header-syntax. constexpr int kObuTypeCodecConfig = 0; @@ -33,25 +45,78 @@ constexpr int kObuTypeSequenceHeader = 31; // From // https://aomediacodec.github.io/iamf/v1.0.0-errata.html#obu-codecconfig. constexpr int kFourccOpus = 0x4f707573; -constexpr int kFourccMp4a = 0x6d703461; -constexpr int kFourccFlac = 0x664c6143; constexpr int kFourccIpcm = 0x6970636d; -constexpr int kLayoutTypeReserved = 1; -constexpr int kLayoutTypeSpeaker = 2; -constexpr int kLayoutTypeBinaural = 3; +} // namespace + +IamfConfigReader::BufferReader::BufferReader(const uint8_t* buf, size_t size) + : buf_(buf), pos_(0), size_(size), error_(false) {} + +bool IamfConfigReader::BufferReader::Read1(uint8_t* ptr) { + if (!HasBytes(sizeof(uint8_t)) || !ptr) { + error_ = true; + return false; + } + *ptr = buf_[pos_++]; + return true; +} + +bool IamfConfigReader::BufferReader::Read4(uint32_t* ptr) { + if (!HasBytes(sizeof(uint32_t)) || !ptr) { + error_ = true; + return false; + } + std::memcpy(ptr, &buf_[pos_], sizeof(uint32_t)); + *ptr = ByteSwap(*ptr); + pos_ += sizeof(uint32_t); + return true; +} + +bool IamfConfigReader::BufferReader::ReadLeb128(uint32_t* ptr) { + if (!HasBytes(sizeof(uint32_t)) || !ptr) { + error_ = true; + return false; + } + int bytes_read = ReadLeb128Internal(buf_ + pos_, ptr); + if (bytes_read < 0) { + error_ = true; + return false; + } + pos_ += bytes_read; + return true; +} + +bool IamfConfigReader::BufferReader::ReadString(std::string& str) { + int bytes_read = ReadStringInternal(buf_ + pos_, str); + if (bytes_read < 0) { + error_ = true; + return false; + } + pos_ += bytes_read; + return true; +} + +bool IamfConfigReader::BufferReader::SkipBytes(size_t size) { + if (!HasBytes(size)) { + error_ = true; + return false; + } + pos_ += size; + return true; +} + +bool IamfConfigReader::BufferReader::SkipLeb128() { + uint32_t val; + return ReadLeb128(&val); +} -inline uint32_t ByteSwap(uint32_t x) { -#if defined(COMPILER_MSVC) - return _byteswap_ulong(x); -#else - return __builtin_bswap32(x); -#endif +bool IamfConfigReader::BufferReader::SkipString() { + std::string str; + return ReadString(str); } -// Decodes an Leb128 value and stores it in |value|. Returns the number of bytes -// read. Returns -1 on error. -int ReadLeb128Value(const uint8_t* buf, uint32_t* value) { +int IamfConfigReader::BufferReader::ReadLeb128Internal(const uint8_t* buf, + uint32_t* value) { SB_DCHECK(buf); SB_DCHECK(value); *value = 0; @@ -72,26 +137,46 @@ int ReadLeb128Value(const uint8_t* buf, uint32_t* value) { return i + 1; } -int ReadString(const uint8_t* buf, std::string& value) { +int IamfConfigReader::BufferReader::ReadStringInternal(const uint8_t* buf, + std::string& str) { SB_DCHECK(buf); - value = ""; - value.reserve(128); - int bytes_read = 0; - bool error = true; - while (bytes_read < 128 && buf[bytes_read] != '\0') { - value.push_back(static_cast(buf[bytes_read])); - bytes_read++; - error = false; + int remaining_size = static_cast(size_) - pos_; + if (remaining_size <= 0) { + return -1; } - if (error) { + // The size of the string is capped to 128 bytes. + int max_str_size = std::min(remaining_size, 128); + str.clear(); + + size_t size = 0; + while (buf[size] != '\0' && size < max_str_size) { + size++; + } + + if (buf[size] != '\0') { return -1; } - return bytes_read; + if (size > 0) { + str.resize(size); + std::memcpy(str.data(), reinterpret_cast(buf), size); + } + + // Account for null terminator byte. + return ++size; +} + +IamfConfigReader::IamfConfigReader(bool prefer_binaural_audio, + bool prefer_surround_audio) + : prefer_binaural_audio_(prefer_binaural_audio), + prefer_surround_audio_(prefer_surround_audio) { +#if SB_IS_BIG_ENDIAN +#error IamfConfigReader assumes little-endianness. +#endif // SB_IS_BIG_ENDIAN + SB_DCHECK(!(prefer_binaural_audio && prefer_surround_audio)); } -} // namespace bool IamfConfigReader::ResetAndRead(scoped_refptr input_buffer) { Reset(); @@ -99,8 +184,8 @@ bool IamfConfigReader::ResetAndRead(scoped_refptr input_buffer) { } void IamfConfigReader::Reset() { - buffer_head_ = 0; mix_presentation_id_ = std::optional(); + binaural_audio_element_ids_ = std::unordered_set(); config_size_ = 0; data_size_ = 0; sample_rate_ = 0; @@ -109,116 +194,295 @@ void IamfConfigReader::Reset() { } bool IamfConfigReader::Read(scoped_refptr input_buffer) { - const uint8_t* buf = input_buffer->data(); - SB_DCHECK(buf); + SB_DCHECK(input_buffer->data()); + BufferReader reader(input_buffer->data(), input_buffer->size()); - bool completed_parsing = false; - while (!completed_parsing && buffer_head_ < input_buffer->size()) { - if (!ReadOBU(buf, completed_parsing)) { - SB_DLOG(INFO) << "Error parsing config OBUs"; - return false; - } + while (!is_valid() && reader.pos() < reader.size()) { + RCHECK(ReadOBU(&reader)); } - SB_CHECK(completed_parsing); + if (reader.error()) { + return false; + } - data_size_ = input_buffer->size() - config_size_; - config_obus_.assign(buf, buf + config_size_); - data_.assign(buf + config_size_, buf + input_buffer->size()); + data_size_ = reader.size() - config_size_; + config_obus_.assign(reader.buf(), reader.buf() + config_size_); + data_.assign(reader.buf() + config_size_, reader.buf() + reader.size()); return true; } -bool IamfConfigReader::ReadOBU(const uint8_t* buf, bool& completed_parsing) { +bool IamfConfigReader::ReadOBU(BufferReader* reader) { + SB_DCHECK(reader); uint8_t obu_type = 0; uint32_t obu_size = 0; - uint32_t header_size = 0; - if (!ReadOBUHeader(buf, &obu_type, &obu_size, &header_size)) { + if (!ReadOBUHeader(reader, &obu_type, &obu_size)) { SB_DLOG(ERROR) << "Error reading OBU header"; return false; } - int next_obu_pos = buffer_head_ + obu_size; - int bytes_read = 0; - - uint32_t count_label = 0; - std::string str; - uint32_t num_sub_mixes = 0; + int next_obu_pos = reader->pos() + obu_size; + + auto skip_param_definition = [&]() { + // parameter_id + RCHECK(reader->SkipLeb128()); + // parameter_rate + RCHECK(reader->SkipLeb128()); + uint8_t param_definition_mode; + RCHECK(reader->Read1(¶m_definition_mode)); + param_definition_mode = param_definition_mode >> 7; + if (param_definition_mode == static_cast(0)) { + // duration + RCHECK(reader->SkipLeb128()); + uint32_t constant_subblock_duration; + RCHECK(reader->ReadLeb128(&constant_subblock_duration)); + if (constant_subblock_duration == 0) { + uint32_t num_subblocks; + RCHECK(reader->ReadLeb128(&num_subblocks)); + for (int i = 0; i < num_subblocks; ++i) { + // subblock_duration + RCHECK(reader->SkipLeb128()); + } + } + } + return true; + }; switch (static_cast(obu_type)) { case kObuTypeCodecConfig: { - sample_rate_ = 0; - uint32_t codec_config_id; - bytes_read = ReadLeb128Value(&buf[buffer_head_], &codec_config_id); - if (bytes_read < 0) { - return false; - } - buffer_head_ += bytes_read; + RCHECK(reader->SkipLeb128()); uint32_t codec_id = 0; - std::memcpy(&codec_id, &buf[buffer_head_], sizeof(uint32_t)); - // Mp4 is in big-endian - codec_id = ByteSwap(codec_id); - buffer_head_ += 4; - - bytes_read = ReadLeb128Value(&buf[buffer_head_], &samples_per_buffer_); - if (bytes_read < 0) { - return false; - } - buffer_head_ += bytes_read; + RCHECK(reader->Read4(&codec_id)); + + RCHECK(reader->ReadLeb128(&samples_per_buffer_)); // audio_roll_distance - buffer_head_ += 2; + RCHECK(reader->SkipBytes(2)); switch (codec_id) { case kFourccOpus: sample_rate_ = 48000; break; - case kFourccMp4a: - // TODO: Properly parse for the sample rate. 48000 is the assumed - // sample rate. - sample_rate_ = 48000; - break; - case kFourccFlac: { - // Skip METADATA_BLOCK_HEADER. - buffer_head_ += 4; - // Skip first 10 bytes of METADATA_BLOCK_STREAMINFO. - buffer_head_ += 10; - - std::memcpy(&sample_rate_, &buf[buffer_head_], sizeof(uint32_t)); - sample_rate_ = ByteSwap(sample_rate_); - sample_rate_ = sample_rate_ >> 12; - break; - } case kFourccIpcm: { // sample_format_flags - buffer_head_++; - uint8_t sample_size_byte = buf[buffer_head_]; - sample_size_ = static_cast(sample_size_byte); - buffer_head_++; + RCHECK(reader->SkipBytes(1)); - std::memcpy(&sample_rate_, &buf[buffer_head_], sizeof(uint32_t)); - sample_rate_ = ByteSwap(sample_rate_); + uint8_t sample_size_unsigned; + RCHECK(reader->Read1(&sample_size_unsigned)); + sample_size_ = static_cast(sample_size_unsigned); + + uint32_t sample_rate_unsigned; + RCHECK(reader->Read4(&sample_rate_unsigned)); + sample_rate_ = static_cast(sample_rate_unsigned); break; } default: SB_NOTREACHED(); + return false; } break; } - case kObuTypeAudioElement: + case kObuTypeAudioElement: { + uint32_t audio_element_id; + RCHECK(reader->ReadLeb128(&audio_element_id)); + + uint8_t audio_element_type; + RCHECK(reader->Read1(&audio_element_type)); + audio_element_type = audio_element_type >> 5; + + // codec_config_id + RCHECK(reader->SkipLeb128()); + + uint32_t num_substreams; + RCHECK(reader->ReadLeb128(&num_substreams)); + + for (int i = 0; i < num_substreams; ++i) { + // audio_substream_id + RCHECK(reader->SkipLeb128()); + } + + uint32_t num_parameters; + RCHECK(reader->ReadLeb128(&num_parameters)); + + for (int i = 0; i < num_parameters; ++i) { + uint32_t param_definition_type; + RCHECK(reader->ReadLeb128(¶m_definition_type)); + + if (param_definition_type == IAMF_PARAMETER_TYPE_DEMIXING) { + skip_param_definition(); + // DemixingParamDefintion + RCHECK(reader->SkipBytes(1)); + } else if (param_definition_type == IAMF_PARAMETER_TYPE_RECON_GAIN) { + skip_param_definition(); + } else if (param_definition_type > 2) { + uint32_t param_definition_size; + RCHECK(reader->ReadLeb128(¶m_definition_size)); + RCHECK(reader->SkipBytes(param_definition_size)); + } + } + + if (static_cast(audio_element_type) == + AUDIO_ELEMENT_CHANNEL_BASED && + (prefer_binaural_audio_ || prefer_surround_audio_)) { + // Parse ScalableChannelLayoutConfig for binaural and surround + // loudspeaker layouts + uint8_t num_layers; + RCHECK(reader->Read1(&num_layers)); + num_layers = num_layers >> 5; + // Read ChannelAudioLayerConfigs + for (int i = 0; i < static_cast(num_layers); ++i) { + uint8_t loudspeaker_layout; + bool output_gain_is_present_flag; + RCHECK(reader->Read1(&loudspeaker_layout)); + output_gain_is_present_flag = (loudspeaker_layout >> 3) & 0x01; + loudspeaker_layout = loudspeaker_layout >> 4; + if (loudspeaker_layout == IA_CHANNEL_LAYOUT_BINAURAL) { + binaural_audio_element_ids_.insert(audio_element_id); + } else if (loudspeaker_layout > IA_CHANNEL_LAYOUT_STEREO && + loudspeaker_layout < IA_CHANNEL_LAYOUT_COUNT) { + surround_audio_element_ids_.insert(audio_element_id); + } + + // substream_count and coupled_substream_count + RCHECK(reader->SkipBytes(2)); + + if (output_gain_is_present_flag) { + // output_gain_flags and output_gain + RCHECK(reader->SkipBytes(3)); + } + + if (i == 1 && loudspeaker_layout == static_cast(15)) { + // expanded_loudspeaker_layout + RCHECK(reader->SkipBytes(1)); + } + } + } + break; + } case kObuTypeSequenceHeader: break; case kObuTypeMixPresentation: { - // TODO: Complete Mix Presentation OBU parsing - uint32_t value; - bytes_read = ReadLeb128Value(&buf[buffer_head_], &value); - if (bytes_read < 0) { - return false; + uint32_t mix_presentation_id; + RCHECK(reader->ReadLeb128(&mix_presentation_id)); + + uint32_t count_label; + RCHECK(reader->ReadLeb128(&count_label)); + for (int i = 0; i < count_label; ++i) { + // language_label; + RCHECK(reader->SkipString()); + } + + for (int i = 0; i < count_label; ++i) { + // MixPresentationAnnotations; + RCHECK(reader->SkipString()); + } + + uint32_t num_sub_mixes; + RCHECK(reader->ReadLeb128(&num_sub_mixes)); + for (int i = 0; i < num_sub_mixes; ++i) { + uint32_t num_audio_elements; + RCHECK(reader->ReadLeb128(&num_audio_elements)); + for (int j = 0; j < num_audio_elements; ++j) { + uint32_t audio_element_id; + RCHECK(reader->ReadLeb128(&audio_element_id)); + + // Set a mix presentation for binaural or surround streams. The mix + // presentation is chosen if there exists an audio element that has + // the qualities it requires - such as an audio element with a + // binaural loudspeaker layout. + if (!mix_presentation_id_.has_value() || + (prefer_binaural_audio_ && + binaural_mix_selection_ > + kBinauralMixSelectionLoudspeakerLayout)) { + if (prefer_binaural_audio_ && + binaural_audio_element_ids_.find(audio_element_id) != + binaural_audio_element_ids_.end()) { + mix_presentation_id_ = mix_presentation_id; + binaural_mix_selection_ = kBinauralMixSelectionLoudspeakerLayout; + } else if (prefer_surround_audio_ && + surround_audio_element_ids_.find(audio_element_id) != + surround_audio_element_ids_.end()) { + mix_presentation_id_ = mix_presentation_id; + } + } + + for (int k = 0; k < count_label; ++k) { + // MixPresentationElementAnnotatoions + RCHECK(reader->SkipString()); + } + + // The following fields are for the RenderingConfig + // headphones_rendering_mode + RCHECK(reader->SkipBytes(1)); + uint32_t rendering_config_extension_size; + RCHECK(reader->ReadLeb128(&rendering_config_extension_size)); + // rendering_config_extension_bytes + RCHECK(reader->SkipBytes(rendering_config_extension_size)); + + // The following fields are for the ElementMixConfig + RCHECK(skip_param_definition()); + // default_mix_gain + RCHECK(reader->SkipBytes(2)); + } + // The following fields are for the OutputMixConfig + RCHECK(skip_param_definition()); + // default_mix_gain + RCHECK(reader->SkipBytes(2)); + + uint32_t num_layouts; + RCHECK(reader->ReadLeb128(&num_layouts)); + for (int j = 0; j < num_layouts; ++j) { + uint8_t layout_type; + RCHECK(reader->Read1(&layout_type)); + layout_type = layout_type >> 6; + // If a binaural mix presentation is preferred and the mix + // presentation id has not yet been set, set the mix presentation id + // if the current mix presentation has a binaural loudness layout. The + // mix presentation id will change if a different mix presentation is + // found that uses an audio element with a binaural loudspeaker + // layout, as that is higher priority. + if (static_cast(layout_type) == IAMF_LAYOUT_TYPE_BINAURAL && + prefer_binaural_audio_ && + (!mix_presentation_id_.has_value() || + binaural_mix_selection_ > kBinauralMixSelectionLoudnessLayout)) { + mix_presentation_id_ = mix_presentation_id; + binaural_mix_selection_ = kBinauralMixSelectionLoudnessLayout; + } + + // The following fields are for the LoudnessInfo + uint8_t info_type; + RCHECK(reader->Read1(&info_type)); + // integrated_loudness and digital_loudness + RCHECK(reader->SkipBytes(4)); + if (info_type & 1) { + // true_peak + RCHECK(reader->SkipBytes(2)); + } + if (info_type & 2) { + uint8_t num_anchored_loudness; + RCHECK(reader->Read1(&num_anchored_loudness)); + for (uint8_t k = 0; k < num_anchored_loudness; ++k) { + // anchor_element and anchored_loudness + RCHECK(reader->SkipBytes(3)); + } + } + if ((info_type & 0b11111100) > 0) { + uint32_t info_type_size; + RCHECK(reader->ReadLeb128(&info_type_size)); + // info_type_bytes + RCHECK(reader->SkipBytes(info_type_size)); + } + } } + + // If the mix presentation id is unassigned at this point, the stream is + // stereo, or a proper mix presentation for binaural or surround preferred + // streams hasn't yet been parsed. Default to the first read mix + // presentation in case a preferred mix does not exist. if (!mix_presentation_id_.has_value()) { - mix_presentation_id_ = value; + mix_presentation_id_ = mix_presentation_id; } break; @@ -226,59 +490,51 @@ bool IamfConfigReader::ReadOBU(const uint8_t* buf, bool& completed_parsing) { default: // Once an OBU is read that is not a descriptor, descriptor parsing is // assumed to be complete. - completed_parsing = true; - break; + SB_DCHECK(mix_presentation_id_.has_value()); + return true; } - if (!completed_parsing) { - // Skip to next OBU - buffer_head_ = next_obu_pos; - config_size_ += obu_size + header_size; - } + // Skip to the next OBU. + const size_t remaining_size = next_obu_pos - reader->pos(); + RCHECK(reader->SkipBytes(remaining_size)); + config_size_ = reader->pos(); return true; } -bool IamfConfigReader::ReadOBUHeader(const uint8_t* buf, +bool IamfConfigReader::ReadOBUHeader(BufferReader* reader, uint8_t* obu_type, - uint32_t* obu_size, - uint32_t* header_size) { - uint8_t header_flags = buf[buffer_head_]; + uint32_t* obu_size) { + uint8_t header_flags; + RCHECK(reader->Read1(&header_flags)); *obu_type = (header_flags >> 3) & 0x1f; - buffer_head_++; const bool obu_redundant_copy = (header_flags >> 2) & 1; const bool obu_trimming_status_flag = (header_flags >> 1) & 1; const bool obu_extension_flag = header_flags & 1; - *header_size = 1; - *obu_size = 0; - int bytes_read = ReadLeb128Value(&buf[buffer_head_], obu_size); - if (bytes_read < 0) { - return false; - } - buffer_head_ += bytes_read; - *header_size += bytes_read; + + RCHECK(reader->ReadLeb128(obu_size)); + + // |obu_size| contains the size of the OBU after its own field. + // If either of the flags are set, subtract the number of bytes read + // from the flags from |obu_size| before returning to ReadOBU(). + size_t reader_pos_before_flags = reader->pos(); if (obu_trimming_status_flag) { - uint32_t value; - bytes_read = ReadLeb128Value(&buf[buffer_head_], &value); - buffer_head_ += bytes_read; - *header_size += bytes_read; - *obu_size -= bytes_read; - bytes_read = ReadLeb128Value(&buf[buffer_head_], &value); - buffer_head_ += bytes_read; - *header_size += bytes_read; - *obu_size -= bytes_read; + RCHECK(reader->SkipLeb128()); + RCHECK(reader->SkipLeb128()); } if (obu_extension_flag) { - uint32_t extension_header_size; - bytes_read = ReadLeb128Value(&buf[buffer_head_], &extension_header_size); - buffer_head_ += extension_header_size; - *obu_size -= extension_header_size; - *header_size += bytes_read; + RCHECK(reader->SkipLeb128()); + } + + size_t flag_bytes_read = reader->pos() - reader_pos_before_flags; + if (flag_bytes_read >= *obu_size) { + return false; } + *obu_size -= flag_bytes_read; return true; } diff --git a/starboard/shared/libiamf/iamf_config_reader.h b/starboard/shared/libiamf/iamf_config_reader.h index aa2e7bc529ab..0012a1d67829 100644 --- a/starboard/shared/libiamf/iamf_config_reader.h +++ b/starboard/shared/libiamf/iamf_config_reader.h @@ -16,6 +16,8 @@ #define STARBOARD_SHARED_LIBIAMF_IAMF_CONFIG_READER_H_ #include +#include +#include #include #include "starboard/common/log.h" @@ -27,45 +29,91 @@ namespace starboard { namespace shared { namespace libiamf { -// TODO: Add handling for non-redundant OBUS -// TODO: Implement or depend on a buffer reader to simplify the implementation. +// TODO: Add handling for non-redundant OBUs class IamfConfigReader { public: typedef ::starboard::shared::starboard::player::InputBuffer InputBuffer; - IamfConfigReader() = default; + IamfConfigReader(bool prefer_binaural_audio, bool prefer_surround_audio); bool ResetAndRead(scoped_refptr input_buffer); - void Reset(); - - bool is_valid() { - return data_size_ > 0 && config_size_ > 0 && has_mix_presentation_id() && - sample_rate_ > 0 && samples_per_buffer_ > 0; + bool is_valid() const { + return mix_presentation_id_.has_value() && sample_rate_ > 0 && + samples_per_buffer_ > 0; } - int sample_rate() { return sample_rate_; } - uint32_t samples_per_buffer() { return samples_per_buffer_; } - uint32_t config_size() { return config_size_; } - // TODO: Allow for selection of multiple mix presentation IDs. Currently, - // only the first mix presentation parsed is selected. - bool has_mix_presentation_id() { return mix_presentation_id_.has_value(); } - uint32_t mix_presentation_id() { + int sample_rate() const { return sample_rate_; } + uint32_t samples_per_buffer() const { return samples_per_buffer_; } + uint32_t config_size() const { return config_size_; } + uint32_t mix_presentation_id() const { SB_DCHECK(mix_presentation_id_.has_value()); return mix_presentation_id_.value(); } - std::vector config_obus() { return config_obus_; } - std::vector data() { return data_; } + const std::vector& config_obus() const { return config_obus_; } + const std::vector& data() const { return data_; } private: + class BufferReader { + public: + BufferReader(const uint8_t* buf, size_t size); + + bool Read1(uint8_t* ptr); + bool Read4(uint32_t* ptr); + bool ReadLeb128(uint32_t* ptr); + bool ReadString(std::string& str); + + bool SkipBytes(size_t size); + bool SkipLeb128(); + bool SkipString(); + + size_t size() const { return size_; } + int pos() const { return pos_; } + const uint8_t* buf() const { return buf_; } + bool error() const { return error_; } + + private: + bool HasBytes(size_t size) const { return size + pos_ <= size_; } + inline uint32_t ByteSwap(uint32_t x) { +#if defined(COMPILER_MSVC) + return _byteswap_ulong(x); +#else + return __builtin_bswap32(x); +#endif + } + // Decodes an Leb128 value and stores it in |value|. Returns the number of + // bytes read, capped to sizeof(uint32_t). Returns the number of bytes read, + // or -1 on error. + int ReadLeb128Internal(const uint8_t* buf, uint32_t* value); + // Reads a c-string into |str|. Returns the number of bytes read, or -1 on + // error. + int ReadStringInternal(const uint8_t* buf, std::string& str); + + int pos_ = 0; + const uint8_t* buf_; + const size_t size_ = 0; + bool error_ = false; + }; + + // Used in the selection of a binaural mix presentation, using the strategy + // defined in + // https://aomediacodec.github.io/iamf/#processing-mixpresentation-selection. + // The preferred methods of choosing a binaural mix presentation are listed + // from high to low. + enum BinauralMixSelection { + kBinauralMixSelectionLoudspeakerLayout, + kBinauralMixSelectionLoudnessLayout, + kBinauralMixSelectionNotFound + }; + + void Reset(); bool Read(scoped_refptr input_buffer); - bool ReadOBU(const uint8_t* buf, bool& completed_parsing); - bool ReadOBUHeader(const uint8_t* buf, + // Reads a single Descriptor OBU. Returns false on error. + bool ReadOBU(BufferReader* reader); + bool ReadOBUHeader(BufferReader* reader, uint8_t* obu_type, - uint32_t* obu_size, - uint32_t* header_size); + uint32_t* obu_size); - int buffer_head_ = 0; int sample_rate_ = 0; int sample_size_ = 0; uint32_t samples_per_buffer_ = 0; @@ -73,9 +121,13 @@ class IamfConfigReader { uint32_t data_size_ = 0; std::optional mix_presentation_id_; + std::unordered_set binaural_audio_element_ids_; + std::unordered_set surround_audio_element_ids_; + const bool prefer_binaural_audio_; + const bool prefer_surround_audio_; + + BinauralMixSelection binaural_mix_selection_ = kBinauralMixSelectionNotFound; - bool has_valid_config_ = false; - bool binaural_mix_presentation_id_ = -1; std::vector config_obus_; std::vector data_; }; From dd0ebdfd143b7376dd2aa3577182a529971cdc7f Mon Sep 17 00:00:00 2001 From: Austin Osagie Date: Tue, 3 Sep 2024 09:28:27 -0700 Subject: [PATCH 09/30] Max IamfAudioDecoder actually servicable --- .../shared/media_is_audio_supported.cc | 2 +- .../shared/player_components_factory.h | 2 +- .../linux/shared/player_components_factory.cc | 2 +- .../shared/libiamf/iamf_audio_decoder.cc | 130 ++++++++++-------- starboard/shared/libiamf/iamf_audio_decoder.h | 14 +- .../shared/libiamf/iamf_config_reader.cc | 24 +--- starboard/shared/libiamf/iamf_config_reader.h | 17 ++- 7 files changed, 94 insertions(+), 97 deletions(-) diff --git a/starboard/android/shared/media_is_audio_supported.cc b/starboard/android/shared/media_is_audio_supported.cc index b2c9d0b9c62f..364a093210da 100644 --- a/starboard/android/shared/media_is_audio_supported.cc +++ b/starboard/android/shared/media_is_audio_supported.cc @@ -64,7 +64,7 @@ bool SbMediaIsAudioSupported(SbMediaAudioCodec audio_codec, if (audio_codec == kSbMediaAudioCodecIamf) { return true; } -#endif // SB_API_VERSION >= 15 +#endif // SB_API_VERSION >= 15 && ENABLE_IAMF_DECODE bool media_codec_supported = MediaCapabilitiesCache::GetInstance()->HasAudioDecoderFor(mime, bitrate); diff --git a/starboard/android/shared/player_components_factory.h b/starboard/android/shared/player_components_factory.h index 83cb08c37651..d5225d71cc4a 100644 --- a/starboard/android/shared/player_components_factory.h +++ b/starboard/android/shared/player_components_factory.h @@ -454,7 +454,7 @@ class PlayerComponentsFactory : public starboard::shared::starboard::player:: std::unique_ptr audio_decoder_impl( new starboard::shared::libiamf::IamfAudioDecoder( - audio_stream_info, /* prefer_binarual_audio */ false)); + audio_stream_info)); if (audio_decoder_impl->is_valid()) { return std::unique_ptr( std::move(audio_decoder_impl)); diff --git a/starboard/linux/shared/player_components_factory.cc b/starboard/linux/shared/player_components_factory.cc index a49a33f934a6..3b9d0b46087e 100644 --- a/starboard/linux/shared/player_components_factory.cc +++ b/starboard/linux/shared/player_components_factory.cc @@ -95,7 +95,7 @@ class PlayerComponentsFactory : public PlayerComponents::Factory { SB_LOG(INFO) << "Playing audio using IamfAudioDecoder."; return std::unique_ptr( new ::starboard::shared::libiamf::IamfAudioDecoder( - audio_stream_info, /* prefer_binarual_audio */ false)); + audio_stream_info)); #endif // SB_API_VERSION >= 15 && ENABLE_IAMF_DECODE } else { std::unique_ptr audio_decoder_impl( diff --git a/starboard/shared/libiamf/iamf_audio_decoder.cc b/starboard/shared/libiamf/iamf_audio_decoder.cc index 9f88f5a83e3c..0c3d5375d710 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.cc +++ b/starboard/shared/libiamf/iamf_audio_decoder.cc @@ -15,8 +15,8 @@ #include "starboard/shared/libiamf/iamf_audio_decoder.h" #include -#include +#include "starboard/common/string.h" #include "third_party/libiamf/source/code/include/IAMF_defines.h" namespace starboard { @@ -27,8 +27,7 @@ namespace { using shared::starboard::player::DecodedAudio; constexpr int kForceBinauralAudio = false; -// Keep disabled as surround audio may require changes further up the SbPlayer. -constexpr int kEnableSurroundAudio = false; +constexpr int kForceSurroundAudio = false; std::string ErrorCodeToString(int code) { switch (code) { @@ -54,19 +53,16 @@ std::string ErrorCodeToString(int code) { } } // namespace -IamfAudioDecoder::IamfAudioDecoder(const AudioStreamInfo& audio_stream_info, - bool prefer_binaural_audio) - : audio_stream_info_(audio_stream_info), - prefer_binarual_audio_(prefer_binaural_audio), - reader_(false, false) { +IamfAudioDecoder::IamfAudioDecoder(const AudioStreamInfo& audio_stream_info) + : audio_stream_info_(audio_stream_info) { decoder_ = IAMF_decoder_open(); if (!decoder_) { - SB_DLOG(ERROR) << "Error creating libiamf decoder"; + SB_LOG(ERROR) << "Error creating libiamf decoder"; } } IamfAudioDecoder::~IamfAudioDecoder() { - TeardownCodec(); + TeardownDecoder(); } bool IamfAudioDecoder::is_valid() const { @@ -93,7 +89,7 @@ void IamfAudioDecoder::Decode(const InputBuffers& input_buffers, SB_DCHECK(output_cb_); if (stream_ended_) { - SB_DLOG(ERROR) << "Decode() is called after WriteEndOfStream() is called."; + SB_LOG(ERROR) << "Decode() is called after WriteEndOfStream() is called."; return; } @@ -141,21 +137,23 @@ bool IamfAudioDecoder::DecodeInternal( const scoped_refptr& input_buffer) { SB_DCHECK(BelongsToCurrentThread()); SB_DCHECK(input_buffer); - SB_DCHECK(input_buffer->size() > 0); SB_DCHECK(output_cb_); SB_DCHECK(!stream_ended_ || !pending_audio_buffers_.empty()); SB_DCHECK(is_valid()); - reader_.ResetAndRead(input_buffer); - if (!reader_.is_valid()) { - SB_DLOG(INFO) << "Failed to parse IA Descriptors"; - error_cb_(kSbPlayerErrorDecode, "Failed to parse IA Descriptors"); + if (input_buffer->size() == 0) { + SB_LOG(ERROR) << "Empty input buffer written to IamfAudioDecoder"; + return false; + } + + IamfConfigReader reader(input_buffer, kForceBinauralAudio, + kForceSurroundAudio); + if (!reader.is_valid()) { + ReportError("Failed to parse IA Descriptors"); return false; } if (!decoder_is_configured_) { - if (!InitializeCodec()) { - SB_DLOG(INFO) << "Failed to initialize IAMF decoder"; - error_cb_(kSbPlayerErrorDecode, "Failed to initialize IAMF decoder"); + if (!ConfigureDecoder(&reader, input_buffer->timestamp())) { return false; } } @@ -163,23 +161,21 @@ bool IamfAudioDecoder::DecodeInternal( scoped_refptr decoded_audio = new DecodedAudio( audio_stream_info_.number_of_channels, GetSampleType(), kSbMediaAudioFrameStorageTypeInterleaved, input_buffer->timestamp(), - audio_stream_info_.number_of_channels * reader_.samples_per_buffer() * + audio_stream_info_.number_of_channels * reader.samples_per_buffer() * starboard::media::GetBytesPerSample(GetSampleType())); int samples_decoded = IAMF_decoder_decode( - decoder_, reader_.data().data(), reader_.data().size(), nullptr, + decoder_, reader.data().data(), reader.data().size(), nullptr, reinterpret_cast(decoded_audio->data())); if (samples_decoded < 1) { - SB_DLOG(INFO) << "IAMF_decoder_decode() error " - << ErrorCodeToString(samples_decoded); - error_cb_(kSbPlayerErrorDecode, "Failed to decode IAMF sample, error " + - ErrorCodeToString(samples_decoded)); + ReportError("Failed to decode IAMF sample, error " + + ErrorCodeToString(samples_decoded)); return false; } - SB_DCHECK(samples_decoded <= reader_.samples_per_buffer()); + SB_DCHECK(samples_decoded <= reader.samples_per_buffer()); decoded_audio->ShrinkTo(audio_stream_info_.number_of_channels * - reader_.samples_per_buffer() * + reader.samples_per_buffer() * starboard::media::GetBytesPerSample(GetSampleType())); // TODO: Enable partial audio once float32 pcm output is available. @@ -211,7 +207,8 @@ void IamfAudioDecoder::WriteEndOfStream() { Schedule(output_cb_); } -bool IamfAudioDecoder::InitializeCodec() { +bool IamfAudioDecoder::ConfigureDecoder(IamfConfigReader* reader, + int64_t timestamp) { SB_DCHECK(is_valid()); SB_DCHECK(!decoder_is_configured_); @@ -219,38 +216,38 @@ bool IamfAudioDecoder::InitializeCodec() { // for now. int error = IAMF_decoder_set_bit_depth(decoder_, 16); if (error != IAMF_OK) { - SB_DLOG(ERROR) << "IAMF_decoder_set_bit_depth() fails with error " - << ErrorCodeToString(error); + ReportError("IAMF_decoder_set_bit_depth() fails with error " + + ErrorCodeToString(error)); return false; } error = IAMF_decoder_set_sampling_rate(decoder_, 48000); if (error != IAMF_OK) { - SB_DLOG(ERROR) << "IAMF_decoder_set_sampling_rate() fails with error " - << ErrorCodeToString(error); + ReportError("IAMF_decoder_set_sampling_rate() fails with error " + + ErrorCodeToString(error)); return false; } - if (kForceBinauralAudio || prefer_binarual_audio_) { - SB_DLOG(INFO) << "Configuring IamfAudioDecoder for binaural output"; + if (kForceBinauralAudio) { + SB_LOG(INFO) << "Configuring IamfAudioDecoder for binaural output"; error = IAMF_decoder_output_layout_set_binaural(decoder_); if (error != IAMF_OK) { - SB_DLOG(ERROR) - << "IAMF_decoder_output_layout_set_binaural() fails with error " - << ErrorCodeToString(error); + ReportError( + "IAMF_decoder_output_layout_set_binaural() fails with error " + + ErrorCodeToString(error)); return false; } } else { - // Default to stereo output. If kEnableSurroundAudio is true, set to a sound + // Default to stereo output. If kForceSurroundAudio is true, set to a sound // system matching, the platform's audio configuration, if available. IAMF_SoundSystem sound_system = SOUND_SYSTEM_A; - if (kEnableSurroundAudio) { + if (kForceSurroundAudio) { SbMediaAudioConfiguration out_config; SbMediaGetAudioConfiguration(0, &out_config); int channels = std::max(out_config.number_of_channels, 2); if (channels > 8 || channels < 1) { - SB_DLOG(ERROR) << "Can't create decoder with " << channels - << " channels"; + ReportError(::starboard::FormatString( + "Can't create decoder with %i channels", channels)); return false; } switch (channels) { @@ -274,6 +271,7 @@ bool IamfAudioDecoder::InitializeCodec() { break; default: SB_NOTREACHED(); + return false; } } else { SB_DLOG(INFO) << "Defaulting to stereo output."; @@ -281,48 +279,49 @@ bool IamfAudioDecoder::InitializeCodec() { error = IAMF_decoder_output_layout_set_sound_system(decoder_, sound_system); if (error != IAMF_OK) { - SB_DLOG(ERROR) - << "IAMF_decoder_output_layout_set_sound_system() fails with error " - << ErrorCodeToString(error); + ReportError( + "IAMF_decoder_output_layout_set_sound_system() fails with error " + + ErrorCodeToString(error)); return false; } } - error = IAMF_decoder_set_pts(decoder_, 0, 90000); + // Time base is set to 9000, as it is in the iamfplayer example + // https://github.com/AOMediaCodec/libiamf/blob/v1.0.0-errata/code/test/tools/iamfplayer/player/iamfplayer.c#L450 + error = IAMF_decoder_set_pts(decoder_, timestamp, 90000); if (error != IAMF_OK) { - SB_DLOG(ERROR) << "IAMF_decoder_set_pts() fails with error " - << ErrorCodeToString(error); + ReportError("IAMF_decoder_set_pts() fails with error " + + ErrorCodeToString(error)); return false; } error = IAMF_decoder_set_mix_presentation_id(decoder_, - reader_.mix_presentation_id()); + reader->mix_presentation_id()); if (error != IAMF_OK) { - SB_DLOG(ERROR) << "IAMF_decoder_set_mix_presentation_id() fails with error " - << ErrorCodeToString(error); + ReportError("IAMF_decoder_set_mix_presentation_id() fails with error " + + ErrorCodeToString(error)); return false; } error = IAMF_decoder_peak_limiter_enable(decoder_, 0); if (error != IAMF_OK) { - SB_DLOG(ERROR) << "IAMF_decoder_peak_limiter_enable() fails with error " - << ErrorCodeToString(error); + ReportError("IAMF_decoder_peak_limiter_enable() fails with error " + + ErrorCodeToString(error)); return false; } error = IAMF_decoder_set_normalization_loudness(decoder_, .0f); if (error != IAMF_OK) { - SB_DLOG(ERROR) - << "IAMF_decoder_set_normalization_loudness() fails with error " - << ErrorCodeToString(error); + ReportError("IAMF_decoder_set_normalization_loudness() fails with error " + + ErrorCodeToString(error)); return false; } - error = IAMF_decoder_configure(decoder_, reader_.config_obus().data(), - reader_.config_size(), nullptr); + error = IAMF_decoder_configure(decoder_, reader->config_obus().data(), + reader->config_size(), nullptr); if (error != IAMF_OK) { - SB_DLOG(ERROR) << "IAMF_decoder_configure() fails with error " - << ErrorCodeToString(error); + ReportError("IAMF_decoder_configure() fails with error " + + ErrorCodeToString(error)); return false; } @@ -331,7 +330,7 @@ bool IamfAudioDecoder::InitializeCodec() { return true; } -void IamfAudioDecoder::TeardownCodec() { +void IamfAudioDecoder::TeardownDecoder() { if (is_valid()) { IAMF_decoder_close(decoder_); decoder_ = NULL; @@ -357,7 +356,7 @@ void IamfAudioDecoder::Reset() { SB_DCHECK(BelongsToCurrentThread()); if (is_valid()) { - TeardownCodec(); + TeardownDecoder(); decoder_ = IAMF_decoder_open(); } @@ -375,9 +374,18 @@ void IamfAudioDecoder::Reset() { SbMediaAudioSampleType IamfAudioDecoder::GetSampleType() const { SB_DCHECK(BelongsToCurrentThread()); +#if SB_API_VERSION <= 15 && SB_HAS_QUIRK(SUPPORT_INT16_AUDIO_SAMPLES) + return kSbMediaAudioSampleTypeInt16; +#endif // SB_API_VERSION <= 15 && SB_HAS_QUIRK(SUPPORT_INT16_AUDIO_SAMPLES) return kSbMediaAudioSampleTypeInt16Deprecated; } +void IamfAudioDecoder::ReportError(const std::string& message) const { + SB_DCHECK(error_cb_); + SB_LOG(ERROR) << "IamfAudioDecoder error: " << message; + error_cb_(kSbPlayerErrorDecode, message); +} + } // namespace libiamf } // namespace shared } // namespace starboard diff --git a/starboard/shared/libiamf/iamf_audio_decoder.h b/starboard/shared/libiamf/iamf_audio_decoder.h index d89958c1fdde..5aee80f5aaf5 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.h +++ b/starboard/shared/libiamf/iamf_audio_decoder.h @@ -17,6 +17,7 @@ #include #include +#include #include "starboard/common/ref_counted.h" #include "starboard/media.h" @@ -38,8 +39,7 @@ class IamfAudioDecoder public: typedef starboard::media::AudioStreamInfo AudioStreamInfo; - explicit IamfAudioDecoder(const AudioStreamInfo& audio_stream_info, - bool prefer_binaural_audio); + explicit IamfAudioDecoder(const AudioStreamInfo& audio_stream_info); ~IamfAudioDecoder() override; bool is_valid() const; @@ -55,13 +55,15 @@ class IamfAudioDecoder private: static constexpr int kMinimumBuffersToDecode = 2; - bool InitializeCodec(); - void TeardownCodec(); + bool ConfigureDecoder(IamfConfigReader* reader, int64_t timestamp); + void TeardownDecoder(); void DecodePendingBuffers(); bool DecodeInternal(const scoped_refptr& input_buffer); SbMediaAudioSampleType GetSampleType() const; + void ReportError(const std::string& message) const; + OutputCB output_cb_; ErrorCB error_cb_; @@ -74,10 +76,6 @@ class IamfAudioDecoder std::deque> pending_audio_buffers_; ConsumedCB consumed_cb_; - - IamfConfigReader reader_; - - const bool prefer_binarual_audio_; }; } // namespace libiamf diff --git a/starboard/shared/libiamf/iamf_config_reader.cc b/starboard/shared/libiamf/iamf_config_reader.cc index d1a4e904d1b6..0a257c6d7136 100644 --- a/starboard/shared/libiamf/iamf_config_reader.cc +++ b/starboard/shared/libiamf/iamf_config_reader.cc @@ -168,32 +168,20 @@ int IamfConfigReader::BufferReader::ReadStringInternal(const uint8_t* buf, return ++size; } -IamfConfigReader::IamfConfigReader(bool prefer_binaural_audio, - bool prefer_surround_audio) +IamfConfigReader::IamfConfigReader( + const scoped_refptr& input_buffer, + bool prefer_binaural_audio, + bool prefer_surround_audio) : prefer_binaural_audio_(prefer_binaural_audio), prefer_surround_audio_(prefer_surround_audio) { #if SB_IS_BIG_ENDIAN #error IamfConfigReader assumes little-endianness. #endif // SB_IS_BIG_ENDIAN SB_DCHECK(!(prefer_binaural_audio && prefer_surround_audio)); + Read(input_buffer); } -bool IamfConfigReader::ResetAndRead(scoped_refptr input_buffer) { - Reset(); - return Read(input_buffer); -} - -void IamfConfigReader::Reset() { - mix_presentation_id_ = std::optional(); - binaural_audio_element_ids_ = std::unordered_set(); - config_size_ = 0; - data_size_ = 0; - sample_rate_ = 0; - samples_per_buffer_ = 0; - sample_size_ = 0; -} - -bool IamfConfigReader::Read(scoped_refptr input_buffer) { +bool IamfConfigReader::Read(const scoped_refptr& input_buffer) { SB_DCHECK(input_buffer->data()); BufferReader reader(input_buffer->data(), input_buffer->size()); diff --git a/starboard/shared/libiamf/iamf_config_reader.h b/starboard/shared/libiamf/iamf_config_reader.h index 0012a1d67829..72653d8f3e51 100644 --- a/starboard/shared/libiamf/iamf_config_reader.h +++ b/starboard/shared/libiamf/iamf_config_reader.h @@ -34,9 +34,9 @@ class IamfConfigReader { public: typedef ::starboard::shared::starboard::player::InputBuffer InputBuffer; - IamfConfigReader(bool prefer_binaural_audio, bool prefer_surround_audio); - - bool ResetAndRead(scoped_refptr input_buffer); + IamfConfigReader(const scoped_refptr& input_buffer, + bool prefer_binaural_audio, + bool prefer_surround_audio); bool is_valid() const { return mix_presentation_id_.has_value() && sample_rate_ > 0 && @@ -47,6 +47,10 @@ class IamfConfigReader { uint32_t config_size() const { return config_size_; } uint32_t mix_presentation_id() const { SB_DCHECK(mix_presentation_id_.has_value()); + if (prefer_binaural_audio_ && + binaural_mix_selection_ == kBinauralMixSelectionNotFound) { + SB_LOG(INFO) << "Could not find binaural mix presentation."; + } return mix_presentation_id_.value(); } @@ -85,8 +89,8 @@ class IamfConfigReader { // bytes read, capped to sizeof(uint32_t). Returns the number of bytes read, // or -1 on error. int ReadLeb128Internal(const uint8_t* buf, uint32_t* value); - // Reads a c-string into |str|. Returns the number of bytes read, or -1 on - // error. + // Reads a c-string into |str|. Returns the number of bytes read, capped to + // 128 bytes, or -1 on error. int ReadStringInternal(const uint8_t* buf, std::string& str); int pos_ = 0; @@ -106,8 +110,7 @@ class IamfConfigReader { kBinauralMixSelectionNotFound }; - void Reset(); - bool Read(scoped_refptr input_buffer); + bool Read(const scoped_refptr& input_buffer); // Reads a single Descriptor OBU. Returns false on error. bool ReadOBU(BufferReader* reader); bool ReadOBUHeader(BufferReader* reader, From 82bc941eb5d5af26be07a6ac1a9ad197bc556d68 Mon Sep 17 00:00:00 2001 From: Austin Osagie Date: Tue, 3 Sep 2024 09:43:39 -0700 Subject: [PATCH 10/30] Minor formatting --- starboard/shared/libiamf/iamf_audio_decoder.cc | 4 ++-- starboard/shared/libiamf/iamf_audio_decoder.h | 5 ++--- starboard/shared/libiamf/iamf_config_reader.cc | 12 ++++++------ starboard/shared/libiamf/iamf_config_reader.h | 6 +++--- 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/starboard/shared/libiamf/iamf_audio_decoder.cc b/starboard/shared/libiamf/iamf_audio_decoder.cc index 0c3d5375d710..ba315b88f6eb 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.cc +++ b/starboard/shared/libiamf/iamf_audio_decoder.cc @@ -221,7 +221,7 @@ bool IamfAudioDecoder::ConfigureDecoder(IamfConfigReader* reader, return false; } - error = IAMF_decoder_set_sampling_rate(decoder_, 48000); + error = IAMF_decoder_set_sampling_rate(decoder_, kDefaultSampleRate); if (error != IAMF_OK) { ReportError("IAMF_decoder_set_sampling_rate() fails with error " + ErrorCodeToString(error)); @@ -348,7 +348,7 @@ scoped_refptr IamfAudioDecoder::Read( result = decoded_audios_.front(); decoded_audios_.pop(); } - *samples_per_second = 48000; + *samples_per_second = kDefaultSampleRate; return result; } diff --git a/starboard/shared/libiamf/iamf_audio_decoder.h b/starboard/shared/libiamf/iamf_audio_decoder.h index 5aee80f5aaf5..a922921ff14b 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.h +++ b/starboard/shared/libiamf/iamf_audio_decoder.h @@ -54,6 +54,7 @@ class IamfAudioDecoder private: static constexpr int kMinimumBuffersToDecode = 2; + static constexpr int kDefaultSampleRate = 48000; bool ConfigureDecoder(IamfConfigReader* reader, int64_t timestamp); void TeardownDecoder(); @@ -66,16 +67,14 @@ class IamfAudioDecoder OutputCB output_cb_; ErrorCB error_cb_; + ConsumedCB consumed_cb_; IAMF_Decoder* decoder_ = nullptr; bool stream_ended_ = false; std::queue> decoded_audios_; AudioStreamInfo audio_stream_info_; - bool decoder_is_configured_ = false; - std::deque> pending_audio_buffers_; - ConsumedCB consumed_cb_; }; } // namespace libiamf diff --git a/starboard/shared/libiamf/iamf_config_reader.cc b/starboard/shared/libiamf/iamf_config_reader.cc index 0a257c6d7136..342b62706967 100644 --- a/starboard/shared/libiamf/iamf_config_reader.cc +++ b/starboard/shared/libiamf/iamf_config_reader.cc @@ -86,7 +86,7 @@ bool IamfConfigReader::BufferReader::ReadLeb128(uint32_t* ptr) { return true; } -bool IamfConfigReader::BufferReader::ReadString(std::string& str) { +bool IamfConfigReader::BufferReader::ReadString(std::string* str) { int bytes_read = ReadStringInternal(buf_ + pos_, str); if (bytes_read < 0) { error_ = true; @@ -112,7 +112,7 @@ bool IamfConfigReader::BufferReader::SkipLeb128() { bool IamfConfigReader::BufferReader::SkipString() { std::string str; - return ReadString(str); + return ReadString(&str); } int IamfConfigReader::BufferReader::ReadLeb128Internal(const uint8_t* buf, @@ -138,7 +138,7 @@ int IamfConfigReader::BufferReader::ReadLeb128Internal(const uint8_t* buf, } int IamfConfigReader::BufferReader::ReadStringInternal(const uint8_t* buf, - std::string& str) { + std::string* str) { SB_DCHECK(buf); int remaining_size = static_cast(size_) - pos_; @@ -148,7 +148,7 @@ int IamfConfigReader::BufferReader::ReadStringInternal(const uint8_t* buf, // The size of the string is capped to 128 bytes. int max_str_size = std::min(remaining_size, 128); - str.clear(); + str->clear(); size_t size = 0; while (buf[size] != '\0' && size < max_str_size) { @@ -160,8 +160,8 @@ int IamfConfigReader::BufferReader::ReadStringInternal(const uint8_t* buf, } if (size > 0) { - str.resize(size); - std::memcpy(str.data(), reinterpret_cast(buf), size); + str->resize(size); + std::memcpy(str->data(), reinterpret_cast(buf), size); } // Account for null terminator byte. diff --git a/starboard/shared/libiamf/iamf_config_reader.h b/starboard/shared/libiamf/iamf_config_reader.h index 72653d8f3e51..d38d0fd9781d 100644 --- a/starboard/shared/libiamf/iamf_config_reader.h +++ b/starboard/shared/libiamf/iamf_config_reader.h @@ -65,14 +65,14 @@ class IamfConfigReader { bool Read1(uint8_t* ptr); bool Read4(uint32_t* ptr); bool ReadLeb128(uint32_t* ptr); - bool ReadString(std::string& str); + bool ReadString(std::string* str); bool SkipBytes(size_t size); bool SkipLeb128(); bool SkipString(); size_t size() const { return size_; } - int pos() const { return pos_; } + size_t pos() const { return pos_; } const uint8_t* buf() const { return buf_; } bool error() const { return error_; } @@ -91,7 +91,7 @@ class IamfConfigReader { int ReadLeb128Internal(const uint8_t* buf, uint32_t* value); // Reads a c-string into |str|. Returns the number of bytes read, capped to // 128 bytes, or -1 on error. - int ReadStringInternal(const uint8_t* buf, std::string& str); + int ReadStringInternal(const uint8_t* buf, std::string* str); int pos_ = 0; const uint8_t* buf_; From 4c3c9576dfb54d81f9176f66b69610855a1253dc Mon Sep 17 00:00:00 2001 From: Austin Osagie Date: Tue, 3 Sep 2024 12:43:12 -0700 Subject: [PATCH 11/30] Update DCHECK in mix_presentation_id() --- starboard/shared/libiamf/iamf_config_reader.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/starboard/shared/libiamf/iamf_config_reader.h b/starboard/shared/libiamf/iamf_config_reader.h index d38d0fd9781d..0a7db26bba9a 100644 --- a/starboard/shared/libiamf/iamf_config_reader.h +++ b/starboard/shared/libiamf/iamf_config_reader.h @@ -46,7 +46,8 @@ class IamfConfigReader { uint32_t samples_per_buffer() const { return samples_per_buffer_; } uint32_t config_size() const { return config_size_; } uint32_t mix_presentation_id() const { - SB_DCHECK(mix_presentation_id_.has_value()); + SB_DCHECK(is_valid()) + << "Called mix_presentation_id() on invalid IamfConfigReader."; if (prefer_binaural_audio_ && binaural_mix_selection_ == kBinauralMixSelectionNotFound) { SB_LOG(INFO) << "Could not find binaural mix presentation."; From 08ccf796e781ca39402f7d642f45a94e5aaa482e Mon Sep 17 00:00:00 2001 From: Austin Osagie Date: Wed, 4 Sep 2024 17:28:32 -0700 Subject: [PATCH 12/30] IamfConfigReader becomes IamfBufferParser --- starboard/android/shared/BUILD.gn | 4 +- starboard/linux/shared/BUILD.gn | 4 +- .../shared/libiamf/iamf_audio_decoder.cc | 86 +-- starboard/shared/libiamf/iamf_audio_decoder.h | 7 +- .../shared/libiamf/iamf_buffer_parser.cc | 588 ++++++++++++++++++ starboard/shared/libiamf/iamf_buffer_parser.h | 109 ++++ .../shared/libiamf/iamf_config_reader.cc | 531 ---------------- starboard/shared/libiamf/iamf_config_reader.h | 143 ----- 8 files changed, 734 insertions(+), 738 deletions(-) create mode 100644 starboard/shared/libiamf/iamf_buffer_parser.cc create mode 100644 starboard/shared/libiamf/iamf_buffer_parser.h delete mode 100644 starboard/shared/libiamf/iamf_config_reader.cc delete mode 100644 starboard/shared/libiamf/iamf_config_reader.h diff --git a/starboard/android/shared/BUILD.gn b/starboard/android/shared/BUILD.gn index 5a05de935791..036107b6b13f 100644 --- a/starboard/android/shared/BUILD.gn +++ b/starboard/android/shared/BUILD.gn @@ -499,8 +499,8 @@ static_library("starboard_platform") { sources += [ "//starboard/shared/libiamf/iamf_audio_decoder.cc", "//starboard/shared/libiamf/iamf_audio_decoder.h", - "//starboard/shared/libiamf/iamf_config_reader.cc", - "//starboard/shared/libiamf/iamf_config_reader.h", + "//starboard/shared/libiamf/iamf_buffer_parser.cc", + "//starboard/shared/libiamf/iamf_buffer_parser.h", ] defines += [ "ENABLE_IAMF_DECODE" ] diff --git a/starboard/linux/shared/BUILD.gn b/starboard/linux/shared/BUILD.gn index bea587c63a10..bf96a97aad6a 100644 --- a/starboard/linux/shared/BUILD.gn +++ b/starboard/linux/shared/BUILD.gn @@ -445,8 +445,8 @@ static_library("starboard_platform_sources") { sources += [ "//starboard/shared/libiamf/iamf_audio_decoder.cc", "//starboard/shared/libiamf/iamf_audio_decoder.h", - "//starboard/shared/libiamf/iamf_config_reader.cc", - "//starboard/shared/libiamf/iamf_config_reader.h", + "//starboard/shared/libiamf/iamf_buffer_parser.cc", + "//starboard/shared/libiamf/iamf_buffer_parser.h", ] defines += [ "ENABLE_IAMF_DECODE" ] diff --git a/starboard/shared/libiamf/iamf_audio_decoder.cc b/starboard/shared/libiamf/iamf_audio_decoder.cc index ba315b88f6eb..58db3987d272 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.cc +++ b/starboard/shared/libiamf/iamf_audio_decoder.cc @@ -93,44 +93,12 @@ void IamfAudioDecoder::Decode(const InputBuffers& input_buffers, return; } - if (input_buffers.size() > kMinimumBuffersToDecode) { - std::copy(std::begin(input_buffers), std::end(input_buffers), - std::back_inserter(pending_audio_buffers_)); - consumed_cb_ = consumed_cb; - DecodePendingBuffers(); - } else { - for (const auto& input_buffer : input_buffers) { - if (!DecodeInternal(input_buffer)) { - return; - } - } - Schedule(consumed_cb); - } -} - -void IamfAudioDecoder::DecodePendingBuffers() { - SB_DCHECK(BelongsToCurrentThread()); - SB_DCHECK(!pending_audio_buffers_.empty()); - SB_DCHECK(consumed_cb_); - - for (int i = 0; i < kMinimumBuffersToDecode; ++i) { - if (!DecodeInternal(pending_audio_buffers_.front())) { - return; - } - pending_audio_buffers_.pop_front(); - if (pending_audio_buffers_.empty()) { - Schedule(consumed_cb_); - consumed_cb_ = nullptr; - if (stream_ended_) { - Schedule(std::bind(&IamfAudioDecoder::WriteEndOfStream, this)); - stream_ended_ = false; - } + for (const auto& input_buffer : input_buffers) { + if (!DecodeInternal(input_buffer)) { return; } } - - SB_DCHECK(!pending_audio_buffers_.empty()); - Schedule(std::bind(&IamfAudioDecoder::DecodePendingBuffers, this)); + Schedule(consumed_cb); } bool IamfAudioDecoder::DecodeInternal( @@ -146,14 +114,15 @@ bool IamfAudioDecoder::DecodeInternal( return false; } - IamfConfigReader reader(input_buffer, kForceBinauralAudio, - kForceSurroundAudio); - if (!reader.is_valid()) { + IamfBufferParser::IamfBufferInfo info; + IamfBufferParser().ParseInputBuffer(input_buffer, &info, kForceBinauralAudio, + kForceSurroundAudio); + if (!info.is_valid()) { ReportError("Failed to parse IA Descriptors"); return false; } if (!decoder_is_configured_) { - if (!ConfigureDecoder(&reader, input_buffer->timestamp())) { + if (!ConfigureDecoder(&info, input_buffer->timestamp())) { return false; } } @@ -161,28 +130,30 @@ bool IamfAudioDecoder::DecodeInternal( scoped_refptr decoded_audio = new DecodedAudio( audio_stream_info_.number_of_channels, GetSampleType(), kSbMediaAudioFrameStorageTypeInterleaved, input_buffer->timestamp(), - audio_stream_info_.number_of_channels * reader.samples_per_buffer() * + audio_stream_info_.number_of_channels * info.num_samples * starboard::media::GetBytesPerSample(GetSampleType())); - int samples_decoded = IAMF_decoder_decode( - decoder_, reader.data().data(), reader.data().size(), nullptr, - reinterpret_cast(decoded_audio->data())); + int samples_decoded = + IAMF_decoder_decode(decoder_, info.data.data(), info.data_size, nullptr, + reinterpret_cast(decoded_audio->data())); if (samples_decoded < 1) { ReportError("Failed to decode IAMF sample, error " + ErrorCodeToString(samples_decoded)); return false; } - SB_DCHECK(samples_decoded <= reader.samples_per_buffer()); + SB_DCHECK(samples_decoded <= info.num_samples) + << "Samples decoded (" << samples_decoded + << ") is greater than the number of samples indicated by the stream (" + << info.num_samples << ")"; - decoded_audio->ShrinkTo(audio_stream_info_.number_of_channels * - reader.samples_per_buffer() * - starboard::media::GetBytesPerSample(GetSampleType())); + if (samples_per_second_ == 0) { + samples_per_second_ = info.sample_rate; + } // TODO: Enable partial audio once float32 pcm output is available. const auto& sample_info = input_buffer->audio_sample_info(); decoded_audio->AdjustForDiscardedDurations( - audio_stream_info_.samples_per_second, - sample_info.discarded_duration_from_front, + samples_per_second_, sample_info.discarded_duration_from_front, sample_info.discarded_duration_from_back); decoded_audios_.push(decoded_audio); @@ -207,7 +178,7 @@ void IamfAudioDecoder::WriteEndOfStream() { Schedule(output_cb_); } -bool IamfAudioDecoder::ConfigureDecoder(IamfConfigReader* reader, +bool IamfAudioDecoder::ConfigureDecoder(IamfBufferInfo* info, int64_t timestamp) { SB_DCHECK(is_valid()); SB_DCHECK(!decoder_is_configured_); @@ -221,7 +192,7 @@ bool IamfAudioDecoder::ConfigureDecoder(IamfConfigReader* reader, return false; } - error = IAMF_decoder_set_sampling_rate(decoder_, kDefaultSampleRate); + error = IAMF_decoder_set_sampling_rate(decoder_, info->sample_rate); if (error != IAMF_OK) { ReportError("IAMF_decoder_set_sampling_rate() fails with error " + ErrorCodeToString(error)); @@ -286,7 +257,7 @@ bool IamfAudioDecoder::ConfigureDecoder(IamfConfigReader* reader, } } - // Time base is set to 9000, as it is in the iamfplayer example + // Time base is set to 90000, as it is in the iamfplayer example // https://github.com/AOMediaCodec/libiamf/blob/v1.0.0-errata/code/test/tools/iamfplayer/player/iamfplayer.c#L450 error = IAMF_decoder_set_pts(decoder_, timestamp, 90000); if (error != IAMF_OK) { @@ -295,8 +266,8 @@ bool IamfAudioDecoder::ConfigureDecoder(IamfConfigReader* reader, return false; } - error = IAMF_decoder_set_mix_presentation_id(decoder_, - reader->mix_presentation_id()); + error = IAMF_decoder_set_mix_presentation_id( + decoder_, info->mix_presentation_id.value()); if (error != IAMF_OK) { ReportError("IAMF_decoder_set_mix_presentation_id() fails with error " + ErrorCodeToString(error)); @@ -317,8 +288,8 @@ bool IamfAudioDecoder::ConfigureDecoder(IamfConfigReader* reader, return false; } - error = IAMF_decoder_configure(decoder_, reader->config_obus().data(), - reader->config_size(), nullptr); + error = IAMF_decoder_configure(decoder_, info->config_obus.data(), + info->config_obus_size, nullptr); if (error != IAMF_OK) { ReportError("IAMF_decoder_configure() fails with error " + ErrorCodeToString(error)); @@ -342,13 +313,14 @@ scoped_refptr IamfAudioDecoder::Read( SB_DCHECK(BelongsToCurrentThread()); SB_DCHECK(output_cb_); SB_DCHECK(!decoded_audios_.empty()); + SB_DCHECK(samples_per_second_ > 0); scoped_refptr result; if (!decoded_audios_.empty()) { result = decoded_audios_.front(); decoded_audios_.pop(); } - *samples_per_second = kDefaultSampleRate; + *samples_per_second = samples_per_second_; return result; } diff --git a/starboard/shared/libiamf/iamf_audio_decoder.h b/starboard/shared/libiamf/iamf_audio_decoder.h index a922921ff14b..6f457d548b90 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.h +++ b/starboard/shared/libiamf/iamf_audio_decoder.h @@ -22,7 +22,7 @@ #include "starboard/common/ref_counted.h" #include "starboard/media.h" #include "starboard/shared/internal_only.h" -#include "starboard/shared/libiamf/iamf_config_reader.h" +#include "starboard/shared/libiamf/iamf_buffer_parser.h" #include "starboard/shared/starboard/media/media_util.h" #include "starboard/shared/starboard/player/decoded_audio_internal.h" #include "starboard/shared/starboard/player/filter/audio_decoder_internal.h" @@ -38,6 +38,7 @@ class IamfAudioDecoder private starboard::player::JobQueue::JobOwner { public: typedef starboard::media::AudioStreamInfo AudioStreamInfo; + typedef shared::libiamf::IamfBufferParser::IamfBufferInfo IamfBufferInfo; explicit IamfAudioDecoder(const AudioStreamInfo& audio_stream_info); ~IamfAudioDecoder() override; @@ -56,9 +57,8 @@ class IamfAudioDecoder static constexpr int kMinimumBuffersToDecode = 2; static constexpr int kDefaultSampleRate = 48000; - bool ConfigureDecoder(IamfConfigReader* reader, int64_t timestamp); + bool ConfigureDecoder(IamfBufferInfo* info, int64_t timestamp); void TeardownDecoder(); - void DecodePendingBuffers(); bool DecodeInternal(const scoped_refptr& input_buffer); SbMediaAudioSampleType GetSampleType() const; @@ -75,6 +75,7 @@ class IamfAudioDecoder AudioStreamInfo audio_stream_info_; bool decoder_is_configured_ = false; std::deque> pending_audio_buffers_; + int samples_per_second_ = 0; }; } // namespace libiamf diff --git a/starboard/shared/libiamf/iamf_buffer_parser.cc b/starboard/shared/libiamf/iamf_buffer_parser.cc new file mode 100644 index 000000000000..02cd04bef7a6 --- /dev/null +++ b/starboard/shared/libiamf/iamf_buffer_parser.cc @@ -0,0 +1,588 @@ +// Copyright 2024 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "starboard/shared/libiamf/iamf_buffer_parser.h" + +#include +#include +#include + +#include "third_party/libiamf/source/code/include/IAMF_defines.h" + +namespace starboard { +namespace shared { +namespace libiamf { + +namespace { + +// From //media/formats/mp4/rcheck.h. +#define RCHECK(condition) \ + do { \ + if (!(condition)) { \ + SB_DLOG(ERROR) << "Failure while parsing IAMF config: " #condition; \ + return false; \ + } \ + } while (0) + +// From +// https://aomediacodec.github.io/iamf/v1.0.0-errata.html#obu-header-syntax. +constexpr int kObuTypeCodecConfig = 0; +constexpr int kObuTypeAudioElement = 1; +constexpr int kObuTypeMixPresentation = 2; +constexpr int kObuTypeSequenceHeader = 31; + +// From +// https://aomediacodec.github.io/iamf/v1.0.0-errata.html#obu-codecconfig. +constexpr int kFourccOpus = 0x4f707573; +constexpr int kFourccIpcm = 0x6970636d; + +} // namespace + +class BufferReader { + public: + BufferReader(const uint8_t* buf, size_t size) + : buf_(buf), pos_(0), size_(size) { +#if SB_IS_BIG_ENDIAN +#error BufferReader assumes little-endianness. +#endif // SB_IS_BIG_ENDIAN + } + + bool Read1(uint8_t* ptr) { + if (!HasBytes(sizeof(uint8_t)) || !ptr) { + return false; + } + *ptr = buf_[pos_++]; + return true; + } + + bool Read4(uint32_t* ptr) { + if (!HasBytes(sizeof(uint32_t)) || !ptr) { + return false; + } + std::memcpy(ptr, &buf_[pos_], sizeof(uint32_t)); + *ptr = ByteSwap(*ptr); + pos_ += sizeof(uint32_t); + return true; + } + + bool ReadLeb128(uint32_t* ptr) { + if (!HasBytes(sizeof(uint32_t)) || !ptr) { + return false; + } + int bytes_read = ReadLeb128Internal(buf_ + pos_, ptr); + if (bytes_read < 0) { + return false; + } + pos_ += bytes_read; + return true; + } + + bool ReadString(std::string* str) { + int bytes_read = ReadStringInternal(buf_ + pos_, str); + if (bytes_read < 0) { + return false; + } + pos_ += bytes_read; + return true; + } + + bool SkipBytes(size_t size) { + if (!HasBytes(size)) { + return false; + } + pos_ += size; + return true; + } + + bool SkipLeb128() { + uint32_t val; + return ReadLeb128(&val); + } + + bool SkipString() { + std::string str; + return ReadString(&str); + } + + size_t size() const { return size_; } + size_t pos() const { return pos_; } + const uint8_t* buf() const { return buf_; } + + private: + bool HasBytes(size_t size) const { return size + pos_ <= size_; } + inline uint32_t ByteSwap(uint32_t x) { +#if defined(COMPILER_MSVC) + return _byteswap_ulong(x); +#else + return __builtin_bswap32(x); +#endif + } + + // Decodes an Leb128 value and stores it in |value|. Returns the number of + // bytes read, capped to sizeof(uint32_t). Returns the number of bytes read, + // or -1 on error. + int ReadLeb128Internal(const uint8_t* buf, uint32_t* value) { + SB_DCHECK(buf); + SB_DCHECK(value); + *value = 0; + bool error = true; + size_t i = 0; + for (; i < sizeof(uint32_t); ++i) { + uint8_t byte = buf[i]; + *value |= ((byte & 0x7f) << (i * 7)); + if (!(byte & 0x80)) { + error = false; + break; + } + } + + if (error) { + return -1; + } + return i + 1; + } + + // Reads a c-string into |str|. Returns the number of bytes read, capped to + // 128 bytes, or -1 on error. + int ReadStringInternal(const uint8_t* buf, std::string* str) { + SB_DCHECK(buf); + + int remaining_size = static_cast(size_) - pos_; + if (remaining_size <= 0) { + return -1; + } + + // The size of the string is capped to 128 bytes. + const int kMaxStringSize = 128; + int str_size = std::min(remaining_size, kMaxStringSize); + str->clear(); + + size_t bytes_read = 0; + while (bytes_read < str_size && buf[bytes_read] != '\0') { + bytes_read++; + } + + if (bytes_read == str_size) { + if (buf[bytes_read - 1] != '\0') { + return -1; + } + } else { + if (buf[bytes_read] != '\0') { + return -1; + } + } + + if (bytes_read > 0) { + str->resize(bytes_read); + std::memcpy(str->data(), reinterpret_cast(buf), bytes_read); + } + + // Account for null terminator byte. + return ++bytes_read; + } + + int pos_ = 0; + const uint8_t* buf_; + const size_t size_ = 0; +}; + +IamfBufferParser::IamfBufferParser() {} + +bool IamfBufferParser::ParseInputBuffer( + const scoped_refptr& input_buffer, + IamfBufferInfo* info, + const bool prefer_binaural_audio, + const bool prefer_surround_audio) { + SB_DCHECK(info); + SB_DCHECK(input_buffer->data()); + SB_DCHECK(!(prefer_binaural_audio && prefer_surround_audio)); + RCHECK(ParseInputBufferInternal(input_buffer, info, prefer_binaural_audio, + prefer_surround_audio)); + return true; +} + +bool IamfBufferParser::ParseInputBufferInternal( + const scoped_refptr& input_buffer, + IamfBufferInfo* info, + const bool prefer_binaural_audio, + const bool prefer_surround_audio) { + BufferReader reader(input_buffer->data(), input_buffer->size()); + + while (!info->is_valid() && reader.pos() < reader.size()) { + RCHECK(ParseDescriptorOBU(&reader, info, prefer_binaural_audio, + prefer_surround_audio)); + } + + info->data_size = reader.size() - info->config_obus_size; + info->config_obus.assign(reader.buf(), reader.buf() + info->config_obus_size); + info->data.assign(reader.buf() + info->config_obus_size, + reader.buf() + reader.size()); + + return true; +} + +bool IamfBufferParser::ParseDescriptorOBU(BufferReader* reader, + IamfBufferInfo* info, + const bool prefer_binaural_audio, + const bool prefer_surround_audio) { + SB_DCHECK(reader); + uint8_t obu_type = 0; + uint32_t obu_size = 0; + if (!ParseOBUHeader(reader, &obu_type, &obu_size)) { + SB_DLOG(ERROR) << "Error parsing OBU header"; + return false; + } + + int next_obu_pos = reader->pos() + obu_size; + + switch (static_cast(obu_type)) { + case kObuTypeCodecConfig: + RCHECK(ParseCodecConfigOBU(reader, info)); + break; + case kObuTypeAudioElement: + RCHECK(ParseAudioElementOBU(reader, info, prefer_binaural_audio, + prefer_surround_audio)); + break; + case kObuTypeSequenceHeader: + break; + case kObuTypeMixPresentation: + RCHECK(ParseMixPresentationOBU(reader, info, prefer_binaural_audio, + prefer_surround_audio)); + break; + default: + // Once an OBU is read that is not a descriptor, descriptor parsing is + // assumed to be complete. + SB_DCHECK(info->is_valid()); + return true; + } + + // Skip to the next OBU. + const size_t remaining_size = next_obu_pos - reader->pos(); + RCHECK(reader->SkipBytes(remaining_size)); + info->config_obus_size = reader->pos(); + return true; +} + +bool IamfBufferParser::ParseOBUHeader(BufferReader* reader, + uint8_t* obu_type, + uint32_t* obu_size) const { + uint8_t header_flags; + RCHECK(reader->Read1(&header_flags)); + *obu_type = (header_flags >> 3) & 0x1f; + + const bool obu_redundant_copy = (header_flags >> 2) & 1; + const bool obu_trimming_status_flag = (header_flags >> 1) & 1; + const bool obu_extension_flag = header_flags & 1; + + *obu_size = 0; + + RCHECK(reader->ReadLeb128(obu_size)); + + // |obu_size| contains the size of the OBU after its own field. + // If either of the flags are set, subtract the number of bytes read + // from the flags from |obu_size| before returning to ParseDescriptorOBU(). + size_t reader_pos_before_flags = reader->pos(); + + if (obu_trimming_status_flag) { + RCHECK(reader->SkipLeb128()); + RCHECK(reader->SkipLeb128()); + } + + if (obu_extension_flag) { + RCHECK(reader->SkipLeb128()); + } + + size_t flag_bytes_read = reader->pos() - reader_pos_before_flags; + if (flag_bytes_read >= *obu_size) { + return false; + } + *obu_size -= flag_bytes_read; + return true; +} + +bool IamfBufferParser::ParseCodecConfigOBU(BufferReader* reader, + IamfBufferInfo* info) { + RCHECK(reader->SkipLeb128()); + + uint32_t codec_id = 0; + RCHECK(reader->Read4(&codec_id)); + + RCHECK(reader->ReadLeb128(&info->num_samples)); + + // audio_roll_distance + RCHECK(reader->SkipBytes(2)); + + const int kOpusSampleRate = 48000; + switch (codec_id) { + case kFourccOpus: + info->sample_rate = kOpusSampleRate; + break; + case kFourccIpcm: { + // sample_format_flags + RCHECK(reader->SkipBytes(1)); + + // sample_size + RCHECK(reader->SkipBytes(1)); + + uint32_t sample_rate_unsigned; + RCHECK(reader->Read4(&sample_rate_unsigned)); + info->sample_rate = static_cast(sample_rate_unsigned); + break; + } + default: + SB_NOTREACHED(); + return false; + } + + return true; +} + +bool IamfBufferParser::ParseAudioElementOBU(BufferReader* reader, + IamfBufferInfo* info, + const bool prefer_binaural_audio, + const bool prefer_surround_audio) { + uint32_t audio_element_id; + RCHECK(reader->ReadLeb128(&audio_element_id)); + + uint8_t audio_element_type; + RCHECK(reader->Read1(&audio_element_type)); + audio_element_type = audio_element_type >> 5; + + // codec_config_id + RCHECK(reader->SkipLeb128()); + + uint32_t num_substreams; + RCHECK(reader->ReadLeb128(&num_substreams)); + + for (int i = 0; i < num_substreams; ++i) { + // audio_substream_id + RCHECK(reader->SkipLeb128()); + } + + uint32_t num_parameters; + RCHECK(reader->ReadLeb128(&num_parameters)); + + for (int i = 0; i < num_parameters; ++i) { + uint32_t param_definition_type; + RCHECK(reader->ReadLeb128(¶m_definition_type)); + + if (param_definition_type == IAMF_PARAMETER_TYPE_DEMIXING) { + SkipParamDefinition(reader); + // DemixingParamDefintion + RCHECK(reader->SkipBytes(1)); + } else if (param_definition_type == IAMF_PARAMETER_TYPE_RECON_GAIN) { + SkipParamDefinition(reader); + } else if (param_definition_type > 2) { + uint32_t param_definition_size; + RCHECK(reader->ReadLeb128(¶m_definition_size)); + RCHECK(reader->SkipBytes(param_definition_size)); + } + } + + if (static_cast(audio_element_type) == + AUDIO_ELEMENT_CHANNEL_BASED && + (prefer_binaural_audio || prefer_surround_audio)) { + // Parse ScalableChannelLayoutConfig for binaural and surround + // loudspeaker layouts + uint8_t num_layers; + RCHECK(reader->Read1(&num_layers)); + num_layers = num_layers >> 5; + // Read ChannelAudioLayerConfigs + for (int i = 0; i < static_cast(num_layers); ++i) { + uint8_t loudspeaker_layout; + bool output_gain_is_present_flag; + RCHECK(reader->Read1(&loudspeaker_layout)); + output_gain_is_present_flag = (loudspeaker_layout >> 3) & 0x01; + loudspeaker_layout = loudspeaker_layout >> 4; + if (loudspeaker_layout == IA_CHANNEL_LAYOUT_BINAURAL) { + binaural_audio_element_ids_.insert(audio_element_id); + } else if (loudspeaker_layout > IA_CHANNEL_LAYOUT_STEREO && + loudspeaker_layout < IA_CHANNEL_LAYOUT_COUNT) { + surround_audio_element_ids_.insert(audio_element_id); + } + + // substream_count and coupled_substream_count + RCHECK(reader->SkipBytes(2)); + + if (output_gain_is_present_flag) { + // output_gain_flags and output_gain + RCHECK(reader->SkipBytes(3)); + } + + if (i == 1 && loudspeaker_layout == static_cast(15)) { + // expanded_loudspeaker_layout + RCHECK(reader->SkipBytes(1)); + } + } + } + return true; +} + +bool IamfBufferParser::ParseMixPresentationOBU( + BufferReader* reader, + IamfBufferInfo* info, + const bool prefer_binaural_audio, + const bool prefer_surround_audio) { + uint32_t mix_presentation_id; + RCHECK(reader->ReadLeb128(&mix_presentation_id)); + + uint32_t count_label; + RCHECK(reader->ReadLeb128(&count_label)); + for (int i = 0; i < count_label; ++i) { + // language_label; + RCHECK(reader->SkipString()); + } + + for (int i = 0; i < count_label; ++i) { + // MixPresentationAnnotations; + RCHECK(reader->SkipString()); + } + + uint32_t num_sub_mixes; + RCHECK(reader->ReadLeb128(&num_sub_mixes)); + for (int i = 0; i < num_sub_mixes; ++i) { + uint32_t num_audio_elements; + RCHECK(reader->ReadLeb128(&num_audio_elements)); + for (int j = 0; j < num_audio_elements; ++j) { + uint32_t audio_element_id; + RCHECK(reader->ReadLeb128(&audio_element_id)); + + // Set a mix presentation for binaural or surround streams. The mix + // presentation is chosen if there exists an audio element that has + // the qualities it requires - such as an audio element with a + // binaural loudspeaker layout. + if (!info->mix_presentation_id.has_value() || + (prefer_binaural_audio && + binaural_mix_selection_ > kBinauralMixSelectionLoudspeakerLayout)) { + if (prefer_binaural_audio && + binaural_audio_element_ids_.find(audio_element_id) != + binaural_audio_element_ids_.end()) { + info->mix_presentation_id = mix_presentation_id; + binaural_mix_selection_ = kBinauralMixSelectionLoudspeakerLayout; + } else if (prefer_surround_audio && + surround_audio_element_ids_.find(audio_element_id) != + surround_audio_element_ids_.end()) { + info->mix_presentation_id = mix_presentation_id; + } + } + + for (int k = 0; k < count_label; ++k) { + // MixPresentationElementAnnotatoions + RCHECK(reader->SkipString()); + } + + // The following fields are for the RenderingConfig + // headphones_rendering_mode + RCHECK(reader->SkipBytes(1)); + uint32_t rendering_config_extension_size; + RCHECK(reader->ReadLeb128(&rendering_config_extension_size)); + // rendering_config_extension_bytes + RCHECK(reader->SkipBytes(rendering_config_extension_size)); + + // The following fields are for the ElementMixConfig + SkipParamDefinition(reader); + // default_mix_gain + RCHECK(reader->SkipBytes(2)); + } + + // The following fields are for the OutputMixConfig + SkipParamDefinition(reader); + // default_mix_gain + RCHECK(reader->SkipBytes(2)); + + uint32_t num_layouts; + RCHECK(reader->ReadLeb128(&num_layouts)); + for (int j = 0; j < num_layouts; ++j) { + uint8_t layout_type; + RCHECK(reader->Read1(&layout_type)); + layout_type = layout_type >> 6; + // If a binaural mix presentation is preferred and the mix + // presentation id has not yet been set, set the mix presentation id + // if the current mix presentation has a binaural loudness layout. The + // mix presentation id will change if a different mix presentation is + // found that uses an audio element with a binaural loudspeaker + // layout, as that is higher priority. + if (static_cast(layout_type) == IAMF_LAYOUT_TYPE_BINAURAL && + prefer_binaural_audio && + (!info->mix_presentation_id.has_value() || + binaural_mix_selection_ > kBinauralMixSelectionLoudnessLayout)) { + info->mix_presentation_id = mix_presentation_id; + binaural_mix_selection_ = kBinauralMixSelectionLoudnessLayout; + } + + // The following fields are for the LoudnessInfo + uint8_t info_type; + RCHECK(reader->Read1(&info_type)); + // integrated_loudness and digital_loudness + RCHECK(reader->SkipBytes(4)); + if (info_type & 1) { + // true_peak + RCHECK(reader->SkipBytes(2)); + } + if (info_type & 2) { + uint8_t num_anchored_loudness; + RCHECK(reader->Read1(&num_anchored_loudness)); + for (uint8_t k = 0; k < num_anchored_loudness; ++k) { + // anchor_element and anchored_loudness + RCHECK(reader->SkipBytes(3)); + } + } + if ((info_type & 0b11111100) > 0) { + uint32_t info_type_size; + RCHECK(reader->ReadLeb128(&info_type_size)); + // info_type_bytes + RCHECK(reader->SkipBytes(info_type_size)); + } + } + } + + // If the mix presentation id is unassigned at this point, the stream is + // stereo, or a proper mix presentation for binaural or surround preferred + // streams hasn't yet been parsed. Default to the first read mix + // presentation in case a preferred mix does not exist. + if (!info->mix_presentation_id.has_value()) { + info->mix_presentation_id = mix_presentation_id; + } + + return true; +} + +bool IamfBufferParser::SkipParamDefinition(BufferReader* reader) const { + // parameter_id + RCHECK(reader->SkipLeb128()); + // parameter_rate + RCHECK(reader->SkipLeb128()); + uint8_t param_definition_mode; + RCHECK(reader->Read1(¶m_definition_mode)); + param_definition_mode = param_definition_mode >> 7; + if (param_definition_mode == static_cast(0)) { + // duration + RCHECK(reader->SkipLeb128()); + uint32_t constant_subblock_duration; + RCHECK(reader->ReadLeb128(&constant_subblock_duration)); + if (constant_subblock_duration == 0) { + uint32_t num_subblocks; + RCHECK(reader->ReadLeb128(&num_subblocks)); + for (int i = 0; i < num_subblocks; ++i) { + // subblock_duration + RCHECK(reader->SkipLeb128()); + } + } + } + return true; +} + +} // namespace libiamf +} // namespace shared +} // namespace starboard diff --git a/starboard/shared/libiamf/iamf_buffer_parser.h b/starboard/shared/libiamf/iamf_buffer_parser.h new file mode 100644 index 000000000000..83154fd0825a --- /dev/null +++ b/starboard/shared/libiamf/iamf_buffer_parser.h @@ -0,0 +1,109 @@ +// Copyright 2024 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef STARBOARD_SHARED_LIBIAMF_IAMF_BUFFER_PARSER_H_ +#define STARBOARD_SHARED_LIBIAMF_IAMF_BUFFER_PARSER_H_ + +#include +#include +#include +#include + +#include "starboard/common/log.h" +#include "starboard/common/ref_counted.h" +#include "starboard/shared/internal_only.h" +#include "starboard/shared/starboard/player/input_buffer_internal.h" + +namespace starboard { +namespace shared { +namespace libiamf { + +class BufferReader; + +// TODO: Skip parsing if the OBUs are redundant +class IamfBufferParser { + public: + typedef ::starboard::shared::starboard::player::InputBuffer InputBuffer; + + struct IamfBufferInfo { + bool is_valid() const { + return mix_presentation_id.has_value() && sample_rate > 0 && + num_samples > 0; + } + + uint32_t num_samples; + int sample_rate; + std::optional mix_presentation_id; + std::vector config_obus; + size_t config_obus_size; + std::vector data; + size_t data_size; + const scoped_refptr input_buffer; + }; + + IamfBufferParser(); + + bool ParseInputBuffer(const scoped_refptr& input_buffer, + IamfBufferInfo* info, + const bool prefer_binaural_audio, + const bool prefer_surround_audio); + + private: + // Used in the selection of a binaural mix presentation, using the strategy + // defined in + // https://aomediacodec.github.io/iamf/#processing-mixpresentation-selection. + // The preferred methods of choosing a binaural mix presentation are listed + // from high to low. + enum BinauralMixSelection { + kBinauralMixSelectionLoudspeakerLayout, + kBinauralMixSelectionLoudnessLayout, + kBinauralMixSelectionNotFound + }; + + bool ParseInputBufferInternal(const scoped_refptr& input_buffer, + IamfBufferInfo* info, + const bool prefer_binaural_audio, + const bool prefer_surround_audio); + // Reads a single Descriptor OBU. Returns false on error. + bool ParseDescriptorOBU(BufferReader* reader, + IamfBufferInfo* info, + const bool prefer_binaural_audio, + const bool prefer_surround_audio); + bool ParseOBUHeader(BufferReader* reader, + uint8_t* obu_type, + uint32_t* obu_size) const; + bool ParseCodecConfigOBU(BufferReader* reader, IamfBufferInfo* info); + bool ParseAudioElementOBU(BufferReader* reader, + IamfBufferInfo* info, + const bool prefer_binaural_audio, + const bool prefer_surround_audio); + bool ParseMixPresentationOBU(BufferReader* reader, + IamfBufferInfo* info, + const bool prefer_binaural_audio, + const bool prefer_surround_audio); + // Helper function to skip parsing ParamDefinitions found in the config OBUs + // https://aomediacodec.github.io/iamf/v1.0.0-errata.html#paramdefinition + bool SkipParamDefinition(BufferReader* reader) const; + + std::unordered_set binaural_audio_element_ids_; + std::unordered_set surround_audio_element_ids_; + + BinauralMixSelection binaural_mix_selection_ = kBinauralMixSelectionNotFound; +}; + +} // namespace libiamf +} // namespace shared +} // namespace starboard + +#endif // STARBOARD_SHARED_LIBIAMF_IAMF_BUFFER_PARSER_H_ diff --git a/starboard/shared/libiamf/iamf_config_reader.cc b/starboard/shared/libiamf/iamf_config_reader.cc deleted file mode 100644 index 342b62706967..000000000000 --- a/starboard/shared/libiamf/iamf_config_reader.cc +++ /dev/null @@ -1,531 +0,0 @@ -// Copyright 2024 The Cobalt Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "starboard/shared/libiamf/iamf_config_reader.h" - -#include -#include - -#include "starboard/common/string.h" -#include "third_party/libiamf/source/code/include/IAMF_defines.h" - -namespace starboard { -namespace shared { -namespace libiamf { - -namespace { - -// From //media/formats/mp4/rcheck.h. -#define RCHECK(condition) \ - do { \ - if (!(condition)) { \ - SB_DLOG(ERROR) << "Failure while parsing IAMF config: " #condition; \ - return false; \ - } \ - } while (0) - -// From -// https://aomediacodec.github.io/iamf/v1.0.0-errata.html#obu-header-syntax. -constexpr int kObuTypeCodecConfig = 0; -constexpr int kObuTypeAudioElement = 1; -constexpr int kObuTypeMixPresentation = 2; -constexpr int kObuTypeSequenceHeader = 31; - -// From -// https://aomediacodec.github.io/iamf/v1.0.0-errata.html#obu-codecconfig. -constexpr int kFourccOpus = 0x4f707573; -constexpr int kFourccIpcm = 0x6970636d; - -} // namespace - -IamfConfigReader::BufferReader::BufferReader(const uint8_t* buf, size_t size) - : buf_(buf), pos_(0), size_(size), error_(false) {} - -bool IamfConfigReader::BufferReader::Read1(uint8_t* ptr) { - if (!HasBytes(sizeof(uint8_t)) || !ptr) { - error_ = true; - return false; - } - *ptr = buf_[pos_++]; - return true; -} - -bool IamfConfigReader::BufferReader::Read4(uint32_t* ptr) { - if (!HasBytes(sizeof(uint32_t)) || !ptr) { - error_ = true; - return false; - } - std::memcpy(ptr, &buf_[pos_], sizeof(uint32_t)); - *ptr = ByteSwap(*ptr); - pos_ += sizeof(uint32_t); - return true; -} - -bool IamfConfigReader::BufferReader::ReadLeb128(uint32_t* ptr) { - if (!HasBytes(sizeof(uint32_t)) || !ptr) { - error_ = true; - return false; - } - int bytes_read = ReadLeb128Internal(buf_ + pos_, ptr); - if (bytes_read < 0) { - error_ = true; - return false; - } - pos_ += bytes_read; - return true; -} - -bool IamfConfigReader::BufferReader::ReadString(std::string* str) { - int bytes_read = ReadStringInternal(buf_ + pos_, str); - if (bytes_read < 0) { - error_ = true; - return false; - } - pos_ += bytes_read; - return true; -} - -bool IamfConfigReader::BufferReader::SkipBytes(size_t size) { - if (!HasBytes(size)) { - error_ = true; - return false; - } - pos_ += size; - return true; -} - -bool IamfConfigReader::BufferReader::SkipLeb128() { - uint32_t val; - return ReadLeb128(&val); -} - -bool IamfConfigReader::BufferReader::SkipString() { - std::string str; - return ReadString(&str); -} - -int IamfConfigReader::BufferReader::ReadLeb128Internal(const uint8_t* buf, - uint32_t* value) { - SB_DCHECK(buf); - SB_DCHECK(value); - *value = 0; - bool error = true; - size_t i = 0; - for (; i < sizeof(uint32_t); ++i) { - uint8_t byte = buf[i]; - *value |= ((byte & 0x7f) << (i * 7)); - if (!(byte & 0x80)) { - error = false; - break; - } - } - - if (error) { - return -1; - } - return i + 1; -} - -int IamfConfigReader::BufferReader::ReadStringInternal(const uint8_t* buf, - std::string* str) { - SB_DCHECK(buf); - - int remaining_size = static_cast(size_) - pos_; - if (remaining_size <= 0) { - return -1; - } - - // The size of the string is capped to 128 bytes. - int max_str_size = std::min(remaining_size, 128); - str->clear(); - - size_t size = 0; - while (buf[size] != '\0' && size < max_str_size) { - size++; - } - - if (buf[size] != '\0') { - return -1; - } - - if (size > 0) { - str->resize(size); - std::memcpy(str->data(), reinterpret_cast(buf), size); - } - - // Account for null terminator byte. - return ++size; -} - -IamfConfigReader::IamfConfigReader( - const scoped_refptr& input_buffer, - bool prefer_binaural_audio, - bool prefer_surround_audio) - : prefer_binaural_audio_(prefer_binaural_audio), - prefer_surround_audio_(prefer_surround_audio) { -#if SB_IS_BIG_ENDIAN -#error IamfConfigReader assumes little-endianness. -#endif // SB_IS_BIG_ENDIAN - SB_DCHECK(!(prefer_binaural_audio && prefer_surround_audio)); - Read(input_buffer); -} - -bool IamfConfigReader::Read(const scoped_refptr& input_buffer) { - SB_DCHECK(input_buffer->data()); - BufferReader reader(input_buffer->data(), input_buffer->size()); - - while (!is_valid() && reader.pos() < reader.size()) { - RCHECK(ReadOBU(&reader)); - } - - if (reader.error()) { - return false; - } - - data_size_ = reader.size() - config_size_; - config_obus_.assign(reader.buf(), reader.buf() + config_size_); - data_.assign(reader.buf() + config_size_, reader.buf() + reader.size()); - - return true; -} - -bool IamfConfigReader::ReadOBU(BufferReader* reader) { - SB_DCHECK(reader); - uint8_t obu_type = 0; - uint32_t obu_size = 0; - if (!ReadOBUHeader(reader, &obu_type, &obu_size)) { - SB_DLOG(ERROR) << "Error reading OBU header"; - return false; - } - - int next_obu_pos = reader->pos() + obu_size; - - auto skip_param_definition = [&]() { - // parameter_id - RCHECK(reader->SkipLeb128()); - // parameter_rate - RCHECK(reader->SkipLeb128()); - uint8_t param_definition_mode; - RCHECK(reader->Read1(¶m_definition_mode)); - param_definition_mode = param_definition_mode >> 7; - if (param_definition_mode == static_cast(0)) { - // duration - RCHECK(reader->SkipLeb128()); - uint32_t constant_subblock_duration; - RCHECK(reader->ReadLeb128(&constant_subblock_duration)); - if (constant_subblock_duration == 0) { - uint32_t num_subblocks; - RCHECK(reader->ReadLeb128(&num_subblocks)); - for (int i = 0; i < num_subblocks; ++i) { - // subblock_duration - RCHECK(reader->SkipLeb128()); - } - } - } - return true; - }; - - switch (static_cast(obu_type)) { - case kObuTypeCodecConfig: { - RCHECK(reader->SkipLeb128()); - - uint32_t codec_id = 0; - RCHECK(reader->Read4(&codec_id)); - - RCHECK(reader->ReadLeb128(&samples_per_buffer_)); - - // audio_roll_distance - RCHECK(reader->SkipBytes(2)); - - switch (codec_id) { - case kFourccOpus: - sample_rate_ = 48000; - break; - case kFourccIpcm: { - // sample_format_flags - RCHECK(reader->SkipBytes(1)); - - uint8_t sample_size_unsigned; - RCHECK(reader->Read1(&sample_size_unsigned)); - sample_size_ = static_cast(sample_size_unsigned); - - uint32_t sample_rate_unsigned; - RCHECK(reader->Read4(&sample_rate_unsigned)); - sample_rate_ = static_cast(sample_rate_unsigned); - break; - } - default: - SB_NOTREACHED(); - return false; - } - - break; - } - case kObuTypeAudioElement: { - uint32_t audio_element_id; - RCHECK(reader->ReadLeb128(&audio_element_id)); - - uint8_t audio_element_type; - RCHECK(reader->Read1(&audio_element_type)); - audio_element_type = audio_element_type >> 5; - - // codec_config_id - RCHECK(reader->SkipLeb128()); - - uint32_t num_substreams; - RCHECK(reader->ReadLeb128(&num_substreams)); - - for (int i = 0; i < num_substreams; ++i) { - // audio_substream_id - RCHECK(reader->SkipLeb128()); - } - - uint32_t num_parameters; - RCHECK(reader->ReadLeb128(&num_parameters)); - - for (int i = 0; i < num_parameters; ++i) { - uint32_t param_definition_type; - RCHECK(reader->ReadLeb128(¶m_definition_type)); - - if (param_definition_type == IAMF_PARAMETER_TYPE_DEMIXING) { - skip_param_definition(); - // DemixingParamDefintion - RCHECK(reader->SkipBytes(1)); - } else if (param_definition_type == IAMF_PARAMETER_TYPE_RECON_GAIN) { - skip_param_definition(); - } else if (param_definition_type > 2) { - uint32_t param_definition_size; - RCHECK(reader->ReadLeb128(¶m_definition_size)); - RCHECK(reader->SkipBytes(param_definition_size)); - } - } - - if (static_cast(audio_element_type) == - AUDIO_ELEMENT_CHANNEL_BASED && - (prefer_binaural_audio_ || prefer_surround_audio_)) { - // Parse ScalableChannelLayoutConfig for binaural and surround - // loudspeaker layouts - uint8_t num_layers; - RCHECK(reader->Read1(&num_layers)); - num_layers = num_layers >> 5; - // Read ChannelAudioLayerConfigs - for (int i = 0; i < static_cast(num_layers); ++i) { - uint8_t loudspeaker_layout; - bool output_gain_is_present_flag; - RCHECK(reader->Read1(&loudspeaker_layout)); - output_gain_is_present_flag = (loudspeaker_layout >> 3) & 0x01; - loudspeaker_layout = loudspeaker_layout >> 4; - if (loudspeaker_layout == IA_CHANNEL_LAYOUT_BINAURAL) { - binaural_audio_element_ids_.insert(audio_element_id); - } else if (loudspeaker_layout > IA_CHANNEL_LAYOUT_STEREO && - loudspeaker_layout < IA_CHANNEL_LAYOUT_COUNT) { - surround_audio_element_ids_.insert(audio_element_id); - } - - // substream_count and coupled_substream_count - RCHECK(reader->SkipBytes(2)); - - if (output_gain_is_present_flag) { - // output_gain_flags and output_gain - RCHECK(reader->SkipBytes(3)); - } - - if (i == 1 && loudspeaker_layout == static_cast(15)) { - // expanded_loudspeaker_layout - RCHECK(reader->SkipBytes(1)); - } - } - } - break; - } - case kObuTypeSequenceHeader: - break; - case kObuTypeMixPresentation: { - uint32_t mix_presentation_id; - RCHECK(reader->ReadLeb128(&mix_presentation_id)); - - uint32_t count_label; - RCHECK(reader->ReadLeb128(&count_label)); - for (int i = 0; i < count_label; ++i) { - // language_label; - RCHECK(reader->SkipString()); - } - - for (int i = 0; i < count_label; ++i) { - // MixPresentationAnnotations; - RCHECK(reader->SkipString()); - } - - uint32_t num_sub_mixes; - RCHECK(reader->ReadLeb128(&num_sub_mixes)); - for (int i = 0; i < num_sub_mixes; ++i) { - uint32_t num_audio_elements; - RCHECK(reader->ReadLeb128(&num_audio_elements)); - for (int j = 0; j < num_audio_elements; ++j) { - uint32_t audio_element_id; - RCHECK(reader->ReadLeb128(&audio_element_id)); - - // Set a mix presentation for binaural or surround streams. The mix - // presentation is chosen if there exists an audio element that has - // the qualities it requires - such as an audio element with a - // binaural loudspeaker layout. - if (!mix_presentation_id_.has_value() || - (prefer_binaural_audio_ && - binaural_mix_selection_ > - kBinauralMixSelectionLoudspeakerLayout)) { - if (prefer_binaural_audio_ && - binaural_audio_element_ids_.find(audio_element_id) != - binaural_audio_element_ids_.end()) { - mix_presentation_id_ = mix_presentation_id; - binaural_mix_selection_ = kBinauralMixSelectionLoudspeakerLayout; - } else if (prefer_surround_audio_ && - surround_audio_element_ids_.find(audio_element_id) != - surround_audio_element_ids_.end()) { - mix_presentation_id_ = mix_presentation_id; - } - } - - for (int k = 0; k < count_label; ++k) { - // MixPresentationElementAnnotatoions - RCHECK(reader->SkipString()); - } - - // The following fields are for the RenderingConfig - // headphones_rendering_mode - RCHECK(reader->SkipBytes(1)); - uint32_t rendering_config_extension_size; - RCHECK(reader->ReadLeb128(&rendering_config_extension_size)); - // rendering_config_extension_bytes - RCHECK(reader->SkipBytes(rendering_config_extension_size)); - - // The following fields are for the ElementMixConfig - RCHECK(skip_param_definition()); - // default_mix_gain - RCHECK(reader->SkipBytes(2)); - } - // The following fields are for the OutputMixConfig - RCHECK(skip_param_definition()); - // default_mix_gain - RCHECK(reader->SkipBytes(2)); - - uint32_t num_layouts; - RCHECK(reader->ReadLeb128(&num_layouts)); - for (int j = 0; j < num_layouts; ++j) { - uint8_t layout_type; - RCHECK(reader->Read1(&layout_type)); - layout_type = layout_type >> 6; - // If a binaural mix presentation is preferred and the mix - // presentation id has not yet been set, set the mix presentation id - // if the current mix presentation has a binaural loudness layout. The - // mix presentation id will change if a different mix presentation is - // found that uses an audio element with a binaural loudspeaker - // layout, as that is higher priority. - if (static_cast(layout_type) == IAMF_LAYOUT_TYPE_BINAURAL && - prefer_binaural_audio_ && - (!mix_presentation_id_.has_value() || - binaural_mix_selection_ > kBinauralMixSelectionLoudnessLayout)) { - mix_presentation_id_ = mix_presentation_id; - binaural_mix_selection_ = kBinauralMixSelectionLoudnessLayout; - } - - // The following fields are for the LoudnessInfo - uint8_t info_type; - RCHECK(reader->Read1(&info_type)); - // integrated_loudness and digital_loudness - RCHECK(reader->SkipBytes(4)); - if (info_type & 1) { - // true_peak - RCHECK(reader->SkipBytes(2)); - } - if (info_type & 2) { - uint8_t num_anchored_loudness; - RCHECK(reader->Read1(&num_anchored_loudness)); - for (uint8_t k = 0; k < num_anchored_loudness; ++k) { - // anchor_element and anchored_loudness - RCHECK(reader->SkipBytes(3)); - } - } - if ((info_type & 0b11111100) > 0) { - uint32_t info_type_size; - RCHECK(reader->ReadLeb128(&info_type_size)); - // info_type_bytes - RCHECK(reader->SkipBytes(info_type_size)); - } - } - } - - // If the mix presentation id is unassigned at this point, the stream is - // stereo, or a proper mix presentation for binaural or surround preferred - // streams hasn't yet been parsed. Default to the first read mix - // presentation in case a preferred mix does not exist. - if (!mix_presentation_id_.has_value()) { - mix_presentation_id_ = mix_presentation_id; - } - - break; - } - default: - // Once an OBU is read that is not a descriptor, descriptor parsing is - // assumed to be complete. - SB_DCHECK(mix_presentation_id_.has_value()); - return true; - } - - // Skip to the next OBU. - const size_t remaining_size = next_obu_pos - reader->pos(); - RCHECK(reader->SkipBytes(remaining_size)); - config_size_ = reader->pos(); - return true; -} - -bool IamfConfigReader::ReadOBUHeader(BufferReader* reader, - uint8_t* obu_type, - uint32_t* obu_size) { - uint8_t header_flags; - RCHECK(reader->Read1(&header_flags)); - *obu_type = (header_flags >> 3) & 0x1f; - - const bool obu_redundant_copy = (header_flags >> 2) & 1; - const bool obu_trimming_status_flag = (header_flags >> 1) & 1; - const bool obu_extension_flag = header_flags & 1; - - *obu_size = 0; - - RCHECK(reader->ReadLeb128(obu_size)); - - // |obu_size| contains the size of the OBU after its own field. - // If either of the flags are set, subtract the number of bytes read - // from the flags from |obu_size| before returning to ReadOBU(). - size_t reader_pos_before_flags = reader->pos(); - - if (obu_trimming_status_flag) { - RCHECK(reader->SkipLeb128()); - RCHECK(reader->SkipLeb128()); - } - - if (obu_extension_flag) { - RCHECK(reader->SkipLeb128()); - } - - size_t flag_bytes_read = reader->pos() - reader_pos_before_flags; - if (flag_bytes_read >= *obu_size) { - return false; - } - *obu_size -= flag_bytes_read; - return true; -} - -} // namespace libiamf -} // namespace shared -} // namespace starboard diff --git a/starboard/shared/libiamf/iamf_config_reader.h b/starboard/shared/libiamf/iamf_config_reader.h deleted file mode 100644 index 0a7db26bba9a..000000000000 --- a/starboard/shared/libiamf/iamf_config_reader.h +++ /dev/null @@ -1,143 +0,0 @@ -// Copyright 2024 The Cobalt Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef STARBOARD_SHARED_LIBIAMF_IAMF_CONFIG_READER_H_ -#define STARBOARD_SHARED_LIBIAMF_IAMF_CONFIG_READER_H_ - -#include -#include -#include -#include - -#include "starboard/common/log.h" -#include "starboard/common/ref_counted.h" -#include "starboard/shared/internal_only.h" -#include "starboard/shared/starboard/player/input_buffer_internal.h" - -namespace starboard { -namespace shared { -namespace libiamf { - -// TODO: Add handling for non-redundant OBUs -class IamfConfigReader { - public: - typedef ::starboard::shared::starboard::player::InputBuffer InputBuffer; - - IamfConfigReader(const scoped_refptr& input_buffer, - bool prefer_binaural_audio, - bool prefer_surround_audio); - - bool is_valid() const { - return mix_presentation_id_.has_value() && sample_rate_ > 0 && - samples_per_buffer_ > 0; - } - int sample_rate() const { return sample_rate_; } - uint32_t samples_per_buffer() const { return samples_per_buffer_; } - uint32_t config_size() const { return config_size_; } - uint32_t mix_presentation_id() const { - SB_DCHECK(is_valid()) - << "Called mix_presentation_id() on invalid IamfConfigReader."; - if (prefer_binaural_audio_ && - binaural_mix_selection_ == kBinauralMixSelectionNotFound) { - SB_LOG(INFO) << "Could not find binaural mix presentation."; - } - return mix_presentation_id_.value(); - } - - const std::vector& config_obus() const { return config_obus_; } - const std::vector& data() const { return data_; } - - private: - class BufferReader { - public: - BufferReader(const uint8_t* buf, size_t size); - - bool Read1(uint8_t* ptr); - bool Read4(uint32_t* ptr); - bool ReadLeb128(uint32_t* ptr); - bool ReadString(std::string* str); - - bool SkipBytes(size_t size); - bool SkipLeb128(); - bool SkipString(); - - size_t size() const { return size_; } - size_t pos() const { return pos_; } - const uint8_t* buf() const { return buf_; } - bool error() const { return error_; } - - private: - bool HasBytes(size_t size) const { return size + pos_ <= size_; } - inline uint32_t ByteSwap(uint32_t x) { -#if defined(COMPILER_MSVC) - return _byteswap_ulong(x); -#else - return __builtin_bswap32(x); -#endif - } - // Decodes an Leb128 value and stores it in |value|. Returns the number of - // bytes read, capped to sizeof(uint32_t). Returns the number of bytes read, - // or -1 on error. - int ReadLeb128Internal(const uint8_t* buf, uint32_t* value); - // Reads a c-string into |str|. Returns the number of bytes read, capped to - // 128 bytes, or -1 on error. - int ReadStringInternal(const uint8_t* buf, std::string* str); - - int pos_ = 0; - const uint8_t* buf_; - const size_t size_ = 0; - bool error_ = false; - }; - - // Used in the selection of a binaural mix presentation, using the strategy - // defined in - // https://aomediacodec.github.io/iamf/#processing-mixpresentation-selection. - // The preferred methods of choosing a binaural mix presentation are listed - // from high to low. - enum BinauralMixSelection { - kBinauralMixSelectionLoudspeakerLayout, - kBinauralMixSelectionLoudnessLayout, - kBinauralMixSelectionNotFound - }; - - bool Read(const scoped_refptr& input_buffer); - // Reads a single Descriptor OBU. Returns false on error. - bool ReadOBU(BufferReader* reader); - bool ReadOBUHeader(BufferReader* reader, - uint8_t* obu_type, - uint32_t* obu_size); - - int sample_rate_ = 0; - int sample_size_ = 0; - uint32_t samples_per_buffer_ = 0; - uint32_t config_size_ = 0; - uint32_t data_size_ = 0; - - std::optional mix_presentation_id_; - std::unordered_set binaural_audio_element_ids_; - std::unordered_set surround_audio_element_ids_; - const bool prefer_binaural_audio_; - const bool prefer_surround_audio_; - - BinauralMixSelection binaural_mix_selection_ = kBinauralMixSelectionNotFound; - - std::vector config_obus_; - std::vector data_; -}; - -} // namespace libiamf -} // namespace shared -} // namespace starboard - -#endif // STARBOARD_SHARED_LIBIAMF_IAMF_CONFIG_READER_H_ From c7e3ab873adb9dfecb3c2d14f7ede8ba90f49b05 Mon Sep 17 00:00:00 2001 From: Austin Osagie Date: Fri, 6 Sep 2024 11:40:12 -0700 Subject: [PATCH 13/30] Update handling of surround audio configurations --- .../shared/libiamf/iamf_audio_decoder.cc | 50 +++++-------------- 1 file changed, 12 insertions(+), 38 deletions(-) diff --git a/starboard/shared/libiamf/iamf_audio_decoder.cc b/starboard/shared/libiamf/iamf_audio_decoder.cc index 58db3987d272..43db908041b0 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.cc +++ b/starboard/shared/libiamf/iamf_audio_decoder.cc @@ -27,7 +27,8 @@ namespace { using shared::starboard::player::DecodedAudio; constexpr int kForceBinauralAudio = false; -constexpr int kForceSurroundAudio = false; +constexpr int kForce6ChannelAudio = false; +constexpr int kForce8ChannelAudio = false; std::string ErrorCodeToString(int code) { switch (code) { @@ -115,8 +116,9 @@ bool IamfAudioDecoder::DecodeInternal( } IamfBufferParser::IamfBufferInfo info; - IamfBufferParser().ParseInputBuffer(input_buffer, &info, kForceBinauralAudio, - kForceSurroundAudio); + IamfBufferParser().ParseInputBuffer( + input_buffer, &info, kForceBinauralAudio, + kForce6ChannelAudio | kForce8ChannelAudio); if (!info.is_valid()) { ReportError("Failed to parse IA Descriptors"); return false; @@ -209,43 +211,15 @@ bool IamfAudioDecoder::ConfigureDecoder(IamfBufferInfo* info, return false; } } else { - // Default to stereo output. If kForceSurroundAudio is true, set to a sound - // system matching, the platform's audio configuration, if available. IAMF_SoundSystem sound_system = SOUND_SYSTEM_A; - if (kForceSurroundAudio) { - SbMediaAudioConfiguration out_config; - SbMediaGetAudioConfiguration(0, &out_config); - int channels = std::max(out_config.number_of_channels, 2); - if (channels > 8 || channels < 1) { - ReportError(::starboard::FormatString( - "Can't create decoder with %i channels", channels)); - return false; - } - switch (channels) { - case 1: - sound_system = SOUND_SYSTEM_MONO; - break; - case 2: - // Stereo output. - sound_system = SOUND_SYSTEM_A; - SB_DLOG(INFO) << "Configuring IamfAudioDecoder for stereo output"; - break; - case 6: - // 5.1 output. - sound_system = SOUND_SYSTEM_B; - SB_DLOG(INFO) << "Configuring IamfAudioDecoder for 5.1 output"; - break; - case 8: - // 7.1 output. - sound_system = SOUND_SYSTEM_C; - SB_DLOG(INFO) << "Configuring IamfAudioDecoder for 7.1 output"; - break; - default: - SB_NOTREACHED(); - return false; - } + if (kForce6ChannelAudio) { + SB_LOG(INFO) << "Configuring IamfAudioDecoder for 5.1 output"; + sound_system = SOUND_SYSTEM_B; + } else if (kForce8ChannelAudio) { + SB_LOG(INFO) << "Configuring IamfAudioDecoder for 7.1 output"; + sound_system = SOUND_SYSTEM_C; } else { - SB_DLOG(INFO) << "Defaulting to stereo output."; + SB_LOG(INFO) << "Configuring IamfAudioDecoder for stereo output"; } error = IAMF_decoder_output_layout_set_sound_system(decoder_, sound_system); From 55d6ae0bf362b94145b831c1a703d64fc805a068 Mon Sep 17 00:00:00 2001 From: Austin Osagie Date: Mon, 7 Oct 2024 15:22:58 -0700 Subject: [PATCH 14/30] Remove Android IAMF support --- starboard/android/shared/BUILD.gn | 12 ------------ starboard/android/shared/media_common.h | 5 ----- .../android/shared/media_is_audio_supported.cc | 6 ------ .../shared/platform_configuration/BUILD.gn | 9 --------- .../android/shared/player_components_factory.h | 18 +----------------- starboard/android/shared/player_create.cc | 6 +----- 6 files changed, 2 insertions(+), 54 deletions(-) diff --git a/starboard/android/shared/BUILD.gn b/starboard/android/shared/BUILD.gn index 036107b6b13f..fa80f7b325e6 100644 --- a/starboard/android/shared/BUILD.gn +++ b/starboard/android/shared/BUILD.gn @@ -493,18 +493,6 @@ static_library("starboard_platform") { if (sb_evergreen_compatible_use_libunwind) { deps += [ "//third_party/llvm-project/libunwind:unwind_starboard" ] } - - defines = [] - if (enable_iamf_decode) { - sources += [ - "//starboard/shared/libiamf/iamf_audio_decoder.cc", - "//starboard/shared/libiamf/iamf_audio_decoder.h", - "//starboard/shared/libiamf/iamf_buffer_parser.cc", - "//starboard/shared/libiamf/iamf_buffer_parser.h", - ] - - defines += [ "ENABLE_IAMF_DECODE" ] - } } static_library("starboard_base_symbolize") { diff --git a/starboard/android/shared/media_common.h b/starboard/android/shared/media_common.h index 4a2b6b86731c..dcbd0b36d108 100644 --- a/starboard/android/shared/media_common.h +++ b/starboard/android/shared/media_common.h @@ -61,11 +61,6 @@ inline const char* SupportedAudioCodecToMimeType( if (audio_codec == kSbMediaAudioCodecOpus) { return "audio/opus"; } -#if SB_API_VERSION >= 15 - if (audio_codec == kSbMediaAudioCodecIamf) { - return "audio/iamf"; - } -#endif // SB_API_VERSION >= 15 return nullptr; } diff --git a/starboard/android/shared/media_is_audio_supported.cc b/starboard/android/shared/media_is_audio_supported.cc index 364a093210da..023cbfc5f8fb 100644 --- a/starboard/android/shared/media_is_audio_supported.cc +++ b/starboard/android/shared/media_is_audio_supported.cc @@ -60,12 +60,6 @@ bool SbMediaIsAudioSupported(SbMediaAudioCodec audio_codec, return true; } -#if SB_API_VERSION >= 15 && ENABLE_IAMF_DECODE - if (audio_codec == kSbMediaAudioCodecIamf) { - return true; - } -#endif // SB_API_VERSION >= 15 && ENABLE_IAMF_DECODE - bool media_codec_supported = MediaCapabilitiesCache::GetInstance()->HasAudioDecoderFor(mime, bitrate); diff --git a/starboard/android/shared/platform_configuration/BUILD.gn b/starboard/android/shared/platform_configuration/BUILD.gn index 2885fccd10f5..f638914483e8 100644 --- a/starboard/android/shared/platform_configuration/BUILD.gn +++ b/starboard/android/shared/platform_configuration/BUILD.gn @@ -175,9 +175,6 @@ config("platform_configuration") { "-Wl,--wrap=readdir_r", ] } - if (enable_iamf_decode) { - configs += [ ":libiamf_config" ] - } } config("size") { @@ -220,9 +217,3 @@ config("pedantic_warnings") { "-Wno-unused-parameter", ] } - -if (enable_iamf_decode) { - config("libiamf_config") { - libs = [ "//third_party/libiamf/platforms/android/libiamf.a" ] - } -} diff --git a/starboard/android/shared/player_components_factory.h b/starboard/android/shared/player_components_factory.h index d5225d71cc4a..cba21570d486 100644 --- a/starboard/android/shared/player_components_factory.h +++ b/starboard/android/shared/player_components_factory.h @@ -48,10 +48,6 @@ #include "starboard/shared/starboard/player/filter/video_renderer_internal_impl.h" #include "starboard/shared/starboard/player/filter/video_renderer_sink.h" -#if ENABLE_IAMF_DECODE -#include "starboard/shared/libiamf/iamf_audio_decoder.h" -#endif // ENABLE_IAMF_DECODE - namespace starboard { namespace android { namespace shared { @@ -118,7 +114,7 @@ class AudioRendererSinkAndroid : public ::starboard::shared::starboard::player:: private: bool IsAudioSampleTypeSupported( SbMediaAudioSampleType audio_sample_type) const override { - if (tunnel_mode_audio_session_id_ != -1) { + if (tunnel_mode_audio_session_id_ != -1 || true) { // Currently the implementation only supports tunnel mode with int16 audio // samples. return audio_sample_type == kSbMediaAudioSampleTypeInt16Deprecated; @@ -448,18 +444,6 @@ class PlayerComponentsFactory : public starboard::shared::starboard::player:: return std::unique_ptr( std::move(audio_decoder_impl)); } -#if SB_API_VERSION >= 15 && ENABLE_IAMF_DECODE - } else if (audio_stream_info.codec == kSbMediaAudioCodecIamf) { - SB_LOG(INFO) << "Creating IAMF audio decoder"; - std::unique_ptr - audio_decoder_impl( - new starboard::shared::libiamf::IamfAudioDecoder( - audio_stream_info)); - if (audio_decoder_impl->is_valid()) { - return std::unique_ptr( - std::move(audio_decoder_impl)); - } -#endif // SB_API_VERSION >= 15 && ENABLE_IAMF_DECODE } else { SB_LOG(ERROR) << "Unsupported audio codec " << audio_stream_info.codec; diff --git a/starboard/android/shared/player_create.cc b/starboard/android/shared/player_create.cc index 2b9a88903b09..78b4ba46f252 100644 --- a/starboard/android/shared/player_create.cc +++ b/starboard/android/shared/player_create.cc @@ -127,11 +127,7 @@ SbPlayer SbPlayerCreate(SbWindow window, audio_codec != kSbMediaAudioCodecAac && audio_codec != kSbMediaAudioCodecAc3 && audio_codec != kSbMediaAudioCodecEac3 && - audio_codec != kSbMediaAudioCodecOpus -#if SB_API_VERSION >= 15 - && audio_codec != kSbMediaAudioCodecIamf -#endif // SB_API_VERSION >= 15 - ) { + audio_codec != kSbMediaAudioCodecOpus) { SB_LOG(ERROR) << "Unsupported audio codec: " << starboard::GetMediaAudioCodecName(audio_codec) << "."; player_error_func( From 1fd6ba5aad32c0a1009e1b665b91e91572eceee0 Mon Sep 17 00:00:00 2001 From: Austin Osagie Date: Mon, 7 Oct 2024 15:22:58 -0700 Subject: [PATCH 15/30] Remove Android IAMF support --- media/base/starboard_utils.cc | 24 +- starboard/linux/shared/BUILD.gn | 4 +- .../shared/libiamf/iamf_audio_decoder.cc | 28 +- starboard/shared/libiamf/iamf_audio_decoder.h | 6 +- starboard/shared/libiamf/iamf_buffer_parser.h | 109 -------- ...buffer_parser.cc => iamf_decoder_utils.cc} | 248 ++++++++++-------- starboard/shared/libiamf/iamf_decoder_utils.h | 52 ++++ 7 files changed, 228 insertions(+), 243 deletions(-) delete mode 100644 starboard/shared/libiamf/iamf_buffer_parser.h rename starboard/shared/libiamf/{iamf_buffer_parser.cc => iamf_decoder_utils.cc} (82%) create mode 100644 starboard/shared/libiamf/iamf_decoder_utils.h diff --git a/media/base/starboard_utils.cc b/media/base/starboard_utils.cc index 4e3156fabb39..c3b1367ac998 100644 --- a/media/base/starboard_utils.cc +++ b/media/base/starboard_utils.cc @@ -29,6 +29,8 @@ #include "starboard/configuration.h" #include "starboard/memory.h" +#include "starboard/common/log.h" + using base::Time; using base::TimeDelta; @@ -55,6 +57,18 @@ int GetBitsPerPixel(const std::string& mime_type) { return 8; } +int GetMaxChannelCount() { + int channels = 2; + int index = 0; + SbMediaAudioConfiguration configuration; + while (SbMediaGetAudioConfiguration(index++, &configuration)) { + SB_LOG(INFO) << "Channels for index " << index-1 << " = " << configuration.number_of_channels; + if (configuration.number_of_channels > channels) + channels = configuration.number_of_channels; + } + return channels; +} + } // namespace SbMediaAudioCodec MediaAudioCodecToSbMediaAudioCodec(AudioCodec codec) { @@ -143,8 +157,14 @@ SbMediaAudioStreamInfo MediaAudioConfigToSbMediaAudioStreamInfo( #if SB_API_VERSION < 15 audio_stream_info.format_tag = 0x00ff; #endif // SB_API_VERSION < 15 - audio_stream_info.number_of_channels = - ChannelLayoutToChannelCount(audio_decoder_config.channel_layout()); + if (audio_stream_info.codec == kSbMediaAudioCodecIamf) { + // IAMF mixes audio signals to the highest available speaker layout. + audio_stream_info.number_of_channels = GetMaxChannelCount(); + } else { + audio_stream_info.number_of_channels = + ChannelLayoutToChannelCount(audio_decoder_config.channel_layout()); + } + audio_stream_info.samples_per_second = audio_decoder_config.samples_per_second(); #if SB_API_VERSION < 15 diff --git a/starboard/linux/shared/BUILD.gn b/starboard/linux/shared/BUILD.gn index bf96a97aad6a..6c01aecfe25e 100644 --- a/starboard/linux/shared/BUILD.gn +++ b/starboard/linux/shared/BUILD.gn @@ -445,8 +445,8 @@ static_library("starboard_platform_sources") { sources += [ "//starboard/shared/libiamf/iamf_audio_decoder.cc", "//starboard/shared/libiamf/iamf_audio_decoder.h", - "//starboard/shared/libiamf/iamf_buffer_parser.cc", - "//starboard/shared/libiamf/iamf_buffer_parser.h", + "//starboard/shared/libiamf/iamf_decoder_utils.cc", + "//starboard/shared/libiamf/iamf_decoder_utils.h", ] defines += [ "ENABLE_IAMF_DECODE" ] diff --git a/starboard/shared/libiamf/iamf_audio_decoder.cc b/starboard/shared/libiamf/iamf_audio_decoder.cc index 43db908041b0..1b8ec3bfc8ee 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.cc +++ b/starboard/shared/libiamf/iamf_audio_decoder.cc @@ -27,8 +27,6 @@ namespace { using shared::starboard::player::DecodedAudio; constexpr int kForceBinauralAudio = false; -constexpr int kForce6ChannelAudio = false; -constexpr int kForce8ChannelAudio = false; std::string ErrorCodeToString(int code) { switch (code) { @@ -55,7 +53,8 @@ std::string ErrorCodeToString(int code) { } // namespace IamfAudioDecoder::IamfAudioDecoder(const AudioStreamInfo& audio_stream_info) - : audio_stream_info_(audio_stream_info) { + : audio_stream_info_(audio_stream_info), + channels_(audio_stream_info.number_of_channels) { decoder_ = IAMF_decoder_open(); if (!decoder_) { SB_LOG(ERROR) << "Error creating libiamf decoder"; @@ -110,16 +109,17 @@ bool IamfAudioDecoder::DecodeInternal( SB_DCHECK(!stream_ended_ || !pending_audio_buffers_.empty()); SB_DCHECK(is_valid()); + SB_LOG(INFO) << "Sample type is: " + << input_buffer->audio_sample_info().stream_info.bits_per_sample; if (input_buffer->size() == 0) { SB_LOG(ERROR) << "Empty input buffer written to IamfAudioDecoder"; + ReportError("Empty input buffer written to IamfAudioDecoder"); return false; } - IamfBufferParser::IamfBufferInfo info; - IamfBufferParser().ParseInputBuffer( - input_buffer, &info, kForceBinauralAudio, - kForce6ChannelAudio | kForce8ChannelAudio); - if (!info.is_valid()) { + IamfBufferInfo info; + if (!ParseInputBuffer(input_buffer, &info, kForceBinauralAudio, + channels_ > 2)) { ReportError("Failed to parse IA Descriptors"); return false; } @@ -130,9 +130,9 @@ bool IamfAudioDecoder::DecodeInternal( } scoped_refptr decoded_audio = new DecodedAudio( - audio_stream_info_.number_of_channels, GetSampleType(), - kSbMediaAudioFrameStorageTypeInterleaved, input_buffer->timestamp(), - audio_stream_info_.number_of_channels * info.num_samples * + channels_, GetSampleType(), kSbMediaAudioFrameStorageTypeInterleaved, + input_buffer->timestamp(), + channels_ * info.num_samples * starboard::media::GetBytesPerSample(GetSampleType())); int samples_decoded = IAMF_decoder_decode(decoder_, info.data.data(), info.data_size, nullptr, @@ -212,12 +212,12 @@ bool IamfAudioDecoder::ConfigureDecoder(IamfBufferInfo* info, } } else { IAMF_SoundSystem sound_system = SOUND_SYSTEM_A; - if (kForce6ChannelAudio) { + if (channels_ == 6) { SB_LOG(INFO) << "Configuring IamfAudioDecoder for 5.1 output"; sound_system = SOUND_SYSTEM_B; - } else if (kForce8ChannelAudio) { + } else if (channels_ == 8) { SB_LOG(INFO) << "Configuring IamfAudioDecoder for 7.1 output"; - sound_system = SOUND_SYSTEM_C; + sound_system = SOUND_SYSTEM_I; } else { SB_LOG(INFO) << "Configuring IamfAudioDecoder for stereo output"; } diff --git a/starboard/shared/libiamf/iamf_audio_decoder.h b/starboard/shared/libiamf/iamf_audio_decoder.h index 6f457d548b90..b6b0a0c04dc2 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.h +++ b/starboard/shared/libiamf/iamf_audio_decoder.h @@ -22,7 +22,7 @@ #include "starboard/common/ref_counted.h" #include "starboard/media.h" #include "starboard/shared/internal_only.h" -#include "starboard/shared/libiamf/iamf_buffer_parser.h" +#include "starboard/shared/libiamf/iamf_decoder_utils.h" #include "starboard/shared/starboard/media/media_util.h" #include "starboard/shared/starboard/player/decoded_audio_internal.h" #include "starboard/shared/starboard/player/filter/audio_decoder_internal.h" @@ -38,7 +38,8 @@ class IamfAudioDecoder private starboard::player::JobQueue::JobOwner { public: typedef starboard::media::AudioStreamInfo AudioStreamInfo; - typedef shared::libiamf::IamfBufferParser::IamfBufferInfo IamfBufferInfo; + // typedef shared::libiamf::IamfBufferInfo IamfBufferInfo; + // using shared::libiamf::IamfBufferInfo; explicit IamfAudioDecoder(const AudioStreamInfo& audio_stream_info); ~IamfAudioDecoder() override; @@ -76,6 +77,7 @@ class IamfAudioDecoder bool decoder_is_configured_ = false; std::deque> pending_audio_buffers_; int samples_per_second_ = 0; + const int channels_; }; } // namespace libiamf diff --git a/starboard/shared/libiamf/iamf_buffer_parser.h b/starboard/shared/libiamf/iamf_buffer_parser.h deleted file mode 100644 index 83154fd0825a..000000000000 --- a/starboard/shared/libiamf/iamf_buffer_parser.h +++ /dev/null @@ -1,109 +0,0 @@ -// Copyright 2024 The Cobalt Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef STARBOARD_SHARED_LIBIAMF_IAMF_BUFFER_PARSER_H_ -#define STARBOARD_SHARED_LIBIAMF_IAMF_BUFFER_PARSER_H_ - -#include -#include -#include -#include - -#include "starboard/common/log.h" -#include "starboard/common/ref_counted.h" -#include "starboard/shared/internal_only.h" -#include "starboard/shared/starboard/player/input_buffer_internal.h" - -namespace starboard { -namespace shared { -namespace libiamf { - -class BufferReader; - -// TODO: Skip parsing if the OBUs are redundant -class IamfBufferParser { - public: - typedef ::starboard::shared::starboard::player::InputBuffer InputBuffer; - - struct IamfBufferInfo { - bool is_valid() const { - return mix_presentation_id.has_value() && sample_rate > 0 && - num_samples > 0; - } - - uint32_t num_samples; - int sample_rate; - std::optional mix_presentation_id; - std::vector config_obus; - size_t config_obus_size; - std::vector data; - size_t data_size; - const scoped_refptr input_buffer; - }; - - IamfBufferParser(); - - bool ParseInputBuffer(const scoped_refptr& input_buffer, - IamfBufferInfo* info, - const bool prefer_binaural_audio, - const bool prefer_surround_audio); - - private: - // Used in the selection of a binaural mix presentation, using the strategy - // defined in - // https://aomediacodec.github.io/iamf/#processing-mixpresentation-selection. - // The preferred methods of choosing a binaural mix presentation are listed - // from high to low. - enum BinauralMixSelection { - kBinauralMixSelectionLoudspeakerLayout, - kBinauralMixSelectionLoudnessLayout, - kBinauralMixSelectionNotFound - }; - - bool ParseInputBufferInternal(const scoped_refptr& input_buffer, - IamfBufferInfo* info, - const bool prefer_binaural_audio, - const bool prefer_surround_audio); - // Reads a single Descriptor OBU. Returns false on error. - bool ParseDescriptorOBU(BufferReader* reader, - IamfBufferInfo* info, - const bool prefer_binaural_audio, - const bool prefer_surround_audio); - bool ParseOBUHeader(BufferReader* reader, - uint8_t* obu_type, - uint32_t* obu_size) const; - bool ParseCodecConfigOBU(BufferReader* reader, IamfBufferInfo* info); - bool ParseAudioElementOBU(BufferReader* reader, - IamfBufferInfo* info, - const bool prefer_binaural_audio, - const bool prefer_surround_audio); - bool ParseMixPresentationOBU(BufferReader* reader, - IamfBufferInfo* info, - const bool prefer_binaural_audio, - const bool prefer_surround_audio); - // Helper function to skip parsing ParamDefinitions found in the config OBUs - // https://aomediacodec.github.io/iamf/v1.0.0-errata.html#paramdefinition - bool SkipParamDefinition(BufferReader* reader) const; - - std::unordered_set binaural_audio_element_ids_; - std::unordered_set surround_audio_element_ids_; - - BinauralMixSelection binaural_mix_selection_ = kBinauralMixSelectionNotFound; -}; - -} // namespace libiamf -} // namespace shared -} // namespace starboard - -#endif // STARBOARD_SHARED_LIBIAMF_IAMF_BUFFER_PARSER_H_ diff --git a/starboard/shared/libiamf/iamf_buffer_parser.cc b/starboard/shared/libiamf/iamf_decoder_utils.cc similarity index 82% rename from starboard/shared/libiamf/iamf_buffer_parser.cc rename to starboard/shared/libiamf/iamf_decoder_utils.cc index 02cd04bef7a6..ef9dc083da36 100644 --- a/starboard/shared/libiamf/iamf_buffer_parser.cc +++ b/starboard/shared/libiamf/iamf_decoder_utils.cc @@ -12,11 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "starboard/shared/libiamf/iamf_buffer_parser.h" +#include "starboard/shared/libiamf/iamf_decoder_utils.h" #include #include #include +#include #include "third_party/libiamf/source/code/include/IAMF_defines.h" @@ -47,7 +48,16 @@ constexpr int kObuTypeSequenceHeader = 31; constexpr int kFourccOpus = 0x4f707573; constexpr int kFourccIpcm = 0x6970636d; -} // namespace +// Used in the selection of a binaural mix presentation, using the strategy +// defined in +// https://aomediacodec.github.io/iamf/#processing-mixpresentation-selection. +// The preferred methods of choosing a binaural mix presentation are listed +// from high to low. +enum BinauralMixSelection { + kBinauralMixSelectionLoudspeakerLayout, + kBinauralMixSelectionLoudnessLayout, + kBinauralMixSelectionNotFound +}; class BufferReader { public: @@ -197,86 +207,42 @@ class BufferReader { const size_t size_ = 0; }; -IamfBufferParser::IamfBufferParser() {} - -bool IamfBufferParser::ParseInputBuffer( - const scoped_refptr& input_buffer, - IamfBufferInfo* info, - const bool prefer_binaural_audio, - const bool prefer_surround_audio) { - SB_DCHECK(info); - SB_DCHECK(input_buffer->data()); - SB_DCHECK(!(prefer_binaural_audio && prefer_surround_audio)); - RCHECK(ParseInputBufferInternal(input_buffer, info, prefer_binaural_audio, - prefer_surround_audio)); - return true; -} - -bool IamfBufferParser::ParseInputBufferInternal( - const scoped_refptr& input_buffer, - IamfBufferInfo* info, - const bool prefer_binaural_audio, - const bool prefer_surround_audio) { - BufferReader reader(input_buffer->data(), input_buffer->size()); - - while (!info->is_valid() && reader.pos() < reader.size()) { - RCHECK(ParseDescriptorOBU(&reader, info, prefer_binaural_audio, - prefer_surround_audio)); +// Helper function to skip parsing ParamDefinitions found in the config OBUs +// https://aomediacodec.github.io/iamf/v1.0.0-errata.html#paramdefinition +bool SkipParamDefinition(BufferReader* reader) { + // parameter_id + RCHECK(reader->SkipLeb128()); + // parameter_rate + RCHECK(reader->SkipLeb128()); + uint8_t param_definition_mode; + RCHECK(reader->Read1(¶m_definition_mode)); + param_definition_mode = param_definition_mode >> 7; + if (param_definition_mode == static_cast(0)) { + // duration + RCHECK(reader->SkipLeb128()); + uint32_t constant_subblock_duration; + RCHECK(reader->ReadLeb128(&constant_subblock_duration)); + if (constant_subblock_duration == 0) { + uint32_t num_subblocks; + RCHECK(reader->ReadLeb128(&num_subblocks)); + for (int i = 0; i < num_subblocks; ++i) { + // subblock_duration + RCHECK(reader->SkipLeb128()); + } + } } - - info->data_size = reader.size() - info->config_obus_size; - info->config_obus.assign(reader.buf(), reader.buf() + info->config_obus_size); - info->data.assign(reader.buf() + info->config_obus_size, - reader.buf() + reader.size()); - return true; } -bool IamfBufferParser::ParseDescriptorOBU(BufferReader* reader, - IamfBufferInfo* info, - const bool prefer_binaural_audio, - const bool prefer_surround_audio) { - SB_DCHECK(reader); - uint8_t obu_type = 0; - uint32_t obu_size = 0; - if (!ParseOBUHeader(reader, &obu_type, &obu_size)) { - SB_DLOG(ERROR) << "Error parsing OBU header"; - return false; - } - - int next_obu_pos = reader->pos() + obu_size; - - switch (static_cast(obu_type)) { - case kObuTypeCodecConfig: - RCHECK(ParseCodecConfigOBU(reader, info)); - break; - case kObuTypeAudioElement: - RCHECK(ParseAudioElementOBU(reader, info, prefer_binaural_audio, - prefer_surround_audio)); - break; - case kObuTypeSequenceHeader: - break; - case kObuTypeMixPresentation: - RCHECK(ParseMixPresentationOBU(reader, info, prefer_binaural_audio, - prefer_surround_audio)); - break; - default: - // Once an OBU is read that is not a descriptor, descriptor parsing is - // assumed to be complete. - SB_DCHECK(info->is_valid()); - return true; - } - - // Skip to the next OBU. - const size_t remaining_size = next_obu_pos - reader->pos(); - RCHECK(reader->SkipBytes(remaining_size)); - info->config_obus_size = reader->pos(); - return true; +// Helper function to check if |info| contains valid information. +bool BufferInfoIsValid(IamfBufferInfo* info) { + return info->mix_presentation_id.has_value() && info->sample_rate > 0 && + info->num_samples > 0; } -bool IamfBufferParser::ParseOBUHeader(BufferReader* reader, - uint8_t* obu_type, - uint32_t* obu_size) const { +bool ParseOBUHeader(BufferReader* reader, + uint8_t* obu_type, + uint32_t* obu_size) { uint8_t header_flags; RCHECK(reader->Read1(&header_flags)); *obu_type = (header_flags >> 3) & 0x1f; @@ -311,8 +277,7 @@ bool IamfBufferParser::ParseOBUHeader(BufferReader* reader, return true; } -bool IamfBufferParser::ParseCodecConfigOBU(BufferReader* reader, - IamfBufferInfo* info) { +bool ParseCodecConfigOBU(BufferReader* reader, IamfBufferInfo* info) { RCHECK(reader->SkipLeb128()); uint32_t codec_id = 0; @@ -348,10 +313,13 @@ bool IamfBufferParser::ParseCodecConfigOBU(BufferReader* reader, return true; } -bool IamfBufferParser::ParseAudioElementOBU(BufferReader* reader, - IamfBufferInfo* info, - const bool prefer_binaural_audio, - const bool prefer_surround_audio) { +bool ParseAudioElementOBU( + BufferReader* reader, + IamfBufferInfo* info, + std::unordered_set* binaural_audio_element_ids, + std::unordered_set* surround_audio_element_ids, + const bool prefer_binaural_audio, + const bool prefer_surround_audio) { uint32_t audio_element_id; RCHECK(reader->ReadLeb128(&audio_element_id)); @@ -406,10 +374,10 @@ bool IamfBufferParser::ParseAudioElementOBU(BufferReader* reader, output_gain_is_present_flag = (loudspeaker_layout >> 3) & 0x01; loudspeaker_layout = loudspeaker_layout >> 4; if (loudspeaker_layout == IA_CHANNEL_LAYOUT_BINAURAL) { - binaural_audio_element_ids_.insert(audio_element_id); + binaural_audio_element_ids->insert(audio_element_id); } else if (loudspeaker_layout > IA_CHANNEL_LAYOUT_STEREO && loudspeaker_layout < IA_CHANNEL_LAYOUT_COUNT) { - surround_audio_element_ids_.insert(audio_element_id); + surround_audio_element_ids->insert(audio_element_id); } // substream_count and coupled_substream_count @@ -429,9 +397,11 @@ bool IamfBufferParser::ParseAudioElementOBU(BufferReader* reader, return true; } -bool IamfBufferParser::ParseMixPresentationOBU( +bool ParseMixPresentationOBU( BufferReader* reader, IamfBufferInfo* info, + std::unordered_set* binaural_audio_element_ids, + std::unordered_set* surround_audio_element_ids, const bool prefer_binaural_audio, const bool prefer_surround_audio) { uint32_t mix_presentation_id; @@ -450,6 +420,7 @@ bool IamfBufferParser::ParseMixPresentationOBU( } uint32_t num_sub_mixes; + BinauralMixSelection binaural_mix_selection = kBinauralMixSelectionNotFound; RCHECK(reader->ReadLeb128(&num_sub_mixes)); for (int i = 0; i < num_sub_mixes; ++i) { uint32_t num_audio_elements; @@ -464,15 +435,15 @@ bool IamfBufferParser::ParseMixPresentationOBU( // binaural loudspeaker layout. if (!info->mix_presentation_id.has_value() || (prefer_binaural_audio && - binaural_mix_selection_ > kBinauralMixSelectionLoudspeakerLayout)) { + binaural_mix_selection > kBinauralMixSelectionLoudspeakerLayout)) { if (prefer_binaural_audio && - binaural_audio_element_ids_.find(audio_element_id) != - binaural_audio_element_ids_.end()) { + binaural_audio_element_ids->find(audio_element_id) != + binaural_audio_element_ids->end()) { info->mix_presentation_id = mix_presentation_id; - binaural_mix_selection_ = kBinauralMixSelectionLoudspeakerLayout; + binaural_mix_selection = kBinauralMixSelectionLoudspeakerLayout; } else if (prefer_surround_audio && - surround_audio_element_ids_.find(audio_element_id) != - surround_audio_element_ids_.end()) { + surround_audio_element_ids->find(audio_element_id) != + surround_audio_element_ids->end()) { info->mix_presentation_id = mix_presentation_id; } } @@ -516,9 +487,9 @@ bool IamfBufferParser::ParseMixPresentationOBU( if (static_cast(layout_type) == IAMF_LAYOUT_TYPE_BINAURAL && prefer_binaural_audio && (!info->mix_presentation_id.has_value() || - binaural_mix_selection_ > kBinauralMixSelectionLoudnessLayout)) { + binaural_mix_selection > kBinauralMixSelectionLoudnessLayout)) { info->mix_presentation_id = mix_presentation_id; - binaural_mix_selection_ = kBinauralMixSelectionLoudnessLayout; + binaural_mix_selection = kBinauralMixSelectionLoudnessLayout; } // The following fields are for the LoudnessInfo @@ -558,28 +529,77 @@ bool IamfBufferParser::ParseMixPresentationOBU( return true; } -bool IamfBufferParser::SkipParamDefinition(BufferReader* reader) const { - // parameter_id - RCHECK(reader->SkipLeb128()); - // parameter_rate - RCHECK(reader->SkipLeb128()); - uint8_t param_definition_mode; - RCHECK(reader->Read1(¶m_definition_mode)); - param_definition_mode = param_definition_mode >> 7; - if (param_definition_mode == static_cast(0)) { - // duration - RCHECK(reader->SkipLeb128()); - uint32_t constant_subblock_duration; - RCHECK(reader->ReadLeb128(&constant_subblock_duration)); - if (constant_subblock_duration == 0) { - uint32_t num_subblocks; - RCHECK(reader->ReadLeb128(&num_subblocks)); - for (int i = 0; i < num_subblocks; ++i) { - // subblock_duration - RCHECK(reader->SkipLeb128()); - } - } +bool ParseDescriptorOBU(BufferReader* reader, + IamfBufferInfo* info, + const bool prefer_binaural_audio, + const bool prefer_surround_audio) { + SB_DCHECK(reader); + uint8_t obu_type = 0; + uint32_t obu_size = 0; + if (!ParseOBUHeader(reader, &obu_type, &obu_size)) { + SB_DLOG(ERROR) << "Error parsing OBU header"; + return false; } + + int next_obu_pos = reader->pos() + obu_size; + + std::unordered_set binaural_audio_element_ids; + std::unordered_set surround_audio_element_ids; + switch (static_cast(obu_type)) { + case kObuTypeCodecConfig: + RCHECK(ParseCodecConfigOBU(reader, info)); + break; + case kObuTypeAudioElement: + RCHECK(ParseAudioElementOBU(reader, info, &binaural_audio_element_ids, + &surround_audio_element_ids, + prefer_binaural_audio, + prefer_surround_audio)); + break; + case kObuTypeSequenceHeader: + break; + case kObuTypeMixPresentation: + RCHECK(ParseMixPresentationOBU(reader, info, &binaural_audio_element_ids, + &surround_audio_element_ids, + prefer_binaural_audio, + prefer_surround_audio)); + break; + default: + // Once an OBU is read that is not a descriptor, descriptor parsing is + // assumed to be complete. + SB_DCHECK(BufferInfoIsValid(info)); + return true; + } + + // Skip to the next OBU. + const size_t remaining_size = next_obu_pos - reader->pos(); + RCHECK(reader->SkipBytes(remaining_size)); + info->config_obus_size = reader->pos(); + return true; +} + +} // namespace + +bool ParseInputBuffer(const scoped_refptr& input_buffer, + IamfBufferInfo* info, + const bool prefer_binaural_audio, + const bool prefer_surround_audio) { + SB_DCHECK(info); + SB_DCHECK(input_buffer->data()); + SB_DCHECK(!(prefer_binaural_audio && prefer_surround_audio)); + + BufferReader reader(input_buffer->data(), input_buffer->size()); + + while (!BufferInfoIsValid(info) && reader.pos() < reader.size()) { + RCHECK(ParseDescriptorOBU(&reader, info, prefer_binaural_audio, + prefer_surround_audio)); + } + RCHECK(BufferInfoIsValid(info)); + + info->data_size = reader.size() - info->config_obus_size; + info->config_obus.assign(reader.buf(), reader.buf() + info->config_obus_size); + info->data.assign(reader.buf() + info->config_obus_size, + reader.buf() + reader.size()); + return true; } diff --git a/starboard/shared/libiamf/iamf_decoder_utils.h b/starboard/shared/libiamf/iamf_decoder_utils.h new file mode 100644 index 000000000000..d2a4577af5e3 --- /dev/null +++ b/starboard/shared/libiamf/iamf_decoder_utils.h @@ -0,0 +1,52 @@ +// Copyright 2024 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef STARBOARD_SHARED_LIBIAMF_IAMF_DECODER_UTILS_H_ +#define STARBOARD_SHARED_LIBIAMF_IAMF_DECODER_UTILS_H_ + +#include +#include + +#include "starboard/common/log.h" +#include "starboard/common/ref_counted.h" +#include "starboard/shared/internal_only.h" +#include "starboard/shared/starboard/player/input_buffer_internal.h" + +namespace starboard { +namespace shared { +namespace libiamf { + +using shared::starboard::player::InputBuffer; + +struct IamfBufferInfo { + uint32_t num_samples; + int sample_rate; + std::optional mix_presentation_id; + std::vector config_obus; + size_t config_obus_size; + std::vector data; + size_t data_size; + const scoped_refptr input_buffer; +}; + +bool ParseInputBuffer(const scoped_refptr& input_buffer, + IamfBufferInfo* info, + const bool prefer_binaural_audio, + const bool prefer_surround_audio); + +} // namespace libiamf +} // namespace shared +} // namespace starboard + +#endif // STARBOARD_SHARED_LIBIAMF_IAMF_DECODER_UTILS_H_ From 57c5c3a246201a4834a145c0cb835fdcae03b637 Mon Sep 17 00:00:00 2001 From: Austin Osagie Date: Tue, 8 Oct 2024 12:10:05 -0700 Subject: [PATCH 16/30] formatting --- media/base/starboard_utils.cc | 28 +++++----- .../shared/player_components_factory.h | 2 +- .../shared/libiamf/iamf_audio_decoder.cc | 53 +++++++++++-------- starboard/shared/libiamf/iamf_audio_decoder.h | 9 ++-- .../shared/libiamf/iamf_decoder_utils.cc | 4 +- 5 files changed, 51 insertions(+), 45 deletions(-) diff --git a/media/base/starboard_utils.cc b/media/base/starboard_utils.cc index c3b1367ac998..cdb71640b975 100644 --- a/media/base/starboard_utils.cc +++ b/media/base/starboard_utils.cc @@ -59,13 +59,12 @@ int GetBitsPerPixel(const std::string& mime_type) { int GetMaxChannelCount() { int channels = 2; - int index = 0; - SbMediaAudioConfiguration configuration; - while (SbMediaGetAudioConfiguration(index++, &configuration)) { - SB_LOG(INFO) << "Channels for index " << index-1 << " = " << configuration.number_of_channels; - if (configuration.number_of_channels > channels) - channels = configuration.number_of_channels; - } + int index = 0; + SbMediaAudioConfiguration configuration; + while (SbMediaGetAudioConfiguration(index++, &configuration)) { + if (configuration.number_of_channels > channels) + channels = configuration.number_of_channels; + } return channels; } @@ -105,7 +104,7 @@ SbMediaAudioCodec MediaAudioCodecToSbMediaAudioCodec(AudioCodec codec) { return kSbMediaAudioCodecPcm; #if SB_API_VERSION >= 15 case AudioCodec::kIAMF: - return kSbMediaAudioCodecIamf; + return kSbMediaAudioCodecIamf; #endif // SB_API_VERSION >= 15 default: // Cobalt only supports a subset of audio codecs defined by Chromium. @@ -156,21 +155,24 @@ SbMediaAudioStreamInfo MediaAudioConfigToSbMediaAudioStreamInfo( #if SB_API_VERSION < 15 audio_stream_info.format_tag = 0x00ff; -#endif // SB_API_VERSION < 15 +#endif // SB_API_VERSION < 15 + +audio_stream_info.number_of_channels = + ChannelLayoutToChannelCount(audio_decoder_config.channel_layout()); + +#if SB_API_VERSION >= 15 if (audio_stream_info.codec == kSbMediaAudioCodecIamf) { // IAMF mixes audio signals to the highest available speaker layout. audio_stream_info.number_of_channels = GetMaxChannelCount(); - } else { - audio_stream_info.number_of_channels = - ChannelLayoutToChannelCount(audio_decoder_config.channel_layout()); } +#endif // SB_API_VERSION >= 15 audio_stream_info.samples_per_second = audio_decoder_config.samples_per_second(); #if SB_API_VERSION < 15 audio_stream_info.average_bytes_per_second = 1; audio_stream_info.block_alignment = 4; -#endif // SB_API_VERSION < 15 +#endif // SB_API_VERSION < 15 audio_stream_info.bits_per_sample = audio_decoder_config.bits_per_channel(); const auto& extra_data = audio_stream_info.codec == kSbMediaAudioCodecAac diff --git a/starboard/android/shared/player_components_factory.h b/starboard/android/shared/player_components_factory.h index cba21570d486..d6060d8406e4 100644 --- a/starboard/android/shared/player_components_factory.h +++ b/starboard/android/shared/player_components_factory.h @@ -114,7 +114,7 @@ class AudioRendererSinkAndroid : public ::starboard::shared::starboard::player:: private: bool IsAudioSampleTypeSupported( SbMediaAudioSampleType audio_sample_type) const override { - if (tunnel_mode_audio_session_id_ != -1 || true) { + if (tunnel_mode_audio_session_id_ != -1) { // Currently the implementation only supports tunnel mode with int16 audio // samples. return audio_sample_type == kSbMediaAudioSampleTypeInt16Deprecated; diff --git a/starboard/shared/libiamf/iamf_audio_decoder.cc b/starboard/shared/libiamf/iamf_audio_decoder.cc index 1b8ec3bfc8ee..bff77ef722c1 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.cc +++ b/starboard/shared/libiamf/iamf_audio_decoder.cc @@ -109,8 +109,6 @@ bool IamfAudioDecoder::DecodeInternal( SB_DCHECK(!stream_ended_ || !pending_audio_buffers_.empty()); SB_DCHECK(is_valid()); - SB_LOG(INFO) << "Sample type is: " - << input_buffer->audio_sample_info().stream_info.bits_per_sample; if (input_buffer->size() == 0) { SB_LOG(ERROR) << "Empty input buffer written to IamfAudioDecoder"; ReportError("Empty input buffer written to IamfAudioDecoder"); @@ -120,11 +118,15 @@ bool IamfAudioDecoder::DecodeInternal( IamfBufferInfo info; if (!ParseInputBuffer(input_buffer, &info, kForceBinauralAudio, channels_ > 2)) { + SB_LOG(ERROR) << "Failed to parse IA Descriptors"; ReportError("Failed to parse IA Descriptors"); return false; } if (!decoder_is_configured_) { - if (!ConfigureDecoder(&info, input_buffer->timestamp())) { + std::string error_message; + if (!ConfigureDecoder(&info, input_buffer->timestamp(), &error_message)) { + SB_LOG(ERROR) << "Failed to configure IAMF decoder"; + ReportError("Failed to configure IAMF decoder. " + error_message); return false; } } @@ -138,6 +140,8 @@ bool IamfAudioDecoder::DecodeInternal( IAMF_decoder_decode(decoder_, info.data.data(), info.data_size, nullptr, reinterpret_cast(decoded_audio->data())); if (samples_decoded < 1) { + SB_LOG(ERROR) << "Failed to decode IAMF sample, error " + << ErrorCodeToString(samples_decoded); ReportError("Failed to decode IAMF sample, error " + ErrorCodeToString(samples_decoded)); return false; @@ -181,7 +185,8 @@ void IamfAudioDecoder::WriteEndOfStream() { } bool IamfAudioDecoder::ConfigureDecoder(IamfBufferInfo* info, - int64_t timestamp) { + int64_t timestamp, + std::string* error_message) { SB_DCHECK(is_valid()); SB_DCHECK(!decoder_is_configured_); @@ -189,15 +194,15 @@ bool IamfAudioDecoder::ConfigureDecoder(IamfBufferInfo* info, // for now. int error = IAMF_decoder_set_bit_depth(decoder_, 16); if (error != IAMF_OK) { - ReportError("IAMF_decoder_set_bit_depth() fails with error " + - ErrorCodeToString(error)); + *error_message = "IAMF_decoder_set_bit_depth() fails with error " + + ErrorCodeToString(error); return false; } error = IAMF_decoder_set_sampling_rate(decoder_, info->sample_rate); if (error != IAMF_OK) { - ReportError("IAMF_decoder_set_sampling_rate() fails with error " + - ErrorCodeToString(error)); + *error_message = "IAMF_decoder_set_sampling_rate() fails with error " + + ErrorCodeToString(error); return false; } @@ -205,9 +210,9 @@ bool IamfAudioDecoder::ConfigureDecoder(IamfBufferInfo* info, SB_LOG(INFO) << "Configuring IamfAudioDecoder for binaural output"; error = IAMF_decoder_output_layout_set_binaural(decoder_); if (error != IAMF_OK) { - ReportError( + *error_message = "IAMF_decoder_output_layout_set_binaural() fails with error " + - ErrorCodeToString(error)); + ErrorCodeToString(error); return false; } } else { @@ -224,9 +229,9 @@ bool IamfAudioDecoder::ConfigureDecoder(IamfBufferInfo* info, error = IAMF_decoder_output_layout_set_sound_system(decoder_, sound_system); if (error != IAMF_OK) { - ReportError( + *error_message = "IAMF_decoder_output_layout_set_sound_system() fails with error " + - ErrorCodeToString(error)); + ErrorCodeToString(error); return false; } } @@ -235,38 +240,40 @@ bool IamfAudioDecoder::ConfigureDecoder(IamfBufferInfo* info, // https://github.com/AOMediaCodec/libiamf/blob/v1.0.0-errata/code/test/tools/iamfplayer/player/iamfplayer.c#L450 error = IAMF_decoder_set_pts(decoder_, timestamp, 90000); if (error != IAMF_OK) { - ReportError("IAMF_decoder_set_pts() fails with error " + - ErrorCodeToString(error)); + *error_message = + "IAMF_decoder_set_pts() fails with error " + ErrorCodeToString(error); return false; } error = IAMF_decoder_set_mix_presentation_id( decoder_, info->mix_presentation_id.value()); if (error != IAMF_OK) { - ReportError("IAMF_decoder_set_mix_presentation_id() fails with error " + - ErrorCodeToString(error)); + *error_message = + "IAMF_decoder_set_mix_presentation_id() fails with error " + + ErrorCodeToString(error); return false; } error = IAMF_decoder_peak_limiter_enable(decoder_, 0); if (error != IAMF_OK) { - ReportError("IAMF_decoder_peak_limiter_enable() fails with error " + - ErrorCodeToString(error)); + *error_message = "IAMF_decoder_peak_limiter_enable() fails with error " + + ErrorCodeToString(error); return false; } error = IAMF_decoder_set_normalization_loudness(decoder_, .0f); if (error != IAMF_OK) { - ReportError("IAMF_decoder_set_normalization_loudness() fails with error " + - ErrorCodeToString(error)); + *error_message = + "IAMF_decoder_set_normalization_loudness() fails with error " + + ErrorCodeToString(error); return false; } error = IAMF_decoder_configure(decoder_, info->config_obus.data(), info->config_obus_size, nullptr); if (error != IAMF_OK) { - ReportError("IAMF_decoder_configure() fails with error " + - ErrorCodeToString(error)); + *error_message = + "IAMF_decoder_configure() fails with error " + ErrorCodeToString(error); return false; } @@ -278,7 +285,7 @@ bool IamfAudioDecoder::ConfigureDecoder(IamfBufferInfo* info, void IamfAudioDecoder::TeardownDecoder() { if (is_valid()) { IAMF_decoder_close(decoder_); - decoder_ = NULL; + decoder_ = nullptr; } } diff --git a/starboard/shared/libiamf/iamf_audio_decoder.h b/starboard/shared/libiamf/iamf_audio_decoder.h index b6b0a0c04dc2..8eb0d118cb7a 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.h +++ b/starboard/shared/libiamf/iamf_audio_decoder.h @@ -38,8 +38,6 @@ class IamfAudioDecoder private starboard::player::JobQueue::JobOwner { public: typedef starboard::media::AudioStreamInfo AudioStreamInfo; - // typedef shared::libiamf::IamfBufferInfo IamfBufferInfo; - // using shared::libiamf::IamfBufferInfo; explicit IamfAudioDecoder(const AudioStreamInfo& audio_stream_info); ~IamfAudioDecoder() override; @@ -55,10 +53,9 @@ class IamfAudioDecoder void Reset() override; private: - static constexpr int kMinimumBuffersToDecode = 2; - static constexpr int kDefaultSampleRate = 48000; - - bool ConfigureDecoder(IamfBufferInfo* info, int64_t timestamp); + bool ConfigureDecoder(IamfBufferInfo* info, + int64_t timestamp, + std::string* error_message); void TeardownDecoder(); bool DecodeInternal(const scoped_refptr& input_buffer); diff --git a/starboard/shared/libiamf/iamf_decoder_utils.cc b/starboard/shared/libiamf/iamf_decoder_utils.cc index ef9dc083da36..02aae7d7d918 100644 --- a/starboard/shared/libiamf/iamf_decoder_utils.cc +++ b/starboard/shared/libiamf/iamf_decoder_utils.cc @@ -564,8 +564,8 @@ bool ParseDescriptorOBU(BufferReader* reader, prefer_surround_audio)); break; default: - // Once an OBU is read that is not a descriptor, descriptor parsing is - // assumed to be complete. + // This executes if a non descriptor OBU is read. The buffer info must be + // valid by this point. SB_DCHECK(BufferInfoIsValid(info)); return true; } From c302b437d54a5e396fadf5563b4a816b63d1d7a1 Mon Sep 17 00:00:00 2001 From: Austin Osagie Date: Tue, 8 Oct 2024 12:24:01 -0700 Subject: [PATCH 17/30] further formatting --- media/base/starboard_utils.cc | 2 -- starboard/shared/libiamf/iamf_audio_decoder.cc | 3 +-- starboard/shared/libiamf/iamf_audio_decoder.h | 1 - starboard/shared/libiamf/iamf_decoder_utils.cc | 11 ++++++----- starboard/shared/libiamf/iamf_decoder_utils.h | 1 - 5 files changed, 7 insertions(+), 11 deletions(-) diff --git a/media/base/starboard_utils.cc b/media/base/starboard_utils.cc index cdb71640b975..ab02b34dc8dd 100644 --- a/media/base/starboard_utils.cc +++ b/media/base/starboard_utils.cc @@ -29,8 +29,6 @@ #include "starboard/configuration.h" #include "starboard/memory.h" -#include "starboard/common/log.h" - using base::Time; using base::TimeDelta; diff --git a/starboard/shared/libiamf/iamf_audio_decoder.cc b/starboard/shared/libiamf/iamf_audio_decoder.cc index bff77ef722c1..019bb905ee01 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.cc +++ b/starboard/shared/libiamf/iamf_audio_decoder.cc @@ -14,8 +14,6 @@ #include "starboard/shared/libiamf/iamf_audio_decoder.h" -#include - #include "starboard/common/string.h" #include "third_party/libiamf/source/code/include/IAMF_defines.h" @@ -58,6 +56,7 @@ IamfAudioDecoder::IamfAudioDecoder(const AudioStreamInfo& audio_stream_info) decoder_ = IAMF_decoder_open(); if (!decoder_) { SB_LOG(ERROR) << "Error creating libiamf decoder"; + ReportError("Error creating libiamf decoder"); } } diff --git a/starboard/shared/libiamf/iamf_audio_decoder.h b/starboard/shared/libiamf/iamf_audio_decoder.h index 8eb0d118cb7a..fff5bbf7275c 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.h +++ b/starboard/shared/libiamf/iamf_audio_decoder.h @@ -20,7 +20,6 @@ #include #include "starboard/common/ref_counted.h" -#include "starboard/media.h" #include "starboard/shared/internal_only.h" #include "starboard/shared/libiamf/iamf_decoder_utils.h" #include "starboard/shared/starboard/media/media_util.h" diff --git a/starboard/shared/libiamf/iamf_decoder_utils.cc b/starboard/shared/libiamf/iamf_decoder_utils.cc index 02aae7d7d918..0f43486fc464 100644 --- a/starboard/shared/libiamf/iamf_decoder_utils.cc +++ b/starboard/shared/libiamf/iamf_decoder_utils.cc @@ -19,6 +19,7 @@ #include #include +#include "starboard/common/log.h" #include "third_party/libiamf/source/code/include/IAMF_defines.h" namespace starboard { @@ -54,9 +55,9 @@ constexpr int kFourccIpcm = 0x6970636d; // The preferred methods of choosing a binaural mix presentation are listed // from high to low. enum BinauralMixSelection { + kBinauralMixSelectionNotFound, kBinauralMixSelectionLoudspeakerLayout, - kBinauralMixSelectionLoudnessLayout, - kBinauralMixSelectionNotFound + kBinauralMixSelectionLoudnessLayout }; class BufferReader { @@ -435,7 +436,7 @@ bool ParseMixPresentationOBU( // binaural loudspeaker layout. if (!info->mix_presentation_id.has_value() || (prefer_binaural_audio && - binaural_mix_selection > kBinauralMixSelectionLoudspeakerLayout)) { + binaural_mix_selection != kBinauralMixSelectionLoudspeakerLayout)) { if (prefer_binaural_audio && binaural_audio_element_ids->find(audio_element_id) != binaural_audio_element_ids->end()) { @@ -487,7 +488,7 @@ bool ParseMixPresentationOBU( if (static_cast(layout_type) == IAMF_LAYOUT_TYPE_BINAURAL && prefer_binaural_audio && (!info->mix_presentation_id.has_value() || - binaural_mix_selection > kBinauralMixSelectionLoudnessLayout)) { + binaural_mix_selection == kBinauralMixSelectionNotFound)) { info->mix_presentation_id = mix_presentation_id; binaural_mix_selection = kBinauralMixSelectionLoudnessLayout; } @@ -537,7 +538,7 @@ bool ParseDescriptorOBU(BufferReader* reader, uint8_t obu_type = 0; uint32_t obu_size = 0; if (!ParseOBUHeader(reader, &obu_type, &obu_size)) { - SB_DLOG(ERROR) << "Error parsing OBU header"; + SB_LOG(ERROR) << "Error parsing OBU header"; return false; } diff --git a/starboard/shared/libiamf/iamf_decoder_utils.h b/starboard/shared/libiamf/iamf_decoder_utils.h index d2a4577af5e3..252d6cbab357 100644 --- a/starboard/shared/libiamf/iamf_decoder_utils.h +++ b/starboard/shared/libiamf/iamf_decoder_utils.h @@ -18,7 +18,6 @@ #include #include -#include "starboard/common/log.h" #include "starboard/common/ref_counted.h" #include "starboard/shared/internal_only.h" #include "starboard/shared/starboard/player/input_buffer_internal.h" From c1b181d0ad008bf78757ffd50309588c5deea5a5 Mon Sep 17 00:00:00 2001 From: Austin Osagie Date: Tue, 8 Oct 2024 12:27:11 -0700 Subject: [PATCH 18/30] Even further formatting --- media/base/starboard_utils.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/media/base/starboard_utils.cc b/media/base/starboard_utils.cc index ab02b34dc8dd..853e186ee287 100644 --- a/media/base/starboard_utils.cc +++ b/media/base/starboard_utils.cc @@ -154,17 +154,14 @@ SbMediaAudioStreamInfo MediaAudioConfigToSbMediaAudioStreamInfo( #if SB_API_VERSION < 15 audio_stream_info.format_tag = 0x00ff; #endif // SB_API_VERSION < 15 - audio_stream_info.number_of_channels = ChannelLayoutToChannelCount(audio_decoder_config.channel_layout()); - #if SB_API_VERSION >= 15 if (audio_stream_info.codec == kSbMediaAudioCodecIamf) { // IAMF mixes audio signals to the highest available speaker layout. audio_stream_info.number_of_channels = GetMaxChannelCount(); } #endif // SB_API_VERSION >= 15 - audio_stream_info.samples_per_second = audio_decoder_config.samples_per_second(); #if SB_API_VERSION < 15 From 06e1d21848d1983584330d5625d7e2226bbce344 Mon Sep 17 00:00:00 2001 From: Austin Osagie Date: Mon, 14 Oct 2024 16:18:05 -0700 Subject: [PATCH 19/30] Remove unused variable --- starboard/shared/libiamf/iamf_audio_decoder.cc | 15 ++++----------- starboard/shared/libiamf/iamf_audio_decoder.h | 1 - 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/starboard/shared/libiamf/iamf_audio_decoder.cc b/starboard/shared/libiamf/iamf_audio_decoder.cc index 019bb905ee01..83231343be49 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.cc +++ b/starboard/shared/libiamf/iamf_audio_decoder.cc @@ -84,7 +84,6 @@ void IamfAudioDecoder::Decode(const InputBuffers& input_buffers, const ConsumedCB& consumed_cb) { SB_DCHECK(BelongsToCurrentThread()); SB_DCHECK(!input_buffers.empty()); - SB_DCHECK(pending_audio_buffers_.empty()); SB_DCHECK(output_cb_); if (stream_ended_) { @@ -105,7 +104,7 @@ bool IamfAudioDecoder::DecodeInternal( SB_DCHECK(BelongsToCurrentThread()); SB_DCHECK(input_buffer); SB_DCHECK(output_cb_); - SB_DCHECK(!stream_ended_ || !pending_audio_buffers_.empty()); + SB_DCHECK(!stream_ended_); SB_DCHECK(is_valid()); if (input_buffer->size() == 0) { @@ -173,9 +172,6 @@ void IamfAudioDecoder::WriteEndOfStream() { SB_DCHECK(output_cb_); stream_ended_ = true; - if (!pending_audio_buffers_.empty()) { - return; - } // Put EOS into the queue. decoded_audios_.push(new DecodedAudio); @@ -306,19 +302,16 @@ scoped_refptr IamfAudioDecoder::Read( void IamfAudioDecoder::Reset() { SB_DCHECK(BelongsToCurrentThread()); + SB_DCHECK(is_valid()); - if (is_valid()) { - TeardownDecoder(); - decoder_ = IAMF_decoder_open(); - } - + TeardownDecoder(); + decoder_ = IAMF_decoder_open(); decoder_is_configured_ = false; stream_ended_ = false; while (!decoded_audios_.empty()) { decoded_audios_.pop(); } - pending_audio_buffers_.clear(); consumed_cb_ = nullptr; CancelPendingJobs(); diff --git a/starboard/shared/libiamf/iamf_audio_decoder.h b/starboard/shared/libiamf/iamf_audio_decoder.h index fff5bbf7275c..a7918db95102 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.h +++ b/starboard/shared/libiamf/iamf_audio_decoder.h @@ -71,7 +71,6 @@ class IamfAudioDecoder std::queue> decoded_audios_; AudioStreamInfo audio_stream_info_; bool decoder_is_configured_ = false; - std::deque> pending_audio_buffers_; int samples_per_second_ = 0; const int channels_; }; From a9b012b1dd59541b87251051167fe43ecde22b94 Mon Sep 17 00:00:00 2001 From: Austin Osagie Date: Mon, 14 Oct 2024 16:50:53 -0700 Subject: [PATCH 20/30] Use IamfMimeUtil to check IAMF support --- .../linux/shared/media_is_audio_supported.cc | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/starboard/linux/shared/media_is_audio_supported.cc b/starboard/linux/shared/media_is_audio_supported.cc index 8541cd76c55b..3dd781c4630a 100644 --- a/starboard/linux/shared/media_is_audio_supported.cc +++ b/starboard/linux/shared/media_is_audio_supported.cc @@ -18,7 +18,12 @@ #include "starboard/configuration.h" #include "starboard/configuration_constants.h" #include "starboard/media.h" +#include "starboard/shared/starboard/media/iamf_util.h" +using ::starboard::shared::starboard::media::IamfMimeUtil; +using ::starboard::shared::starboard::media::kIamfProfileBase; +using ::starboard::shared::starboard::media::kIamfProfileSimple; +using ::starboard::shared::starboard::media::kIamfSubstreamCodecOpus; using ::starboard::shared::starboard::media::MimeType; bool SbMediaIsAudioSupported(SbMediaAudioCodec audio_codec, @@ -34,6 +39,25 @@ bool SbMediaIsAudioSupported(SbMediaAudioCodec audio_codec, #if SB_API_VERSION >= 15 && ENABLE_IAMF_DECODE if (audio_codec == kSbMediaAudioCodecIamf) { + if (!mime_type || !mime_type->is_valid()) { + return false; + } + const std::vector& codecs = mime_type->GetCodecs(); + if (codecs.size() != 1) { + return false; + } + + IamfMimeUtil mime_util(codecs[0]); + if (!mime_util.is_valid()) { + return false; + } + uint32_t profile = mime_util.primary_profile(); + // Support only IAMF streams with a base or simple profile and an Opus + // substream. + if (mime_util.substream_codec() != kIamfSubstreamCodecOpus || + (profile != kIamfProfileSimple && profile != kIamfProfileBase)) { + return false; + } return bitrate <= kSbMediaMaxAudioBitrateInBitsPerSecond; } #endif // SB_API_VERSION >= 15 From f98d3448cfb074a28020516e3fa9c626a854cf07 Mon Sep 17 00:00:00 2001 From: Austin Osagie Date: Tue, 15 Oct 2024 16:56:44 -0700 Subject: [PATCH 21/30] Various updates --- media/base/starboard_utils.cc | 5 +- .../linux/shared/media_is_audio_supported.cc | 32 ++-- .../shared/libiamf/iamf_audio_decoder.cc | 55 +++--- starboard/shared/libiamf/iamf_audio_decoder.h | 8 +- .../shared/libiamf/iamf_decoder_utils.cc | 172 ++++++++++-------- starboard/shared/libiamf/iamf_decoder_utils.h | 3 +- 6 files changed, 147 insertions(+), 128 deletions(-) diff --git a/media/base/starboard_utils.cc b/media/base/starboard_utils.cc index 853e186ee287..1fd23b3edfd3 100644 --- a/media/base/starboard_utils.cc +++ b/media/base/starboard_utils.cc @@ -60,8 +60,7 @@ int GetMaxChannelCount() { int index = 0; SbMediaAudioConfiguration configuration; while (SbMediaGetAudioConfiguration(index++, &configuration)) { - if (configuration.number_of_channels > channels) - channels = configuration.number_of_channels; + channels = std::max(configuration.number_of_channels, channels); } return channels; } @@ -154,7 +153,7 @@ SbMediaAudioStreamInfo MediaAudioConfigToSbMediaAudioStreamInfo( #if SB_API_VERSION < 15 audio_stream_info.format_tag = 0x00ff; #endif // SB_API_VERSION < 15 -audio_stream_info.number_of_channels = + audio_stream_info.number_of_channels = ChannelLayoutToChannelCount(audio_decoder_config.channel_layout()); #if SB_API_VERSION >= 15 if (audio_stream_info.codec == kSbMediaAudioCodecIamf) { diff --git a/starboard/linux/shared/media_is_audio_supported.cc b/starboard/linux/shared/media_is_audio_supported.cc index 3dd781c4630a..fe05b9008be9 100644 --- a/starboard/linux/shared/media_is_audio_supported.cc +++ b/starboard/linux/shared/media_is_audio_supported.cc @@ -43,22 +43,24 @@ bool SbMediaIsAudioSupported(SbMediaAudioCodec audio_codec, return false; } const std::vector& codecs = mime_type->GetCodecs(); - if (codecs.size() != 1) { - return false; - } - - IamfMimeUtil mime_util(codecs[0]); - if (!mime_util.is_valid()) { - return false; + bool stream_is_supported = false; + for (auto& codec : codecs) { + IamfMimeUtil mime_util(codec); + if (!mime_util.is_valid()) { + continue; + } + uint32_t profile = mime_util.primary_profile(); + // Support only IAMF streams with a base or simple profile and an Opus + // substream. + if (mime_util.substream_codec() != kIamfSubstreamCodecOpus || + (profile != kIamfProfileSimple && profile != kIamfProfileBase)) { + continue; + } + stream_is_supported = true; + break; } - uint32_t profile = mime_util.primary_profile(); - // Support only IAMF streams with a base or simple profile and an Opus - // substream. - if (mime_util.substream_codec() != kIamfSubstreamCodecOpus || - (profile != kIamfProfileSimple && profile != kIamfProfileBase)) { - return false; - } - return bitrate <= kSbMediaMaxAudioBitrateInBitsPerSecond; + return stream_is_supported && + bitrate <= kSbMediaMaxAudioBitrateInBitsPerSecond; } #endif // SB_API_VERSION >= 15 diff --git a/starboard/shared/libiamf/iamf_audio_decoder.cc b/starboard/shared/libiamf/iamf_audio_decoder.cc index 83231343be49..6a8526fc3ed0 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.cc +++ b/starboard/shared/libiamf/iamf_audio_decoder.cc @@ -14,6 +14,8 @@ #include "starboard/shared/libiamf/iamf_audio_decoder.h" +#include "starboard/audio_sink.h" +#include "starboard/common/log.h" #include "starboard/common/string.h" #include "third_party/libiamf/source/code/include/IAMF_defines.h" @@ -51,12 +53,12 @@ std::string ErrorCodeToString(int code) { } // namespace IamfAudioDecoder::IamfAudioDecoder(const AudioStreamInfo& audio_stream_info) - : audio_stream_info_(audio_stream_info), - channels_(audio_stream_info.number_of_channels) { + : audio_stream_info_(audio_stream_info) { + SB_DCHECK(audio_stream_info_.number_of_channels <= + SbAudioSinkGetMaxChannels()); decoder_ = IAMF_decoder_open(); if (!decoder_) { SB_LOG(ERROR) << "Error creating libiamf decoder"; - ReportError("Error creating libiamf decoder"); } } @@ -65,7 +67,7 @@ IamfAudioDecoder::~IamfAudioDecoder() { } bool IamfAudioDecoder::is_valid() const { - return decoder_ != NULL; + return decoder_ != nullptr; } void IamfAudioDecoder::Initialize(const OutputCB& output_cb, @@ -108,47 +110,47 @@ bool IamfAudioDecoder::DecodeInternal( SB_DCHECK(is_valid()); if (input_buffer->size() == 0) { - SB_LOG(ERROR) << "Empty input buffer written to IamfAudioDecoder"; ReportError("Empty input buffer written to IamfAudioDecoder"); return false; } IamfBufferInfo info; if (!ParseInputBuffer(input_buffer, &info, kForceBinauralAudio, - channels_ > 2)) { - SB_LOG(ERROR) << "Failed to parse IA Descriptors"; + audio_stream_info_.number_of_channels > 2)) { ReportError("Failed to parse IA Descriptors"); return false; } + SB_DCHECK(info.is_valid()); if (!decoder_is_configured_) { std::string error_message; if (!ConfigureDecoder(&info, input_buffer->timestamp(), &error_message)) { - SB_LOG(ERROR) << "Failed to configure IAMF decoder"; ReportError("Failed to configure IAMF decoder. " + error_message); return false; } + decoder_is_configured_ = true; } scoped_refptr decoded_audio = new DecodedAudio( - channels_, GetSampleType(), kSbMediaAudioFrameStorageTypeInterleaved, - input_buffer->timestamp(), - channels_ * info.num_samples * + audio_stream_info_.number_of_channels, GetSampleType(), + kSbMediaAudioFrameStorageTypeInterleaved, input_buffer->timestamp(), + audio_stream_info_.number_of_channels * info.num_samples * starboard::media::GetBytesPerSample(GetSampleType())); int samples_decoded = - IAMF_decoder_decode(decoder_, info.data.data(), info.data_size, nullptr, + IAMF_decoder_decode(decoder_, info.data.data(), info.data.size(), nullptr, reinterpret_cast(decoded_audio->data())); if (samples_decoded < 1) { - SB_LOG(ERROR) << "Failed to decode IAMF sample, error " - << ErrorCodeToString(samples_decoded); ReportError("Failed to decode IAMF sample, error " + ErrorCodeToString(samples_decoded)); return false; } - SB_DCHECK(samples_decoded <= info.num_samples) - << "Samples decoded (" << samples_decoded - << ") is greater than the number of samples indicated by the stream (" - << info.num_samples << ")"; + if (samples_decoded > info.num_samples) { + ReportError(::starboard::FormatString( + "Samples decoded (%i) is greater than the number of samples indicated " + "by the stream (%i)", + samples_decoded, info.num_samples)); + return false; + } if (samples_per_second_ == 0) { samples_per_second_ = info.sample_rate; @@ -162,7 +164,7 @@ bool IamfAudioDecoder::DecodeInternal( decoded_audios_.push(decoded_audio); - output_cb_(); + Schedule(output_cb_); return true; } @@ -179,14 +181,16 @@ void IamfAudioDecoder::WriteEndOfStream() { Schedule(output_cb_); } -bool IamfAudioDecoder::ConfigureDecoder(IamfBufferInfo* info, +bool IamfAudioDecoder::ConfigureDecoder(const IamfBufferInfo* info, int64_t timestamp, - std::string* error_message) { + std::string* error_message) const { + SB_DCHECK(BelongsToCurrentThread()); SB_DCHECK(is_valid()); + SB_DCHECK(info->is_valid()); SB_DCHECK(!decoder_is_configured_); // TODO: libiamf has an issue outputting 32 bit float samples, set to 16 bit - // for now. + // integer for now. int error = IAMF_decoder_set_bit_depth(decoder_, 16); if (error != IAMF_OK) { *error_message = "IAMF_decoder_set_bit_depth() fails with error " + @@ -212,13 +216,14 @@ bool IamfAudioDecoder::ConfigureDecoder(IamfBufferInfo* info, } } else { IAMF_SoundSystem sound_system = SOUND_SYSTEM_A; - if (channels_ == 6) { + if (audio_stream_info_.number_of_channels == 6) { SB_LOG(INFO) << "Configuring IamfAudioDecoder for 5.1 output"; sound_system = SOUND_SYSTEM_B; - } else if (channels_ == 8) { + } else if (audio_stream_info_.number_of_channels == 8) { SB_LOG(INFO) << "Configuring IamfAudioDecoder for 7.1 output"; sound_system = SOUND_SYSTEM_I; } else { + SB_DCHECK(audio_stream_info_.number_of_channels == 2); SB_LOG(INFO) << "Configuring IamfAudioDecoder for stereo output"; } @@ -272,8 +277,6 @@ bool IamfAudioDecoder::ConfigureDecoder(IamfBufferInfo* info, return false; } - decoder_is_configured_ = true; - return true; } diff --git a/starboard/shared/libiamf/iamf_audio_decoder.h b/starboard/shared/libiamf/iamf_audio_decoder.h index a7918db95102..3898f14ed3a3 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.h +++ b/starboard/shared/libiamf/iamf_audio_decoder.h @@ -52,9 +52,9 @@ class IamfAudioDecoder void Reset() override; private: - bool ConfigureDecoder(IamfBufferInfo* info, + bool ConfigureDecoder(const IamfBufferInfo* info, int64_t timestamp, - std::string* error_message); + std::string* error_message) const; void TeardownDecoder(); bool DecodeInternal(const scoped_refptr& input_buffer); @@ -62,6 +62,8 @@ class IamfAudioDecoder void ReportError(const std::string& message) const; + const AudioStreamInfo audio_stream_info_; + OutputCB output_cb_; ErrorCB error_cb_; ConsumedCB consumed_cb_; @@ -69,10 +71,8 @@ class IamfAudioDecoder IAMF_Decoder* decoder_ = nullptr; bool stream_ended_ = false; std::queue> decoded_audios_; - AudioStreamInfo audio_stream_info_; bool decoder_is_configured_ = false; int samples_per_second_ = 0; - const int channels_; }; } // namespace libiamf diff --git a/starboard/shared/libiamf/iamf_decoder_utils.cc b/starboard/shared/libiamf/iamf_decoder_utils.cc index 0f43486fc464..315ad7572f24 100644 --- a/starboard/shared/libiamf/iamf_decoder_utils.cc +++ b/starboard/shared/libiamf/iamf_decoder_utils.cc @@ -20,6 +20,7 @@ #include #include "starboard/common/log.h" +#include "starboard/common/string.h" #include "third_party/libiamf/source/code/include/IAMF_defines.h" namespace starboard { @@ -62,8 +63,7 @@ enum BinauralMixSelection { class BufferReader { public: - BufferReader(const uint8_t* buf, size_t size) - : buf_(buf), pos_(0), size_(size) { + BufferReader(const uint8_t* buf, size_t size) : buf_(buf), size_(size) { #if SB_IS_BIG_ENDIAN #error BufferReader assumes little-endianness. #endif // SB_IS_BIG_ENDIAN @@ -88,10 +88,12 @@ class BufferReader { } bool ReadLeb128(uint32_t* ptr) { - if (!HasBytes(sizeof(uint32_t)) || !ptr) { + if (RemainingSize() < 1 || !ptr) { return false; } - int bytes_read = ReadLeb128Internal(buf_ + pos_, ptr); + int bytes_read = ReadLeb128Internal( + buf_ + pos_, ptr, + std::min(static_cast(RemainingSize()), sizeof(uint32_t))); if (bytes_read < 0) { return false; } @@ -100,6 +102,9 @@ class BufferReader { } bool ReadString(std::string* str) { + if (RemainingSize() < 1 || !str) { + return false; + } int bytes_read = ReadStringInternal(buf_ + pos_, str); if (bytes_read < 0) { return false; @@ -132,6 +137,7 @@ class BufferReader { private: bool HasBytes(size_t size) const { return size + pos_ <= size_; } + int RemainingSize() const { return size_ - pos_; } inline uint32_t ByteSwap(uint32_t x) { #if defined(COMPILER_MSVC) return _byteswap_ulong(x); @@ -141,15 +147,18 @@ class BufferReader { } // Decodes an Leb128 value and stores it in |value|. Returns the number of - // bytes read, capped to sizeof(uint32_t). Returns the number of bytes read, - // or -1 on error. - int ReadLeb128Internal(const uint8_t* buf, uint32_t* value) { + // bytes read, capped to |max_bytes_to_read|. Returns the number of bytes + // read, or -1 on error. + int ReadLeb128Internal(const uint8_t* buf, + uint32_t* value, + const int max_bytes_to_read) { SB_DCHECK(buf); SB_DCHECK(value); + *value = 0; bool error = true; size_t i = 0; - for (; i < sizeof(uint32_t); ++i) { + for (; i < max_bytes_to_read; ++i) { uint8_t byte = buf[i]; *value |= ((byte & 0x7f) << (i * 7)); if (!(byte & 0x80)) { @@ -168,36 +177,24 @@ class BufferReader { // 128 bytes, or -1 on error. int ReadStringInternal(const uint8_t* buf, std::string* str) { SB_DCHECK(buf); - - int remaining_size = static_cast(size_) - pos_; - if (remaining_size <= 0) { - return -1; - } + SB_DCHECK(str); // The size of the string is capped to 128 bytes. - const int kMaxStringSize = 128; - int str_size = std::min(remaining_size, kMaxStringSize); + // https://aomediacodec.github.io/iamf/v1.0.0-errata.html#convention-data-types + const int kMaxIamfStringSize = 128; + int max_bytes_to_read = std::min(RemainingSize(), kMaxIamfStringSize); str->clear(); + str->resize(max_bytes_to_read); - size_t bytes_read = 0; - while (bytes_read < str_size && buf[bytes_read] != '\0') { - bytes_read++; - } - - if (bytes_read == str_size) { - if (buf[bytes_read - 1] != '\0') { - return -1; - } - } else { + int bytes_read = ::starboard::strlcpy( + str->data(), reinterpret_cast(buf), max_bytes_to_read); + if (bytes_read == kMaxIamfStringSize) { + // Ensure that the read string is null terminated. if (buf[bytes_read] != '\0') { - return -1; + return false; } } - - if (bytes_read > 0) { - str->resize(bytes_read); - std::memcpy(str->data(), reinterpret_cast(buf), bytes_read); - } + str->resize(bytes_read); // Account for null terminator byte. return ++bytes_read; @@ -205,7 +202,7 @@ class BufferReader { int pos_ = 0; const uint8_t* buf_; - const size_t size_ = 0; + const size_t size_; }; // Helper function to skip parsing ParamDefinitions found in the config OBUs @@ -235,12 +232,8 @@ bool SkipParamDefinition(BufferReader* reader) { return true; } -// Helper function to check if |info| contains valid information. -bool BufferInfoIsValid(IamfBufferInfo* info) { - return info->mix_presentation_id.has_value() && info->sample_rate > 0 && - info->num_samples > 0; -} - +// Parses an IAMF OBU header. +// https://aomediacodec.github.io/iamf/v1.0.0-errata.html#obu-header-syntax bool ParseOBUHeader(BufferReader* reader, uint8_t* obu_type, uint32_t* obu_size) { @@ -248,7 +241,6 @@ bool ParseOBUHeader(BufferReader* reader, RCHECK(reader->Read1(&header_flags)); *obu_type = (header_flags >> 3) & 0x1f; - const bool obu_redundant_copy = (header_flags >> 2) & 1; const bool obu_trimming_status_flag = (header_flags >> 1) & 1; const bool obu_extension_flag = header_flags & 1; @@ -278,6 +270,9 @@ bool ParseOBUHeader(BufferReader* reader, return true; } +// Parses an IAMF Config OBU for the substream sample rate. This only handles +// Opus and ipcm substreams, FLAC and AAC substreams are unsupported. +// https://aomediacodec.github.io/iamf/v1.0.0-errata.html#obu-codecconfig bool ParseCodecConfigOBU(BufferReader* reader, IamfBufferInfo* info) { RCHECK(reader->SkipLeb128()); @@ -289,10 +284,11 @@ bool ParseCodecConfigOBU(BufferReader* reader, IamfBufferInfo* info) { // audio_roll_distance RCHECK(reader->SkipBytes(2)); - const int kOpusSampleRate = 48000; switch (codec_id) { case kFourccOpus: - info->sample_rate = kOpusSampleRate; + // The Opus sample rate is always 48kHz. + // https://aomediacodec.github.io/iamf/v1.0.0-errata.html#opus-specific + info->sample_rate = 48000; break; case kFourccIpcm: { // sample_format_flags @@ -314,13 +310,17 @@ bool ParseCodecConfigOBU(BufferReader* reader, IamfBufferInfo* info) { return true; } -bool ParseAudioElementOBU( - BufferReader* reader, - IamfBufferInfo* info, - std::unordered_set* binaural_audio_element_ids, - std::unordered_set* surround_audio_element_ids, - const bool prefer_binaural_audio, - const bool prefer_surround_audio) { +// Parses an IAMF Audio Element OBU for audio element ids. When +// |prefer_binaural_audio| is true, |binaural_audio_element_id| the first audio +// element id containing a binaural loudspeaker layout. The same is done for +// |surround_audio_element_id| when |prefer_surround_audio| is true. +// https://aomediacodec.github.io/iamf/v1.0.0-errata.html#obu-audioelement +bool ParseAudioElementOBU(BufferReader* reader, + IamfBufferInfo* info, + std::optional* binaural_audio_element_id, + std::optional* surround_audio_element_id, + const bool prefer_binaural_audio, + const bool prefer_surround_audio) { uint32_t audio_element_id; RCHECK(reader->ReadLeb128(&audio_element_id)); @@ -347,12 +347,12 @@ bool ParseAudioElementOBU( RCHECK(reader->ReadLeb128(¶m_definition_type)); if (param_definition_type == IAMF_PARAMETER_TYPE_DEMIXING) { - SkipParamDefinition(reader); + RCHECK(SkipParamDefinition(reader)); // DemixingParamDefintion RCHECK(reader->SkipBytes(1)); } else if (param_definition_type == IAMF_PARAMETER_TYPE_RECON_GAIN) { - SkipParamDefinition(reader); - } else if (param_definition_type > 2) { + RCHECK(SkipParamDefinition(reader)); + } else if (param_definition_type > IAMF_PARAMETER_TYPE_RECON_GAIN) { uint32_t param_definition_size; RCHECK(reader->ReadLeb128(¶m_definition_size)); RCHECK(reader->SkipBytes(param_definition_size)); @@ -374,11 +374,13 @@ bool ParseAudioElementOBU( RCHECK(reader->Read1(&loudspeaker_layout)); output_gain_is_present_flag = (loudspeaker_layout >> 3) & 0x01; loudspeaker_layout = loudspeaker_layout >> 4; - if (loudspeaker_layout == IA_CHANNEL_LAYOUT_BINAURAL) { - binaural_audio_element_ids->insert(audio_element_id); + if (loudspeaker_layout == IA_CHANNEL_LAYOUT_BINAURAL && + !binaural_audio_element_id->has_value()) { + *binaural_audio_element_id = audio_element_id; } else if (loudspeaker_layout > IA_CHANNEL_LAYOUT_STEREO && - loudspeaker_layout < IA_CHANNEL_LAYOUT_COUNT) { - surround_audio_element_ids->insert(audio_element_id); + loudspeaker_layout < IA_CHANNEL_LAYOUT_COUNT && + !surround_audio_element_id->has_value()) { + *surround_audio_element_id = audio_element_id; } // substream_count and coupled_substream_count @@ -398,11 +400,22 @@ bool ParseAudioElementOBU( return true; } +// Parses a Mix Presentation OBU for a mix presentation to be used when +// configuring the IAMF decoder. +// When |prefer_surround_audio| is true, |info->mix_presentation_id| is +// set to |surround_audio_element_id|, if it has a value. +// When |prefer_binaural_audio| is true, |info->mix_presentation_id| is +// set to |binaural_audio_element_id|, if it has a value. If +// |binaural_audio_element_id| is unset, |info->mix_presentation_id| will be set +// to a mix presentation containing a loudness layout. +// |info->mix_presentation_id| is set to the first read mix presentation by +// default. +// https://aomediacodec.github.io/iamf/v1.0.0-errata.html#obu-mixpresentation bool ParseMixPresentationOBU( BufferReader* reader, IamfBufferInfo* info, - std::unordered_set* binaural_audio_element_ids, - std::unordered_set* surround_audio_element_ids, + const std::optional* binaural_audio_element_id, + const std::optional* surround_audio_element_id, const bool prefer_binaural_audio, const bool prefer_surround_audio) { uint32_t mix_presentation_id; @@ -431,20 +444,20 @@ bool ParseMixPresentationOBU( RCHECK(reader->ReadLeb128(&audio_element_id)); // Set a mix presentation for binaural or surround streams. The mix - // presentation is chosen if there exists an audio element that has + // presentation is chosen if an audio element exists that has // the qualities it requires - such as an audio element with a // binaural loudspeaker layout. if (!info->mix_presentation_id.has_value() || (prefer_binaural_audio && binaural_mix_selection != kBinauralMixSelectionLoudspeakerLayout)) { if (prefer_binaural_audio && - binaural_audio_element_ids->find(audio_element_id) != - binaural_audio_element_ids->end()) { + (binaural_audio_element_id->has_value() && + binaural_audio_element_id->value() == audio_element_id)) { info->mix_presentation_id = mix_presentation_id; binaural_mix_selection = kBinauralMixSelectionLoudspeakerLayout; } else if (prefer_surround_audio && - surround_audio_element_ids->find(audio_element_id) != - surround_audio_element_ids->end()) { + (surround_audio_element_id->has_value() && + surround_audio_element_id->value() == audio_element_id)) { info->mix_presentation_id = mix_presentation_id; } } @@ -463,13 +476,13 @@ bool ParseMixPresentationOBU( RCHECK(reader->SkipBytes(rendering_config_extension_size)); // The following fields are for the ElementMixConfig - SkipParamDefinition(reader); + RCHECK(SkipParamDefinition(reader)); // default_mix_gain RCHECK(reader->SkipBytes(2)); } // The following fields are for the OutputMixConfig - SkipParamDefinition(reader); + RCHECK(SkipParamDefinition(reader)); // default_mix_gain RCHECK(reader->SkipBytes(2)); @@ -544,31 +557,29 @@ bool ParseDescriptorOBU(BufferReader* reader, int next_obu_pos = reader->pos() + obu_size; - std::unordered_set binaural_audio_element_ids; - std::unordered_set surround_audio_element_ids; + std::optional binaural_audio_element_id; + std::optional surround_audio_element_id; switch (static_cast(obu_type)) { case kObuTypeCodecConfig: RCHECK(ParseCodecConfigOBU(reader, info)); break; case kObuTypeAudioElement: - RCHECK(ParseAudioElementOBU(reader, info, &binaural_audio_element_ids, - &surround_audio_element_ids, - prefer_binaural_audio, - prefer_surround_audio)); + RCHECK(ParseAudioElementOBU( + reader, info, &binaural_audio_element_id, &surround_audio_element_id, + prefer_binaural_audio, prefer_surround_audio)); break; case kObuTypeSequenceHeader: break; case kObuTypeMixPresentation: - RCHECK(ParseMixPresentationOBU(reader, info, &binaural_audio_element_ids, - &surround_audio_element_ids, - prefer_binaural_audio, - prefer_surround_audio)); + RCHECK(ParseMixPresentationOBU( + reader, info, &binaural_audio_element_id, &surround_audio_element_id, + prefer_binaural_audio, prefer_surround_audio)); break; default: // This executes if a non descriptor OBU is read. The buffer info must be // valid by this point. - SB_DCHECK(BufferInfoIsValid(info)); - return true; + SB_DCHECK(info->is_valid()); + return info->is_valid(); } // Skip to the next OBU. @@ -580,6 +591,10 @@ bool ParseDescriptorOBU(BufferReader* reader, } // namespace +bool IamfBufferInfo::is_valid() const { + return mix_presentation_id.has_value() && sample_rate > 0 && num_samples > 0; +} + bool ParseInputBuffer(const scoped_refptr& input_buffer, IamfBufferInfo* info, const bool prefer_binaural_audio, @@ -590,13 +605,12 @@ bool ParseInputBuffer(const scoped_refptr& input_buffer, BufferReader reader(input_buffer->data(), input_buffer->size()); - while (!BufferInfoIsValid(info) && reader.pos() < reader.size()) { + while (!info->is_valid() && reader.pos() < reader.size()) { RCHECK(ParseDescriptorOBU(&reader, info, prefer_binaural_audio, prefer_surround_audio)); } - RCHECK(BufferInfoIsValid(info)); + RCHECK(info->is_valid()); - info->data_size = reader.size() - info->config_obus_size; info->config_obus.assign(reader.buf(), reader.buf() + info->config_obus_size); info->data.assign(reader.buf() + info->config_obus_size, reader.buf() + reader.size()); diff --git a/starboard/shared/libiamf/iamf_decoder_utils.h b/starboard/shared/libiamf/iamf_decoder_utils.h index 252d6cbab357..4e3ef59f8e34 100644 --- a/starboard/shared/libiamf/iamf_decoder_utils.h +++ b/starboard/shared/libiamf/iamf_decoder_utils.h @@ -35,8 +35,9 @@ struct IamfBufferInfo { std::vector config_obus; size_t config_obus_size; std::vector data; - size_t data_size; const scoped_refptr input_buffer; + + bool is_valid() const; }; bool ParseInputBuffer(const scoped_refptr& input_buffer, From bbe198418005204328ea7bebf1ad7b7b440c0712 Mon Sep 17 00:00:00 2001 From: Austin Osagie Date: Wed, 16 Oct 2024 02:38:29 -0700 Subject: [PATCH 22/30] Check additional profile --- starboard/linux/shared/media_is_audio_supported.cc | 8 ++++++-- starboard/shared/libiamf/iamf_audio_decoder.cc | 7 +++---- starboard/shared/libiamf/iamf_audio_decoder.h | 3 +-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/starboard/linux/shared/media_is_audio_supported.cc b/starboard/linux/shared/media_is_audio_supported.cc index fe05b9008be9..70328a35b61f 100644 --- a/starboard/linux/shared/media_is_audio_supported.cc +++ b/starboard/linux/shared/media_is_audio_supported.cc @@ -49,11 +49,15 @@ bool SbMediaIsAudioSupported(SbMediaAudioCodec audio_codec, if (!mime_util.is_valid()) { continue; } - uint32_t profile = mime_util.primary_profile(); + uint32_t primary_profile = mime_util.primary_profile(); + uint32_t additional_profile = mime_util.additional_profile(); // Support only IAMF streams with a base or simple profile and an Opus // substream. if (mime_util.substream_codec() != kIamfSubstreamCodecOpus || - (profile != kIamfProfileSimple && profile != kIamfProfileBase)) { + (primary_profile != kIamfProfileSimple && + primary_profile != kIamfProfileBase) || + (additional_profile != kIamfProfileSimple && + additional_profile != kIamfProfileBase)) { continue; } stream_is_supported = true; diff --git a/starboard/shared/libiamf/iamf_audio_decoder.cc b/starboard/shared/libiamf/iamf_audio_decoder.cc index 6a8526fc3ed0..5078f18125d1 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.cc +++ b/starboard/shared/libiamf/iamf_audio_decoder.cc @@ -135,9 +135,9 @@ bool IamfAudioDecoder::DecodeInternal( kSbMediaAudioFrameStorageTypeInterleaved, input_buffer->timestamp(), audio_stream_info_.number_of_channels * info.num_samples * starboard::media::GetBytesPerSample(GetSampleType())); - int samples_decoded = - IAMF_decoder_decode(decoder_, info.data.data(), info.data.size(), nullptr, - reinterpret_cast(decoded_audio->data())); + int samples_decoded = IAMF_decoder_decode( + decoder_, const_cast(info.data.data()), info.data.size(), + nullptr, reinterpret_cast(decoded_audio->data())); if (samples_decoded < 1) { ReportError("Failed to decode IAMF sample, error " + ErrorCodeToString(samples_decoded)); @@ -315,7 +315,6 @@ void IamfAudioDecoder::Reset() { while (!decoded_audios_.empty()) { decoded_audios_.pop(); } - consumed_cb_ = nullptr; CancelPendingJobs(); } diff --git a/starboard/shared/libiamf/iamf_audio_decoder.h b/starboard/shared/libiamf/iamf_audio_decoder.h index 3898f14ed3a3..dfa731373185 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.h +++ b/starboard/shared/libiamf/iamf_audio_decoder.h @@ -66,9 +66,8 @@ class IamfAudioDecoder OutputCB output_cb_; ErrorCB error_cb_; - ConsumedCB consumed_cb_; - IAMF_Decoder* decoder_ = nullptr; + IAMF_DecoderHandle decoder_ = nullptr; bool stream_ended_ = false; std::queue> decoded_audios_; bool decoder_is_configured_ = false; From 6dac7cb05508d0d2f644fdcd872e029ad9a3238b Mon Sep 17 00:00:00 2001 From: Austin Osagie Date: Wed, 16 Oct 2024 03:10:46 -0700 Subject: [PATCH 23/30] Fix handling of profile rejection --- .../linux/shared/media_is_audio_supported.cc | 12 +++++---- .../shared/libiamf/iamf_audio_decoder.cc | 4 +-- .../shared/libiamf/iamf_decoder_utils.cc | 26 +++++++++---------- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/starboard/linux/shared/media_is_audio_supported.cc b/starboard/linux/shared/media_is_audio_supported.cc index 70328a35b61f..280f48b966cd 100644 --- a/starboard/linux/shared/media_is_audio_supported.cc +++ b/starboard/linux/shared/media_is_audio_supported.cc @@ -49,13 +49,15 @@ bool SbMediaIsAudioSupported(SbMediaAudioCodec audio_codec, if (!mime_util.is_valid()) { continue; } + // Support only IAMF streams with an Opus substream. + if (mime_util.substream_codec() != kIamfSubstreamCodecOpus) { + continue; + } + // At least one of the profiles must be Base or Simple. uint32_t primary_profile = mime_util.primary_profile(); uint32_t additional_profile = mime_util.additional_profile(); - // Support only IAMF streams with a base or simple profile and an Opus - // substream. - if (mime_util.substream_codec() != kIamfSubstreamCodecOpus || - (primary_profile != kIamfProfileSimple && - primary_profile != kIamfProfileBase) || + if ((primary_profile != kIamfProfileSimple && + primary_profile != kIamfProfileBase) && (additional_profile != kIamfProfileSimple && additional_profile != kIamfProfileBase)) { continue; diff --git a/starboard/shared/libiamf/iamf_audio_decoder.cc b/starboard/shared/libiamf/iamf_audio_decoder.cc index 5078f18125d1..77e4beb05ea2 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.cc +++ b/starboard/shared/libiamf/iamf_audio_decoder.cc @@ -312,9 +312,7 @@ void IamfAudioDecoder::Reset() { decoder_is_configured_ = false; stream_ended_ = false; - while (!decoded_audios_.empty()) { - decoded_audios_.pop(); - } + decoded_audios_ = std::queue>(); // clear CancelPendingJobs(); } diff --git a/starboard/shared/libiamf/iamf_decoder_utils.cc b/starboard/shared/libiamf/iamf_decoder_utils.cc index 315ad7572f24..c0a27122fb65 100644 --- a/starboard/shared/libiamf/iamf_decoder_utils.cc +++ b/starboard/shared/libiamf/iamf_decoder_utils.cc @@ -17,7 +17,6 @@ #include #include #include -#include #include "starboard/common/log.h" #include "starboard/common/string.h" @@ -88,7 +87,7 @@ class BufferReader { } bool ReadLeb128(uint32_t* ptr) { - if (RemainingSize() < 1 || !ptr) { + if (!HasBytes(1) || !ptr) { return false; } int bytes_read = ReadLeb128Internal( @@ -102,7 +101,7 @@ class BufferReader { } bool ReadString(std::string* str) { - if (RemainingSize() < 1 || !str) { + if (!HasBytes(1) || !str) { return false; } int bytes_read = ReadStringInternal(buf_ + pos_, str); @@ -138,7 +137,7 @@ class BufferReader { private: bool HasBytes(size_t size) const { return size + pos_ <= size_; } int RemainingSize() const { return size_ - pos_; } - inline uint32_t ByteSwap(uint32_t x) { + inline uint32_t ByteSwap(uint32_t x) const { #if defined(COMPILER_MSVC) return _byteswap_ulong(x); #else @@ -188,7 +187,7 @@ class BufferReader { int bytes_read = ::starboard::strlcpy( str->data(), reinterpret_cast(buf), max_bytes_to_read); - if (bytes_read == kMaxIamfStringSize) { + if (bytes_read == max_bytes_to_read) { // Ensure that the read string is null terminated. if (buf[bytes_read] != '\0') { return false; @@ -311,12 +310,12 @@ bool ParseCodecConfigOBU(BufferReader* reader, IamfBufferInfo* info) { } // Parses an IAMF Audio Element OBU for audio element ids. When -// |prefer_binaural_audio| is true, |binaural_audio_element_id| the first audio -// element id containing a binaural loudspeaker layout. The same is done for -// |surround_audio_element_id| when |prefer_surround_audio| is true. +// |prefer_binaural_audio| is true, |binaural_audio_element_id| is set to +// the first audioelement id containing a binaural loudspeaker layout. +// The same is done for |surround_audio_element_id| when +// |prefer_surround_audio| is true. // https://aomediacodec.github.io/iamf/v1.0.0-errata.html#obu-audioelement bool ParseAudioElementOBU(BufferReader* reader, - IamfBufferInfo* info, std::optional* binaural_audio_element_id, std::optional* surround_audio_element_id, const bool prefer_binaural_audio, @@ -407,9 +406,10 @@ bool ParseAudioElementOBU(BufferReader* reader, // When |prefer_binaural_audio| is true, |info->mix_presentation_id| is // set to |binaural_audio_element_id|, if it has a value. If // |binaural_audio_element_id| is unset, |info->mix_presentation_id| will be set -// to a mix presentation containing a loudness layout. -// |info->mix_presentation_id| is set to the first read mix presentation by -// default. +// to a mix presentation containing a binaural loudness layout. +// If a matching audio element wasn't found, or if a stereo layout is requested, +// |info->mix_presentation_id| is set to the first read mix presentation +// by default. // https://aomediacodec.github.io/iamf/v1.0.0-errata.html#obu-mixpresentation bool ParseMixPresentationOBU( BufferReader* reader, @@ -565,7 +565,7 @@ bool ParseDescriptorOBU(BufferReader* reader, break; case kObuTypeAudioElement: RCHECK(ParseAudioElementOBU( - reader, info, &binaural_audio_element_id, &surround_audio_element_id, + reader, &binaural_audio_element_id, &surround_audio_element_id, prefer_binaural_audio, prefer_surround_audio)); break; case kObuTypeSequenceHeader: From 8889907028f3bf8611fad282c06217f6c30d2ae5 Mon Sep 17 00:00:00 2001 From: Austin Osagie Date: Wed, 16 Oct 2024 03:50:15 -0700 Subject: [PATCH 24/30] Minor abstraction --- .../linux/shared/media_is_audio_supported.cc | 8 +- .../shared/libiamf/iamf_decoder_utils.cc | 102 ++++++++++-------- 2 files changed, 61 insertions(+), 49 deletions(-) diff --git a/starboard/linux/shared/media_is_audio_supported.cc b/starboard/linux/shared/media_is_audio_supported.cc index 280f48b966cd..b88b07b5cedf 100644 --- a/starboard/linux/shared/media_is_audio_supported.cc +++ b/starboard/linux/shared/media_is_audio_supported.cc @@ -56,10 +56,10 @@ bool SbMediaIsAudioSupported(SbMediaAudioCodec audio_codec, // At least one of the profiles must be Base or Simple. uint32_t primary_profile = mime_util.primary_profile(); uint32_t additional_profile = mime_util.additional_profile(); - if ((primary_profile != kIamfProfileSimple && - primary_profile != kIamfProfileBase) && - (additional_profile != kIamfProfileSimple && - additional_profile != kIamfProfileBase)) { + if (primary_profile != kIamfProfileSimple && + primary_profile != kIamfProfileBase && + additional_profile != kIamfProfileSimple && + additional_profile != kIamfProfileBase) { continue; } stream_is_supported = true; diff --git a/starboard/shared/libiamf/iamf_decoder_utils.cc b/starboard/shared/libiamf/iamf_decoder_utils.cc index c0a27122fb65..689f55f38ad1 100644 --- a/starboard/shared/libiamf/iamf_decoder_utils.cc +++ b/starboard/shared/libiamf/iamf_decoder_utils.cc @@ -188,14 +188,14 @@ class BufferReader { int bytes_read = ::starboard::strlcpy( str->data(), reinterpret_cast(buf), max_bytes_to_read); if (bytes_read == max_bytes_to_read) { - // Ensure that the read string is null terminated. + // Ensure that the string is null terminated. if (buf[bytes_read] != '\0') { return false; } } str->resize(bytes_read); - // Account for null terminator byte. + // Account for null terminator. return ++bytes_read; } @@ -296,9 +296,7 @@ bool ParseCodecConfigOBU(BufferReader* reader, IamfBufferInfo* info) { // sample_size RCHECK(reader->SkipBytes(1)); - uint32_t sample_rate_unsigned; - RCHECK(reader->Read4(&sample_rate_unsigned)); - info->sample_rate = static_cast(sample_rate_unsigned); + RCHECK(reader->Read4(reinterpret_cast(&info->sample_rate))); break; } default: @@ -309,6 +307,53 @@ bool ParseCodecConfigOBU(BufferReader* reader, IamfBufferInfo* info) { return true; } +// Checks if the Audio Element indicated by |audio_element_id| contains layouts +// for binaural and surround configurations, and sets +// |binaural_audio_element_id| and |surround_audio_element_id| if so. +bool CheckForAdvancedAudioElements( + BufferReader* reader, + uint32_t audio_element_id, + std::optional* binaural_audio_element_id, + std::optional* surround_audio_element_id) { + // Parse ScalableChannelLayoutConfig for binaural and surround + // loudspeaker layouts + uint8_t num_layers; + RCHECK(reader->Read1(&num_layers)); + num_layers = num_layers >> 5; + // Read ChannelAudioLayerConfigs + uint8_t max_loudspeaker_layout = 0; + for (int i = 0; i < static_cast(num_layers); ++i) { + uint8_t loudspeaker_layout; + bool output_gain_is_present_flag; + RCHECK(reader->Read1(&loudspeaker_layout)); + output_gain_is_present_flag = (loudspeaker_layout >> 3) & 0x01; + loudspeaker_layout = loudspeaker_layout >> 4; + if (loudspeaker_layout == IA_CHANNEL_LAYOUT_BINAURAL && + !binaural_audio_element_id->has_value()) { + *binaural_audio_element_id = audio_element_id; + } else if (loudspeaker_layout > IA_CHANNEL_LAYOUT_STEREO && + loudspeaker_layout < IA_CHANNEL_LAYOUT_COUNT && + loudspeaker_layout > max_loudspeaker_layout) { + *surround_audio_element_id = audio_element_id; + max_loudspeaker_layout = loudspeaker_layout; + } + + // substream_count and coupled_substream_count + RCHECK(reader->SkipBytes(2)); + + if (output_gain_is_present_flag) { + // output_gain_flags and output_gain + RCHECK(reader->SkipBytes(3)); + } + + if (i == 1 && loudspeaker_layout == static_cast(15)) { + // expanded_loudspeaker_layout + RCHECK(reader->SkipBytes(1)); + } + } + return true; +} + // Parses an IAMF Audio Element OBU for audio element ids. When // |prefer_binaural_audio| is true, |binaural_audio_element_id| is set to // the first audioelement id containing a binaural loudspeaker layout. @@ -361,40 +406,9 @@ bool ParseAudioElementOBU(BufferReader* reader, if (static_cast(audio_element_type) == AUDIO_ELEMENT_CHANNEL_BASED && (prefer_binaural_audio || prefer_surround_audio)) { - // Parse ScalableChannelLayoutConfig for binaural and surround - // loudspeaker layouts - uint8_t num_layers; - RCHECK(reader->Read1(&num_layers)); - num_layers = num_layers >> 5; - // Read ChannelAudioLayerConfigs - for (int i = 0; i < static_cast(num_layers); ++i) { - uint8_t loudspeaker_layout; - bool output_gain_is_present_flag; - RCHECK(reader->Read1(&loudspeaker_layout)); - output_gain_is_present_flag = (loudspeaker_layout >> 3) & 0x01; - loudspeaker_layout = loudspeaker_layout >> 4; - if (loudspeaker_layout == IA_CHANNEL_LAYOUT_BINAURAL && - !binaural_audio_element_id->has_value()) { - *binaural_audio_element_id = audio_element_id; - } else if (loudspeaker_layout > IA_CHANNEL_LAYOUT_STEREO && - loudspeaker_layout < IA_CHANNEL_LAYOUT_COUNT && - !surround_audio_element_id->has_value()) { - *surround_audio_element_id = audio_element_id; - } - - // substream_count and coupled_substream_count - RCHECK(reader->SkipBytes(2)); - - if (output_gain_is_present_flag) { - // output_gain_flags and output_gain - RCHECK(reader->SkipBytes(3)); - } - - if (i == 1 && loudspeaker_layout == static_cast(15)) { - // expanded_loudspeaker_layout - RCHECK(reader->SkipBytes(1)); - } - } + RCHECK(CheckForAdvancedAudioElements(reader, audio_element_id, + binaural_audio_element_id, + surround_audio_element_id)); } return true; } @@ -492,12 +506,10 @@ bool ParseMixPresentationOBU( uint8_t layout_type; RCHECK(reader->Read1(&layout_type)); layout_type = layout_type >> 6; - // If a binaural mix presentation is preferred and the mix - // presentation id has not yet been set, set the mix presentation id - // if the current mix presentation has a binaural loudness layout. The - // mix presentation id will change if a different mix presentation is - // found that uses an audio element with a binaural loudspeaker - // layout, as that is higher priority. + // Set the mix presentation ID for a binaural loudness layout. This mix + // presentation ID will be overridden if a different mix presentation has + // a binarual loudspeaker layout, as that is prioritized. + // https://aomediacodec.github.io/iamf/v1.0.0-errata.html#processing-mixpresentation-selection if (static_cast(layout_type) == IAMF_LAYOUT_TYPE_BINAURAL && prefer_binaural_audio && (!info->mix_presentation_id.has_value() || From 4e08474d49c3244c68210e25e9e80ff7dc25172c Mon Sep 17 00:00:00 2001 From: Austin Osagie Date: Wed, 16 Oct 2024 04:01:18 -0700 Subject: [PATCH 25/30] Add missing const --- starboard/shared/libiamf/iamf_decoder_utils.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/starboard/shared/libiamf/iamf_decoder_utils.cc b/starboard/shared/libiamf/iamf_decoder_utils.cc index 689f55f38ad1..5dcb8a5146f6 100644 --- a/starboard/shared/libiamf/iamf_decoder_utils.cc +++ b/starboard/shared/libiamf/iamf_decoder_utils.cc @@ -150,7 +150,7 @@ class BufferReader { // read, or -1 on error. int ReadLeb128Internal(const uint8_t* buf, uint32_t* value, - const int max_bytes_to_read) { + const int max_bytes_to_read) const { SB_DCHECK(buf); SB_DCHECK(value); @@ -174,7 +174,7 @@ class BufferReader { // Reads a c-string into |str|. Returns the number of bytes read, capped to // 128 bytes, or -1 on error. - int ReadStringInternal(const uint8_t* buf, std::string* str) { + int ReadStringInternal(const uint8_t* buf, std::string* str) const { SB_DCHECK(buf); SB_DCHECK(str); From e16809e93dedc259e9c64b767168aa5d9b7ca198 Mon Sep 17 00:00:00 2001 From: Austin Osagie Date: Wed, 16 Oct 2024 16:42:45 -0700 Subject: [PATCH 26/30] Some simplification --- .../linux/shared/media_is_audio_supported.cc | 32 +++++++------------ .../shared/libiamf/iamf_audio_decoder.cc | 8 +++-- .../shared/libiamf/iamf_decoder_utils.cc | 21 ++++++------ 3 files changed, 29 insertions(+), 32 deletions(-) diff --git a/starboard/linux/shared/media_is_audio_supported.cc b/starboard/linux/shared/media_is_audio_supported.cc index b88b07b5cedf..aca1d779004d 100644 --- a/starboard/linux/shared/media_is_audio_supported.cc +++ b/starboard/linux/shared/media_is_audio_supported.cc @@ -26,6 +26,11 @@ using ::starboard::shared::starboard::media::kIamfProfileSimple; using ::starboard::shared::starboard::media::kIamfSubstreamCodecOpus; using ::starboard::shared::starboard::media::MimeType; +bool HasSupportedIamfProfile(const IamfMimeUtil* mime_util) { + return mime_util->primary_profile() == kIamfProfileSimple || + mime_util->additional_profile() == kIamfProfileBase; +} + bool SbMediaIsAudioSupported(SbMediaAudioCodec audio_codec, const MimeType* mime_type, int64_t bitrate) { @@ -43,30 +48,17 @@ bool SbMediaIsAudioSupported(SbMediaAudioCodec audio_codec, return false; } const std::vector& codecs = mime_type->GetCodecs(); - bool stream_is_supported = false; for (auto& codec : codecs) { IamfMimeUtil mime_util(codec); - if (!mime_util.is_valid()) { - continue; - } - // Support only IAMF streams with an Opus substream. - if (mime_util.substream_codec() != kIamfSubstreamCodecOpus) { - continue; - } - // At least one of the profiles must be Base or Simple. - uint32_t primary_profile = mime_util.primary_profile(); - uint32_t additional_profile = mime_util.additional_profile(); - if (primary_profile != kIamfProfileSimple && - primary_profile != kIamfProfileBase && - additional_profile != kIamfProfileSimple && - additional_profile != kIamfProfileBase) { - continue; + // We support only IAMF Base or Simple profile streams with an Opus + // substream. + if (mime_util.is_valid() && + mime_util.substream_codec() == kIamfSubstreamCodecOpus && + HasSupportedIamfProfile(&mime_util)) { + return bitrate <= kSbMediaMaxAudioBitrateInBitsPerSecond; } - stream_is_supported = true; - break; } - return stream_is_supported && - bitrate <= kSbMediaMaxAudioBitrateInBitsPerSecond; + return false; } #endif // SB_API_VERSION >= 15 diff --git a/starboard/shared/libiamf/iamf_audio_decoder.cc b/starboard/shared/libiamf/iamf_audio_decoder.cc index 77e4beb05ea2..57e7652d4d68 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.cc +++ b/starboard/shared/libiamf/iamf_audio_decoder.cc @@ -135,9 +135,11 @@ bool IamfAudioDecoder::DecodeInternal( kSbMediaAudioFrameStorageTypeInterleaved, input_buffer->timestamp(), audio_stream_info_.number_of_channels * info.num_samples * starboard::media::GetBytesPerSample(GetSampleType())); - int samples_decoded = IAMF_decoder_decode( - decoder_, const_cast(info.data.data()), info.data.size(), - nullptr, reinterpret_cast(decoded_audio->data())); + + int samples_decoded = + IAMF_decoder_decode(decoder_, info.data.data(), info.data.size(), nullptr, + const_cast(decoded_audio->data())); + if (samples_decoded < 1) { ReportError("Failed to decode IAMF sample, error " + ErrorCodeToString(samples_decoded)); diff --git a/starboard/shared/libiamf/iamf_decoder_utils.cc b/starboard/shared/libiamf/iamf_decoder_utils.cc index 5dcb8a5146f6..cef03f23e806 100644 --- a/starboard/shared/libiamf/iamf_decoder_utils.cc +++ b/starboard/shared/libiamf/iamf_decoder_utils.cc @@ -104,7 +104,12 @@ class BufferReader { if (!HasBytes(1) || !str) { return false; } - int bytes_read = ReadStringInternal(buf_ + pos_, str); + + // The size of the string is capped to 128 bytes. + // https://aomediacodec.github.io/iamf/v1.0.0-errata.html#convention-data-types + const int kMaxIamfStringSize = 128; + int bytes_read = ReadStringInternal( + buf_ + pos_, str, std::min(RemainingSize(), kMaxIamfStringSize)); if (bytes_read < 0) { return false; } @@ -148,9 +153,9 @@ class BufferReader { // Decodes an Leb128 value and stores it in |value|. Returns the number of // bytes read, capped to |max_bytes_to_read|. Returns the number of bytes // read, or -1 on error. - int ReadLeb128Internal(const uint8_t* buf, - uint32_t* value, - const int max_bytes_to_read) const { + static int ReadLeb128Internal(const uint8_t* buf, + uint32_t* value, + const int max_bytes_to_read) { SB_DCHECK(buf); SB_DCHECK(value); @@ -174,14 +179,12 @@ class BufferReader { // Reads a c-string into |str|. Returns the number of bytes read, capped to // 128 bytes, or -1 on error. - int ReadStringInternal(const uint8_t* buf, std::string* str) const { + static int ReadStringInternal(const uint8_t* buf, + std::string* str, + int max_bytes_to_read) { SB_DCHECK(buf); SB_DCHECK(str); - // The size of the string is capped to 128 bytes. - // https://aomediacodec.github.io/iamf/v1.0.0-errata.html#convention-data-types - const int kMaxIamfStringSize = 128; - int max_bytes_to_read = std::min(RemainingSize(), kMaxIamfStringSize); str->clear(); str->resize(max_bytes_to_read); From 76a242c1f4855df5975f0269fd2a54889a0226f1 Mon Sep 17 00:00:00 2001 From: Austin Osagie Date: Fri, 18 Oct 2024 13:06:38 -0700 Subject: [PATCH 27/30] Check decoder validity --- media/base/starboard_utils.cc | 4 +++- starboard/linux/shared/media_is_audio_supported.cc | 2 ++ starboard/linux/shared/player_components_factory.cc | 11 +++++++---- starboard/shared/libiamf/iamf_audio_decoder.cc | 9 ++------- starboard/shared/libiamf/iamf_decoder_utils.cc | 6 ++++-- 5 files changed, 18 insertions(+), 14 deletions(-) diff --git a/media/base/starboard_utils.cc b/media/base/starboard_utils.cc index 1fd23b3edfd3..f25093f1cd6c 100644 --- a/media/base/starboard_utils.cc +++ b/media/base/starboard_utils.cc @@ -25,6 +25,7 @@ #include "base/strings/string_util.h" #include "media/base/decoder_buffer.h" #include "media/base/decrypt_config.h" +#include "starboard/audio_sink.h" #include "starboard/common/media.h" #include "starboard/configuration.h" #include "starboard/memory.h" @@ -62,7 +63,7 @@ int GetMaxChannelCount() { while (SbMediaGetAudioConfiguration(index++, &configuration)) { channels = std::max(configuration.number_of_channels, channels); } - return channels; + return std::min(channels, SbAudioSinkGetMaxChannels()); } } // namespace @@ -158,6 +159,7 @@ SbMediaAudioStreamInfo MediaAudioConfigToSbMediaAudioStreamInfo( #if SB_API_VERSION >= 15 if (audio_stream_info.codec == kSbMediaAudioCodecIamf) { // IAMF mixes audio signals to the highest available speaker layout. + // TODO: Handle this logic below Starboard. audio_stream_info.number_of_channels = GetMaxChannelCount(); } #endif // SB_API_VERSION >= 15 diff --git a/starboard/linux/shared/media_is_audio_supported.cc b/starboard/linux/shared/media_is_audio_supported.cc index aca1d779004d..16c72b022bf4 100644 --- a/starboard/linux/shared/media_is_audio_supported.cc +++ b/starboard/linux/shared/media_is_audio_supported.cc @@ -28,6 +28,8 @@ using ::starboard::shared::starboard::media::MimeType; bool HasSupportedIamfProfile(const IamfMimeUtil* mime_util) { return mime_util->primary_profile() == kIamfProfileSimple || + mime_util->primary_profile() == kIamfProfileBase || + mime_util->additional_profile() == kIamfProfileSimple || mime_util->additional_profile() == kIamfProfileBase; } diff --git a/starboard/linux/shared/player_components_factory.cc b/starboard/linux/shared/player_components_factory.cc index 3b9d0b46087e..44fce6c72d5d 100644 --- a/starboard/linux/shared/player_components_factory.cc +++ b/starboard/linux/shared/player_components_factory.cc @@ -92,10 +92,13 @@ class PlayerComponentsFactory : public PlayerComponents::Factory { return std::unique_ptr(new FdkAacAudioDecoder()); #if SB_API_VERSION >= 15 && ENABLE_IAMF_DECODE } else if (audio_stream_info.codec == kSbMediaAudioCodecIamf) { - SB_LOG(INFO) << "Playing audio using IamfAudioDecoder."; - return std::unique_ptr( - new ::starboard::shared::libiamf::IamfAudioDecoder( - audio_stream_info)); + using ::starboard::shared::libiamf::IamfAudioDecoder; + std::unique_ptr audio_decoder_impl( + new IamfAudioDecoder(audio_stream_info)); + if (audio_decoder_impl->is_valid()) { + SB_LOG(INFO) << "Playing audio using IamfAudioDecoder."; + return std::unique_ptr(std::move(audio_decoder_impl)); + } #endif // SB_API_VERSION >= 15 && ENABLE_IAMF_DECODE } else { std::unique_ptr audio_decoder_impl( diff --git a/starboard/shared/libiamf/iamf_audio_decoder.cc b/starboard/shared/libiamf/iamf_audio_decoder.cc index 57e7652d4d68..4231ba98a821 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.cc +++ b/starboard/shared/libiamf/iamf_audio_decoder.cc @@ -87,6 +87,7 @@ void IamfAudioDecoder::Decode(const InputBuffers& input_buffers, SB_DCHECK(BelongsToCurrentThread()); SB_DCHECK(!input_buffers.empty()); SB_DCHECK(output_cb_); + SB_DCHECK(consumed_cb); if (stream_ended_) { SB_LOG(ERROR) << "Decode() is called after WriteEndOfStream() is called."; @@ -121,6 +122,7 @@ bool IamfAudioDecoder::DecodeInternal( return false; } SB_DCHECK(info.is_valid()); + if (!decoder_is_configured_) { std::string error_message; if (!ConfigureDecoder(&info, input_buffer->timestamp(), &error_message)) { @@ -159,11 +161,6 @@ bool IamfAudioDecoder::DecodeInternal( } // TODO: Enable partial audio once float32 pcm output is available. - const auto& sample_info = input_buffer->audio_sample_info(); - decoded_audio->AdjustForDiscardedDurations( - samples_per_second_, sample_info.discarded_duration_from_front, - sample_info.discarded_duration_from_back); - decoded_audios_.push(decoded_audio); Schedule(output_cb_); @@ -315,8 +312,6 @@ void IamfAudioDecoder::Reset() { stream_ended_ = false; decoded_audios_ = std::queue>(); // clear - - CancelPendingJobs(); } SbMediaAudioSampleType IamfAudioDecoder::GetSampleType() const { diff --git a/starboard/shared/libiamf/iamf_decoder_utils.cc b/starboard/shared/libiamf/iamf_decoder_utils.cc index cef03f23e806..80fdce81e9e2 100644 --- a/starboard/shared/libiamf/iamf_decoder_utils.cc +++ b/starboard/shared/libiamf/iamf_decoder_utils.cc @@ -193,7 +193,7 @@ class BufferReader { if (bytes_read == max_bytes_to_read) { // Ensure that the string is null terminated. if (buf[bytes_read] != '\0') { - return false; + return -1; } } str->resize(bytes_read); @@ -202,9 +202,9 @@ class BufferReader { return ++bytes_read; } - int pos_ = 0; const uint8_t* buf_; const size_t size_; + int pos_ = 0; }; // Helper function to skip parsing ParamDefinitions found in the config OBUs @@ -610,6 +610,8 @@ bool IamfBufferInfo::is_valid() const { return mix_presentation_id.has_value() && sample_rate > 0 && num_samples > 0; } +// TODO: Implement a way to skip parsing the Config OBUs entirely once the +// decoder is configured. bool ParseInputBuffer(const scoped_refptr& input_buffer, IamfBufferInfo* info, const bool prefer_binaural_audio, From b47672e05509df261d956213ad09f90d2284dd2c Mon Sep 17 00:00:00 2001 From: Austin Osagie Date: Fri, 18 Oct 2024 13:39:43 -0700 Subject: [PATCH 28/30] update comment --- media/base/starboard_utils.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/media/base/starboard_utils.cc b/media/base/starboard_utils.cc index f25093f1cd6c..f8172b95edfb 100644 --- a/media/base/starboard_utils.cc +++ b/media/base/starboard_utils.cc @@ -159,7 +159,7 @@ SbMediaAudioStreamInfo MediaAudioConfigToSbMediaAudioStreamInfo( #if SB_API_VERSION >= 15 if (audio_stream_info.codec == kSbMediaAudioCodecIamf) { // IAMF mixes audio signals to the highest available speaker layout. - // TODO: Handle this logic below Starboard. + // TODO: Set the number of channels below Starboard. audio_stream_info.number_of_channels = GetMaxChannelCount(); } #endif // SB_API_VERSION >= 15 From 609328b7072cf83f49970ab3798b4a2121517a32 Mon Sep 17 00:00:00 2001 From: Austin Osagie Date: Fri, 18 Oct 2024 13:45:36 -0700 Subject: [PATCH 29/30] update comments --- .../linux/shared/player_components_factory.cc | 4 +- .../shared/libiamf/iamf_audio_decoder.cc | 2 +- starboard/shared/libiamf/iamf_audio_decoder.h | 2 +- .../shared/libiamf/iamf_decoder_utils.cc | 46 +++++++++---------- starboard/shared/libiamf/iamf_decoder_utils.h | 2 + 5 files changed, 28 insertions(+), 28 deletions(-) diff --git a/starboard/linux/shared/player_components_factory.cc b/starboard/linux/shared/player_components_factory.cc index 44fce6c72d5d..0e64ca9e8854 100644 --- a/starboard/linux/shared/player_components_factory.cc +++ b/starboard/linux/shared/player_components_factory.cc @@ -41,9 +41,9 @@ #include "starboard/shared/starboard/player/filter/video_render_algorithm_impl.h" #include "starboard/shared/starboard/player/filter/video_renderer_sink.h" -#if ENABLE_IAMF_DECODE +#if SB_API_VERSION >= 15 && ENABLE_IAMF_DECODE #include "starboard/shared/libiamf/iamf_audio_decoder.h" -#endif // ENABLE_IAMF_DECODE +#endif // SB_API_VERSION >= 15 && ENABLE_IAMF_DECODE namespace starboard { namespace shared { diff --git a/starboard/shared/libiamf/iamf_audio_decoder.cc b/starboard/shared/libiamf/iamf_audio_decoder.cc index 4231ba98a821..4582a3c94608 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.cc +++ b/starboard/shared/libiamf/iamf_audio_decoder.cc @@ -235,7 +235,7 @@ bool IamfAudioDecoder::ConfigureDecoder(const IamfBufferInfo* info, } } - // Time base is set to 90000, as it is in the iamfplayer example + // Time base is set to 90000, as it is in the iamfplayer example. // https://github.com/AOMediaCodec/libiamf/blob/v1.0.0-errata/code/test/tools/iamfplayer/player/iamfplayer.c#L450 error = IAMF_decoder_set_pts(decoder_, timestamp, 90000); if (error != IAMF_OK) { diff --git a/starboard/shared/libiamf/iamf_audio_decoder.h b/starboard/shared/libiamf/iamf_audio_decoder.h index dfa731373185..75deb4993a7d 100644 --- a/starboard/shared/libiamf/iamf_audio_decoder.h +++ b/starboard/shared/libiamf/iamf_audio_decoder.h @@ -43,7 +43,7 @@ class IamfAudioDecoder bool is_valid() const; - // AudioDecoder functions + // AudioDecoder functions. void Initialize(const OutputCB& output_cb, const ErrorCB& error_cb) override; void Decode(const InputBuffers& input_buffers, const ConsumedCB& consumed_cb) override; diff --git a/starboard/shared/libiamf/iamf_decoder_utils.cc b/starboard/shared/libiamf/iamf_decoder_utils.cc index 80fdce81e9e2..3093aa8a15c2 100644 --- a/starboard/shared/libiamf/iamf_decoder_utils.cc +++ b/starboard/shared/libiamf/iamf_decoder_utils.cc @@ -319,11 +319,11 @@ bool CheckForAdvancedAudioElements( std::optional* binaural_audio_element_id, std::optional* surround_audio_element_id) { // Parse ScalableChannelLayoutConfig for binaural and surround - // loudspeaker layouts + // loudspeaker layouts. uint8_t num_layers; RCHECK(reader->Read1(&num_layers)); num_layers = num_layers >> 5; - // Read ChannelAudioLayerConfigs + // Read ChannelAudioLayerConfigs. uint8_t max_loudspeaker_layout = 0; for (int i = 0; i < static_cast(num_layers); ++i) { uint8_t loudspeaker_layout; @@ -341,16 +341,16 @@ bool CheckForAdvancedAudioElements( max_loudspeaker_layout = loudspeaker_layout; } - // substream_count and coupled_substream_count + // substream_count and coupled_substream_count. RCHECK(reader->SkipBytes(2)); if (output_gain_is_present_flag) { - // output_gain_flags and output_gain + // output_gain_flags and output_gain. RCHECK(reader->SkipBytes(3)); } if (i == 1 && loudspeaker_layout == static_cast(15)) { - // expanded_loudspeaker_layout + // expanded_loudspeaker_layout. RCHECK(reader->SkipBytes(1)); } } @@ -375,14 +375,14 @@ bool ParseAudioElementOBU(BufferReader* reader, RCHECK(reader->Read1(&audio_element_type)); audio_element_type = audio_element_type >> 5; - // codec_config_id + // codec_config_id. RCHECK(reader->SkipLeb128()); uint32_t num_substreams; RCHECK(reader->ReadLeb128(&num_substreams)); for (int i = 0; i < num_substreams; ++i) { - // audio_substream_id + // audio_substream_id. RCHECK(reader->SkipLeb128()); } @@ -395,7 +395,7 @@ bool ParseAudioElementOBU(BufferReader* reader, if (param_definition_type == IAMF_PARAMETER_TYPE_DEMIXING) { RCHECK(SkipParamDefinition(reader)); - // DemixingParamDefintion + // DemixingParamDefintion. RCHECK(reader->SkipBytes(1)); } else if (param_definition_type == IAMF_PARAMETER_TYPE_RECON_GAIN) { RCHECK(SkipParamDefinition(reader)); @@ -441,12 +441,12 @@ bool ParseMixPresentationOBU( uint32_t count_label; RCHECK(reader->ReadLeb128(&count_label)); for (int i = 0; i < count_label; ++i) { - // language_label; + // language_label. RCHECK(reader->SkipString()); } for (int i = 0; i < count_label; ++i) { - // MixPresentationAnnotations; + // MixPresentationAnnotations. RCHECK(reader->SkipString()); } @@ -480,27 +480,27 @@ bool ParseMixPresentationOBU( } for (int k = 0; k < count_label; ++k) { - // MixPresentationElementAnnotatoions + // MixPresentationElementAnnotatoions. RCHECK(reader->SkipString()); } // The following fields are for the RenderingConfig - // headphones_rendering_mode + // headphones_rendering_mode. RCHECK(reader->SkipBytes(1)); uint32_t rendering_config_extension_size; RCHECK(reader->ReadLeb128(&rendering_config_extension_size)); - // rendering_config_extension_bytes + // rendering_config_extension_bytes. RCHECK(reader->SkipBytes(rendering_config_extension_size)); - // The following fields are for the ElementMixConfig + // The following fields are for the ElementMixConfig. RCHECK(SkipParamDefinition(reader)); - // default_mix_gain + // default_mix_gain. RCHECK(reader->SkipBytes(2)); } - // The following fields are for the OutputMixConfig + // The following fields are for the OutputMixConfig. RCHECK(SkipParamDefinition(reader)); - // default_mix_gain + // default_mix_gain. RCHECK(reader->SkipBytes(2)); uint32_t num_layouts; @@ -521,27 +521,27 @@ bool ParseMixPresentationOBU( binaural_mix_selection = kBinauralMixSelectionLoudnessLayout; } - // The following fields are for the LoudnessInfo + // The following fields are for the LoudnessInfo. uint8_t info_type; RCHECK(reader->Read1(&info_type)); - // integrated_loudness and digital_loudness + // integrated_loudness and digital_loudness. RCHECK(reader->SkipBytes(4)); if (info_type & 1) { - // true_peak + // true_peak. RCHECK(reader->SkipBytes(2)); } if (info_type & 2) { uint8_t num_anchored_loudness; RCHECK(reader->Read1(&num_anchored_loudness)); for (uint8_t k = 0; k < num_anchored_loudness; ++k) { - // anchor_element and anchored_loudness + // anchor_element and anchored_loudness. RCHECK(reader->SkipBytes(3)); } } if ((info_type & 0b11111100) > 0) { uint32_t info_type_size; RCHECK(reader->ReadLeb128(&info_type_size)); - // info_type_bytes + // info_type_bytes. RCHECK(reader->SkipBytes(info_type_size)); } } @@ -610,8 +610,6 @@ bool IamfBufferInfo::is_valid() const { return mix_presentation_id.has_value() && sample_rate > 0 && num_samples > 0; } -// TODO: Implement a way to skip parsing the Config OBUs entirely once the -// decoder is configured. bool ParseInputBuffer(const scoped_refptr& input_buffer, IamfBufferInfo* info, const bool prefer_binaural_audio, diff --git a/starboard/shared/libiamf/iamf_decoder_utils.h b/starboard/shared/libiamf/iamf_decoder_utils.h index 4e3ef59f8e34..8f2e5d802a47 100644 --- a/starboard/shared/libiamf/iamf_decoder_utils.h +++ b/starboard/shared/libiamf/iamf_decoder_utils.h @@ -40,6 +40,8 @@ struct IamfBufferInfo { bool is_valid() const; }; +// TODO: Implement a way to skip parsing the Config OBUs once the IAMF +// decoder is configured. bool ParseInputBuffer(const scoped_refptr& input_buffer, IamfBufferInfo* info, const bool prefer_binaural_audio, From bf43e00f4bfda8f0ac776b460556d09fd646e694 Mon Sep 17 00:00:00 2001 From: Austin Osagie Date: Thu, 24 Oct 2024 13:06:08 -0700 Subject: [PATCH 30/30] Filter tests --- starboard/linux/shared/test_filters.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/starboard/linux/shared/test_filters.py b/starboard/linux/shared/test_filters.py index 83fa5dddcda1..15e6c25df472 100644 --- a/starboard/linux/shared/test_filters.py +++ b/starboard/linux/shared/test_filters.py @@ -39,7 +39,16 @@ _FILTERED_TESTS = { 'nplb': [ # TODO(b/286249595): This test crashes when coverage is enabled. - 'SbMemoryMapTest.CanChangeMemoryProtection' + 'SbMemoryMapTest.CanChangeMemoryProtection', + # The IamfAudioDecoder doesn't support partial audio. + 'SbPlayerWriteSampleTests/SbPlayerWriteSampleTest.DiscardAllAudio/audio_iamf_*', + 'SbPlayerWriteSampleTests/SbPlayerWriteSampleTest.PartialAudio/audio_iamf_*', + ], + 'player_filter_tests': [ + # TODO(b/375238561): Investigate test failures. + 'AdaptiveAudioDecoderTests/AdaptiveAudioDecoderTest.SingleInput/*iamf*', + 'AdaptiveAudioDecoderTests/AdaptiveAudioDecoderTest.MultipleInput*iamf*', + 'AudioDecoderTests/AudioDecoderTest.SingleInvalidInput/*iamf*', ], } if os.getenv('MODULAR_BUILD', '0') == '1':