Skip to content

Commit

Permalink
Make stub audio and video decoders asynchronous
Browse files Browse the repository at this point in the history 
To behave more similarly to real decoders, the stub decoders use
a separate thread for decoding and sending out data.

The stub audio decoder has been changed to send up to two decoded audio
objects per input buffer, and the stub video decoder can now send
kBufferFull before WriteEndOfStream().

b/154000421

Change-Id: I2ed7012f376495ad9725c6bfe69507f89b69d09c
  • Loading branch information
Cobalt Team committed Mar 8, 2021
1 parent 68a762b commit b6f48bf
Show file tree
Hide file tree
Showing 6 changed files with 202 additions and 79 deletions.
4 changes: 4 additions & 0 deletions starboard/shared/starboard/player/filter/adaptive_audio_decoder_internal.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,10 @@ void AdaptiveAudioDecoder::TeardownAudioDecoder() {
}

void AdaptiveAudioDecoder::OnDecoderOutput() {
if (!BelongsToCurrentThread()) {
Schedule(std::bind(&AdaptiveAudioDecoder::OnDecoderOutput, this));
return;
}
SB_DCHECK(BelongsToCurrentThread());
SB_DCHECK(output_cb_);

Expand Down
169 changes: 116 additions & 53 deletions starboard/shared/starboard/player/filter/stub_audio_decoder.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,18 +41,69 @@ StubAudioDecoder::StubAudioDecoder(
const SbMediaAudioSampleInfo& audio_sample_info)
: sample_type_(GetSupportedSampleType()),
audio_codec_(audio_codec),
audio_sample_info_(audio_sample_info),
stream_ended_(false) {}
audio_sample_info_(audio_sample_info) {}

void StubAudioDecoder::Initialize(const OutputCB& output_cb,
const ErrorCB& error_cb) {
SB_DCHECK(BelongsToCurrentThread());

output_cb_ = output_cb;
error_cb_ = error_cb;
}

void StubAudioDecoder::Decode(const scoped_refptr<InputBuffer>& input_buffer,
const ConsumedCB& consumed_cb) {
SB_DCHECK(BelongsToCurrentThread());
SB_DCHECK(input_buffer);

if (!decoder_thread_) {
decoder_thread_.reset(new JobThread("stub_audio_decoder"));
}
decoder_thread_->job_queue()->Schedule(std::bind(
&StubAudioDecoder::DecodeOneBuffer, this, input_buffer, consumed_cb));
}

void StubAudioDecoder::WriteEndOfStream() {
SB_DCHECK(BelongsToCurrentThread());
if (decoder_thread_) {
decoder_thread_->job_queue()->Schedule(
std::bind(&StubAudioDecoder::DecodeEndOfStream, this));
return;
}
decoded_audios_.push(new DecodedAudio());
output_cb_();
}

scoped_refptr<DecodedAudio> StubAudioDecoder::Read(int* samples_per_second) {
SB_DCHECK(BelongsToCurrentThread());

*samples_per_second = audio_sample_info_.samples_per_second;
ScopedLock lock(decoded_audios_mutex_);
if (decoded_audios_.empty()) {
return scoped_refptr<DecodedAudio>();
}
auto result = decoded_audios_.front();
decoded_audios_.pop();
return result;
}

void StubAudioDecoder::Reset() {
SB_DCHECK(BelongsToCurrentThread());

decoder_thread_.reset();
last_input_buffer_ = NULL;
total_input_count_ = 0;
while (!decoded_audios_.empty()) {
decoded_audios_.pop();
}
CancelPendingJobs();
}

void StubAudioDecoder::DecodeOneBuffer(
const scoped_refptr<InputBuffer>& input_buffer,
const ConsumedCB& consumed_cb) {
SB_DCHECK(decoder_thread_->job_queue()->BelongsToCurrentThread());

// Values to represent what kind of dummy audio to fill the decoded audio
// we produce with.
enum FillType {
Expand All @@ -61,48 +112,79 @@ void StubAudioDecoder::Decode(const scoped_refptr<InputBuffer>& input_buffer,
};
// Can be set locally to fill with different types.
const FillType fill_type = kSilence;
const int kMaxInputBeforeMultipleDecodedAudios = 4;

if (last_input_buffer_) {
SbTime diff = input_buffer->timestamp() - last_input_buffer_->timestamp();
SB_DCHECK(diff >= 0);
SbTime total_output_duration =
input_buffer->timestamp() - last_input_buffer_->timestamp();
if (total_output_duration < 0) {
SB_DLOG(ERROR) << "Total output duration " << total_output_duration
<< " is invalid.";
error_cb_(kSbPlayerErrorDecode, "Total output duration is less than 0.");
return;
}
size_t sample_size =
sample_type_ == kSbMediaAudioSampleTypeInt16Deprecated ? 2 : 4;
size_t size = diff * audio_sample_info_.samples_per_second * sample_size *
audio_sample_info_.number_of_channels / kSbTimeSecond;
size -= size % (sample_size * audio_sample_info_.number_of_channels);
size_t total_frame_size = total_output_duration *
audio_sample_info_.samples_per_second /
kSbTimeSecond;
if (audio_codec_ == kSbMediaAudioCodecAac) {
// Frame size for AAC is fixed at 1024, so fake the output size such that
// number of frames matches up.
size = sample_size * audio_sample_info_.number_of_channels * 1024;
// Frame size for AAC is fixed at 1024 samples.
total_frame_size = 1024;
}

decoded_audios_.push(
new DecodedAudio(audio_sample_info_.number_of_channels, sample_type_,
kSbMediaAudioFrameStorageTypeInterleaved,
last_input_buffer_->timestamp(), size));

if (fill_type == kSilence) {
SbMemorySet(decoded_audios_.back()->buffer(), 0, size);
} else {
SB_DCHECK(fill_type == kWave);
for (int i = 0; i < size / sample_size; ++i) {
if (sample_size == 2) {
*(reinterpret_cast<int16_t*>(decoded_audios_.back()->buffer()) + i) =
i;
} else {
SB_DCHECK(sample_size == 4);
*(reinterpret_cast<float*>(decoded_audios_.back()->buffer()) + i) =
((i % 1024) - 512) / 512.0f;
// Send 3 decoded audio objects on every 5th call to DecodeOneBuffer().
int num_decoded_audio_objects =
total_input_count_ % kMaxInputBeforeMultipleDecodedAudios == 0 ? 3 : 1;
size_t output_frame_size = total_frame_size / num_decoded_audio_objects;
size_t output_byte_size =
output_frame_size * sample_size * audio_sample_info_.number_of_channels;

for (int i = 0; i < num_decoded_audio_objects; ++i) {
SbTime timestamp =
last_input_buffer_->timestamp() +
((total_output_duration / num_decoded_audio_objects) * i);
// Calculate the output frame size of the last decoded audio object, which
// may be larger than the rest.
if (i == num_decoded_audio_objects - 1 && num_decoded_audio_objects > 1) {
output_frame_size = total_frame_size -
(num_decoded_audio_objects - 1) * output_frame_size;
output_byte_size = output_frame_size * sample_size *
audio_sample_info_.number_of_channels;
}
scoped_refptr<DecodedAudio> decoded_audio(
new DecodedAudio(audio_sample_info_.number_of_channels, sample_type_,
kSbMediaAudioFrameStorageTypeInterleaved, timestamp,
output_byte_size));

if (fill_type == kSilence) {
SbMemorySet(decoded_audio.get()->buffer(), 0, output_byte_size);
} else {
SB_DCHECK(fill_type == kWave);
for (int j = 0; j < output_byte_size / sample_size; ++j) {
if (sample_size == 2) {
*(reinterpret_cast<int16_t*>(decoded_audio.get()->buffer()) + j) =
j;
} else {
SB_DCHECK(sample_size == 4);
*(reinterpret_cast<float*>(decoded_audio.get()->buffer()) + j) =
((j % 1024) - 512) / 512.0f;
}
}
}
ScopedLock lock(decoded_audios_mutex_);
decoded_audios_.push(decoded_audio);
decoder_thread_->job_queue()->Schedule(output_cb_);
}
Schedule(output_cb_);
}
Schedule(consumed_cb);
decoder_thread_->job_queue()->Schedule(consumed_cb);
last_input_buffer_ = input_buffer;
total_input_count_++;
}

void StubAudioDecoder::WriteEndOfStream() {
void StubAudioDecoder::DecodeEndOfStream() {
SB_DCHECK(decoder_thread_->job_queue()->BelongsToCurrentThread());

if (last_input_buffer_) {
// There won't be a next pts, so just guess that the decoded size is
// 4 times the encoded size.
Expand All @@ -116,35 +198,16 @@ void StubAudioDecoder::WriteEndOfStream() {
// number of frames matches up.
fake_size = sample_size * audio_sample_info_.number_of_channels * 1024;
}
ScopedLock lock(decoded_audios_mutex_);
decoded_audios_.push(
new DecodedAudio(audio_sample_info_.number_of_channels, sample_type_,
kSbMediaAudioFrameStorageTypeInterleaved,
last_input_buffer_->timestamp(), fake_size));
Schedule(output_cb_);
decoder_thread_->job_queue()->Schedule(output_cb_);
}
ScopedLock lock(decoded_audios_mutex_);
decoded_audios_.push(new DecodedAudio());
stream_ended_ = true;
Schedule(output_cb_);
}

scoped_refptr<DecodedAudio> StubAudioDecoder::Read(int* samples_per_second) {
scoped_refptr<DecodedAudio> result;
if (!decoded_audios_.empty()) {
result = decoded_audios_.front();
decoded_audios_.pop();
}
*samples_per_second = audio_sample_info_.samples_per_second;
return result;
}

void StubAudioDecoder::Reset() {
while (!decoded_audios_.empty()) {
decoded_audios_.pop();
}
stream_ended_ = false;
last_input_buffer_ = NULL;

CancelPendingJobs();
decoder_thread_->job_queue()->Schedule(output_cb_);
}

} // namespace filter
Expand Down
16 changes: 12 additions & 4 deletions starboard/shared/starboard/player/filter/stub_audio_decoder.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
#include "starboard/shared/starboard/media/media_util.h"
#include "starboard/shared/starboard/player/filter/audio_decoder_internal.h"
#include "starboard/shared/starboard/player/job_queue.h"
#include "starboard/shared/starboard/player/job_thread.h"

namespace starboard {
namespace shared {
Expand All @@ -33,26 +34,33 @@ class StubAudioDecoder : public AudioDecoder, private JobQueue::JobOwner {
public:
StubAudioDecoder(SbMediaAudioCodec audio_codec,
const SbMediaAudioSampleInfo& audio_sample_info);
~StubAudioDecoder() { Reset(); }

void Initialize(const OutputCB& output_cb, const ErrorCB& error_cb) override;

void Decode(const scoped_refptr<InputBuffer>& input_buffer,
const ConsumedCB& consumed_cb) override;

void WriteEndOfStream() override;

scoped_refptr<DecodedAudio> Read(int* samples_per_second) override;

void Reset() override;

private:
void DecodeOneBuffer(const scoped_refptr<InputBuffer>& input_buffer,
const ConsumedCB& consumed_cb);
void DecodeEndOfStream();

OutputCB output_cb_;
ErrorCB error_cb_;
SbMediaAudioSampleType sample_type_;
SbMediaAudioCodec audio_codec_;
starboard::media::AudioSampleInfo audio_sample_info_;
bool stream_ended_;

scoped_ptr<starboard::player::JobThread> decoder_thread_;
Mutex decoded_audios_mutex_;
std::queue<scoped_refptr<DecodedAudio> > decoded_audios_;
scoped_refptr<InputBuffer> last_input_buffer_;
// Used to determine when to send multiple DecodedAudios in DecodeOneBuffer().
int total_input_count_ = 0;
};

} // namespace filter
Expand Down
75 changes: 59 additions & 16 deletions starboard/shared/starboard/player/filter/stub_video_decoder.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ namespace filter {

void StubVideoDecoder::Initialize(const DecoderStatusCB& decoder_status_cb,
const ErrorCB& error_cb) {
SB_DCHECK(BelongsToCurrentThread());
SB_DCHECK(decoder_status_cb);
SB_DCHECK(!decoder_status_cb_);
decoder_status_cb_ = decoder_status_cb;
Expand All @@ -43,8 +44,44 @@ size_t StubVideoDecoder::GetMaxNumberOfCachedFrames() const {

void StubVideoDecoder::WriteInputBuffer(
const scoped_refptr<InputBuffer>& input_buffer) {
SB_DCHECK(BelongsToCurrentThread());
SB_DCHECK(input_buffer);

if (!decoder_thread_) {
decoder_thread_.reset(new JobThread("stub_video_decoder"));
}
decoder_thread_->job_queue()->Schedule(
std::bind(&StubVideoDecoder::DecodeOneBuffer, this, input_buffer));
}

void StubVideoDecoder::WriteEndOfStream() {
SB_DCHECK(BelongsToCurrentThread());

if (decoder_thread_) {
decoder_thread_->job_queue()->Schedule(
std::bind(&StubVideoDecoder::DecodeEndOfStream, this));
return;
}
decoder_status_cb_(kBufferFull, VideoFrame::CreateEOSFrame());
}

void StubVideoDecoder::Reset() {
SB_DCHECK(BelongsToCurrentThread());

video_sample_info_ = media::VideoSampleInfo();
decoder_thread_.reset();
output_frame_timestamps_.clear();
total_input_count_ = 0;
CancelPendingJobs();
}

SbDecodeTarget StubVideoDecoder::GetCurrentDecodeTarget() {
return kSbDecodeTargetInvalid;
}

void StubVideoDecoder::DecodeOneBuffer(
const scoped_refptr<InputBuffer>& input_buffer) {
SB_DCHECK(decoder_thread_->job_queue()->BelongsToCurrentThread());
auto& video_sample_info = input_buffer->video_sample_info();
if (video_sample_info.is_key_frame) {
if (video_sample_info_ != video_sample_info) {
Expand All @@ -53,38 +90,44 @@ void StubVideoDecoder::WriteInputBuffer(
}
}

output_event_frame_times_.insert(input_buffer->timestamp());
// Defer sending frames out until we've accumulated a reasonable number.
// This allows for input buffers to be out of order, and we expect that
// after buffering 8 (arbitrarily chosen) that the first timestamp in the
// sorted buffer will be the "correct" timestamp to send out.
const int kMaxFramesToDelay = 8;
// Send kBufferFull on every 5th input buffer received, starting with the
// first.
const int kMaxInputBeforeBufferFull = 5;
scoped_refptr<VideoFrame> output_frame = NULL;
if (output_event_frame_times_.size() > kMaxFramesToDelay) {
output_frame = new VideoFrame(*output_event_frame_times_.begin());
output_event_frame_times_.erase(output_event_frame_times_.begin());

output_frame_timestamps_.insert(input_buffer->timestamp());
if (output_frame_timestamps_.size() > kMaxFramesToDelay) {
output_frame = new VideoFrame(*output_frame_timestamps_.begin());
output_frame_timestamps_.erase(output_frame_timestamps_.begin());
}

if (total_input_count_ % kMaxInputBeforeBufferFull == 0) {
total_input_count_++;
decoder_status_cb_(kBufferFull, output_frame);
decoder_status_cb_(kNeedMoreInput, nullptr);
return;
}
total_input_count_++;
decoder_status_cb_(kNeedMoreInput, output_frame);
}

void StubVideoDecoder::WriteEndOfStream() {
void StubVideoDecoder::DecodeEndOfStream() {
SB_DCHECK(decoder_thread_->job_queue()->BelongsToCurrentThread());

// If there are any remaining frames we need to output, send them all out
// before writing EOS.
for (const auto& time : output_event_frame_times_) {
decoder_status_cb_(kBufferFull, new VideoFrame(time));
for (const auto time : output_frame_timestamps_) {
scoped_refptr<VideoFrame> output_frame = new VideoFrame(time);
decoder_status_cb_(kBufferFull, output_frame);
}
decoder_status_cb_(kBufferFull, VideoFrame::CreateEOSFrame());
}

void StubVideoDecoder::Reset() {
output_event_frame_times_.clear();
video_sample_info_ = media::VideoSampleInfo();
}

SbDecodeTarget StubVideoDecoder::GetCurrentDecodeTarget() {
return kSbDecodeTargetInvalid;
}

} // namespace filter
} // namespace player
} // namespace starboard
Expand Down
Loading

0 comments on commit b6f48bf

Please sign in to comment.