From 58eba6a31c089d7a554f20ee4308c75b52a28aa6 Mon Sep 17 00:00:00 2001 From: Jacob Su Date: Sun, 29 Sep 2024 16:22:06 +0800 Subject: [PATCH 1/5] fix mp4 stts box bug. --- trunk/src/kernel/srs_kernel_mp4.cpp | 31 ++++++++++++++--------------- trunk/src/kernel/srs_kernel_mp4.hpp | 5 ++++- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/trunk/src/kernel/srs_kernel_mp4.cpp b/trunk/src/kernel/srs_kernel_mp4.cpp index b4d597f7ee..f1223286b6 100644 --- a/trunk/src/kernel/srs_kernel_mp4.cpp +++ b/trunk/src/kernel/srs_kernel_mp4.cpp @@ -5062,21 +5062,15 @@ srs_error_t SrsMp4SampleManager::write_track(SrsFrameType track, if (sample->frame_type == SrsVideoAvcFrameTypeKeyFrame) { stss_entries.push_back(sample->index + 1); } - - if (stts) { - if (previous) { - uint32_t delta = (uint32_t)(sample->dts - previous->dts); - if (stts_entry.sample_delta == 0 || stts_entry.sample_delta == delta) { - stts_entry.sample_delta = delta; - stts_entry.sample_count++; - } else { - stts_entries.push_back(stts_entry); - stts_entry.sample_count = 1; - stts_entry.sample_delta = delta; - } - } else { - // The first sample always in the STTS table. - stts_entry.sample_count++; + + if (stts && previous) { + if (sample->dts >= previous->dts && previous->nb_samples > 0) { + uint32_t delta = (uint32_t)(sample->dts - previous->dts) / previous->nb_samples; + stts_entry.sample_count = previous->nb_samples; + // calcaulate delta in the time-scale of the media. + // moov->mvhd->timescale which is hardcoded to 1000, sample->tbn also being hardcoded to 1000. + stts_entry.sample_delta = delta * previous->tbn / 1000; + stts_entries.push_back(stts_entry); } } @@ -5097,7 +5091,10 @@ srs_error_t SrsMp4SampleManager::write_track(SrsFrameType track, previous = sample; } - if (stts && stts_entry.sample_count) { + if (stts && previous && previous->nb_samples > 0) { + stts_entry.sample_count = previous->nb_samples; + // Can't calculate last sample duration, so set sample_delta to 1. + stts_entry.sample_delta = 1; stts_entries.push_back(stts_entry); } @@ -5828,10 +5825,12 @@ srs_error_t SrsMp4Encoder::write_sample( ps->type = SrsFrameTypeVideo; ps->frame_type = (SrsVideoAvcFrameType)ft; ps->index = nb_videos++; + ps->nb_samples = format->video->nb_samples; vduration = dts; } else if (ht == SrsMp4HandlerTypeSOUN) { ps->type = SrsFrameTypeAudio; ps->index = nb_audios++; + ps->nb_samples = format->audio->nb_samples; aduration = dts; } else { srs_freep(ps); diff --git a/trunk/src/kernel/srs_kernel_mp4.hpp b/trunk/src/kernel/srs_kernel_mp4.hpp index 23805773e6..4718107ee2 100644 --- a/trunk/src/kernel/srs_kernel_mp4.hpp +++ b/trunk/src/kernel/srs_kernel_mp4.hpp @@ -1554,7 +1554,7 @@ class SrsMp4SampleDescriptionBox : public SrsMp4FullBox }; // 8.6.1.2 Decoding Time to Sample Box (stts), for Audio/Video. -// ISO_IEC_14496-12-base-format-2012.pdf, page 48 +// ISO_IEC_14496-12-base-format-2012.pdf, page 36 class SrsMp4SttsEntry { public: @@ -1893,6 +1893,9 @@ class SrsMp4Sample // The sample data. uint32_t nb_data; uint8_t* data; + // number of nalu|audio-frames in this sample. + uint32_t nb_samples; + public: SrsMp4Sample(); virtual ~SrsMp4Sample(); From 42ea5cfbf9e39ec1d7716011058bd40b7ca9fba8 Mon Sep 17 00:00:00 2001 From: Jacob Su Date: Mon, 30 Sep 2024 00:45:19 +0800 Subject: [PATCH 2/5] fix mp4 stts box decoder UT. --- trunk/src/kernel/srs_kernel_mp4.cpp | 58 ++++------------------------- trunk/src/kernel/srs_kernel_mp4.hpp | 10 +---- 2 files changed, 8 insertions(+), 60 deletions(-) diff --git a/trunk/src/kernel/srs_kernel_mp4.cpp b/trunk/src/kernel/srs_kernel_mp4.cpp index f1223286b6..574101f0c3 100644 --- a/trunk/src/kernel/srs_kernel_mp4.cpp +++ b/trunk/src/kernel/srs_kernel_mp4.cpp @@ -3962,52 +3962,12 @@ stringstream& SrsMp4SttsEntry::dumps_detail(stringstream& ss, SrsMp4DumpContext SrsMp4DecodingTime2SampleBox::SrsMp4DecodingTime2SampleBox() { type = SrsMp4BoxTypeSTTS; - - index = count = 0; } SrsMp4DecodingTime2SampleBox::~SrsMp4DecodingTime2SampleBox() { } -srs_error_t SrsMp4DecodingTime2SampleBox::initialize_counter() -{ - srs_error_t err = srs_success; - - // If only sps/pps and no frames, there is no stts entries. - if (entries.empty()) { - return err; - } - - index = 0; - if (index >= entries.size()) { - return srs_error_new(ERROR_MP4_ILLEGAL_TIMESTAMP, "illegal ts, empty stts"); - } - - count = entries[0].sample_count; - - return err; -} - -srs_error_t SrsMp4DecodingTime2SampleBox::on_sample(uint32_t sample_index, SrsMp4SttsEntry** ppentry) -{ - srs_error_t err = srs_success; - - if (sample_index + 1 > count) { - index++; - - if (index >= entries.size()) { - return srs_error_new(ERROR_MP4_ILLEGAL_TIMESTAMP, "illegal ts, stts overflow, count=%zd", entries.size()); - } - - count += entries[index].sample_count; - } - - *ppentry = &entries[index]; - - return err; -} - int SrsMp4DecodingTime2SampleBox::nb_header() { return SrsMp4FullBox::nb_header() + 4 + 8*(int)entries.size(); @@ -5213,11 +5173,6 @@ srs_error_t SrsMp4SampleManager::load_trak(map& tses, S // Samples per chunk. stsc->initialize_counter(); - // DTS box. - if ((err = stts->initialize_counter()) != srs_success) { - return srs_error_wrap(err, "stts init counter"); - } - // CTS/PTS box. if (ctts && (err = ctts->initialize_counter()) != srs_success) { return srs_error_wrap(err, "ctts init counter"); @@ -5246,13 +5201,14 @@ srs_error_t SrsMp4SampleManager::load_trak(map& tses, S } sample_relative_offset += sample_size; - SrsMp4SttsEntry* stts_entry = NULL; - if ((err = stts->on_sample(sample->index, &stts_entry)) != srs_success) { - srs_freep(sample); - return srs_error_wrap(err, "stts on sample"); - } if (previous) { - sample->pts = sample->dts = previous->dts + stts_entry->sample_delta; + uint32_t stts_index = sample->index - 1; + if (stts_index >= 0 && stts_index < stts->entries.size()) { + SrsMp4SttsEntry* stts_entry = &stts->entries[stts_index]; + sample->pts = sample->dts = previous->dts + stts_entry->sample_count * stts_entry->sample_delta; + } else { + sample->pts = sample->dts = previous->dts; + } } SrsMp4CttsEntry* ctts_entry = NULL; diff --git a/trunk/src/kernel/srs_kernel_mp4.hpp b/trunk/src/kernel/srs_kernel_mp4.hpp index 4718107ee2..c53a6ea8c8 100644 --- a/trunk/src/kernel/srs_kernel_mp4.hpp +++ b/trunk/src/kernel/srs_kernel_mp4.hpp @@ -1581,18 +1581,10 @@ class SrsMp4DecodingTime2SampleBox : public SrsMp4FullBox public: // An integer that gives the number of entries in the following table. std::vector entries; -private: - // The index for counter to calc the dts for samples. - uint32_t index; - uint32_t count; + public: SrsMp4DecodingTime2SampleBox(); virtual ~SrsMp4DecodingTime2SampleBox(); -public: - // Initialize the counter. - virtual srs_error_t initialize_counter(); - // When got an sample, index starts from 0. - virtual srs_error_t on_sample(uint32_t sample_index, SrsMp4SttsEntry** ppentry); protected: virtual int nb_header(); virtual srs_error_t encode_header(SrsBuffer* buf); From 652cdefaa2130880cf693d441f2cbb9b876a9019 Mon Sep 17 00:00:00 2001 From: Jacob Su Date: Mon, 30 Sep 2024 08:34:11 +0800 Subject: [PATCH 3/5] mp4 stts: fix multiplication duplication overflow warning. --- trunk/src/kernel/srs_kernel_mp4.cpp | 16 ++++++++-------- trunk/src/kernel/srs_kernel_mp4.hpp | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/trunk/src/kernel/srs_kernel_mp4.cpp b/trunk/src/kernel/srs_kernel_mp4.cpp index 574101f0c3..eaed242dfc 100644 --- a/trunk/src/kernel/srs_kernel_mp4.cpp +++ b/trunk/src/kernel/srs_kernel_mp4.cpp @@ -5024,9 +5024,9 @@ srs_error_t SrsMp4SampleManager::write_track(SrsFrameType track, } if (stts && previous) { - if (sample->dts >= previous->dts && previous->nb_samples > 0) { - uint32_t delta = (uint32_t)(sample->dts - previous->dts) / previous->nb_samples; - stts_entry.sample_count = previous->nb_samples; + if (sample->dts >= previous->dts && previous->nb_subsamples > 0) { + uint32_t delta = (uint32_t)(sample->dts - previous->dts) / previous->nb_subsamples; + stts_entry.sample_count = previous->nb_subsamples; // calcaulate delta in the time-scale of the media. // moov->mvhd->timescale which is hardcoded to 1000, sample->tbn also being hardcoded to 1000. stts_entry.sample_delta = delta * previous->tbn / 1000; @@ -5051,8 +5051,8 @@ srs_error_t SrsMp4SampleManager::write_track(SrsFrameType track, previous = sample; } - if (stts && previous && previous->nb_samples > 0) { - stts_entry.sample_count = previous->nb_samples; + if (stts && previous && previous->nb_subsamples > 0) { + stts_entry.sample_count = previous->nb_subsamples; // Can't calculate last sample duration, so set sample_delta to 1. stts_entry.sample_delta = 1; stts_entries.push_back(stts_entry); @@ -5205,7 +5205,7 @@ srs_error_t SrsMp4SampleManager::load_trak(map& tses, S uint32_t stts_index = sample->index - 1; if (stts_index >= 0 && stts_index < stts->entries.size()) { SrsMp4SttsEntry* stts_entry = &stts->entries[stts_index]; - sample->pts = sample->dts = previous->dts + stts_entry->sample_count * stts_entry->sample_delta; + sample->pts = sample->dts = previous->dts + (uint64_t) stts_entry->sample_count * (uint64_t) stts_entry->sample_delta; } else { sample->pts = sample->dts = previous->dts; } @@ -5781,12 +5781,12 @@ srs_error_t SrsMp4Encoder::write_sample( ps->type = SrsFrameTypeVideo; ps->frame_type = (SrsVideoAvcFrameType)ft; ps->index = nb_videos++; - ps->nb_samples = format->video->nb_samples; + ps->nb_subsamples = format->video->nb_samples; vduration = dts; } else if (ht == SrsMp4HandlerTypeSOUN) { ps->type = SrsFrameTypeAudio; ps->index = nb_audios++; - ps->nb_samples = format->audio->nb_samples; + ps->nb_subsamples = format->audio->nb_samples; aduration = dts; } else { srs_freep(ps); diff --git a/trunk/src/kernel/srs_kernel_mp4.hpp b/trunk/src/kernel/srs_kernel_mp4.hpp index c53a6ea8c8..2d1f947019 100644 --- a/trunk/src/kernel/srs_kernel_mp4.hpp +++ b/trunk/src/kernel/srs_kernel_mp4.hpp @@ -1886,7 +1886,7 @@ class SrsMp4Sample uint32_t nb_data; uint8_t* data; // number of nalu|audio-frames in this sample. - uint32_t nb_samples; + uint32_t nb_subsamples; public: SrsMp4Sample(); From ad6a185ec061166ef55bb43ebb80c649a463c868 Mon Sep 17 00:00:00 2001 From: Jacob Su Date: Mon, 30 Sep 2024 14:54:19 +0800 Subject: [PATCH 4/5] mp4: VLC and quicktime player compatible. --- trunk/src/app/srs_app_dvr.cpp | 24 +++++++++++++++++++++++- trunk/src/kernel/srs_kernel_mp4.cpp | 2 +- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/trunk/src/app/srs_app_dvr.cpp b/trunk/src/app/srs_app_dvr.cpp index f56c06da07..50aee24e69 100644 --- a/trunk/src/app/srs_app_dvr.cpp +++ b/trunk/src/app/srs_app_dvr.cpp @@ -642,7 +642,29 @@ srs_error_t SrsDvrPlan::on_video(SrsSharedPtrMessage* shared_video, SrsFormat* f if (!dvr_enabled) { return err; } - + + // skip any SEI type nalus + if (format->vcodec->id == SrsVideoCodecIdAVC) { + for (int i = 0; i < format->video->nb_samples; ++i) { + SrsSample* sample = &format->video->samples[i]; + SrsAvcNaluType avc_nalu_type; + + if ((err = SrsVideoFrame::parse_avc_nalu_type(sample, avc_nalu_type)) != srs_success) { + return srs_error_wrap(err, "parse avc nalu_type"); + } + + if (avc_nalu_type == SrsAvcNaluTypeSEI) { + return err; + } + } + } + + // quicktime player compatible: skip the packet without any nalu. + if (!format->is_avc_sequence_header() && format->video->nb_samples == 0) { + srs_warn("skip video segment has empty samples and is not sequence header."); + return err; + } + if ((err = segment->write_video(shared_video, format)) != srs_success) { return srs_error_wrap(err, "write video"); } diff --git a/trunk/src/kernel/srs_kernel_mp4.cpp b/trunk/src/kernel/srs_kernel_mp4.cpp index eaed242dfc..7658f76760 100644 --- a/trunk/src/kernel/srs_kernel_mp4.cpp +++ b/trunk/src/kernel/srs_kernel_mp4.cpp @@ -5696,7 +5696,7 @@ srs_error_t SrsMp4Encoder::initialize(ISrsWriteSeeker* ws) ftyp->major_brand = SrsMp4BoxBrandISOM; ftyp->minor_version = 512; - ftyp->set_compatible_brands(SrsMp4BoxBrandISOM, SrsMp4BoxBrandISO2, SrsMp4BoxBrandMP41); + ftyp->set_compatible_brands(SrsMp4BoxBrandISOM, SrsMp4BoxBrandISO2, SrsMp4BoxBrandAVC1, SrsMp4BoxBrandMP41); int nb_data = ftyp->nb_bytes(); std::vector data(nb_data); From 09cbd5b3302dd8a18a7d6af388077abc52198731 Mon Sep 17 00:00:00 2001 From: Jacob Su Date: Sun, 13 Oct 2024 00:13:50 +0800 Subject: [PATCH 5/5] fix code review problems: 1. fix stts box calculation; 2. filter nalu SEI for h.265 codec. --- trunk/src/app/srs_app_dvr.cpp | 10 ++++++++++ trunk/src/kernel/srs_kernel_codec.hpp | 1 + trunk/src/kernel/srs_kernel_mp4.cpp | 25 ++++++++++++++----------- trunk/src/kernel/srs_kernel_mp4.hpp | 2 -- 4 files changed, 25 insertions(+), 13 deletions(-) diff --git a/trunk/src/app/srs_app_dvr.cpp b/trunk/src/app/srs_app_dvr.cpp index 50aee24e69..3bebc79af9 100644 --- a/trunk/src/app/srs_app_dvr.cpp +++ b/trunk/src/app/srs_app_dvr.cpp @@ -657,6 +657,16 @@ srs_error_t SrsDvrPlan::on_video(SrsSharedPtrMessage* shared_video, SrsFormat* f return err; } } + } else if (format->vcodec->id == SrsVideoCodecIdHEVC) { +#ifdef SRS_H265 + for (int i = 0; i < format->video->nb_samples; ++i) { + SrsSample* sample = &format->video->samples[i]; + SrsHevcNaluType hevc_nalu_type = SrsHevcNaluTypeParse(sample->bytes[0]); + if (hevc_nalu_type == SrsHevcNaluType_SEI || hevc_nalu_type == SrsHevcNaluType_SEI_SUFFIX) { + return err; + } + } +#endif } // quicktime player compatible: skip the packet without any nalu. diff --git a/trunk/src/kernel/srs_kernel_codec.hpp b/trunk/src/kernel/srs_kernel_codec.hpp index a1462d3328..2d92e9a6b7 100644 --- a/trunk/src/kernel/srs_kernel_codec.hpp +++ b/trunk/src/kernel/srs_kernel_codec.hpp @@ -1308,6 +1308,7 @@ class SrsVideoFrame : public SrsFrame bool has_sps_pps; // The first nalu type. SrsAvcNaluType first_nalu_type; + public: SrsVideoFrame(); virtual ~SrsVideoFrame(); diff --git a/trunk/src/kernel/srs_kernel_mp4.cpp b/trunk/src/kernel/srs_kernel_mp4.cpp index 7658f76760..937d8afbc1 100644 --- a/trunk/src/kernel/srs_kernel_mp4.cpp +++ b/trunk/src/kernel/srs_kernel_mp4.cpp @@ -5024,13 +5024,19 @@ srs_error_t SrsMp4SampleManager::write_track(SrsFrameType track, } if (stts && previous) { - if (sample->dts >= previous->dts && previous->nb_subsamples > 0) { - uint32_t delta = (uint32_t)(sample->dts - previous->dts) / previous->nb_subsamples; - stts_entry.sample_count = previous->nb_subsamples; + uint32_t delta = (uint32_t)(sample->dts - previous->dts); + + if (delta > 0 && stts_entry.sample_count == 0) { + stts_entry.sample_count = 1; + stts_entry.sample_delta = delta; + } else if (delta > 0 && stts_entry.sample_delta == delta) { + stts_entry.sample_count++; + } else if (stts_entry.sample_count > 0) { + stts_entries.push_back(stts_entry); + stts_entry.sample_count = delta > 0 ? 1 : 0; // calcaulate delta in the time-scale of the media. // moov->mvhd->timescale which is hardcoded to 1000, sample->tbn also being hardcoded to 1000. - stts_entry.sample_delta = delta * previous->tbn / 1000; - stts_entries.push_back(stts_entry); + stts_entry.sample_delta = delta; } } @@ -5051,10 +5057,9 @@ srs_error_t SrsMp4SampleManager::write_track(SrsFrameType track, previous = sample; } - if (stts && previous && previous->nb_subsamples > 0) { - stts_entry.sample_count = previous->nb_subsamples; - // Can't calculate last sample duration, so set sample_delta to 1. - stts_entry.sample_delta = 1; + if (stts && stts_entry.sample_count) { + // Can't calculate last sample duration, so set the sample_delta to previous entry. + stts_entry.sample_delta = stts_entries.size() > 0 ? stts_entries.back().sample_delta : 1; stts_entries.push_back(stts_entry); } @@ -5781,12 +5786,10 @@ srs_error_t SrsMp4Encoder::write_sample( ps->type = SrsFrameTypeVideo; ps->frame_type = (SrsVideoAvcFrameType)ft; ps->index = nb_videos++; - ps->nb_subsamples = format->video->nb_samples; vduration = dts; } else if (ht == SrsMp4HandlerTypeSOUN) { ps->type = SrsFrameTypeAudio; ps->index = nb_audios++; - ps->nb_subsamples = format->audio->nb_samples; aduration = dts; } else { srs_freep(ps); diff --git a/trunk/src/kernel/srs_kernel_mp4.hpp b/trunk/src/kernel/srs_kernel_mp4.hpp index 2d1f947019..66804d327e 100644 --- a/trunk/src/kernel/srs_kernel_mp4.hpp +++ b/trunk/src/kernel/srs_kernel_mp4.hpp @@ -1885,8 +1885,6 @@ class SrsMp4Sample // The sample data. uint32_t nb_data; uint8_t* data; - // number of nalu|audio-frames in this sample. - uint32_t nb_subsamples; public: SrsMp4Sample();