From 8059ded69570aa737d2a186eeb1fe3b76ffd58b4 Mon Sep 17 00:00:00 2001 From: chundonglinlin Date: Sat, 7 Jan 2023 00:31:03 +0800 Subject: [PATCH] H265: Support HEVC over SRT.(#465) --- trunk/src/app/srs_app_srt_source.cpp | 198 ++++++- trunk/src/app/srs_app_srt_source.hpp | 14 + trunk/src/kernel/srs_kernel_error.hpp | 6 + trunk/src/protocol/srs_protocol_raw_avc.cpp | 591 ++++++++++++++++++++ trunk/src/protocol/srs_protocol_raw_avc.hpp | 45 ++ 5 files changed, 853 insertions(+), 1 deletion(-) diff --git a/trunk/src/app/srs_app_srt_source.cpp b/trunk/src/app/srs_app_srt_source.cpp index 1c549fe431..ed34a6fc8f 100644 --- a/trunk/src/app/srs_app_srt_source.cpp +++ b/trunk/src/app/srs_app_srt_source.cpp @@ -338,7 +338,7 @@ srs_error_t SrsRtmpFromSrtBridge::on_ts_message(SrsTsMessage* msg) } // check supported codec - if (msg->channel->stream != SrsTsStreamVideoH264 && msg->channel->stream != SrsTsStreamAudioAAC) { + if (msg->channel->stream != SrsTsStreamVideoH264 && msg->channel->stream != SrsTsStreamVideoHEVC && msg->channel->stream != SrsTsStreamAudioAAC) { return srs_error_new(ERROR_STREAM_CASTER_TS_CODEC, "ts: unsupported stream codec=%d", msg->channel->stream); } @@ -358,6 +358,14 @@ srs_error_t SrsRtmpFromSrtBridge::on_ts_message(SrsTsMessage* msg) } // TODO: FIXME: implements other codec? +#ifdef SRS_H265 + if (msg->channel->stream == SrsTsStreamVideoHEVC) { + if ((err = on_ts_hevc(msg, &avs)) != srs_success) { + return srs_error_wrap(err, "ts: consume hevc video"); + } + } +#endif + return err; } @@ -525,6 +533,194 @@ srs_error_t SrsRtmpFromSrtBridge::on_h264_frame(SrsTsMessage* msg, vector > ipb_frames; + + SrsRawHEVCStream *hevc = new SrsRawHEVCStream(); + SrsAutoFree(SrsRawHEVCStream, hevc); + + // send each frame. + while (!avs->empty()) { + char* frame = NULL; + int frame_size = 0; + if ((err = hevc->annexb_demux(avs, &frame, &frame_size)) != srs_success) { + return srs_error_wrap(err, "demux hevc annexb"); + } + + if (frame == NULL || frame_size == 0) { + continue; + } + + // for vps + if (hevc->is_vps(frame, frame_size)) { + std::string vps; + if ((err = hevc->sps_demux(frame, frame_size, vps)) != srs_success) { + return srs_error_wrap(err, "demux vps"); + } + + if (!vps.empty() && hevc_vps_ != vps) { + vps_sps_pps_change_ = true; + } + + hevc_vps_ = vps; + continue; + } + + // for sps + if (hevc->is_sps(frame, frame_size)) { + std::string sps; + if ((err = hevc->sps_demux(frame, frame_size, sps)) != srs_success) { + return srs_error_wrap(err, "demux sps"); + } + + if (! sps.empty() && hevc_sps_ != sps) { + vps_sps_pps_change_ = true; + } + + hevc_sps_ = sps; + continue; + } + + // for pps + if (hevc->is_pps(frame, frame_size)) { + std::string pps; + if ((err = hevc->pps_demux(frame, frame_size, pps)) != srs_success) { + return srs_error_wrap(err, "demux pps"); + } + + if (! pps.empty() && hevc_pps_ != pps) { + vps_sps_pps_change_ = true; + } + + hevc_pps_ = pps; + continue; + } + + ipb_frames.push_back(make_pair(frame, frame_size)); + } + + if ((err = check_vps_sps_pps_change(msg)) != srs_success) { + return srs_error_wrap(err, "check vps sps pps"); + } + + return on_hevc_frame(msg, ipb_frames); +} + +srs_error_t SrsRtmpFromSrtBridge::check_vps_sps_pps_change(SrsTsMessage* msg) +{ + srs_error_t err = srs_success; + + if (!vps_sps_pps_change_) { + return err; + } + + if (hevc_vps_.empty() || hevc_sps_.empty() || hevc_pps_.empty()) { + return srs_error_new(ERROR_SRT_TO_RTMP_EMPTY_SPS_PPS, "vps or sps or pps empty"); + } + + // vps/sps/pps changed, generate new video sh frame and dispatch it. + vps_sps_pps_change_ = false; + + // ts tbn to flv tbn. + uint32_t dts = (uint32_t)(msg->dts / 90); + + std::string sh; + SrsRawHEVCStream* hevc = new SrsRawHEVCStream(); + SrsAutoFree(SrsRawHEVCStream, hevc); + + if ((err = hevc->mux_sequence_header(hevc_vps_, hevc_sps_, hevc_pps_, sh)) != srs_success) { + return srs_error_wrap(err, "mux sequence header"); + } + + // h264 packet to flv packet. + char* flv = NULL; + int nb_flv = 0; + if ((err = hevc->mux_avc2flv(sh, SrsVideoAvcFrameTypeKeyFrame, SrsVideoAvcFrameTraitSequenceHeader, dts, dts, &flv, &nb_flv)) != srs_success) { + return srs_error_wrap(err, "avc to flv"); + } + + SrsMessageHeader header; + header.initialize_video(nb_flv, dts, video_streamid_); + SrsCommonMessage rtmp; + if ((err = rtmp.create(&header, flv, nb_flv)) != srs_success) { + return srs_error_wrap(err, "create rtmp"); + } + + if ((err = live_source_->on_video(&rtmp)) != srs_success) { + return srs_error_wrap(err, "srt to rtmp sps/pps"); + } + + return err; +} + +srs_error_t SrsRtmpFromSrtBridge::on_hevc_frame(SrsTsMessage* msg, vector >& ipb_frames) +{ + srs_error_t err = srs_success; + + if (ipb_frames.empty()) { + return srs_error_new(ERROR_SRT_CONN, "empty frame"); + } + + // ts tbn to flv tbn. + uint32_t dts = (uint32_t)(msg->dts / 90); + uint32_t pts = (uint32_t)(msg->pts / 90); + int32_t cts = pts - dts; + + // for IDR frame, the frame is keyframe. + SrsVideoAvcFrameType frame_type = SrsVideoAvcFrameTypeInterFrame; + + // 5bytes video tag header + int frame_size = 5; + for (size_t i = 0; i != ipb_frames.size(); ++i) { + // 4 bytes for nalu length. + frame_size += 4 + ipb_frames[i].second; + SrsHevcNaluType nal_unit_type = (SrsHevcNaluType)((ipb_frames[i].first[0] & 0x7e) >> 1); + if ((nal_unit_type >= SrsHevcNaluType_CODED_SLICE_BLA) && (nal_unit_type <= SrsHevcNaluType_RESERVED_23)) { + frame_type = SrsVideoAvcFrameTypeKeyFrame; + } + } + + SrsCommonMessage rtmp; + rtmp.header.initialize_video(frame_size, dts, video_streamid_); + rtmp.create_payload(frame_size); + rtmp.size = frame_size; + SrsBuffer payload(rtmp.payload, rtmp.size); + + // Write 5bytes video tag header. + + // @see: E.4.3 Video Tags, video_file_format_spec_v10_1.pdf, page 78 + // Frame Type, Type of video frame. + // CodecID, Codec Identifier. + // set the rtmp header + payload.write_1bytes((frame_type << 4) | SrsVideoCodecIdHEVC); + // hevc_type: nalu + payload.write_1bytes(0x01); + // composition time + payload.write_3bytes(cts); + + // Write video nalus. + for (size_t i = 0; i != ipb_frames.size(); ++i) { + char* nal = ipb_frames[i].first; + int nal_size = ipb_frames[i].second; + + // write 4 bytes of nalu length. + payload.write_4bytes(nal_size); + // write nalu + payload.write_bytes(nal, nal_size); + } + + if ((err = live_source_->on_video(&rtmp)) != srs_success) { + return srs_error_wrap(err ,"srt ts hevc video to rtmp"); + } + + return err; +} +#endif + srs_error_t SrsRtmpFromSrtBridge::on_ts_audio(SrsTsMessage* msg, SrsBuffer* avs) { srs_error_t err = srs_success; diff --git a/trunk/src/app/srs_app_srt_source.hpp b/trunk/src/app/srs_app_srt_source.hpp index a11a94b1b8..42e7018654 100644 --- a/trunk/src/app/srs_app_srt_source.hpp +++ b/trunk/src/app/srs_app_srt_source.hpp @@ -123,6 +123,13 @@ class SrsRtmpFromSrtBridge : public ISrsSrtSourceBridge, public ISrsTsHandler srs_error_t on_h264_frame(SrsTsMessage* msg, std::vector >& ipb_frames); srs_error_t check_audio_sh_change(SrsTsMessage* msg, uint32_t pts); srs_error_t on_aac_frame(SrsTsMessage* msg, uint32_t pts, char* frame, int frame_size); + +#ifdef SRS_H265 + srs_error_t on_ts_hevc(SrsTsMessage *msg, SrsBuffer *avs); + srs_error_t check_vps_sps_pps_change(SrsTsMessage *msg); + srs_error_t on_hevc_frame(SrsTsMessage *msg, std::vector> &ipb_frames); +#endif + private: SrsTsContext* ts_ctx_; @@ -131,6 +138,13 @@ class SrsRtmpFromSrtBridge : public ISrsSrtSourceBridge, public ISrsTsHandler std::string sps_; std::string pps_; +#ifdef SRS_H265 + bool vps_sps_pps_change_; + std::string hevc_vps_; + std::string hevc_sps_; + std::string hevc_pps_; +#endif + // Record audio sepcific config had changed, if change, need to generate new audio sh frame. bool audio_sh_change_; std::string audio_sh_; diff --git a/trunk/src/kernel/srs_kernel_error.hpp b/trunk/src/kernel/srs_kernel_error.hpp index 0e339702a7..21259cf59c 100644 --- a/trunk/src/kernel/srs_kernel_error.hpp +++ b/trunk/src/kernel/srs_kernel_error.hpp @@ -275,6 +275,9 @@ XX(ERROR_HEVC_DISABLED , 3098, "HevcDisabled", "HEVC is disabled") \ XX(ERROR_HEVC_DECODE_ERROR , 3099, "HevcDecode", "HEVC decode av stream failed") \ XX(ERROR_MP4_HVCC_CHANGE , 3100, "Mp4HvcCChange", "MP4 does not support video HvcC change") + XX(ERROR_HEVC_DROP_BEFORE_SPS_PPS , 4050, "HevcDropBeforeSequence", "HEVC Drop frames before get sps and pps") \ + XX(ERROR_HEVC_API_NO_PREFIXED , 4051, "HevcAnnexbPrefix", "No annexb prefix for HEVC decoder") + /**************************************************/ /* HTTP/StreamConverter protocol error. */ #define SRS_ERRNO_MAP_HTTP(XX) \ @@ -324,6 +327,9 @@ XX(ERROR_GB_SSRC_GENERATE , 4051, "GbSsrcGenerate", "Failed to generate SSRC for GB28181") \ XX(ERROR_GB_CONFIG , 4052, "GbConfig", "Invalid configuration for GB28181") \ XX(ERROR_GB_TIMEOUT , 4053, "GbTimeout", "SIP or media connection timeout for GB28181") \ + XX(ERROR_STREAM_CASTER_HEVC_VPS , 4060, "CasterTsHevcVps", "Invalid ts HEVC VPS for stream caster") \ + XX(ERROR_STREAM_CASTER_HEVC_SPS , 4061, "CasterTsHevcSps", "Invalid ts HEVC SPS for stream caster") \ + XX(ERROR_STREAM_CASTER_HEVC_PPS , 4062, "CasterTsHevcPps", "Invalid ts HEVC PPS for stream caster") /**************************************************/ /* RTC protocol error. */ diff --git a/trunk/src/protocol/srs_protocol_raw_avc.cpp b/trunk/src/protocol/srs_protocol_raw_avc.cpp index dde9f0be57..db0e86cb7b 100644 --- a/trunk/src/protocol/srs_protocol_raw_avc.cpp +++ b/trunk/src/protocol/srs_protocol_raw_avc.cpp @@ -263,6 +263,597 @@ srs_error_t SrsRawH264Stream::mux_avc2flv(string video, int8_t frame_type, int8_ return err; } +#ifdef SRS_H265 + +#define SRS_STREAM_BIT(ptr, off) (((ptr)[(off) / 8] >> (7 - ((off) % 8))) & 0x01) + +struct mpeg4_hevc_t { + uint8_t configurationVersion; // 1-only + uint8_t general_profile_space; // 2bit,[0,3] + uint8_t general_tier_flag; // 1bit,[0,1] + uint8_t general_profile_idc; // 5bit,[0,31] + uint32_t general_profile_compatibility_flags; + uint64_t general_constraint_indicator_flags; + uint8_t general_level_idc; + uint16_t min_spatial_segmentation_idc; + uint8_t parallelismType; // 2bit,[0,3] + uint8_t chromaFormat; // 2bit,[0,3] + uint8_t bitDepthLumaMinus8; // 3bit,[0,7] + uint8_t bitDepthChromaMinus8; // 3bit,[0,7] + uint16_t avgFrameRate; + uint8_t constantFrameRate; // 2bit,[0,3] + uint8_t numTemporalLayers; // 3bit,[0,7] + uint8_t temporalIdNested; // 1bit,[0,1] + uint8_t lengthSizeMinusOne; // 2bit,[0,3] + + uint8_t numOfArrays; + struct { + uint8_t array_completeness; + uint8_t type; // nalu type + uint16_t bytes; + uint8_t* data; + } nalu[64]; + + uint8_t array_completeness; + uint8_t data[4 * 1024]; + int off; +}; + +static uint8_t mpeg4_hevc_read_ue(const uint8_t *data, int bytes, int *offset) +{ + int bit; + int leadingZeroBits = -1; + + for (bit = 0; !bit && *offset / 8 < bytes; ++leadingZeroBits) { + bit = (data[*offset / 8] >> (7 - (*offset % 8))) & 0x01; + ++*offset; + } + + srs_assert(leadingZeroBits < 32); + + bit = 0; + for (int i = 0; i < leadingZeroBits && *offset / 8 < bytes; i++) { + bit = (bit << 1) | ((data[*offset / 8] >> (7 - (*offset % 8))) & 0x01); + ++*offset; + } + + return (uint8_t)((1 << leadingZeroBits) - 1 + bit); +} + +static int hevc_rbsp_decode(const uint8_t* nalu, int bytes, uint8_t* sodb) +{ + int n = 0; + + for (int i = 0; i < bytes; i++) { + if (i + 2 < bytes && 0 == nalu[i] && 0 == nalu[i + 1] && 0x03 == nalu[i + 2]) { + sodb[n++] = nalu[i]; + sodb[n++] = nalu[i + 1]; + i += 2; + } else { + sodb[n++] = nalu[i]; + } + } + + return n; +} + +static int hevc_profile_tier_level(const uint8_t* nalu, int bytes, uint8_t maxNumSubLayersMinus1, struct mpeg4_hevc_t* hevc) +{ + if (bytes < 12) return -1; + if (maxNumSubLayersMinus1 < 1) return 12; + + hevc->general_profile_space = (nalu[0] >> 6) & 0x03; + hevc->general_tier_flag = (nalu[0] >> 5) & 0x01; + hevc->general_profile_idc = nalu[0] & 0x1f; + + hevc->general_profile_compatibility_flags = 0; + hevc->general_profile_compatibility_flags |= nalu[1] << 24; + hevc->general_profile_compatibility_flags |= nalu[2] << 16; + hevc->general_profile_compatibility_flags |= nalu[3] << 8; + hevc->general_profile_compatibility_flags |= nalu[4]; + + hevc->general_constraint_indicator_flags = 0; + hevc->general_constraint_indicator_flags |= ((uint64_t)nalu[5]) << 40; + hevc->general_constraint_indicator_flags |= ((uint64_t)nalu[6]) << 32; + hevc->general_constraint_indicator_flags |= ((uint64_t)nalu[7]) << 24; + hevc->general_constraint_indicator_flags |= ((uint64_t)nalu[8]) << 16; + hevc->general_constraint_indicator_flags |= ((uint64_t)nalu[9]) << 8; + hevc->general_constraint_indicator_flags |= nalu[10]; + + hevc->general_level_idc = nalu[11]; + + uint8_t sub_layer_profile_present_flag[8]; + uint8_t sub_layer_level_present_flag[8]; + for (int i = 0; i < maxNumSubLayersMinus1; i++) { + sub_layer_profile_present_flag[i] = SRS_STREAM_BIT(nalu, 12 * 8 + i * 2); + sub_layer_level_present_flag[i] = SRS_STREAM_BIT(nalu, 12 * 8 + i * 2 + 1); + } + + int n = 12 + 2; + for (int i = 0; i < maxNumSubLayersMinus1; i++) { + if (sub_layer_profile_present_flag[i]) n += 11; + if (sub_layer_level_present_flag[i]) n += 1; + } + + return bytes < n ? n : -1; +} + +static uint8_t hevc_vps_id(const uint8_t* rbsp, int bytes, struct mpeg4_hevc_t* hevc, uint8_t* ptr) +{ + int sodb = hevc_rbsp_decode(rbsp, bytes, ptr); + if (sodb < 16 + 2) return 0xFF; + + uint8_t vps = ptr[2] >> 4; // 2-nalu type + uint8_t vps_max_sub_layers_minus1 = (ptr[3] >> 1) & 0x07; + uint8_t vps_temporal_id_nesting_flag = ptr[3] & 0x01; + hevc->numTemporalLayers = srs_max(hevc->numTemporalLayers, vps_max_sub_layers_minus1 + 1); + hevc->temporalIdNested = (hevc->temporalIdNested || vps_temporal_id_nesting_flag) ? 1 : 0; + hevc_profile_tier_level(ptr + 6, sodb - 6, vps_max_sub_layers_minus1, hevc); + + return vps; +} + +static uint8_t hevc_sps_id(const uint8_t* rbsp, int bytes, struct mpeg4_hevc_t* hevc, uint8_t* ptr, uint8_t* vps) +{ + int sodb = hevc_rbsp_decode(rbsp, bytes, ptr); + if (sodb < 12 + 3) return 0xFF; + + *vps = ptr[2] >> 4; // 2-nalu type + + uint8_t sps_max_sub_layers_minus1 = (ptr[2] >> 1) & 0x07; + uint8_t sps_temporal_id_nesting_flag = ptr[2] & 0x01; + (void)sps_temporal_id_nesting_flag; + + int n = hevc_profile_tier_level(ptr + 3, sodb - 3, sps_max_sub_layers_minus1, hevc); + if (n <= 0) return 0xFF; + n = (n + 3) * 8; + + uint8_t sps = mpeg4_hevc_read_ue(ptr, sodb, &n); + hevc->chromaFormat = mpeg4_hevc_read_ue(ptr, sodb, &n); + if (3 == hevc->chromaFormat) n++; + + mpeg4_hevc_read_ue(ptr, sodb, &n); // pic_width_in_luma_samples + mpeg4_hevc_read_ue(ptr, sodb, &n); // pic_height_in_luma_samples + + uint8_t conformance_window_flag = SRS_STREAM_BIT(ptr, n); n++; // conformance_window_flag + if (conformance_window_flag) { + mpeg4_hevc_read_ue(ptr, sodb, &n); // conf_win_left_offset + mpeg4_hevc_read_ue(ptr, sodb, &n); // conf_win_right_offset + mpeg4_hevc_read_ue(ptr, sodb, &n); // conf_win_top_offset + mpeg4_hevc_read_ue(ptr, sodb, &n); // conf_win_bottom_offset + } + hevc->bitDepthLumaMinus8 = mpeg4_hevc_read_ue(ptr, sodb, &n); + hevc->bitDepthChromaMinus8 = mpeg4_hevc_read_ue(ptr, sodb, &n); + + // TODO: vui_parameters + //mp4->hevc->min_spatial_segmentation_idc; // min_spatial_segmentation_idc + + return sps; +} + +static uint8_t hevc_pps_id(const uint8_t* rbsp, int bytes, struct mpeg4_hevc_t* hevc, uint8_t* ptr, uint8_t* sps) +{ + // FIXME: TODO: + //hevc->parallelismType; // entropy_coding_sync_enabled_flag + (void)hevc; + + int sodb = hevc_rbsp_decode(rbsp, bytes, ptr); + if (sodb < 3) return 0xFF; + + int offset = 2 * 8; // 2-nalu type + *sps = mpeg4_hevc_read_ue(ptr, sodb, &offset); + return mpeg4_hevc_read_ue(ptr, sodb, &offset); +} + +static void mpeg4_hevc_remove(struct mpeg4_hevc_t* hevc, uint8_t* ptr, int bytes, const uint8_t* end) +{ + srs_assert(ptr >= hevc->data && ptr + bytes <= end && end <= hevc->data + sizeof(hevc->data)); + + memmove(ptr, ptr + bytes, end - ptr - bytes); + + for (int i = 0; i < hevc->numOfArrays; i++) { + if (hevc->nalu[i].data > ptr) hevc->nalu[i].data -= bytes; + } +} + +static int mpeg4_hevc_update(struct mpeg4_hevc_t *hevc, int i, const uint8_t *nalu, int bytes) +{ + if (bytes == hevc->nalu[i].bytes && 0 == memcmp(nalu, hevc->nalu[i].data, bytes)) + return 0; // do nothing + + if (bytes > hevc->nalu[i].bytes && hevc->off + (bytes - hevc->nalu[i].bytes) > (int)sizeof(hevc->data)) { + return -1; // too big + } + + mpeg4_hevc_remove(hevc, hevc->nalu[i].data, hevc->nalu[i].bytes, hevc->data + hevc->off); + hevc->off -= hevc->nalu[i].bytes; + + hevc->nalu[i].data = hevc->data + hevc->off; + hevc->nalu[i].bytes = (uint16_t)bytes; + memcpy(hevc->nalu[i].data, nalu, bytes); + hevc->off += bytes; + + return 1; +} + +static int mpeg4_hevc_add(struct mpeg4_hevc_t* hevc, uint8_t type, const uint8_t* nalu, int bytes) +{ + // copy new + srs_assert(hevc->numOfArrays < (int)sizeof(hevc->nalu) / (int)sizeof(hevc->nalu[0])); + + if (hevc->numOfArrays >= (int)sizeof(hevc->nalu) / (int)sizeof(hevc->nalu[0]) || hevc->off + bytes > (int)sizeof(hevc->data)) { + return -1; + } + + hevc->nalu[hevc->numOfArrays].type = type; + hevc->nalu[hevc->numOfArrays].bytes = (uint16_t)bytes; + hevc->nalu[hevc->numOfArrays].array_completeness = 1; + hevc->nalu[hevc->numOfArrays].data = hevc->data + hevc->off; + memcpy(hevc->nalu[hevc->numOfArrays].data, nalu, bytes); + hevc->off += bytes; + ++hevc->numOfArrays; + + return 1; +} + +static int h265_vps_copy(struct mpeg4_hevc_t* hevc, const uint8_t* nalu, int bytes) +{ + if (bytes < 3) return -1; // invalid length + + uint8_t vpsid = hevc_vps_id(nalu, bytes, hevc, hevc->data + hevc->off); + for (int i = 0; i < hevc->numOfArrays; i++) { + if (SrsHevcNaluType_VPS == hevc->nalu[i].type && vpsid == hevc_vps_id(hevc->nalu[i].data, hevc->nalu[i].bytes, hevc, hevc->data + hevc->off)) + return mpeg4_hevc_update(hevc, i, nalu, bytes); + } + + return mpeg4_hevc_add(hevc, SrsHevcNaluType_VPS, nalu, bytes); +} + +static int h265_sps_copy(struct mpeg4_hevc_t* hevc, const uint8_t* nalu, int bytes) +{ + if (bytes < 13 + 2) return -1; // invalid length + + uint8_t vpsid, vpsid2; + uint8_t spsid = hevc_sps_id(nalu, bytes, hevc, hevc->data + hevc->off, &vpsid); + for (int i = 0; i < hevc->numOfArrays; i++) { + if (SrsHevcNaluType_SPS == hevc->nalu[i].type && spsid == hevc_sps_id(hevc->nalu[i].data, hevc->nalu[i].bytes, hevc, hevc->data + hevc->off, &vpsid2) && vpsid == vpsid2) + return mpeg4_hevc_update(hevc, i, nalu, bytes); + } + + return mpeg4_hevc_add(hevc, SrsHevcNaluType_SPS, nalu, bytes); +} + +static int h265_pps_copy(struct mpeg4_hevc_t* hevc, const uint8_t* nalu, int bytes) +{ + if (bytes < 1 + 2) return -1; // invalid length + + uint8_t spsid, spsid2; + uint8_t ppsid = hevc_pps_id(nalu, bytes, hevc, hevc->data + hevc->off, &spsid); + for (int i = 0; i < hevc->numOfArrays; i++) { + if (SrsHevcNaluType_PPS == hevc->nalu[i].type && ppsid == hevc_pps_id(hevc->nalu[i].data, hevc->nalu[i].bytes, hevc, hevc->data + hevc->off, &spsid2) && spsid == spsid2) + return mpeg4_hevc_update(hevc, i, nalu, bytes); + } + + return mpeg4_hevc_add(hevc, SrsHevcNaluType_PPS, nalu, bytes); +} + +SrsRawHEVCStream::SrsRawHEVCStream() +{ +} + +SrsRawHEVCStream::~SrsRawHEVCStream() +{ +} + +srs_error_t SrsRawHEVCStream::annexb_demux(SrsBuffer *stream, char **pframe, int *pnb_frame) +{ + srs_error_t err = srs_success; + + *pframe = NULL; + *pnb_frame = 0; + + while (!stream->empty()) { + // each frame must prefixed by annexb format. + // about annexb, @see ISO_IEC_14496-10-AVC-2003.pdf, page 211. + int pnb_start_code = 0; + if (!srs_avc_startswith_annexb(stream, &pnb_start_code)) { + return srs_error_new(ERROR_HEVC_API_NO_PREFIXED, "hevc annexb start code"); + } + int start = stream->pos() + pnb_start_code; + + // find the last frame prefixed by annexb format. + stream->skip(pnb_start_code); + while (!stream->empty()) { + if (srs_avc_startswith_annexb(stream, NULL)) { + break; + } + stream->skip(1); + } + + // demux the frame. + *pnb_frame = stream->pos() - start; + *pframe = stream->data() + start; + break; + } + + return err; +} + +// whether the frame is vps or sps or pps. +bool SrsRawHEVCStream::is_vps(char *frame, int nb_frame) +{ + srs_assert(nb_frame > 0); + + // 5bits, 7.3.1 NAL unit syntax, + // ISO_IEC_14496-10-AVC-2003.pdf, page 44. + // 7: SPS, 8: PPS, 5: I Frame, 1: P Frame + SrsHevcNaluType nal_unit_type = (SrsHevcNaluType)((frame[0] & 0x7E) >> 1); + + return nal_unit_type == SrsHevcNaluType_VPS; +} + +bool SrsRawHEVCStream::is_sps(char *frame, int nb_frame) +{ + srs_assert(nb_frame > 0); + + // 5bits, 7.3.1 NAL unit syntax, + // ISO_IEC_14496-10-AVC-2003.pdf, page 44. + // 7: SPS, 8: PPS, 5: I Frame, 1: P Frame + SrsHevcNaluType nal_unit_type = (SrsHevcNaluType)((frame[0] & 0x7E) >> 1); + + return nal_unit_type == SrsHevcNaluType_SPS; +} + +bool SrsRawHEVCStream::is_pps(char *frame, int nb_frame) +{ + srs_assert(nb_frame > 0); + + // 5bits, 7.3.1 NAL unit syntax, + // ISO_IEC_14496-10-AVC-2003.pdf, page 44. + // 7: SPS, 8: PPS, 5: I Frame, 1: P Frame + SrsHevcNaluType nal_unit_type = (SrsHevcNaluType)((frame[0] & 0x7E) >> 1); + + return nal_unit_type == SrsHevcNaluType_PPS; +} + +srs_error_t SrsRawHEVCStream::vps_demux(char *frame, int nb_frame, std::string &vps) +{ + srs_error_t err = srs_success; + + if (nb_frame <= 0) { + return srs_error_new(ERROR_STREAM_CASTER_HEVC_VPS, "no hevc vps"); + } + + vps = string(frame, nb_frame); + + return err; +} + +srs_error_t SrsRawHEVCStream::sps_demux(char *frame, int nb_frame, std::string &sps) +{ + srs_error_t err = srs_success; + + // atleast 1bytes for SPS to decode the type, profile, constrain and level. + if (nb_frame < 4) { + return err; + } + + sps = string(frame, nb_frame); + + return err; +} + +srs_error_t SrsRawHEVCStream::pps_demux(char *frame, int nb_frame, std::string &pps) +{ + srs_error_t err = srs_success; + + if (nb_frame <= 0) { + return srs_error_new(ERROR_STREAM_CASTER_HEVC_PPS, "no hevc pps"); + } + + pps = string(frame, nb_frame); + + return err; +} + +srs_error_t SrsRawHEVCStream::mux_sequence_header(std::string vps, std::string sps, std::string pps, std::string &hvcC) +{ + srs_error_t err = srs_success; + + uint8_t temp8bits = 0; + struct mpeg4_hevc_t hevc_info; + + memset(&hevc_info, 0, sizeof(hevc_info)); + + if (h265_vps_copy(&hevc_info, (uint8_t *)vps.data(), (int)vps.length()) < 0) { + return srs_error_new(ERROR_STREAM_CASTER_HEVC_VPS, "decode vps error"); + } + + if (h265_sps_copy(&hevc_info, (uint8_t *)sps.data(), (int)sps.length()) < 0) { + return srs_error_new(ERROR_STREAM_CASTER_HEVC_SPS, "decode sps error"); + } + + if (h265_pps_copy(&hevc_info, (uint8_t *)pps.data(), (int)pps.length()) < 0) { + return srs_error_new(ERROR_STREAM_CASTER_HEVC_PPS, "decode pps error"); + } + + // hevc header information: + // 23bytes header: + // configurationVersion, general_profile_space, general_tier_flag, general_profile_idc + // general_profile_compatibility_flags, general_constraint_indicator_flags, + // general_level_idc, min_spatial_segmentation_idc, parallelismType, + // chromaFormat, bitDepthLumaMinus8, bitDepthChromaMinus8, + // avgFrameRate, constantFrameRate, numTemporalLayers, temporalIdNested, + // lengthSizeMinusOne, numOfArrays + // 5bytes size of vps/sps/pps: + // array_completeness, nal_unit_type, numNalus, nalUnitLength, + // Nbytes of vps/sps/pps. + // sequenceParameterSetNALUnit + + // use simple mode: nalu size + nalu data + int nb_packet = 23 + 5 + (int)vps.length() + 5 + (int)sps.length() + 5 + (int)pps.length(); + char *packet = new char[nb_packet]; + SrsAutoFreeA(char, packet); + + // use stream to generate the hevc packet. + SrsBuffer stream(packet, nb_packet); + + hevc_info.configurationVersion = 1; + stream.write_1bytes(hevc_info.configurationVersion); + + temp8bits = 0; + temp8bits |= ((hevc_info.general_profile_space << 6) & 0xc0); + temp8bits |= ((hevc_info.general_tier_flag << 5) & 0x20); + temp8bits |= hevc_info.general_profile_idc & 0x1f; + stream.write_1bytes(temp8bits); + + stream.write_4bytes(hevc_info.general_profile_compatibility_flags); + + stream.write_2bytes((hevc_info.general_constraint_indicator_flags >> 32) & 0xffff); + stream.write_4bytes(hevc_info.general_constraint_indicator_flags & 0xffffffff); + + stream.write_1bytes(hevc_info.general_level_idc); + + stream.write_2bytes(0xf000 | (hevc_info.min_spatial_segmentation_idc & 0x0fff)); + + stream.write_1bytes(0xfc | (hevc_info.parallelismType & 0x03)); + stream.write_1bytes(0xfc | (hevc_info.chromaFormat & 0x03)); + + stream.write_1bytes(0xf8 | (hevc_info.bitDepthLumaMinus8 & 0x07)); + + stream.write_1bytes(0xf8 | (hevc_info.bitDepthChromaMinus8 & 0x07)); + + stream.write_2bytes(hevc_info.avgFrameRate); + + hevc_info.lengthSizeMinusOne = 3; + temp8bits = 0; + temp8bits |= (hevc_info.constantFrameRate << 6) | 0xc0; + temp8bits |= (hevc_info.numTemporalLayers << 3) | 0x38; + temp8bits |= (hevc_info.temporalIdNested << 2) | 0x04; + temp8bits |= hevc_info.lengthSizeMinusOne & 0x03; + + stream.write_1bytes(temp8bits); + + uint8_t numOfArrays = 3; // vps,sps,pps + stream.write_1bytes(numOfArrays); + + uint8_t array_completeness = 0; // 1bit + // uint8_t reserved = 0;//1bit + uint8_t nal_unit_type = 0; // 6bits; + + // vps + nal_unit_type = ((array_completeness << 7) & 0x80) | (SrsHevcNaluType_VPS & 0x3f); + stream.write_1bytes(nal_unit_type); + + uint16_t namNalus = 1; + stream.write_2bytes(namNalus); + + uint16_t nalUnitLength = vps.length(); + stream.write_2bytes(nalUnitLength); + + stream.write_string(vps); + + // sps + nal_unit_type = ((array_completeness << 7) & 0x80) | (SrsHevcNaluType_SPS & 0x3f); + stream.write_1bytes(nal_unit_type); + + namNalus = 1; + stream.write_2bytes(namNalus); + + nalUnitLength = sps.length(); + stream.write_2bytes(nalUnitLength); + + stream.write_string(sps); + + // pps + nal_unit_type = ((array_completeness << 7) & 0x80) | (SrsHevcNaluType_PPS & 0x3f); + stream.write_1bytes(nal_unit_type); + + namNalus = 1; + stream.write_2bytes(namNalus); + + nalUnitLength = pps.length(); + stream.write_2bytes(nalUnitLength); + stream.write_string(pps); + + hvcC = string(packet, nb_packet); + + return err; +} + +srs_error_t SrsRawHEVCStream::mux_ipb_frame(char *frame, int nb_frame, std::string &ibp) +{ + srs_error_t err = srs_success; + + // 4bytes size of nalu: + // NALUnitLength + // Nbytes of nalu. + // NALUnit + int nb_packet = 4 + nb_frame; + char *packet = new char[nb_packet]; + SrsAutoFreeA(char, packet); + + // use stream to generate the h264 packet. + SrsBuffer stream(packet, nb_packet); + + // 5.3.4.2.1 Syntax, ISO_IEC_14496-15-AVC-format-2012.pdf, page 16 + // lengthSizeMinusOne, or NAL_unit_length, always use 4bytes size + uint32_t NAL_unit_length = nb_frame; + + // mux the avc NALU in "ISO Base Media File Format" + // from ISO_IEC_14496-15-AVC-format-2012.pdf, page 20 + // NALUnitLength + stream.write_4bytes(NAL_unit_length); + // NALUnit + stream.write_bytes(frame, nb_frame); + + ibp = string(packet, nb_packet); + + return err; +} + +srs_error_t SrsRawHEVCStream::mux_avc2flv(std::string video, int8_t frame_type, int8_t avc_packet_type, uint32_t dts, uint32_t pts, char **flv, int *nb_flv) +{ + srs_error_t err = srs_success; + + // for h264 in RTMP video payload, there is 5bytes header: + // 1bytes, FrameType | CodecID + // 1bytes, AVCPacketType + // 3bytes, CompositionTime, the cts. + // @see: E.4.3 Video Tags, video_file_format_spec_v10_1.pdf, page 78 + int size = (int)video.length() + 5; + char *data = new char[size]; + char *p = data; + + // @see: E.4.3 Video Tags, video_file_format_spec_v10_1.pdf, page 78 + // Frame Type, Type of video frame. + // CodecID, Codec Identifier. + // set the rtmp header + *p++ = (frame_type << 4) | SrsVideoCodecIdHEVC; + + // AVCPacketType + *p++ = avc_packet_type; + + // CompositionTime + // pts = dts + cts, or + // cts = pts - dts. + // where cts is the header in rtmp video packet payload header. + uint32_t cts = pts - dts; + char *pp = (char *)&cts; + *p++ = pp[2]; + *p++ = pp[1]; + *p++ = pp[0]; + + // hevc raw data. + memcpy(p, video.data(), video.length()); + + *flv = data; + *nb_flv = size; + + return err; +} +#endif + SrsRawAacStream::SrsRawAacStream() { } diff --git a/trunk/src/protocol/srs_protocol_raw_avc.hpp b/trunk/src/protocol/srs_protocol_raw_avc.hpp index b352ecf16d..d8239e2b04 100644 --- a/trunk/src/protocol/srs_protocol_raw_avc.hpp +++ b/trunk/src/protocol/srs_protocol_raw_avc.hpp @@ -53,6 +53,51 @@ class SrsRawH264Stream virtual srs_error_t mux_avc2flv(std::string video, int8_t frame_type, int8_t avc_packet_type, uint32_t dts, uint32_t pts, char** flv, int* nb_flv); }; +#ifdef SRS_H265 +// The raw h.265 stream, in annexb. +class SrsRawHEVCStream +{ +public: + SrsRawHEVCStream(); + virtual ~SrsRawHEVCStream(); + +public: + // Demux the stream in annexb format. + // @param stream the input stream bytes. + // @param pframe the output hevc frame in stream. user should never free it. + // @param pnb_frame the output hevc frame size. + virtual srs_error_t annexb_demux(SrsBuffer *stream, char **pframe, int *pnb_frame); + // whether the frame is sps or pps or vps. + virtual bool is_sps(char *frame, int nb_frame); + virtual bool is_pps(char *frame, int nb_frame); + virtual bool is_vps(char *frame, int nb_frame); + + // Demux the sps or pps or vps to string. + // @param sps/pps output the sps/pps/vps. + virtual srs_error_t sps_demux(char *frame, int nb_frame, std::string &sps); + virtual srs_error_t pps_demux(char *frame, int nb_frame, std::string &pps); + virtual srs_error_t vps_demux(char *frame, int nb_frame, std::string &vps); + +public: + // The hevc raw data to hevc packet, without flv payload header. + // Mux the sps/pps/vps to flv sequence header packet. + // @param sh output the sequence header. + virtual srs_error_t mux_sequence_header(std::string vps, std::string sps, std::string pps, std::string &sh); + // The hevc raw data to hevc packet, without flv payload header. + // Mux the ibp to flv ibp packet. + // @param ibp output the packet. + // @param frame_type output the frame type. + virtual srs_error_t mux_ipb_frame(char *frame, int nb_frame, std::string &ibp); + // Mux the hevc video packet to flv video packet. + // @param frame_type, SrsVideoAvcFrameTypeKeyFrame or SrsVideoAvcFrameTypeInterFrame. + // @param avc_packet_type, SrsVideoAvcFrameTraitSequenceHeader or SrsVideoAvcFrameTraitNALU. + // @param video the hevc raw data. + // @param flv output the muxed flv packet. + // @param nb_flv output the muxed flv size. + virtual srs_error_t mux_avc2flv(std::string video, int8_t frame_type, int8_t avc_packet_type, uint32_t dts, uint32_t pts, char **flv, int *nb_flv); +}; +#endif + // The header of adts sample. struct SrsRawAacStreamCodec {