diff --git a/trunk/conf/full.conf b/trunk/conf/full.conf index 0579788fac..361c5d7229 100644 --- a/trunk/conf/full.conf +++ b/trunk/conf/full.conf @@ -1787,6 +1787,13 @@ vhost hls.srs.com { # default: off enabled on; + # whether to use fmp4 as container + # The default value is off, then HLS use ts as container format, + # if on, HLS use fmp4 as container format. + # Overwrite by env SRS_VHOST_HLS_HLS_USE_FMP4 for all vhosts. + # default: off + hls_use_fmp4 on; + # the hls fragment in seconds, the duration of a piece of ts. # Overwrite by env SRS_VHOST_HLS_HLS_FRAGMENT for all vhosts. # default: 10 @@ -1852,6 +1859,26 @@ vhost hls.srs.com { # Overwrite by env SRS_VHOST_HLS_HLS_TS_FILE for all vhosts. # default: [app]/[stream]-[seq].ts hls_ts_file [app]/[stream]-[seq].ts; + # the hls fmp4 file name. + # we supports some variables to generate the filename. + # [vhost], the vhost of stream. + # [app], the app of stream. + # [stream], the stream name of stream. + # [2006], replace this const to current year. + # [01], replace this const to current month. + # [02], replace this const to current date. + # [15], replace this const to current hour. + # [04], replace this const to current minute. + # [05], replace this const to current second.p + # [999], replace this const to current millisecond. + # [timestamp],replace this const to current UNIX timestamp in ms. + # [seq], the sequence number of fmp4. + # [duration], replace this const to current ts duration. + # @see https://ossrs.net/lts/zh-cn/docs/v4/doc/dvr#custom-path + # @see https://ossrs.net/lts/zh-cn/docs/v4/doc/delivery-hls#hls-config + # Overwrite by env SRS_VHOST_HLS_HLS_FMP4_FILE for all vhosts. + # default: [app]/[stream]-[seq].m4s + hls_fmp4_file [app]/[stream]-[seq].m4s; # the hls entry prefix, which is base url of ts url. # for example, the prefix is: # http://your-server/ diff --git a/trunk/conf/hls.mp4.conf b/trunk/conf/hls.mp4.conf new file mode 100644 index 0000000000..b957ca2a26 --- /dev/null +++ b/trunk/conf/hls.mp4.conf @@ -0,0 +1,22 @@ +# the config for srs to delivery hls +# @see https://ossrs.net/lts/zh-cn/docs/v4/doc/sample-hls +# @see full.conf for detail config. + +listen 1935; +max_connections 1000; +daemon off; +srs_log_tank console; +http_server { + enabled on; + listen 8080; + dir ./objs/nginx/html; +} +vhost __defaultVhost__ { + hls { + enabled on; + hls_use_fmp4 on; + hls_path ./objs/nginx/html; + hls_fragment 10; + hls_window 60; + } +} diff --git a/trunk/src/app/srs_app_config.cpp b/trunk/src/app/srs_app_config.cpp index 0731f3cd2b..c3da69efcb 100644 --- a/trunk/src/app/srs_app_config.cpp +++ b/trunk/src/app/srs_app_config.cpp @@ -2683,7 +2683,7 @@ srs_error_t SrsConfig::check_normal_config() && m != "hls_storage" && m != "hls_mount" && m != "hls_td_ratio" && m != "hls_aof_ratio" && m != "hls_acodec" && m != "hls_vcodec" && m != "hls_m3u8_file" && m != "hls_ts_file" && m != "hls_ts_floor" && m != "hls_cleanup" && m != "hls_nb_notify" && m != "hls_wait_keyframe" && m != "hls_dispose" && m != "hls_keys" && m != "hls_fragments_per_key" && m != "hls_key_file" - && m != "hls_key_file_path" && m != "hls_key_url" && m != "hls_dts_directly" && m != "hls_ctx" && m != "hls_ts_ctx") { + && m != "hls_key_file_path" && m != "hls_key_url" && m != "hls_dts_directly" && m != "hls_ctx" && m != "hls_ts_ctx" && m != "hls_use_fmp4" && m != "hls_fmp4_file") { return srs_error_new(ERROR_SYSTEM_CONFIG_INVALID, "illegal vhost.hls.%s of %s", m.c_str(), vhost->arg0().c_str()); } @@ -6936,6 +6936,31 @@ bool SrsConfig::get_hls_enabled(SrsConfDirective* vhost) return SRS_CONF_PREFER_FALSE(conf->arg0()); } +bool SrsConfig::get_hls_use_fmp4(std::string vhost) +{ + SRS_OVERWRITE_BY_ENV_BOOL("srs.vhost.hls.hls_use_fmp4"); // SRS_VHOST_HLS_HLS_USE_FMP4 + + static bool DEFAULT = false; + + SrsConfDirective* conf = get_vhost(vhost); + if (!conf) { + return DEFAULT; + } + + conf = conf->get("hls"); + + if (!conf) { + return DEFAULT; + } + + conf = conf->get("hls_use_fmp4"); + if (!conf || conf->arg0().empty()) { + return DEFAULT; + } + + return SRS_CONF_PREFER_FALSE(conf->arg0()); +} + string SrsConfig::get_hls_entry_prefix(string vhost) { SRS_OVERWRITE_BY_ENV_STRING("srs.vhost.hls.hls_entry_prefix"); // SRS_VHOST_HLS_HLS_ENTRY_PREFIX @@ -7012,6 +7037,25 @@ string SrsConfig::get_hls_ts_file(string vhost) return conf->arg0(); } +string SrsConfig::get_hls_fmp4_file(std::string vhost) +{ + SRS_OVERWRITE_BY_ENV_STRING("srs.vhost.hls.hls_fmp4_file"); // SRS_VHOST_HLS_HLS_FMP4_FILE + + static string DEFAULT = "[app]/[stream]-[seq].m4s"; + + SrsConfDirective* conf = get_hls(vhost); + if (!conf) { + return DEFAULT; + } + + conf = conf->get("hls_fmp4_file"); + if (!conf || conf->arg0().empty()) { + return DEFAULT; + } + + return conf->arg0(); +} + bool SrsConfig::get_hls_ts_floor(string vhost) { SRS_OVERWRITE_BY_ENV_BOOL("srs.vhost.hls.hls_ts_floor"); // SRS_VHOST_HLS_HLS_TS_FLOOR diff --git a/trunk/src/app/srs_app_config.hpp b/trunk/src/app/srs_app_config.hpp index 28aec179db..c06b3a3941 100644 --- a/trunk/src/app/srs_app_config.hpp +++ b/trunk/src/app/srs_app_config.hpp @@ -933,6 +933,8 @@ class SrsConfig // Whether HLS is enabled. virtual bool get_hls_enabled(std::string vhost); virtual bool get_hls_enabled(SrsConfDirective* vhost); + // Whether HLS use fmp4 container format + virtual bool get_hls_use_fmp4(std::string vhost); // Get the HLS m3u8 list ts segment entry prefix info. virtual std::string get_hls_entry_prefix(std::string vhost); // Get the HLS ts/m3u8 file store path. @@ -941,6 +943,8 @@ class SrsConfig virtual std::string get_hls_m3u8_file(std::string vhost); // Get the HLS ts file path template. virtual std::string get_hls_ts_file(std::string vhost); + // Get the HLS fmp4 file path template. + virtual std::string get_hls_fmp4_file(std::string vhost); // Whether enable the floor(timestamp/hls_fragment) for variable timestamp. virtual bool get_hls_ts_floor(std::string vhost); // Get the hls fragment time, in srs_utime_t. diff --git a/trunk/src/app/srs_app_hls.cpp b/trunk/src/app/srs_app_hls.cpp index 9bc9bdfcb6..d0a13cec32 100644 --- a/trunk/src/app/srs_app_hls.cpp +++ b/trunk/src/app/srs_app_hls.cpp @@ -31,6 +31,7 @@ using namespace std; #include #include #include +#include #include #include #include @@ -76,6 +77,123 @@ srs_error_t SrsHlsSegment::rename() return SrsFragment::rename(); } +SrsInitMp4Segment::SrsInitMp4Segment() +{ + fw_ = new SrsFileWriter(); + init_ = new SrsMp4M2tsInitEncoder(); +} + +SrsInitMp4Segment::~SrsInitMp4Segment() +{ + srs_freep(init_); + srs_freep(fw_); +} + +srs_error_t SrsInitMp4Segment::write(SrsFormat* format, int v_tid, int a_tid) +{ + srs_error_t err = srs_success; + + string path_tmp = tmppath(); + if ((err = fw_->open(path_tmp)) != srs_success) { + return srs_error_wrap(err, "Open init mp4 failed, path=%s", path_tmp.c_str()); + } + + if ((err = init_->initialize(fw_)) != srs_success) { + return srs_error_wrap(err, "init"); + } + + if ((err = init_->write(format, v_tid, a_tid)) != srs_success) { + return srs_error_wrap(err, "write init"); + } + + return err; +} + +SrsHlsM4sSegment::SrsHlsM4sSegment() +{ + fw_ = new SrsFileWriter(); + enc_ = new SrsFmp4SegmentEncoder(); +} + +SrsHlsM4sSegment::~SrsHlsM4sSegment() +{ + srs_freep(enc_); + srs_freep(fw_); +} + +srs_error_t SrsHlsM4sSegment::initialize(int64_t time, uint32_t v_tid, uint32_t a_tid, int sequence_number, std::string m4s_path) +{ + srs_error_t err = srs_success; + + set_path(m4s_path); + + set_number(sequence_number); + if ((err = create_dir()) != srs_success) { + return srs_error_wrap(err, "create dir"); + } + + if ((err = fw_->open(tmppath())) != srs_success) { + return srs_error_wrap(err, "fw open"); + } + + if ((err = enc_->initialize(fw_, sequence_number, time, v_tid, a_tid)) != srs_success) + { + return srs_error_wrap(err, "initialize SrsFmp4SegmentEncoder"); + } + + return err; +} + +srs_error_t SrsHlsM4sSegment::write(SrsSharedPtrMessage* shared_msg, SrsFormat* format) +{ + srs_error_t err = srs_success; + + if (shared_msg->is_audio()) { + uint8_t* sample = (uint8_t*)format->raw; + uint32_t nb_sample = (uint32_t)format->nb_raw; + + uint32_t dts = (uint32_t)shared_msg->timestamp; + if ((err = enc_->write_sample(SrsMp4HandlerTypeSOUN, 0x00, dts, dts, sample, nb_sample)) != srs_success) { + return srs_error_wrap(err, "m4s segment write audio sample"); + } + } else if (shared_msg->is_video()) { + SrsVideoAvcFrameType frame_type = format->video->frame_type; + uint32_t cts = (uint32_t)format->video->cts; + + uint32_t dts = (uint32_t)shared_msg->timestamp; + uint32_t pts = dts + cts; + + uint8_t* sample = (uint8_t*)format->raw; + uint32_t nb_sample = (uint32_t)format->nb_raw; + if ((err = enc_->write_sample(SrsMp4HandlerTypeVIDE, frame_type, dts, pts, sample, nb_sample)) != srs_success) { + return srs_error_wrap(err, "m4s segment write video sample"); + } + } else { + return err; + } + + append(shared_msg->timestamp); + + return err; +} + +srs_error_t SrsHlsM4sSegment::reap(uint64_t& dts) +{ + srs_error_t err = srs_success; + + if ((err = enc_->flush(dts)) != srs_success) { + return srs_error_wrap(err, "Flush encoder failed"); + } + + srs_freep(fw_); + + if ((err = rename()) != srs_success) { + return srs_error_wrap(err, "rename"); + } + + return err; +} + SrsDvrAsyncCallOnHls::SrsDvrAsyncCallOnHls(SrsContextId c, SrsRequest* r, string p, string t, string m, string mu, int s, srs_utime_t d) { req = r->copy(); @@ -151,37 +269,633 @@ srs_error_t SrsDvrAsyncCallOnHlsNotify::call() return err; } - // the http hooks will cause context switch, - // so we must copy all hooks for the on_connect may freed. - // @see https://github.com/ossrs/srs/issues/475 - vector hooks; + // the http hooks will cause context switch, + // so we must copy all hooks for the on_connect may freed. + // @see https://github.com/ossrs/srs/issues/475 + vector hooks; + + if (true) { + SrsConfDirective* conf = _srs_config->get_vhost_on_hls_notify(req->vhost); + + if (!conf) { + return err; + } + + hooks = conf->args; + } + + int nb_notify = _srs_config->get_vhost_hls_nb_notify(req->vhost); + for (int i = 0; i < (int)hooks.size(); i++) { + std::string url = hooks.at(i); + if ((err = SrsHttpHooks::on_hls_notify(cid, url, req, ts_url, nb_notify)) != srs_success) { + return srs_error_wrap(err, "callback on_hls_notify %s", url.c_str()); + } + } + + return err; +} + +string SrsDvrAsyncCallOnHlsNotify::to_string() +{ + return "on_hls_notify: " + ts_url; +} + +SrsHlsFmp4Muxer::SrsHlsFmp4Muxer() +{ + req_ = NULL; + hls_fragment_ = hls_window_ = 0; + hls_aof_ratio_ = 1.0; + deviation_ts_ = 0; + hls_cleanup_ = true; + hls_wait_keyframe_ = true; + previous_floor_ts_ = 0; + accept_floor_ts_ = 0; + hls_ts_floor_ = false; + max_td_ = 0; + writer_ = NULL; + sequence_no_ = 0; + current_ = NULL; + hls_keys_ = false; + hls_fragments_per_key_ = 0; + async_ = new SrsAsyncCallWorker(); + segments_ = new SrsFragmentWindow(); + latest_acodec_ = SrsAudioCodecIdForbidden; + latest_vcodec_ = SrsVideoCodecIdForbidden; + video_track_id_ = 0; + audio_track_id_ = 0; + init_mp4_ready_ = false; + video_dts_ = 0; + + memset(key_, 0, 16); + memset(iv_, 0, 16); +} + +SrsHlsFmp4Muxer::~SrsHlsFmp4Muxer() +{ + srs_freep(segments_); + srs_freep(current_); + srs_freep(req_); + srs_freep(async_); + srs_freep(writer_); +} + +void SrsHlsFmp4Muxer::dispose() +{ + srs_error_t err = srs_success; + + segments_->dispose(); + + if (current_) { + if ((err = current_->unlink_tmpfile()) != srs_success) { + srs_warn("Unlink tmp ts failed %s", srs_error_desc(err).c_str()); + srs_freep(err); + } + srs_freep(current_); + } + + if (unlink(m3u8_.c_str()) < 0) { + srs_warn("dispose unlink path failed. file=%s", m3u8_.c_str()); + } + + srs_trace("gracefully dispose hls %s", req_ ? req_->get_stream_url().c_str() : ""); +} + +int SrsHlsFmp4Muxer::sequence_no() +{ + return sequence_no_; +} + +std::string SrsHlsFmp4Muxer::ts_url() +{ + // return current_ ? current_->uri : ""; + // TODO: impl segment url for fmp4 segment. + return ""; +} + +srs_utime_t SrsHlsFmp4Muxer::duration() +{ + return current_ ? current_->duration() : 0; +} + +int SrsHlsFmp4Muxer::deviation() +{ + // no floor, no deviation. + if (!hls_ts_floor_) { + return 0; + } + + return deviation_ts_; +} + +SrsAudioCodecId SrsHlsFmp4Muxer::latest_acodec() +{ + return latest_acodec_; +} + +void SrsHlsFmp4Muxer::set_latest_acodec(SrsAudioCodecId v) +{ + latest_acodec_ = v; +} + +SrsVideoCodecId SrsHlsFmp4Muxer::latest_vcodec() +{ + return latest_vcodec_; +} + +void SrsHlsFmp4Muxer::set_latest_vcodec(SrsVideoCodecId v) +{ + latest_vcodec_ = v; +} + +srs_error_t SrsHlsFmp4Muxer::initialize(int v_tid, int a_tid) +{ + srs_error_t err = srs_success; + + video_track_id_ = v_tid; + audio_track_id_ = a_tid; + + return err; +} + +srs_error_t SrsHlsFmp4Muxer::on_publish(SrsRequest* req) +{ + srs_error_t err = srs_success; + + if ((err = async_->start()) != srs_success) { + return srs_error_wrap(err, "async start"); + } + + return err; +} + +srs_error_t SrsHlsFmp4Muxer::write_init_mp4(SrsFormat* format) +{ + srs_error_t err = srs_success; + + std::string vhost = req_->vhost; + std::string stream = req_->stream; + std::string app = req_->app; + std::string path = _srs_config->get_hls_path(vhost); + + path = path + "/" + app + "/" + stream; + if ((err = srs_create_dir_recursively(path)) != srs_success) { + return srs_error_wrap(err, "Create media home failed, home=%s", path.c_str()); + } + + path += "/init.mp4"; + + SrsUniquePtr init_mp4(new SrsInitMp4Segment()); + + init_mp4->set_path(path); + + if ((err = init_mp4->write(format, video_track_id_, audio_track_id_)) != srs_success) { + return srs_error_wrap(err, "write hls init.mp4 with audio and video"); + } + + if ((err = init_mp4->rename()) != srs_success) { + return srs_error_wrap(err, "rename hls init.mp4"); + } + + init_mp4_ready_ = true; + return err; +} + +srs_error_t SrsHlsFmp4Muxer::write_audio(SrsSharedPtrMessage* shared_audio, SrsFormat* format) +{ + srs_error_t err = srs_success; + + // audio_dts_ = shared_audio->timestamp; + + if (!current_) { + if ((err = segment_open(shared_audio->timestamp * SRS_UTIME_MILLISECONDS)) != srs_success) { + return srs_error_wrap(err, "open segment"); + } + } + + current_->write(shared_audio, format); + return err; +} + +srs_error_t SrsHlsFmp4Muxer::write_video(SrsSharedPtrMessage* shared_video, SrsFormat* format) +{ + srs_error_t err = srs_success; + + video_dts_ = shared_video->timestamp; + + if (!current_) { + if ((err = segment_open(shared_video->timestamp * SRS_UTIME_MILLISECONDS)) != srs_success) { + return srs_error_wrap(err, "open segment"); + } + } + + bool reopen = format->video->frame_type == SrsVideoAvcFrameTypeKeyFrame && current_->duration() >= hls_fragment_; + if (reopen) { + if ((err = segment_close()) != srs_success) { + return srs_error_wrap(err, "segment close"); + } + + if ((err = segment_open(shared_video->timestamp * SRS_UTIME_MILLISECONDS)) != srs_success) { + return srs_error_wrap(err, "open segment"); + } + } + + // TODO: do reap segment here. + current_->write(shared_video, format); + + return err; +} + +srs_error_t SrsHlsFmp4Muxer::on_unpublish() +{ + async_->stop(); + return srs_success; +} + +srs_error_t SrsHlsFmp4Muxer::update_config(SrsRequest* r) +{ + srs_error_t err = srs_success; + + srs_freep(req_); + req_ = r->copy(); + + std::string vhost = req_->vhost; + std::string stream = req_->stream; + std::string app = req_->app; + + hls_fragment_ = _srs_config->get_hls_fragment(vhost); + double hls_td_ratio = _srs_config->get_hls_td_ratio(vhost); + hls_window_ = _srs_config->get_hls_window(vhost); + + // get the hls m3u8 ts list entry prefix config + hls_entry_prefix_ = _srs_config->get_hls_entry_prefix(vhost); + // get the hls path config + hls_path_ = _srs_config->get_hls_path(vhost); + m3u8_url_ = _srs_config->get_hls_m3u8_file(vhost); + hls_m4s_file_ = _srs_config->get_hls_fmp4_file(vhost); + hls_cleanup_ = _srs_config->get_hls_cleanup(vhost); + hls_wait_keyframe_ = _srs_config->get_hls_wait_keyframe(vhost); + // the audio overflow, for pure audio to reap segment. + hls_aof_ratio_ = _srs_config->get_hls_aof_ratio(vhost); + // whether use floor(timestamp/hls_fragment) for variable timestamp + hls_ts_floor_ = _srs_config->get_hls_ts_floor(vhost); + + hls_keys_ = _srs_config->get_hls_keys(vhost); + hls_fragments_per_key_ = _srs_config->get_hls_fragments_per_key(vhost); + hls_key_file_ = _srs_config->get_hls_key_file(vhost); + hls_key_file_path_ = _srs_config->get_hls_key_file_path(vhost); + hls_key_url_ = _srs_config->get_hls_key_url(vhost); + + previous_floor_ts_ = 0; + accept_floor_ts_ = 0; + deviation_ts_ = 0; + + // generate the m3u8 dir and path. + m3u8_url_ = srs_path_build_stream(m3u8_url_, vhost, app, stream); + m3u8_ = hls_path_ + "/" + m3u8_url_; + + // when update config, reset the history target duration. + max_td_ = hls_fragment_ * hls_td_ratio; + + // create m3u8 dir once. + m3u8_dir_ = srs_path_dirname(m3u8_); + if ((err = srs_create_dir_recursively(m3u8_dir_)) != srs_success) { + return srs_error_wrap(err, "create dir"); + } + + if (hls_keys_ && (hls_path_ != hls_key_file_path_)) { + string key_file = srs_path_build_stream(hls_key_file_, vhost, app, stream); + string key_url = hls_key_file_path_ + "/" + key_file; + string key_dir = srs_path_dirname(key_url); + if ((err = srs_create_dir_recursively(key_dir)) != srs_success) { + return srs_error_wrap(err, "create dir"); + } + } + + if(hls_keys_) { + writer_ = new SrsEncFileWriter(); + } else { + writer_ = new SrsFileWriter(); + } + + return err; +} + +srs_error_t SrsHlsFmp4Muxer::segment_open(srs_utime_t basetime) +{ + srs_error_t err = srs_success; + + if (current_) { + srs_warn("ignore the segment open, for segment is already open."); + return err; + } + + // new segment. + current_ = new SrsHlsM4sSegment(); + current_->sequence_no = sequence_no_++; + + if ((err = write_hls_key()) != srs_success) { + return srs_error_wrap(err, "write hls key"); + } + + // generate filename. + std::string m4s_file = hls_m4s_file_; + m4s_file = srs_path_build_stream(m4s_file, req_->vhost, req_->app, req_->stream); + if (hls_ts_floor_) { + // accept the floor ts for the first piece. + int64_t current_floor_ts = srs_update_system_time() / hls_fragment_; + if (!accept_floor_ts_) { + accept_floor_ts_ = current_floor_ts - 1; + } else { + accept_floor_ts_++; + } + + // jump when deviation more than 10p + if (accept_floor_ts_ - current_floor_ts > SRS_JUMP_WHEN_PIECE_DEVIATION) { + srs_warn("hls: jmp for ts deviation, current=%" PRId64 ", accept=%" PRId64, current_floor_ts, accept_floor_ts_); + accept_floor_ts_ = current_floor_ts - 1; + } + + // when reap ts, adjust the deviation. + deviation_ts_ = (int)(accept_floor_ts_ - current_floor_ts); + + // dup/jmp detect for ts in floor mode. + if (previous_floor_ts_ && previous_floor_ts_ != current_floor_ts - 1) { + srs_warn("hls: dup/jmp ts, previous=%" PRId64 ", current=%" PRId64 ", accept=%" PRId64 ", deviation=%d", + previous_floor_ts_, current_floor_ts, accept_floor_ts_, deviation_ts_); + } + previous_floor_ts_ = current_floor_ts; + + // we always ensure the piece is increase one by one. + std::stringstream ts_floor; + ts_floor << accept_floor_ts_; + m4s_file = srs_string_replace(m4s_file, "[timestamp]", ts_floor.str()); + + // TODO: FIMXE: we must use the accept ts floor time to generate the hour variable. + m4s_file = srs_path_build_timestamp(m4s_file); + } else { + m4s_file = srs_path_build_timestamp(m4s_file); + } + if (true) { + std::stringstream ss; + ss << current_->sequence_no; + m4s_file = srs_string_replace(m4s_file, "[seq]", ss.str()); + } + current_->set_path(hls_path_ + "/" + m4s_file); + + std::string m4s_path = hls_path_ + "/" + m4s_file; + + // the ts url, relative or absolute url. + // TODO: FIXME: Use url and path manager. + std::string ts_url = current_->fullpath(); + if (srs_string_starts_with(ts_url, m3u8_dir_)) { + ts_url = ts_url.substr(m3u8_dir_.length()); + } + while (srs_string_starts_with(ts_url, "/")) { + ts_url = ts_url.substr(1); + } + // current->uri += hls_entry_prefix; + if (!hls_entry_prefix_.empty() && !srs_string_ends_with(hls_entry_prefix_, "/")) { + // current_->uri += "/"; + + // add the http dir to uri. + string http_dir = srs_path_dirname(m3u8_url_); + if (!http_dir.empty()) { + // current->uri += http_dir + "/"; + } + } + + current_->initialize(basetime, video_track_id_, audio_track_id_, sequence_no_, m4s_path); + + return err; +} + +srs_error_t SrsHlsFmp4Muxer::on_sequence_header() +{ + return srs_success; +} + +bool SrsHlsFmp4Muxer::is_segment_overflow() +{ + srs_assert(current_); + + // to prevent very small segment. + if (current_->duration() < 2 * SRS_HLS_SEGMENT_MIN_DURATION) { + return false; + } + + // Use N% deviation, to smoother. + srs_utime_t deviation = hls_ts_floor_ ? SRS_HLS_FLOOR_REAP_PERCENT * deviation_ts_ * hls_fragment_ : 0; + + // Keep in mind that we use max_td for the base duration, not the hls_fragment. To calculate + // max_td, multiply hls_fragment by hls_td_ratio. + return current_->duration() >= max_td_ + deviation; +} + +bool SrsHlsFmp4Muxer::wait_keyframe() +{ + return hls_wait_keyframe_; +} + +bool SrsHlsFmp4Muxer::is_segment_absolutely_overflow() +{ + srs_assert(current_); + + // to prevent very small segment. + if (current_->duration() < 2 * SRS_HLS_SEGMENT_MIN_DURATION) { + return false; + } + + // use N% deviation, to smoother. + srs_utime_t deviation = hls_ts_floor_? SRS_HLS_FLOOR_REAP_PERCENT * deviation_ts_ * hls_fragment_ : 0; + return current_->duration() >= hls_aof_ratio_ * hls_fragment_ + deviation; +} + +void SrsHlsFmp4Muxer::update_duration(uint64_t dts) +{ + current_->append(dts / 90); +} + +srs_error_t SrsHlsFmp4Muxer::segment_close() +{ + srs_error_t err = do_segment_close(); + + return err; +} + +srs_error_t SrsHlsFmp4Muxer::do_segment_close() +{ + srs_error_t err = srs_success; + + if (!current_) { + srs_warn("ignore the segment close, for segment is not open."); + return err; + } + + if ((err = current_->reap(video_dts_)) != srs_success) { + return srs_error_wrap(err, "reap segment"); + } + + // // use async to call the http hooks, for it will cause thread switch. + // if ((err = async_->execute(new SrsDvrAsyncCallOnHls(_srs_context->get_id(), req_, current_->fullpath(), + // current_->uri, m3u8_, m3u8_url_, current_->sequence_no, current_->duration()))) != srs_success) { + // return srs_error_wrap(err, "segment close"); + // } + + // // use async to call the http hooks, for it will cause thread switch. + // if ((err = async_->execute(new SrsDvrAsyncCallOnHlsNotify(_srs_context->get_id(), req_, current_->uri))) != srs_success) { + // return srs_error_wrap(err, "segment close"); + // } + + segments_->append(current_); + current_ = NULL; + + // shrink the segments. + segments_->shrink(hls_window_); + + // refresh the m3u8, donot contains the removed ts + if ((err = refresh_m3u8()) != srs_success) { + return srs_error_wrap(err, "refresh m3u8"); + } + + // remove the ts file. + segments_->clear_expired(hls_cleanup_); + + return err; +} + +srs_error_t SrsHlsFmp4Muxer::write_hls_key() +{ + return srs_success; +} + +srs_error_t SrsHlsFmp4Muxer::refresh_m3u8() +{ + srs_error_t err = srs_success; + + // no segments, also no m3u8, return. + if (segments_->empty()) { + return err; + } + + std::string temp_m3u8 = m3u8_ + ".temp"; + if ((err = _refresh_m3u8(temp_m3u8)) == srs_success) { + if (rename(temp_m3u8.c_str(), m3u8_.c_str()) < 0) { + err = srs_error_new(ERROR_HLS_WRITE_FAILED, "hls: rename m3u8 file failed. %s => %s", temp_m3u8.c_str(), m3u8_.c_str()); + } + } + + // remove the temp file. + if (srs_path_exists(temp_m3u8)) { + if (unlink(temp_m3u8.c_str()) < 0) { + srs_warn("ignore remove m3u8 failed, %s", temp_m3u8.c_str()); + } + } + + return err; +} + +srs_error_t SrsHlsFmp4Muxer::_refresh_m3u8(std::string m3u8_file) +{ + srs_error_t err = srs_success; + + // no segments, return. + if (segments_->empty()) { + return err; + } + + SrsFileWriter writer; + if ((err = writer.open(m3u8_file)) != srs_success) { + return srs_error_wrap(err, "hls: open m3u8 file %s", m3u8_file.c_str()); + } + + // #EXTM3U\n + // #EXT-X-VERSION:3\n + std::stringstream ss; + ss << "#EXTM3U" << SRS_CONSTS_LF; + // TODO: for fmp4 set #EXT-X-VERSION:7, need support tag #EXT-X-MAP:URI="init.mp4", which + // at least version:5 + // DOC: https://developer.apple.com/documentation/http-live-streaming/about-the-ext-x-version-tag + ss << "#EXT-X-VERSION:7" << SRS_CONSTS_LF; + + // #EXT-X-MEDIA-SEQUENCE:4294967295\n + SrsHlsM4sSegment* first = dynamic_cast(segments_->first()); + if (first == NULL) { + return srs_error_new(ERROR_HLS_WRITE_FAILED, "segments cast"); + } + + ss << "#EXT-X-MEDIA-SEQUENCE:" << first->sequence_no << SRS_CONSTS_LF; + + // #EXT-X-TARGETDURATION:4294967295\n + /** + * @see hls-m3u8-draft-pantos-http-live-streaming-12.pdf, page 25 + * The Media Playlist file MUST contain an EXT-X-TARGETDURATION tag. + * Its value MUST be equal to or greater than the EXTINF duration of any + * media segment that appears or will appear in the Playlist file, + * rounded to the nearest integer. Its value MUST NOT change. A + * typical target duration is 10 seconds. + */ + srs_utime_t max_duration = segments_->max_duration(); + int target_duration = (int)ceil(srsu2msi(srs_max(max_duration, max_td_)) / 1000.0); + + ss << "#EXT-X-TARGETDURATION:" << target_duration << SRS_CONSTS_LF; + + // TODO: add #EXT-X-MAP:URI="init.mp4" for fmp4 + ss << "#EXT-X-MAP:URI=\""<< req_->stream << "/init.mp4\"" << SRS_CONSTS_LF; - if (true) { - SrsConfDirective* conf = _srs_config->get_vhost_on_hls_notify(req->vhost); + // write all segments + for (int i = 0; i < segments_->size(); i++) { + SrsHlsM4sSegment* segment = dynamic_cast(segments_->at(i)); - if (!conf) { - return err; + if (segment->is_sequence_header()) { + // #EXT-X-DISCONTINUITY\n + ss << "#EXT-X-DISCONTINUITY" << SRS_CONSTS_LF; + } + +#if 0 + if(hls_keys_ && ((segment->sequence_no % hls_fragments_per_key_) == 0)) { + char hexiv[33]; + srs_data_to_hex(hexiv, segment->iv, 16); + hexiv[32] = '\0'; + + string key_file = srs_path_build_stream(hls_key_file_, req_->vhost, req_->app, req_->stream); + key_file = srs_string_replace(key_file, "[seq]", srs_int2str(segment->sequence_no)); + + string key_path = key_file; + //if key_url is not set,only use the file name + if (!hls_key_url_.empty()) { + key_path = hls_key_url_ + key_file; + } + + ss << "#EXT-X-KEY:METHOD=AES-128,URI=" << "\"" << key_path << "\",IV=0x" << hexiv << SRS_CONSTS_LF; } +#endif - hooks = conf->args; + // "#EXTINF:4294967295.208,\n" + ss.precision(3); + ss.setf(std::ios::fixed, std::ios::floatfield); + ss << "#EXTINF:" << srsu2msi(segment->duration()) / 1000.0 << ", no desc" << SRS_CONSTS_LF; + + // {file name}\n + // TODO get segment name in relative path. + std::string seg_uri = segment->fullpath(); + if (true) { + std::stringstream stemp; + stemp << srsu2msi(segment->duration()); + seg_uri = srs_string_replace(seg_uri, "[duration]", stemp.str()); + } + //ss << segment->uri << SRS_CONSTS_LF; + ss << srs_path_basename(seg_uri) << SRS_CONSTS_LF; } - int nb_notify = _srs_config->get_vhost_hls_nb_notify(req->vhost); - for (int i = 0; i < (int)hooks.size(); i++) { - std::string url = hooks.at(i); - if ((err = SrsHttpHooks::on_hls_notify(cid, url, req, ts_url, nb_notify)) != srs_success) { - return srs_error_wrap(err, "callback on_hls_notify %s", url.c_str()); - } + // write m3u8 to writer. + std::string m3u8 = ss.str(); + if ((err = writer.write((char*)m3u8.c_str(), (int)m3u8.length(), NULL)) != srs_success) { + return srs_error_wrap(err, "hls: write m3u8"); } return err; } -string SrsDvrAsyncCallOnHlsNotify::to_string() -{ - return "on_hls_notify: " + ts_url; -} - SrsHlsMuxer::SrsHlsMuxer() { req = NULL; @@ -800,6 +1514,9 @@ srs_error_t SrsHlsMuxer::_refresh_m3u8(string m3u8_file) // #EXT-X-VERSION:3\n std::stringstream ss; ss << "#EXTM3U" << SRS_CONSTS_LF; + // TODO: for fmp4 set #EXT-X-VERSION:7, need support tag #EXT-X-MAP:URI="init.mp4", which + // at least version:5 + // DOC: https://developer.apple.com/documentation/http-live-streaming/about-the-ext-x-version-tag ss << "#EXT-X-VERSION:3" << SRS_CONSTS_LF; // #EXT-X-MEDIA-SEQUENCE:4294967295\n @@ -823,6 +1540,8 @@ srs_error_t SrsHlsMuxer::_refresh_m3u8(string m3u8_file) int target_duration = (int)ceil(srsu2msi(srs_max(max_duration, max_td)) / 1000.0); ss << "#EXT-X-TARGETDURATION:" << target_duration << SRS_CONSTS_LF; + + // TODO: add #EXT-X-MAP:URI="init.mp4" for fmp4 // write all segments for (int i = 0; i < segments->size(); i++) { @@ -875,10 +1594,22 @@ srs_error_t SrsHlsMuxer::_refresh_m3u8(string m3u8_file) return err; } +ISrsHlsController::ISrsHlsController() +{ +} + +ISrsHlsController::~ISrsHlsController() +{ +} + SrsHlsController::SrsHlsController() { tsmc = new SrsTsMessageCache(); muxer = new SrsHlsMuxer(); + + hls_dts_directly = false; + previous_audio_dts = 0; + aac_samples = 0; } SrsHlsController::~SrsHlsController() @@ -972,7 +1703,9 @@ srs_error_t SrsHlsController::on_publish(SrsRequest* req) } // This config item is used in SrsHls, we just log its value here. - bool hls_dts_directly = _srs_config->get_vhost_hls_dts_directly(req->vhost); + // If enabled, directly turn FLV timestamp to TS DTS. + // @remark It'll be reloaded automatically, because the origin hub will republish while reloading. + hls_dts_directly = _srs_config->get_vhost_hls_dts_directly(req->vhost); srs_trace("hls: win=%dms, frag=%dms, prefix=%s, path=%s, m3u8=%s, ts=%s, tdr=%.2f, aof=%.2f, floor=%d, clean=%d, waitk=%d, dispose=%dms, dts_directly=%d", srsu2msi(hls_window), srsu2msi(hls_fragment), entry_prefix.c_str(), path.c_str(), m3u8_file.c_str(), ts_file.c_str(), @@ -1000,7 +1733,7 @@ srs_error_t SrsHlsController::on_unpublish() return err; } -srs_error_t SrsHlsController::on_sequence_header() +srs_error_t SrsHlsController::on_sequence_header(SrsSharedPtrMessage* msg, SrsFormat* format) { // TODO: support discontinuity for the same stream // currently we reap and insert discontinity when encoder republish, @@ -1011,10 +1744,50 @@ srs_error_t SrsHlsController::on_sequence_header() return muxer->on_sequence_header(); } -srs_error_t SrsHlsController::write_audio(SrsAudioFrame* frame, int64_t pts) +srs_error_t SrsHlsController::write_audio(SrsSharedPtrMessage* shared_audio, SrsFormat* format) { srs_error_t err = srs_success; + SrsAudioFrame* frame = format->audio; + + // Reset the aac samples counter when DTS jitter. + if (previous_audio_dts > shared_audio->timestamp) { + previous_audio_dts = shared_audio->timestamp; + aac_samples = 0; + } + + // The diff duration in ms between two FLV audio packets. + int diff = ::abs((int)(shared_audio->timestamp - previous_audio_dts)); + previous_audio_dts = shared_audio->timestamp; + + // Guess the number of samples for each AAC frame. + // If samples is 1024, the sample-rate is 8000HZ, the diff should be 1024/8000s=128ms. + // If samples is 1024, the sample-rate is 44100HZ, the diff should be 1024/44100s=23ms. + // If samples is 2048, the sample-rate is 44100HZ, the diff should be 2048/44100s=46ms. + int nb_samples_per_frame = 0; + int guessNumberOfSamples = diff * srs_flv_srates[format->acodec->sound_rate] / 1000; + if (guessNumberOfSamples > 0) { + if (guessNumberOfSamples < 960) { + nb_samples_per_frame = 960; + } else if (guessNumberOfSamples < 1536) { + nb_samples_per_frame = 1024; + } else if (guessNumberOfSamples < 3072) { + nb_samples_per_frame = 2048; + } else { + nb_samples_per_frame = 4096; + } + } + + // Recalc the DTS by the samples of AAC. + aac_samples += nb_samples_per_frame; + int64_t dts = 90000 * aac_samples / srs_flv_srates[format->acodec->sound_rate]; + // If directly turn FLV timestamp, overwrite the guessed DTS. + // @doc https://github.com/ossrs/srs/issues/1506#issuecomment-562063095 + if (hls_dts_directly) { + dts = shared_audio->timestamp * 90; + } + + // Refresh the codec ASAP. if (muxer->latest_acodec() != frame->acodec()->id) { srs_trace("HLS: Switch audio codec %d(%s) to %d(%s)", muxer->latest_acodec(), srs_audio_codec_id2str(muxer->latest_acodec()).c_str(), @@ -1023,7 +1796,7 @@ srs_error_t SrsHlsController::write_audio(SrsAudioFrame* frame, int64_t pts) } // write audio to cache. - if ((err = tsmc->cache_audio(frame, pts)) != srs_success) { + if ((err = tsmc->cache_audio(frame, dts)) != srs_success) { return srs_error_wrap(err, "hls: cache audio"); } @@ -1046,7 +1819,7 @@ srs_error_t SrsHlsController::write_audio(SrsAudioFrame* frame, int64_t pts) // for pure audio, aggregate some frame to one. // TODO: FIXME: Check whether it's necessary. if (muxer->pure_audio() && tsmc->audio) { - if (pts - tsmc->audio->start_pts < SRS_CONSTS_HLS_PURE_AUDIO_AGGREGATE) { + if (dts - tsmc->audio->start_pts < SRS_CONSTS_HLS_PURE_AUDIO_AGGREGATE) { return err; } } @@ -1062,9 +1835,11 @@ srs_error_t SrsHlsController::write_audio(SrsAudioFrame* frame, int64_t pts) return err; } -srs_error_t SrsHlsController::write_video(SrsVideoFrame* frame, int64_t dts) +srs_error_t SrsHlsController::write_video(SrsSharedPtrMessage* shared_video, SrsFormat* format) { srs_error_t err = srs_success; + SrsVideoFrame* frame = format->video; + int64_t dts = shared_video->timestamp * 90; // Refresh the codec ASAP. if (muxer->latest_vcodec() != frame->vcodec()->id) { @@ -1142,6 +1917,160 @@ srs_error_t SrsHlsController::reap_segment() return err; } +SrsHlsMp4Controller::SrsHlsMp4Controller() +{ + has_video_sh_ = false; + has_audio_sh_ = false; + + video_track_id_ = 1; + audio_track_id_ = 2; + + audio_dts_ = 0; + video_dts_ = 0; + + req_ = NULL; + muxer_ = new SrsHlsFmp4Muxer(); +} + +SrsHlsMp4Controller::~SrsHlsMp4Controller() +{ + srs_freep(muxer_); +} + +srs_error_t SrsHlsMp4Controller::initialize() +{ + srs_error_t err = srs_success; + if ((err = muxer_->initialize(video_track_id_, audio_track_id_)) != srs_success) { + return srs_error_wrap(err, "initialize SrsHlsFmp4Muxer"); + } + + return err; +} + +void SrsHlsMp4Controller::dispose() +{ + muxer_->dispose(); +} + +srs_error_t SrsHlsMp4Controller::on_publish(SrsRequest* req) +{ + srs_error_t err = srs_success; + + req_ = req; + std::string vhost = req->vhost; + std::string stream = req->stream; + std::string app = req->app; + + // get the hls m3u8 ts list entry prefix config + std::string entry_prefix = _srs_config->get_hls_entry_prefix(vhost); + // get the hls path config + std::string path = _srs_config->get_hls_path(vhost); + std::string m3u8_file = _srs_config->get_hls_m3u8_file(vhost); + std::string ts_file = _srs_config->get_hls_ts_file(vhost); + + if ((err = muxer_->on_publish(req)) != srs_success) { + return srs_error_wrap(err, "muxer publish"); + } + + if ((err = muxer_->update_config(req)) != srs_success ) { + return srs_error_wrap(err, "hls: update config"); + } + + return err; +} + +srs_error_t SrsHlsMp4Controller::on_unpublish() +{ + srs_error_t err = srs_success; + req_ = NULL; + + if ((err = muxer_->segment_close()) != srs_success) { + return srs_error_wrap(err, "hls: segment close"); + } + + if ((err = muxer_->on_unpublish()) != srs_success) { + return srs_error_wrap(err, "muxer unpublish"); + } + + return err; +} + +srs_error_t SrsHlsMp4Controller::write_audio(SrsSharedPtrMessage* shared_audio, SrsFormat* format) +{ + srs_error_t err = srs_success; + + // Ignore audio sequence header + if (format->is_aac_sequence_header() || format->is_mp3_sequence_header()) { + return err; + } + + audio_dts_ = shared_audio->timestamp; + muxer_->write_audio(shared_audio, format); + + return err; +} + +srs_error_t SrsHlsMp4Controller::write_video(SrsSharedPtrMessage* shared_video, SrsFormat* format) +{ + srs_error_t err = srs_success; + SrsVideoFrame* frame = format->video; + + // Refresh the codec ASAP. + if (muxer_->latest_vcodec() != frame->vcodec()->id) { + srs_trace("HLS: Switch video codec %d(%s) to %d(%s)", muxer_->latest_acodec(), srs_video_codec_id2str(muxer_->latest_vcodec()).c_str(), + frame->vcodec()->id, srs_video_codec_id2str(frame->vcodec()->id).c_str()); + muxer_->set_latest_vcodec(frame->vcodec()->id); + } + + video_dts_ = shared_video->timestamp; + + muxer_->write_video(shared_video, format); + + return err; +} + +srs_error_t SrsHlsMp4Controller::on_sequence_header(SrsSharedPtrMessage* msg, SrsFormat* format) +{ + srs_error_t err = srs_success; + + if (req_ == NULL) { + return srs_error_new(ERROR_HLS_NO_STREAM, "no req yet"); + } + + // TODO: on av sequence header, doing generate the init.mp4? + if (msg->is_video()) { + has_video_sh_ = true; + } + + if (msg->is_audio()) { + has_audio_sh_ = true; + } + + muxer_->write_init_mp4(format); + + return err; +} + +int SrsHlsMp4Controller::sequence_no() +{ + return 0; +} + +std::string SrsHlsMp4Controller::ts_url() +{ + return ""; +} + +srs_utime_t SrsHlsMp4Controller::duration() +{ + return 0; +} + +int SrsHlsMp4Controller::deviation() +{ + return 0; +} + SrsHls::SrsHls() { req = NULL; @@ -1152,13 +2081,10 @@ SrsHls::SrsHls() unpublishing_ = false; async_reload_ = reloading_ = false; last_update_time = 0; - hls_dts_directly = false; - - previous_audio_dts = 0; - aac_samples = 0; jitter = new SrsRtmpJitter(); - controller = new SrsHlsController(); + // TODO: replace NULL by a dummy ISrsHlsController + controller = NULL; pprint = SrsPithyPrint::create_hls(); } @@ -1292,6 +2218,16 @@ srs_error_t SrsHls::initialize(SrsOriginHub* h, SrsRequest* r) hub = h; req = r; + + bool is_fmp4_enabled = _srs_config->get_hls_use_fmp4(r->vhost); + + if (!controller) { + if (is_fmp4_enabled) { + controller = new SrsHlsMp4Controller(); + } else { + controller = new SrsHlsController(); + } + } if ((err = controller->initialize()) != srs_success) { return srs_error_wrap(err, "controller initialize"); @@ -1319,10 +2255,6 @@ srs_error_t SrsHls::on_publish() if ((err = controller->on_publish(req)) != srs_success) { return srs_error_wrap(err, "hls: on publish"); } - - // If enabled, directly turn FLV timestamp to TS DTS. - // @remark It'll be reloaded automatically, because the origin hub will republish while reloading. - hls_dts_directly = _srs_config->get_vhost_hls_dts_directly(req->vhost); // if enabled, open the muxer. enabled = true; @@ -1367,6 +2299,7 @@ srs_error_t SrsHls::on_audio(SrsSharedPtrMessage* shared_audio, SrsFormat* forma // Ignore if no format->acodec, it means the codec is not parsed, or unknown codec. // @issue https://github.com/ossrs/srs/issues/1506#issuecomment-562079474 + // TODO: format->acodec is always not-nil, remove this check. if (!format->acodec) { return err; } @@ -1385,7 +2318,7 @@ srs_error_t SrsHls::on_audio(SrsSharedPtrMessage* shared_audio, SrsFormat* forma // ignore sequence header srs_assert(format->audio); if (acodec == SrsAudioCodecIdAAC && format->audio->aac_packet_type == SrsAudioAacFrameTraitSequenceHeader) { - return controller->on_sequence_header(); + return controller->on_sequence_header(audio.get(), format); } // TODO: FIXME: config the jitter of HLS. @@ -1393,45 +2326,7 @@ srs_error_t SrsHls::on_audio(SrsSharedPtrMessage* shared_audio, SrsFormat* forma return srs_error_wrap(err, "hls: jitter"); } - // Reset the aac samples counter when DTS jitter. - if (previous_audio_dts > audio->timestamp) { - previous_audio_dts = audio->timestamp; - aac_samples = 0; - } - - // The diff duration in ms between two FLV audio packets. - int diff = ::abs((int)(audio->timestamp - previous_audio_dts)); - previous_audio_dts = audio->timestamp; - - // Guess the number of samples for each AAC frame. - // If samples is 1024, the sample-rate is 8000HZ, the diff should be 1024/8000s=128ms. - // If samples is 1024, the sample-rate is 44100HZ, the diff should be 1024/44100s=23ms. - // If samples is 2048, the sample-rate is 44100HZ, the diff should be 2048/44100s=46ms. - int nb_samples_per_frame = 0; - int guessNumberOfSamples = diff * srs_flv_srates[format->acodec->sound_rate] / 1000; - if (guessNumberOfSamples > 0) { - if (guessNumberOfSamples < 960) { - nb_samples_per_frame = 960; - } else if (guessNumberOfSamples < 1536) { - nb_samples_per_frame = 1024; - } else if (guessNumberOfSamples < 3072) { - nb_samples_per_frame = 2048; - } else { - nb_samples_per_frame = 4096; - } - } - - // Recalc the DTS by the samples of AAC. - aac_samples += nb_samples_per_frame; - int64_t dts = 90000 * aac_samples / srs_flv_srates[format->acodec->sound_rate]; - - // If directly turn FLV timestamp, overwrite the guessed DTS. - // @doc https://github.com/ossrs/srs/issues/1506#issuecomment-562063095 - if (hls_dts_directly) { - dts = audio->timestamp * 90; - } - - if ((err = controller->write_audio(format->audio, dts)) != srs_success) { + if ((err = controller->write_audio(audio.get(), format)) != srs_success) { return srs_error_wrap(err, "hls: write audio"); } @@ -1471,7 +2366,7 @@ srs_error_t SrsHls::on_video(SrsSharedPtrMessage* shared_video, SrsFormat* forma // ignore sequence header if (format->video->avc_packet_type == SrsVideoAvcFrameTraitSequenceHeader) { - return controller->on_sequence_header(); + return controller->on_sequence_header(video.get(), format); } // TODO: FIXME: config the jitter of HLS. @@ -1479,8 +2374,7 @@ srs_error_t SrsHls::on_video(SrsSharedPtrMessage* shared_video, SrsFormat* forma return srs_error_wrap(err, "hls: jitter"); } - int64_t dts = video->timestamp * 90; - if ((err = controller->write_video(format->video, dts)) != srs_success) { + if ((err = controller->write_video(video.get(), format)) != srs_success) { return srs_error_wrap(err, "hls: write video"); } diff --git a/trunk/src/app/srs_app_hls.hpp b/trunk/src/app/srs_app_hls.hpp index e0866da252..4bb8d45ae5 100644 --- a/trunk/src/app/srs_app_hls.hpp +++ b/trunk/src/app/srs_app_hls.hpp @@ -32,11 +32,14 @@ class SrsTsAacJitter; class SrsTsMessageCache; class SrsHlsSegment; class SrsTsContext; +class SrsMp4M2tsInitEncoder; +class SrsFmp4SegmentEncoder; // The wrapper of m3u8 segment from specification: // // 3.3.2. EXTINF // The EXTINF tag specifies the duration of a media segment. +// TODO: refactor this to support fmp4 segment. class SrsHlsSegment : public SrsFragment { public: @@ -61,6 +64,40 @@ class SrsHlsSegment : public SrsFragment virtual srs_error_t rename(); }; +class SrsInitMp4Segment : public SrsFragment +{ +private: + SrsFileWriter* fw_; + SrsMp4M2tsInitEncoder* init_; + +public: + SrsInitMp4Segment(); + virtual ~SrsInitMp4Segment(); + +public: + + // Write the init mp4 file, with the v_tid(video track id) and a_tid (audio track id). + virtual srs_error_t write(SrsFormat* format, int v_tid, int a_tid); +}; + +// TODO: merge this code with SrsFragmentedMp4 in dash +class SrsHlsM4sSegment : public SrsFragment +{ +private: + SrsFileWriter* fw_; + SrsFmp4SegmentEncoder* enc_; +public: + // sequence number in m3u8. + int sequence_no; +public: + SrsHlsM4sSegment(); + virtual ~SrsHlsM4sSegment(); + + virtual srs_error_t initialize(int64_t time, uint32_t v_tid, uint32_t a_tid, int sequence_number, std::string m4s_path); + virtual srs_error_t write(SrsSharedPtrMessage* shared_msg, SrsFormat* format); + virtual srs_error_t reap(uint64_t& dts); +}; + // The hls async call: on_hls class SrsDvrAsyncCallOnHls : public ISrsAsyncCallTask { @@ -217,6 +254,154 @@ class SrsHlsMuxer virtual srs_error_t _refresh_m3u8(std::string m3u8_file); }; +// Mux the HLS stream(m3u8 and m4s files). +// Generally, the m3u8 muxer only provides methods to open/close segments, +// to flush video/audio, without any mechenisms. +// +// That is, user must use HlsCache, which will control the methods of muxer, +// and provides HLS mechenisms. +class SrsHlsFmp4Muxer +{ +private: + SrsRequest* req_; +private: + std::string hls_entry_prefix_; + std::string hls_path_; + std::string hls_m4s_file_; + bool hls_cleanup_; + bool hls_wait_keyframe_; + std::string m3u8_dir_; + double hls_aof_ratio_; + // TODO: FIXME: Use TBN 1000. + srs_utime_t hls_fragment_; + srs_utime_t hls_window_; + SrsAsyncCallWorker* async_; +private: + // Whether use floor algorithm for timestamp. + bool hls_ts_floor_; + // The deviation in piece to adjust the fragment to be more + // bigger or smaller. + int deviation_ts_; + // The previous reap floor timestamp, + // used to detect the dup or jmp or ts. + int64_t accept_floor_ts_; + int64_t previous_floor_ts_; + bool init_mp4_ready_; +private: + // Whether encrypted or not + bool hls_keys_; + int hls_fragments_per_key_; + // The key file name + std::string hls_key_file_; + // The key file path + std::string hls_key_file_path_; + // The key file url + std::string hls_key_url_; + // The key and iv. + unsigned char key_[16]; + unsigned char iv_[16]; + // The underlayer file writer. + SrsFileWriter* writer_; +private: + int sequence_no_; + srs_utime_t max_td_; + std::string m3u8_; + std::string m3u8_url_; + int video_track_id_; + int audio_track_id_; + uint64_t video_dts_; +private: + // The available cached segments in m3u8. + SrsFragmentWindow* segments_; + // The current writing segment. + SrsHlsM4sSegment* current_; + +private: + // Latest audio codec, parsed from stream. + SrsAudioCodecId latest_acodec_; + // Latest audio codec, parsed from stream. + SrsVideoCodecId latest_vcodec_; +public: + SrsHlsFmp4Muxer(); + virtual ~SrsHlsFmp4Muxer(); +public: + virtual void dispose(); +public: + virtual int sequence_no(); + virtual std::string ts_url(); + virtual srs_utime_t duration(); + virtual int deviation(); +public: + SrsAudioCodecId latest_acodec(); + void set_latest_acodec(SrsAudioCodecId v); + SrsVideoCodecId latest_vcodec(); + void set_latest_vcodec(SrsVideoCodecId v); +public: + // Initialize the hls muxer. + virtual srs_error_t initialize(int v_tid, int a_tid); + // When publish or unpublish stream. + virtual srs_error_t on_publish(SrsRequest* req); + + virtual srs_error_t write_init_mp4(SrsFormat* format); + virtual srs_error_t write_audio(SrsSharedPtrMessage* shared_audio, SrsFormat* format); + virtual srs_error_t write_video(SrsSharedPtrMessage* shared_video, SrsFormat* format); + + virtual srs_error_t on_unpublish(); + // When publish, update the config for muxer. + virtual srs_error_t update_config(SrsRequest* r); + // Open a new segment(a new ts file) + virtual srs_error_t segment_open(srs_utime_t basetime); + virtual srs_error_t on_sequence_header(); + // Whether segment overflow, + // that is whether the current segment duration>=(the segment in config) + virtual bool is_segment_overflow(); + // Whether wait keyframe to reap the ts. + virtual bool wait_keyframe(); + // Whether segment absolutely overflow, for pure audio to reap segment, + // that is whether the current segment duration>=2*(the segment in config) + virtual bool is_segment_absolutely_overflow(); +public: + // Whether current hls muxer is pure audio mode. +// virtual bool pure_audio(); +// virtual srs_error_t flush_audio(SrsTsMessageCache* cache); +// virtual srs_error_t flush_video(SrsTsMessageCache* cache); + // When flushing video or audio, we update the duration. But, we should also update the + // duration before closing the segment. Keep in mind that it's fine to update the duration + // several times using the same dts timestamp. + void update_duration(uint64_t dts); + // Close segment(ts). + virtual srs_error_t segment_close(); +private: + virtual srs_error_t do_segment_close(); + virtual srs_error_t write_hls_key(); + virtual srs_error_t refresh_m3u8(); + virtual srs_error_t _refresh_m3u8(std::string m3u8_file); +}; + +// The base class for HLS controller +class ISrsHlsController +{ +public: + ISrsHlsController(); + virtual ~ISrsHlsController(); + +public: + virtual srs_error_t initialize() = 0; + virtual void dispose() = 0; + // When publish or unpublish stream. + virtual srs_error_t on_publish(SrsRequest* req) = 0; + virtual srs_error_t on_unpublish() = 0; + + virtual srs_error_t write_audio(SrsSharedPtrMessage* shared_audio, SrsFormat* format) = 0; + virtual srs_error_t write_video(SrsSharedPtrMessage* shared_video, SrsFormat* format) = 0; + + virtual srs_error_t on_sequence_header(SrsSharedPtrMessage* msg, SrsFormat* format) = 0; + virtual int sequence_no() = 0; + virtual std::string ts_url() = 0; + virtual srs_utime_t duration() = 0; + virtual int deviation() = 0; +}; + // The hls stream cache, // use to cache hls stream and flush to hls muxer. // @@ -232,14 +417,23 @@ class SrsHlsMuxer // when timestamp convert to flv tbn, it will loose precise, // so we must gather audio frame together, and recalc the timestamp @see SrsTsAacJitter, // we use a aac jitter to correct the audio pts. -class SrsHlsController +class SrsHlsController : public ISrsHlsController { private: // The HLS muxer to reap ts and m3u8. // The TS is cached to SrsTsMessageCache then flush to ts segment. SrsHlsMuxer* muxer; // The TS cache + // TODO: support both fmp4 and ts format SrsTsMessageCache* tsmc; + + // If the diff=dts-previous_audio_dts is about 23, + // that's the AAC samples is 1024, and we use the samples to calc the dts. + int64_t previous_audio_dts; + // The total aac samples. + uint64_t aac_samples; + // Whether directly turn FLV timestamp to TS DTS. + bool hls_dts_directly; public: SrsHlsController(); virtual ~SrsHlsController(); @@ -258,11 +452,11 @@ class SrsHlsController // must write a #EXT-X-DISCONTINUITY to m3u8. // @see: hls-m3u8-draft-pantos-http-live-streaming-12.txt // @see: 3.4.11. EXT-X-DISCONTINUITY - virtual srs_error_t on_sequence_header(); + virtual srs_error_t on_sequence_header(SrsSharedPtrMessage* shared_audio, SrsFormat* format); // write audio to cache, if need to flush, flush to muxer. - virtual srs_error_t write_audio(SrsAudioFrame* frame, int64_t pts); + virtual srs_error_t write_audio(SrsSharedPtrMessage* shared_audio, SrsFormat* format); // write video to muxer. - virtual srs_error_t write_video(SrsVideoFrame* frame, int64_t dts); + virtual srs_error_t write_video(SrsSharedPtrMessage* shared_video, SrsFormat* format); private: // Reopen the muxer for a new hls segment, // close current segment, open a new segment, @@ -271,12 +465,51 @@ class SrsHlsController virtual srs_error_t reap_segment(); }; -// Transmux RTMP stream to HLS(m3u8 and ts). +class SrsHlsMp4Controller : public ISrsHlsController +{ +private: + bool has_video_sh_; + bool has_audio_sh_; + + int video_track_id_; + int audio_track_id_; + + // Current audio dts. + uint64_t audio_dts_; + // Current video dts. + uint64_t video_dts_; + + SrsRequest* req_; + + SrsHlsFmp4Muxer* muxer_; + +public: + SrsHlsMp4Controller(); + virtual ~SrsHlsMp4Controller(); + +public: + virtual srs_error_t initialize(); + virtual void dispose(); + // When publish or unpublish stream. + virtual srs_error_t on_publish(SrsRequest* req); + virtual srs_error_t on_unpublish(); + virtual srs_error_t write_audio(SrsSharedPtrMessage* shared_audio, SrsFormat* format); + virtual srs_error_t write_video(SrsSharedPtrMessage* shared_video, SrsFormat* format); + + virtual srs_error_t on_sequence_header(SrsSharedPtrMessage* shared_audio, SrsFormat* format); + virtual int sequence_no(); + virtual std::string ts_url(); + virtual srs_utime_t duration(); + virtual int deviation(); +}; + + +// Transmux RTMP stream to HLS(m3u8 and ts,fmp4). // TODO: FIXME: add utest for hls. class SrsHls { private: - SrsHlsController* controller; + ISrsHlsController* controller; private: SrsRequest* req; // Whether the HLS is enabled. @@ -290,14 +523,7 @@ class SrsHls bool reloading_; // To detect heartbeat and dispose it if configured. srs_utime_t last_update_time; -private: - // If the diff=dts-previous_audio_dts is about 23, - // that's the AAC samples is 1024, and we use the samples to calc the dts. - int64_t previous_audio_dts; - // The total aac samples. - uint64_t aac_samples; - // Whether directly turn FLV timestamp to TS DTS. - bool hls_dts_directly; + private: SrsOriginHub* hub; SrsRtmpJitter* jitter; diff --git a/trunk/src/kernel/srs_kernel_codec.hpp b/trunk/src/kernel/srs_kernel_codec.hpp index a1462d3328..07636bcc3e 100644 --- a/trunk/src/kernel/srs_kernel_codec.hpp +++ b/trunk/src/kernel/srs_kernel_codec.hpp @@ -1361,6 +1361,7 @@ class SrsFormat public: virtual bool is_aac_sequence_header(); virtual bool is_mp3_sequence_header(); + // TODO: is avc|hevc|av1 sequence header virtual bool is_avc_sequence_header(); private: // Demux the video packet in H.264 codec. diff --git a/trunk/src/kernel/srs_kernel_mp4.cpp b/trunk/src/kernel/srs_kernel_mp4.cpp index b4d597f7ee..868071730e 100644 --- a/trunk/src/kernel/srs_kernel_mp4.cpp +++ b/trunk/src/kernel/srs_kernel_mp4.cpp @@ -759,15 +759,8 @@ void SrsMp4MovieFragmentBox::set_mfhd(SrsMp4MovieFragmentHeaderBox* v) boxes.push_back(v); } -SrsMp4TrackFragmentBox* SrsMp4MovieFragmentBox::traf() +void SrsMp4MovieFragmentBox::add_traf(SrsMp4TrackFragmentBox* v) { - SrsMp4Box* box = get(SrsMp4BoxTypeTRAF); - return dynamic_cast(box); -} - -void SrsMp4MovieFragmentBox::set_traf(SrsMp4TrackFragmentBox* v) -{ - remove(SrsMp4BoxTypeTRAF); boxes.push_back(v); } @@ -1647,15 +1640,8 @@ SrsMp4MovieExtendsBox::~SrsMp4MovieExtendsBox() { } -SrsMp4TrackExtendsBox* SrsMp4MovieExtendsBox::trex() -{ - SrsMp4Box* box = get(SrsMp4BoxTypeTREX); - return dynamic_cast(box); -} - -void SrsMp4MovieExtendsBox::set_trex(SrsMp4TrackExtendsBox* v) +void SrsMp4MovieExtendsBox::add_trex(SrsMp4TrackExtendsBox* v) { - remove(SrsMp4BoxTypeTREX); boxes.push_back(v); } @@ -4989,13 +4975,11 @@ srs_error_t SrsMp4SampleManager::write(SrsMp4MovieBox* moov) return err; } -srs_error_t SrsMp4SampleManager::write(SrsMp4MovieFragmentBox* moof, uint64_t dts) +srs_error_t SrsMp4SampleManager::write(SrsMp4TrackFragmentBox* traf, uint64_t dts) { srs_error_t err = srs_success; - SrsMp4TrackFragmentBox* traf = moof->traf(); SrsMp4TrackFragmentRunBox* trun = traf->trun(); - trun->flags = SrsMp4TrunFlagsDataOffset | SrsMp4TrunFlagsSampleDuration | SrsMp4TrunFlagsSampleSize | SrsMp4TrunFlagsSampleFlag | SrsMp4TrunFlagsSampleCtsOffset; @@ -6327,7 +6311,7 @@ srs_error_t SrsMp4M2tsInitEncoder::write(SrsFormat* format, bool video, int tid) moov->set_mvex(mvex); SrsMp4TrackExtendsBox* trex = new SrsMp4TrackExtendsBox(); - mvex->set_trex(trex); + mvex->add_trex(trex); trex->track_ID = tid; trex->default_sample_description_index = 1; @@ -6428,7 +6412,7 @@ srs_error_t SrsMp4M2tsInitEncoder::write(SrsFormat* format, bool video, int tid) moov->set_mvex(mvex); SrsMp4TrackExtendsBox* trex = new SrsMp4TrackExtendsBox(); - mvex->set_trex(trex); + mvex->add_trex(trex); trex->track_ID = tid; trex->default_sample_description_index = 1; @@ -6442,6 +6426,248 @@ srs_error_t SrsMp4M2tsInitEncoder::write(SrsFormat* format, bool video, int tid) return err; } +srs_error_t SrsMp4M2tsInitEncoder::write(SrsFormat* format, int v_tid, int a_tid) +{ + srs_error_t err = srs_success; + + // Write ftyp box. + if (true) { + SrsUniquePtr ftyp(new SrsMp4FileTypeBox()); + + ftyp->major_brand = SrsMp4BoxBrandMP42; // SrsMp4BoxBrandISO5; + ftyp->minor_version = 512; + ftyp->set_compatible_brands(SrsMp4BoxBrandISO6, SrsMp4BoxBrandMP41); + + if ((err = srs_mp4_write_box(writer, ftyp.get())) != srs_success) { + return srs_error_wrap(err, "write ftyp"); + } + } + + // Write moov. + if (true) { + SrsUniquePtr moov(new SrsMp4MovieBox()); + + SrsMp4MovieHeaderBox* mvhd = new SrsMp4MovieHeaderBox(); + moov->set_mvhd(mvhd); + + mvhd->timescale = 1000; // Use tbn ms. + mvhd->duration_in_tbn = 0; + mvhd->next_track_ID = 4294967295; // 2^32 - 1 + + // write video track + if (format->vcodec) { + SrsMp4TrackBox* trak = new SrsMp4TrackBox(); + moov->add_trak(trak); + + SrsMp4TrackHeaderBox* tkhd = new SrsMp4TrackHeaderBox(); + trak->set_tkhd(tkhd); + + tkhd->track_ID = v_tid; + tkhd->duration = 0; + tkhd->width = (format->vcodec->width << 16); + tkhd->height = (format->vcodec->height << 16); + + SrsMp4MediaBox* mdia = new SrsMp4MediaBox(); + trak->set_mdia(mdia); + + SrsMp4MediaHeaderBox* mdhd = new SrsMp4MediaHeaderBox(); + mdia->set_mdhd(mdhd); + + mdhd->timescale = 1000; + mdhd->duration = 0; + mdhd->set_language0('u'); + mdhd->set_language1('n'); + mdhd->set_language2('d'); + + SrsMp4HandlerReferenceBox* hdlr = new SrsMp4HandlerReferenceBox(); + mdia->set_hdlr(hdlr); + + hdlr->handler_type = SrsMp4HandlerTypeVIDE; + hdlr->name = "VideoHandler"; + + SrsMp4MediaInformationBox* minf = new SrsMp4MediaInformationBox(); + mdia->set_minf(minf); + + SrsMp4VideoMeidaHeaderBox* vmhd = new SrsMp4VideoMeidaHeaderBox(); + minf->set_vmhd(vmhd); + + SrsMp4DataInformationBox* dinf = new SrsMp4DataInformationBox(); + minf->set_dinf(dinf); + + SrsMp4DataReferenceBox* dref = new SrsMp4DataReferenceBox(); + dinf->set_dref(dref); + + SrsMp4DataEntryBox* url = new SrsMp4DataEntryUrlBox(); + dref->append(url); + + SrsMp4SampleTableBox* stbl = new SrsMp4SampleTableBox(); + minf->set_stbl(stbl); + + SrsMp4SampleDescriptionBox* stsd = new SrsMp4SampleDescriptionBox(); + stbl->set_stsd(stsd); + + if (format->vcodec->id == SrsVideoCodecIdAVC) { + SrsMp4VisualSampleEntry* avc1 = new SrsMp4VisualSampleEntry(SrsMp4BoxTypeAVC1); + stsd->append(avc1); + + avc1->width = format->vcodec->width; + avc1->height = format->vcodec->height; + avc1->data_reference_index = 1; + + SrsMp4AvccBox* avcC = new SrsMp4AvccBox(); + avc1->set_avcC(avcC); + + avcC->avc_config = format->vcodec->avc_extra_data; + } else { + SrsMp4VisualSampleEntry* hev1 = new SrsMp4VisualSampleEntry(SrsMp4BoxTypeHEV1); + stsd->append(hev1); + + hev1->width = format->vcodec->width; + hev1->height = format->vcodec->height; + hev1->data_reference_index = 1; + + SrsMp4HvcCBox* hvcC = new SrsMp4HvcCBox(); + hev1->set_hvcC(hvcC); + + hvcC->hevc_config = format->vcodec->avc_extra_data; + } + + SrsMp4DecodingTime2SampleBox* stts = new SrsMp4DecodingTime2SampleBox(); + stbl->set_stts(stts); + + SrsMp4Sample2ChunkBox* stsc = new SrsMp4Sample2ChunkBox(); + stbl->set_stsc(stsc); + + SrsMp4SampleSizeBox* stsz = new SrsMp4SampleSizeBox(); + stbl->set_stsz(stsz); + + // TODO: FIXME: need to check using stco or co64? + SrsMp4ChunkOffsetBox* stco = new SrsMp4ChunkOffsetBox(); + stbl->set_stco(stco); + } + + // write audio track + if (format->acodec) { + SrsMp4TrackBox* trak = new SrsMp4TrackBox(); + moov->add_trak(trak); + + SrsMp4TrackHeaderBox* tkhd = new SrsMp4TrackHeaderBox(); + tkhd->volume = 0x0100; + trak->set_tkhd(tkhd); + + tkhd->track_ID = a_tid; + tkhd->duration = 0; + + SrsMp4MediaBox* mdia = new SrsMp4MediaBox(); + trak->set_mdia(mdia); + + SrsMp4MediaHeaderBox* mdhd = new SrsMp4MediaHeaderBox(); + mdia->set_mdhd(mdhd); + + mdhd->timescale = 1000; + mdhd->duration = 0; + mdhd->set_language0('u'); + mdhd->set_language1('n'); + mdhd->set_language2('d'); + + SrsMp4HandlerReferenceBox* hdlr = new SrsMp4HandlerReferenceBox(); + mdia->set_hdlr(hdlr); + + hdlr->handler_type = SrsMp4HandlerTypeSOUN; + hdlr->name = "SoundHandler"; + + SrsMp4MediaInformationBox* minf = new SrsMp4MediaInformationBox(); + mdia->set_minf(minf); + + SrsMp4SoundMeidaHeaderBox* smhd = new SrsMp4SoundMeidaHeaderBox(); + minf->set_smhd(smhd); + + SrsMp4DataInformationBox* dinf = new SrsMp4DataInformationBox(); + minf->set_dinf(dinf); + + SrsMp4DataReferenceBox* dref = new SrsMp4DataReferenceBox(); + dinf->set_dref(dref); + + SrsMp4DataEntryBox* url = new SrsMp4DataEntryUrlBox(); + dref->append(url); + + SrsMp4SampleTableBox* stbl = new SrsMp4SampleTableBox(); + minf->set_stbl(stbl); + + SrsMp4SampleDescriptionBox* stsd = new SrsMp4SampleDescriptionBox(); + stbl->set_stsd(stsd); + + SrsMp4AudioSampleEntry* mp4a = new SrsMp4AudioSampleEntry(); + mp4a->data_reference_index = 1; + mp4a->samplerate = uint32_t(srs_flv_srates[format->acodec->sound_rate]) << 16; + if (format->acodec->sound_size == SrsAudioSampleBits16bit) { + mp4a->samplesize = 16; + } else { + mp4a->samplesize = 8; + } + if (format->acodec->sound_type == SrsAudioChannelsStereo) { + mp4a->channelcount = 2; + } else { + mp4a->channelcount = 1; + } + stsd->append(mp4a); + + SrsMp4EsdsBox* esds = new SrsMp4EsdsBox(); + mp4a->set_esds(esds); + + SrsMp4ES_Descriptor* es = esds->es; + es->ES_ID = 0x02; + + SrsMp4DecoderConfigDescriptor& desc = es->decConfigDescr; + desc.objectTypeIndication = SrsMp4ObjectTypeAac; + desc.streamType = SrsMp4StreamTypeAudioStream; + srs_freep(desc.decSpecificInfo); + + SrsMp4DecoderSpecificInfo* asc = new SrsMp4DecoderSpecificInfo(); + desc.decSpecificInfo = asc; + asc->asc = format->acodec->aac_extra_data; + + SrsMp4DecodingTime2SampleBox* stts = new SrsMp4DecodingTime2SampleBox(); + stbl->set_stts(stts); + + SrsMp4Sample2ChunkBox* stsc = new SrsMp4Sample2ChunkBox(); + stbl->set_stsc(stsc); + + SrsMp4SampleSizeBox* stsz = new SrsMp4SampleSizeBox(); + stbl->set_stsz(stsz); + + // TODO: FIXME: need to check using stco or co64? + SrsMp4ChunkOffsetBox* stco = new SrsMp4ChunkOffsetBox(); + stbl->set_stco(stco); + } + + if (true) { + SrsMp4MovieExtendsBox* mvex = new SrsMp4MovieExtendsBox(); + moov->set_mvex(mvex); + + // video trex + SrsMp4TrackExtendsBox* v_trex = new SrsMp4TrackExtendsBox(); + mvex->add_trex(v_trex); + + v_trex->track_ID = v_tid; + v_trex->default_sample_description_index = 1; + + // audio trex + SrsMp4TrackExtendsBox* a_trex = new SrsMp4TrackExtendsBox(); + mvex->add_trex(a_trex); + + a_trex->track_ID = a_tid; + a_trex->default_sample_description_index = 1; + } + + if ((err = srs_mp4_write_box(writer, moov.get())) != srs_success) { + return srs_error_wrap(err, "write moov"); + } + } + + return err; +} + SrsMp4M2tsSegmentEncoder::SrsMp4M2tsSegmentEncoder() { writer = NULL; @@ -6568,7 +6794,7 @@ srs_error_t SrsMp4M2tsSegmentEncoder::flush(uint64_t& dts) mfhd->sequence_number = sequence_number; SrsMp4TrackFragmentBox* traf = new SrsMp4TrackFragmentBox(); - moof->set_traf(traf); + moof->add_traf(traf); SrsMp4TrackFragmentHeaderBox* tfhd = new SrsMp4TrackFragmentHeaderBox(); traf->set_tfhd(tfhd); @@ -6585,7 +6811,7 @@ srs_error_t SrsMp4M2tsSegmentEncoder::flush(uint64_t& dts) SrsMp4TrackFragmentRunBox* trun = new SrsMp4TrackFragmentRunBox(); traf->set_trun(trun); - if ((err = samples->write(moof.get(), dts)) != srs_success) { + if ((err = samples->write(traf, dts)) != srs_success) { return srs_error_wrap(err, "write samples"); } @@ -6635,3 +6861,207 @@ srs_error_t SrsMp4M2tsSegmentEncoder::flush(uint64_t& dts) return err; } +SrsFmp4SegmentEncoder::SrsFmp4SegmentEncoder() +{ + writer_ = NULL; + sequence_number_ = 0; + decode_basetime_ = 0; + audio_track_id_ = 0; + video_track_id_ = 0; + nb_audios_ = 0; + nb_videos_ = 0; + styp_bytes_ = 0; + mdat_audio_bytes_ = 0; + mdat_video_bytes_ = 0; + audio_samples_ = new SrsMp4SampleManager(); + video_samples_ = new SrsMp4SampleManager(); +} + +SrsFmp4SegmentEncoder::~SrsFmp4SegmentEncoder() +{ + srs_freep(audio_samples_); + srs_freep(video_samples_); +} + + +srs_error_t SrsFmp4SegmentEncoder::initialize(ISrsWriter* w, uint32_t sequence, srs_utime_t basetime, uint32_t v_tid, uint32_t a_tid) +{ + srs_error_t err = srs_success; + + writer_ = w; + sequence_number_ = sequence; + decode_basetime_ = basetime; + video_track_id_ = v_tid; + audio_track_id_ = a_tid; + + return err; +} + +srs_error_t SrsFmp4SegmentEncoder::write_sample(SrsMp4HandlerType ht, uint16_t ft, + uint32_t dts, uint32_t pts, uint8_t* sample, uint32_t nb_sample) +{ + srs_error_t err = srs_success; + + SrsMp4Sample* ps = new SrsMp4Sample(); + + if (ht == SrsMp4HandlerTypeVIDE) { + ps->type = SrsFrameTypeVideo; + ps->frame_type = (SrsVideoAvcFrameType)ft; + ps->index = nb_videos_++; + video_samples_->append(ps); + mdat_video_bytes_ += nb_sample; + } else if (ht == SrsMp4HandlerTypeSOUN) { + ps->type = SrsFrameTypeAudio; + ps->index = nb_audios_++; + audio_samples_->append(ps); + mdat_audio_bytes_ += nb_sample; + } else { + srs_freep(ps); + return err; + } + + ps->tbn = 1000; + ps->dts = dts; + ps->pts = pts; + + // We should copy the sample data, which is shared ptr from video/audio message. + // Furthermore, we do free the data when freeing the sample. + ps->data = new uint8_t[nb_sample]; + memcpy(ps->data, sample, nb_sample); + ps->nb_data = nb_sample; + + return err; +} + +srs_error_t SrsFmp4SegmentEncoder::flush(uint64_t& dts) +{ + srs_error_t err = srs_success; + SrsMp4TrackFragmentRunBox* video_trun = NULL; + SrsMp4TrackFragmentRunBox* audio_trun = NULL; + + if (nb_videos_ == 0 && nb_audios_ == 0) { + return srs_error_new(ERROR_MP4_ILLEGAL_MDAT, "empty samples"); + } + // Create a mdat box. + // its payload will be writen by samples, + // and we will update its header(size) when flush. + SrsUniquePtr mdat(new SrsMp4MediaDataBox()); + + SrsUniquePtr moof(new SrsMp4MovieFragmentBox()); + + SrsMp4MovieFragmentHeaderBox* mfhd = new SrsMp4MovieFragmentHeaderBox(); + moof->set_mfhd(mfhd); + mfhd->sequence_number = sequence_number_; + + // write video traf + if (mdat_video_bytes_ > 0) { + // video traf + SrsMp4TrackFragmentBox* traf = new SrsMp4TrackFragmentBox(); + moof->add_traf(traf); + + SrsMp4TrackFragmentHeaderBox* tfhd = new SrsMp4TrackFragmentHeaderBox(); + traf->set_tfhd(tfhd); + + tfhd->track_id = video_track_id_; + tfhd->flags = SrsMp4TfhdFlagsDefaultBaseIsMoof; + + SrsMp4TrackFragmentDecodeTimeBox* tfdt = new SrsMp4TrackFragmentDecodeTimeBox(); + traf->set_tfdt(tfdt); + + tfdt->version = 1; + tfdt->base_media_decode_time = srsu2ms(decode_basetime_); + + SrsMp4TrackFragmentRunBox* trun = new SrsMp4TrackFragmentRunBox(); + traf->set_trun(trun); + video_trun = trun; + + if ((err = video_samples_->write(traf, dts)) != srs_success) { + return srs_error_wrap(err, "write samples"); + } + } + + // write audio traf + if (mdat_audio_bytes_ > 0) { + // audio traf + SrsMp4TrackFragmentBox* traf = new SrsMp4TrackFragmentBox(); + moof->add_traf(traf); + + SrsMp4TrackFragmentHeaderBox* tfhd = new SrsMp4TrackFragmentHeaderBox(); + traf->set_tfhd(tfhd); + + tfhd->track_id = audio_track_id_; + tfhd->flags = SrsMp4TfhdFlagsDefaultBaseIsMoof; + + SrsMp4TrackFragmentDecodeTimeBox* tfdt = new SrsMp4TrackFragmentDecodeTimeBox(); + traf->set_tfdt(tfdt); + + tfdt->version = 1; + tfdt->base_media_decode_time = srsu2ms(decode_basetime_); + + SrsMp4TrackFragmentRunBox* trun = new SrsMp4TrackFragmentRunBox(); + traf->set_trun(trun); + audio_trun = trun; + + if ((err = audio_samples_->write(traf, dts)) != srs_success) { + return srs_error_wrap(err, "write samples"); + } + } + + // @remark Remember the data_offset of turn is size(moof)+header(mdat) + int moof_bytes = moof->nb_bytes(); + // rewrite video data_offset + if (video_trun != NULL) { + video_trun->data_offset = (int32_t)(moof_bytes + mdat->sz_header() + 0); + } + + if (audio_trun != NULL) { + audio_trun->data_offset = (int32_t)(moof_bytes + mdat->sz_header() + mdat_video_bytes_); + } + + // srs_trace("seq: %d, moof_bytes=%d, mdat->sz_header=%d", sequence_number_, moof->nb_bytes(), mdat->sz_header()); + // srs_trace("mdat_video_bytes_ = %d, mdat_audio_bytes_ = %d", mdat_video_bytes_, mdat_audio_bytes_); + + if ((err = srs_mp4_write_box(writer_, moof.get())) != srs_success) { + return srs_error_wrap(err, "write moof"); + } + + mdat->nb_data = mdat_video_bytes_ + mdat_audio_bytes_; + // Write mdat. + if (true) { + int nb_data = mdat->sz_header(); + SrsUniquePtr data(new uint8_t[nb_data]); + + SrsUniquePtr buffer(new SrsBuffer((char*)data.get(), nb_data)); + if ((err = mdat->encode(buffer.get())) != srs_success) { + return srs_error_wrap(err, "encode mdat"); + } + + // TODO: FIXME: Ensure all bytes are writen. + if ((err = writer_->write(data.get(), nb_data, NULL)) != srs_success) { + return srs_error_wrap(err, "write mdat"); + } + + vector::iterator it; + // write video sample data + for (it = video_samples_->samples.begin(); it != video_samples_->samples.end(); ++it) { + SrsMp4Sample* sample = *it; + + // TODO: FIXME: Ensure all bytes are writen. + if ((err = writer_->write(sample->data, sample->nb_data, NULL)) != srs_success) { + return srs_error_wrap(err, "write sample"); + } + } + + // write audio sample data + for (it = audio_samples_->samples.begin(); it != audio_samples_->samples.end(); ++it) { + SrsMp4Sample* sample = *it; + + // TODO: FIXME: Ensure all bytes are writen. + if ((err = writer_->write(sample->data, sample->nb_data, NULL)) != srs_success) { + return srs_error_wrap(err, "write sample"); + } + } + } + + return err; +} diff --git a/trunk/src/kernel/srs_kernel_mp4.hpp b/trunk/src/kernel/srs_kernel_mp4.hpp index 23805773e6..a091d5f93f 100644 --- a/trunk/src/kernel/srs_kernel_mp4.hpp +++ b/trunk/src/kernel/srs_kernel_mp4.hpp @@ -317,9 +317,9 @@ class SrsMp4MovieFragmentBox : public SrsMp4Box // Get the header of moof. virtual SrsMp4MovieFragmentHeaderBox* mfhd(); virtual void set_mfhd(SrsMp4MovieFragmentHeaderBox* v); - // Get the traf. - virtual SrsMp4TrackFragmentBox* traf(); - virtual void set_traf(SrsMp4TrackFragmentBox* v); + + // Let moof support more than one traf + virtual void add_traf(SrsMp4TrackFragmentBox* v); }; // 8.8.5 Movie Fragment Header Box (mfhd) @@ -499,7 +499,7 @@ class SrsMp4TrackFragmentRunBox : public SrsMp4FullBox public: // The number of samples being added in this run; also the number of rows in the following // table (the rows can be empty) - //uint32_t sample_count; + // uint32_t sample_count; // The following are optional fields public: // added to the implicit or explicit data_offset established in the track fragment header. @@ -710,8 +710,7 @@ class SrsMp4MovieExtendsBox : public SrsMp4Box virtual ~SrsMp4MovieExtendsBox(); public: // Get the track extends box. - virtual SrsMp4TrackExtendsBox* trex(); - virtual void set_trex(SrsMp4TrackExtendsBox* v); + virtual void add_trex(SrsMp4TrackExtendsBox* v); }; // 8.8.3 Track Extends Box(trex) @@ -1931,7 +1930,7 @@ class SrsMp4SampleManager virtual srs_error_t write(SrsMp4MovieBox* moov); // Write the samples info to moof. // @param The dts is the dts of last segment. - virtual srs_error_t write(SrsMp4MovieFragmentBox* moof, uint64_t dts); + virtual srs_error_t write(SrsMp4TrackFragmentBox* traf, uint64_t dts); private: virtual srs_error_t write_track(SrsFrameType track, SrsMp4DecodingTime2SampleBox* stts, SrsMp4SyncSampleBox* stss, SrsMp4CompositionTime2SampleBox* ctts, @@ -2111,6 +2110,7 @@ class SrsMp4Encoder }; // A fMP4 encoder, to write the init.mp4 with sequence header. +// TODO: What the M2ts short for? class SrsMp4M2tsInitEncoder { private: @@ -2122,11 +2122,31 @@ class SrsMp4M2tsInitEncoder // Initialize the encoder with a writer w. virtual srs_error_t initialize(ISrsWriter* w); // Write the sequence header. + // TODO: merge this method to its sibling. virtual srs_error_t write(SrsFormat* format, bool video, int tid); + + /** + * The mp4 box format for init.mp4. + * + * |ftyp| + * |moov| + * | |mvhd| + * | |trak| + * | |trak| + * | |....| + * | |mvex| + * | | |trex| + * | | |trex| + * | | |....| + * + * Write the sequence header with both video and audio track. + */ + virtual srs_error_t write(SrsFormat* format, int v_tid, int a_tid); }; // A fMP4 encoder, to cache segments then flush to disk, because the fMP4 should write // trun box before mdat. +// TODO: fmp4 support package more than one tracks. class SrsMp4M2tsSegmentEncoder { private: @@ -2160,6 +2180,47 @@ class SrsMp4M2tsSegmentEncoder virtual srs_error_t flush(uint64_t& dts); }; +// A fMP4 encoder, to cache segments then flush to disk, because the fMP4 should write +// trun box before mdat. +// TODO: fmp4 support package more than one tracks. +class SrsFmp4SegmentEncoder +{ +private: + ISrsWriter* writer_; + uint32_t sequence_number_; + // TODO: audio, video may have different basetime. + srs_utime_t decode_basetime_; + uint32_t audio_track_id_; + uint32_t video_track_id_; +private: + uint32_t nb_audios_; + uint32_t nb_videos_; + uint32_t styp_bytes_; + uint64_t mdat_audio_bytes_; + uint64_t mdat_video_bytes_; + SrsMp4SampleManager* audio_samples_; + SrsMp4SampleManager* video_samples_; +public: + SrsFmp4SegmentEncoder(); + virtual ~SrsFmp4SegmentEncoder(); +public: + // Initialize the encoder with a writer w. + virtual srs_error_t initialize(ISrsWriter* w, uint32_t sequence, srs_utime_t basetime, uint32_t v_tid, uint32_t a_tid); + // Cache a sample. + // @param ht, The sample handler type, audio/soun or video/vide. + // @param ft, The frame type. For video, it's SrsVideoAvcFrameType. + // @param dts The output dts in milliseconds. + // @param pts The output pts in milliseconds. + // @param sample The output payload, user must free it. + // @param nb_sample The output size of payload. + // @remark All samples are RAW AAC/AVC data, because sequence header is writen to init.mp4. + virtual srs_error_t write_sample(SrsMp4HandlerType ht, uint16_t ft, + uint32_t dts, uint32_t pts, uint8_t* sample, uint32_t nb_sample); + // Flush the encoder, to write the moof and mdat. + virtual srs_error_t flush(uint64_t& dts); +}; + + // LCOV_EXCL_START ///////////////////////////////////////////////////////////////////////////////// // MP4 dumps functions. diff --git a/trunk/src/utest/srs_utest_config.cpp b/trunk/src/utest/srs_utest_config.cpp index 8999dabfeb..fa78053ae3 100644 --- a/trunk/src/utest/srs_utest_config.cpp +++ b/trunk/src/utest/srs_utest_config.cpp @@ -3732,12 +3732,14 @@ VOID TEST(ConfigMainTest, CheckVhostConfig5) if (true) { MockSrsConfig conf; - HELPER_ASSERT_SUCCESS(conf.parse(_MIN_OK_CONF "vhost ossrs.net{hls{hls_keys on;hls_fragments_per_key 5;hls_key_file xxx;hls_key_file_path xxx2;hls_key_url xxx3;}}")); + HELPER_ASSERT_SUCCESS(conf.parse(_MIN_OK_CONF "vhost ossrs.net{hls{hls_keys on;hls_fragments_per_key 5;hls_key_file xxx;hls_key_file_path xxx2;hls_key_url xxx3;hls_use_fmp4 on;hls_fmp4_file xx.m4s;}}")); EXPECT_TRUE(conf.get_hls_keys("ossrs.net")); EXPECT_EQ(5, conf.get_hls_fragments_per_key("ossrs.net")); EXPECT_STREQ("xxx", conf.get_hls_key_file("ossrs.net").c_str()); EXPECT_STREQ("xxx2", conf.get_hls_key_file_path("ossrs.net").c_str()); EXPECT_STREQ("xxx3", conf.get_hls_key_url("ossrs.net").c_str()); + EXPECT_TRUE(conf.get_hls_use_fmp4("ossrs.net")); + EXPECT_STREQ("xx.m4s", conf.get_hls_fmp4_file("ossrs.net").c_str()); } if (true) { @@ -5046,6 +5048,18 @@ VOID TEST(ConfigEnvTest, CheckEnvValuesHls) SrsSetEnvConfig(hls_dts_directly, "SRS_VHOST_HLS_HLS_DTS_DIRECTLY", "off"); EXPECT_FALSE(conf.get_vhost_hls_dts_directly("__defaultVhost__")); + + SrsSetEnvConfig(hls_use_fmp4_on, "SRS_VHOST_HLS_HLS_USE_FMP4", "on"); + EXPECT_TRUE(conf.get_hls_use_fmp4("__defaultVhost__")); + + SrsSetEnvConfig(hls_use_fmp4_off, "SRS_VHOST_HLS_HLS_USE_FMP4", "off"); + EXPECT_FALSE(conf.get_hls_use_fmp4("__defaultVhost__")); + + SrsSetEnvConfig(hls_use_fmp4_unexpected, "SRS_VHOST_HLS_HLS_USE_FMP4", "xx"); + EXPECT_FALSE(conf.get_hls_use_fmp4("__defaultVhost__")); + + SrsSetEnvConfig(hls_fmp4_file, "SRS_VHOST_HLS_HLS_FMP4_FILE", "xxx.m4s"); + EXPECT_STREQ("xxx.m4s", conf.get_hls_fmp4_file("__defaultVhost__").c_str()); } } diff --git a/trunk/src/utest/srs_utest_mp4.cpp b/trunk/src/utest/srs_utest_mp4.cpp index 77f9355fea..5499bb1fc0 100644 --- a/trunk/src/utest/srs_utest_mp4.cpp +++ b/trunk/src/utest/srs_utest_mp4.cpp @@ -898,11 +898,10 @@ VOID TEST(KernelMp4Test, TREXBox) } SrsMp4MovieExtendsBox box; - EXPECT_TRUE(NULL == box.trex()); SrsMp4TrackExtendsBox* trex = new SrsMp4TrackExtendsBox(); - box.set_trex(trex); - EXPECT_TRUE(trex == box.trex()); + box.add_trex(trex); + EXPECT_TRUE(trex == box.get(SrsMp4BoxTypeTREX)); } VOID TEST(KernelMp4Test, TKHDBox)