Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Creating a webvtt hls ll playlist #4

Open
wants to merge 5 commits into
base: webvtt-hlsll
Choose a base branch
from
Open

Creating a webvtt hls ll playlist #4

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
faa5f66
Creating a webvtt hls ll playlist
estigma88 May 7, 2024
61289e5
Creating a webvtt hls ll playlist
estigma88 May 8, 2024
570ff0a
Creating a webvtt hls ll playlist
estigma88 May 8, 2024
be25c1a
Creating a webvtt hls ll playlist
estigma88 May 9, 2024
1d769e0
Creating a webvtt hls ll playlist
estigma88 May 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 48 additions & 0 deletions docs/webvtt/Server.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,13 @@
<Port>9090</Port>
</WebVTT>
</Providers>
<Publishers>
<LLHLS>
<Port>3333</Port>
<TLSPort>3334</TLSPort>
<WorkerCount>1</WorkerCount>
</LLHLS>
</Publishers>
</Bind>
<Managers>
<Host>
Expand Down Expand Up @@ -66,9 +73,50 @@
<DumpToFile>true</DumpToFile>
</WebVTT>
</Providers>
<OutputProfiles>
<HardwareAcceleration>false</HardwareAcceleration>
<OutputProfile>
<Name>bypass_stream</Name>
<OutputStreamName>${OriginStreamName}</OutputStreamName>
<Encodes>
<Video>
<Bypass>true</Bypass>
</Video>
<Audio>
<Bypass>true</Bypass>
</Audio>
</Encodes>
</OutputProfile>
</OutputProfiles>
<Publishers>
<AppWorkerCount>10</AppWorkerCount>
<StreamWorkerCount>10</StreamWorkerCount>
<LLHLS>
<ChunkDuration>0.2</ChunkDuration>
<SegmentDuration>4</SegmentDuration>
<SegmentCount>5</SegmentCount>
<Dumps>
<Dump>
<Enable>false</Enable>
<TargetStreamName>live</TargetStreamName>

<Playlists>
<Playlist>playlist.m3u8</Playlist>
</Playlists>
<OutputPath>
/tmp/vod/html/${VHostName}_${AppName}_${StreamName}/${YYYY}_${MM}_${DD}_${hh}_${mm}_${ss}
</OutputPath>
</Dump>
</Dumps>
<CrossDomains>
<Url>*</Url>
</CrossDomains>
</LLHLS>
</Publishers>
</Application>
</Applications>
</VirtualHost>
</VirtualHosts>
</Server>


7 changes: 7 additions & 0 deletions src/projects/base/info/media_track.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -374,6 +374,13 @@ bool MediaTrack::IsValid()
return true;
}

// Subtitle type is always valid
if(GetMediaType() == MediaType::Subtitle)
{
_is_valid = true;
return true;
}

switch (GetCodecId())
{
case MediaCodecId::H264: {
Expand Down
1 change: 1 addition & 0 deletions src/projects/base/mediarouter/media_type.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@ namespace cmn
Opus,
Jpeg,
Png,
WebVTT
};

enum class MediaCodecModuleId : uint8_t
Expand Down
4 changes: 4 additions & 0 deletions src/projects/base/publisher/application.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,10 @@ namespace pub
{
stream_data->_stream->SendDataFrame(stream_data->_media_packet);
}
else if (stream_data->_media_packet->GetMediaType() == cmn::MediaType::Subtitle)
{
stream_data->_stream->SendSubtitleFrame(stream_data->_media_packet);
}
else
{
// Nothing can do
Expand Down
1 change: 1 addition & 0 deletions src/projects/base/publisher/stream.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@ namespace pub
virtual void SendVideoFrame(const std::shared_ptr<MediaPacket> &media_packet) = 0;
virtual void SendAudioFrame(const std::shared_ptr<MediaPacket> &media_packet) = 0;
virtual void SendDataFrame(const std::shared_ptr<MediaPacket> &media_packet) = 0;
virtual void SendSubtitleFrame(const std::shared_ptr<MediaPacket> &media_packet) = 0;

virtual bool Start();
virtual bool Stop();
Expand Down
3 changes: 3 additions & 0 deletions src/projects/mediarouter/mediarouter_stream.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -821,6 +821,9 @@ bool MediaRouteStream::NormalizeMediaPacket(std::shared_ptr<MediaTrack> &media_t
case cmn::BitstreamFormat::ID3v2:
result = true;
break;
case cmn::BitstreamFormat::WebVTT:
result = true;
break;
case cmn::BitstreamFormat::JPEG:
case cmn::BitstreamFormat::PNG:
{
Expand Down
11 changes: 8 additions & 3 deletions src/projects/providers/webvtt/webvtt_stream.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@
#include "modules/subtitles/webvtt/webvtt_decoder.h"
#include "webvtt_provider_private.h"

// Fix track id
#define WEBVTT_TRACK_ID 0

namespace pvd
{
std::shared_ptr<WebVTTStream> WebVTTStream::Create(StreamSourceType source_type, uint32_t channel_id, const std::shared_ptr<ov::Socket> &client_socket, const std::shared_ptr<PushProvider> &provider)
Expand Down Expand Up @@ -85,12 +88,13 @@ namespace pvd

auto media_packet = std::make_shared<MediaPacket>(GetMsid(),
cmn::MediaType::Subtitle,
1,
WEBVTT_TRACK_ID,
cueData,
cue->_time_start,
cue->_duration,
cue->_time_start,
cmn::BitstreamFormat::WebVTT,
cmn::PacketType::SUBTITLE_EVENT);
media_packet->SetDuration(cue->_duration);

return SendFrame(media_packet);
}
Expand All @@ -102,10 +106,11 @@ namespace pvd
{
auto subtitle_track = std::make_shared<MediaTrack>();

subtitle_track->SetId(0);
subtitle_track->SetId(WEBVTT_TRACK_ID);
subtitle_track->SetMediaType(cmn::MediaType::Subtitle);
subtitle_track->SetTimeBase(1, 1000);
subtitle_track->SetOriginBitstream(cmn::BitstreamFormat::WebVTT);
subtitle_track->SetCodecId(cmn::MediaCodecId::WebVTT);

AddTrack(subtitle_track);

Expand Down
1 change: 1 addition & 0 deletions src/projects/publishers/file/file_stream.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ namespace pub
void SendVideoFrame(const std::shared_ptr<MediaPacket> &media_packet) override;
void SendAudioFrame(const std::shared_ptr<MediaPacket> &media_packet) override;
void SendDataFrame(const std::shared_ptr<MediaPacket> &media_packet) override {} // Not supported
void SendSubtitleFrame(const std::shared_ptr<MediaPacket> &media_packet) override {} // Not supported

std::shared_ptr<FileSession> CreateSession();
bool DeleteSession(uint32_t session_id);
Expand Down
22 changes: 22 additions & 0 deletions src/projects/publishers/llhls/llhls_stream.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -524,6 +524,7 @@ bool LLHlsStream::IsSupportedCodec(cmn::MediaCodecId codec_id) const
case cmn::MediaCodecId::H264:
case cmn::MediaCodecId::H265:
case cmn::MediaCodecId::Aac:
case cmn::MediaCodecId::WebVTT:
return true;
default:
return false;
Expand Down Expand Up @@ -1079,6 +1080,27 @@ void LLHlsStream::SendDataFrame(const std::shared_ptr<MediaPacket> &media_packet
}
}

void LLHlsStream::SendSubtitleFrame(const std::shared_ptr<MediaPacket> &media_packet)
{
if (media_packet == nullptr || media_packet->GetData() == nullptr)
{
return;
}

if (GetState() == State::CREATED)
{
BufferMediaPacketUntilReadyToPlay(media_packet);
return;
}

if (_initial_media_packet_buffer.IsEmpty() == false)
{
SendBufferedPackets();
}

AppendMediaPacket(media_packet);
}

bool LLHlsStream::AppendMediaPacket(const std::shared_ptr<MediaPacket> &media_packet)
{
auto track = GetTrack(media_packet->GetTrackId());
Expand Down
1 change: 1 addition & 0 deletions src/projects/publishers/llhls/llhls_stream.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ class LLHlsStream : public pub::Stream, public bmff::FMp4StorageObserver
void SendVideoFrame(const std::shared_ptr<MediaPacket> &media_packet) override;
void SendAudioFrame(const std::shared_ptr<MediaPacket> &media_packet) override;
void SendDataFrame(const std::shared_ptr<MediaPacket> &media_packet) override;
void SendSubtitleFrame(const std::shared_ptr<MediaPacket> &media_packet) override;

enum class RequestResult : uint8_t
{
Expand Down
1 change: 1 addition & 0 deletions src/projects/publishers/mpegtspush/mpegtspush_stream.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ class MpegtsPushStream : public pub::Stream
void SendVideoFrame(const std::shared_ptr<MediaPacket> &media_packet) override;
void SendAudioFrame(const std::shared_ptr<MediaPacket> &media_packet) override;
void SendDataFrame(const std::shared_ptr<MediaPacket> &media_packet) override {} // Not supported
void SendSubtitleFrame(const std::shared_ptr<MediaPacket> &media_packet) override {} // Not supported

std::shared_ptr<pub::Session> CreatePushSession(std::shared_ptr<info::Push> &push) override;

Expand Down
1 change: 1 addition & 0 deletions src/projects/publishers/ovt/ovt_stream.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ class OvtStream : public pub::Stream, public OvtPacketizerInterface
void SendVideoFrame(const std::shared_ptr<MediaPacket> &media_packet) override;
void SendAudioFrame(const std::shared_ptr<MediaPacket> &media_packet) override;
void SendDataFrame(const std::shared_ptr<MediaPacket> &media_packet) override {} // Not supported yet
void SendSubtitleFrame(const std::shared_ptr<MediaPacket> &media_packet) override {} // Not supported


bool OnOvtPacketized(std::shared_ptr<OvtPacket> &packet) override;
Expand Down
1 change: 1 addition & 0 deletions src/projects/publishers/rtmppush/rtmppush_stream.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ class RtmpPushStream : public pub::Stream
void SendVideoFrame(const std::shared_ptr<MediaPacket> &media_packet) override;
void SendAudioFrame(const std::shared_ptr<MediaPacket> &media_packet) override;
void SendDataFrame(const std::shared_ptr<MediaPacket> &media_packet) override {} // Not supported
void SendSubtitleFrame(const std::shared_ptr<MediaPacket> &media_packet) override {} // Not supported

std::shared_ptr<pub::Session> CreatePushSession(std::shared_ptr<info::Push> &push) override;

Expand Down
1 change: 1 addition & 0 deletions src/projects/publishers/segment/segment_stream/segment_stream.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ class SegmentStream : public pub::Stream
void SendVideoFrame(const std::shared_ptr<MediaPacket> &media_packet) override;
void SendAudioFrame(const std::shared_ptr<MediaPacket> &media_packet) override;
void SendDataFrame(const std::shared_ptr<MediaPacket> &media_packet) override{} // Not supported
void SendSubtitleFrame(const std::shared_ptr<MediaPacket> &media_packet) override {} // Not supported
int GetSegmentDuration() { return _segment_duration; }

protected:
Expand Down
1 change: 1 addition & 0 deletions src/projects/publishers/thumbnail/thumbnail_stream.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ class ThumbnailStream : public pub::Stream
void SendVideoFrame(const std::shared_ptr<MediaPacket> &media_packet) override;
void SendAudioFrame(const std::shared_ptr<MediaPacket> &media_packet) override;
void SendDataFrame(const std::shared_ptr<MediaPacket> &media_packet) override {} // Not supported
void SendSubtitleFrame(const std::shared_ptr<MediaPacket> &media_packet) override {} // Not supported

std::shared_ptr<ov::Data> GetVideoFrameByCodecId(cmn::MediaCodecId codec_id, int64_t timeout_ms = 0);
private:
Expand Down
1 change: 1 addition & 0 deletions src/projects/publishers/webrtc/rtc_stream.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ class RtcStream : public pub::Stream, public RtpPacketizerInterface
void SendVideoFrame(const std::shared_ptr<MediaPacket> &media_packet) override;
void SendAudioFrame(const std::shared_ptr<MediaPacket> &media_packet) override;
void SendDataFrame(const std::shared_ptr<MediaPacket> &media_packet) override {} // Not supported
void SendSubtitleFrame(const std::shared_ptr<MediaPacket> &media_packet) override {} // Not supported

std::shared_ptr<RtxRtpPacket> GetRtxRtpPacket(uint32_t track_id, uint8_t origin_payload_type, uint16_t origin_sequence_number);

Expand Down
18 changes: 17 additions & 1 deletion src/projects/transcoder/transcoder_stream.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -336,11 +336,12 @@ bool TranscoderStream::IsAvailableSmoothTransitionStream(const std::shared_ptr<i

bool TranscoderStream::Push(std::shared_ptr<MediaPacket> packet)
{
// SendFrame()
if(_create_success == false)
{
return false;
}

//
DecodePacket(std::move(packet));

return true;
Expand Down Expand Up @@ -531,6 +532,21 @@ std::shared_ptr<info::Stream> TranscoderStream::CreateOutputStream(const cfg::vh
AddComposite(GetIdentifiedForDataProfile(input_track_id), _input_stream, input_track, output_stream, output_track);
}
break;

// If there is a subtitle type track in the input stream, it must be created equally in all output streams.
case cmn::MediaType::Subtitle: {
auto output_track = CreateOutputTrackSubtitleType(input_track);
if (output_track == nullptr)
{
logtw("Failed to create media tracks. Encoding options need to be checked. track_id(%d)", input_track_id);
continue;
}

output_stream->AddTrack(output_track);

AddComposite(GetIdentifiedForSubtitleProfile(input_track_id), _input_stream, input_track, output_stream, output_track);
}
break;
default: {
logtw("Unsupported media type of input track. type(%d)", input_track->GetMediaType());
continue;
Expand Down
32 changes: 31 additions & 1 deletion src/projects/transcoder/transcoder_stream_internal.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,11 @@ ov::String TranscoderStreamInternal::GetIdentifiedForDataProfile(const uint32_t
return ov::String::FormatString("I=%d,O=bypass", track_id);
}

ov::String TranscoderStreamInternal::GetIdentifiedForSubtitleProfile(const uint32_t track_id)
{
return ov::String::FormatString("I=%d,O=bypass", track_id);
}

cmn::Timebase TranscoderStreamInternal::GetDefaultTimebaseByCodecId(cmn::MediaCodecId codec_id)
{
cmn::Timebase timebase(1, 1000);
Expand Down Expand Up @@ -412,6 +417,31 @@ std::shared_ptr<MediaTrack> TranscoderStreamInternal::CreateOutputTrackDataType(
return output_track;
}

std::shared_ptr<MediaTrack> TranscoderStreamInternal::CreateOutputTrackSubtitleType(const std::shared_ptr<MediaTrack> &input_track)
{
auto output_track = std::make_shared<MediaTrack>();
if (output_track == nullptr)
{
return nullptr;
}

output_track->SetMediaType(cmn::MediaType::Subtitle);
output_track->SetId(NewTrackId());
output_track->SetVariantName("");
output_track->SetPublicName(input_track->GetPublicName());
output_track->SetLanguage(input_track->GetLanguage());
output_track->SetBypass(true);
output_track->SetCodecId(input_track->GetCodecId());
output_track->SetCodecModules("");
output_track->SetCodecModuleId(input_track->GetCodecModuleId());
output_track->SetOriginBitstream(input_track->GetOriginBitstream());
output_track->SetWidth(input_track->GetWidth());
output_track->SetHeight(input_track->GetHeight());
output_track->SetFrameRateByMeasured(input_track->GetFrameRate());
output_track->SetTimeBase(input_track->GetTimeBase());

return output_track;
}

bool TranscoderStreamInternal::IsMatchesBypassCondition(const std::shared_ptr<MediaTrack> &input_track, const cfg::vhost::app::oprf::VideoProfile &profile)
{
Expand Down Expand Up @@ -699,4 +729,4 @@ void TranscoderStreamInternal::UpdateOutputTrackTranscode(const std::shared_ptr<
output_track->SetChannel(buffer->GetChannels());
}
}
}
}
4 changes: 3 additions & 1 deletion src/projects/transcoder/transcoder_stream_internal.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,13 +33,15 @@ class TranscoderStreamInternal
static ov::String GetIdentifiedForAudioProfile(const uint32_t track_id, const cfg::vhost::app::oprf::AudioProfile &profile);
static ov::String GetIdentifiedForImageProfile(const uint32_t track_id, const cfg::vhost::app::oprf::ImageProfile &profile);
static ov::String GetIdentifiedForDataProfile(const uint32_t track_id);
static ov::String GetIdentifiedForSubtitleProfile(const uint32_t track_id);

static cmn::Timebase GetDefaultTimebaseByCodecId(cmn::MediaCodecId codec_id);

std::shared_ptr<MediaTrack> CreateOutputTrack(const std::shared_ptr<MediaTrack> &input_track, const cfg::vhost::app::oprf::VideoProfile &profile);
std::shared_ptr<MediaTrack> CreateOutputTrack(const std::shared_ptr<MediaTrack> &input_track, const cfg::vhost::app::oprf::AudioProfile &profile);
std::shared_ptr<MediaTrack> CreateOutputTrack(const std::shared_ptr<MediaTrack> &input_track, const cfg::vhost::app::oprf::ImageProfile &profile);
std::shared_ptr<MediaTrack> CreateOutputTrackDataType(const std::shared_ptr<MediaTrack> &input_track);
std::shared_ptr<MediaTrack> CreateOutputTrackDataType(const std::shared_ptr<MediaTrack> &input_track);
std::shared_ptr<MediaTrack> CreateOutputTrackSubtitleType(const std::shared_ptr<MediaTrack> &input_track);

bool IsMatchesBypassCondition(const std::shared_ptr<MediaTrack> &input_track, const cfg::vhost::app::oprf::VideoProfile &profile);
bool IsMatchesBypassCondition(const std::shared_ptr<MediaTrack> &input_track, const cfg::vhost::app::oprf::AudioProfile &profile);
Expand Down