Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Filter out streams with mismatched content length #760

Merged
merged 7 commits into from
Jan 2, 2024
Merged

Filter out streams with mismatched content length #760

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
858162d
Filter out streams with mismatched content length
Tyrrrz Jan 2, 2024
4b95090
Add tests
Tyrrrz Jan 2, 2024
89feae9
Retry with a different client if no streams are returned
Tyrrrz Jan 2, 2024
4b8b1dc
Test
Tyrrrz Jan 2, 2024
f935cbd
Only filter out 404s
Tyrrrz Jan 2, 2024
6aabc9e
Refactor
Tyrrrz Jan 2, 2024
174bad0
jl
Tyrrrz Jan 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions YoutubeExplode.Tests/ClosedCaptionSpecs.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,8 @@ public async Task I_can_get_a_specific_closed_caption_track_from_a_video()
var manifest = await youtube.Videos.ClosedCaptions.GetManifestAsync(
VideoIds.WithClosedCaptions
);
var trackInfo = manifest.GetByLanguage("en-US");

var trackInfo = manifest.GetByLanguage("en-US");
var track = await youtube.Videos.ClosedCaptions.GetAsync(trackInfo);

// Assert
Expand All @@ -80,8 +80,8 @@ public async Task I_can_get_a_specific_closed_caption_track_from_a_video_that_ha
var manifest = await youtube.Videos.ClosedCaptions.GetManifestAsync(
VideoIds.WithBrokenClosedCaptions
);
var trackInfo = manifest.GetByLanguage("en");

var trackInfo = manifest.GetByLanguage("en");
var track = await youtube.Videos.ClosedCaptions.GetAsync(trackInfo);

// Assert
Expand All @@ -98,8 +98,8 @@ public async Task I_can_get_an_individual_closed_caption_from_a_video()
var manifest = await youtube.Videos.ClosedCaptions.GetManifestAsync(
VideoIds.WithClosedCaptions
);
var trackInfo = manifest.GetByLanguage("en-US");

var trackInfo = manifest.GetByLanguage("en-US");
var track = await youtube.Videos.ClosedCaptions.GetAsync(trackInfo);

var caption = track.GetByTime(TimeSpan.FromSeconds(641));
Expand All @@ -118,8 +118,8 @@ public async Task I_can_get_an_individual_closed_caption_part_from_a_video()
var manifest = await youtube.Videos.ClosedCaptions.GetManifestAsync(
VideoIds.WithClosedCaptions
);
var trackInfo = manifest.GetByLanguage("en");

var trackInfo = manifest.GetByLanguage("en");
var track = await youtube.Videos.ClosedCaptions.GetAsync(trackInfo);

var captionPart = track
Expand All @@ -141,8 +141,8 @@ public async Task I_can_download_a_specific_closed_caption_track_from_a_video()
var manifest = await youtube.Videos.ClosedCaptions.GetManifestAsync(
VideoIds.WithClosedCaptions
);
var trackInfo = manifest.GetByLanguage("en-US");

var trackInfo = manifest.GetByLanguage("en-US");
await youtube.Videos.ClosedCaptions.DownloadAsync(trackInfo, file.Path);

// Assert
Expand Down
6 changes: 3 additions & 3 deletions YoutubeExplode.Tests/StreamSpecs.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,6 @@ public async Task I_can_get_the_list_of_available_streams_of_a_video()

// Assert
manifest.Streams.Should().NotBeEmpty();
manifest.GetMuxedStreams().Should().NotBeEmpty();
manifest.GetAudioStreams().Should().NotBeEmpty();
manifest.GetVideoStreams().Should().NotBeEmpty();

manifest
.GetVideoStreams()
Expand Down Expand Up @@ -71,6 +68,7 @@ public async Task I_can_get_the_list_of_available_streams_of_a_video()
[InlineData(VideoIds.AgeRestrictedSexual)]
[InlineData(VideoIds.AgeRestrictedEmbedRestricted)]
[InlineData(VideoIds.LiveStreamRecording)]
[InlineData(VideoIds.WithBrokenStreams)]
[InlineData(VideoIds.WithOmnidirectionalStreams)]
[InlineData(VideoIds.WithHighDynamicRangeStreams)]
public async Task I_can_get_the_list_of_available_streams_of_any_playable_video(string videoId)
Expand Down Expand Up @@ -134,6 +132,7 @@ public async Task I_can_try_to_get_the_list_of_available_streams_of_a_video_and_
[InlineData(VideoIds.AgeRestrictedViolent)]
[InlineData(VideoIds.AgeRestrictedSexual)]
[InlineData(VideoIds.LiveStreamRecording)]
[InlineData(VideoIds.WithBrokenStreams)]
[InlineData(VideoIds.WithOmnidirectionalStreams)]
public async Task I_can_get_a_specific_stream_of_a_video(string videoId)
{
Expand Down Expand Up @@ -163,6 +162,7 @@ public async Task I_can_get_a_specific_stream_of_a_video(string videoId)
[InlineData(VideoIds.AgeRestrictedSexual)]
[InlineData(VideoIds.AgeRestrictedEmbedRestricted)]
[InlineData(VideoIds.LiveStreamRecording)]
[InlineData(VideoIds.WithBrokenStreams)]
[InlineData(VideoIds.WithOmnidirectionalStreams)]
public async Task I_can_download_a_specific_stream_of_a_video(string videoId)
{
Expand Down
1 change: 1 addition & 0 deletions YoutubeExplode.Tests/TestData/VideoIds.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ internal static class VideoIds
public const string LiveStream = "jfKfPfyJRdk";
public const string LiveStreamRecording = "rsAAeyAr-9Y";
public const string WithBrokenTitle = "4ZJWv6t-PfY";
public const string WithBrokenStreams = "JQgKhZZyBYg";
public const string WithHighQualityStreams = "V5Fsj_sCKdg";
public const string WithOmnidirectionalStreams = "-xNN-bJQ4vI";
public const string WithHighDynamicRangeStreams = "vX2vsvdq8nw";
Expand Down
15 changes: 0 additions & 15 deletions YoutubeExplode/Utils/Extensions/HttpExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,19 +59,4 @@ public static async ValueTask<HttpResponseMessage> HeadAsync(
cancellationToken
);
}

public static async ValueTask<long?> TryGetContentLengthAsync(
this HttpClient http,
string requestUri,
bool ensureSuccess = true,
CancellationToken cancellationToken = default
)
{
using var response = await http.HeadAsync(requestUri, cancellationToken);

if (ensureSuccess)
response.EnsureSuccessStatusCode();

return response.Content.Headers.ContentLength;
}
}
28 changes: 16 additions & 12 deletions YoutubeExplode/Videos/Streams/MediaStream.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,14 @@
namespace YoutubeExplode.Videos.Streams;

// Works around YouTube's rate throttling, provides seeking support, and some resiliency
internal class MediaStream(HttpClient http, IStreamInfo streamInfo) : Stream
internal partial class MediaStream(HttpClient http, IStreamInfo streamInfo) : Stream
{
// For most streams, YouTube limits transfer speed to match the video playback rate.
// This helps them avoid unnecessary bandwidth, but for us it's a hindrance because
// we want to download the stream as fast as possible.
// To solve this, we divide the logical stream up into multiple segments and download
// them all separately.

private readonly long _segmentLength = streamInfo.IsThrottled()
? 9_898_989
: streamInfo.Size.Bytes;
Expand All @@ -31,12 +37,6 @@ internal class MediaStream(HttpClient http, IStreamInfo streamInfo) : Stream

public override long Position { get; set; }

// For most streams, YouTube limits transfer speed to match the video playback rate.
// This helps them avoid unnecessary bandwidth, but for us it's a hindrance because
// we want to download the stream as fast as possible.
// To solve this, we divide the logical stream up into multiple segments and download
// them all separately.

private void ResetSegment()
{
_segmentStream?.Dispose();
Expand All @@ -50,10 +50,7 @@ private async ValueTask<Stream> ResolveSegmentAsync(
if (_segmentStream is not null)
return _segmentStream;

var from = Position;
var to = Position + _segmentLength - 1;
var url = UrlEx.SetQueryParameter(streamInfo.Url, "range", $"{from}-{to}");

var url = GetSegmentUrl(streamInfo.Url, Position, Position + _segmentLength - 1);
var stream = await http.GetStreamAsync(url, cancellationToken);

return _segmentStream = stream;
Expand All @@ -77,7 +74,8 @@ private async ValueTask<int> ReadSegmentAsync(
return await stream.ReadAsync(buffer, offset, count, cancellationToken);
}
// Retry on connectivity issues
catch (IOException) when (retriesRemaining > 0)
catch (Exception ex)
when (ex is HttpRequestException or IOException && retriesRemaining > 0)
{
ResetSegment();
}
Expand Down Expand Up @@ -147,3 +145,9 @@ protected override void Dispose(bool disposing)
base.Dispose(disposing);
}
}

internal partial class MediaStream
{
public static string GetSegmentUrl(string streamUrl, long from, long to) =>
UrlEx.SetQueryParameter(streamUrl, "range", $"{from}-{to}");
}
Loading