diff --git a/YoutubeExplode.Tests/ClosedCaptionSpecs.cs b/YoutubeExplode.Tests/ClosedCaptionSpecs.cs index 3b86dead..ca1613a5 100644 --- a/YoutubeExplode.Tests/ClosedCaptionSpecs.cs +++ b/YoutubeExplode.Tests/ClosedCaptionSpecs.cs @@ -62,8 +62,8 @@ public async Task I_can_get_a_specific_closed_caption_track_from_a_video() var manifest = await youtube.Videos.ClosedCaptions.GetManifestAsync( VideoIds.WithClosedCaptions ); - var trackInfo = manifest.GetByLanguage("en-US"); + var trackInfo = manifest.GetByLanguage("en-US"); var track = await youtube.Videos.ClosedCaptions.GetAsync(trackInfo); // Assert @@ -80,8 +80,8 @@ public async Task I_can_get_a_specific_closed_caption_track_from_a_video_that_ha var manifest = await youtube.Videos.ClosedCaptions.GetManifestAsync( VideoIds.WithBrokenClosedCaptions ); - var trackInfo = manifest.GetByLanguage("en"); + var trackInfo = manifest.GetByLanguage("en"); var track = await youtube.Videos.ClosedCaptions.GetAsync(trackInfo); // Assert @@ -98,8 +98,8 @@ public async Task I_can_get_an_individual_closed_caption_from_a_video() var manifest = await youtube.Videos.ClosedCaptions.GetManifestAsync( VideoIds.WithClosedCaptions ); - var trackInfo = manifest.GetByLanguage("en-US"); + var trackInfo = manifest.GetByLanguage("en-US"); var track = await youtube.Videos.ClosedCaptions.GetAsync(trackInfo); var caption = track.GetByTime(TimeSpan.FromSeconds(641)); @@ -118,8 +118,8 @@ public async Task I_can_get_an_individual_closed_caption_part_from_a_video() var manifest = await youtube.Videos.ClosedCaptions.GetManifestAsync( VideoIds.WithClosedCaptions ); - var trackInfo = manifest.GetByLanguage("en"); + var trackInfo = manifest.GetByLanguage("en"); var track = await youtube.Videos.ClosedCaptions.GetAsync(trackInfo); var captionPart = track @@ -141,8 +141,8 @@ public async Task I_can_download_a_specific_closed_caption_track_from_a_video() var manifest = await youtube.Videos.ClosedCaptions.GetManifestAsync( VideoIds.WithClosedCaptions ); - var trackInfo = manifest.GetByLanguage("en-US"); + var trackInfo = manifest.GetByLanguage("en-US"); await youtube.Videos.ClosedCaptions.DownloadAsync(trackInfo, file.Path); // Assert diff --git a/YoutubeExplode.Tests/StreamSpecs.cs b/YoutubeExplode.Tests/StreamSpecs.cs index d57759e3..36c5a555 100644 --- a/YoutubeExplode.Tests/StreamSpecs.cs +++ b/YoutubeExplode.Tests/StreamSpecs.cs @@ -27,9 +27,6 @@ public async Task I_can_get_the_list_of_available_streams_of_a_video() // Assert manifest.Streams.Should().NotBeEmpty(); - manifest.GetMuxedStreams().Should().NotBeEmpty(); - manifest.GetAudioStreams().Should().NotBeEmpty(); - manifest.GetVideoStreams().Should().NotBeEmpty(); manifest .GetVideoStreams() @@ -71,6 +68,7 @@ public async Task I_can_get_the_list_of_available_streams_of_a_video() [InlineData(VideoIds.AgeRestrictedSexual)] [InlineData(VideoIds.AgeRestrictedEmbedRestricted)] [InlineData(VideoIds.LiveStreamRecording)] + [InlineData(VideoIds.WithBrokenStreams)] [InlineData(VideoIds.WithOmnidirectionalStreams)] [InlineData(VideoIds.WithHighDynamicRangeStreams)] public async Task I_can_get_the_list_of_available_streams_of_any_playable_video(string videoId) @@ -134,6 +132,7 @@ public async Task I_can_try_to_get_the_list_of_available_streams_of_a_video_and_ [InlineData(VideoIds.AgeRestrictedViolent)] [InlineData(VideoIds.AgeRestrictedSexual)] [InlineData(VideoIds.LiveStreamRecording)] + [InlineData(VideoIds.WithBrokenStreams)] [InlineData(VideoIds.WithOmnidirectionalStreams)] public async Task I_can_get_a_specific_stream_of_a_video(string videoId) { @@ -163,6 +162,7 @@ public async Task I_can_get_a_specific_stream_of_a_video(string videoId) [InlineData(VideoIds.AgeRestrictedSexual)] [InlineData(VideoIds.AgeRestrictedEmbedRestricted)] [InlineData(VideoIds.LiveStreamRecording)] + [InlineData(VideoIds.WithBrokenStreams)] [InlineData(VideoIds.WithOmnidirectionalStreams)] public async Task I_can_download_a_specific_stream_of_a_video(string videoId) { diff --git a/YoutubeExplode.Tests/TestData/VideoIds.cs b/YoutubeExplode.Tests/TestData/VideoIds.cs index bbf30715..9beef14f 100644 --- a/YoutubeExplode.Tests/TestData/VideoIds.cs +++ b/YoutubeExplode.Tests/TestData/VideoIds.cs @@ -15,6 +15,7 @@ internal static class VideoIds public const string LiveStream = "jfKfPfyJRdk"; public const string LiveStreamRecording = "rsAAeyAr-9Y"; public const string WithBrokenTitle = "4ZJWv6t-PfY"; + public const string WithBrokenStreams = "JQgKhZZyBYg"; public const string WithHighQualityStreams = "V5Fsj_sCKdg"; public const string WithOmnidirectionalStreams = "-xNN-bJQ4vI"; public const string WithHighDynamicRangeStreams = "vX2vsvdq8nw"; diff --git a/YoutubeExplode/Utils/Extensions/HttpExtensions.cs b/YoutubeExplode/Utils/Extensions/HttpExtensions.cs index 16a9a77b..3146a053 100644 --- a/YoutubeExplode/Utils/Extensions/HttpExtensions.cs +++ b/YoutubeExplode/Utils/Extensions/HttpExtensions.cs @@ -59,19 +59,4 @@ public static async ValueTask HeadAsync( cancellationToken ); } - - public static async ValueTask TryGetContentLengthAsync( - this HttpClient http, - string requestUri, - bool ensureSuccess = true, - CancellationToken cancellationToken = default - ) - { - using var response = await http.HeadAsync(requestUri, cancellationToken); - - if (ensureSuccess) - response.EnsureSuccessStatusCode(); - - return response.Content.Headers.ContentLength; - } } diff --git a/YoutubeExplode/Videos/Streams/MediaStream.cs b/YoutubeExplode/Videos/Streams/MediaStream.cs index 7ca3fe04..0f729aa2 100644 --- a/YoutubeExplode/Videos/Streams/MediaStream.cs +++ b/YoutubeExplode/Videos/Streams/MediaStream.cs @@ -9,8 +9,14 @@ namespace YoutubeExplode.Videos.Streams; // Works around YouTube's rate throttling, provides seeking support, and some resiliency -internal class MediaStream(HttpClient http, IStreamInfo streamInfo) : Stream +internal partial class MediaStream(HttpClient http, IStreamInfo streamInfo) : Stream { + // For most streams, YouTube limits transfer speed to match the video playback rate. + // This helps them avoid unnecessary bandwidth, but for us it's a hindrance because + // we want to download the stream as fast as possible. + // To solve this, we divide the logical stream up into multiple segments and download + // them all separately. + private readonly long _segmentLength = streamInfo.IsThrottled() ? 9_898_989 : streamInfo.Size.Bytes; @@ -31,12 +37,6 @@ internal class MediaStream(HttpClient http, IStreamInfo streamInfo) : Stream public override long Position { get; set; } - // For most streams, YouTube limits transfer speed to match the video playback rate. - // This helps them avoid unnecessary bandwidth, but for us it's a hindrance because - // we want to download the stream as fast as possible. - // To solve this, we divide the logical stream up into multiple segments and download - // them all separately. - private void ResetSegment() { _segmentStream?.Dispose(); @@ -50,10 +50,7 @@ private async ValueTask ResolveSegmentAsync( if (_segmentStream is not null) return _segmentStream; - var from = Position; - var to = Position + _segmentLength - 1; - var url = UrlEx.SetQueryParameter(streamInfo.Url, "range", $"{from}-{to}"); - + var url = GetSegmentUrl(streamInfo.Url, Position, Position + _segmentLength - 1); var stream = await http.GetStreamAsync(url, cancellationToken); return _segmentStream = stream; @@ -77,7 +74,8 @@ private async ValueTask ReadSegmentAsync( return await stream.ReadAsync(buffer, offset, count, cancellationToken); } // Retry on connectivity issues - catch (IOException) when (retriesRemaining > 0) + catch (Exception ex) + when (ex is HttpRequestException or IOException && retriesRemaining > 0) { ResetSegment(); } @@ -147,3 +145,9 @@ protected override void Dispose(bool disposing) base.Dispose(disposing); } } + +internal partial class MediaStream +{ + public static string GetSegmentUrl(string streamUrl, long from, long to) => + UrlEx.SetQueryParameter(streamUrl, "range", $"{from}-{to}"); +} diff --git a/YoutubeExplode/Videos/Streams/StreamClient.cs b/YoutubeExplode/Videos/Streams/StreamClient.cs index f0fc04f2..09d89f83 100644 --- a/YoutubeExplode/Videos/Streams/StreamClient.cs +++ b/YoutubeExplode/Videos/Streams/StreamClient.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Net; using System.Net.Http; using System.Runtime.CompilerServices; using System.Threading; @@ -50,6 +51,48 @@ CancellationToken cancellationToken ?? throw new YoutubeExplodeException("Failed to extract the cipher manifest."); } + private async ValueTask TryGetContentLengthAsync( + IStreamData streamData, + string url, + CancellationToken cancellationToken = default + ) + { + var contentLength = streamData.ContentLength; + + // If content length is not available in the metadata, get it by + // sending a HEAD request and parsing the Content-Length header. + if (contentLength is null) + { + using var response = await _http.HeadAsync(url, cancellationToken); + + // 404 error indicates that the stream is not available + if (response.StatusCode == HttpStatusCode.NotFound) + return null; + + response.EnsureSuccessStatusCode(); + } + + if (contentLength is not null) + { + // Streams may have mismatched content length, so ensure that the obtained value is correct + // https://github.com/Tyrrrz/YoutubeExplode/issues/759 + using var response = await _http.GetAsync( + // Try to access the last byte of the stream + MediaStream.GetSegmentUrl(url, contentLength.Value - 2, contentLength.Value - 1), + HttpCompletionOption.ResponseHeadersRead, + cancellationToken + ); + + // 404 error indicates that the stream has mismatched content length or is not available + if (response.StatusCode == HttpStatusCode.NotFound) + return null; + + response.EnsureSuccessStatusCode(); + } + + return contentLength; + } + private async IAsyncEnumerable GetStreamInfosAsync( IEnumerable streamDatas, [EnumeratorCancellation] CancellationToken cancellationToken = default @@ -77,13 +120,8 @@ private async IAsyncEnumerable GetStreamInfosAsync( ); } - var contentLength = - streamData.ContentLength - ?? await _http.TryGetContentLengthAsync(url, false, cancellationToken) - ?? 0; - - // Stream cannot be accessed - if (contentLength <= 0) + var contentLength = await TryGetContentLengthAsync(streamData, url, cancellationToken); + if (contentLength is null) continue; var container = @@ -114,7 +152,7 @@ streamData.VideoWidth is not null && streamData.VideoHeight is not null var streamInfo = new MuxedStreamInfo( url, container, - new FileSize(contentLength), + new FileSize(contentLength.Value), bitrate, streamData.AudioCodec, streamData.VideoCodec, @@ -130,7 +168,7 @@ streamData.VideoWidth is not null && streamData.VideoHeight is not null var streamInfo = new VideoOnlyStreamInfo( url, container, - new FileSize(contentLength), + new FileSize(contentLength.Value), bitrate, streamData.VideoCodec, videoQuality, @@ -146,7 +184,7 @@ streamData.VideoWidth is not null && streamData.VideoHeight is not null var streamInfo = new AudioOnlyStreamInfo( url, container, - new FileSize(contentLength), + new FileSize(contentLength.Value), bitrate, streamData.AudioCodec ); @@ -160,14 +198,13 @@ streamData.VideoWidth is not null && streamData.VideoHeight is not null } } - private async IAsyncEnumerable GetStreamInfosAsync( + private async ValueTask> GetStreamInfosAsync( VideoId videoId, - [EnumeratorCancellation] CancellationToken cancellationToken = default + PlayerResponse playerResponse, + CancellationToken cancellationToken = default ) { - var playerResponse = await _controller.GetPlayerResponseAsync(videoId, cancellationToken); - - // If the video is pay-to-play, error out + // Video is pay-to-play if (!string.IsNullOrWhiteSpace(playerResponse.PreviewVideoId)) { throw new VideoRequiresPurchaseException( @@ -176,56 +213,85 @@ private async IAsyncEnumerable GetStreamInfosAsync( ); } - // If the video is unplayable, try one more time by fetching the player response - // with signature deciphering. This is (only) required for age-restricted videos. - if (!playerResponse.IsPlayable) - { - var cipherManifest = await ResolveCipherManifestAsync(cancellationToken); - playerResponse = await _controller.GetPlayerResponseAsync( - videoId, - cipherManifest.SignatureTimestamp, - cancellationToken - ); - } - - // If the video is still unplayable, error out + // Video is unplayable if (!playerResponse.IsPlayable) { throw new VideoUnplayableException( - $"Video '{videoId}' is unplayable. " - + $"Reason: '{playerResponse.PlayabilityError}'." + $"Video '{videoId}' is unplayable. Reason: '{playerResponse.PlayabilityError}'." ); } + var streamInfos = new List(); + // Extract streams from the player response await foreach ( var streamInfo in GetStreamInfosAsync(playerResponse.Streams, cancellationToken) ) - yield return streamInfo; + { + streamInfos.Add(streamInfo); + } // Extract streams from the DASH manifest if (!string.IsNullOrWhiteSpace(playerResponse.DashManifestUrl)) { - var dashManifest = default(DashManifest?); - try { - dashManifest = await _controller.GetDashManifestAsync( + var dashManifest = await _controller.GetDashManifestAsync( playerResponse.DashManifestUrl, cancellationToken ); - } - // Some DASH manifest URLs return 404 for whatever reason - // https://github.com/Tyrrrz/YoutubeExplode/issues/728 - catch (HttpRequestException) { } - if (dashManifest is not null) - { await foreach ( var streamInfo in GetStreamInfosAsync(dashManifest.Streams, cancellationToken) ) - yield return streamInfo; + { + streamInfos.Add(streamInfo); + } } + // Some DASH manifest URLs return 404 for whatever reason + // https://github.com/Tyrrrz/YoutubeExplode/issues/728 + catch (HttpRequestException) { } + } + + // Error if no streams were found + if (!streamInfos.Any()) + { + throw new VideoUnplayableException( + $"Video '{videoId}' does not contain any playable streams." + ); + } + + return streamInfos; + } + + private async ValueTask> GetStreamInfosAsync( + VideoId videoId, + CancellationToken cancellationToken = default + ) + { + // Try to get player response from a cipher-less client + try + { + var playerResponse = await _controller.GetPlayerResponseAsync( + videoId, + cancellationToken + ); + + return await GetStreamInfosAsync(videoId, playerResponse, cancellationToken); + } + catch (VideoUnplayableException) { } + + // Try to get player response from a client with cipher + { + var cipherManifest = await ResolveCipherManifestAsync(cancellationToken); + + var playerResponse = await _controller.GetPlayerResponseAsync( + videoId, + cipherManifest.SignatureTimestamp, + cancellationToken + ); + + return await GetStreamInfosAsync(videoId, playerResponse, cancellationToken); } } @@ -239,25 +305,13 @@ public async ValueTask GetManifestAsync( { for (var retriesRemaining = 5; ; retriesRemaining--) { - var streamInfos = await GetStreamInfosAsync(videoId, cancellationToken); - - if (!streamInfos.Any()) + try { - throw new VideoUnplayableException( - $"Video '{videoId}' does not contain any playable streams." - ); + return new StreamManifest(await GetStreamInfosAsync(videoId, cancellationToken)); } - - // YouTube sometimes returns stream URLs that produce 403 Forbidden errors when accessed. - // This happens for both protected and non-protected streams, so the cause is unclear. - // As a workaround, we can access one of the stream URLs and retry if it fails. - using var response = await _http.HeadAsync(streamInfos.First().Url, cancellationToken); - if ((int)response.StatusCode == 403 && retriesRemaining > 0) - continue; - - response.EnsureSuccessStatusCode(); - - return new StreamManifest(streamInfos); + // Retry on connectivity issues + catch (Exception ex) + when (ex is HttpRequestException or IOException && retriesRemaining > 0) { } } } @@ -273,16 +327,14 @@ public async ValueTask GetHttpLiveStreamUrlAsync( if (!playerResponse.IsPlayable) { throw new VideoUnplayableException( - $"Video '{videoId}' is unplayable. " - + $"Reason: '{playerResponse.PlayabilityError}'." + $"Video '{videoId}' is unplayable. Reason: '{playerResponse.PlayabilityError}'." ); } if (string.IsNullOrWhiteSpace(playerResponse.HlsManifestUrl)) { throw new YoutubeExplodeException( - "Failed to extract the HTTP Live Stream manifest URL. " - + $"Video '{videoId}' is likely not a live stream." + $"Failed to extract the HTTP Live Stream manifest URL. Video '{videoId}' is likely not a live stream." ); } diff --git a/YoutubeExplode/YoutubeHttpHandler.cs b/YoutubeExplode/YoutubeHttpHandler.cs index a5f8c246..9e470ae6 100644 --- a/YoutubeExplode/YoutubeHttpHandler.cs +++ b/YoutubeExplode/YoutubeHttpHandler.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Net; using System.Net.Http; @@ -186,7 +187,8 @@ await base.SendAsync( return response; } // Retry on connectivity issues - catch (HttpRequestException) when (retriesRemaining > 0) { } + catch (Exception ex) + when (ex is HttpRequestException or IOException && retriesRemaining > 0) { } } } }