Implement YoutubeApiInterface, Add ytClient parameter to getManifest,…

… getManifest does no longer throw, add android_music client.
Hexer10 · Oct 9, 2024 · 8a485fb · 8a485fb
1 parent 41e4475
commit 8a485fb
Show file tree

Hide file tree

Showing 8 changed files with 173 additions and 166 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,9 @@
+## 2.3.0
+- BREAKING CHANGE: Now if the manifest cannot be fetched an exception is not thrown but an empty list is returned.
+- Implement `YoutubeApiClient` interface.
+- Add `ytClient` parameter to `StreamClient.getManifest`
+- Workaround: implement `android_music` to fetch muxed streams for music videos.
+
 ## 2.2.3
 - Impersonate ios client to extract manifest.
 

diff --git a/example/video_download.dart b/example/video_download.dart
@@ -44,6 +44,7 @@ Future<void> download(String id) async {
       .replaceAll('"', '')
       .replaceAll('<', '')
       .replaceAll('>', '')
+      .replaceAll(':', '')
       .replaceAll('|', '');
   final file = File('downloads/$fileName');
 

diff --git a/lib/src/retry.dart b/lib/src/retry.dart
@@ -37,5 +37,8 @@ int getExceptionCost(Exception e) {
   if (e is FatalFailureException) {
     return 3;
   }
+  if (e is VideoUnplayableException) {
+    return 4;
+  }
   return 1;
 }
diff --git a/lib/src/reverse_engineering/youtube_http_client.dart b/lib/src/reverse_engineering/youtube_http_client.dart
@@ -3,6 +3,7 @@ import 'dart:convert';
 
 import 'package:collection/collection.dart';
 import 'package:http/http.dart' as http;
+import 'package:logging/logging.dart';
 
 import '../exceptions/exceptions.dart';
 import '../extensions/helpers_extension.dart';
@@ -12,6 +13,7 @@ import '../videos/streams/streams.dart';
 /// HttpClient wrapper for YouTube
 class YoutubeHttpClient extends http.BaseClient {
   final http.Client _httpClient;
+  final _logger = Logger('YoutubeExplode.HttpClient');
 
   // Flag to interrupt receiving stream.
   bool _closed = false;
@@ -341,8 +343,7 @@ class YoutubeHttpClient extends http.BaseClient {
       }
     });
 
-    // print(request);
-    // print(StackTrace.current);
+    _logger.fine('Sending request: ${request.url}', null, StackTrace.current);
     return _httpClient.send(request);
   }
-}
+}
diff --git a/lib/src/videos/streams/stream_client.dart b/lib/src/videos/streams/stream_client.dart
@@ -1,5 +1,7 @@
 import 'dart:collection';
 
+import 'package:logging/logging.dart';
+
 import '../../exceptions/exceptions.dart';
 import '../../extensions/helpers_extension.dart';
 import '../../retry.dart';
@@ -8,13 +10,14 @@ import '../../reverse_engineering/heuristics.dart';
 import '../../reverse_engineering/models/stream_info_provider.dart';
 import '../../reverse_engineering/pages/watch_page.dart';
 import '../../reverse_engineering/youtube_http_client.dart';
-import '../video_controller.dart';
 import '../video_id.dart';
+import '../youtube_api_client.dart';
 import 'stream_controller.dart';
 import 'streams.dart';
 
 /// Queries related to media streams of YouTube videos.
 class StreamClient {
+  final _logger = Logger('YoutubeExplode.StreamsClient');
   final YoutubeHttpClient _httpClient;
   final StreamController _controller;
 
@@ -26,48 +29,71 @@ class StreamClient {
   /// Gets the manifest that contains information
   /// about available streams in the specified video.
   ///
-  /// If [fullManifest] is set to `true`, more streams types will be fetched
-  /// and track of different languages (if present) will be included.
+  /// See [YoutubeApiClient] for all the possible clients.
+  /// By default ios and android_music are used.
+  ///
+  /// If the extraction fails an empty list is returned, to diagnose the issue enabled the logging from the `logging` package, and open an issue with the output.
+  /// For example:
+  /// ```dart
+  /// Logger.root.level = Level.ALL
+  /// Logger.root.onRecord.listen(print);
+  /// // run yt related code ...
+  ///
+  /// ```
   Future<StreamManifest> getManifest(dynamic videoId,
-      {bool fullManifest = false}) {
+      {@Deprecated(
+          'Use the ytClient parameter instead passing the proper [YoutubeApiClient]s')
+      bool fullManifest = false,
+      List<YoutubeApiClient> ytClients = const [
+        YoutubeApiClient.androidMusic,
+        YoutubeApiClient.iosClient,
+      ]}) async {
     videoId = VideoId.fromString(videoId);
 
-    return retry(_httpClient, () async {
-      final streams =
-          await _getStreams(videoId, fullManifest: fullManifest).toList();
-      if (streams.isEmpty) {
-        throw VideoUnavailableException(
-          'Video "$videoId" does not contain any playable streams.',
-        );
-      }
-
-      final response = await _httpClient.head(streams.first.url);
-      if (response.statusCode == 403) {
-        throw YoutubeExplodeException(
-          'Video $videoId returned 403 (stream: ${streams.first.tag}',
-        );
-      }
+    final uniqueStreams = LinkedHashSet<StreamInfo>(
+      equals: (a, b) {
+        if (a.runtimeType != b.runtimeType) return false;
+        if (a is AudioStreamInfo && b is AudioStreamInfo) {
+          return a.tag == b.tag && a.audioTrack == b.audioTrack;
+        }
+        return a.tag == b.tag;
+      },
+      hashCode: (e) {
+        if (e is AudioStreamInfo) {
+          return e.tag.hashCode ^ e.audioTrack.hashCode;
+        }
+        return e.tag.hashCode;
+      },
+    );
 
-      // Remove duplicate streams (must have same tag and audioTrack (if it's an audio stream))
-      final uniqueStreams = LinkedHashSet<StreamInfo>(
-        equals: (a, b) {
-          if (a.runtimeType != b.runtimeType) return false;
-          if (a is AudioStreamInfo && b is AudioStreamInfo) {
-            return a.tag == b.tag && a.audioTrack == b.audioTrack;
-          }
-          return a.tag == b.tag;
-        },
-        hashCode: (e) {
-          if (e is AudioStreamInfo) {
-            return e.tag.hashCode ^ e.audioTrack.hashCode;
+    for (final client in ytClients) {
+      _logger.fine(
+          'Getting stream manifest for video $videoId with client: ${client.payload['context']['client']['clientName']}');
+      try {
+        await retry(_httpClient, () async {
+          final streams = await _getStreams(videoId, ytClient: client).toList();
+          if (streams.isEmpty) {
+            throw VideoUnavailableException(
+              'Video "$videoId" does not contain any playable streams.',
+            );
           }
-          return e.tag.hashCode;
-        },
-      );
-      uniqueStreams.addAll(streams);
 
-      return StreamManifest(uniqueStreams.toList());
-    });
+          final response = await _httpClient.head(streams.first.url);
+          if (response.statusCode == 403) {
+            throw YoutubeExplodeException(
+              'Video $videoId returned 403 (stream: ${streams.first.tag})',
+            );
+          }
+          uniqueStreams.addAll(streams);
+        });
+      } on YoutubeExplodeException catch (e, s) {
+        _logger.severe(
+            'Failed to get stream manifest for video $videoId with client: ${client.payload['context']['client']['clientName']}. Reason: $e\n',
+            e,
+            s);
+      }
+    }
+    return StreamManifest(uniqueStreams.toList());
   }
 
   /// Gets the HTTP Live Stream (HLS) manifest URL
@@ -117,29 +143,15 @@ class StreamClient {
   }
 
   Stream<StreamInfo> _getStreams(VideoId videoId,
-      {required bool fullManifest}) async* {
-    try {
-      // Use await for instead of yield* to catch exceptions
-      await for (final stream
-          in _getStream(videoId, VideoController.iosClient)) {
-        yield stream;
-      }
-      if (fullManifest) {
-        await for (final stream
-            in _getStream(videoId, VideoController.androidClient)) {
-          yield stream;
-        }
-      }
-    } on VideoUnplayableException catch (e) {
-      if (e is VideoRequiresPurchaseException) {
-        rethrow;
-      }
-      yield* _getCipherStream(videoId);
+      {required YoutubeApiClient ytClient}) async* {
+    // Use await for instead of yield* to catch exceptions
+    await for (final stream in _getStream(videoId, ytClient)) {
+      yield stream;
     }
   }
 
-  Stream<StreamInfo> _getStream(VideoId videoId,
-      Map<String, Map<String, Map<String, Object>>> ytClient) async* {
+  Stream<StreamInfo> _getStream(
+      VideoId videoId, YoutubeApiClient ytClient) async* {
     final playerResponse =
         await _controller.getPlayerResponse(videoId, ytClient);
     if (!playerResponse.previewVideoId.isNullOrWhiteSpace) {
@@ -168,23 +180,6 @@ class StreamClient {
     }
   }
 
-  Stream<StreamInfo> _getCipherStream(VideoId videoId) async* {
-    final cipherManifest = await _getCipherManifest();
-    final playerResponse = await _controller.getPlayerResponseWithSignature(
-      videoId,
-      cipherManifest.signatureTimestamp,
-    );
-
-    if (!playerResponse.isVideoPlayable) {
-      throw VideoUnplayableException.unplayable(
-        videoId,
-        reason: playerResponse.videoPlayabilityError ?? '',
-      );
-    }
-
-    yield* _parseStreamInfo(playerResponse.streams, videoId);
-  }
-
   Stream<StreamInfo> _parseStreamInfo(
       Iterable<StreamInfoProvider> streams, VideoId videoId) async* {
     for (final stream in streams) {

diff --git a/lib/src/videos/video_controller.dart b/lib/src/videos/video_controller.dart
@@ -3,60 +3,10 @@ import '../../youtube_explode_dart.dart';
 
 import '../reverse_engineering/pages/watch_page.dart';
 import '../reverse_engineering/player/player_response.dart';
+import 'youtube_api_client.dart';
 
 @internal
 class VideoController {
-  /// Used to fetch streams without signature deciphering, but has limited streams.
-  static const iosClient = {
-    "context": {
-      "client": {
-        "clientName": "IOS",
-        "clientVersion": "19.29.1",
-        "deviceMake": "Apple",
-        "deviceModel": "iPhone16,2",
-        "hl": "en",
-        "osName": "iPhone",
-        "osVersion": "17.5.1.21F90",
-        "timeZone": "UTC",
-        "userAgent": "com.google.ios.youtube/19.29.1 (iPhone16,2; U; CPU iOS 17_5_1 like Mac OS X;)",
-        "gl": "US",
-        "utcOffsetMinutes": 0
-      }
-    },
-  };
-
-  /// Used to to fetch all the video streams (but has signature deciphering).
-  static const androidClient = {
-    'context': {
-      'client': {
-        'clientName': 'ANDROID',
-        'clientVersion': '19.09.37',
-        'androidSdkVersion': 30,
-        'userAgent':
-            'com.google.android.youtube/19.09.37 (Linux; U; Android 11) gzip',
-        'hl': 'en',
-        'timeZone': 'UTC',
-        'utcOffsetMinutes': 0,
-      },
-    },
-  };
-
-  /// Used to fetch streams for age-restricted videos without authentication.
-  static const tvClient = {
-    'context': {
-      'client': {
-        'clientName': 'TVHTML5_SIMPLY_EMBEDDED_PLAYER',
-        'clientVersion': '2.0',
-        'hl': 'en',
-        'gl': 'US',
-        'utcOffsetMinutes': 0,
-      },
-      'thirdParty': {
-        'embedUrl': 'https://www.youtube.com',
-      },
-    },
-  };
-
   @protected
   final YoutubeHttpClient httpClient;
 
@@ -67,50 +17,22 @@ class VideoController {
   }
 
   Future<PlayerResponse> getPlayerResponse(VideoId videoId,
-      Map<String, Map<String, Map<String, Object>>> client) async {
-    assert(client['context'] != null, 'client must contain a context');
-    assert(client['context']!['client'] != null,
+      YoutubeApiClient client) async {
+    final payload = client.payload;
+    assert(payload['context'] != null, 'client must contain a context');
+    assert(payload['context']!['client'] != null,
         'client must contain a context.client');
 
-    final userAgent = client['context']!['client']!['userAgent'] as String?;
-    // From https://github.com/Tyrrrz/YoutubeExplode:
-    // The most optimal client to impersonate is the Android client, because
-    // it doesn't require signature deciphering (for both normal and n-parameter signatures).
-    // However, the regular Android client has a limitation, preventing it from downloading
-    // multiple streams from the same manifest (or the same stream multiple times).
-    // As a workaround, we're using ANDROID_TESTSUITE which appears to offer the same
-    // functionality, but doesn't impose the aforementioned limitation.
-    // https://github.com/Tyrrrz/YoutubeExplode/issues/705
-    final content = await httpClient.postString(
-      'https://www.youtube.com/youtubei/v1/player?key=AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w&prettyPrint=false',
+    final userAgent = payload['context']!['client']!['userAgent'] as String?;
+
+    final content = await httpClient.postString(client.apiUrl,
       body: {
-        ...client,
+        ...payload,
         'videoId': videoId.value,
       },
       headers: {
         if (userAgent != null) 'User-Agent': userAgent,
-      },
-    );
-    return PlayerResponse.parse(content);
-  }
-
-  Future<PlayerResponse> getPlayerResponseWithSignature(
-    VideoId videoId,
-    String? signatureTimestamp,
-  ) async {
-    /// The only client that can handle age-restricted videos without authentication is the
-    ///  TVHTML5_SIMPLY_EMBEDDED_PLAYER client.
-    ///  This client does require signature deciphering, so we only use it as a fallback.
-    final content = await httpClient.postString(
-      'https://www.youtube.com/youtubei/v1/player?key=AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w&prettyPrint=false',
-      body: {
-        ...tvClient,
-        'videoId': videoId.value,
-        'playbackContext': {
-          'contentPlaybackContext': {
-            'signatureTimestamp': signatureTimestamp ?? '19369',
-          },
-        },
+        ...client.headers,
       },
     );
     return PlayerResponse.parse(content);