From 3c5ce69b6ce48770bbc51399d09aafa8a15dbdf2 Mon Sep 17 00:00:00 2001 From: GautamMKGarg Date: Sun, 18 Sep 2022 00:06:16 +0530 Subject: [PATCH 1/6] Fixed issue #76 --- service.py | 101 ++++++++++++++++++++++++++++++++--------------------- 1 file changed, 61 insertions(+), 40 deletions(-) diff --git a/service.py b/service.py index 3db3a4d1a..bb9ddb0e9 100644 --- a/service.py +++ b/service.py @@ -51,6 +51,16 @@ def showErrorNotification(message): xbmcgui.NOTIFICATION_ERROR, 5000) +def get_user_input(): + kb = xbmc.Keyboard('', 'Please enter the URL') + kb.doModal() # Onscreen keyboard appears + if not kb.isConfirmed(): + return '' + + # User input + return kb.getText() + + # Get the plugin url in plugin:// notation. __url__ = sys.argv[0] # Get the plugin handle as an integer number. @@ -60,6 +70,10 @@ def showErrorNotification(message): def getParams(): result = {} paramstring = sys.argv[2] + + if paramstring == '': + paramstring = "?" + get_user_input() + additionalParamsIndex = paramstring.find(' ') if additionalParamsIndex == -1: result['url'] = paramstring[1:] @@ -72,6 +86,50 @@ def getParams(): return result +def play(url, ydl_opts): + if xbmcplugin.getSetting(int(sys.argv[1]),"usemanifest"): + ydl_opts['format'] = 'bestvideo*+bestaudio/best' + ydl = YoutubeDL(ydl_opts) + ydl.add_default_info_extractors() + + with ydl: + showInfoNotification("Resolving stream(s) for " + url) + result = ydl.extract_info(url, download=False) + + if 'entries' in result: + # more than one video + pl = xbmc.PlayList(1) + pl.clear() + + # determine which index in the queue to start playing from + indexToStartAt = playlistIndex(url, result) + if indexToStartAt == None: + indexToStartAt = 0 + + unresolvedEntries = list(result['entries']) + startingEntry = unresolvedEntries.pop(indexToStartAt) + + # populate the queue with unresolved entries so that the starting entry can be inserted + for video in unresolvedEntries: + list_item = createListItemFromFlatPlaylistItem(video) + pl.add(list_item.getPath(), list_item) + + # make sure the starting ListItem has a resolved url, to avoid recursion and crashes + startingVideoUrl = startingEntry['url'] + startingItem = createListItemFromVideo(ydl.extract_info(startingVideoUrl, download=False)) + pl.add(startingItem.getPath(), startingItem, indexToStartAt) + + #xbmc.Player().play(pl) # this probably works again + # ...but start playback the same way the Youtube plugin does it: + xbmc.executebuiltin('Playlist.PlayOffset(%s,%d)' % ('video', indexToStartAt)) + + showInfoNotification("Playing playlist " + result['title']) + else: + # Just a video, pass the item to the Kodi player. + showInfoNotification("Playing title " + result['title']) + xbmcplugin.setResolvedUrl(__handle__, True, listitem=createListItemFromVideo(result)) + + def extract_manifest_url(result): # sometimes there is an url directly # but for some extractors this is only one quality and sometimes not even a real manifest @@ -252,44 +310,7 @@ def playlistIndex(url, playlist): params = getParams() url = str(params['url']) ydl_opts.update(params['ydlOpts']) -if xbmcplugin.getSetting(int(sys.argv[1]),"usemanifest"): - ydl_opts['format'] = 'bestvideo*+bestaudio/best' -ydl = YoutubeDL(ydl_opts) -ydl.add_default_info_extractors() - -with ydl: - showInfoNotification("Resolving stream(s) for " + url) - result = ydl.extract_info(url, download=False) - -if 'entries' in result: - # more than one video - pl = xbmc.PlayList(1) - pl.clear() - - # determine which index in the queue to start playing from - indexToStartAt = playlistIndex(url, result) - if indexToStartAt == None: - indexToStartAt = 0 - - unresolvedEntries = list(result['entries']) - startingEntry = unresolvedEntries.pop(indexToStartAt) - - # populate the queue with unresolved entries so that the starting entry can be inserted - for video in unresolvedEntries: - list_item = createListItemFromFlatPlaylistItem(video) - pl.add(list_item.getPath(), list_item) - - # make sure the starting ListItem has a resolved url, to avoid recursion and crashes - startingVideoUrl = startingEntry['url'] - startingItem = createListItemFromVideo(ydl.extract_info(startingVideoUrl, download=False)) - pl.add(startingItem.getPath(), startingItem, indexToStartAt) - - #xbmc.Player().play(pl) # this probably works again - # ...but start playback the same way the Youtube plugin does it: - xbmc.executebuiltin('Playlist.PlayOffset(%s,%d)' % ('video', indexToStartAt)) - - showInfoNotification("Playing playlist " + result['title']) +if url == '': + showInfoNotification("Kindly provide the valid URL.") else: - # Just a video, pass the item to the Kodi player. - showInfoNotification("Playing title " + result['title']) - xbmcplugin.setResolvedUrl(__handle__, True, listitem=createListItemFromVideo(result)) \ No newline at end of file + play(url, ydl_opts) From 45c9150a8962129b89bf3a2927f428b0ec7621e7 Mon Sep 17 00:00:00 2001 From: Gautam Garg <65900807+knit-pay@users.noreply.github.com> Date: Thu, 22 Sep 2022 20:48:13 +0530 Subject: [PATCH 2/6] Implemented Code Review Changes --- service.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/service.py b/service.py index c824db9a8..5b2d8fd23 100644 --- a/service.py +++ b/service.py @@ -51,8 +51,8 @@ def showErrorNotification(message): xbmcgui.NOTIFICATION_ERROR, 5000) -def get_user_input(): - kb = xbmc.Keyboard('', 'Please enter the URL') +def get_local_user_input(): + kb = xbmc.Keyboard('', 'Please enter a URL') kb.doModal() # Onscreen keyboard appears if not kb.isConfirmed(): return '' @@ -71,8 +71,8 @@ def getParams(): result = {} paramstring = sys.argv[2] - if paramstring == '': - paramstring = "?" + get_user_input() + if not paramstring: + paramstring = "?" + get_local_user_input() additionalParamsIndex = paramstring.find(' ') if additionalParamsIndex == -1: From ce5b02bd7026c05df7f2c7e9aa6798870370b466 Mon Sep 17 00:00:00 2001 From: Gautam Garg <65900807+knit-pay@users.noreply.github.com> Date: Thu, 22 Sep 2022 21:17:51 +0530 Subject: [PATCH 3/6] Revert "Merge tag 'v0.9.285'" This reverts commit 343a6f54e0f2b487a9dbe9bd4d5124f7cdc99e5c, reversing changes made to 3c5ce69b6ce48770bbc51399d09aafa8a15dbdf2. --- README.md | 10 +- lib/yt_dlp/extractor/_extractors.py | 2 +- lib/yt_dlp/extractor/goplay.py | 395 ---------------------------- lib/yt_dlp/extractor/tiktok.py | 34 ++- lib/yt_dlp/extractor/vier.py | 261 ++++++++++++++++++ lib/yt_dlp_version | 2 +- service.py | 83 +++--- 7 files changed, 324 insertions(+), 463 deletions(-) delete mode 100644 lib/yt_dlp/extractor/goplay.py create mode 100644 lib/yt_dlp/extractor/vier.py diff --git a/README.md b/README.md index 18f97c926..cca0d83ff 100644 --- a/README.md +++ b/README.md @@ -2,19 +2,13 @@ [![build-publish-addon](https://github.com/firsttris/plugin.video.sendtokodi/actions/workflows/build-publish.yml/badge.svg)](https://github.com/firsttris/plugin.video.sendtokodi/actions/workflows/build-publish.yml) -This add-on receives video or audio URLs and plays them in [Kodi](https://kodi.tv). It resolves sent websites automatically with [yt-dlp](https://github.com/yt-dlp/yt-dlp) into a playable stream. The URLs can be sent with one of the supported apps listed below. - -drawing - - -## Installation -The plugin is not in the offical kodi addon repo, to install it with automatic updates you need to add our repo first. Download the repo file for your kodi version and [install the repo from zip](https://kodi.wiki/view/Add-on_manager). Afterwards the addon `sendtokodi` can be found in the [install from repository](https://kodi.wiki/view/Add-on_manager) section. +This [kodi](https://github.com/xbmc/xbmc) plugin receives URLs and resolves almost all of them with [yt-dlp](https://github.com/yt-dlp/yt-dlp) creating a playable video stream for kodi. The URLs can be send with one of the supported apps listed below. The plugin itself is not in the offical kodi addon repo, to install it with automatic updates you need to add our repo first. Download the repo file for your kodi version and [install it from zip](https://kodi.wiki/view/Add-on_manager). Afterwards the repositoy should be listed and can be selected to install the actual plugin itself. [Download repo for kodi 18](https://github.com/firsttris/repository.sendtokodi/raw/master/repository.sendtokodi/repository.sendtokodi-0.0.1.zip) [Download repo for kodi 19+](https://github.com/firsttris/repository.sendtokodi.python3/raw/master/repository.sendtokodi.python3/repository.sendtokodi.python3-0.0.1.zip) -*Please note that kodi 18 is internally limited to python2 but the addon uses yt-dlp to resolve URLs which requires python 3.6+. Therefore, the kodi 18 version uses [youtube-dl](https://youtube-dl.org/) instead. Unfortunately, the development of youtube-dl was stuck but it has been resumed. So the kodi 18 version of this plugin might not be as stable as the kodi 19 version.* +*Please note that kodi 18 is limited to python 2 only, but the used URL resolver yt-dlp requires python 3.6+. Therefore, the kodi 18 version uses [youtube-dl](https://youtube-dl.org/) instead. Unfortunately, the development of youtube-dl was stuck and development has just been resumed lately. So the kodi 18 version of this plugin might not be as stable as the kodi 19 version.* ## Features - [supported sites](https://github.com/yt-dlp/yt-dlp/blob/master/supportedsites.md) diff --git a/lib/yt_dlp/extractor/_extractors.py b/lib/yt_dlp/extractor/_extractors.py index 43e2f93d3..6bf769a9e 100644 --- a/lib/yt_dlp/extractor/_extractors.py +++ b/lib/yt_dlp/extractor/_extractors.py @@ -649,7 +649,6 @@ ) from .googlesearch import GoogleSearchIE from .gopro import GoProIE -from .goplay import GoPlayIE from .goshgay import GoshgayIE from .gotostage import GoToStageIE from .gputechconf import GPUTechConfIE @@ -2022,6 +2021,7 @@ VidioLiveIE ) from .vidlii import VidLiiIE +from .vier import VierIE, VierVideosIE from .viewlift import ( ViewLiftIE, ViewLiftEmbedIE, diff --git a/lib/yt_dlp/extractor/goplay.py b/lib/yt_dlp/extractor/goplay.py deleted file mode 100644 index 31267e1aa..000000000 --- a/lib/yt_dlp/extractor/goplay.py +++ /dev/null @@ -1,395 +0,0 @@ -import base64 -import binascii -import datetime -import hashlib -import hmac -import json -import os - -from .common import InfoExtractor -from ..utils import ( - ExtractorError, - traverse_obj, - unescapeHTML, -) - - -class GoPlayIE(InfoExtractor): - _VALID_URL = r'https?://(www\.)?goplay\.be/video/([^/]+/[^/]+/|)(?P[^/#]+)' - - _NETRC_MACHINE = 'goplay' - - _TESTS = [{ - 'url': 'https://www.goplay.be/video/de-container-cup/de-container-cup-s3/de-container-cup-s3-aflevering-2#autoplay', - 'info_dict': { - 'id': '9c4214b8-e55d-4e4b-a446-f015f6c6f811', - 'ext': 'mp4', - 'title': 'S3 - Aflevering 2', - 'series': 'De Container Cup', - 'season': 'Season 3', - 'season_number': 3, - 'episode': 'Episode 2', - 'episode_number': 2, - }, - 'skip': 'This video is only available for registered users' - }, { - 'url': 'https://www.goplay.be/video/a-family-for-thr-holidays-s1-aflevering-1#autoplay', - 'info_dict': { - 'id': '74e3ed07-748c-49e4-85a0-393a93337dbf', - 'ext': 'mp4', - 'title': 'A Family for the Holidays', - }, - 'skip': 'This video is only available for registered users' - }] - - _id_token = None - - def _perform_login(self, username, password): - self.report_login() - aws = AwsIdp(ie=self, pool_id='eu-west-1_dViSsKM5Y', client_id='6s1h851s8uplco5h6mqh1jac8m') - self._id_token, _ = aws.authenticate(username=username, password=password) - - def _real_initialize(self): - if not self._id_token: - raise self.raise_login_required(method='password') - - def _real_extract(self, url): - url, display_id = self._match_valid_url(url).group(0, 'display_id') - webpage = self._download_webpage(url, display_id) - video_data_json = self._html_search_regex(r'_). - E.g.: eu-west-1_aLkOfYN3T - :param str client_id: The client application ID (the ID of the application connecting) - """ - - self.ie = ie - - self.pool_id = pool_id - if "_" not in self.pool_id: - raise ValueError("Invalid pool_id format. Should be _.") - - self.client_id = client_id - self.region = self.pool_id.split("_")[0] - self.url = "https://cognito-idp.%s.amazonaws.com/" % (self.region,) - - # Initialize the values - # https://github.com/aws/amazon-cognito-identity-js/blob/master/src/AuthenticationHelper.js#L22 - self.n_hex = 'FFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD1' + \ - '29024E088A67CC74020BBEA63B139B22514A08798E3404DD' + \ - 'EF9519B3CD3A431B302B0A6DF25F14374FE1356D6D51C245' + \ - 'E485B576625E7EC6F44C42E9A637ED6B0BFF5CB6F406B7ED' + \ - 'EE386BFB5A899FA5AE9F24117C4B1FE649286651ECE45B3D' + \ - 'C2007CB8A163BF0598DA48361C55D39A69163FA8FD24CF5F' + \ - '83655D23DCA3AD961C62F356208552BB9ED529077096966D' + \ - '670C354E4ABC9804F1746C08CA18217C32905E462E36CE3B' + \ - 'E39E772C180E86039B2783A2EC07A28FB5C55DF06F4C52C9' + \ - 'DE2BCBF6955817183995497CEA956AE515D2261898FA0510' + \ - '15728E5A8AAAC42DAD33170D04507A33A85521ABDF1CBA64' + \ - 'ECFB850458DBEF0A8AEA71575D060C7DB3970F85A6E1E4C7' + \ - 'ABF5AE8CDB0933D71E8C94E04A25619DCEE3D2261AD2EE6B' + \ - 'F12FFA06D98A0864D87602733EC86A64521F2B18177B200C' + \ - 'BBE117577A615D6C770988C0BAD946E208E24FA074E5AB31' + \ - '43DB5BFCE0FD108E4B82D120A93AD2CAFFFFFFFFFFFFFFFF' - - # https://github.com/aws/amazon-cognito-identity-js/blob/master/src/AuthenticationHelper.js#L49 - self.g_hex = '2' - self.info_bits = bytearray('Caldera Derived Key', 'utf-8') - - self.big_n = self.__hex_to_long(self.n_hex) - self.g = self.__hex_to_long(self.g_hex) - self.k = self.__hex_to_long(self.__hex_hash('00' + self.n_hex + '0' + self.g_hex)) - self.small_a_value = self.__generate_random_small_a() - self.large_a_value = self.__calculate_a() - - def authenticate(self, username, password): - """ Authenticate with a username and password. """ - # Step 1: First initiate an authentication request - auth_data_dict = self.__get_authentication_request(username) - auth_data = json.dumps(auth_data_dict).encode("utf-8") - auth_headers = { - "X-Amz-Target": "AWSCognitoIdentityProviderService.InitiateAuth", - "Accept-Encoding": "identity", - "Content-Type": "application/x-amz-json-1.1" - } - auth_response_json = self.ie._download_json( - self.url, None, data=auth_data, headers=auth_headers, - note='Authenticating username', errnote='Invalid username') - challenge_parameters = auth_response_json.get("ChallengeParameters") - - if auth_response_json.get("ChallengeName") != "PASSWORD_VERIFIER": - raise AuthenticationException(auth_response_json["message"]) - - # Step 2: Respond to the Challenge with a valid ChallengeResponse - challenge_request = self.__get_challenge_response_request(challenge_parameters, password) - challenge_data = json.dumps(challenge_request).encode("utf-8") - challenge_headers = { - "X-Amz-Target": "AWSCognitoIdentityProviderService.RespondToAuthChallenge", - "Content-Type": "application/x-amz-json-1.1" - } - auth_response_json = self.ie._download_json( - self.url, None, data=challenge_data, headers=challenge_headers, - note='Authenticating password', errnote='Invalid password') - - if 'message' in auth_response_json: - raise InvalidLoginException(auth_response_json['message']) - return ( - auth_response_json['AuthenticationResult']['IdToken'], - auth_response_json['AuthenticationResult']['RefreshToken'] - ) - - def __get_authentication_request(self, username): - """ - - :param str username: The username to use - - :return: A full Authorization request. - :rtype: dict - """ - auth_request = { - "AuthParameters": { - "USERNAME": username, - "SRP_A": self.__long_to_hex(self.large_a_value) - }, - "AuthFlow": "USER_SRP_AUTH", - "ClientId": self.client_id - } - return auth_request - - def __get_challenge_response_request(self, challenge_parameters, password): - """ Create a Challenge Response Request object. - - :param dict[str,str|imt] challenge_parameters: The parameters for the challenge. - :param str password: The password. - - :return: A valid and full request data object to use as a response for a challenge. - :rtype: dict - """ - user_id = challenge_parameters["USERNAME"] - user_id_for_srp = challenge_parameters["USER_ID_FOR_SRP"] - srp_b = challenge_parameters["SRP_B"] - salt = challenge_parameters["SALT"] - secret_block = challenge_parameters["SECRET_BLOCK"] - - timestamp = self.__get_current_timestamp() - - # Get a HKDF key for the password, SrpB and the Salt - hkdf = self.__get_hkdf_key_for_password( - user_id_for_srp, - password, - self.__hex_to_long(srp_b), - salt - ) - secret_block_bytes = base64.standard_b64decode(secret_block) - - # the message is a combo of the pool_id, provided SRP userId, the Secret and Timestamp - msg = \ - bytearray(self.pool_id.split('_')[1], 'utf-8') + \ - bytearray(user_id_for_srp, 'utf-8') + \ - bytearray(secret_block_bytes) + \ - bytearray(timestamp, 'utf-8') - hmac_obj = hmac.new(hkdf, msg, digestmod=hashlib.sha256) - signature_string = base64.standard_b64encode(hmac_obj.digest()).decode('utf-8') - challenge_request = { - "ChallengeResponses": { - "USERNAME": user_id, - "TIMESTAMP": timestamp, - "PASSWORD_CLAIM_SECRET_BLOCK": secret_block, - "PASSWORD_CLAIM_SIGNATURE": signature_string - }, - "ChallengeName": "PASSWORD_VERIFIER", - "ClientId": self.client_id - } - return challenge_request - - def __get_hkdf_key_for_password(self, username, password, server_b_value, salt): - """ Calculates the final hkdf based on computed S value, and computed U value and the key. - - :param str username: Username. - :param str password: Password. - :param int server_b_value: Server B value. - :param int salt: Generated salt. - - :return Computed HKDF value. - :rtype: object - """ - - u_value = self.__calculate_u(self.large_a_value, server_b_value) - if u_value == 0: - raise ValueError('U cannot be zero.') - username_password = '%s%s:%s' % (self.pool_id.split('_')[1], username, password) - username_password_hash = self.__hash_sha256(username_password.encode('utf-8')) - - x_value = self.__hex_to_long(self.__hex_hash(self.__pad_hex(salt) + username_password_hash)) - g_mod_pow_xn = pow(self.g, x_value, self.big_n) - int_value2 = server_b_value - self.k * g_mod_pow_xn - s_value = pow(int_value2, self.small_a_value + u_value * x_value, self.big_n) - hkdf = self.__compute_hkdf( - bytearray.fromhex(self.__pad_hex(s_value)), - bytearray.fromhex(self.__pad_hex(self.__long_to_hex(u_value))) - ) - return hkdf - - def __compute_hkdf(self, ikm, salt): - """ Standard hkdf algorithm - - :param {Buffer} ikm Input key material. - :param {Buffer} salt Salt value. - :return {Buffer} Strong key material. - """ - - prk = hmac.new(salt, ikm, hashlib.sha256).digest() - info_bits_update = self.info_bits + bytearray(chr(1), 'utf-8') - hmac_hash = hmac.new(prk, info_bits_update, hashlib.sha256).digest() - return hmac_hash[:16] - - def __calculate_u(self, big_a, big_b): - """ Calculate the client's value U which is the hash of A and B - - :param int big_a: Large A value. - :param int big_b: Server B value. - - :return Computed U value. - :rtype: int - """ - - u_hex_hash = self.__hex_hash(self.__pad_hex(big_a) + self.__pad_hex(big_b)) - return self.__hex_to_long(u_hex_hash) - - def __generate_random_small_a(self): - """ Helper function to generate a random big integer - - :return a random value. - :rtype: int - """ - random_long_int = self.__get_random(128) - return random_long_int % self.big_n - - def __calculate_a(self): - """ Calculate the client's public value A = g^a%N with the generated random number a - - :return Computed large A. - :rtype: int - """ - - big_a = pow(self.g, self.small_a_value, self.big_n) - # safety check - if (big_a % self.big_n) == 0: - raise ValueError('Safety check for A failed') - return big_a - - @staticmethod - def __long_to_hex(long_num): - return '%x' % long_num - - @staticmethod - def __hex_to_long(hex_string): - return int(hex_string, 16) - - @staticmethod - def __hex_hash(hex_string): - return AwsIdp.__hash_sha256(bytearray.fromhex(hex_string)) - - @staticmethod - def __hash_sha256(buf): - """AuthenticationHelper.hash""" - digest = hashlib.sha256(buf).hexdigest() - return (64 - len(digest)) * '0' + digest - - @staticmethod - def __pad_hex(long_int): - """ Converts a Long integer (or hex string) to hex format padded with zeroes for hashing - - :param int|str long_int: Number or string to pad. - - :return Padded hex string. - :rtype: str - """ - - if not isinstance(long_int, str): - hash_str = AwsIdp.__long_to_hex(long_int) - else: - hash_str = long_int - if len(hash_str) % 2 == 1: - hash_str = '0%s' % hash_str - elif hash_str[0] in '89ABCDEFabcdef': - hash_str = '00%s' % hash_str - return hash_str - - @staticmethod - def __get_random(nbytes): - random_hex = binascii.hexlify(os.urandom(nbytes)) - return AwsIdp.__hex_to_long(random_hex) - - @staticmethod - def __get_current_timestamp(): - """ Creates a timestamp with the correct English format. - - :return: timestamp in format 'Sun Jan 27 19:00:04 UTC 2019' - :rtype: str - """ - - # We need US only data, so we cannot just do a strftime: - # Sun Jan 27 19:00:04 UTC 2019 - months = [None, 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'] - days = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'] - - time_now = datetime.datetime.utcnow() - format_string = "{} {} {} %H:%M:%S UTC %Y".format(days[time_now.weekday()], months[time_now.month], time_now.day) - time_string = datetime.datetime.utcnow().strftime(format_string) - return time_string - - def __str__(self): - return "AWS IDP Client for:\nRegion: %s\nPoolId: %s\nAppId: %s" % ( - self.region, self.pool_id.split("_")[1], self.client_id - ) diff --git a/lib/yt_dlp/extractor/tiktok.py b/lib/yt_dlp/extractor/tiktok.py index 4a35a241c..c58538394 100644 --- a/lib/yt_dlp/extractor/tiktok.py +++ b/lib/yt_dlp/extractor/tiktok.py @@ -25,7 +25,7 @@ class TikTokBaseIE(InfoExtractor): - _APP_VERSIONS = [('26.1.3', '260103'), ('26.1.2', '260102'), ('26.1.1', '260101'), ('25.6.2', '250602')] + _APP_VERSIONS = [('20.9.3', '293'), ('20.4.3', '243'), ('20.2.1', '221'), ('20.1.2', '212'), ('20.0.4', '204')] _WORKING_APP_VERSION = None _APP_NAME = 'trill' _AID = 1180 @@ -33,6 +33,7 @@ class TikTokBaseIE(InfoExtractor): _UPLOADER_URL_FORMAT = 'https://www.tiktok.com/@%s' _WEBPAGE_HOST = 'https://www.tiktok.com/' QUALITIES = ('360p', '540p', '720p', '1080p') + _session_initialized = False @staticmethod def _create_url(user_id, video_id): @@ -42,6 +43,12 @@ def _get_sigi_state(self, webpage, display_id): return self._parse_json(get_element_by_id( 'SIGI_STATE|sigi-persisted-data', webpage, escape_value=False), display_id) + def _real_initialize(self): + if self._session_initialized: + return + self._request_webpage(HEADRequest('https://www.tiktok.com'), None, note='Setting up session', fatal=False) + TikTokBaseIE._session_initialized = True + def _call_api_impl(self, ep, query, manifest_app_version, video_id, fatal=True, note='Downloading API JSON', errnote='Unable to download API page'): self._set_cookie(self._API_HOSTNAME, 'odin_tt', ''.join(random.choice('0123456789abcdef') for _ in range(160))) @@ -282,7 +289,7 @@ def extract_addr(addr, add_meta={}): 'uploader_url': user_url, 'track': music_track, 'album': str_or_none(music_info.get('album')) or None, - 'artist': music_author or None, + 'artist': music_author, 'timestamp': int_or_none(aweme_detail.get('create_time')), 'formats': formats, 'subtitles': self.extract_subtitles(aweme_detail, aweme_id), @@ -515,7 +522,7 @@ class TikTokIE(TikTokBaseIE): 'repost_count': int, 'comment_count': int, }, - 'skip': 'This video is unavailable', + 'expected_warnings': ['trying feed workaround', 'Unable to find video in feed'] }, { # Auto-captions available 'url': 'https://www.tiktok.com/@hankgreen1/video/7047596209028074758', @@ -523,11 +530,18 @@ class TikTokIE(TikTokBaseIE): }] def _extract_aweme_app(self, aweme_id): - feed_list = self._call_api('feed', {'aweme_id': aweme_id}, aweme_id, - note='Downloading video feed', errnote='Unable to download video feed').get('aweme_list') or [] - aweme_detail = next((aweme for aweme in feed_list if str(aweme.get('aweme_id')) == aweme_id), None) - if not aweme_detail: - raise ExtractorError('Unable to find video in feed', video_id=aweme_id) + try: + aweme_detail = self._call_api('aweme/detail', {'aweme_id': aweme_id}, aweme_id, + note='Downloading video details', errnote='Unable to download video details').get('aweme_detail') + if not aweme_detail: + raise ExtractorError('Video not available', video_id=aweme_id) + except ExtractorError as e: + self.report_warning(f'{e.orig_msg}; trying feed workaround') + feed_list = self._call_api('feed', {'aweme_id': aweme_id}, aweme_id, + note='Downloading video feed', errnote='Unable to download video feed').get('aweme_list') or [] + aweme_detail = next((aweme for aweme in feed_list if str(aweme.get('aweme_id')) == aweme_id), None) + if not aweme_detail: + raise ExtractorError('Unable to find video in feed', video_id=aweme_id) return self._parse_aweme_video_app(aweme_detail) def _real_extract(self, url): @@ -558,7 +572,6 @@ def _real_extract(self, url): class TikTokUserIE(TikTokBaseIE): IE_NAME = 'tiktok:user' _VALID_URL = r'https?://(?:www\.)?tiktok\.com/@(?P[\w\.-]+)/?(?:$|[#?])' - _WORKING = False _TESTS = [{ 'url': 'https://tiktok.com/@corgibobaa?lang=en', 'playlist_mincount': 45, @@ -695,7 +708,6 @@ def _real_extract(self, url): class TikTokSoundIE(TikTokBaseListIE): IE_NAME = 'tiktok:sound' _VALID_URL = r'https?://(?:www\.)?tiktok\.com/music/[\w\.-]+-(?P[\d]+)[/?#&]?' - _WORKING = False _QUERY_NAME = 'music_id' _API_ENDPOINT = 'music/aweme' _TESTS = [{ @@ -719,7 +731,6 @@ class TikTokSoundIE(TikTokBaseListIE): class TikTokEffectIE(TikTokBaseListIE): IE_NAME = 'tiktok:effect' _VALID_URL = r'https?://(?:www\.)?tiktok\.com/sticker/[\w\.-]+-(?P[\d]+)[/?#&]?' - _WORKING = False _QUERY_NAME = 'sticker_id' _API_ENDPOINT = 'sticker/aweme' _TESTS = [{ @@ -739,7 +750,6 @@ class TikTokEffectIE(TikTokBaseListIE): class TikTokTagIE(TikTokBaseListIE): IE_NAME = 'tiktok:tag' _VALID_URL = r'https?://(?:www\.)?tiktok\.com/tag/(?P[^/?#&]+)' - _WORKING = False _QUERY_NAME = 'ch_id' _API_ENDPOINT = 'challenge/aweme' _TESTS = [{ diff --git a/lib/yt_dlp/extractor/vier.py b/lib/yt_dlp/extractor/vier.py new file mode 100644 index 000000000..eab894ab6 --- /dev/null +++ b/lib/yt_dlp/extractor/vier.py @@ -0,0 +1,261 @@ +import re +import itertools + +from .common import InfoExtractor +from ..utils import ( + urlencode_postdata, + int_or_none, + unified_strdate, +) + + +class VierIE(InfoExtractor): + IE_NAME = 'vier' + IE_DESC = 'vier.be and vijf.be' + _VALID_URL = r'''(?x) + https?:// + (?:www\.)?(?Pvier|vijf)\.be/ + (?: + (?: + [^/]+/videos| + video(?:/[^/]+)* + )/ + (?P[^/]+)(?:/(?P\d+))?| + (?: + video/v3/embed| + embed/video/public + )/(?P\d+) + ) + ''' + _NETRC_MACHINE = 'vier' + _TESTS = [{ + 'url': 'http://www.vier.be/planb/videos/het-wordt-warm-de-moestuin/16129', + 'md5': 'e4ae2054a6b040ef1e289e20d111b46e', + 'info_dict': { + 'id': '16129', + 'display_id': 'het-wordt-warm-de-moestuin', + 'ext': 'mp4', + 'title': 'Het wordt warm in De Moestuin', + 'description': 'De vele uren werk eisen hun tol. Wim droomt van assistentie...', + 'upload_date': '20121025', + 'series': 'Plan B', + 'tags': ['De Moestuin', 'Moestuin', 'meisjes', 'Tomaat', 'Wim', 'Droom'], + }, + }, { + 'url': 'http://www.vijf.be/temptationisland/videos/zo-grappig-temptation-island-hosts-moeten-kiezen-tussen-onmogelijke-dilemmas/2561614', + 'info_dict': { + 'id': '2561614', + 'display_id': 'zo-grappig-temptation-island-hosts-moeten-kiezen-tussen-onmogelijke-dilemmas', + 'ext': 'mp4', + 'title': 'md5:84f45fe48b8c1fa296a7f6d208d080a7', + 'description': 'md5:0356d4981e58b8cbee19355cbd51a8fe', + 'upload_date': '20170228', + 'series': 'Temptation Island', + 'tags': list, + }, + 'params': { + 'skip_download': True, + }, + }, { + 'url': 'http://www.vier.be/janigaat/videos/jani-gaat-naar-tokio-aflevering-4/2674839', + 'info_dict': { + 'id': '2674839', + 'display_id': 'jani-gaat-naar-tokio-aflevering-4', + 'ext': 'mp4', + 'title': 'Jani gaat naar Tokio - Aflevering 4', + 'description': 'md5:aa8d611541db6ae9e863125704511f88', + 'upload_date': '20170501', + 'series': 'Jani gaat', + 'episode_number': 4, + 'tags': ['Jani Gaat', 'Volledige Aflevering'], + }, + 'params': { + 'skip_download': True, + }, + 'skip': 'Requires account credentials', + }, { + # Requires account credentials but bypassed extraction via v3/embed page + # without metadata + 'url': 'http://www.vier.be/janigaat/videos/jani-gaat-naar-tokio-aflevering-4/2674839', + 'info_dict': { + 'id': '2674839', + 'display_id': 'jani-gaat-naar-tokio-aflevering-4', + 'ext': 'mp4', + 'title': 'jani-gaat-naar-tokio-aflevering-4', + }, + 'params': { + 'skip_download': True, + }, + 'expected_warnings': ['Log in to extract metadata'], + }, { + # Without video id in URL + 'url': 'http://www.vier.be/planb/videos/dit-najaar-plan-b', + 'only_matching': True, + }, { + 'url': 'http://www.vier.be/video/v3/embed/16129', + 'only_matching': True, + }, { + 'url': 'https://www.vijf.be/embed/video/public/4093', + 'only_matching': True, + }, { + 'url': 'https://www.vier.be/video/blockbusters/in-juli-en-augustus-summer-classics', + 'only_matching': True, + }, { + 'url': 'https://www.vier.be/video/achter-de-rug/2017/achter-de-rug-seizoen-1-aflevering-6', + 'only_matching': True, + }] + + def _real_initialize(self): + self._logged_in = False + + def _login(self, site): + username, password = self._get_login_info() + if username is None or password is None: + return + + login_page = self._download_webpage( + 'http://www.%s.be/user/login' % site, + None, note='Logging in', errnote='Unable to log in', + data=urlencode_postdata({ + 'form_id': 'user_login', + 'name': username, + 'pass': password, + }), + headers={'Content-Type': 'application/x-www-form-urlencoded'}) + + login_error = self._html_search_regex( + r'(?s)
\s*
\s*(.+?)<', + login_page, 'login error', default=None) + if login_error: + self.report_warning('Unable to log in: %s' % login_error) + else: + self._logged_in = True + + def _real_extract(self, url): + mobj = self._match_valid_url(url) + embed_id = mobj.group('embed_id') + display_id = mobj.group('display_id') or embed_id + video_id = mobj.group('id') or embed_id + site = mobj.group('site') + + if not self._logged_in: + self._login(site) + + webpage = self._download_webpage(url, display_id) + + if r'id="user-login"' in webpage: + self.report_warning( + 'Log in to extract metadata', video_id=display_id) + webpage = self._download_webpage( + 'http://www.%s.be/video/v3/embed/%s' % (site, video_id), + display_id) + + video_id = self._search_regex( + [r'data-nid="(\d+)"', r'"nid"\s*:\s*"(\d+)"'], + webpage, 'video id', default=video_id or display_id) + + playlist_url = self._search_regex( + r'data-file=(["\'])(?P(?:https?:)?//[^/]+/.+?\.m3u8.*?)\1', + webpage, 'm3u8 url', default=None, group='url') + + if not playlist_url: + application = self._search_regex( + [r'data-application="([^"]+)"', r'"application"\s*:\s*"([^"]+)"'], + webpage, 'application', default=site + '_vod') + filename = self._search_regex( + [r'data-filename="([^"]+)"', r'"filename"\s*:\s*"([^"]+)"'], + webpage, 'filename') + playlist_url = 'http://vod.streamcloud.be/%s/_definst_/mp4:%s.mp4/playlist.m3u8' % (application, filename) + + formats = self._extract_wowza_formats( + playlist_url, display_id, skip_protocols=['dash']) + self._sort_formats(formats) + + title = self._og_search_title(webpage, default=display_id) + description = self._html_search_regex( + r'(?s)]+\bclass=(["\'])[^>]*?\bfield-type-text-with-summary\b[^>]*?\1[^>]*>.*?

(?P.+?)

', + webpage, 'description', default=None, group='value') + thumbnail = self._og_search_thumbnail(webpage, default=None) + upload_date = unified_strdate(self._html_search_regex( + r'(?s)]+\bclass=(["\'])[^>]*?\bfield-name-post-date\b[^>]*?\1[^>]*>.*?(?P\d{2}/\d{2}/\d{4})', + webpage, 'upload date', default=None, group='value')) + + series = self._search_regex( + r'data-program=(["\'])(?P(?:(?!\1).)+)\1', webpage, + 'series', default=None, group='value') + episode_number = int_or_none(self._search_regex( + r'(?i)aflevering (\d+)', title, 'episode number', default=None)) + tags = re.findall(r']+\bhref=["\']/tags/[^>]+>([^<]+)<', webpage) + + return { + 'id': video_id, + 'display_id': display_id, + 'title': title, + 'description': description, + 'thumbnail': thumbnail, + 'upload_date': upload_date, + 'series': series, + 'episode_number': episode_number, + 'tags': tags, + 'formats': formats, + } + + +class VierVideosIE(InfoExtractor): + IE_NAME = 'vier:videos' + _VALID_URL = r'https?://(?:www\.)?(?Pvier|vijf)\.be/(?P[^/]+)/videos(?:\?.*\bpage=(?P\d+)|$)' + _TESTS = [{ + 'url': 'http://www.vier.be/demoestuin/videos', + 'info_dict': { + 'id': 'demoestuin', + }, + 'playlist_mincount': 153, + }, { + 'url': 'http://www.vijf.be/temptationisland/videos', + 'info_dict': { + 'id': 'temptationisland', + }, + 'playlist_mincount': 159, + }, { + 'url': 'http://www.vier.be/demoestuin/videos?page=6', + 'info_dict': { + 'id': 'demoestuin-page6', + }, + 'playlist_mincount': 20, + }, { + 'url': 'http://www.vier.be/demoestuin/videos?page=7', + 'info_dict': { + 'id': 'demoestuin-page7', + }, + 'playlist_mincount': 13, + }] + + def _real_extract(self, url): + mobj = self._match_valid_url(url) + program = mobj.group('program') + site = mobj.group('site') + + page_id = mobj.group('page') + if page_id: + page_id = int(page_id) + start_page = page_id + playlist_id = '%s-page%d' % (program, page_id) + else: + start_page = 0 + playlist_id = program + + entries = [] + for current_page_id in itertools.count(start_page): + current_page = self._download_webpage( + 'http://www.%s.be/%s/videos?page=%d' % (site, program, current_page_id), + program, + 'Downloading page %d' % (current_page_id + 1)) + page_entries = [ + self.url_result('http://www.' + site + '.be' + video_url, 'Vier') + for video_url in re.findall( + r'', current_page)] + entries.extend(page_entries) + if page_id or '>Meer<' not in current_page: + break + + return self.playlist_result(entries, playlist_id) diff --git a/lib/yt_dlp_version b/lib/yt_dlp_version index 67d502c59..8a909edcf 100644 --- a/lib/yt_dlp_version +++ b/lib/yt_dlp_version @@ -1 +1 @@ -f7c5a5e96756636379a0b1afbeadb08b9c643bef \ No newline at end of file +19b4e59a1e1bf368078f90e7f735fa4576f97b64 \ No newline at end of file diff --git a/service.py b/service.py index 5b2d8fd23..8304ed1af 100644 --- a/service.py +++ b/service.py @@ -87,51 +87,42 @@ def getParams(): def play(url, ydl_opts): - if xbmcplugin.getSetting(int(sys.argv[1]),"usemanifest") == 'true': - ydl_opts['format'] = 'bestvideo*+bestaudio/best' - ydl = YoutubeDL(ydl_opts) - ydl.add_default_info_extractors() - - with ydl: - progress = xbmcgui.DialogProgressBG() - progress.create("Resolving " + url) - try: - result = ydl.extract_info(url, download=False) - except: - progress.close() - showErrorNotification("Could not resolve the url, check the log for more info") - import traceback - log(msg=traceback.format_exc(), level=xbmc.LOGERROR) - exit() - progress.close() - - if 'entries' in result: - # more than one video - pl = xbmc.PlayList(1) - pl.clear() - - # determine which index in the queue to start playing from - indexToStartAt = playlistIndex(url, result) - if indexToStartAt == None: - indexToStartAt = 0 - - unresolvedEntries = list(result['entries']) - startingEntry = unresolvedEntries.pop(indexToStartAt) - - # populate the queue with unresolved entries so that the starting entry can be inserted - for video in unresolvedEntries: - list_item = createListItemFromFlatPlaylistItem(video) - pl.add(list_item.getPath(), list_item) - - # make sure the starting ListItem has a resolved url, to avoid recursion and crashes - startingVideoUrl = startingEntry['url'] - startingItem = createListItemFromVideo(ydl.extract_info(startingVideoUrl, download=False)) - pl.add(startingItem.getPath(), startingItem, indexToStartAt) - - #xbmc.Player().play(pl) # this probably works again - # ...but start playback the same way the Youtube plugin does it: - xbmc.executebuiltin('Playlist.PlayOffset(%s,%d)' % ('video', indexToStartAt)) - + if xbmcplugin.getSetting(int(sys.argv[1]),"usemanifest"): + ydl_opts['format'] = 'bestvideo*+bestaudio/best' + ydl = YoutubeDL(ydl_opts) + ydl.add_default_info_extractors() + + with ydl: + showInfoNotification("Resolving stream(s) for " + url) + result = ydl.extract_info(url, download=False) + + if 'entries' in result: + # more than one video + pl = xbmc.PlayList(1) + pl.clear() + + # determine which index in the queue to start playing from + indexToStartAt = playlistIndex(url, result) + if indexToStartAt == None: + indexToStartAt = 0 + + unresolvedEntries = list(result['entries']) + startingEntry = unresolvedEntries.pop(indexToStartAt) + + # populate the queue with unresolved entries so that the starting entry can be inserted + for video in unresolvedEntries: + list_item = createListItemFromFlatPlaylistItem(video) + pl.add(list_item.getPath(), list_item) + + # make sure the starting ListItem has a resolved url, to avoid recursion and crashes + startingVideoUrl = startingEntry['url'] + startingItem = createListItemFromVideo(ydl.extract_info(startingVideoUrl, download=False)) + pl.add(startingItem.getPath(), startingItem, indexToStartAt) + + #xbmc.Player().play(pl) # this probably works again + # ...but start playback the same way the Youtube plugin does it: + xbmc.executebuiltin('Playlist.PlayOffset(%s,%d)' % ('video', indexToStartAt)) + showInfoNotification("Playing playlist " + result['title']) else: # Just a video, pass the item to the Kodi player. @@ -211,7 +202,7 @@ def check_if_kodi_supports_manifest(url): def createListItemFromVideo(result): debug(result) adaptive_type = False - if xbmcplugin.getSetting(int(sys.argv[1]),"usemanifest") == 'true': + if xbmcplugin.getSetting(int(sys.argv[1]),"usemanifest"): url = extract_manifest_url(result) if url is not None: log("found original manifest: " + url) From 6f84c2cbd55c7c56c98e995e4bb77e3454b35337 Mon Sep 17 00:00:00 2001 From: Gautam Garg Date: Thu, 22 Sep 2022 21:35:41 +0530 Subject: [PATCH 4/6] Merge tag 'v0.9.285' --- README.md | 10 +- lib/yt_dlp/extractor/_extractors.py | 2 +- lib/yt_dlp/extractor/goplay.py | 395 ++++++++++++++++++++++++++++ lib/yt_dlp/extractor/tiktok.py | 34 +-- lib/yt_dlp/extractor/vier.py | 261 ------------------ lib/yt_dlp_version | 2 +- service.py | 83 +++--- 7 files changed, 463 insertions(+), 324 deletions(-) create mode 100644 lib/yt_dlp/extractor/goplay.py delete mode 100644 lib/yt_dlp/extractor/vier.py diff --git a/README.md b/README.md index cca0d83ff..18f97c926 100644 --- a/README.md +++ b/README.md @@ -2,13 +2,19 @@ [![build-publish-addon](https://github.com/firsttris/plugin.video.sendtokodi/actions/workflows/build-publish.yml/badge.svg)](https://github.com/firsttris/plugin.video.sendtokodi/actions/workflows/build-publish.yml) -This [kodi](https://github.com/xbmc/xbmc) plugin receives URLs and resolves almost all of them with [yt-dlp](https://github.com/yt-dlp/yt-dlp) creating a playable video stream for kodi. The URLs can be send with one of the supported apps listed below. The plugin itself is not in the offical kodi addon repo, to install it with automatic updates you need to add our repo first. Download the repo file for your kodi version and [install it from zip](https://kodi.wiki/view/Add-on_manager). Afterwards the repositoy should be listed and can be selected to install the actual plugin itself. +This add-on receives video or audio URLs and plays them in [Kodi](https://kodi.tv). It resolves sent websites automatically with [yt-dlp](https://github.com/yt-dlp/yt-dlp) into a playable stream. The URLs can be sent with one of the supported apps listed below. + +drawing + + +## Installation +The plugin is not in the offical kodi addon repo, to install it with automatic updates you need to add our repo first. Download the repo file for your kodi version and [install the repo from zip](https://kodi.wiki/view/Add-on_manager). Afterwards the addon `sendtokodi` can be found in the [install from repository](https://kodi.wiki/view/Add-on_manager) section. [Download repo for kodi 18](https://github.com/firsttris/repository.sendtokodi/raw/master/repository.sendtokodi/repository.sendtokodi-0.0.1.zip) [Download repo for kodi 19+](https://github.com/firsttris/repository.sendtokodi.python3/raw/master/repository.sendtokodi.python3/repository.sendtokodi.python3-0.0.1.zip) -*Please note that kodi 18 is limited to python 2 only, but the used URL resolver yt-dlp requires python 3.6+. Therefore, the kodi 18 version uses [youtube-dl](https://youtube-dl.org/) instead. Unfortunately, the development of youtube-dl was stuck and development has just been resumed lately. So the kodi 18 version of this plugin might not be as stable as the kodi 19 version.* +*Please note that kodi 18 is internally limited to python2 but the addon uses yt-dlp to resolve URLs which requires python 3.6+. Therefore, the kodi 18 version uses [youtube-dl](https://youtube-dl.org/) instead. Unfortunately, the development of youtube-dl was stuck but it has been resumed. So the kodi 18 version of this plugin might not be as stable as the kodi 19 version.* ## Features - [supported sites](https://github.com/yt-dlp/yt-dlp/blob/master/supportedsites.md) diff --git a/lib/yt_dlp/extractor/_extractors.py b/lib/yt_dlp/extractor/_extractors.py index 6bf769a9e..43e2f93d3 100644 --- a/lib/yt_dlp/extractor/_extractors.py +++ b/lib/yt_dlp/extractor/_extractors.py @@ -649,6 +649,7 @@ ) from .googlesearch import GoogleSearchIE from .gopro import GoProIE +from .goplay import GoPlayIE from .goshgay import GoshgayIE from .gotostage import GoToStageIE from .gputechconf import GPUTechConfIE @@ -2021,7 +2022,6 @@ VidioLiveIE ) from .vidlii import VidLiiIE -from .vier import VierIE, VierVideosIE from .viewlift import ( ViewLiftIE, ViewLiftEmbedIE, diff --git a/lib/yt_dlp/extractor/goplay.py b/lib/yt_dlp/extractor/goplay.py new file mode 100644 index 000000000..31267e1aa --- /dev/null +++ b/lib/yt_dlp/extractor/goplay.py @@ -0,0 +1,395 @@ +import base64 +import binascii +import datetime +import hashlib +import hmac +import json +import os + +from .common import InfoExtractor +from ..utils import ( + ExtractorError, + traverse_obj, + unescapeHTML, +) + + +class GoPlayIE(InfoExtractor): + _VALID_URL = r'https?://(www\.)?goplay\.be/video/([^/]+/[^/]+/|)(?P[^/#]+)' + + _NETRC_MACHINE = 'goplay' + + _TESTS = [{ + 'url': 'https://www.goplay.be/video/de-container-cup/de-container-cup-s3/de-container-cup-s3-aflevering-2#autoplay', + 'info_dict': { + 'id': '9c4214b8-e55d-4e4b-a446-f015f6c6f811', + 'ext': 'mp4', + 'title': 'S3 - Aflevering 2', + 'series': 'De Container Cup', + 'season': 'Season 3', + 'season_number': 3, + 'episode': 'Episode 2', + 'episode_number': 2, + }, + 'skip': 'This video is only available for registered users' + }, { + 'url': 'https://www.goplay.be/video/a-family-for-thr-holidays-s1-aflevering-1#autoplay', + 'info_dict': { + 'id': '74e3ed07-748c-49e4-85a0-393a93337dbf', + 'ext': 'mp4', + 'title': 'A Family for the Holidays', + }, + 'skip': 'This video is only available for registered users' + }] + + _id_token = None + + def _perform_login(self, username, password): + self.report_login() + aws = AwsIdp(ie=self, pool_id='eu-west-1_dViSsKM5Y', client_id='6s1h851s8uplco5h6mqh1jac8m') + self._id_token, _ = aws.authenticate(username=username, password=password) + + def _real_initialize(self): + if not self._id_token: + raise self.raise_login_required(method='password') + + def _real_extract(self, url): + url, display_id = self._match_valid_url(url).group(0, 'display_id') + webpage = self._download_webpage(url, display_id) + video_data_json = self._html_search_regex(r'_). + E.g.: eu-west-1_aLkOfYN3T + :param str client_id: The client application ID (the ID of the application connecting) + """ + + self.ie = ie + + self.pool_id = pool_id + if "_" not in self.pool_id: + raise ValueError("Invalid pool_id format. Should be _.") + + self.client_id = client_id + self.region = self.pool_id.split("_")[0] + self.url = "https://cognito-idp.%s.amazonaws.com/" % (self.region,) + + # Initialize the values + # https://github.com/aws/amazon-cognito-identity-js/blob/master/src/AuthenticationHelper.js#L22 + self.n_hex = 'FFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD1' + \ + '29024E088A67CC74020BBEA63B139B22514A08798E3404DD' + \ + 'EF9519B3CD3A431B302B0A6DF25F14374FE1356D6D51C245' + \ + 'E485B576625E7EC6F44C42E9A637ED6B0BFF5CB6F406B7ED' + \ + 'EE386BFB5A899FA5AE9F24117C4B1FE649286651ECE45B3D' + \ + 'C2007CB8A163BF0598DA48361C55D39A69163FA8FD24CF5F' + \ + '83655D23DCA3AD961C62F356208552BB9ED529077096966D' + \ + '670C354E4ABC9804F1746C08CA18217C32905E462E36CE3B' + \ + 'E39E772C180E86039B2783A2EC07A28FB5C55DF06F4C52C9' + \ + 'DE2BCBF6955817183995497CEA956AE515D2261898FA0510' + \ + '15728E5A8AAAC42DAD33170D04507A33A85521ABDF1CBA64' + \ + 'ECFB850458DBEF0A8AEA71575D060C7DB3970F85A6E1E4C7' + \ + 'ABF5AE8CDB0933D71E8C94E04A25619DCEE3D2261AD2EE6B' + \ + 'F12FFA06D98A0864D87602733EC86A64521F2B18177B200C' + \ + 'BBE117577A615D6C770988C0BAD946E208E24FA074E5AB31' + \ + '43DB5BFCE0FD108E4B82D120A93AD2CAFFFFFFFFFFFFFFFF' + + # https://github.com/aws/amazon-cognito-identity-js/blob/master/src/AuthenticationHelper.js#L49 + self.g_hex = '2' + self.info_bits = bytearray('Caldera Derived Key', 'utf-8') + + self.big_n = self.__hex_to_long(self.n_hex) + self.g = self.__hex_to_long(self.g_hex) + self.k = self.__hex_to_long(self.__hex_hash('00' + self.n_hex + '0' + self.g_hex)) + self.small_a_value = self.__generate_random_small_a() + self.large_a_value = self.__calculate_a() + + def authenticate(self, username, password): + """ Authenticate with a username and password. """ + # Step 1: First initiate an authentication request + auth_data_dict = self.__get_authentication_request(username) + auth_data = json.dumps(auth_data_dict).encode("utf-8") + auth_headers = { + "X-Amz-Target": "AWSCognitoIdentityProviderService.InitiateAuth", + "Accept-Encoding": "identity", + "Content-Type": "application/x-amz-json-1.1" + } + auth_response_json = self.ie._download_json( + self.url, None, data=auth_data, headers=auth_headers, + note='Authenticating username', errnote='Invalid username') + challenge_parameters = auth_response_json.get("ChallengeParameters") + + if auth_response_json.get("ChallengeName") != "PASSWORD_VERIFIER": + raise AuthenticationException(auth_response_json["message"]) + + # Step 2: Respond to the Challenge with a valid ChallengeResponse + challenge_request = self.__get_challenge_response_request(challenge_parameters, password) + challenge_data = json.dumps(challenge_request).encode("utf-8") + challenge_headers = { + "X-Amz-Target": "AWSCognitoIdentityProviderService.RespondToAuthChallenge", + "Content-Type": "application/x-amz-json-1.1" + } + auth_response_json = self.ie._download_json( + self.url, None, data=challenge_data, headers=challenge_headers, + note='Authenticating password', errnote='Invalid password') + + if 'message' in auth_response_json: + raise InvalidLoginException(auth_response_json['message']) + return ( + auth_response_json['AuthenticationResult']['IdToken'], + auth_response_json['AuthenticationResult']['RefreshToken'] + ) + + def __get_authentication_request(self, username): + """ + + :param str username: The username to use + + :return: A full Authorization request. + :rtype: dict + """ + auth_request = { + "AuthParameters": { + "USERNAME": username, + "SRP_A": self.__long_to_hex(self.large_a_value) + }, + "AuthFlow": "USER_SRP_AUTH", + "ClientId": self.client_id + } + return auth_request + + def __get_challenge_response_request(self, challenge_parameters, password): + """ Create a Challenge Response Request object. + + :param dict[str,str|imt] challenge_parameters: The parameters for the challenge. + :param str password: The password. + + :return: A valid and full request data object to use as a response for a challenge. + :rtype: dict + """ + user_id = challenge_parameters["USERNAME"] + user_id_for_srp = challenge_parameters["USER_ID_FOR_SRP"] + srp_b = challenge_parameters["SRP_B"] + salt = challenge_parameters["SALT"] + secret_block = challenge_parameters["SECRET_BLOCK"] + + timestamp = self.__get_current_timestamp() + + # Get a HKDF key for the password, SrpB and the Salt + hkdf = self.__get_hkdf_key_for_password( + user_id_for_srp, + password, + self.__hex_to_long(srp_b), + salt + ) + secret_block_bytes = base64.standard_b64decode(secret_block) + + # the message is a combo of the pool_id, provided SRP userId, the Secret and Timestamp + msg = \ + bytearray(self.pool_id.split('_')[1], 'utf-8') + \ + bytearray(user_id_for_srp, 'utf-8') + \ + bytearray(secret_block_bytes) + \ + bytearray(timestamp, 'utf-8') + hmac_obj = hmac.new(hkdf, msg, digestmod=hashlib.sha256) + signature_string = base64.standard_b64encode(hmac_obj.digest()).decode('utf-8') + challenge_request = { + "ChallengeResponses": { + "USERNAME": user_id, + "TIMESTAMP": timestamp, + "PASSWORD_CLAIM_SECRET_BLOCK": secret_block, + "PASSWORD_CLAIM_SIGNATURE": signature_string + }, + "ChallengeName": "PASSWORD_VERIFIER", + "ClientId": self.client_id + } + return challenge_request + + def __get_hkdf_key_for_password(self, username, password, server_b_value, salt): + """ Calculates the final hkdf based on computed S value, and computed U value and the key. + + :param str username: Username. + :param str password: Password. + :param int server_b_value: Server B value. + :param int salt: Generated salt. + + :return Computed HKDF value. + :rtype: object + """ + + u_value = self.__calculate_u(self.large_a_value, server_b_value) + if u_value == 0: + raise ValueError('U cannot be zero.') + username_password = '%s%s:%s' % (self.pool_id.split('_')[1], username, password) + username_password_hash = self.__hash_sha256(username_password.encode('utf-8')) + + x_value = self.__hex_to_long(self.__hex_hash(self.__pad_hex(salt) + username_password_hash)) + g_mod_pow_xn = pow(self.g, x_value, self.big_n) + int_value2 = server_b_value - self.k * g_mod_pow_xn + s_value = pow(int_value2, self.small_a_value + u_value * x_value, self.big_n) + hkdf = self.__compute_hkdf( + bytearray.fromhex(self.__pad_hex(s_value)), + bytearray.fromhex(self.__pad_hex(self.__long_to_hex(u_value))) + ) + return hkdf + + def __compute_hkdf(self, ikm, salt): + """ Standard hkdf algorithm + + :param {Buffer} ikm Input key material. + :param {Buffer} salt Salt value. + :return {Buffer} Strong key material. + """ + + prk = hmac.new(salt, ikm, hashlib.sha256).digest() + info_bits_update = self.info_bits + bytearray(chr(1), 'utf-8') + hmac_hash = hmac.new(prk, info_bits_update, hashlib.sha256).digest() + return hmac_hash[:16] + + def __calculate_u(self, big_a, big_b): + """ Calculate the client's value U which is the hash of A and B + + :param int big_a: Large A value. + :param int big_b: Server B value. + + :return Computed U value. + :rtype: int + """ + + u_hex_hash = self.__hex_hash(self.__pad_hex(big_a) + self.__pad_hex(big_b)) + return self.__hex_to_long(u_hex_hash) + + def __generate_random_small_a(self): + """ Helper function to generate a random big integer + + :return a random value. + :rtype: int + """ + random_long_int = self.__get_random(128) + return random_long_int % self.big_n + + def __calculate_a(self): + """ Calculate the client's public value A = g^a%N with the generated random number a + + :return Computed large A. + :rtype: int + """ + + big_a = pow(self.g, self.small_a_value, self.big_n) + # safety check + if (big_a % self.big_n) == 0: + raise ValueError('Safety check for A failed') + return big_a + + @staticmethod + def __long_to_hex(long_num): + return '%x' % long_num + + @staticmethod + def __hex_to_long(hex_string): + return int(hex_string, 16) + + @staticmethod + def __hex_hash(hex_string): + return AwsIdp.__hash_sha256(bytearray.fromhex(hex_string)) + + @staticmethod + def __hash_sha256(buf): + """AuthenticationHelper.hash""" + digest = hashlib.sha256(buf).hexdigest() + return (64 - len(digest)) * '0' + digest + + @staticmethod + def __pad_hex(long_int): + """ Converts a Long integer (or hex string) to hex format padded with zeroes for hashing + + :param int|str long_int: Number or string to pad. + + :return Padded hex string. + :rtype: str + """ + + if not isinstance(long_int, str): + hash_str = AwsIdp.__long_to_hex(long_int) + else: + hash_str = long_int + if len(hash_str) % 2 == 1: + hash_str = '0%s' % hash_str + elif hash_str[0] in '89ABCDEFabcdef': + hash_str = '00%s' % hash_str + return hash_str + + @staticmethod + def __get_random(nbytes): + random_hex = binascii.hexlify(os.urandom(nbytes)) + return AwsIdp.__hex_to_long(random_hex) + + @staticmethod + def __get_current_timestamp(): + """ Creates a timestamp with the correct English format. + + :return: timestamp in format 'Sun Jan 27 19:00:04 UTC 2019' + :rtype: str + """ + + # We need US only data, so we cannot just do a strftime: + # Sun Jan 27 19:00:04 UTC 2019 + months = [None, 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'] + days = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'] + + time_now = datetime.datetime.utcnow() + format_string = "{} {} {} %H:%M:%S UTC %Y".format(days[time_now.weekday()], months[time_now.month], time_now.day) + time_string = datetime.datetime.utcnow().strftime(format_string) + return time_string + + def __str__(self): + return "AWS IDP Client for:\nRegion: %s\nPoolId: %s\nAppId: %s" % ( + self.region, self.pool_id.split("_")[1], self.client_id + ) diff --git a/lib/yt_dlp/extractor/tiktok.py b/lib/yt_dlp/extractor/tiktok.py index c58538394..4a35a241c 100644 --- a/lib/yt_dlp/extractor/tiktok.py +++ b/lib/yt_dlp/extractor/tiktok.py @@ -25,7 +25,7 @@ class TikTokBaseIE(InfoExtractor): - _APP_VERSIONS = [('20.9.3', '293'), ('20.4.3', '243'), ('20.2.1', '221'), ('20.1.2', '212'), ('20.0.4', '204')] + _APP_VERSIONS = [('26.1.3', '260103'), ('26.1.2', '260102'), ('26.1.1', '260101'), ('25.6.2', '250602')] _WORKING_APP_VERSION = None _APP_NAME = 'trill' _AID = 1180 @@ -33,7 +33,6 @@ class TikTokBaseIE(InfoExtractor): _UPLOADER_URL_FORMAT = 'https://www.tiktok.com/@%s' _WEBPAGE_HOST = 'https://www.tiktok.com/' QUALITIES = ('360p', '540p', '720p', '1080p') - _session_initialized = False @staticmethod def _create_url(user_id, video_id): @@ -43,12 +42,6 @@ def _get_sigi_state(self, webpage, display_id): return self._parse_json(get_element_by_id( 'SIGI_STATE|sigi-persisted-data', webpage, escape_value=False), display_id) - def _real_initialize(self): - if self._session_initialized: - return - self._request_webpage(HEADRequest('https://www.tiktok.com'), None, note='Setting up session', fatal=False) - TikTokBaseIE._session_initialized = True - def _call_api_impl(self, ep, query, manifest_app_version, video_id, fatal=True, note='Downloading API JSON', errnote='Unable to download API page'): self._set_cookie(self._API_HOSTNAME, 'odin_tt', ''.join(random.choice('0123456789abcdef') for _ in range(160))) @@ -289,7 +282,7 @@ def extract_addr(addr, add_meta={}): 'uploader_url': user_url, 'track': music_track, 'album': str_or_none(music_info.get('album')) or None, - 'artist': music_author, + 'artist': music_author or None, 'timestamp': int_or_none(aweme_detail.get('create_time')), 'formats': formats, 'subtitles': self.extract_subtitles(aweme_detail, aweme_id), @@ -522,7 +515,7 @@ class TikTokIE(TikTokBaseIE): 'repost_count': int, 'comment_count': int, }, - 'expected_warnings': ['trying feed workaround', 'Unable to find video in feed'] + 'skip': 'This video is unavailable', }, { # Auto-captions available 'url': 'https://www.tiktok.com/@hankgreen1/video/7047596209028074758', @@ -530,18 +523,11 @@ class TikTokIE(TikTokBaseIE): }] def _extract_aweme_app(self, aweme_id): - try: - aweme_detail = self._call_api('aweme/detail', {'aweme_id': aweme_id}, aweme_id, - note='Downloading video details', errnote='Unable to download video details').get('aweme_detail') - if not aweme_detail: - raise ExtractorError('Video not available', video_id=aweme_id) - except ExtractorError as e: - self.report_warning(f'{e.orig_msg}; trying feed workaround') - feed_list = self._call_api('feed', {'aweme_id': aweme_id}, aweme_id, - note='Downloading video feed', errnote='Unable to download video feed').get('aweme_list') or [] - aweme_detail = next((aweme for aweme in feed_list if str(aweme.get('aweme_id')) == aweme_id), None) - if not aweme_detail: - raise ExtractorError('Unable to find video in feed', video_id=aweme_id) + feed_list = self._call_api('feed', {'aweme_id': aweme_id}, aweme_id, + note='Downloading video feed', errnote='Unable to download video feed').get('aweme_list') or [] + aweme_detail = next((aweme for aweme in feed_list if str(aweme.get('aweme_id')) == aweme_id), None) + if not aweme_detail: + raise ExtractorError('Unable to find video in feed', video_id=aweme_id) return self._parse_aweme_video_app(aweme_detail) def _real_extract(self, url): @@ -572,6 +558,7 @@ def _real_extract(self, url): class TikTokUserIE(TikTokBaseIE): IE_NAME = 'tiktok:user' _VALID_URL = r'https?://(?:www\.)?tiktok\.com/@(?P[\w\.-]+)/?(?:$|[#?])' + _WORKING = False _TESTS = [{ 'url': 'https://tiktok.com/@corgibobaa?lang=en', 'playlist_mincount': 45, @@ -708,6 +695,7 @@ def _real_extract(self, url): class TikTokSoundIE(TikTokBaseListIE): IE_NAME = 'tiktok:sound' _VALID_URL = r'https?://(?:www\.)?tiktok\.com/music/[\w\.-]+-(?P[\d]+)[/?#&]?' + _WORKING = False _QUERY_NAME = 'music_id' _API_ENDPOINT = 'music/aweme' _TESTS = [{ @@ -731,6 +719,7 @@ class TikTokSoundIE(TikTokBaseListIE): class TikTokEffectIE(TikTokBaseListIE): IE_NAME = 'tiktok:effect' _VALID_URL = r'https?://(?:www\.)?tiktok\.com/sticker/[\w\.-]+-(?P[\d]+)[/?#&]?' + _WORKING = False _QUERY_NAME = 'sticker_id' _API_ENDPOINT = 'sticker/aweme' _TESTS = [{ @@ -750,6 +739,7 @@ class TikTokEffectIE(TikTokBaseListIE): class TikTokTagIE(TikTokBaseListIE): IE_NAME = 'tiktok:tag' _VALID_URL = r'https?://(?:www\.)?tiktok\.com/tag/(?P[^/?#&]+)' + _WORKING = False _QUERY_NAME = 'ch_id' _API_ENDPOINT = 'challenge/aweme' _TESTS = [{ diff --git a/lib/yt_dlp/extractor/vier.py b/lib/yt_dlp/extractor/vier.py deleted file mode 100644 index eab894ab6..000000000 --- a/lib/yt_dlp/extractor/vier.py +++ /dev/null @@ -1,261 +0,0 @@ -import re -import itertools - -from .common import InfoExtractor -from ..utils import ( - urlencode_postdata, - int_or_none, - unified_strdate, -) - - -class VierIE(InfoExtractor): - IE_NAME = 'vier' - IE_DESC = 'vier.be and vijf.be' - _VALID_URL = r'''(?x) - https?:// - (?:www\.)?(?Pvier|vijf)\.be/ - (?: - (?: - [^/]+/videos| - video(?:/[^/]+)* - )/ - (?P[^/]+)(?:/(?P\d+))?| - (?: - video/v3/embed| - embed/video/public - )/(?P\d+) - ) - ''' - _NETRC_MACHINE = 'vier' - _TESTS = [{ - 'url': 'http://www.vier.be/planb/videos/het-wordt-warm-de-moestuin/16129', - 'md5': 'e4ae2054a6b040ef1e289e20d111b46e', - 'info_dict': { - 'id': '16129', - 'display_id': 'het-wordt-warm-de-moestuin', - 'ext': 'mp4', - 'title': 'Het wordt warm in De Moestuin', - 'description': 'De vele uren werk eisen hun tol. Wim droomt van assistentie...', - 'upload_date': '20121025', - 'series': 'Plan B', - 'tags': ['De Moestuin', 'Moestuin', 'meisjes', 'Tomaat', 'Wim', 'Droom'], - }, - }, { - 'url': 'http://www.vijf.be/temptationisland/videos/zo-grappig-temptation-island-hosts-moeten-kiezen-tussen-onmogelijke-dilemmas/2561614', - 'info_dict': { - 'id': '2561614', - 'display_id': 'zo-grappig-temptation-island-hosts-moeten-kiezen-tussen-onmogelijke-dilemmas', - 'ext': 'mp4', - 'title': 'md5:84f45fe48b8c1fa296a7f6d208d080a7', - 'description': 'md5:0356d4981e58b8cbee19355cbd51a8fe', - 'upload_date': '20170228', - 'series': 'Temptation Island', - 'tags': list, - }, - 'params': { - 'skip_download': True, - }, - }, { - 'url': 'http://www.vier.be/janigaat/videos/jani-gaat-naar-tokio-aflevering-4/2674839', - 'info_dict': { - 'id': '2674839', - 'display_id': 'jani-gaat-naar-tokio-aflevering-4', - 'ext': 'mp4', - 'title': 'Jani gaat naar Tokio - Aflevering 4', - 'description': 'md5:aa8d611541db6ae9e863125704511f88', - 'upload_date': '20170501', - 'series': 'Jani gaat', - 'episode_number': 4, - 'tags': ['Jani Gaat', 'Volledige Aflevering'], - }, - 'params': { - 'skip_download': True, - }, - 'skip': 'Requires account credentials', - }, { - # Requires account credentials but bypassed extraction via v3/embed page - # without metadata - 'url': 'http://www.vier.be/janigaat/videos/jani-gaat-naar-tokio-aflevering-4/2674839', - 'info_dict': { - 'id': '2674839', - 'display_id': 'jani-gaat-naar-tokio-aflevering-4', - 'ext': 'mp4', - 'title': 'jani-gaat-naar-tokio-aflevering-4', - }, - 'params': { - 'skip_download': True, - }, - 'expected_warnings': ['Log in to extract metadata'], - }, { - # Without video id in URL - 'url': 'http://www.vier.be/planb/videos/dit-najaar-plan-b', - 'only_matching': True, - }, { - 'url': 'http://www.vier.be/video/v3/embed/16129', - 'only_matching': True, - }, { - 'url': 'https://www.vijf.be/embed/video/public/4093', - 'only_matching': True, - }, { - 'url': 'https://www.vier.be/video/blockbusters/in-juli-en-augustus-summer-classics', - 'only_matching': True, - }, { - 'url': 'https://www.vier.be/video/achter-de-rug/2017/achter-de-rug-seizoen-1-aflevering-6', - 'only_matching': True, - }] - - def _real_initialize(self): - self._logged_in = False - - def _login(self, site): - username, password = self._get_login_info() - if username is None or password is None: - return - - login_page = self._download_webpage( - 'http://www.%s.be/user/login' % site, - None, note='Logging in', errnote='Unable to log in', - data=urlencode_postdata({ - 'form_id': 'user_login', - 'name': username, - 'pass': password, - }), - headers={'Content-Type': 'application/x-www-form-urlencoded'}) - - login_error = self._html_search_regex( - r'(?s)
\s*
\s*(.+?)<', - login_page, 'login error', default=None) - if login_error: - self.report_warning('Unable to log in: %s' % login_error) - else: - self._logged_in = True - - def _real_extract(self, url): - mobj = self._match_valid_url(url) - embed_id = mobj.group('embed_id') - display_id = mobj.group('display_id') or embed_id - video_id = mobj.group('id') or embed_id - site = mobj.group('site') - - if not self._logged_in: - self._login(site) - - webpage = self._download_webpage(url, display_id) - - if r'id="user-login"' in webpage: - self.report_warning( - 'Log in to extract metadata', video_id=display_id) - webpage = self._download_webpage( - 'http://www.%s.be/video/v3/embed/%s' % (site, video_id), - display_id) - - video_id = self._search_regex( - [r'data-nid="(\d+)"', r'"nid"\s*:\s*"(\d+)"'], - webpage, 'video id', default=video_id or display_id) - - playlist_url = self._search_regex( - r'data-file=(["\'])(?P(?:https?:)?//[^/]+/.+?\.m3u8.*?)\1', - webpage, 'm3u8 url', default=None, group='url') - - if not playlist_url: - application = self._search_regex( - [r'data-application="([^"]+)"', r'"application"\s*:\s*"([^"]+)"'], - webpage, 'application', default=site + '_vod') - filename = self._search_regex( - [r'data-filename="([^"]+)"', r'"filename"\s*:\s*"([^"]+)"'], - webpage, 'filename') - playlist_url = 'http://vod.streamcloud.be/%s/_definst_/mp4:%s.mp4/playlist.m3u8' % (application, filename) - - formats = self._extract_wowza_formats( - playlist_url, display_id, skip_protocols=['dash']) - self._sort_formats(formats) - - title = self._og_search_title(webpage, default=display_id) - description = self._html_search_regex( - r'(?s)]+\bclass=(["\'])[^>]*?\bfield-type-text-with-summary\b[^>]*?\1[^>]*>.*?

(?P.+?)

', - webpage, 'description', default=None, group='value') - thumbnail = self._og_search_thumbnail(webpage, default=None) - upload_date = unified_strdate(self._html_search_regex( - r'(?s)]+\bclass=(["\'])[^>]*?\bfield-name-post-date\b[^>]*?\1[^>]*>.*?(?P\d{2}/\d{2}/\d{4})', - webpage, 'upload date', default=None, group='value')) - - series = self._search_regex( - r'data-program=(["\'])(?P(?:(?!\1).)+)\1', webpage, - 'series', default=None, group='value') - episode_number = int_or_none(self._search_regex( - r'(?i)aflevering (\d+)', title, 'episode number', default=None)) - tags = re.findall(r']+\bhref=["\']/tags/[^>]+>([^<]+)<', webpage) - - return { - 'id': video_id, - 'display_id': display_id, - 'title': title, - 'description': description, - 'thumbnail': thumbnail, - 'upload_date': upload_date, - 'series': series, - 'episode_number': episode_number, - 'tags': tags, - 'formats': formats, - } - - -class VierVideosIE(InfoExtractor): - IE_NAME = 'vier:videos' - _VALID_URL = r'https?://(?:www\.)?(?Pvier|vijf)\.be/(?P[^/]+)/videos(?:\?.*\bpage=(?P\d+)|$)' - _TESTS = [{ - 'url': 'http://www.vier.be/demoestuin/videos', - 'info_dict': { - 'id': 'demoestuin', - }, - 'playlist_mincount': 153, - }, { - 'url': 'http://www.vijf.be/temptationisland/videos', - 'info_dict': { - 'id': 'temptationisland', - }, - 'playlist_mincount': 159, - }, { - 'url': 'http://www.vier.be/demoestuin/videos?page=6', - 'info_dict': { - 'id': 'demoestuin-page6', - }, - 'playlist_mincount': 20, - }, { - 'url': 'http://www.vier.be/demoestuin/videos?page=7', - 'info_dict': { - 'id': 'demoestuin-page7', - }, - 'playlist_mincount': 13, - }] - - def _real_extract(self, url): - mobj = self._match_valid_url(url) - program = mobj.group('program') - site = mobj.group('site') - - page_id = mobj.group('page') - if page_id: - page_id = int(page_id) - start_page = page_id - playlist_id = '%s-page%d' % (program, page_id) - else: - start_page = 0 - playlist_id = program - - entries = [] - for current_page_id in itertools.count(start_page): - current_page = self._download_webpage( - 'http://www.%s.be/%s/videos?page=%d' % (site, program, current_page_id), - program, - 'Downloading page %d' % (current_page_id + 1)) - page_entries = [ - self.url_result('http://www.' + site + '.be' + video_url, 'Vier') - for video_url in re.findall( - r'', current_page)] - entries.extend(page_entries) - if page_id or '>Meer<' not in current_page: - break - - return self.playlist_result(entries, playlist_id) diff --git a/lib/yt_dlp_version b/lib/yt_dlp_version index 8a909edcf..67d502c59 100644 --- a/lib/yt_dlp_version +++ b/lib/yt_dlp_version @@ -1 +1 @@ -19b4e59a1e1bf368078f90e7f735fa4576f97b64 \ No newline at end of file +f7c5a5e96756636379a0b1afbeadb08b9c643bef \ No newline at end of file diff --git a/service.py b/service.py index 8304ed1af..5b2d8fd23 100644 --- a/service.py +++ b/service.py @@ -87,42 +87,51 @@ def getParams(): def play(url, ydl_opts): - if xbmcplugin.getSetting(int(sys.argv[1]),"usemanifest"): - ydl_opts['format'] = 'bestvideo*+bestaudio/best' - ydl = YoutubeDL(ydl_opts) - ydl.add_default_info_extractors() - - with ydl: - showInfoNotification("Resolving stream(s) for " + url) - result = ydl.extract_info(url, download=False) - - if 'entries' in result: - # more than one video - pl = xbmc.PlayList(1) - pl.clear() - - # determine which index in the queue to start playing from - indexToStartAt = playlistIndex(url, result) - if indexToStartAt == None: - indexToStartAt = 0 - - unresolvedEntries = list(result['entries']) - startingEntry = unresolvedEntries.pop(indexToStartAt) - - # populate the queue with unresolved entries so that the starting entry can be inserted - for video in unresolvedEntries: - list_item = createListItemFromFlatPlaylistItem(video) - pl.add(list_item.getPath(), list_item) - - # make sure the starting ListItem has a resolved url, to avoid recursion and crashes - startingVideoUrl = startingEntry['url'] - startingItem = createListItemFromVideo(ydl.extract_info(startingVideoUrl, download=False)) - pl.add(startingItem.getPath(), startingItem, indexToStartAt) - - #xbmc.Player().play(pl) # this probably works again - # ...but start playback the same way the Youtube plugin does it: - xbmc.executebuiltin('Playlist.PlayOffset(%s,%d)' % ('video', indexToStartAt)) - + if xbmcplugin.getSetting(int(sys.argv[1]),"usemanifest") == 'true': + ydl_opts['format'] = 'bestvideo*+bestaudio/best' + ydl = YoutubeDL(ydl_opts) + ydl.add_default_info_extractors() + + with ydl: + progress = xbmcgui.DialogProgressBG() + progress.create("Resolving " + url) + try: + result = ydl.extract_info(url, download=False) + except: + progress.close() + showErrorNotification("Could not resolve the url, check the log for more info") + import traceback + log(msg=traceback.format_exc(), level=xbmc.LOGERROR) + exit() + progress.close() + + if 'entries' in result: + # more than one video + pl = xbmc.PlayList(1) + pl.clear() + + # determine which index in the queue to start playing from + indexToStartAt = playlistIndex(url, result) + if indexToStartAt == None: + indexToStartAt = 0 + + unresolvedEntries = list(result['entries']) + startingEntry = unresolvedEntries.pop(indexToStartAt) + + # populate the queue with unresolved entries so that the starting entry can be inserted + for video in unresolvedEntries: + list_item = createListItemFromFlatPlaylistItem(video) + pl.add(list_item.getPath(), list_item) + + # make sure the starting ListItem has a resolved url, to avoid recursion and crashes + startingVideoUrl = startingEntry['url'] + startingItem = createListItemFromVideo(ydl.extract_info(startingVideoUrl, download=False)) + pl.add(startingItem.getPath(), startingItem, indexToStartAt) + + #xbmc.Player().play(pl) # this probably works again + # ...but start playback the same way the Youtube plugin does it: + xbmc.executebuiltin('Playlist.PlayOffset(%s,%d)' % ('video', indexToStartAt)) + showInfoNotification("Playing playlist " + result['title']) else: # Just a video, pass the item to the Kodi player. @@ -202,7 +211,7 @@ def check_if_kodi_supports_manifest(url): def createListItemFromVideo(result): debug(result) adaptive_type = False - if xbmcplugin.getSetting(int(sys.argv[1]),"usemanifest"): + if xbmcplugin.getSetting(int(sys.argv[1]),"usemanifest") == 'true': url = extract_manifest_url(result) if url is not None: log("found original manifest: " + url) From a14acfae80648ef2fe20c1500972801ed21d9225 Mon Sep 17 00:00:00 2001 From: Gautam Garg Date: Thu, 22 Sep 2022 21:52:35 +0530 Subject: [PATCH 5/6] Code Review Changes --- service.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/service.py b/service.py index 5b2d8fd23..981c7118f 100644 --- a/service.py +++ b/service.py @@ -68,7 +68,7 @@ def get_local_user_input(): def getParams(): - result = {} + params = {} paramstring = sys.argv[2] if not paramstring: @@ -76,14 +76,15 @@ def getParams(): additionalParamsIndex = paramstring.find(' ') if additionalParamsIndex == -1: - result['url'] = paramstring[1:] - result['ydlOpts'] = {} + params['url'] = paramstring[1:] + params['ydlOpts'] = {} else: - result['url'] = paramstring[1:additionalParamsIndex] + params['url'] = paramstring[1:additionalParamsIndex] additionalParamsString = paramstring[additionalParamsIndex:] additionalParams = json.loads(additionalParamsString) - result['ydlOpts'] = additionalParams['ydlOpts'] - return result + params['ydlOpts'] = additionalParams['ydlOpts'] + params['url'] = str(params['url']) + return params def play(url, ydl_opts): @@ -317,7 +318,7 @@ def playlistIndex(url, playlist): } params = getParams() -url = str(params['url']) +url = params['url']) ydl_opts.update(params['ydlOpts']) if url == '': showInfoNotification("Kindly provide the valid URL.") From 6753c5c0d17f0b09aece2a4f0dab6313cf2e2493 Mon Sep 17 00:00:00 2001 From: Gautam Garg Date: Thu, 22 Sep 2022 22:14:05 +0530 Subject: [PATCH 6/6] Error Fix --- service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service.py b/service.py index 981c7118f..d55aa48fb 100644 --- a/service.py +++ b/service.py @@ -318,7 +318,7 @@ def playlistIndex(url, playlist): } params = getParams() -url = params['url']) +url = params['url'] ydl_opts.update(params['ydlOpts']) if url == '': showInfoNotification("Kindly provide the valid URL.")