diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 01dfc4f..72446b5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,18 @@ Changelog ********* +v3.6 (2022-07-09) +======================================== + +- preload support for mopidy-tubeify + + +v3.5 (2021-12-20) +======================================== + +- changes + + v3.4 (2021-04-11) ======================================== diff --git a/mopidy_youtube/apis/youtube_japi.py b/mopidy_youtube/apis/youtube_japi.py index 50973f1..0ba2bd7 100644 --- a/mopidy_youtube/apis/youtube_japi.py +++ b/mopidy_youtube/apis/youtube_japi.py @@ -16,6 +16,7 @@ watchVideoPath, ) from mopidy_youtube.comms import Client +from mopidy_youtube.timeformat import format_duration from mopidy_youtube.youtube import Video @@ -490,9 +491,7 @@ def json_to_items(result_json): try: duration_text = video["lengthText"]["simpleText"] - duration = "PT" + Client.format_duration( - re.match(Client.time_regex, duration_text) - ) + duration = "PT" + format_duration(duration_text) logger.debug(f"video {videoId} duration: {duration}") except Exception as e: logger.warn(f"video {videoId} no video-time, possibly live: {e}") diff --git a/mopidy_youtube/apis/youtube_music.py b/mopidy_youtube/apis/youtube_music.py index 4a083c1..c28c243 100644 --- a/mopidy_youtube/apis/youtube_music.py +++ b/mopidy_youtube/apis/youtube_music.py @@ -1,5 +1,4 @@ import json - from concurrent.futures import as_completed from concurrent.futures.thread import ThreadPoolExecutor from itertools import repeat @@ -34,7 +33,7 @@ def __init__(self, proxy, headers, *args, **kwargs): else json.dumps(headers) ) try: - ytmusic = YTMusic(auth=auth) + ytmusic = YTMusic(auth=auth, requests_session=self.session) except Exception as e: logger.error("YTMusic init error: %s", str(e)) ytmusic = YTMusic() @@ -63,48 +62,54 @@ def list_related_videos(cls, video_id): returns related videos for a given video_id """ - logger.debug( - f"youtube_music list_related_videos triggered " - f"ytmusic.get_watch_playlist {video_id}" - ) - # this is untested - try to add artist and channel to related # videos by calling get_song for each related song # this would be faster with threading, but it all happens in the # background, so who cares? # What is better: get_watch_playlist or get_song_related? Are they different? - get_song_related_tracks = [] - get_watch_playlist = ytmusic.get_watch_playlist(video_id) - logger.debug( - f"youtube_music list_related_videos triggered " - f"ytmusic.get_watch_playlist: {video_id}" - ) + get_watch_playlist = {} - related_browseId = get_watch_playlist["related"] + try: + logger.debug( + f"youtube_music list_related_videos triggered " + f"ytmusic.get_watch_playlist: {video_id}" + ) + + get_watch_playlist = ytmusic.get_watch_playlist(video_id) + related_browseId = get_watch_playlist.get("related", "none") + except Exception as e: + logger.error( + f"youtube_music list_related_videos get_watch_playlist " + f"error:{e}. videoId: {video_id}" + ) + + related_videos = [] + get_song_related_tracks = [] try: + logger.debug( + f"youtube_music list_related_videos triggered " + f"ytmusic.get_song_related ({related_browseId})" + ) get_song_related_tracks = ytmusic.get_song_related(related_browseId)[0][ "contents" ] + logger.debug( + f"youtube_music list_related_videos triggered " + f"ytmusic.get_song for {len(related_videos)} tracks." + ) related_videos = [ ytmusic.get_song(track["videoId"])["videoDetails"] for track in get_song_related_tracks ] - logger.debug( - f"youtube_music list_related_videos triggered " - f"ytmusic.get_song_related ({related_browseId}), and ytmusic.get_song " - f"for {len(related_videos)} tracks." - ) - except Exception as e: logger.error( f"youtube_music list_related_videos error:{e} " f"Related_browseId: {related_browseId}" ) - related_videos = [] if len(related_videos) < 10: logger.warn( @@ -112,17 +117,17 @@ def list_related_videos(cls, video_id): f"Trying get_watch_playlist['tracks'] for more" ) try: + logger.debug( + f"youtube_music list_related_videos triggered " + f"ytmusic.get_song for {len(get_watch_playlist['tracks'])} tracks" + ) + related_videos.extend( [ ytmusic.get_song(track["videoId"])["videoDetails"] for track in get_watch_playlist["tracks"] ] ) - - logger.debug( - f"youtube_music list_related_videos triggered " - f"ytmusic.get_song for {len(get_watch_playlist['tracks'])} tracks" - ) except Exception as e: logger.error(f"youtube_music list_related_videos error:{e}") @@ -146,8 +151,7 @@ def list_related_videos(cls, video_id): if len(tracks) < 10: logger.warn( f"get_song_related and get_watch_playlist only returned " - f"{len(tracks)} tracks. " - f"Trying youtube_japi.jAPI.list_related_videos" + f"{len(tracks)} tracks. Trying youtube_japi.jAPI.list_related_videos" ) japi_related_videos = youtube_japi.jAPI.list_related_videos(video_id) tracks.extend(japi_related_videos["items"]) @@ -229,7 +233,8 @@ def list_playlists(cls, ids): results = [] logger.debug( - f"youtube_music list_playlists triggered _get_playlist_or_album x {len(ids)}: {ids}" + f"youtube_music list_playlists triggered " + f"_get_playlist_or_album x {len(ids)}: {ids}" ) with ThreadPoolExecutor() as executor: @@ -483,7 +488,6 @@ def yt_listitem_to_playlist(item, channelTitle=None): "contentDetails": {"itemCount": itemCount}, "artists": item.get("artists", None), } - if "tracks" in item: fields = ["artists", "thumbnails"] [ diff --git a/mopidy_youtube/apis/ytm_item_to_video.py b/mopidy_youtube/apis/ytm_item_to_video.py index f898745..145b695 100644 --- a/mopidy_youtube/apis/ytm_item_to_video.py +++ b/mopidy_youtube/apis/ytm_item_to_video.py @@ -1,7 +1,5 @@ -import re - from mopidy_youtube import logger -from mopidy_youtube.comms import Client +from mopidy_youtube.timeformat import format_duration def ytm_item_to_video(item): @@ -35,7 +33,7 @@ def _convertMillis(milliseconds): logger.error(f"youtube_music yt_item_to_video duration error {e}: {item}") try: - duration = "PT" + Client.format_duration(re.match(Client.time_regex, duration)) + duration = "PT" + format_duration(duration) except Exception as e: logger.error( f"youtube_music yt_item_to_video format duration error {e}: {item}" diff --git a/mopidy_youtube/backend.py b/mopidy_youtube/backend.py index 9f5ec5b..c2630e7 100644 --- a/mopidy_youtube/backend.py +++ b/mopidy_youtube/backend.py @@ -2,6 +2,7 @@ import os import pykka +from cachetools import TTLCache, cached from mopidy import backend, httpclient from mopidy.core import CoreListener from mopidy.models import Image, Ref, SearchResult, Track, model_json_decoder @@ -144,7 +145,12 @@ class YouTubeLibraryProvider(backend.LibraryProvider): When enabled makes possible to browse public playlists of the channel as well as browse separate tracks in playlists. """ + cache_max_len = 4000 + cache_ttl = 21600 + youtube_library_cache = TTLCache(maxsize=cache_max_len, ttl=cache_ttl) + + @cached(cache=youtube_library_cache) def browse(self, uri): if uri == "youtube:browse": return [ @@ -325,15 +331,10 @@ def lookup(self, uri): preload = extract_preload_tracks(uri) if preload: for track in preload[1]: - logger.info(track) video = Video.get(track["id"]["videoId"]) minimum_fields = ["title", "length", "channel"] item, extended_fields = video.extend_fields(track, minimum_fields) - logger.info(f"{extended_fields}") - video._set_api_data( - extended_fields, - item, - ) + video._set_api_data(extended_fields, item) return [self.lookup_video_track(preload[0])] playlist_id = extract_playlist_id(uri) diff --git a/mopidy_youtube/comms.py b/mopidy_youtube/comms.py index 61ee834..56258cc 100644 --- a/mopidy_youtube/comms.py +++ b/mopidy_youtube/comms.py @@ -23,12 +23,6 @@ def init_poolmanager(self, *args, **kwargs): class Client: - time_regex = ( - r"(?:(?:(?P[0-9]+)\:)?" - r"(?P[0-9]+)\:" - r"(?P[0-9]{2}))" - ) - def __init__(self, proxy, headers): if not hasattr(type(self), "session"): self._create_session(proxy, headers) @@ -58,14 +52,3 @@ def _create_session( cls.session.mount("https://", adapter) cls.session.proxies = {"http": proxy, "https": proxy} cls.session.headers = headers - - @classmethod - def format_duration(cls, match): - duration = "" - if match.group("durationHours") is not None: - duration += match.group("durationHours") + "H" - if match.group("durationMinutes") is not None: - duration += match.group("durationMinutes") + "M" - if match.group("durationSeconds") is not None: - duration += match.group("durationSeconds") + "S" - return duration diff --git a/mopidy_youtube/data.py b/mopidy_youtube/data.py index 87be6d7..fd98445 100644 --- a/mopidy_youtube/data.py +++ b/mopidy_youtube/data.py @@ -34,6 +34,8 @@ def format_channel_uri(id) -> str: def extract_video_id(uri) -> str: + if uri is None: + return "" if "youtube.com" in uri: url = urlparse(uri.replace("yt:", "").replace("youtube:", "")) req = parse_qs(url.query) diff --git a/mopidy_youtube/frontend.py b/mopidy_youtube/frontend.py index 4a90dee..4c06bfc 100644 --- a/mopidy_youtube/frontend.py +++ b/mopidy_youtube/frontend.py @@ -70,9 +70,9 @@ def track_playback_started(self, tl_track): "better results" ) return None - + logger.debug(f"getting current track id for {track.uri}") current_track_id = extract_video_id(track.uri) - + logger.debug(f"track id {current_track_id}") if self.max_degrees_of_separation: if self.degrees_of_separation < self.max_degrees_of_separation: self.degrees_of_separation += 1 @@ -81,31 +81,38 @@ def track_playback_started(self, tl_track): current_track_id = self.base_track_id self.degrees_of_separation = 0 logger.debug("resetting to autoplay base track id") + logger.debug(f"degrees of sep {self.degrees_of_separation}") if current_track_id not in autoplayed: self.base_track_id = current_track_id autoplayed.append(current_track_id) # avoid replaying track self.degrees_of_separation = 0 logger.debug("setting new autoplay base id") + logger.debug(f"base track id {self.base_track_id}") current_track = youtube.Video.get(current_track_id) + logger.debug(f"triggered related videos for {current_track.id}") current_track.related_videos + logger.debug("getting related videos") related_videos = current_track.related_videos.get() logger.debug( f"autoplayer is adding a track related to {current_track.title.get()}" ) + logger.debug(f"related videos {related_videos}") # remove already autoplayed related_videos[:] = [ related_video for related_video in related_videos if related_video.id not in autoplayed ] + logger.debug(f"related videos edit 1 {related_videos}") # remove if track_length is 0 (probably a live video) or None related_videos[:] = [ related_video for related_video in related_videos if related_video.length.get() ] + logger.debug(f"related videos edit 2 {related_videos}") # remove if too long if self.max_autoplay_length: related_videos[:] = [ @@ -113,6 +120,7 @@ def track_playback_started(self, tl_track): for related_video in related_videos if related_video.length.get() < self.max_autoplay_length ] + logger.debug(f"related videos edit 3{related_videos}") if len(related_videos) == 0: logger.warn( @@ -122,6 +130,7 @@ def track_playback_started(self, tl_track): return None else: next_video = random.choice(related_videos) + logger.debug(f"next video {next_video.id}") autoplayed.append(next_video.id) uri = [format_video_uri(next_video.id)] tl.add(uris=uri).get() diff --git a/mopidy_youtube/timeformat.py b/mopidy_youtube/timeformat.py new file mode 100644 index 0000000..8e09ac0 --- /dev/null +++ b/mopidy_youtube/timeformat.py @@ -0,0 +1,47 @@ +import re + + +def format_duration(duration_text): + + time_regex = ( + r"(?:(?:(?P[0-9]+)\:)?" + r"(?P[0-9]+)\:" + r"(?P[0-9]{2}))" + ) + + match = re.match(time_regex, duration_text) + + duration = "" + if match.group("durationHours") is not None: + duration += match.group("durationHours") + "H" + if match.group("durationMinutes") is not None: + duration += match.group("durationMinutes") + "M" + if match.group("durationSeconds") is not None: + duration += match.group("durationSeconds") + "S" + + return duration + + +def ISO8601_to_seconds(iso_duration): + + # convert PT1H2M10S to 3730 + m = re.search( + r"P((?P\d+)W)?" + + r"((?P\d+)D)?" + + r"T((?P\d+)H)?" + + r"((?P\d+)M)?" + + r"((?P\d+)S)?", + iso_duration, + ) + if m: + val = ( + int(m.group("weeks") or 0) * 604800 + + int(m.group("days") or 0) * 86400 + + int(m.group("hours") or 0) * 3600 + + int(m.group("minutes") or 0) * 60 + + int(m.group("seconds") or 0) + ) + else: + val = 0 + + return val diff --git a/mopidy_youtube/youtube.py b/mopidy_youtube/youtube.py index efd4d4c..6a3f744 100644 --- a/mopidy_youtube/youtube.py +++ b/mopidy_youtube/youtube.py @@ -1,7 +1,6 @@ import importlib import json import os -import re import shutil from concurrent.futures.thread import ThreadPoolExecutor @@ -11,6 +10,7 @@ from mopidy_youtube import logger from mopidy_youtube.converters import convert_video_to_track +from mopidy_youtube.timeformat import ISO8601_to_seconds api_enabled = False channel = None @@ -168,25 +168,8 @@ def _set_api_data(self, fields, item): elif k == "artists": val = item["artists"] elif k == "length": - # convert PT1H2M10S to 3730 - m = re.search( - r"P((?P\d+)W)?" - + r"((?P\d+)D)?" - + r"T((?P\d+)H)?" - + r"((?P\d+)M)?" - + r"((?P\d+)S)?", - item["contentDetails"]["duration"], - ) - if m: - val = ( - int(m.group("weeks") or 0) * 604800 - + int(m.group("days") or 0) * 86400 - + int(m.group("hours") or 0) * 3600 - + int(m.group("minutes") or 0) * 60 - + int(m.group("seconds") or 0) - ) - else: - val = 0 + # convert ISO8601 (PT1H2M10S) to s (3730) + val = ISO8601_to_seconds(item["contentDetails"]["duration"]) elif k == "video_count": val = min( int(item["contentDetails"]["itemCount"]), @@ -194,9 +177,13 @@ def _set_api_data(self, fields, item): ) elif k == "thumbnails": val = [ - Image(uri=val["url"], width=val["width"], height=val["height"]) - for (key, val) in item["snippet"]["thumbnails"].items() - if key in ["default", "medium", "high"] + Image( + uri=details["url"], + width=details["width"], + height=details["height"], + ) + for (quality, details) in item["snippet"]["thumbnails"].items() + if quality in ["default", "medium", "high"] ] or None # is this "or None" necessary? elif k == "channelId": val = item["snippet"]["channelId"] @@ -204,7 +191,6 @@ def _set_api_data(self, fields, item): @classmethod def extend_fields(self, item, fields): - # logger.info(f"in: {item, fields}") extended_fields = set(fields) if "snippet" in item: if "channelId" in item["snippet"]: @@ -240,7 +226,6 @@ def extend_fields(self, item, fields): extended_fields.add("length") elif "itemCount" in item["contentDetails"]: extended_fields.add("video_count") - # logger.info(f"out: {item, list(extended_fields)}") return (item, list(extended_fields)) @@ -258,7 +243,6 @@ def load_info(cls, listOfVideos): listOfVideos = cls._add_futures(listOfVideos, minimum_fields) def job(sublist): - # logger.info(sublist) try: data = cls.api.list_videos([x.id for x in sublist]) except Exception as e: @@ -277,10 +261,9 @@ def job(sublist): # extended_fields = minimum_fields video._set_api_data(extended_fields, item_dict.get(video.id)) else: - logger.info( + logger.warn( f"no dict: {video.id, type(video), item_dict.get(video.id)}" ) - logger.info(f"job done: {sublist}") with ThreadPoolExecutor() as executor: # make sure order is deterministic so that HTTP requests are replayable in tests @@ -428,8 +411,12 @@ def my_hook(d): "proxy": self.proxy, "cachedir": False, "nopart": True, + "retries": 10, } + if youtube_dl_package == "yt_dlp": + ytdl_options["no_color"] = True + ytdl_extract_info_options = { "url": f"https://www.youtube.com/watch?v={self.id}", "ie_key": None, @@ -536,12 +523,10 @@ def job(sublist): if data: item_dict = {item["id"]: item for item in data["items"]} - # logger.info(f"data {data}") for pl in sublist: item_dict[pl.id], extended_fields = cls.extend_fields( item_dict.get(pl.id), minimum_fields ) - # extended_fields = minimum_fields pl._set_api_data(extended_fields, item_dict.get(pl.id)) with ThreadPoolExecutor() as executor: @@ -659,7 +644,6 @@ def playlists(cls, channel_id=None): extended_fields = minimum_fields pl._set_api_data(extended_fields, item) channel_playlists.append(pl) - logger.info(f"channel_playlists {channel_playlists}") Playlist.load_info(channel_playlists) return channel_playlists except Exception as e: diff --git a/setup.cfg b/setup.cfg index be02d76..58645a6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = Mopidy-YouTube -version = 3.5 +version = 3.6 url = https://github.com/natumbri/mopidy-youtube author = nik tumbri author_email = natumbri@gmail.com diff --git a/tests/fixtures/api/api_search.yaml b/tests/fixtures/api/api_search.yaml index baf21a4..5632785 100644 --- a/tests/fixtures/api/api_search.yaml +++ b/tests/fixtures/api/api_search.yaml @@ -837,4 +837,54 @@ interactions: status: code: 200 message: OK +- request: + body: null + headers: + Accept-Language: + - en;q=0.8 + Cookie: + - PREF=hl=en; CONSENT=YES+20210329; + user-agent: + - Mopidy-YouTube/3.6 Mopidy/3.3.0 CPython/3.8.10 + method: GET + uri: https://www.googleapis.com/youtube/v3/videos?fields=items%28id%2Csnippet%28title%2CchannelTitle%2CchannelId%29%2CcontentDetails%28duration%29%29&id=du4kNAyjVCg%2Ce1YqueG2gtQ%2CfyyiJc0Wk2M%2CCjwwmFrsX_E%2C-HYI26sG8u0%2C81RqEnvczV8%2Cktoaj1IpTbw%2CLDSVLQoPc8w%2Cifr3O33UpWs&key=fake_key&part=id%2Csnippet%2CcontentDetails + response: + body: + string: "{\n \"error\": {\n \"code\": 400,\n \"message\": \"API key not + valid. Please pass a valid API key.\",\n \"errors\": [\n {\n \"message\": + \"API key not valid. Please pass a valid API key.\",\n \"domain\": + \"global\",\n \"reason\": \"badRequest\"\n }\n ],\n \"status\": + \"INVALID_ARGUMENT\",\n \"details\": [\n {\n \"@type\": \"type.googleapis.com/google.rpc.ErrorInfo\",\n + \ \"reason\": \"API_KEY_INVALID\",\n \"domain\": \"googleapis.com\",\n + \ \"metadata\": {\n \"service\": \"youtube.googleapis.com\"\n + \ }\n }\n ]\n }\n}\n" + headers: + Accept-Ranges: + - none + Alt-Svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000,h3-Q050=":443"; ma=2592000,h3-Q046=":443"; + ma=2592000,h3-Q043=":443"; ma=2592000,quic=":443"; ma=2592000; v="46,43" + Cache-Control: + - private + Content-Type: + - application/json; charset=UTF-8 + Date: + - Sat, 09 Jul 2022 04:45:02 GMT + Server: + - scaffolding on HTTPServer2 + Transfer-Encoding: + - chunked + Vary: + - X-Origin + - Referer + - Origin,Accept-Encoding + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-XSS-Protection: + - '0' + status: + code: 400 + message: Bad Request version: 1 diff --git a/tests/fixtures/japi/api_search.yaml b/tests/fixtures/japi/api_search.yaml index 3342515..6a2e06e 100644 --- a/tests/fixtures/japi/api_search.yaml +++ b/tests/fixtures/japi/api_search.yaml @@ -13150,8 +13150,6 @@ interactions: 10:01:26 GMT; Path=/; Secure; HttpOnly; SameSite=none Strict-Transport-Security: - max-age=31536000 - Transfer-Encoding: - - chunked Vary: - Accept-Encoding X-Content-Type-Options: @@ -14558,8 +14556,6 @@ interactions: 10:04:15 GMT; Path=/; Secure; HttpOnly; SameSite=none Strict-Transport-Security: - max-age=31536000 - Transfer-Encoding: - - chunked Vary: - Accept-Encoding X-Content-Type-Options: @@ -21462,4 +21458,1453 @@ interactions: status: code: 200 message: OK +- request: + body: null + headers: + Accept-Language: + - en;q=0.8 + Cookie: + - PREF=hl=en; CONSENT=YES+20210329; + user-agent: + - Mopidy-YouTube/3.6 Mopidy/3.3.0 CPython/3.8.10 + method: GET + uri: https://www.youtube.com/playlist?app=desktop&list=PLUQWP2nyiMrwf_qfshJC4ZmN3u5GvLtfg&persist_app=1 + response: + body: + string: "
AboutPressCopyrightContact usCreatorsAdvertiseDevelopersTermsPrivacyPolicy & SafetyHow YouTube worksTest new features
© + 2022 Google LLC
CHVRCHES + Live - Glastonbury 2016 - Full Show - YouTube
" + headers: + Accept-Ranges: + - none + Alt-Svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000,h3-Q050=":443"; ma=2592000,h3-Q046=":443"; + ma=2592000,h3-Q043=":443"; ma=2592000,quic=":443"; ma=2592000; v="46,43" + Cache-Control: + - no-cache, no-store, max-age=0, must-revalidate + Content-Type: + - text/html; charset=utf-8 + Cross-Origin-Opener-Policy-Report-Only: + - same-origin-allow-popups; report-to="youtube_main" + Date: + - Sat, 09 Jul 2022 04:44:57 GMT + Expires: + - Mon, 01 Jan 1990 00:00:00 GMT + P3P: + - CP="This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl=en + for more info." + Permissions-Policy: + - ch-ua-arch=*, ch-ua-bitness=*, ch-ua-full-version=*, ch-ua-full-version-list=*, + ch-ua-model=*, ch-ua-platform=*, ch-ua-platform-version=* + Pragma: + - no-cache + Report-To: + - '{"group":"youtube_main","max_age":2592000,"endpoints":[{"url":"https://csp.withgoogle.com/csp/report-to/youtube_main"}]}' + Server: + - ESF + Set-Cookie: + - GPS=1; Domain=.youtube.com; Expires=Sat, 09-Jul-2022 05:14:57 GMT; Path=/; + Secure; HttpOnly + - YSC=MT7W0P9iooc; Domain=.youtube.com; Path=/; Secure; HttpOnly; SameSite=none + - PREF=app=desktop&hl=en; Domain=.youtube.com; Expires=Thu, 09-Mar-2023 16:37:57 + GMT; Path=/; Secure + - __Secure-YEC=; Domain=.youtube.com; Expires=Sun, 13-Oct-2019 04:44:57 GMT; + Path=/; Secure; HttpOnly; SameSite=lax + - VISITOR_INFO1_LIVE=cqG-moexTrs; Domain=.youtube.com; Expires=Thu, 05-Jan-2023 + 04:44:57 GMT; Path=/; Secure; HttpOnly; SameSite=none + Strict-Transport-Security: + - max-age=31536000 + Transfer-Encoding: + - chunked + Vary: + - Accept-Encoding + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-XSS-Protection: + - '0' + status: + code: 200 + message: OK version: 1