diff --git a/.gitignore b/.gitignore index e7310d676..e17044bdb 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ cachefile.dbm bazarr.pid /venv /data +/.vscode # Allow !*.dll \ No newline at end of file diff --git a/README.md b/README.md index 9548ff8bf..542e769dc 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,7 @@ If you need something that is not already part of Bazarr, feel free to create a * TVSubtitles * Wizdom * XSubs +* Yavka.net * Zimuku ## Screenshot diff --git a/bazarr.py b/bazarr.py index 581689f76..f9e5d1c3f 100644 --- a/bazarr.py +++ b/bazarr.py @@ -2,29 +2,26 @@ import os import platform -import signal import subprocess import sys import time +import atexit from bazarr.get_args import args -from libs.six import PY3 def check_python_version(): python_version = platform.python_version_tuple() - minimum_py2_tuple = (2, 7, 13) minimum_py3_tuple = (3, 7, 0) - minimum_py2_str = ".".join(str(i) for i in minimum_py2_tuple) minimum_py3_str = ".".join(str(i) for i in minimum_py3_tuple) - if (int(python_version[0]) == minimum_py3_tuple[0] and int(python_version[1]) < minimum_py3_tuple[1]) or \ - (int(python_version[0]) != minimum_py3_tuple[0] and int(python_version[0]) != minimum_py2_tuple[0]): + if int(python_version[0]) < minimum_py3_tuple[0]: print("Python " + minimum_py3_str + " or greater required. " "Current version is " + platform.python_version() + ". Please upgrade Python.") sys.exit(1) - elif int(python_version[0]) == minimum_py2_tuple[0] and int(python_version[1]) < minimum_py2_tuple[1]: - print("Python " + minimum_py2_str + " or greater required. " + elif (int(python_version[0]) == minimum_py3_tuple[0] and int(python_version[1]) < minimum_py3_tuple[1]) or \ + (int(python_version[0]) != minimum_py3_tuple[0]): + print("Python " + minimum_py3_str + " or greater required. " "Current version is " + platform.python_version() + ". Please upgrade Python.") sys.exit(1) @@ -34,156 +31,58 @@ def check_python_version(): dir_name = os.path.dirname(__file__) -class ProcessRegistry: - - def register(self, process): - pass - - def unregister(self, process): - pass - - -class DaemonStatus(ProcessRegistry): - - def __init__(self): - self.__should_stop = False - self.__processes = set() - - def register(self, process): - self.__processes.add(process) - - def unregister(self, process): - self.__processes.remove(process) - - @staticmethod - def __wait_for_processes(processes, timeout): - """ - Waits all the provided processes for the specified amount of time in seconds. - """ - reference_ts = time.time() - elapsed = 0 - remaining_processes = list(processes) - while elapsed < timeout and len(remaining_processes) > 0: - remaining_time = timeout - elapsed - for ep in list(remaining_processes): - if ep.poll() is not None: - remaining_processes.remove(ep) - else: - if remaining_time > 0: - if PY3: - try: - ep.wait(remaining_time) - remaining_processes.remove(ep) - except subprocess.TimeoutExpired: - pass - else: - # In python 2 there is no such thing as some mechanism to wait with a timeout - time.sleep(1) - elapsed = time.time() - reference_ts - remaining_time = timeout - elapsed - return remaining_processes - - @staticmethod - def __send_signal(processes, signal_no, live_processes=None): - """ - Sends to every single of the specified processes the given signal and (if live_processes is not None) append to - it processes which are still alive. - """ - for ep in processes: - if ep.poll() is None: - if live_processes is not None: - live_processes.append(ep) - try: - ep.send_signal(signal_no) - except Exception as e: - print('Failed sending signal %s to process %s because of an unexpected error: %s' % ( - signal_no, ep.pid, e)) - return live_processes - - def stop(self): - """ - Flags this instance as should stop and terminates as smoothly as possible children processes. - """ - self.__should_stop = True - live_processes = DaemonStatus.__send_signal(self.__processes, signal.SIGINT, list()) - live_processes = DaemonStatus.__wait_for_processes(live_processes, 120) - DaemonStatus.__send_signal(live_processes, signal.SIGTERM) - - def should_stop(self): - return self.__should_stop - - -def start_bazarr(process_registry=ProcessRegistry()): +def start_bazarr(): script = [sys.executable, "-u", os.path.normcase(os.path.join(dir_name, 'bazarr', 'main.py'))] + sys.argv[1:] - - print("Bazarr starting...") - if PY3: - ep = subprocess.Popen(script, stdout=None, stderr=None, stdin=subprocess.DEVNULL) - else: - ep = subprocess.Popen(script, stdout=None, stderr=None, stdin=None) - process_registry.register(ep) - try: - ep.wait() - process_registry.unregister(ep) - except KeyboardInterrupt: - pass + ep = subprocess.Popen(script, stdout=None, stderr=None, stdin=subprocess.DEVNULL) + atexit.register(lambda: ep.kill()) + + +def check_status(): + if os.path.exists(stopfile): + try: + os.remove(stopfile) + except Exception: + print('Unable to delete stop file.') + finally: + print('Bazarr exited.') + sys.exit(0) + + if os.path.exists(restartfile): + try: + os.remove(restartfile) + except Exception: + print('Unable to delete restart file.') + else: + print("Bazarr is restarting...") + start_bazarr() if __name__ == '__main__': - restartfile = os.path.normcase(os.path.join(args.config_dir, 'bazarr.restart')) - stopfile = os.path.normcase(os.path.join(args.config_dir, 'bazarr.stop')) + restartfile = os.path.join(args.config_dir, 'bazarr.restart') + stopfile = os.path.join(args.config_dir, 'bazarr.stop') + # Cleanup leftover files try: os.remove(restartfile) - except Exception: + except FileNotFoundError: pass try: os.remove(stopfile) - except Exception: + except FileNotFoundError: pass - - def daemon(bazarr_runner=lambda: start_bazarr()): - if os.path.exists(stopfile): - try: - os.remove(stopfile) - except Exception: - print('Unable to delete stop file.') - else: - print('Bazarr exited.') - sys.exit(0) - - if os.path.exists(restartfile): - try: - os.remove(restartfile) - except Exception: - print('Unable to delete restart file.') - else: - bazarr_runner() - - - bazarr_runner = lambda: start_bazarr() - - should_stop = lambda: False - - if PY3: - daemonStatus = DaemonStatus() - - def shutdown(): - # indicates that everything should stop - daemonStatus.stop() - # emulate a Ctrl C command on itself (bypasses the signal thing but, then, emulates the "Ctrl+C break") - os.kill(os.getpid(), signal.SIGINT) - - signal.signal(signal.SIGTERM, lambda signal_no, frame: shutdown()) - - should_stop = lambda: daemonStatus.should_stop() - bazarr_runner = lambda: start_bazarr(daemonStatus) - - bazarr_runner() + # Initial start of main bazarr process + print("Bazarr starting...") + start_bazarr() # Keep the script running forever until stop is requested through term or keyboard interrupt - while not should_stop(): - daemon(bazarr_runner) - time.sleep(1) + while True: + check_status() + try: + if sys.platform.startswith('win'): + time.sleep(5) + else: + os.wait() + except (KeyboardInterrupt, SystemExit): + pass diff --git a/bazarr/check_update.py b/bazarr/check_update.py index 4938724a2..5d62bb417 100644 --- a/bazarr/check_update.py +++ b/bazarr/check_update.py @@ -34,6 +34,8 @@ def gitconfig(): logging.debug('BAZARR Settings git email') config_write.set_value("user", "email", "bazarr@fake.email") + config_write.release() + def check_and_apply_update(): check_releases() diff --git a/bazarr/config.py b/bazarr/config.py index b64040ebf..b0bb193f6 100644 --- a/bazarr/config.py +++ b/bazarr/config.py @@ -104,7 +104,8 @@ }, 'legendasdivx': { 'username': '', - 'password': '' + 'password': '', + 'skip_wrong_fps': 'False' }, 'legendastv': { 'username': '', @@ -150,6 +151,7 @@ settings = simpleconfigparser(defaults=defaults) settings.read(os.path.join(args.config_dir, 'config', 'config.ini')) +settings.general.base_url = settings.general.base_url if settings.general.base_url else '/' base_url = settings.general.base_url diff --git a/bazarr/get_providers.py b/bazarr/get_providers.py index d345b79ff..b0f892afa 100644 --- a/bazarr/get_providers.py +++ b/bazarr/get_providers.py @@ -8,12 +8,24 @@ from get_args import args from config import settings -from subliminal_patch.exceptions import TooManyRequests, APIThrottled, ParseResponseError +from subliminal_patch.exceptions import TooManyRequests, APIThrottled, ParseResponseError, IPAddressBlocked from subliminal.exceptions import DownloadLimitExceeded, ServiceUnavailable from subliminal import region as subliminal_cache_region +def time_until_end_of_day(dt=None): + # type: (datetime.datetime) -> datetime.timedelta + """ + Get timedelta until end of day on the datetime passed, or current time. + """ + if dt is None: + dt = datetime.datetime.now() + tomorrow = dt + datetime.timedelta(days=1) + return datetime.datetime.combine(tomorrow, datetime.time.min) - dt + +hours_until_end_of_day = time_until_end_of_day().seconds // 3600 + 1 + VALID_THROTTLE_EXCEPTIONS = (TooManyRequests, DownloadLimitExceeded, ServiceUnavailable, APIThrottled, - ParseResponseError) + ParseResponseError, IPAddressBlocked) VALID_COUNT_EXCEPTIONS = ('TooManyRequests', 'ServiceUnavailable', 'APIThrottled') PROVIDER_THROTTLE_MAP = { @@ -32,9 +44,16 @@ "addic7ed": { DownloadLimitExceeded: (datetime.timedelta(hours=3), "3 hours"), TooManyRequests: (datetime.timedelta(minutes=5), "5 minutes"), + IPAddressBlocked: (datetime.timedelta(hours=1), "1 hours"), + }, "titulky": { DownloadLimitExceeded: (datetime.timedelta(hours=3), "3 hours") + }, + "legendasdivx": { + TooManyRequests: (datetime.timedelta(hours=3), "3 hours"), + DownloadLimitExceeded: (datetime.timedelta(hours=hours_until_end_of_day), "{} hours".format(str(hours_until_end_of_day))), + IPAddressBlocked: (datetime.timedelta(hours=hours_until_end_of_day), "{} hours".format(str(hours_until_end_of_day))), } } @@ -116,6 +135,7 @@ def get_providers_auth(): }, 'legendasdivx': {'username': settings.legendasdivx.username, 'password': settings.legendasdivx.password, + 'skip_wrong_fps': settings.legendasdivx.getboolean('skip_wrong_fps'), }, 'legendastv': {'username': settings.legendastv.username, 'password': settings.legendastv.password, diff --git a/bazarr/get_subtitle.py b/bazarr/get_subtitle.py index 232d62858..8baf5c259 100644 --- a/bazarr/get_subtitle.py +++ b/bazarr/get_subtitle.py @@ -207,7 +207,7 @@ def download_subtitle(path, language, audio_language, hi, forced, providers, pro path_decoder=force_unicode ) except Exception as e: - logging.exception('BAZARR Error saving Subtitles file to disk for this file:' + path) + logging.exception('BAZARR Error saving Subtitles file to disk for this file:' + path + ': ' + repr(e)) pass else: saved_any = True @@ -473,11 +473,14 @@ def manual_download_subtitle(path, language, audio_language, hi, forced, subtitl logging.debug('BAZARR Ended manually downloading Subtitles for file: ' + path) -def manual_upload_subtitle(path, language, forced, title, scene_name, media_type, subtitle): +def manual_upload_subtitle(path, language, forced, title, scene_name, media_type, subtitle, audio_language): logging.debug('BAZARR Manually uploading subtitles for this file: ' + path) single = settings.general.getboolean('single_language') + use_postprocessing = settings.general.getboolean('use_postprocessing') + postprocessing_cmd = settings.general.postprocessing_cmd + chmod = int(settings.general.chmod, 8) if not sys.platform.startswith( 'win') and settings.general.getboolean('chmod_enabled') else None @@ -540,6 +543,20 @@ def manual_upload_subtitle(path, language, forced, title, scene_name, media_type os.chmod(subtitle_path, chmod) message = language_from_alpha3(language) + (" forced" if forced else "") + " Subtitles manually uploaded." + + uploaded_language_code3 = language + uploaded_language = language_from_alpha3(uploaded_language_code3) + uploaded_language_code2 = alpha2_from_alpha3(uploaded_language_code3) + audio_language_code2 = alpha2_from_language(audio_language) + audio_language_code3 = alpha3_from_language(audio_language) + + + if use_postprocessing is True: + command = pp_replace(postprocessing_cmd, path, subtitle_path, uploaded_language, + uploaded_language_code2, uploaded_language_code3, audio_language, + audio_language_code2, audio_language_code3, forced) + postprocessing(command, path) + if media_type == 'series': reversed_path = path_replace_reverse(path) @@ -973,7 +990,10 @@ def refine_from_ffprobe(path, video): video.video_codec = data['video'][0]['codec'] if 'frame_rate' in data['video'][0]: if not video.fps: - video.fps = data['video'][0]['frame_rate'] + if isinstance(data['video'][0]['frame_rate'], float): + video.fps = data['video'][0]['frame_rate'] + else: + video.fps = data['video'][0]['frame_rate'].magnitude if 'audio' not in data: logging.debug('BAZARR FFprobe was unable to find audio tracks in the file!') diff --git a/bazarr/list_subtitles.py b/bazarr/list_subtitles.py index c1a8794e5..b19219b6b 100644 --- a/bazarr/list_subtitles.py +++ b/bazarr/list_subtitles.py @@ -40,7 +40,7 @@ def store_subtitles(original_path, reversed_path): subtitle_languages = embedded_subs_reader.list_languages(reversed_path) for subtitle_language, subtitle_forced, subtitle_codec in subtitle_languages: try: - if settings.general.getboolean("ignore_pgs_subs") and subtitle_codec == "hdmv_pgs_subtitle": + if settings.general.getboolean("ignore_pgs_subs") and subtitle_codec == "PGS": logging.debug("BAZARR skipping pgs sub for language: " + str(alpha2_from_alpha3(subtitle_language))) continue @@ -116,7 +116,7 @@ def store_subtitles_movie(original_path, reversed_path): subtitle_languages = embedded_subs_reader.list_languages(reversed_path) for subtitle_language, subtitle_forced, subtitle_codec in subtitle_languages: try: - if settings.general.getboolean("ignore_pgs_subs") and subtitle_codec == "hdmv_pgs_subtitle": + if settings.general.getboolean("ignore_pgs_subs") and subtitle_codec == "PGS": logging.debug("BAZARR skipping pgs sub for language: " + str(alpha2_from_alpha3(subtitle_language))) continue @@ -383,7 +383,7 @@ def guess_external_subtitles(dest_folder, subtitles): logging.debug('BAZARR detected encoding %r', guess) if guess["confidence"] < 0.6: raise UnicodeError - if guess["confidence"] < 0.8 or guess["encoding"] == "ascii": + if guess["encoding"] == "ascii": guess["encoding"] = "utf-8" text = text.decode(guess["encoding"]) detected_language = guess_language(text) diff --git a/bazarr/main.py b/bazarr/main.py index f96b1e810..12e8c213d 100644 --- a/bazarr/main.py +++ b/bazarr/main.py @@ -1,6 +1,6 @@ # coding=utf-8 -bazarr_version = '0.8.4.3' +bazarr_version = '0.8.4.4' import os os.environ["SZ_USER_AGENT"] = "Bazarr/1" @@ -65,7 +65,7 @@ from check_update import check_and_apply_update from subliminal_patch.extensions import provider_registry as provider_manager from subliminal_patch.core import SUBTITLE_EXTENSIONS - +from subliminal.cache import region scheduler = Scheduler() @@ -222,7 +222,7 @@ def doShutdown(): else: stop_file.write(six.text_type('')) stop_file.close() - sys.exit(0) + os._exit(0) @route(base_url + 'restart') @@ -243,7 +243,7 @@ def restart(): logging.info('Bazarr is being restarted...') restart_file.write(six.text_type('')) restart_file.close() - sys.exit(0) + os._exit(0) @route(base_url + 'wizard') @@ -401,12 +401,19 @@ def save_wizard(): else: settings_opensubtitles_skip_wrong_fps = 'True' + settings_legendasdivx_skip_wrong_fps = request.forms.get('settings_legendasdivx_skip_wrong_fps') + if settings_legendasdivx_skip_wrong_fps is None: + settings_legendasdivx_skip_wrong_fps = 'False' + else: + settings_legendasdivx_skip_wrong_fps = 'True' + settings.addic7ed.username = request.forms.get('settings_addic7ed_username') settings.addic7ed.password = request.forms.get('settings_addic7ed_password') settings.addic7ed.random_agents = text_type(settings_addic7ed_random_agents) settings.assrt.token = request.forms.get('settings_assrt_token') settings.legendasdivx.username = request.forms.get('settings_legendasdivx_username') settings.legendasdivx.password = request.forms.get('settings_legendasdivx_password') + settings.legendasdivx.skip_wrong_fps = text_type(settings_legendasdivx_skip_wrong_fps) settings.legendastv.username = request.forms.get('settings_legendastv_username') settings.legendastv.password = request.forms.get('settings_legendastv_password') settings.opensubtitles.username = request.forms.get('settings_opensubtitles_username') @@ -1536,13 +1543,26 @@ def save_settings(): settings_opensubtitles_skip_wrong_fps = 'False' else: settings_opensubtitles_skip_wrong_fps = 'True' - + + if (settings.opensubtitles.username != request.forms.get('settings_opensubtitles_username') or + settings.opensubtitles.password != request.forms.get('settings_opensubtitles_password') or + settings.opensubtitles.vip != text_type(settings_opensubtitles_vip)): + region.delete("os_token") + region.delete("os_server_url") + + settings_legendasdivx_skip_wrong_fps = request.forms.get('settings_legendasdivx_skip_wrong_fps') + if settings_legendasdivx_skip_wrong_fps is None: + settings_legendasdivx_skip_wrong_fps = 'False' + else: + settings_legendasdivx_skip_wrong_fps = 'True' + settings.addic7ed.username = request.forms.get('settings_addic7ed_username') settings.addic7ed.password = request.forms.get('settings_addic7ed_password') settings.addic7ed.random_agents = text_type(settings_addic7ed_random_agents) settings.assrt.token = request.forms.get('settings_assrt_token') settings.legendasdivx.username = request.forms.get('settings_legendasdivx_username') settings.legendasdivx.password = request.forms.get('settings_legendasdivx_password') + settings.legendasdivx.skip_wrong_fps = text_type(settings_legendasdivx_skip_wrong_fps) settings.legendastv.username = request.forms.get('settings_legendastv_username') settings.legendastv.password = request.forms.get('settings_legendastv_password') settings.opensubtitles.username = request.forms.get('settings_opensubtitles_username') @@ -1848,14 +1868,17 @@ def perform_manual_upload_subtitle(): authorize() ref = request.environ['HTTP_REFERER'] - episodePath = request.forms.episodePath - sceneName = request.forms.sceneName + episodePath = request.forms.get('episodePath') + sceneName = request.forms.get('sceneName') language = request.forms.get('language') forced = True if request.forms.get('forced') == '1' else False upload = request.files.get('upload') sonarrSeriesId = request.forms.get('sonarrSeriesId') sonarrEpisodeId = request.forms.get('sonarrEpisodeId') - title = request.forms.title + title = request.forms.get('title') + + data = database.execute("SELECT audio_language FROM table_shows WHERE sonarrSeriesId=?", (sonarrSeriesId,), only_one=True) + audio_language = data['audio_language'] _, ext = os.path.splitext(upload.filename) @@ -1869,7 +1892,8 @@ def perform_manual_upload_subtitle(): title=title, scene_name=sceneName, media_type='series', - subtitle=upload) + subtitle=upload, + audio_language=audio_language) if result is not None: message = result[0] @@ -1988,13 +2012,16 @@ def perform_manual_upload_subtitle_movie(): authorize() ref = request.environ['HTTP_REFERER'] - moviePath = request.forms.moviePath - sceneName = request.forms.sceneName + moviePath = request.forms.get('moviePath') + sceneName = request.forms.get('sceneName') language = request.forms.get('language') forced = True if request.forms.get('forced') == '1' else False upload = request.files.get('upload') radarrId = request.forms.get('radarrId') - title = request.forms.title + title = request.forms.get('title') + + data = database.execute("SELECT audio_language FROM table_movies WHERE radarrId=?", (radarrId,), only_one=True) + audio_language = data['audio_language'] _, ext = os.path.splitext(upload.filename) @@ -2008,7 +2035,8 @@ def perform_manual_upload_subtitle_movie(): title=title, scene_name=sceneName, media_type='movie', - subtitle=upload) + subtitle=upload, + audio_language=audio_language) if result is not None: message = result[0] @@ -2093,10 +2121,11 @@ def api_history(): @custom_auth_basic(check_credentials) def test_url(protocol, url): authorize() - url = six.moves.urllib.parse.unquote(url) + url = protocol + "://" + six.moves.urllib.parse.unquote(url) try: - result = requests.get(protocol + "://" + url, allow_redirects=False, verify=False).json()['version'] - except: + result = requests.get(url, allow_redirects=False, verify=False).json()['version'] + except Exception as e: + logging.exception('BAZARR cannot successfully contact this URL: ' + url) return dict(status=False) else: return dict(status=True, version=result) diff --git a/bazarr/scheduler.py b/bazarr/scheduler.py index 5d272f823..b999316f4 100644 --- a/bazarr/scheduler.py +++ b/bazarr/scheduler.py @@ -18,8 +18,6 @@ from apscheduler.triggers.date import DateTrigger from apscheduler.events import EVENT_JOB_SUBMITTED, EVENT_JOB_EXECUTED, EVENT_JOB_ERROR from datetime import datetime -import pytz -from tzlocal import get_localzone from calendar import day_name import pretty from six import PY2 @@ -30,10 +28,7 @@ class Scheduler: def __init__(self): self.__running_tasks = [] - if str(get_localzone()) == "local": - self.aps_scheduler = BackgroundScheduler(timezone=pytz.timezone('UTC')) - else: - self.aps_scheduler = BackgroundScheduler() + self.aps_scheduler = BackgroundScheduler() # task listener def task_listener_add(event): diff --git a/bin/Linux/i386/ffprobe/ffprobe b/bin/Linux/i386/ffprobe/ffprobe index 1e48486d2..4cc6fabcc 100755 Binary files a/bin/Linux/i386/ffprobe/ffprobe and b/bin/Linux/i386/ffprobe/ffprobe differ diff --git a/bin/Linux/x86_64/ffprobe/ffprobe b/bin/Linux/x86_64/ffprobe/ffprobe index 7dfcd0621..17ed7409d 100755 Binary files a/bin/Linux/x86_64/ffprobe/ffprobe and b/bin/Linux/x86_64/ffprobe/ffprobe differ diff --git a/bin/Windows/i386/ffprobe/ffprobe.exe b/bin/Windows/i386/ffprobe/ffprobe.exe index b34db114b..b764bccfb 100644 Binary files a/bin/Windows/i386/ffprobe/ffprobe.exe and b/bin/Windows/i386/ffprobe/ffprobe.exe differ diff --git a/libs/subliminal/providers/tvsubtitles.py b/libs/subliminal/providers/tvsubtitles.py index 277dbdaa6..188d4448c 100644 --- a/libs/subliminal/providers/tvsubtitles.py +++ b/libs/subliminal/providers/tvsubtitles.py @@ -209,7 +209,7 @@ def list_subtitles(self, video, languages): if subtitles: return subtitles else: - logger.error('No show id found for %r (%r)', video.series, {'year': video.year}) + logger.debug('No show id found for %r (%r)', video.series, {'year': video.year}) return [] diff --git a/libs/subliminal_patch/exceptions.py b/libs/subliminal_patch/exceptions.py index 82f33ade8..6cd73e769 100644 --- a/libs/subliminal_patch/exceptions.py +++ b/libs/subliminal_patch/exceptions.py @@ -7,11 +7,13 @@ class TooManyRequests(ProviderError): """Exception raised by providers when too many requests are made.""" pass - class APIThrottled(ProviderError): pass - class ParseResponseError(ProviderError): """Exception raised by providers when they are not able to parse the response.""" pass + +class IPAddressBlocked(ProviderError): + """Exception raised when providers block requests from IP Address.""" + pass \ No newline at end of file diff --git a/libs/subliminal_patch/providers/addic7ed.py b/libs/subliminal_patch/providers/addic7ed.py index 880a4729c..af6813b4c 100644 --- a/libs/subliminal_patch/providers/addic7ed.py +++ b/libs/subliminal_patch/providers/addic7ed.py @@ -9,13 +9,14 @@ from dogpile.cache.api import NO_VALUE from requests import Session +from requests.exceptions import ConnectionError from subliminal.cache import region from subliminal.exceptions import DownloadLimitExceeded, AuthenticationError, ConfigurationError from subliminal.providers.addic7ed import Addic7edProvider as _Addic7edProvider, \ Addic7edSubtitle as _Addic7edSubtitle, ParserBeautifulSoup from subliminal.subtitle import fix_line_ending from subliminal_patch.utils import sanitize -from subliminal_patch.exceptions import TooManyRequests +from subliminal_patch.exceptions import TooManyRequests, IPAddressBlocked from subliminal_patch.pitcher import pitchers, load_verification, store_verification from subzero.language import Language @@ -91,15 +92,19 @@ def initialize(self): # login if self.username and self.password: def check_verification(cache_region): - rr = self.session.get(self.server_url + 'panel.php', allow_redirects=False, timeout=10, - headers={"Referer": self.server_url}) - if rr.status_code == 302: - logger.info('Addic7ed: Login expired') - cache_region.delete("addic7ed_data") - else: - logger.info('Addic7ed: Re-using old login') - self.logged_in = True - return True + try: + rr = self.session.get(self.server_url + 'panel.php', allow_redirects=False, timeout=10, + headers={"Referer": self.server_url}) + if rr.status_code == 302: + logger.info('Addic7ed: Login expired') + cache_region.delete("addic7ed_data") + else: + logger.info('Addic7ed: Re-using old login') + self.logged_in = True + return True + except ConnectionError as e: + logger.debug("Addic7ed: There was a problem reaching the server: %s." % e) + raise IPAddressBlocked("Addic7ed: Your IP is temporarily blocked.") if load_verification("addic7ed", self.session, callback=check_verification): return diff --git a/libs/subliminal_patch/providers/argenteam.py b/libs/subliminal_patch/providers/argenteam.py index c3707925f..5faf027ba 100644 --- a/libs/subliminal_patch/providers/argenteam.py +++ b/libs/subliminal_patch/providers/argenteam.py @@ -55,7 +55,7 @@ def release_info(self): return self._release_info combine = [] - for attr in ("format", "version", "video_codec"): + for attr in ("format", "version"): value = getattr(self, attr) if value: combine.append(value) @@ -76,9 +76,11 @@ def get_matches(self, video): if video.series and (sanitize(self.title) in ( sanitize(name) for name in [video.series] + video.alternative_series)): matches.add('series') + # season if video.season and self.season == video.season: matches.add('season') + # episode if video.episode and self.episode == video.episode: matches.add('episode') @@ -87,6 +89,9 @@ def get_matches(self, video): if video.tvdb_id and str(self.tvdb_id) == str(video.tvdb_id): matches.add('tvdb_id') + # year (year is not available for series, but we assume it matches) + matches.add('year') + elif isinstance(video, Movie) and self.movie_kind == 'movie': # title if video.title and (sanitize(self.title) in ( @@ -230,29 +235,29 @@ def query(self, title, video, titles=None): has_multiple_ids = len(argenteam_ids) > 1 for aid in argenteam_ids: response = self.session.get(url, params={'id': aid}, timeout=10) - response.raise_for_status() content = response.json() - imdb_id = year = None - returned_title = title - if not is_episode and "info" in content: - imdb_id = content["info"].get("imdb") - year = content["info"].get("year") - returned_title = content["info"].get("title", title) - - for r in content['releases']: - for s in r['subtitles']: - movie_kind = "episode" if is_episode else "movie" - page_link = self.BASE_URL + movie_kind + "/" + str(aid) - # use https and new domain - download_link = s['uri'].replace('http://www.argenteam.net/', self.BASE_URL) - sub = ArgenteamSubtitle(language, page_link, download_link, movie_kind, returned_title, - season, episode, year, r.get('team'), r.get('tags'), - r.get('source'), r.get('codec'), content.get("tvdb"), imdb_id, - asked_for_release_group=video.release_group, - asked_for_episode=episode) - subtitles.append(sub) + if content is not None: # eg https://argenteam.net/api/v1/episode?id=11534 + imdb_id = year = None + returned_title = title + if not is_episode and "info" in content: + imdb_id = content["info"].get("imdb") + year = content["info"].get("year") + returned_title = content["info"].get("title", title) + + for r in content['releases']: + for s in r['subtitles']: + movie_kind = "episode" if is_episode else "movie" + page_link = self.BASE_URL + movie_kind + "/" + str(aid) + # use https and new domain + download_link = s['uri'].replace('http://www.argenteam.net/', self.BASE_URL) + sub = ArgenteamSubtitle(language, page_link, download_link, movie_kind, returned_title, + season, episode, year, r.get('team'), r.get('tags'), + r.get('source'), r.get('codec'), content.get("tvdb"), imdb_id, + asked_for_release_group=video.release_group, + asked_for_episode=episode) + subtitles.append(sub) if has_multiple_ids: time.sleep(self.multi_result_throttle) diff --git a/libs/subliminal_patch/providers/legendasdivx.py b/libs/subliminal_patch/providers/legendasdivx.py index 6247792af..3864d1547 100644 --- a/libs/subliminal_patch/providers/legendasdivx.py +++ b/libs/subliminal_patch/providers/legendasdivx.py @@ -3,19 +3,27 @@ import logging import io import os -import rarfile +import re import zipfile +from time import sleep +from urllib.parse import quote +from requests.exceptions import HTTPError +import rarfile -from requests import Session from guessit import guessit -from subliminal_patch.exceptions import ParseResponseError -from subliminal_patch.providers import Provider +from subliminal.cache import region +from subliminal.exceptions import ConfigurationError, AuthenticationError, ServiceUnavailable, DownloadLimitExceeded from subliminal.providers import ParserBeautifulSoup -from subliminal_patch.subtitle import Subtitle +from subliminal.subtitle import SUBTITLE_EXTENSIONS, fix_line_ending, guess_matches +from subliminal.utils import sanitize, sanitize_release_group from subliminal.video import Episode, Movie -from subliminal.subtitle import SUBTITLE_EXTENSIONS, fix_line_ending,guess_matches +from subliminal_patch.exceptions import TooManyRequests, IPAddressBlocked +from subliminal_patch.http import RetryingCFSession +from subliminal_patch.providers import Provider +from subliminal_patch.score import get_scores, framerate_equal +from subliminal_patch.subtitle import Subtitle from subzero.language import Language -from subliminal_patch.score import get_scores +from dogpile.cache.api import NO_VALUE logger = logging.getLogger(__name__) @@ -23,15 +31,18 @@ class LegendasdivxSubtitle(Subtitle): """Legendasdivx Subtitle.""" provider_name = 'legendasdivx' - def __init__(self, language, video, data): + def __init__(self, language, video, data, skip_wrong_fps=True): super(LegendasdivxSubtitle, self).__init__(language) self.language = language self.page_link = data['link'] - self.hits=data['hits'] - self.exact_match=data['exact_match'] - self.description=data['description'].lower() + self.hits = data['hits'] + self.exact_match = data['exact_match'] + self.description = data['description'] self.video = video - self.videoname =data['videoname'] + self.sub_frame_rate = data['frame_rate'] + self.uploader = data['uploader'] + self.wrong_fps = False + self.skip_wrong_fps = skip_wrong_fps @property def id(self): @@ -44,40 +55,67 @@ def release_info(self): def get_matches(self, video): matches = set() - if self.videoname.lower() in self.description: - matches.update(['title']) - matches.update(['season']) - matches.update(['episode']) + # if skip_wrong_fps = True no point to continue if they don't match + subtitle_fps = None + try: + subtitle_fps = float(self.sub_frame_rate) + except ValueError: + pass + + # check fps match and skip based on configuration + if video.fps and subtitle_fps and not framerate_equal(video.fps, subtitle_fps): + self.wrong_fps = True + + if self.skip_wrong_fps: + logger.debug("Legendasdivx :: Skipping subtitle due to FPS mismatch (expected: %s, got: %s)", video.fps, self.sub_frame_rate) + # not a single match :) + return set() + logger.debug("Legendasdivx :: Frame rate mismatch (expected: %s, got: %s, but continuing...)", video.fps, self.sub_frame_rate) + + description = sanitize(self.description) - # episode - if video.title and video.title.lower() in self.description: + video_filename = video.name + video_filename = os.path.basename(video_filename) + video_filename, _ = os.path.splitext(video_filename) + video_filename = sanitize_release_group(video_filename) + + if sanitize(video_filename) in description: matches.update(['title']) - if video.year and '{:04d}'.format(video.year) in self.description: + # relying people won' use just S01E01 for the file name + if isinstance(video, Episode): + matches.update(['series']) + matches.update(['season']) + matches.update(['episode']) + + # can match both movies and series + if video.year and '{:04d}'.format(video.year) in description: matches.update(['year']) + # match movie title (include alternative movie names) + if isinstance(video, Movie): + if video.title: + for movie_name in [video.title] + video.alternative_titles: + if sanitize(movie_name) in description: + matches.update(['title']) + if isinstance(video, Episode): - # already matched in search query - if video.season and 's{:02d}'.format(video.season) in self.description: + if video.title and sanitize(video.title) in description: + matches.update(['title']) + if video.series: + for series_name in [video.series] + video.alternative_series: + if sanitize(series_name) in description: + matches.update(['series']) + if video.season and 's{:02d}'.format(video.season) in description: matches.update(['season']) - if video.episode and 'e{:02d}'.format(video.episode) in self.description: + if video.episode and 'e{:02d}'.format(video.episode) in description: matches.update(['episode']) - if video.episode and video.season and video.series: - if '{}.s{:02d}e{:02d}'.format(video.series.lower(),video.season,video.episode) in self.description: - matches.update(['series']) - matches.update(['season']) - matches.update(['episode']) - if '{} s{:02d}e{:02d}'.format(video.series.lower(),video.season,video.episode) in self.description: - matches.update(['series']) - matches.update(['season']) - matches.update(['episode']) # release_group - if video.release_group and video.release_group.lower() in self.description: + if video.release_group and sanitize_release_group(video.release_group) in sanitize_release_group(description): matches.update(['release_group']) # resolution - - if video.resolution and video.resolution.lower() in self.description: + if video.resolution and video.resolution.lower() in description: matches.update(['resolution']) # format @@ -87,9 +125,9 @@ def get_matches(self, video): if formats[0] == "web-dl": formats.append("webdl") formats.append("webrip") - formats.append("web ") + formats.append("web") for frmt in formats: - if frmt.lower() in self.description: + if frmt in description: matches.update(['format']) break @@ -97,21 +135,16 @@ def get_matches(self, video): if video.video_codec: video_codecs = [video.video_codec.lower()] if video_codecs[0] == "h264": - formats.append("x264") + video_codecs.append("x264") elif video_codecs[0] == "h265": - formats.append("x265") - for vc in formats: - if vc.lower() in self.description: + video_codecs.append("x265") + for vc in video_codecs: + if vc in description: matches.update(['video_codec']) break - # running guessit on a huge description may break guessit - # matches |= guess_matches(video, guessit(self.description)) return matches - - - class LegendasdivxProvider(Provider): """Legendasdivx Provider.""" languages = {Language('por', 'BR')} | {Language('por')} @@ -121,147 +154,223 @@ class LegendasdivxProvider(Provider): 'User-Agent': os.environ.get("SZ_USER_AGENT", "Sub-Zero/2"), 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', 'Origin': 'https://www.legendasdivx.pt', - 'Referer': 'https://www.legendasdivx.pt', - 'Pragma': 'no-cache', - 'Cache-Control': 'no-cache' + 'Referer': 'https://www.legendasdivx.pt' } loginpage = site + '/forum/ucp.php?mode=login' searchurl = site + '/modules.php?name=Downloads&file=jz&d_op=search&op=_jz00&query={query}' - language_list = list(languages) + download_link = site + '/modules.php{link}' - def __init__(self, username, password): + def __init__(self, username, password, skip_wrong_fps=True): + # make sure login credentials are configured. + if any((username, password)) and not all((username, password)): + raise ConfigurationError('Legendasdivx.pt :: Username and password must be specified') self.username = username self.password = password + self.skip_wrong_fps = skip_wrong_fps def initialize(self): - self.session = Session() - self.login() + logger.debug("Legendasdivx.pt :: Creating session for requests") + self.session = RetryingCFSession() + # re-use PHP Session if present + prev_cookies = region.get("legendasdivx_cookies2") + if prev_cookies != NO_VALUE: + logger.debug("Legendasdivx.pt :: Re-using previous legendasdivx cookies: %s", prev_cookies) + self.session.cookies.update(prev_cookies) + # login if session has expired + else: + logger.debug("Legendasdivx.pt :: Session cookies not found!") + self.session.headers.update(self.headers) + self.login() def terminate(self): - self.logout() + # session close self.session.close() def login(self): - logger.info('Logging in') - self.headers['Referer'] = self.site + '/index.php' - self.session.headers.update(self.headers.items()) - res = self.session.get(self.loginpage) - bsoup = ParserBeautifulSoup(res.content, ['lxml']) - - _allinputs = bsoup.findAll('input') - fields = {} - for field in _allinputs: - fields[field.get('name')] = field.get('value') - - fields['username'] = self.username - fields['password'] = self.password - fields['autologin'] = 'on' - fields['viewonline'] = 'on' - - self.headers['Referer'] = self.loginpage - self.session.headers.update(self.headers.items()) - res = self.session.post(self.loginpage, fields) + logger.debug('Legendasdivx.pt :: Logging in') try: - logger.debug('Got session id %s' % - self.session.cookies.get_dict()['PHPSESSID']) - except KeyError as e: - logger.error(repr(e)) - logger.error("Didn't get session id, check your credentials") - return False + res = self.session.get(self.loginpage) + res.raise_for_status() + bsoup = ParserBeautifulSoup(res.content, ['lxml']) + + _allinputs = bsoup.findAll('input') + data = {} + # necessary to set 'sid' for POST request + for field in _allinputs: + data[field.get('name')] = field.get('value') + + data['username'] = self.username + data['password'] = self.password + + res = self.session.post(self.loginpage, data) + res.raise_for_status() + # make sure we're logged in + logger.debug('Legendasdivx.pt :: Logged in successfully: PHPSESSID: %s', self.session.cookies.get_dict()['PHPSESSID']) + cj = self.session.cookies.copy() + store_cks = ("PHPSESSID", "phpbb3_2z8zs_sid", "phpbb3_2z8zs_k", "phpbb3_2z8zs_u", "lang") + for cn in iter(self.session.cookies.keys()): + if cn not in store_cks: + del cj[cn] + # store session cookies on cache + logger.debug("Legendasdivx.pt :: Storing legendasdivx session cookies: %r", cj) + region.set("legendasdivx_cookies2", cj) + + except KeyError: + logger.error("Legendasdivx.pt :: Couldn't get session ID, check your credentials") + raise AuthenticationError("Legendasdivx.pt :: Couldn't get session ID, check your credentials") + except HTTPError as e: + if "bloqueado" in res.text.lower(): + logger.error("LegendasDivx.pt :: Your IP is blocked on this server.") + raise IPAddressBlocked("LegendasDivx.pt :: Your IP is blocked on this server.") + logger.error("Legendasdivx.pt :: HTTP Error %s", e) + raise TooManyRequests("Legendasdivx.pt :: HTTP Error %s", e) except Exception as e: - logger.error(repr(e)) - logger.error('uncached error #legendasdivx #AA') - return False + logger.error("LegendasDivx.pt :: Uncaught error: %r", e) + raise ServiceUnavailable("LegendasDivx.pt :: Uncaught error: %r", e) - return True + def _process_page(self, video, bsoup): - def logout(self): - # need to figure this out - return True - - def _process_page(self, video, bsoup, querytext, videoname): subtitles = [] + _allsubs = bsoup.findAll("div", {"class": "sub_box"}) - lang = Language.fromopensubtitles("pob") + for _subbox in _allsubs: - hits=0 - for th in _subbox.findAll("th", {"class": "color2"}): - if th.string == 'Hits:': - hits = int(th.parent.find("td").string) - if th.string == 'Idioma:': - lang = th.parent.find("td").find ("img").get ('src') - if 'brazil' in lang: + + hits = 0 + for th in _subbox.findAll("th"): + if th.text == 'Hits:': + hits = int(th.find_next("td").text) + if th.text == 'Idioma:': + lang = th.find_next("td").find("img").get('src') + if 'brazil' in lang.lower(): lang = Language.fromopensubtitles('pob') - else: + elif 'portugal' in lang.lower(): lang = Language.fromopensubtitles('por') + else: + continue + if th.text == "Frame Rate:": + frame_rate = th.find_next("td").text.strip() + + # get description for matches + description = _subbox.find("td", {"class": "td_desc brd_up"}).get_text() - description = _subbox.find("td", {"class": "td_desc brd_up"}) - download = _subbox.find("a", {"class": "sub_download"}) + # get subtitle link from footer + sub_footer = _subbox.find("div", {"class": "sub_footer"}) + download = sub_footer.find("a", {"class": "sub_download"}) if sub_footer else None + + # sometimes 'a' tag is not found and returns None. Most likely HTML format error! try: - # sometimes BSoup just doesn't get the link - logger.debug(download.get('href')) - except Exception as e: - logger.warning('skipping subbox on %s' % self.searchurl.format(query=querytext)) + download_link = self.download_link.format(link=download.get('href')) + logger.debug("Legendasdivx.pt :: Found subtitle link on: %s ", download_link) + except: + logger.debug("Legendasdivx.pt :: Couldn't find download link. Trying next...") continue + # get subtitle uploader + sub_header = _subbox.find("div", {"class" :"sub_header"}) + uploader = sub_header.find("a").text if sub_header else 'anonymous' + exact_match = False - if video.name.lower() in description.get_text().lower(): + if video.name.lower() in description.lower(): exact_match = True - data = {'link': self.site + '/modules.php' + download.get('href'), + + data = {'link': download_link, 'exact_match': exact_match, 'hits': hits, - 'videoname': videoname, - 'description': description.get_text() } + 'uploader': uploader, + 'frame_rate': frame_rate, + 'description': description + } subtitles.append( - LegendasdivxSubtitle(lang, video, data) + LegendasdivxSubtitle(lang, video, data, skip_wrong_fps=self.skip_wrong_fps) ) return subtitles - def query(self, video, language): - try: - logger.debug('Got session id %s' % - self.session.cookies.get_dict()['PHPSESSID']) - except Exception as e: - self.login() + def query(self, video, languages): - language_ids = '0' - if isinstance(language, (tuple, list, set)): - if len(language) == 1: - language_ids = ','.join(sorted(l.opensubtitles for l in language)) - if language_ids == 'por': - language_ids = '&form_cat=28' - else: - language_ids = '&form_cat=29' - - videoname = video.name - videoname = os.path.basename(videoname) - videoname, _ = os.path.splitext(videoname) - # querytext = videoname.lower() _searchurl = self.searchurl - if video.imdb_id is None: - if isinstance(video, Episode): - querytext = "{} S{:02d}E{:02d}".format(video.series, video.season, video.episode) - elif isinstance(video, Movie): - querytext = video.title - else: - querytext = video.imdb_id - - - # querytext = querytext.replace( - # ".", "+").replace("[", "").replace("]", "") - if language_ids != '0': - querytext = querytext + language_ids - self.headers['Referer'] = self.site + '/index.php' - self.session.headers.update(self.headers.items()) - res = self.session.get(_searchurl.format(query=querytext)) - # form_cat=28 = br - # form_cat=29 = pt - if "A legenda não foi encontrada" in res.text: - logger.warning('%s not found', querytext) - return [] + + if isinstance(video, Movie): + querytext = video.imdb_id if video.imdb_id else video.title + + if isinstance(video, Episode): + querytext = '"{} S{:02d}E{:02d}"'.format(video.series, video.season, video.episode) + querytext = quote(quote(querytext)) + + # language query filter + if isinstance(languages, (tuple, list, set)): + language_ids = ','.join(sorted(l.opensubtitles for l in languages)) + if 'por' in language_ids: # prioritize portuguese subtitles + lang_filter = '&form_cat=28' + elif 'pob' in language_ids: + lang_filter = '&form_cat=29' + else: + lang_filter = '' + + querytext = querytext + lang_filter if lang_filter else querytext + + try: + # sleep for a 1 second before another request + sleep(1) + self.headers['Referer'] = self.site + '/index.php' + self.session.headers.update(self.headers) + res = self.session.get(_searchurl.format(query=querytext), allow_redirects=False) + res.raise_for_status() + if (res.status_code == 200 and "A legenda não foi encontrada" in res.text): + logger.warning('Legendasdivx.pt :: query %s return no results!', querytext) + # for series, if no results found, try again just with series and season (subtitle packs) + if isinstance(video, Episode): + logger.debug("Legendasdivx.pt :: trying again with just series and season on query.") + querytext = re.sub("(e|E)(\d{2})", "", querytext) + res = self.session.get(_searchurl.format(query=querytext), allow_redirects=False) + res.raise_for_status() + if (res.status_code == 200 and "A legenda não foi encontrada" in res.text): + logger.warning('Legendasdivx.pt :: query %s return no results (for series and season only).', querytext) + return [] + if res.status_code == 302: # got redirected to login page. + # seems that our session cookies are no longer valid... clean them from cache + region.delete("legendasdivx_cookies2") + logger.debug("Legendasdivx.pt :: Logging in again. Cookies have expired!") + # login and try again + self.login() + res = self.session.get(_searchurl.format(query=querytext)) + res.raise_for_status() + except HTTPError as e: + if "bloqueado" in res.text.lower(): + logger.error("LegendasDivx.pt :: Your IP is blocked on this server.") + raise IPAddressBlocked("LegendasDivx.pt :: Your IP is blocked on this server.") + logger.error("Legendasdivx.pt :: HTTP Error %s", e) + raise TooManyRequests("Legendasdivx.pt :: HTTP Error %s", e) + except Exception as e: + logger.error("LegendasDivx.pt :: Uncaught error: %r", e) + raise ServiceUnavailable("LegendasDivx.pt :: Uncaught error: %r", e) bsoup = ParserBeautifulSoup(res.content, ['html.parser']) - subtitles = self._process_page(video, bsoup, querytext, videoname) + + # search for more than 10 results (legendasdivx uses pagination) + # don't throttle - maximum results = 6 * 10 + MAX_PAGES = 6 + + # get number of pages bases on results found + page_header = bsoup.find("div", {"class": "pager_bar"}) + results_found = re.search(r'\((.*?) encontradas\)', page_header.text).group(1) if page_header else 0 + logger.debug("Legendasdivx.pt :: Found %s subtitles", str(results_found)) + num_pages = (int(results_found) // 10) + 1 + num_pages = min(MAX_PAGES, num_pages) + + # process first page + subtitles = self._process_page(video, bsoup) + + # more pages? + if num_pages > 1: + for num_page in range(2, num_pages+1): + sleep(1) # another 1 sec before requesting... + _search_next = self.searchurl.format(query=querytext) + "&page={0}".format(str(num_page)) + logger.debug("Legendasdivx.pt :: Moving on to next page: %s", _search_next) + res = self.session.get(_search_next) + next_page = ParserBeautifulSoup(res.content, ['html.parser']) + subs = self._process_page(video, next_page) + subtitles.extend(subs) return subtitles @@ -269,34 +378,47 @@ def list_subtitles(self, video, languages): return self.query(video, languages) def download_subtitle(self, subtitle): - res = self.session.get(subtitle.page_link) - if res: - if res.text == '500': - raise ValueError('Error 500 on server') - archive = self._get_archive(res.content) - # extract the subtitle - subtitle_content = self._get_subtitle_from_archive(archive, subtitle) - subtitle.content = fix_line_ending(subtitle_content) - subtitle.normalize() + try: + res = self.session.get(subtitle.page_link) + res.raise_for_status() + except HTTPError as e: + if "bloqueado" in res.text.lower(): + logger.error("LegendasDivx.pt :: Your IP is blocked on this server.") + raise IPAddressBlocked("LegendasDivx.pt :: Your IP is blocked on this server.") + logger.error("Legendasdivx.pt :: HTTP Error %s", e) + raise TooManyRequests("Legendasdivx.pt :: HTTP Error %s", e) + except Exception as e: + logger.error("LegendasDivx.pt :: Uncaught error: %r", e) + raise ServiceUnavailable("LegendasDivx.pt :: Uncaught error: %r", e) - return subtitle - raise ValueError('Problems conecting to the server') + # make sure we haven't maxed out our daily limit + if (res.status_code == 200 and 'limite de downloads diário atingido' in res.text.lower()): + logger.error("LegendasDivx.pt :: Daily download limit reached!") + raise DownloadLimitExceeded("Legendasdivx.pt :: Daily download limit reached!") + + archive = self._get_archive(res.content) + # extract the subtitle + if archive: + subtitle_content = self._get_subtitle_from_archive(archive, subtitle) + if subtitle_content: + subtitle.content = fix_line_ending(subtitle_content) + subtitle.normalize() + return subtitle + return def _get_archive(self, content): # open the archive - # stole^H^H^H^H^H inspired from subvix provider archive_stream = io.BytesIO(content) if rarfile.is_rarfile(archive_stream): - logger.debug('Identified rar archive') + logger.debug('Legendasdivx.pt :: Identified rar archive') archive = rarfile.RarFile(archive_stream) elif zipfile.is_zipfile(archive_stream): - logger.debug('Identified zip archive') + logger.debug('Legendasdivx.pt :: Identified zip archive') archive = zipfile.ZipFile(archive_stream) else: - # raise ParseResponseError('Unsupported compressed format') - raise Exception('Unsupported compressed format') - + logger.error('Legendasdivx.pt :: Unsupported compressed format') + return None return archive def _get_subtitle_from_archive(self, archive, subtitle): @@ -305,7 +427,7 @@ def _get_subtitle_from_archive(self, archive, subtitle): _tmp.remove('.txt') _subtitle_extensions = tuple(_tmp) _max_score = 0 - _scores = get_scores (subtitle.video) + _scores = get_scores(subtitle.video) for name in archive.namelist(): # discard hidden files @@ -316,26 +438,27 @@ def _get_subtitle_from_archive(self, archive, subtitle): if not name.lower().endswith(_subtitle_extensions): continue - _guess = guessit (name) + _guess = guessit(name) if isinstance(subtitle.video, Episode): - logger.debug ("guessing %s" % name) - logger.debug("subtitle S{}E{} video S{}E{}".format(_guess['season'],_guess['episode'],subtitle.video.season,subtitle.video.episode)) + logger.debug("Legendasdivx.pt :: guessing %s", name) + logger.debug("Legendasdivx.pt :: subtitle S%sE%s video S%sE%s", _guess['season'], _guess['episode'], subtitle.video.season, subtitle.video.episode) if subtitle.video.episode != _guess['episode'] or subtitle.video.season != _guess['season']: - logger.debug('subtitle does not match video, skipping') + logger.debug('Legendasdivx.pt :: subtitle does not match video, skipping') continue matches = set() - matches |= guess_matches (subtitle.video, _guess) - logger.debug('srt matches: %s' % matches) - _score = sum ((_scores.get (match, 0) for match in matches)) + matches |= guess_matches(subtitle.video, _guess) + logger.debug('Legendasdivx.pt :: sub matches: %s', matches) + _score = sum((_scores.get(match, 0) for match in matches)) if _score > _max_score: _max_name = name _max_score = _score - logger.debug("new max: {} {}".format(name, _score)) + logger.debug("Legendasdivx.pt :: new max: %s %s", name, _score) if _max_score > 0: - logger.debug("returning from archive: {} scored {}".format(_max_name, _max_score)) + logger.debug("Legendasdivx.pt :: returning from archive: %s scored %s", _max_name, _max_score) return archive.read(_max_name) - raise ParseResponseError('Can not find the subtitle in the compressed file') + logger.error("Legendasdivx.pt :: No subtitle found on compressed file. Max score was 0") + return None diff --git a/libs/subliminal_patch/providers/opensubtitles.py b/libs/subliminal_patch/providers/opensubtitles.py index 012fe6c13..bcda1db85 100644 --- a/libs/subliminal_patch/providers/opensubtitles.py +++ b/libs/subliminal_patch/providers/opensubtitles.py @@ -44,6 +44,12 @@ def __init__(self, language, hearing_impaired, page_link, subtitle_id, matched_b self.wrong_fps = False self.skip_wrong_fps = skip_wrong_fps + def get_fps(self): + try: + return float(self.fps) + except: + return None + def get_matches(self, video, hearing_impaired=False): matches = super(OpenSubtitlesSubtitle, self).get_matches(video) @@ -138,11 +144,9 @@ def get_server_proxy(self, url, timeout=None): return ServerProxy(url, SubZeroRequestsTransport(use_https=self.use_ssl, timeout=timeout or self.timeout, user_agent=os.environ.get("SZ_USER_AGENT", "Sub-Zero/2"))) - def log_in(self, server_url=None): - if server_url: - self.terminate() - - self.server = self.get_server_proxy(server_url) + def log_in_url(self, server_url): + self.token = None + self.server = self.get_server_proxy(server_url) response = self.retry( lambda: checked( @@ -155,6 +159,25 @@ def log_in(self, server_url=None): logger.debug('Logged in with token %r', self.token[:10]+"X"*(len(self.token)-10)) region.set("os_token", bytearray(self.token, encoding='utf-8')) + region.set("os_server_url", bytearray(server_url, encoding='utf-8')) + + def log_in(self): + logger.info('Logging in') + + try: + self.log_in_url(self.vip_url if self.is_vip else self.default_url) + + except Unauthorized: + if self.is_vip: + logger.info("VIP server login failed, falling back") + try: + self.log_in_url(self.default_url) + except Unauthorized: + pass + + if not self.token: + logger.error("Login failed, please check your credentials") + raise Unauthorized def use_token_or_login(self, func): if not self.token: @@ -167,45 +190,18 @@ def use_token_or_login(self, func): return func() def initialize(self): - if self.is_vip: - self.server = self.get_server_proxy(self.vip_url) - logger.info("Using VIP server") - else: - self.server = self.get_server_proxy(self.default_url) - - logger.info('Logging in') - - token = str(region.get("os_token")) - if token is not NO_VALUE: - try: - logger.debug('Trying previous token: %r', token[:10]+"X"*(len(token)-10)) - checked(lambda: self.server.NoOperation(token)) - self.token = token - logger.debug("Using previous login token: %r", token[:10]+"X"*(len(token)-10)) - return - except (NoSession, Unauthorized): - logger.debug('Token not valid.') - pass - - try: - self.log_in() + token_cache = region.get("os_token") + url_cache = region.get("os_server_url") - except Unauthorized: - if self.is_vip: - logger.info("VIP server login failed, falling back") - self.log_in(self.default_url) - if self.token: - return + if token_cache is not NO_VALUE and url_cache is not NO_VALUE: + self.token = token_cache.decode("utf-8") + self.server = self.get_server_proxy(url_cache.decode("utf-8")) + logger.debug("Using previous login token: %r", self.token[:10] + "X" * (len(self.token) - 10)) + else: + self.server = None + self.token = None - logger.error("Login failed, please check your credentials") - def terminate(self): - if self.token: - try: - checked(lambda: self.server.LogOut(self.token)) - except: - logger.error("Logout failed: %s", traceback.format_exc()) - self.server = None self.token = None diff --git a/libs/subliminal_patch/providers/subssabbz.py b/libs/subliminal_patch/providers/subssabbz.py index f2bb05450..66edc1c4f 100644 --- a/libs/subliminal_patch/providers/subssabbz.py +++ b/libs/subliminal_patch/providers/subssabbz.py @@ -13,7 +13,6 @@ from subliminal_patch.providers import Provider from subliminal_patch.subtitle import Subtitle from subliminal_patch.utils import sanitize, fix_inconsistent_naming -from subliminal.exceptions import ProviderError from subliminal.utils import sanitize_release_group from subliminal.subtitle import guess_matches from subliminal.video import Episode, Movie @@ -35,25 +34,31 @@ def fix_tv_naming(title): "Marvel's Luke Cage": "Luke Cage", "Marvel's Iron Fist": "Iron Fist", "Marvel's Jessica Jones": "Jessica Jones", - "DC's Legends of Tomorrow": "Legends of Tomorrow" + "DC's Legends of Tomorrow": "Legends of Tomorrow", + "Doctor Who (2005)": "Doctor Who", }, True) class SubsSabBzSubtitle(Subtitle): """SubsSabBz Subtitle.""" provider_name = 'subssabbz' - def __init__(self, langauge, filename, type, video, link): + def __init__(self, langauge, filename, type, video, link, fps, num_cds): super(SubsSabBzSubtitle, self).__init__(langauge) self.langauge = langauge self.filename = filename self.page_link = link self.type = type self.video = video + self.fps = fps + self.num_cds = num_cds self.release_info = os.path.splitext(filename)[0] @property def id(self): - return self.filename + return self.page_link + self.filename + + def get_fps(self): + return self.fps def make_picklable(self): self.content = None @@ -75,13 +80,21 @@ def get_matches(self, video): if video_filename == subtitle_filename: matches.add('hash') - matches |= guess_matches(video, guessit(self.filename, {'type': self.type})) + if video.year and self.year == video.year: + matches.add('year') + + if isinstance(video, Movie): + if video.imdb_id and self.imdb_id == video.imdb_id: + matches.add('imdb_id') + + matches |= guess_matches(video, guessit(self.title, {'type': self.type, 'allowed_countries': [None]})) + matches |= guess_matches(video, guessit(self.filename, {'type': self.type, 'allowed_countries': [None]})) return matches class SubsSabBzProvider(Provider): """SubsSabBz Provider.""" - languages = {Language('por', 'BR')} | {Language(l) for l in [ + languages = {Language(l) for l in [ 'bul', 'eng' ]} @@ -135,19 +148,51 @@ def query(self, language, video): soup = BeautifulSoup(response.content, 'lxml') rows = soup.findAll('tr', {'class': 'subs-row'}) - # Search on first 20 rows only - for row in rows[:20]: + # Search on first 25 rows only + for row in rows[:25]: a_element_wrapper = row.find('td', { 'class': 'c2field' }) if a_element_wrapper: element = a_element_wrapper.find('a') if element: link = element.get('href') - element = row.find('a', href = re.compile(r'.*showuser=.*')) - uploader = element.get_text() if element else None + notes = element.get('onmouseover') + title = element.get_text() + + try: + year = int(str(element.next_sibling).strip(' ()')) + except: + year = None + + td = row.findAll('td') + + try: + num_cds = int(td[6].get_text()) + except: + num_cds = None + + try: + fps = float(td[7].get_text()) + except: + fps = None + + try: + uploader = td[8].get_text() + except: + uploader = None + + try: + imdb_id = re.findall(r'imdb.com/title/(tt\d+)/?$', td[9].find('a').get('href'))[0] + except: + imdb_id = None + logger.info('Found subtitle link %r', link) - sub = self.download_archive_and_add_subtitle_files(link, language, video) - for s in sub: + sub = self.download_archive_and_add_subtitle_files(link, language, video, fps, num_cds) + for s in sub: + s.title = title + s.notes = notes + s.year = year s.uploader = uploader + s.imdb_id = imdb_id subtitles = subtitles + sub return subtitles @@ -159,23 +204,24 @@ def download_subtitle(self, subtitle): pass else: seeking_subtitle_file = subtitle.filename - arch = self.download_archive_and_add_subtitle_files(subtitle.page_link, subtitle.language, subtitle.video) + arch = self.download_archive_and_add_subtitle_files(subtitle.page_link, subtitle.language, subtitle.video, + subtitle.fps, subtitle.num_cds) for s in arch: if s.filename == seeking_subtitle_file: subtitle.content = s.content - def process_archive_subtitle_files(self, archiveStream, language, video, link): + def process_archive_subtitle_files(self, archiveStream, language, video, link, fps, num_cds): subtitles = [] type = 'episode' if isinstance(video, Episode) else 'movie' - for file_name in archiveStream.namelist(): + for file_name in sorted(archiveStream.namelist()): if file_name.lower().endswith(('.srt', '.sub')): logger.info('Found subtitle file %r', file_name) - subtitle = SubsSabBzSubtitle(language, file_name, type, video, link) - subtitle.content = archiveStream.read(file_name) + subtitle = SubsSabBzSubtitle(language, file_name, type, video, link, fps, num_cds) + subtitle.content = fix_line_ending(archiveStream.read(file_name)) subtitles.append(subtitle) return subtitles - def download_archive_and_add_subtitle_files(self, link, language, video ): + def download_archive_and_add_subtitle_files(self, link, language, video, fps, num_cds): logger.info('Downloading subtitle %r', link) request = self.session.get(link, headers={ 'Referer': 'http://subs.sab.bz/index.php?' @@ -184,8 +230,9 @@ def download_archive_and_add_subtitle_files(self, link, language, video ): archive_stream = io.BytesIO(request.content) if is_rarfile(archive_stream): - return self.process_archive_subtitle_files( RarFile(archive_stream), language, video, link ) + return self.process_archive_subtitle_files(RarFile(archive_stream), language, video, link, fps, num_cds) elif is_zipfile(archive_stream): - return self.process_archive_subtitle_files( ZipFile(archive_stream), language, video, link ) + return self.process_archive_subtitle_files(ZipFile(archive_stream), language, video, link, fps, num_cds) else: - raise ValueError('Not a valid archive') + logger.error('Ignore unsupported archive %r', request.headers) + return [] diff --git a/libs/subliminal_patch/providers/subsunacs.py b/libs/subliminal_patch/providers/subsunacs.py index 87c97c486..b4edc1af1 100644 --- a/libs/subliminal_patch/providers/subsunacs.py +++ b/libs/subliminal_patch/providers/subsunacs.py @@ -13,7 +13,6 @@ from subliminal_patch.providers import Provider from subliminal_patch.subtitle import Subtitle from subliminal_patch.utils import sanitize, fix_inconsistent_naming -from subliminal.exceptions import ProviderError from subliminal.utils import sanitize_release_group from subliminal.subtitle import guess_matches from subliminal.video import Episode, Movie @@ -34,25 +33,31 @@ def fix_tv_naming(title): return fix_inconsistent_naming(title, {"Marvel's Daredevil": "Daredevil", "Marvel's Luke Cage": "Luke Cage", "Marvel's Iron Fist": "Iron Fist", - "DC's Legends of Tomorrow": "Legends of Tomorrow" + "DC's Legends of Tomorrow": "Legends of Tomorrow", + "Doctor Who (2005)": "Doctor Who", }, True) class SubsUnacsSubtitle(Subtitle): """SubsUnacs Subtitle.""" provider_name = 'subsunacs' - def __init__(self, langauge, filename, type, video, link): + def __init__(self, langauge, filename, type, video, link, fps, num_cds): super(SubsUnacsSubtitle, self).__init__(langauge) self.langauge = langauge self.filename = filename self.page_link = link self.type = type self.video = video + self.fps = fps + self.num_cds = num_cds self.release_info = os.path.splitext(filename)[0] @property def id(self): - return self.filename + return self.page_link + self.filename + + def get_fps(self): + return self.fps def make_picklable(self): self.content = None @@ -74,13 +79,17 @@ def get_matches(self, video): if video_filename == subtitle_filename: matches.add('hash') - matches |= guess_matches(video, guessit(self.filename, {'type': self.type})) + if video.year and self.year == video.year: + matches.add('year') + + matches |= guess_matches(video, guessit(self.title, {'type': self.type, 'allowed_countries': [None]})) + matches |= guess_matches(video, guessit(self.filename, {'type': self.type, 'allowed_countries': [None]})) return matches class SubsUnacsProvider(Provider): """SubsUnacs Provider.""" - languages = {Language('por', 'BR')} | {Language(l) for l in [ + languages = {Language(l) for l in [ 'bul', 'eng' ]} @@ -145,11 +154,43 @@ def query(self, language, video): element = a_element_wrapper.find('a', {'class': 'tooltip'}) if element: link = element.get('href') - element = row.find('a', href = re.compile(r'.*/search\.php\?t=1\&(memid|u)=.*')) - uploader = element.get_text() if element else None + notes = element.get('title') + title = element.get_text() + + try: + year = int(element.find_next_sibling('span', {'class' : 'smGray'}).text.strip('\xa0()')) + except: + year = None + + td = row.findAll('td') + + try: + num_cds = int(td[1].get_text()) + except: + num_cds = None + + try: + fps = float(td[2].get_text()) + except: + fps = None + + try: + rating = float(td[3].find('img').get('title')) + except: + rating = None + + try: + uploader = td[5].get_text() + except: + uploader = None + logger.info('Found subtitle link %r', link) - sub = self.download_archive_and_add_subtitle_files('https://subsunacs.net' + link, language, video) - for s in sub: + sub = self.download_archive_and_add_subtitle_files('https://subsunacs.net' + link, language, video, fps, num_cds) + for s in sub: + s.title = title + s.notes = notes + s.year = year + s.rating = rating s.uploader = uploader subtitles = subtitles + sub return subtitles @@ -162,28 +203,29 @@ def download_subtitle(self, subtitle): pass else: seeking_subtitle_file = subtitle.filename - arch = self.download_archive_and_add_subtitle_files(subtitle.page_link, subtitle.language, subtitle.video) + arch = self.download_archive_and_add_subtitle_files(subtitle.page_link, subtitle.language, subtitle.video, + subtitle.fps, subtitle.num_cds) for s in arch: if s.filename == seeking_subtitle_file: subtitle.content = s.content - def process_archive_subtitle_files(self, archiveStream, language, video, link): + def process_archive_subtitle_files(self, archiveStream, language, video, link, fps, num_cds): subtitles = [] type = 'episode' if isinstance(video, Episode) else 'movie' - for file_name in archiveStream.namelist(): + for file_name in sorted(archiveStream.namelist()): if file_name.lower().endswith(('.srt', '.sub', '.txt')): file_is_txt = True if file_name.lower().endswith('.txt') else False if file_is_txt and re.search(r'subsunacs\.net|танете част|прочети|^read ?me|procheti', file_name, re.I): logger.info('Ignore readme txt file %r', file_name) continue logger.info('Found subtitle file %r', file_name) - subtitle = SubsUnacsSubtitle(language, file_name, type, video, link) - subtitle.content = archiveStream.read(file_name) + subtitle = SubsUnacsSubtitle(language, file_name, type, video, link, fps, num_cds) + subtitle.content = fix_line_ending(archiveStream.read(file_name)) if file_is_txt == False or subtitle.is_valid(): subtitles.append(subtitle) return subtitles - def download_archive_and_add_subtitle_files(self, link, language, video ): + def download_archive_and_add_subtitle_files(self, link, language, video, fps, num_cds): logger.info('Downloading subtitle %r', link) request = self.session.get(link, headers={ 'Referer': 'https://subsunacs.net/search.php' @@ -192,8 +234,9 @@ def download_archive_and_add_subtitle_files(self, link, language, video ): archive_stream = io.BytesIO(request.content) if is_rarfile(archive_stream): - return self.process_archive_subtitle_files( RarFile(archive_stream), language, video, link ) + return self.process_archive_subtitle_files(RarFile(archive_stream), language, video, link, fps, num_cds) elif is_zipfile(archive_stream): - return self.process_archive_subtitle_files( ZipFile(archive_stream), language, video, link ) + return self.process_archive_subtitle_files(ZipFile(archive_stream), language, video, link, fps, num_cds) else: - raise ValueError('Not a valid archive') + logger.error('Ignore unsupported archive %r', request.headers) + return [] diff --git a/libs/subliminal_patch/providers/xsubs.py b/libs/subliminal_patch/providers/xsubs.py index 9f8854b92..98160d62c 100644 --- a/libs/subliminal_patch/providers/xsubs.py +++ b/libs/subliminal_patch/providers/xsubs.py @@ -19,6 +19,8 @@ logger = logging.getLogger(__name__) article_re = re.compile(r'^([A-Za-z]{1,3}) (.*)$') episode_re = re.compile(r'^(\d+)(-(\d+))*$') +episode_name_re = re.compile(r'^(.*?)( [\[(].{2,4}[\])])*$') +series_sanitize_re = re.compile(r'^(.*?)( \[\D+\])*$') class XSubsSubtitle(Subtitle): @@ -143,7 +145,11 @@ def _get_show_ids(self): for show_category in soup.findAll('seriesl'): if show_category.attrs['category'] == u'Σειρές': for show in show_category.findAll('series'): - show_ids[sanitize(show.text)] = int(show['srsid']) + series = show.text + series_match = series_sanitize_re.match(series) + if series_match: + series = series_match.group(1) + show_ids[sanitize(series)] = int(show['srsid']) break logger.debug('Found %d show ids', len(show_ids)) @@ -195,6 +201,9 @@ def query(self, show_id, series, season, year=None, country=None): soup = ParserBeautifulSoup(r.content, ['lxml', 'html.parser']) series = soup.find('name').text + series_match = episode_name_re.match(series) + if series_match: + series = series_match.group(1) # loop over season rows seasons = soup.findAll('series_group') diff --git a/libs/subliminal_patch/providers/yavkanet.py b/libs/subliminal_patch/providers/yavkanet.py index d695245ee..42029634e 100644 --- a/libs/subliminal_patch/providers/yavkanet.py +++ b/libs/subliminal_patch/providers/yavkanet.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import import logging -import re import io import os from random import randint @@ -13,7 +12,6 @@ from subliminal_patch.providers import Provider from subliminal_patch.subtitle import Subtitle from subliminal_patch.utils import sanitize -from subliminal.exceptions import ProviderError from subliminal.utils import sanitize_release_group from subliminal.subtitle import guess_matches from subliminal.video import Episode, Movie @@ -27,18 +25,22 @@ class YavkaNetSubtitle(Subtitle): """YavkaNet Subtitle.""" provider_name = 'yavkanet' - def __init__(self, langauge, filename, type, video, link): + def __init__(self, langauge, filename, type, video, link, fps): super(YavkaNetSubtitle, self).__init__(langauge) self.langauge = langauge self.filename = filename self.page_link = link self.type = type self.video = video + self.fps = fps self.release_info = os.path.splitext(filename)[0] @property def id(self): - return self.filename + return self.page_link + self.filename + + def get_fps(self): + return self.fps def make_picklable(self): self.content = None @@ -60,7 +62,11 @@ def get_matches(self, video): if video_filename == subtitle_filename: matches.add('hash') - matches |= guess_matches(video, guessit(self.filename, {'type': self.type})) + if video.year and self.year == video.year: + matches.add('year') + + matches |= guess_matches(video, guessit(self.title, {'type': self.type, 'allowed_countries': [None]})) + matches |= guess_matches(video, guessit(self.filename, {'type': self.type, 'allowed_countries': [None]})) return matches @@ -122,18 +128,34 @@ def query(self, language, video): return subtitles soup = BeautifulSoup(response.content, 'lxml') - rows = soup.findAll('tr', {'class': 'info'}) + rows = soup.findAll('tr') - # Search on first 20 rows only - for row in rows[:20]: + # Search on first 25 rows only + for row in rows[:25]: element = row.find('a', {'class': 'selector'}) if element: link = element.get('href') + notes = element.get('content') + title = element.get_text() + + try: + year = int(element.find_next_sibling('span').text.strip('()')) + except: + year = None + + try: + fps = float(row.find('span', {'title': 'Кадри в секунда'}).text.strip()) + except: + fps = None + element = row.find('a', {'class': 'click'}) uploader = element.get_text() if element else None logger.info('Found subtitle link %r', link) - sub = self.download_archive_and_add_subtitle_files('http://yavka.net/' + link, language, video) - for s in sub: + sub = self.download_archive_and_add_subtitle_files('http://yavka.net/' + link, language, video, fps) + for s in sub: + s.title = title + s.notes = notes + s.year = year s.uploader = uploader subtitles = subtitles + sub return subtitles @@ -146,23 +168,24 @@ def download_subtitle(self, subtitle): pass else: seeking_subtitle_file = subtitle.filename - arch = self.download_archive_and_add_subtitle_files(subtitle.page_link, subtitle.language, subtitle.video) + arch = self.download_archive_and_add_subtitle_files(subtitle.page_link, subtitle.language, subtitle.video, + subtitle.fps) for s in arch: if s.filename == seeking_subtitle_file: subtitle.content = s.content - def process_archive_subtitle_files(self, archiveStream, language, video, link): + def process_archive_subtitle_files(self, archiveStream, language, video, link, fps): subtitles = [] type = 'episode' if isinstance(video, Episode) else 'movie' for file_name in archiveStream.namelist(): if file_name.lower().endswith(('.srt', '.sub')): logger.info('Found subtitle file %r', file_name) - subtitle = YavkaNetSubtitle(language, file_name, type, video, link) - subtitle.content = archiveStream.read(file_name) + subtitle = YavkaNetSubtitle(language, file_name, type, video, link, fps) + subtitle.content = fix_line_ending(archiveStream.read(file_name)) subtitles.append(subtitle) return subtitles - def download_archive_and_add_subtitle_files(self, link, language, video ): + def download_archive_and_add_subtitle_files(self, link, language, video, fps): logger.info('Downloading subtitle %r', link) request = self.session.get(link, headers={ 'Referer': 'http://yavka.net/subtitles.php' @@ -171,9 +194,9 @@ def download_archive_and_add_subtitle_files(self, link, language, video ): archive_stream = io.BytesIO(request.content) if is_rarfile(archive_stream): - return self.process_archive_subtitle_files( RarFile(archive_stream), language, video, link ) + return self.process_archive_subtitle_files(RarFile(archive_stream), language, video, link, fps) elif is_zipfile(archive_stream): - return self.process_archive_subtitle_files( ZipFile(archive_stream), language, video, link ) + return self.process_archive_subtitle_files(ZipFile(archive_stream), language, video, link, fps) else: - raise ValueError('Not a valid archive') - + logger.error('Ignore unsupported archive %r', request.headers) + return [] diff --git a/libs/subliminal_patch/subtitle.py b/libs/subliminal_patch/subtitle.py index ce89e74d3..249fe633f 100644 --- a/libs/subliminal_patch/subtitle.py +++ b/libs/subliminal_patch/subtitle.py @@ -89,6 +89,13 @@ def text(self): def numeric_id(self): raise NotImplemented + def get_fps(self): + """ + :return: frames per second or None if not supported + :rtype: float + """ + return None + def make_picklable(self): """ some subtitle instances might have unpicklable objects stored; clean them up here @@ -264,10 +271,14 @@ def is_valid(self): else: logger.info("Got format: %s", subs.format) except pysubs2.UnknownFPSError: - # if parsing failed, suggest our media file's fps - logger.info("No FPS info in subtitle. Using our own media FPS for the MicroDVD subtitle: %s", - self.plex_media_fps) - subs = pysubs2.SSAFile.from_string(text, fps=self.plex_media_fps) + # if parsing failed, use frame rate from provider + sub_fps = self.get_fps() + if not isinstance(sub_fps, float) or sub_fps < 10.0: + # or use our media file's fps as a fallback + sub_fps = self.plex_media_fps + logger.info("No FPS info in subtitle. Using our own media FPS for the MicroDVD subtitle: %s", + self.plex_media_fps) + subs = pysubs2.SSAFile.from_string(text, fps=sub_fps) unicontent = self.pysubs2_to_unicode(subs) self.content = unicontent.encode(self._guessed_encoding) diff --git a/libs/tzlocal/unix.py b/libs/tzlocal/unix.py index c62a03418..8574965a5 100644 --- a/libs/tzlocal/unix.py +++ b/libs/tzlocal/unix.py @@ -84,11 +84,10 @@ def _get_localzone(_root='/'): if not etctz: continue tz = pytz.timezone(etctz.replace(' ', '_')) - # Disabling this offset valdation due to issue with some timezone: https://github.com/regebro/tzlocal/issues/80 - # if _root == '/': + if _root == '/': # We are using a file in etc to name the timezone. # Verify that the timezone specified there is actually used: - # utils.assert_tz_offset(tz) + utils.assert_tz_offset(tz) return tz except IOError: diff --git a/libs/tzlocal/utils.py b/libs/tzlocal/utils.py index bd9d663e8..5a6779903 100644 --- a/libs/tzlocal/utils.py +++ b/libs/tzlocal/utils.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- +import time import datetime +import calendar def get_system_offset(): @@ -11,8 +13,14 @@ def get_system_offset(): To keep compatibility with Windows, we're always importing time module here. """ - import time - if time.daylight and time.localtime().tm_isdst > 0: + + localtime = calendar.timegm(time.localtime()) + gmtime = calendar.timegm(time.gmtime()) + offset = gmtime - localtime + # We could get the localtime and gmtime on either side of a second switch + # so we check that the difference is less than one minute, because nobody + # has that small DST differences. + if abs(offset - time.altzone) < 60: return -time.altzone else: return -time.timezone diff --git a/libs/tzlocal/windows_tz.py b/libs/tzlocal/windows_tz.py index 3d691c85a..86ba807d0 100644 --- a/libs/tzlocal/windows_tz.py +++ b/libs/tzlocal/windows_tz.py @@ -87,6 +87,7 @@ 'Pacific Standard Time (Mexico)': 'America/Tijuana', 'Pakistan Standard Time': 'Asia/Karachi', 'Paraguay Standard Time': 'America/Asuncion', + 'Qyzylorda Standard Time': 'Asia/Qyzylorda', 'Romance Standard Time': 'Europe/Paris', 'Russia Time Zone 10': 'Asia/Srednekolymsk', 'Russia Time Zone 11': 'Asia/Kamchatka', @@ -127,6 +128,7 @@ 'Ulaanbaatar Standard Time': 'Asia/Ulaanbaatar', 'Venezuela Standard Time': 'America/Caracas', 'Vladivostok Standard Time': 'Asia/Vladivostok', + 'Volgograd Standard Time': 'Europe/Volgograd', 'W. Australia Standard Time': 'Australia/Perth', 'W. Central Africa Standard Time': 'Africa/Lagos', 'W. Europe Standard Time': 'Europe/Berlin', @@ -287,7 +289,7 @@ 'America/Mendoza': 'Argentina Standard Time', 'America/Menominee': 'Central Standard Time', 'America/Merida': 'Central Standard Time (Mexico)', - 'America/Metlakatla': 'Pacific Standard Time', + 'America/Metlakatla': 'Alaskan Standard Time', 'America/Mexico_City': 'Central Standard Time (Mexico)', 'America/Miquelon': 'Saint Pierre Standard Time', 'America/Moncton': 'Atlantic Standard Time', @@ -347,13 +349,13 @@ 'America/Winnipeg': 'Central Standard Time', 'America/Yakutat': 'Alaskan Standard Time', 'America/Yellowknife': 'Mountain Standard Time', - 'Antarctica/Casey': 'W. Australia Standard Time', + 'Antarctica/Casey': 'Singapore Standard Time', 'Antarctica/Davis': 'SE Asia Standard Time', 'Antarctica/DumontDUrville': 'West Pacific Standard Time', 'Antarctica/Macquarie': 'Central Pacific Standard Time', 'Antarctica/Mawson': 'West Asia Standard Time', 'Antarctica/McMurdo': 'New Zealand Standard Time', - 'Antarctica/Palmer': 'Magallanes Standard Time', + 'Antarctica/Palmer': 'SA Eastern Standard Time', 'Antarctica/Rothera': 'SA Eastern Standard Time', 'Antarctica/South_Pole': 'New Zealand Standard Time', 'Antarctica/Syowa': 'E. Africa Standard Time', @@ -424,7 +426,7 @@ 'Asia/Pyongyang': 'North Korea Standard Time', 'Asia/Qatar': 'Arab Standard Time', 'Asia/Qostanay': 'Central Asia Standard Time', - 'Asia/Qyzylorda': 'West Asia Standard Time', + 'Asia/Qyzylorda': 'Qyzylorda Standard Time', 'Asia/Rangoon': 'Myanmar Standard Time', 'Asia/Riyadh': 'Arab Standard Time', 'Asia/Saigon': 'SE Asia Standard Time', @@ -592,7 +594,7 @@ 'Europe/Vatican': 'W. Europe Standard Time', 'Europe/Vienna': 'W. Europe Standard Time', 'Europe/Vilnius': 'FLE Standard Time', - 'Europe/Volgograd': 'Russian Standard Time', + 'Europe/Volgograd': 'Volgograd Standard Time', 'Europe/Warsaw': 'Central European Standard Time', 'Europe/Zagreb': 'Central European Standard Time', 'Europe/Zaporozhye': 'FLE Standard Time', diff --git a/libs/version.txt b/libs/version.txt index 5e35b7131..c4da8022b 100644 --- a/libs/version.txt +++ b/libs/version.txt @@ -1,4 +1,4 @@ -apprise=0.8.2 +apprise=0.8.5 apscheduler=3.5.1 babelfish=0.5.5 backports.functools-lru-cache=1.5 @@ -13,7 +13,6 @@ gitpython=2.1.9 guessit=2.1.4 guess_language-spirit=0.5.3 knowit=0.3.0-dev -peewee=3.9.6 py-pretty=1 pycountry=18.2.23 pyga=2.6.1 @@ -25,6 +24,6 @@ six=1.11.0 SimpleConfigParser=0.1.0 <-- modified version: do not update!!! stevedore=1.28.0 subliminal=2.1.0dev -tzlocal=1.5.1 +tzlocal=2.1b1 urllib3=1.23 Js2Py=0.63 <-- modified: manually merged from upstream: https://github.com/PiotrDabkowski/Js2Py/pull/192/files diff --git a/views/providers.tpl b/views/providers.tpl index 0bf3f019e..3d93e5e9c 100644 --- a/views/providers.tpl +++ b/views/providers.tpl @@ -265,6 +265,24 @@ +
+
+ +
+
+
+ + +
+
+ +
@@ -500,8 +518,9 @@
- +
+
@@ -733,7 +752,7 @@
-
+
@@ -755,6 +774,28 @@
+
+
+ +
+
+
+ + +
+
+ +
+
+ +
+
@@ -811,7 +852,7 @@
-
+
@@ -922,6 +963,12 @@ $("#settings_opensubtitles_skip_wrong_fps").checkbox('uncheck'); } + if ($('#settings_legendasdivx_skip_wrong_fps').data("ldfps") === "True") { + $("#settings_legendasdivx_skip_wrong_fps").checkbox('check'); + } else { + $("#settings_legendasdivx_skip_wrong_fps").checkbox('uncheck'); + } + $('#settings_providers').dropdown('clear'); $('#settings_providers').dropdown('set selected',{{!enabled_providers}}); $('#settings_providers').dropdown(); @@ -949,4 +996,4 @@ $('#'+$(this).parent().attr('id')+'_option').hide(); } }); - + \ No newline at end of file diff --git a/views/system.tpl b/views/system.tpl index e3a04ca81..dd73ba382 100644 --- a/views/system.tpl +++ b/views/system.tpl @@ -403,15 +403,12 @@ }); $('#shutdown').on('click', function(){ - $.ajax({ - url: "{{base_url}}shutdown", - async: false + document.open(); + document.write('Bazarr has shutdown.'); + document.close(); + $.ajax({ + url: "{{base_url}}shutdown" }) - .always(function(){ - document.open(); - document.write('Bazarr has shutdown.'); - document.close(); - }); }); $('#logout').on('click', function(){ @@ -422,11 +419,9 @@ $('#loader_text').text("Bazarr is restarting, please wait..."); $.ajax({ url: "{{base_url}}restart", - async: true, - error: (function () { - setTimeout(function () { setInterval(ping, 2000); }, 8000); - }) + async: true }); + setTimeout(function () { setInterval(ping, 2000); }, 8000); }); % from config import settings