Merge pull request #9792 from pymedusa/release/release-0.5.16

Release/release 0.5.16

.github/workflows/python-backend.yml

@@ -23,4 +23,4 @@ jobs:
     - name: Test with tox
       run: tox
     - name: Upload coverage to Codecov
-      uses: codecov/codecov-action@v1
+      uses: codecov/codecov-action@v2

CHANGELOG.md

@@ -1,3 +1,27 @@
+## 0.5.16 (13-08-2021)
+
+#### New Features
+- Implemented recommended shows v2. ([5782](https://github.com/pymedusa/Medusa/pull/5782))
+  - Added recommended list from anilist.co
+  - Recommended lists are cached nightly
+  - Configure which lists to cache
+  - Improvements to the recommended list UI
+  - Added plot and genre information when available from the recommended list
+  - Add shows from recommended lists through one click to the show search, or by id (if a tvdbid, tmdbid or tvmazeid is available)
+
+#### Improvements
+- adba lib: Reduced startup time for libraries with many anime shows. ([5782](https://github.com/pymedusa/Medusa/pull/5782))
+  - anime-list.xml was read for each anime show on startup
+
+#### Fixes
+- Fixed postprocessing of archives with multiple video files caused a pp of the complete download dir. ([9775](https://github.com/pymedusa/Medusa/pull/9775))
+- Fixed download handler wrongly untrack downloads when connection errors occurred. ([9774](https://github.com/pymedusa/Medusa/pull/9774))
+- Removed anonomized redirect service (derefer.org is down) in favor of "noreferrer noopener" headers ([5782](https://github.com/pymedusa/Medusa/pull/5782))
+- Fixed schedule page not showing day of week for shows airing on sunday (banner/poster layouts) ([9791](https://github.com/pymedusa/Medusa/pull/9791))
+- Group history compact layout results by quality ([9788](https://github.com/pymedusa/Medusa/pull/9788))
+
+----
+
 ## 0.5.15 (23-07-2021)
 
 #### Improvements

ext/adba/aniDBfileInfo.py

@@ -21,9 +21,10 @@
 import logging
 import os
 import pickle
-import requests
 import sys
+import requests
 import time
+import ttl_cache
 import xml.etree.cElementTree as etree
 
 
@@ -119,6 +120,7 @@ def read_tvdb_map_xml(file_path=None):
     return read_xml_into_etree(file_path)
 
 
+@ttl_cache(180.0)
 def read_xml_into_etree(filePath):
     if not filePath:
         return None

ext/readme.md

@@ -1,7 +1,7 @@
 ## ext
 Folder | Package | Version / Commit | Used By | Notes / Modules
 :----: | :-----: | :--------------: | :------ | :--------------
-ext | **`adba`** | pymedusa/[ad9a011](https://github.com/pymedusa/adba/tree/ad9a011b26e24b276baed5435981fee21a93c254) | **`medusa`** | -
+ext | **`adba`** | pymedusa/[37b0c74](https://github.com/pymedusa/adba/tree/37b0c74e76b40b3dbde29e71da75a1808eb121de) | **`medusa`** | -
 ext | `appdirs` | [1.4.3](https://pypi.org/project/appdirs/1.4.3/) | `simpleanidb`, `subliminal` (cli only) | File: `appdirs.py`
 ext | `attrs` | [18.2.0](https://pypi.org/project/attrs/18.2.0/) | `imdbpie` | Module: `attr`
 ext | **`babelfish`** | [f403000](https://github.com/Diaoul/babelfish/tree/f403000dd63092cfaaae80be9f309fd85c7f20c9) | **`medusa`**, `guessit`, `knowit`, `subliminal` | -
@@ -60,7 +60,7 @@ ext | **`tornado`** | [6.1](https://pypi.org/project/tornado/6.1/) | **`medusa`*
 ext | **`tornroutes`** | [0.5.1](https://pypi.org/project/tornroutes/0.5.1/) | **`medusa`** | -
 ext | **`trakt`** | [3.1.0](https://pypi.org/project/trakt/3.1.0/) | **`medusa`** | -
 ext | `trans` | [2.1.0](https://pypi.org/project/trans/2.1.0/) | `imdbpie` | File: `trans.py`
-ext | `ttl-cache` | [1.6](https://pypi.org/project/ttl-cache/1.6/) | **`medusa`** | File: `ttl_cache.py`
+ext | `ttl-cache` | [1.6](https://pypi.org/project/ttl-cache/1.6/) | **`medusa`**, `adba` | File: `ttl_cache.py`
 ext | **`tvdbapiv2`** | pymedusa/[d6d0e9d](https://github.com/pymedusa/tvdbv2/tree/d6d0e9d98071c2d646beb997b336edbb0e98dfb7) | **`medusa`** | -
 ext | **`urllib3`** | [1.24.1](https://pypi.org/project/urllib3/1.24.1/) | `requests` | -
 ext | **`validators`** | [0.18.2](https://pypi.org/project/validators/0.18.2/) | **`medusa`** | -

medusa/__main__.py

@@ -77,8 +77,9 @@
     check_setting_list, check_setting_str,
     load_provider_setting, save_provider_setting
 )
-from medusa.databases import cache_db, failed_db, main_db
+from medusa.databases import cache_db, failed_db, main_db, recommended_db
 from medusa.failed_history import trim_history
+from medusa.generic_update_queue import GenericQueueScheduler, RecommendedShowUpdateScheduler
 from medusa.indexers.config import INDEXER_TVDBV2, INDEXER_TVMAZE
 from medusa.init.filesystem import is_valid_encoding
 from medusa.providers.generic_provider import GenericProvider
@@ -1031,6 +1032,16 @@ def initialize(self, console_logging=True):
             app.FALLBACK_PLEX_NOTIFICATIONS = check_setting_int(app.CFG, 'General', 'fallback_plex_notifications', 1)
             app.FALLBACK_PLEX_TIMEOUT = check_setting_int(app.CFG, 'General', 'fallback_plex_timeout', 3)
 
+            app.CACHE_RECOMMENDED_SHOWS = check_setting_int(app.CFG, 'Recommended', 'cache_shows', 1)
+            app.CACHE_RECOMMENDED_TRAKT = check_setting_int(app.CFG, 'Recommended', 'cache_trakt', 1)
+            app.CACHE_RECOMMENDED_IMDB = check_setting_int(app.CFG, 'Recommended', 'cache_imdb', 1)
+            app.CACHE_RECOMMENDED_ANIDB = check_setting_int(app.CFG, 'Recommended', 'cache_anidb', 1)
+            app.CACHE_RECOMMENDED_ANILIST = check_setting_int(app.CFG, 'Recommended', 'cache_anilist', 1)
+            app.RECOMMENDED_SHOW_UPDATE_HOUR = max(
+                0, min(23, check_setting_int(app.CFG, 'Recommended', 'recommended_show_update_hour', app.DEFAULT_RECOMMENDED_SHOW_UPDATE_HOUR))
+            )
+            app.CACHE_RECOMMENDED_TRAKT_LISTS = check_setting_list(app.CFG, 'Recommended', 'trakt_lists', app.CACHE_RECOMMENDED_TRAKT_LISTS)
+
             # Initialize trakt config path.
             trakt.core.CONFIG_PATH = os.path.join(app.CACHE_DIR, '.pytrakt.json')
             trakt.core.load_config()
@@ -1179,10 +1190,18 @@ def initialize(self, console_logging=True):
             cache_db_con = db.DBConnection('cache.db')
             db.upgradeDatabase(cache_db_con, cache_db.InitialSchema)
 
+            # initialize the recommended shows database
+            recommended_db_con = db.DBConnection('recommended.db')
+            db.upgradeDatabase(recommended_db_con, recommended_db.InitialSchema)
+
             # Performs a vacuum on cache.db
             logger.debug(u'Performing a vacuum on the CACHE database')
             cache_db_con.action('VACUUM')
 
+            # Performs a vacuum on recommended.db
+            logger.debug(u'Performing a vacuum on the RECOMMENDED database')
+            recommended_db_con.action('VACUUM')
+
             # initialize the failed downloads database
             failed_db_con = db.DBConnection('failed.db')
             db.upgradeDatabase(failed_db_con, failed_db.InitialSchema)
@@ -1224,6 +1243,18 @@ def initialize(self, console_logging=True):
                                                            cycleTime=datetime.timedelta(seconds=3),
                                                            threadName='SHOWQUEUE')
 
+            app.generic_queue_scheduler = scheduler.Scheduler(
+                GenericQueueScheduler(),
+                cycleTime=datetime.timedelta(seconds=3),
+                threadName='GENERICQUEUESCHEDULER'
+            )
+
+            app.recommended_show_update_scheduler = scheduler.Scheduler(
+                RecommendedShowUpdateScheduler(),
+                threadName='RECOMMENDEDSHOWUPDATESCHEDULER',
+                start_time=datetime.time(hour=app.RECOMMENDED_SHOW_UPDATE_HOUR,
+                                         minute=random.randint(0, 59)))
+
             app.show_update_scheduler = scheduler.Scheduler(show_updater.ShowUpdater(),
                                                             threadName='SHOWUPDATER',
                                                             start_time=datetime.time(hour=app.SHOWUPDATE_HOUR,
@@ -1377,6 +1408,14 @@ def start_threads():
             app.backlog_search_scheduler.enable = True
             app.backlog_search_scheduler.start()
 
+            # start the generic queue checker
+            app.generic_queue_scheduler.enable = True
+            app.generic_queue_scheduler.start()
+
+            # start the recommended show update scheduler
+            app.recommended_show_update_scheduler.enable = True
+            app.recommended_show_update_scheduler.start()
+
             # start the show updater
             app.show_update_scheduler.enable = True
             app.show_update_scheduler.start()
@@ -1472,6 +1511,7 @@ def halt():
                 app.show_update_scheduler,
                 app.episode_update_scheduler,
                 app.version_check_scheduler,
+                app.generic_update_scheduler,
                 app.show_queue_scheduler,
                 app.search_queue_scheduler,
                 app.forced_search_queue_scheduler,
@@ -1690,6 +1730,15 @@ def save_config():
         new_config['General']['fallback_plex_notifications'] = app.FALLBACK_PLEX_NOTIFICATIONS
         new_config['General']['fallback_plex_timeout'] = app.FALLBACK_PLEX_TIMEOUT
 
+        new_config['Recommended'] = {}
+        new_config['Recommended']['cache_shows'] = app.CACHE_RECOMMENDED_SHOWS
+        new_config['Recommended']['cache_trakt'] = app.CACHE_RECOMMENDED_TRAKT
+        new_config['Recommended']['cache_imdb'] = app.CACHE_RECOMMENDED_IMDB
+        new_config['Recommended']['cache_anidb'] = app.CACHE_RECOMMENDED_ANIDB
+        new_config['Recommended']['cache_anilist'] = app.CACHE_RECOMMENDED_ANILIST
+        new_config['Recommended']['recommended_show_update_hour'] = int(app.RECOMMENDED_SHOW_UPDATE_HOUR)
+        new_config['Recommended']['trakt_lists'] = app.CACHE_RECOMMENDED_TRAKT_LISTS
+
         new_config['Blackhole'] = {}
         new_config['Blackhole']['nzb_dir'] = app.NZB_DIR
         new_config['Blackhole']['torrent_dir'] = app.TORRENT_DIR
@@ -2257,7 +2306,7 @@ def restore_db(src_dir, dst_dir):
         :return:
         """
         try:
-            files_list = [app.APPLICATION_DB, app.CONFIG_INI, app.FAILED_DB, app.CACHE_DB]
+            files_list = [app.APPLICATION_DB, app.CONFIG_INI, app.FAILED_DB, app.CACHE_DB, app.RECOMMENDED_DB]
 
             for filename in files_list:
                 src_file = os.path.join(src_dir, filename)

medusa/app.py

@@ -43,6 +43,7 @@ def __init__(self):
         self.APPLICATION_DB = 'main.db'
         self.FAILED_DB = 'failed.db'
         self.CACHE_DB = 'cache.db'
+        self.RECOMMENDED_DB = 'recommended.db'
         self.LOG_FILENAME = 'application.log'
         self.CONFIG_INI = 'config.ini'
         self.GIT_ORG = 'pymedusa'
@@ -101,6 +102,9 @@ def __init__(self):
         self.backlog_search_scheduler = None
         self.show_update_scheduler = None
         self.version_check_scheduler = None
+        self.generic_queue_scheduler = None
+        self.recommended_show_update_queue = None
+        self.recommended_show_update_scheduler = None
         self.show_queue_scheduler = None
         self.search_queue_scheduler = None
         self.forced_search_queue_scheduler = None
@@ -281,12 +285,14 @@ def __init__(self):
         self._UPDATE_FREQUENCY = None
         self._BACKLOG_FREQUENCY = None
         self._SHOWUPDATE_HOUR = None
+        self._RECOMMENDED_SHOW_UPDATE_HOUR = None
 
         self.DEFAULT_DOWNLOAD_HANDLER_FREQUENCY = 60
         self.DEFAULT_DAILYSEARCH_FREQUENCY = 40
         self.DEFAULT_BACKLOG_FREQUENCY = 21
         self.DEFAULT_UPDATE_FREQUENCY = 1
         self.DEFAULT_SHOWUPDATE_HOUR = random.randint(2, 4)
+        self.DEFAULT_RECOMMENDED_SHOW_UPDATE_HOUR = random.randint(0, 2)
 
         self.MIN_AUTOPOSTPROCESSOR_FREQUENCY = 1
         self.MIN_DOWNLOAD_HANDLER_FREQUENCY = 5
@@ -706,6 +712,17 @@ def __init__(self):
         self.FALLBACK_PLEX_API_URL = 'https://tvdb2.plex.tv'
         self.TVDB_API_KEY = 'd99c8e7dac2307355af4ab88720a6c32'
 
+        # Recommended Shows settings
+        self.CACHE_RECOMMENDED_SHOWS = True
+        self.CACHE_RECOMMENDED_TRAKT = True
+        self.CACHE_RECOMMENDED_IMDB = True
+        self.CACHE_RECOMMENDED_ANIDB = True
+        self.CACHE_RECOMMENDED_ANILIST = True
+        self.CACHE_RECOMMENDED_TRAKT_LISTS = [
+            'trending', 'popular', 'anticipated', 'collected',
+            'watched', 'played', 'recommendations', 'newshow', 'newseason'
+        ]
+
     def _init_scheduler(self, app_prop=None, scheduler=None, enabled=None):
         from medusa.logger.adapters.style import BraceAdapter
         log = BraceAdapter(logging.getLogger(__name__))
@@ -986,6 +1003,16 @@ def SHOWUPDATE_HOUR(self, value):
         """Change SHOWUPDATE_HOUR."""
         self.handle_prop('SHOWUPDATE_HOUR', value)
 
+    @property
+    def RECOMMENDED_SHOW_UPDATE_HOUR(self):
+        """Return app.SHOWUPDATE_HOUR."""
+        return self._RECOMMENDED_SHOW_UPDATE_HOUR
+
+    @RECOMMENDED_SHOW_UPDATE_HOUR.setter
+    def RECOMMENDED_SHOW_UPDATE_HOUR(self, value):
+        """Change RECOMMENDED_SHOW_UPDATE_HOUR."""
+        self.handle_prop('RECOMMENDED_SHOW_UPDATE_HOUR', value)
+
     @property
     def SUBTITLES_FINDER_FREQUENCY(self):
         """Return app.SUBTITLES_FINDER_FREQUENCY."""

medusa/clients/nzb/nzbget.py

@@ -12,6 +12,7 @@
 from medusa import app
 from medusa.common import Quality
 from medusa.helper.common import try_int
+from medusa.helper.exceptions import DownloadClientConnectionException
 from medusa.logger.adapters.style import BraceAdapter
 
 import ttl_cache
@@ -56,7 +57,7 @@ def nzb_connection(url):
         return False
 
 
-def test_authentication(host, username, password, use_https):
+def test_authentication(host=None, username=None, password=None, use_https=False):
     """
     Test NZBget client connection.
 
@@ -197,10 +198,13 @@ def _get_nzb_queue():
     )
 
     if not nzb_connection(url):
-        return False
+        raise DownloadClientConnectionException('Error while fetching nzbget queue')
 
     nzb_get_rpc = ServerProxy(url)
-    nzb_groups = nzb_get_rpc.listgroups()
+    try:
+        nzb_groups = nzb_get_rpc.listgroups()
+    except ConnectionRefusedError as error:
+        raise DownloadClientConnectionException(f'Error while fetching nzbget history. Error: {error}')
 
     return nzb_groups
 
@@ -216,10 +220,13 @@ def _get_nzb_history():
     )
 
     if not nzb_connection(url):
-        return False
+        raise DownloadClientConnectionException('Error while fetching nzbget history')
 
     nzb_get_rpc = ServerProxy(url)
-    nzb_groups = nzb_get_rpc.history()
+    try:
+        nzb_groups = nzb_get_rpc.history()
+    except ConnectionRefusedError as error:
+        raise DownloadClientConnectionException(f'Error while fetching nzbget history. Error: {error}')
 
     return nzb_groups
 

medusa/clients/nzb/sab.py

@@ -16,17 +16,19 @@
 
 from medusa import app
 from medusa.helper.common import sanitize_filename
+from medusa.helper.exceptions import DownloadClientConnectionException
 from medusa.logger.adapters.style import BraceAdapter
-from medusa.session.core import ClientSession
+from medusa.session.core import MedusaSession
 
 from requests.compat import urljoin
+from requests.exceptions import ConnectionError
 
 import ttl_cache
 
 log = BraceAdapter(logging.getLogger(__name__))
 log.logger.addHandler(logging.NullHandler())
 
-session = ClientSession()
+session = MedusaSession()
 
 
 def send_nzb(nzb):
@@ -209,7 +211,11 @@ def _get_nzb_queue(host=None):
         'limit': 100
     }
 
-    data = session.get_json(url, params=params, verify=False)
+    try:
+        data = session.get_json(url, params=params, verify=False)
+    except ConnectionError as error:
+        raise DownloadClientConnectionException(f'Error while fetching sabnzbd queue. Error: {error}')
+
     if not data:
         log.info('Error connecting to sab, no data returned')
     else:
@@ -234,7 +240,11 @@ def _get_nzb_history(host=None):
         'limit': 100
     }
 
-    data = session.get_json(url, params=params, verify=False)
+    try:
+        data = session.get_json(url, params=params, verify=False)
+    except ConnectionError as error:
+        raise DownloadClientConnectionException(f'Error while fetching sabnzbd history. Error: {error}')
+
     if not data:
         log.info('Error connecting to sab, no data returned')
     else:

medusa/clients/torrent/deluge.py

@@ -9,6 +9,7 @@
 
 from medusa import app
 from medusa.clients.torrent.generic import GenericClient
+from medusa.helper.exceptions import DownloadClientConnectionException
 from medusa.helpers import (
     get_extension,
     is_already_processed_media,
@@ -215,7 +216,6 @@ def _get_auth(self):
                           {'name': self.name})
                 return None
 
-        self._torrent_properties('e4d44da9e71a8f4411bc3fd82aad7689cfa0f07f')
         return self.auth
 
     def _add_torrent_uri(self, result):
@@ -489,9 +489,13 @@ def _torrent_properties(self, info_hash):
         })
 
         log.debug('Checking {client} torrent {hash} status.', {'client': self.name, 'hash': info_hash})
-        if not self._request(method='post', data=post_data) or self.response.json()['error']:
-            log.warning('Error while fetching torrent {hash} status.', {'hash': info_hash})
-            return
+
+        try:
+            if not self._request(method='post', data=post_data) or self.response.json()['error']:
+                log.warning('Error while fetching torrent {hash} status.', {'hash': info_hash})
+                return
+        except RequestException as error:
+            raise DownloadClientConnectionException(f'Error while fetching torrent info_hash {info_hash}. Error: {error}')
 
         return self.response.json()['result']