diff --git a/deluge/core/core.py b/deluge/core/core.py index e2130f595a..529218928e 100644 --- a/deluge/core/core.py +++ b/deluge/core/core.py @@ -10,7 +10,9 @@ import glob import logging import os +import random import shutil +import string import tempfile from base64 import b64decode, b64encode from typing import Any, Dict, List, Optional, Tuple, Union @@ -46,6 +48,7 @@ InvalidTorrentError, ) from deluge.event import ( + CallbackHandlingEvent, NewVersionAvailableEvent, SessionPausedEvent, SessionResumedEvent, @@ -508,6 +511,84 @@ async def add_torrents(): return task.deferLater(reactor, 0, add_torrents) + @maybe_coroutine + async def _download_file( + self, + url, + callback=None, + headers=None, + allow_compression=True, + handle_redirects=True, + ) -> bytes: + tmp_fd, tmp_file = tempfile.mkstemp(prefix='deluge_url.') + try: + filename = await download_file( + url=url, + filename=tmp_file, + callback=callback, + headers=headers, + force_filename=True, + allow_compression=allow_compression, + handle_redirects=handle_redirects, + ) + except Exception: + raise + else: + with open(filename, 'rb') as _file: + data = _file.read() + return data + finally: + try: + os.close(tmp_fd) + os.remove(tmp_file) + except OSError as ex: + log.warning(f'Unable to delete temp file {tmp_file}: , {ex}') + + @export + @maybe_coroutine + async def download_file( + self, + url, + callback=None, + headers=None, + allow_compression=True, + handle_redirects=True, + ) -> 'defer.Deferred[Optional[bytes]]': + """Downloads a file from a URL and returns the content as bytes. + + Use this method to download from the daemon itself (like a proxy). + + Args: + url (str): The url to download from. + callback (func, str): A function to be called when partial data is received, + it's signature should be: func(data, current_length, total_length). + headers (dict): Any optional headers to send. + allow_compression (bool): Allows gzip & deflate decoding. + handle_redirects (bool): HTTP redirects handled automatically or not. + + Returns: + a Deferred which returns the content as bytes or None + """ + log.info(f'Attempting to download URL {url}') + + if isinstance(callback, str): + original_callback = callback + + def emit(*args, **kwargs): + component.get('EventManager').emit( + CallbackHandlingEvent(original_callback, *args, **kwargs) + ) + + callback = emit + + try: + return await self._download_file( + url, callback, headers, allow_compression, handle_redirects + ) + except Exception: + log.error(f'Failed to download file from URL {url}') + raise + @export @maybe_coroutine async def add_torrent_url( @@ -524,26 +605,17 @@ async def add_torrent_url( Returns: a Deferred which returns the torrent_id as a str or None """ - log.info('Attempting to add URL %s', url) + log.info(f'Attempting to add URL {url}') - tmp_fd, tmp_file = tempfile.mkstemp(prefix='deluge_url.', suffix='.torrent') try: - filename = await download_file( - url, tmp_file, headers=headers, force_filename=True - ) + data = await self._download_file(url, headers=headers) except Exception: - log.error('Failed to add torrent from URL %s', url) + log.error(f'Failed to add torrent from URL {url}') raise else: - with open(filename, 'rb') as _file: - data = _file.read() - return self.add_torrent_file(filename, b64encode(data), options) - finally: - try: - os.close(tmp_fd) - os.remove(tmp_file) - except OSError as ex: - log.warning(f'Unable to delete temp file {tmp_file}: , {ex}') + chars = string.ascii_letters + string.digits + tmp_file_name = ''.join(random.choices(chars, k=7)) + return self.add_torrent_file(tmp_file_name, b64encode(data), options) @export def add_torrent_magnet(self, uri: str, options: dict) -> str: diff --git a/deluge/event.py b/deluge/event.py index c51fca4de1..1d11e1aeec 100644 --- a/deluge/event.py +++ b/deluge/event.py @@ -318,3 +318,8 @@ def __init__(self, external_ip): external_ip (str): The IP address. """ self._args = [external_ip] + + +class CallbackHandlingEvent(DelugeEvent): + def __init__(self, callback, *args, **kwargs): + self._args = [callback, args, kwargs] diff --git a/deluge/plugins/Blocklist/deluge_blocklist/core.py b/deluge/plugins/Blocklist/deluge_blocklist/core.py index 1765767408..4da5b7ae3d 100644 --- a/deluge/plugins/Blocklist/deluge_blocklist/core.py +++ b/deluge/plugins/Blocklist/deluge_blocklist/core.py @@ -23,7 +23,6 @@ import deluge.configmanager from deluge.common import is_url from deluge.core.rpcserver import export -from deluge.httpdownloader import download_file from deluge.plugins.pluginbase import CorePluginBase from .common import IP, BadIP @@ -326,7 +325,7 @@ def on_retrieve_data(data, current_length, total_length): log.debug('Attempting to download blocklist %s', url) log.debug('Sending headers: %s', headers) self.is_downloading = True - return download_file( + return self.core.download_file( url, deluge.configmanager.get_config_dir('blocklist.download'), on_retrieve_data, diff --git a/deluge/tests/test_core.py b/deluge/tests/test_core.py index 28b590250a..d494174c80 100644 --- a/deluge/tests/test_core.py +++ b/deluge/tests/test_core.py @@ -509,3 +509,57 @@ def test_create_torrent(self, path, tmp_path, piece_length): assert f.read() == filecontent lt.torrent_info(filecontent) + + @pytest.fixture + def _download_file_content(self): + with open( + common.get_test_data_file('ubuntu-9.04-desktop-i386.iso.torrent'), 'rb' + ) as _file: + data = _file.read() + return data + + @pytest_twisted.inlineCallbacks + def test_download_file(self, mock_mkstemp, _download_file_content): + url = ( + f'http://localhost:{self.listen_port}/ubuntu-9.04-desktop-i386.iso.torrent' + ) + + file_content = yield self.core.download_file(url) + assert file_content == _download_file_content + assert not os.path.isfile(mock_mkstemp[1]) + + async def test_download_file_with_cookie(self, _download_file_content): + url = f'http://localhost:{self.listen_port}/cookie' + headers = {'Cookie': 'password=deluge'} + + with pytest.raises(Exception): + await self.core.download_file(url) + + file_content = await self.core.download_file(url, headers=headers) + assert file_content == _download_file_content + + async def test_download_file_with_redirect(self, _download_file_content): + url = f'http://localhost:{self.listen_port}/redirect' + + with pytest.raises(Exception): + await self.core.download_file(url, handle_redirects=False) + + file_content = await self.core.download_file(url) + assert file_content == _download_file_content + + async def test_download_file_with_callback(self, _download_file_content): + url = ( + f'http://localhost:{self.listen_port}/ubuntu-9.04-desktop-i386.iso.torrent' + ) + called_callback = False + data_valid = False + + def on_retrieve_data(data, current_length, total_length): + nonlocal called_callback, data_valid + data_valid |= data in _download_file_content + called_callback = True + + file_content = await self.core.download_file(url, callback=on_retrieve_data) + assert file_content == _download_file_content + assert data_valid + assert called_callback diff --git a/deluge/tests/test_metafile.py b/deluge/tests/test_metafile.py index 1b1675052c..14b50ec602 100644 --- a/deluge/tests/test_metafile.py +++ b/deluge/tests/test_metafile.py @@ -51,6 +51,18 @@ def test_save_multifile(self): os.close(tmp_fd) os.remove(tmp_file) + def test_save_empty_file(self): + with tempfile.TemporaryDirectory() as tmp_dir: + with open(tmp_dir + '/empty', 'wb') as tmp_file: + pass + with open(tmp_dir + '/file', 'wb') as tmp_file: + tmp_file.write(b'c' * (11 * 1024)) + + tmp_torrent = tmp_dir + '/test.torrent' + metafile.make_meta_file(tmp_dir, '', 32768, target=tmp_torrent) + + check_torrent(tmp_torrent) + def test_save_singlefile(self): with tempfile.TemporaryDirectory() as tmp_dir: tmp_data = tmp_dir + '/testdata' diff --git a/deluge/transfer.py b/deluge/transfer.py index ed7d6dd9a0..1fa9bc2644 100644 --- a/deluge/transfer.py +++ b/deluge/transfer.py @@ -122,16 +122,45 @@ def _handle_complete_message(self, data): :param data: a zlib compressed string encoded with rencode. """ + + def log_exception(exception): + log.warning( + 'Failed to decompress (%d bytes) and load serialized data with rencode: %s', + len(data), + exception, + ) + + def build_part(part): + if isinstance(part, bytes): + try: + new_part = part.decode('UTF-8') + except UnicodeDecodeError: + new_part = part + elif isinstance(part, dict): + new_part = {} + for k, v in part.items(): + new_part[build_part(k)] = build_part(v) + elif isinstance(part, list): + new_part = [build_part(i) for i in part] + elif isinstance(part, tuple): + new_part = [build_part(i) for i in part] + new_part = tuple(new_part) + else: + new_part = part + return new_part + try: self.message_received( rencode.loads(zlib.decompress(data), decode_utf8=True) ) + except UnicodeDecodeError: + try: + decoded_data = rencode.loads(zlib.decompress(data)) + self.message_received(build_part(decoded_data)) + except Exception as ex: + log_exception(ex) except Exception as ex: - log.warning( - 'Failed to decompress (%d bytes) and load serialized data with rencode: %s', - len(data), - ex, - ) + log_exception(ex) def get_bytes_recv(self): """ diff --git a/deluge/ui/client.py b/deluge/ui/client.py index 0fef66767a..3e00961801 100644 --- a/deluge/ui/client.py +++ b/deluge/ui/client.py @@ -102,7 +102,7 @@ def message_received(self, request): return if len(request) < 3: log.debug( - 'Received invalid message: number of items in ' 'response is %s', + 'Received invalid message: number of items in response is %s', len(request), ) return @@ -556,6 +556,7 @@ class Client: """ __event_handlers = {} + __callback_handlers = {} def __init__(self): self._daemon_proxy = None @@ -598,6 +599,9 @@ def on_connect_fail(reason): def on_authenticate(result, daemon_info): log.debug('Authentication successful: %s', result) + self.register_event_handler( + 'CallbackHandlingEvent', self._handle_callback_event + ) return result def on_authenticate_fail(reason): @@ -624,6 +628,10 @@ def disconnect(self): """ Disconnects from the daemon. """ + self.deregister_event_handler( + 'CallbackHandlingEvent', self._handle_callback_event + ) + if self.is_standalone(): self._daemon_proxy.disconnect() self.stop_standalone() @@ -790,6 +798,32 @@ def deregister_event_handler(self, event, handler): if self._daemon_proxy: self._daemon_proxy.deregister_event_handler(event, handler) + @staticmethod + def register_callback_handler(callback_id: str, callback: callable): + """ + Registers a callback handler for supported methods on the daemon. + + Args: + callback_id: a unique identifier for the callback + callback: the callback function to call + """ + Client.__callback_handlers[callback_id] = callback + + @staticmethod + def deregister_callback_handler(callback_id: str): + """ + Deregisters a callback handler + + Args: + callback_id: the identifier to remove + """ + if callback_id in Client.__callback_handlers: + Client.__callback_handlers.pop(callback_id) + + def _handle_callback_event(self, callback_id, *args): + if callback_id in self.__callback_handlers: + self.__callback_handlers[callback_id](*(args[0]), **(args[1])) + def force_call(self, block=False): # no-op for now.. we'll see if we need this in the future pass diff --git a/deluge/ui/gtk3/addtorrentdialog.py b/deluge/ui/gtk3/addtorrentdialog.py index 7e752bc990..c14455debb 100644 --- a/deluge/ui/gtk3/addtorrentdialog.py +++ b/deluge/ui/gtk3/addtorrentdialog.py @@ -780,6 +780,8 @@ def on_button_url_clicked(self, widget): ).run() def add_from_url(self, url): + daemon_dl_supported = client.daemon_version_check_min('2.2') + dialog = Gtk.Dialog( _('Downloading...'), flags=Gtk.DialogFlags.MODAL | Gtk.DialogFlags.DESTROY_WITH_PARENT, @@ -791,11 +793,6 @@ def add_from_url(self, url): dialog.vbox.pack_start(pb, True, True, 0) dialog.show_all() - # Create a tmp file path - import tempfile - - tmp_fd, tmp_file = tempfile.mkstemp(prefix='deluge_url.', suffix='.torrent') - def on_part(data, current_length, total_length): if total_length: percent = current_length / total_length @@ -808,10 +805,6 @@ def on_part(data, current_length, total_length): pb.pulse() pb.set_text('%s' % fsize(current_length)) - def on_download_success(result): - self.add_from_files([result]) - dialog.destroy() - def on_download_fail(result): log.debug('Download failed: %s', result) dialog.destroy() @@ -823,8 +816,36 @@ def on_download_fail(result): ).run() return result - d = download_file(url, tmp_file, on_part) - os.close(tmp_fd) + if daemon_dl_supported: + + def on_download_success(result): + # Create a tmp file path + import tempfile + + tmp_fd, tmp_file = tempfile.mkstemp( + prefix='deluge_url.', suffix='.torrent' + ) + with open(tmp_file, 'wb') as _file: + _file.write(result) + os.close(tmp_fd) + self.add_from_files([tmp_file]) + dialog.destroy() + client.deregister_callback_handler(on_part.__qualname__) + + client.register_callback_handler(on_part.__qualname__, on_part) + d = client.core.download_file(url, on_part.__qualname__) + else: + # Create a tmp file path + import tempfile + + tmp_fd, tmp_file = tempfile.mkstemp(prefix='deluge_url.', suffix='.torrent') + + def on_download_success(result): + self.add_from_files([result]) + dialog.destroy() + + d = download_file(url, tmp_file, on_part) + os.close(tmp_fd) d.addCallbacks(on_download_success, on_download_fail) def on_button_hash_clicked(self, widget): diff --git a/deluge/ui/gtk3/connectionmanager.py b/deluge/ui/gtk3/connectionmanager.py index b53dd8e042..73bed29c3a 100644 --- a/deluge/ui/gtk3/connectionmanager.py +++ b/deluge/ui/gtk3/connectionmanager.py @@ -224,9 +224,7 @@ def _update_widget_buttons(self): try: getaddrinfo(host, None) except gaierror as ex: - log.error( - 'Error resolving host %s to ip: %s', row[HOSTLIST_COL_HOST], ex.args[1] - ) + log.error(f'Error resolving host {host} to ip: {ex.args[1]}') self.builder.get_object('button_connect').set_sensitive(False) return diff --git a/deluge/ui/tracker_icons.py b/deluge/ui/tracker_icons.py index d9e080a988..d3a239e2ba 100644 --- a/deluge/ui/tracker_icons.py +++ b/deluge/ui/tracker_icons.py @@ -20,6 +20,7 @@ from deluge.configmanager import get_config_dir from deluge.decorators import proxy from deluge.httpdownloader import download_file +from deluge.ui.client import client try: import chardet @@ -254,11 +255,24 @@ def download_page( Returns: The filename of the tracker host's page """ + daemon_dl_supported = client.daemon_version_check_min('2.2') + if not url: url = self.host_to_url(host) log.debug(f'Downloading {host} {url} to {filename}') - return download_file(url, filename, force_filename=True) + if daemon_dl_supported: + + def on_download(data): + with open(filename, 'w') as icon_file: + icon_file.write(data) + return filename + + d = client.core.download_file(url) + d.addCallback(on_download) + return d + else: + return download_file(url, filename, force_filename=True) def on_download_page_complete(self, page): """ @@ -352,14 +366,29 @@ def download_icon(self, icons, host): :returns: a Deferred which fires with the downloaded icon's filename :rtype: Deferred """ + daemon_dl_supported = client.daemon_version_check_min('2.1') + if len(icons) == 0: raise NoIconsError('empty icons list') (url, mimetype) = icons.pop(0) - d = download_file( - url, - os.path.join(self.dir, host_to_icon_name(host, mimetype)), - force_filename=True, - ) + + icon_download_path = os.path.join(self.dir, host_to_icon_name(host, mimetype)) + + if daemon_dl_supported: + + def on_download(data): + with open(icon_download_path, 'wb') as icon_file: + icon_file.write(data) + return icon_download_path + + d = client.core.download_file(url) + d.addCallback(on_download) + else: + d = download_file( + url, + icon_download_path, + force_filename=True, + ) d.addCallback(self.check_icon_is_valid) if icons: d.addErrback(self.on_download_icon_fail, host, icons)