From 6c2ac17d549f4cddd464e7a05f8defc6c2962af4 Mon Sep 17 00:00:00 2001 From: DjLegolas Date: Wed, 21 Feb 2024 19:55:44 +0200 Subject: [PATCH] [3595][Core] add ability to download files from daemon When we use thin client mode, we will download tracker's icon from the client endpoint. This means we are leaking the IP of the client location, instead using the daemon, which is already connected to the tracker. Therefor, an ability to download files from the daemon is added. Closes https://dev.deluge-torrent.org/ticket/3595 --- deluge/core/core.py | 102 +++++++++++++++--- deluge/event.py | 5 + .../Blocklist/deluge_blocklist/core.py | 3 +- deluge/tests/test_core.py | 54 ++++++++++ deluge/tests/test_metafile.py | 12 +++ deluge/transfer.py | 39 ++++++- deluge/ui/client.py | 36 ++++++- deluge/ui/gtk3/addtorrentdialog.py | 43 ++++++-- deluge/ui/gtk3/connectionmanager.py | 4 +- deluge/ui/tracker_icons.py | 41 +++++-- 10 files changed, 296 insertions(+), 43 deletions(-) diff --git a/deluge/core/core.py b/deluge/core/core.py index e2130f595a..529218928e 100644 --- a/deluge/core/core.py +++ b/deluge/core/core.py @@ -10,7 +10,9 @@ import glob import logging import os +import random import shutil +import string import tempfile from base64 import b64decode, b64encode from typing import Any, Dict, List, Optional, Tuple, Union @@ -46,6 +48,7 @@ InvalidTorrentError, ) from deluge.event import ( + CallbackHandlingEvent, NewVersionAvailableEvent, SessionPausedEvent, SessionResumedEvent, @@ -508,6 +511,84 @@ async def add_torrents(): return task.deferLater(reactor, 0, add_torrents) + @maybe_coroutine + async def _download_file( + self, + url, + callback=None, + headers=None, + allow_compression=True, + handle_redirects=True, + ) -> bytes: + tmp_fd, tmp_file = tempfile.mkstemp(prefix='deluge_url.') + try: + filename = await download_file( + url=url, + filename=tmp_file, + callback=callback, + headers=headers, + force_filename=True, + allow_compression=allow_compression, + handle_redirects=handle_redirects, + ) + except Exception: + raise + else: + with open(filename, 'rb') as _file: + data = _file.read() + return data + finally: + try: + os.close(tmp_fd) + os.remove(tmp_file) + except OSError as ex: + log.warning(f'Unable to delete temp file {tmp_file}: , {ex}') + + @export + @maybe_coroutine + async def download_file( + self, + url, + callback=None, + headers=None, + allow_compression=True, + handle_redirects=True, + ) -> 'defer.Deferred[Optional[bytes]]': + """Downloads a file from a URL and returns the content as bytes. + + Use this method to download from the daemon itself (like a proxy). + + Args: + url (str): The url to download from. + callback (func, str): A function to be called when partial data is received, + it's signature should be: func(data, current_length, total_length). + headers (dict): Any optional headers to send. + allow_compression (bool): Allows gzip & deflate decoding. + handle_redirects (bool): HTTP redirects handled automatically or not. + + Returns: + a Deferred which returns the content as bytes or None + """ + log.info(f'Attempting to download URL {url}') + + if isinstance(callback, str): + original_callback = callback + + def emit(*args, **kwargs): + component.get('EventManager').emit( + CallbackHandlingEvent(original_callback, *args, **kwargs) + ) + + callback = emit + + try: + return await self._download_file( + url, callback, headers, allow_compression, handle_redirects + ) + except Exception: + log.error(f'Failed to download file from URL {url}') + raise + @export @maybe_coroutine async def add_torrent_url( @@ -524,26 +605,17 @@ async def add_torrent_url( Returns: a Deferred which returns the torrent_id as a str or None """ - log.info('Attempting to add URL %s', url) + log.info(f'Attempting to add URL {url}') - tmp_fd, tmp_file = tempfile.mkstemp(prefix='deluge_url.', suffix='.torrent') try: - filename = await download_file( - url, tmp_file, headers=headers, force_filename=True - ) + data = await self._download_file(url, headers=headers) except Exception: - log.error('Failed to add torrent from URL %s', url) + log.error(f'Failed to add torrent from URL {url}') raise else: - with open(filename, 'rb') as _file: - data = _file.read() - return self.add_torrent_file(filename, b64encode(data), options) - finally: - try: - os.close(tmp_fd) - os.remove(tmp_file) - except OSError as ex: - log.warning(f'Unable to delete temp file {tmp_file}: , {ex}') + chars = string.ascii_letters + string.digits + tmp_file_name = ''.join(random.choices(chars, k=7)) + return self.add_torrent_file(tmp_file_name, b64encode(data), options) @export def add_torrent_magnet(self, uri: str, options: dict) -> str: diff --git a/deluge/event.py b/deluge/event.py index c51fca4de1..1d11e1aeec 100644 --- a/deluge/event.py +++ b/deluge/event.py @@ -318,3 +318,8 @@ def __init__(self, external_ip): external_ip (str): The IP address. """ self._args = [external_ip] + + +class CallbackHandlingEvent(DelugeEvent): + def __init__(self, callback, *args, **kwargs): + self._args = [callback, args, kwargs] diff --git a/deluge/plugins/Blocklist/deluge_blocklist/core.py b/deluge/plugins/Blocklist/deluge_blocklist/core.py index 1765767408..4da5b7ae3d 100644 --- a/deluge/plugins/Blocklist/deluge_blocklist/core.py +++ b/deluge/plugins/Blocklist/deluge_blocklist/core.py @@ -23,7 +23,6 @@ import deluge.configmanager from deluge.common import is_url from deluge.core.rpcserver import export -from deluge.httpdownloader import download_file from deluge.plugins.pluginbase import CorePluginBase from .common import IP, BadIP @@ -326,7 +325,7 @@ def on_retrieve_data(data, current_length, total_length): log.debug('Attempting to download blocklist %s', url) log.debug('Sending headers: %s', headers) self.is_downloading = True - return download_file( + return self.core.download_file( url, deluge.configmanager.get_config_dir('blocklist.download'), on_retrieve_data, diff --git a/deluge/tests/test_core.py b/deluge/tests/test_core.py index 28b590250a..d494174c80 100644 --- a/deluge/tests/test_core.py +++ b/deluge/tests/test_core.py @@ -509,3 +509,57 @@ def test_create_torrent(self, path, tmp_path, piece_length): assert f.read() == filecontent lt.torrent_info(filecontent) + + @pytest.fixture + def _download_file_content(self): + with open( + common.get_test_data_file('ubuntu-9.04-desktop-i386.iso.torrent'), 'rb' + ) as _file: + data = _file.read() + return data + + @pytest_twisted.inlineCallbacks + def test_download_file(self, mock_mkstemp, _download_file_content): + url = ( + f'http://localhost:{self.listen_port}/ubuntu-9.04-desktop-i386.iso.torrent' + ) + + file_content = yield self.core.download_file(url) + assert file_content == _download_file_content + assert not os.path.isfile(mock_mkstemp[1]) + + async def test_download_file_with_cookie(self, _download_file_content): + url = f'http://localhost:{self.listen_port}/cookie' + headers = {'Cookie': 'password=deluge'} + + with pytest.raises(Exception): + await self.core.download_file(url) + + file_content = await self.core.download_file(url, headers=headers) + assert file_content == _download_file_content + + async def test_download_file_with_redirect(self, _download_file_content): + url = f'http://localhost:{self.listen_port}/redirect' + + with pytest.raises(Exception): + await self.core.download_file(url, handle_redirects=False) + + file_content = await self.core.download_file(url) + assert file_content == _download_file_content + + async def test_download_file_with_callback(self, _download_file_content): + url = ( + f'http://localhost:{self.listen_port}/ubuntu-9.04-desktop-i386.iso.torrent' + ) + called_callback = False + data_valid = False + + def on_retrieve_data(data, current_length, total_length): + nonlocal called_callback, data_valid + data_valid |= data in _download_file_content + called_callback = True + + file_content = await self.core.download_file(url, callback=on_retrieve_data) + assert file_content == _download_file_content + assert data_valid + assert called_callback diff --git a/deluge/tests/test_metafile.py b/deluge/tests/test_metafile.py index 1b1675052c..14b50ec602 100644 --- a/deluge/tests/test_metafile.py +++ b/deluge/tests/test_metafile.py @@ -51,6 +51,18 @@ def test_save_multifile(self): os.close(tmp_fd) os.remove(tmp_file) + def test_save_empty_file(self): + with tempfile.TemporaryDirectory() as tmp_dir: + with open(tmp_dir + '/empty', 'wb') as tmp_file: + pass + with open(tmp_dir + '/file', 'wb') as tmp_file: + tmp_file.write(b'c' * (11 * 1024)) + + tmp_torrent = tmp_dir + '/test.torrent' + metafile.make_meta_file(tmp_dir, '', 32768, target=tmp_torrent) + + check_torrent(tmp_torrent) + def test_save_singlefile(self): with tempfile.TemporaryDirectory() as tmp_dir: tmp_data = tmp_dir + '/testdata' diff --git a/deluge/transfer.py b/deluge/transfer.py index ed7d6dd9a0..1fa9bc2644 100644 --- a/deluge/transfer.py +++ b/deluge/transfer.py @@ -122,16 +122,45 @@ def _handle_complete_message(self, data): :param data: a zlib compressed string encoded with rencode. """ + + def log_exception(exception): + log.warning( + 'Failed to decompress (%d bytes) and load serialized data with rencode: %s', + len(data), + exception, + ) + + def build_part(part): + if isinstance(part, bytes): + try: + new_part = part.decode('UTF-8') + except UnicodeDecodeError: + new_part = part + elif isinstance(part, dict): + new_part = {} + for k, v in part.items(): + new_part[build_part(k)] = build_part(v) + elif isinstance(part, list): + new_part = [build_part(i) for i in part] + elif isinstance(part, tuple): + new_part = [build_part(i) for i in part] + new_part = tuple(new_part) + else: + new_part = part + return new_part + try: self.message_received( rencode.loads(zlib.decompress(data), decode_utf8=True) ) + except UnicodeDecodeError: + try: + decoded_data = rencode.loads(zlib.decompress(data)) + self.message_received(build_part(decoded_data)) + except Exception as ex: + log_exception(ex) except Exception as ex: - log.warning( - 'Failed to decompress (%d bytes) and load serialized data with rencode: %s', - len(data), - ex, - ) + log_exception(ex) def get_bytes_recv(self): """ diff --git a/deluge/ui/client.py b/deluge/ui/client.py index 0fef66767a..3e00961801 100644 --- a/deluge/ui/client.py +++ b/deluge/ui/client.py @@ -102,7 +102,7 @@ def message_received(self, request): return if len(request) < 3: log.debug( - 'Received invalid message: number of items in ' 'response is %s', + 'Received invalid message: number of items in response is %s', len(request), ) return @@ -556,6 +556,7 @@ class Client: """ __event_handlers = {} + __callback_handlers = {} def __init__(self): self._daemon_proxy = None @@ -598,6 +599,9 @@ def on_connect_fail(reason): def on_authenticate(result, daemon_info): log.debug('Authentication successful: %s', result) + self.register_event_handler( + 'CallbackHandlingEvent', self._handle_callback_event + ) return result def on_authenticate_fail(reason): @@ -624,6 +628,10 @@ def disconnect(self): """ Disconnects from the daemon. """ + self.deregister_event_handler( + 'CallbackHandlingEvent', self._handle_callback_event + ) + if self.is_standalone(): self._daemon_proxy.disconnect() self.stop_standalone() @@ -790,6 +798,32 @@ def deregister_event_handler(self, event, handler): if self._daemon_proxy: self._daemon_proxy.deregister_event_handler(event, handler) + @staticmethod + def register_callback_handler(callback_id: str, callback: callable): + """ + Registers a callback handler for supported methods on the daemon. + + Args: + callback_id: a unique identifier for the callback + callback: the callback function to call + """ + Client.__callback_handlers[callback_id] = callback + + @staticmethod + def deregister_callback_handler(callback_id: str): + """ + Deregisters a callback handler + + Args: + callback_id: the identifier to remove + """ + if callback_id in Client.__callback_handlers: + Client.__callback_handlers.pop(callback_id) + + def _handle_callback_event(self, callback_id, *args): + if callback_id in self.__callback_handlers: + self.__callback_handlers[callback_id](*(args[0]), **(args[1])) + def force_call(self, block=False): # no-op for now.. we'll see if we need this in the future pass diff --git a/deluge/ui/gtk3/addtorrentdialog.py b/deluge/ui/gtk3/addtorrentdialog.py index 7e752bc990..c14455debb 100644 --- a/deluge/ui/gtk3/addtorrentdialog.py +++ b/deluge/ui/gtk3/addtorrentdialog.py @@ -780,6 +780,8 @@ def on_button_url_clicked(self, widget): ).run() def add_from_url(self, url): + daemon_dl_supported = client.daemon_version_check_min('2.2') + dialog = Gtk.Dialog( _('Downloading...'), flags=Gtk.DialogFlags.MODAL | Gtk.DialogFlags.DESTROY_WITH_PARENT, @@ -791,11 +793,6 @@ def add_from_url(self, url): dialog.vbox.pack_start(pb, True, True, 0) dialog.show_all() - # Create a tmp file path - import tempfile - - tmp_fd, tmp_file = tempfile.mkstemp(prefix='deluge_url.', suffix='.torrent') - def on_part(data, current_length, total_length): if total_length: percent = current_length / total_length @@ -808,10 +805,6 @@ def on_part(data, current_length, total_length): pb.pulse() pb.set_text('%s' % fsize(current_length)) - def on_download_success(result): - self.add_from_files([result]) - dialog.destroy() - def on_download_fail(result): log.debug('Download failed: %s', result) dialog.destroy() @@ -823,8 +816,36 @@ def on_download_fail(result): ).run() return result - d = download_file(url, tmp_file, on_part) - os.close(tmp_fd) + if daemon_dl_supported: + + def on_download_success(result): + # Create a tmp file path + import tempfile + + tmp_fd, tmp_file = tempfile.mkstemp( + prefix='deluge_url.', suffix='.torrent' + ) + with open(tmp_file, 'wb') as _file: + _file.write(result) + os.close(tmp_fd) + self.add_from_files([tmp_file]) + dialog.destroy() + client.deregister_callback_handler(on_part.__qualname__) + + client.register_callback_handler(on_part.__qualname__, on_part) + d = client.core.download_file(url, on_part.__qualname__) + else: + # Create a tmp file path + import tempfile + + tmp_fd, tmp_file = tempfile.mkstemp(prefix='deluge_url.', suffix='.torrent') + + def on_download_success(result): + self.add_from_files([result]) + dialog.destroy() + + d = download_file(url, tmp_file, on_part) + os.close(tmp_fd) d.addCallbacks(on_download_success, on_download_fail) def on_button_hash_clicked(self, widget): diff --git a/deluge/ui/gtk3/connectionmanager.py b/deluge/ui/gtk3/connectionmanager.py index b53dd8e042..73bed29c3a 100644 --- a/deluge/ui/gtk3/connectionmanager.py +++ b/deluge/ui/gtk3/connectionmanager.py @@ -224,9 +224,7 @@ def _update_widget_buttons(self): try: getaddrinfo(host, None) except gaierror as ex: - log.error( - 'Error resolving host %s to ip: %s', row[HOSTLIST_COL_HOST], ex.args[1] - ) + log.error(f'Error resolving host {host} to ip: {ex.args[1]}') self.builder.get_object('button_connect').set_sensitive(False) return diff --git a/deluge/ui/tracker_icons.py b/deluge/ui/tracker_icons.py index d9e080a988..d3a239e2ba 100644 --- a/deluge/ui/tracker_icons.py +++ b/deluge/ui/tracker_icons.py @@ -20,6 +20,7 @@ from deluge.configmanager import get_config_dir from deluge.decorators import proxy from deluge.httpdownloader import download_file +from deluge.ui.client import client try: import chardet @@ -254,11 +255,24 @@ def download_page( Returns: The filename of the tracker host's page """ + daemon_dl_supported = client.daemon_version_check_min('2.2') + if not url: url = self.host_to_url(host) log.debug(f'Downloading {host} {url} to {filename}') - return download_file(url, filename, force_filename=True) + if daemon_dl_supported: + + def on_download(data): + with open(filename, 'w') as icon_file: + icon_file.write(data) + return filename + + d = client.core.download_file(url) + d.addCallback(on_download) + return d + else: + return download_file(url, filename, force_filename=True) def on_download_page_complete(self, page): """ @@ -352,14 +366,29 @@ def download_icon(self, icons, host): :returns: a Deferred which fires with the downloaded icon's filename :rtype: Deferred """ + daemon_dl_supported = client.daemon_version_check_min('2.1') + if len(icons) == 0: raise NoIconsError('empty icons list') (url, mimetype) = icons.pop(0) - d = download_file( - url, - os.path.join(self.dir, host_to_icon_name(host, mimetype)), - force_filename=True, - ) + + icon_download_path = os.path.join(self.dir, host_to_icon_name(host, mimetype)) + + if daemon_dl_supported: + + def on_download(data): + with open(icon_download_path, 'wb') as icon_file: + icon_file.write(data) + return icon_download_path + + d = client.core.download_file(url) + d.addCallback(on_download) + else: + d = download_file( + url, + icon_download_path, + force_filename=True, + ) d.addCallback(self.check_icon_is_valid) if icons: d.addErrback(self.on_download_icon_fail, host, icons)