From a1d67a1bb998b65c8b21f019b53fb0f502369204 Mon Sep 17 00:00:00 2001 From: drew2a Date: Thu, 1 Feb 2024 17:46:29 +0700 Subject: [PATCH] Add `seeder.py` --- scripts/seedbox/README.md | 56 +++++++++ scripts/seedbox/requirements.txt | 2 + scripts/seedbox/seeder.py | 208 +++++++++++++++++++++++++++++++ 3 files changed, 266 insertions(+) create mode 100644 scripts/seedbox/README.md create mode 100644 scripts/seedbox/requirements.txt create mode 100644 scripts/seedbox/seeder.py diff --git a/scripts/seedbox/README.md b/scripts/seedbox/README.md new file mode 100644 index 00000000000..aba98e17b33 --- /dev/null +++ b/scripts/seedbox/README.md @@ -0,0 +1,56 @@ +# Seedbox + +This folder contains scripts for effortlessly setting up a seedbox. + +## Prerequisites + +Clone the tribler repo: + + ```bash + git clone https://github.com/Tribler/tribler.git + ``` + +Install requirements: + + ```bash + python3 -m pip install -r scripts/seeedbox/requirements.txt + ``` + +## Torrent seeding + +To start torrents' seeding run the following script: + +```bash +python3 seeder.py +``` + +Consider the following folder structure: + +```text +source folder +├ sub_directory +| ├ file1 +| └file2 +├ sub_directory2 +| ├ file3 +| └ file4 +├ thumbnail.png +└ description.md +``` + +In this particular example, `seeder.py` will create two torrents: +`sub_directory.torrent` and `sub_directory2.torrent`. + +`seeder.py` will start to seed them through BitTorrent protocol after creating. + +### Error reporting + +In case you want errors to be reported, you can use [Sentry](https://develop.sentry.dev/) + +To enable error reporting, specify the following environment variable: + +```bash +export SENTRY_URL= +``` + +URL can be taken directly from a corresponding Sentry project. diff --git a/scripts/seedbox/requirements.txt b/scripts/seedbox/requirements.txt new file mode 100644 index 00000000000..71d0a0abd75 --- /dev/null +++ b/scripts/seedbox/requirements.txt @@ -0,0 +1,2 @@ +libtorrent==1.2.19 +sentry-sdk==1.31.0 diff --git a/scripts/seedbox/seeder.py b/scripts/seedbox/seeder.py new file mode 100644 index 00000000000..a703b43256d --- /dev/null +++ b/scripts/seedbox/seeder.py @@ -0,0 +1,208 @@ +""" +This script generates torrents in input folder and seed them. +For available parameters see "parse_args" function below. + +Folder structure: + +# my channel +# ├ sub_directory +# | ├ file1 +# | └ file2 +# ├ sub_directory2 +# | ├ file3 +# | └ file4 +# ├ file5 +# └ file6 + +The script generates torrents for each folder contains files. +There are a possibility to add ignored files (see "_ignore_glob" below). +""" +import argparse +import logging +import os +import time +from collections import defaultdict +from pathlib import Path + +import libtorrent +import sentry_sdk + +UNLIMITED = -1 + +_creator = 'TU Delft' + +_dht_routers = [ + ('router.utorrent.com', 6881), + ("router.utorrent.com", 6881), + ("router.bittorrent.com", 6881), + ("dht.transmissionbt.com", 6881), + ("dht.aelitis.com", 6881), + ("router.bitcomet.com", 6881), +] +_port_range = (6881, 7000) +_log_statistics_interval_in_sec = 10 +_add_torrent_delay_in_sec = 1 +_ignore_glob = [ + '*DS_Store', + '*.torrent', + 'thumbnail.png', + 'description.md', +] + +_logger = logging.getLogger('Seeder') + +sentry_sdk.init( + os.environ.get('SENTRY_URL'), + traces_sample_rate=1.0 +) + + +def parse_args(): + parser = argparse.ArgumentParser(description='Seed data by using the LibTorrent protocol') + + parser.add_argument('-s', '--source', type=str, help='path to data folder', default='.') + parser.add_argument('-v', '--verbosity', help='increase output verbosity', action='store_true') + parser.add_argument('-t', '--testnet', help='Testnet run', action='store_true') + + return parser.parse_args() + + +def setup_logger(verbosity): + logging_level = logging.DEBUG if verbosity else logging.INFO + logging.basicConfig(level=logging_level) + + +def get_folders_with_files(source): + """ Return all folders that contains files + + Args: + source: a source folder + + Returns: + Dictionary where + * key: is a folder + * value: is a file list + """ + result = {} + + for file in Path(source).rglob('*'): + ignore = any(file.match(a) for a in _ignore_glob) + if file.is_file() and not ignore: + result.setdefault(file.parent, set()).add(file) + + return result + + +def create_torrents(folders, source): + _logger.info(f'Creating {len(folders)} torrent files...') + + for folder in folders: + if folder.match(source): + continue + + torrent_file = folder.parent / f'{folder.name}.torrent' + + if not torrent_file.exists(): + original, encoded = create_torrent_from_folder(folder, folders[folder]) + torrent_file.write_bytes(encoded) + _logger.info(f'Created: {torrent_file}') + + yield original, folder + else: + _logger.info(f'Skipped (file already exists): {torrent_file}') + + encoded = torrent_file.read_bytes() + decoded = libtorrent.bdecode(encoded) + + yield decoded, folder + + +def create_torrent_from_folder(folder, files): + file_storage = libtorrent.file_storage() + file_storage.set_name(folder.name) + + for file in files: + relative = file.relative_to(folder.parent) + size = file.stat().st_size + + file_storage.add_file(str(relative), size) + + flags = libtorrent.create_torrent_flags_t.optimize + torrent = libtorrent.create_torrent(file_storage, flags=flags) + + torrent.set_creator(_creator) + libtorrent.set_piece_hashes(torrent, str(folder.parent)) + + torrent_data = torrent.generate() + return torrent_data, libtorrent.bencode(torrent_data) + + +def log_all_alerts(session): + for a in session.pop_alerts(): + if a.category() & libtorrent.alert.category_t.error_notification: + _logger.error(a) + else: + _logger.info(a) + + +def log_statistics(session, handlers, interval): + while True: + time.sleep(interval) + log_all_alerts(session) + + states = defaultdict(int) + errors = defaultdict(int) + + for h in handlers: + status = h.status() + states[status.state] += 1 + if status.errc.value() != 0: + errors[status.errc.message()] += 1 + + _logger.info(f'Torrents states: {states}') + if errors: + _logger.info(f'Torrents errors: {errors}') + + +def seed(torrents): + _logger.info(f'Create torrent session in port range: {_port_range}') + session = libtorrent.session() + session.listen_on(*_port_range) + for router in _dht_routers: + session.add_dht_router(*router) + + session.start_dht() + + session.apply_settings({ + 'active_seeds': UNLIMITED, + 'active_limit': UNLIMITED + }) + + handlers = [] + for torrent, folder in torrents: + torrent_info = libtorrent.torrent_info(torrent) + params = { + 'save_path': str(folder.parent), + 'ti': torrent_info, + 'name': folder.name, + } + + _logger.info(f'Add torrent: {params}') + result = session.add_torrent(params) + handlers.append(result) + + time.sleep(_add_torrent_delay_in_sec) + log_all_alerts(session) + + log_statistics(session, handlers, _log_statistics_interval_in_sec) + + +if __name__ == "__main__": + _arguments = parse_args() + print(f"Arguments: {_arguments}") + + setup_logger(_arguments.verbosity) + _folders = get_folders_with_files(_arguments.source) + _torrents = list(create_torrents(_folders, _arguments.source)) + if not _arguments.testnet: + seed(_torrents)