From d92e9da014a3a5dcc7aee2de174711c410ebf415 Mon Sep 17 00:00:00 2001 From: Mengxuan Xia Date: Fri, 5 Apr 2013 23:19:28 -0400 Subject: [PATCH 1/4] Added support for yaaw/aria2-jsonrpc --- lixian_download_tools.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/lixian_download_tools.py b/lixian_download_tools.py index 4d1185b..b966228 100644 --- a/lixian_download_tools.py +++ b/lixian_download_tools.py @@ -5,6 +5,8 @@ import subprocess import urllib2 import os.path +import json +import time download_tools = {} @@ -112,6 +114,28 @@ def __call__(self): if exit_code != 0: raise Exception('aria2c exited abnormally') +@download_tool('aria2rpc') +def aria2rpc_download(client, download_url, path, resuming=False): + gdriveid = str(client.get_gdriveid()) + dir = os.path.dirname(path) + filename = os.path.basename(path) + aria2rpchost = get_config('aria2-rpc-host', 'localhost') + aria2rpcport = get_config('aria2-rpc-port', '6800') + ts = str(int(time.time()*1000)) + data = {"jsonrpc": "2.0", + "method":"aria2.addUri", + "id": ts, + "params": [[download_url], + {"out": filename, + "header": "Cookie: gdriveid="+gdriveid} + + ] + } + data = json.dumps(data) + request_url = "http://"+aria2rpchost+":"+aria2rpcport+"/jsonrpc?tm="+ts + urllib2.urlopen(request_url, data) + + @download_tool('axel') def axel_download(client, download_url, path, resuming=False): gdriveid = str(client.get_gdriveid()) From d82cbc4412d426dfa46f130bb351b753fccd1938 Mon Sep 17 00:00:00 2001 From: Leeroy Ding Date: Mon, 22 Jul 2013 14:06:46 +0800 Subject: [PATCH 2/4] Added async option and aria2-rpc support --- lixian_commands/download.py | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/lixian_commands/download.py b/lixian_commands/download.py index 4f4ba51..da2a9c5 100644 --- a/lixian_commands/download.py +++ b/lixian_commands/download.py @@ -77,6 +77,7 @@ def download_file(client, path, task, options): overwrite = options.get('overwrite') mini_hash = options.get('mini_hash') no_hash = options.get('no_hash') + async = options.get('async') url = str(task['xunlei_url']) if options['node']: @@ -121,7 +122,18 @@ def download2(client, url, path, task): if not verify(path, task): raise Exception('hash check failed') - download2(client, url, path, task) + def download_async(client, url, path, task): + size=task['size'] + download = download_tool(client=client, url=url, path=path, size=size, resuming=resuming, async=True) + download1(download, path) + + def download3(client, url, path, task, async): + if async: + download_async(client, url, path, task) + else: + download2(client, url, path, task) + + download3(client, url, path, task, async) def download_single_task(client, task, options): @@ -252,6 +264,7 @@ def download_multiple_tasks(client, tasks, options): @command_line_value('category') @command_line_option('delete', default=get_config('delete')) @command_line_option('continue', alias='c', default=get_config('continue')) +@command_line_option('async', default=get_config('async')) @command_line_option('overwrite') @command_line_option('mini-hash', default=get_config('mini-hash')) @command_line_option('hash', default=get_config('hash', True)) @@ -269,6 +282,7 @@ def download_task(args): 'output_dir': args.output_dir, 'delete': args.delete, 'resuming': args._args['continue'], + 'async': args._args['async'], 'overwrite': args.overwrite, 'mini_hash': args.mini_hash, 'no_hash': not args.hash, @@ -303,16 +317,28 @@ def sleep(n): elif args.watch: assert not args.output, 'not supported with watch option yet' + old_tasks = [] tasks = query.pull_completed() while True: - if tasks: - download_multiple_tasks(client, tasks, download_args) + new_tasks = [] + for nt in tasks: + is_new = True + for ot in old_tasks: + if nt['id'] == ot['id']: + is_new = False + break + if is_new: + new_tasks.append(nt) + + if new_tasks: + download_multiple_tasks(client, new_tasks, download_args) if (not query.download_jobs) and (not query.queries): break - if not tasks: + if not new_tasks: sleep(args.watch_interval) query.refresh_status() query.query_search() + old_tasks = tasks tasks = query.pull_completed() else: From dc421bf8a449e477c8e3475c4d6cfc86c21351c2 Mon Sep 17 00:00:00 2001 From: Zhao Cai Date: Sat, 23 Mar 2013 18:27:10 -0400 Subject: [PATCH 3/4] use to separate output_tasks which is aligned and easy for other program to parse. --- lixian_commands/list.py | 2 +- lixian_commands/util.py | 28 ++++++++++++++-------------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/lixian_commands/list.py b/lixian_commands/list.py index 3347324..7470ea7 100644 --- a/lixian_commands/list.py +++ b/lixian_commands/list.py @@ -48,7 +48,7 @@ def list_task(args): if len(args) == 1 and re.match(r'\d+/', args[0]) and len(tasks) == 1 and 'files' in tasks[0]: parent_ids = [tasks[0]['id']] tasks = tasks[0]['files'] - columns = ['n', 'id', 'name', 'status', 'size', 'progress', 'speed', 'date', 'dcid', 'gcid', 'original-url', 'download-url'] + columns = ['n', 'id', 'status', 'size', 'progress', 'speed', 'date', 'dcid', 'gcid', 'name', 'original-url', 'download-url'] columns = filter(lambda k: getattr(args, k), columns) output_tasks(tasks, columns, args, not parent_ids) diff --git a/lixian_commands/util.py b/lixian_commands/util.py index 1b50cf5..ca3d7dd 100644 --- a/lixian_commands/util.py +++ b/lixian_commands/util.py @@ -66,34 +66,34 @@ def output_tasks(tasks, columns, args, top=True): for k in columns: if k == 'n': if top: - print '#%d' % t['#'], + print '#%d\t' % t['#'], elif k == 'id': - print t.get('index', t['id']), - elif k == 'name': - print t['name'].encode(default_encoding), + print '%s\t' % t.get('index', t['id']), elif k == 'status': with colors(args.colors).bold(): - print t['status_text'], + print '%s\t' % t['status_text'], elif k == 'size': if args.format_size: from lixian_util import format_size - print format_size(t['size']), + print '%s\t' % format_size(t['size']), else: - print t['size'], + print '%s\t' % t['size'], elif k == 'progress': - print t['progress'], + print '%s\t' % t['progress'], elif k == 'speed': - print t['speed'], + print '%s\t' % t['speed'], elif k == 'date': - print t['date'], + print '%s\t' % t['date'], elif k == 'dcid': - print t['dcid'], + print '%s\t' % t['dcid'], elif k == 'gcid': - print t['gcid'], + print '%s\t' % t['gcid'], elif k == 'original-url': - print t['original_url'], + print '%s\t' % t['original_url'], elif k == 'download-url': - print t['xunlei_url'], + print '%s\t' % t['xunlei_url'], + elif k == 'name': + print '%s\t' % t['name'].encode(default_encoding), else: raise NotImplementedError(k) print From 7cf294d303323284611e521de3049292cdc75e76 Mon Sep 17 00:00:00 2001 From: Zhao Cai Date: Sat, 7 Sep 2013 20:39:39 -0400 Subject: [PATCH 4/4] [minor] support user/passwd for aria2 rpc download --- lixian_download_tools.py | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/lixian_download_tools.py b/lixian_download_tools.py index b966228..c274806 100644 --- a/lixian_download_tools.py +++ b/lixian_download_tools.py @@ -3,7 +3,7 @@ from lixian_config import * import subprocess -import urllib2 +import urllib2, base64 import os.path import json import time @@ -116,24 +116,36 @@ def __call__(self): @download_tool('aria2rpc') def aria2rpc_download(client, download_url, path, resuming=False): - gdriveid = str(client.get_gdriveid()) - dir = os.path.dirname(path) - filename = os.path.basename(path) + gdriveid = str(client.get_gdriveid()) + dir = os.path.dirname(path) + filename = os.path.basename(path) aria2rpchost = get_config('aria2-rpc-host', 'localhost') aria2rpcport = get_config('aria2-rpc-port', '6800') + aria2rpcuser = get_config('aria2-rpc-user', None) + aria2rpcpass = get_config('aria2-rpc-pass', '') + ts = str(int(time.time()*1000)) - data = {"jsonrpc": "2.0", + data = {"jsonrpc": "2.0", "method":"aria2.addUri", "id": ts, - "params": [[download_url], - {"out": filename, + "params": [[download_url], + { + "dir": dir, + "out": filename, "header": "Cookie: gdriveid="+gdriveid} - ] } data = json.dumps(data) + request_url = "http://"+aria2rpchost+":"+aria2rpcport+"/jsonrpc?tm="+ts - urllib2.urlopen(request_url, data) + + if aria2rpcuser: + request = urllib2.Request(request_url) + base64string = base64.encodestring('%s:%s' % (aria2rpcuser, aria2rpcpass)).replace('\n', '') + request.add_header("Authorization", "Basic %s" % base64string) + urllib2.urlopen(request, data) + else: + urllib2.urlopen(request_url, data) @download_tool('axel')