From bfb527ee44687be770bb6a4fe9fb5d45180da830 Mon Sep 17 00:00:00 2001 From: Fred Chevitarese Date: Mon, 30 Jan 2017 13:11:18 -0200 Subject: [PATCH 1/6] Each queue should be able to have its custom_exception or None. --- django_rq/management/commands/rqworker.py | 3 ++- django_rq/queues.py | 7 ++++- django_rq/test_exception_handler.py | 3 +++ django_rq/test_settings.py | 3 ++- django_rq/workers.py | 32 ++++++++++++++++------- 5 files changed, 35 insertions(+), 13 deletions(-) create mode 100644 django_rq/test_exception_handler.py diff --git a/django_rq/management/commands/rqworker.py b/django_rq/management/commands/rqworker.py index f70a8336..b6d92b36 100644 --- a/django_rq/management/commands/rqworker.py +++ b/django_rq/management/commands/rqworker.py @@ -77,7 +77,8 @@ def handle(self, *args, **options): queues, connection=queues[0].connection, name=options['name'], - exception_handlers=get_exception_handlers() or None, + exception_handlers=get_exception_handlers( + queues[0].exception_handlers) or None, default_worker_ttl=options['worker_ttl'] ) diff --git a/django_rq/queues.py b/django_rq/queues.py index 86cf53de..717f2fe7 100644 --- a/django_rq/queues.py +++ b/django_rq/queues.py @@ -49,6 +49,7 @@ class DjangoRQ(Queue): def __init__(self, *args, **kwargs): autocommit = kwargs.pop('autocommit', None) self._autocommit = get_commit_mode() if autocommit is None else autocommit + self.exception_handlers = kwargs.pop('exception_handlers', None) super(DjangoRQ, self).__init__(*args, **kwargs) @@ -138,9 +139,13 @@ def get_queue(name='default', default_timeout=None, async=None, default_timeout = QUEUES[name].get('DEFAULT_TIMEOUT') if queue_class is None: queue_class = get_queue_class(QUEUES[name]) + + exception_handlers = QUEUES[name].get('EXCEPTION_HANDLERS', None) + return queue_class(name, default_timeout=default_timeout, connection=get_connection(name), async=async, - autocommit=autocommit) + autocommit=autocommit, + exception_handlers=exception_handlers) def get_queue_by_index(index): diff --git a/django_rq/test_exception_handler.py b/django_rq/test_exception_handler.py new file mode 100644 index 00000000..b2fbfc1e --- /dev/null +++ b/django_rq/test_exception_handler.py @@ -0,0 +1,3 @@ + +def my_custom_exception(): + pass \ No newline at end of file diff --git a/django_rq/test_settings.py b/django_rq/test_settings.py index 28256508..d039027e 100644 --- a/django_rq/test_settings.py +++ b/django_rq/test_settings.py @@ -99,7 +99,8 @@ 'HOST': REDIS_HOST, 'PORT': 6379, 'DB': 0, - 'DEFAULT_TIMEOUT': 500 + 'DEFAULT_TIMEOUT': 500, + 'EXCEPTION_HANDLERS': ['test_exception_handler.my_custom_exception'], }, 'test': { 'HOST': REDIS_HOST, diff --git a/django_rq/workers.py b/django_rq/workers.py index 057ae449..90c6ce95 100644 --- a/django_rq/workers.py +++ b/django_rq/workers.py @@ -2,17 +2,27 @@ from rq.utils import import_attribute from .queues import get_queues -from .settings import EXCEPTION_HANDLERS -def get_exception_handlers(): - """ - Custom exception handlers could be defined in settings.py: - RQ = { - 'EXCEPTION_HANDLERS': ['path.to.handler'], +def get_exception_handlers(exception_handlers): + """Custom exception handler defined in QUEUE settings: + RQ_QUEUES = { + 'default': { + 'HOST': 'localhost', + 'PORT': 6379, + 'DB': 0, + 'PASSWORD': '', + 'DEFAULT_TIMEOUT': 360, + 'EXCEPTION_HANDLERS': [ + 'test_exception_handler.my_custom_exception' + ], + } } """ - return [import_attribute(path) for path in EXCEPTION_HANDLERS] + if exception_handlers: + return [import_attribute(exception_handler) for exception_handler in + exception_handlers] + return None def get_worker(*queue_names): @@ -20,6 +30,8 @@ def get_worker(*queue_names): Returns a RQ worker for all queues or specified ones. """ queues = get_queues(*queue_names) - return Worker(queues, - connection=queues[0].connection, - exception_handlers=get_exception_handlers() or None) + return Worker( + queues, + connection=queues[0].connection, + exception_handlers=get_exception_handlers(queues[0].exception_handlers) + ) From 3c959b314a04819ded6cd031260a3f2e3bf2d83c Mon Sep 17 00:00:00 2001 From: Fred Chevitarese Date: Mon, 30 Jan 2017 13:15:36 -0200 Subject: [PATCH 2/6] Forgot the README ... --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index a924b83a..db36da9d 100644 --- a/README.rst +++ b/README.rst @@ -46,6 +46,7 @@ Installation 'DB': 0, 'PASSWORD': 'some-password', 'DEFAULT_TIMEOUT': 360, + 'EXCEPTION_HANDLERS': ['path.to.my.handler'], # If you need custom exception handlers in your queue. If not just delete this. }, 'high': { 'URL': os.getenv('REDISTOGO_URL', 'redis://localhost:6379/0'), # If you're on Heroku @@ -58,7 +59,6 @@ Installation } } - RQ_EXCEPTION_HANDLERS = ['path.to.my.handler'] # If you need custom exception handlers * Include ``django_rq.urls`` in your ``urls.py``: From defa71729d3e433e5ccf8e6eca10b979818e26a7 Mon Sep 17 00:00:00 2001 From: Fred Chevitarese Date: Mon, 30 Jan 2017 14:03:57 -0200 Subject: [PATCH 3/6] Removing custom exception from config because it's leading to a break of a test. I cannot send a failed job to failed --- django_rq/test_exception_handler.py | 4 ++-- django_rq/test_settings.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/django_rq/test_exception_handler.py b/django_rq/test_exception_handler.py index b2fbfc1e..fded0e9a 100644 --- a/django_rq/test_exception_handler.py +++ b/django_rq/test_exception_handler.py @@ -1,3 +1,3 @@ -def my_custom_exception(): - pass \ No newline at end of file +def my_custom_exception(job, exc_type, exc_value, traceback): + return False \ No newline at end of file diff --git a/django_rq/test_settings.py b/django_rq/test_settings.py index d039027e..1208d7e9 100644 --- a/django_rq/test_settings.py +++ b/django_rq/test_settings.py @@ -100,7 +100,6 @@ 'PORT': 6379, 'DB': 0, 'DEFAULT_TIMEOUT': 500, - 'EXCEPTION_HANDLERS': ['test_exception_handler.my_custom_exception'], }, 'test': { 'HOST': REDIS_HOST, From 0c31ac39f581e914948d56f75ef8e9b422da694e Mon Sep 17 00:00:00 2001 From: Fred Chevitarese Date: Thu, 9 Feb 2017 19:21:19 -0200 Subject: [PATCH 4/6] Importing RQ_EXCEPTION_HANDLERS from settings to mantain backward compatibility --- django_rq/workers.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/django_rq/workers.py b/django_rq/workers.py index 90c6ce95..c762eb12 100644 --- a/django_rq/workers.py +++ b/django_rq/workers.py @@ -19,6 +19,12 @@ def get_exception_handlers(exception_handlers): } } """ + try: + from django.settings import RQ_EXCEPTION_HANDLERS + exception_handlers.extends(RQ_EXCEPTION_HANDLERS) + except ImportError: + pass + if exception_handlers: return [import_attribute(exception_handler) for exception_handler in exception_handlers] From 54d4e70b503f62ca12f6cfd07446df49c26a7f67 Mon Sep 17 00:00:00 2001 From: Fred Chevitarese Date: Thu, 16 Feb 2017 14:27:05 -0200 Subject: [PATCH 5/6] Fallback --- django_rq/workers.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/django_rq/workers.py b/django_rq/workers.py index c762eb12..5a26725f 100644 --- a/django_rq/workers.py +++ b/django_rq/workers.py @@ -1,6 +1,7 @@ from rq import Worker from rq.utils import import_attribute +from .settings import EXCEPTION_HANDLERS from .queues import get_queues @@ -19,16 +20,12 @@ def get_exception_handlers(exception_handlers): } } """ - try: - from django.settings import RQ_EXCEPTION_HANDLERS - exception_handlers.extends(RQ_EXCEPTION_HANDLERS) - except ImportError: - pass if exception_handlers: return [import_attribute(exception_handler) for exception_handler in exception_handlers] - return None + else: + return [import_attribute(path) for path in EXCEPTION_HANDLERS] def get_worker(*queue_names): From c3cf7a0fcc80c99ef6c050bda2c8ce29f469b6c3 Mon Sep 17 00:00:00 2001 From: Fred Chevitarese Date: Wed, 22 Mar 2017 09:44:07 -0300 Subject: [PATCH 6/6] Changing the signature of get_exception_handler function; Fixing tests by checking if have a custom exception, otherwise, it start a Worker without the parameter. --- django_rq/queues.py | 5 +++-- django_rq/workers.py | 21 +++++++++++++-------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/django_rq/queues.py b/django_rq/queues.py index 717f2fe7..8bf705b5 100644 --- a/django_rq/queues.py +++ b/django_rq/queues.py @@ -175,9 +175,10 @@ def filter_connection_params(queue_params): """ NON_CONNECTION_PARAMS = ('DEFAULT_TIMEOUT',) - #return {p:v for p,v in queue_params.items() if p not in NON_CONNECTION_PARAMS} + # return {p:v for p,v in queue_params.items() if p not in NON_CONNECTION_PARAMS} # Dict comprehension compatible with python 2.6 - return dict((p,v) for (p,v) in queue_params.items() if p not in NON_CONNECTION_PARAMS) + return dict((p, v) for (p, v) in queue_params.items() + if p not in NON_CONNECTION_PARAMS) def get_queues(*queue_names, **kwargs): diff --git a/django_rq/workers.py b/django_rq/workers.py index 5a26725f..c924eff8 100644 --- a/django_rq/workers.py +++ b/django_rq/workers.py @@ -5,7 +5,7 @@ from .queues import get_queues -def get_exception_handlers(exception_handlers): +def get_exception_handlers(queue_name): """Custom exception handler defined in QUEUE settings: RQ_QUEUES = { 'default': { @@ -21,9 +21,9 @@ def get_exception_handlers(exception_handlers): } """ - if exception_handlers: + if queue_name: return [import_attribute(exception_handler) for exception_handler in - exception_handlers] + queue_name] else: return [import_attribute(path) for path in EXCEPTION_HANDLERS] @@ -33,8 +33,13 @@ def get_worker(*queue_names): Returns a RQ worker for all queues or specified ones. """ queues = get_queues(*queue_names) - return Worker( - queues, - connection=queues[0].connection, - exception_handlers=get_exception_handlers(queues[0].exception_handlers) - ) + exception_handlers = get_exception_handlers(queues[0].exception_handlers) + + if exception_handlers: + return Worker( + queues, + connection=queues[0].connection, + exception_handlers=exception_handlers + ) + else: + return Worker(queues, connection=queues[0].connection)