From 58ff6f7f8ce204b2fb9b29d630b36d348b2b2f9f Mon Sep 17 00:00:00 2001 From: Simon Hewitt Date: Sat, 2 Jul 2011 12:50:19 +0100 Subject: [PATCH 01/17] return number of times a job has been cancelled --- tests/test_job.py | 12 +++++++++--- thoonk/feeds/job.py | 8 +++++++- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/tests/test_job.py b/tests/test_job.py index fa15a8f..03e4ccc 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -36,7 +36,7 @@ def test_10_basic_job(self): #worker testjobworker = self.ps.job("testjob") - id_worker, query_worker = testjobworker.get(timeout=3) + id_worker, query_worker, cancelled = testjobworker.get(timeout=3) result_worker = math.sqrt(float(query_worker)) testjobworker.finish(id_worker, result_worker, True) @@ -51,13 +51,19 @@ def test_20_cancel_job(self): #publisher id = j.put(9.0) #worker claims - id, query = j.get() + id, query, cancelled = j.get() + self.assertEqual(cancelled, 0) #publisher or worker cancels j.cancel(id) - id2, query2 = j.get() + id2, query2, cancelled2 = j.get() + self.assertEqual(cancelled2, 1) self.assertEqual(id, id2) #cancel the work again j.cancel(id) + # check the cancelled increment again + id3, query3, cancelled3 = j.get() + self.assertEqual(cancelled3, 2) + self.assertEqual(id, id3) #cleanup -- remove the job from the queue j.retract(id) self.assertEqual(j.get_ids(), []) diff --git a/thoonk/feeds/job.py b/thoonk/feeds/job.py index 43d06c6..bb6721e 100644 --- a/thoonk/feeds/job.py +++ b/thoonk/feeds/job.py @@ -178,6 +178,11 @@ def get(self, timeout=0): Arguments: timeout -- Optional time in seconds to wait before raising an exception. + + Returns: + id -- The id of the job + job -- The job content + cancelled -- The number of times the job has been cancelled """ id = self.redis.brpop(self.feed_ids, timeout) if id is None: @@ -187,8 +192,9 @@ def get(self, timeout=0): pipe = self.redis.pipeline() pipe.zadd(self.feed_job_claimed, id, time.time()) pipe.hget(self.feed_items, id) + pipe.hget(self.feed_cancelled, id) result = pipe.execute() - return id, result[1] + return id, result[1], int(result[2]) if isinstance(result[2], basestring) else 0 def finish(self, id, item=None, result=False, timeout=None): """ From 309ec1fc6aee6d7658e3e9059d551be268708679 Mon Sep 17 00:00:00 2001 From: Simon Hewitt Date: Sat, 2 Jul 2011 12:57:58 +0100 Subject: [PATCH 02/17] reworked cancelled if then statement --- thoonk/feeds/job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/thoonk/feeds/job.py b/thoonk/feeds/job.py index bb6721e..087693c 100644 --- a/thoonk/feeds/job.py +++ b/thoonk/feeds/job.py @@ -194,7 +194,7 @@ def get(self, timeout=0): pipe.hget(self.feed_items, id) pipe.hget(self.feed_cancelled, id) result = pipe.execute() - return id, result[1], int(result[2]) if isinstance(result[2], basestring) else 0 + return id, result[1], 0 if result[2] is None else int(result[2]) def finish(self, id, item=None, result=False, timeout=None): """ From 8442f34f2abe875cb4d90d0f76e3f2b661f5d3d4 Mon Sep 17 00:00:00 2001 From: Simon Hewitt Date: Sat, 2 Jul 2011 13:21:12 +0100 Subject: [PATCH 03/17] job.get now raises Empty if it times out --- tests/test_job.py | 5 ++++- thoonk/feeds/job.py | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_job.py b/tests/test_job.py index 03e4ccc..2414ef9 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -68,6 +68,9 @@ def test_20_cancel_job(self): j.retract(id) self.assertEqual(j.get_ids(), []) - + def test_30_no_job(self): + j = self.ps.job("testjob") + self.assertRaises(Exception, j.get, timeout=1) + suite = unittest.TestLoader().loadTestsFromTestCase(TestJob) diff --git a/thoonk/feeds/job.py b/thoonk/feeds/job.py index 087693c..ae38c63 100644 --- a/thoonk/feeds/job.py +++ b/thoonk/feeds/job.py @@ -8,6 +8,7 @@ from thoonk.exceptions import * from thoonk.feeds import Queue +from thoonk.feeds.queue import Empty class JobDoesNotExist(Exception): @@ -186,7 +187,7 @@ def get(self, timeout=0): """ id = self.redis.brpop(self.feed_ids, timeout) if id is None: - return # raise exception? + raise Empty id = id[1] pipe = self.redis.pipeline() From 0dbbf8fca6d8a967172eda662d773f5378795321 Mon Sep 17 00:00:00 2001 From: Simon Hewitt Date: Sat, 2 Jul 2011 13:23:57 +0100 Subject: [PATCH 04/17] made assertRaises explicitly catch Empty exception --- tests/test_job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_job.py b/tests/test_job.py index 2414ef9..7f2cf28 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -70,7 +70,7 @@ def test_20_cancel_job(self): def test_30_no_job(self): j = self.ps.job("testjob") - self.assertRaises(Exception, j.get, timeout=1) + self.assertRaises(thoonk.feeds.queue.Empty, j.get, timeout=1) suite = unittest.TestLoader().loadTestsFromTestCase(TestJob) From c915e77dbefca932b4b2a3e6a6894e67ccb9c02c Mon Sep 17 00:00:00 2001 From: Simon Hewitt Date: Mon, 25 Jul 2011 15:44:52 +0100 Subject: [PATCH 05/17] added publish events for job feeds --- tests/test_job.py | 4 +- tests/test_notice.py | 181 +++++++++++++++++++++++++++++++++++++++++++ thoonk/feeds/feed.py | 2 +- thoonk/feeds/job.py | 39 +++++++--- thoonk/pubsub.py | 116 ++++++++++++++++++++++++++- 5 files changed, 327 insertions(+), 15 deletions(-) create mode 100644 tests/test_notice.py diff --git a/tests/test_job.py b/tests/test_job.py index 7f2cf28..95967ef 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -32,7 +32,7 @@ def test_10_basic_job(self): """JOB publish, retrieve, finish, get result""" #publisher testjob = self.ps.job("testjob") - id = testjob.put(9.0) + id = testjob.put('9.0') #worker testjobworker = self.ps.job("testjob") @@ -49,7 +49,7 @@ def test_20_cancel_job(self): """Test cancelling a job""" j = self.ps.job("testjob") #publisher - id = j.put(9.0) + id = j.put('9.0') #worker claims id, query, cancelled = j.get() self.assertEqual(cancelled, 0) diff --git a/tests/test_notice.py b/tests/test_notice.py new file mode 100644 index 0000000..dac741a --- /dev/null +++ b/tests/test_notice.py @@ -0,0 +1,181 @@ +import thoonk +import unittest +import time +from ConfigParser import ConfigParser + +class TestNotice(unittest.TestCase): + + def __init__(self, *args, **kwargs): + unittest.TestCase.__init__(self, *args, **kwargs) + + conf = ConfigParser() + conf.read('test.cfg') + if conf.sections() == ['Test']: + self.ps = thoonk.Thoonk(host=conf.get('Test', 'host'), + port=conf.getint('Test', 'port'), + db=conf.getint('Test', 'db'), + listen=True) + self.ps.redis.flushdb() + else: + print 'No test configuration found in test.cfg' + exit() + + + def tearDown(self): + self.ps.close() + + "claimed, cancelled, stalled, finished" + + def test_05_publish_notice(self): + notice_received = [False] + ids = [None, None] + + def received_handler(feed, item, id): + self.assertEqual(feed, "testfeed") + ids[1] = id + notice_received[0] = True + + self.ps.register_handler('publish_notice', received_handler) + + j = self.ps.feed("testfeed") + + self.assertFalse(notice_received[0]) + + #publisher + ids[0] = j.publish('a') + + # block while waiting for notice + i = 0 + while not notice_received[0] and i < 3: + i += 1 + time.sleep(1) + + self.assertEqual(ids[0], ids[1]) + + self.assertTrue(notice_received[0], "Notice not received") + + self.ps.remove_handler('publish_notice', received_handler) + + def test_10_job_notices(self): + notices_received = [False] + ids = [None, None] + + def publish_handler(feed, item, id): + self.assertEqual(feed, "testjob") + ids[-1] = id + notices_received[-1] = "publish" + + def claimed_handler(feed, id): + self.assertEqual(feed, "testjob") + ids[-1] = id + notices_received[-1] = "claimed" + + def cancelled_handler(feed, id): + self.assertEqual(feed, "testjob") + ids[-1] = id + notices_received[-1] = "cancelled" + + def stalled_handler(feed, id): + self.assertEqual(feed, "testjob") + ids[-1] = id + notices_received[-1] = "stalled" + + def retried_handler(feed, id): + self.assertEqual(feed, "testjob") + ids[-1] = id + notices_received[-1] = "retried" + + def finished_handler(feed, id, result): + self.assertEqual(feed, "testjob") + ids[-1] = id + notices_received[-1] = "finished" + + def do_wait(): + i = 0 + while not notices_received[-1] and i < 2: + i += 1 + time.sleep(1) + + self.ps.register_handler('publish_notice', publish_handler) + self.ps.register_handler('claimed_notice', claimed_handler) + self.ps.register_handler('cancelled_notice', cancelled_handler) + self.ps.register_handler('stalled_notice', stalled_handler) + self.ps.register_handler('retried_notice', retried_handler) + self.ps.register_handler('finished_notice', finished_handler) + + j = self.ps.job("testjob") + + self.assertFalse(notices_received[0]) + + # create the job + ids[0] = j.put('b') + do_wait() + self.assertEqual(notices_received[0], "publish", "Notice not received") + self.assertEqual(ids[0], ids[-1]) + + notices_received.append(False); ids.append(None); + # claim the job + id, job, cancelled = j.get() + self.assertEqual(job, 'b') + self.assertEqual(cancelled, 0) + self.assertEqual(ids[0], id) + do_wait() + self.assertEqual(notices_received[-1], "claimed", "Claimed notice not received") + self.assertEqual(ids[0], ids[-1]) + + notices_received.append(False); ids.append(None); + # cancel the job + j.cancel(id) + do_wait() + self.assertEqual(notices_received[-1], "cancelled", "Cancelled notice not received") + self.assertEqual(ids[0], ids[-1]) + + notices_received.append(False); ids.append(None); + # get the job again + id, job, cancelled = j.get() + self.assertEqual(job, 'b') + self.assertEqual(cancelled, 1) + self.assertEqual(ids[0], id) + do_wait() + self.assertEqual(notices_received[-1], "claimed", "Claimed notice not received") + self.assertEqual(ids[0], ids[-1]) + + notices_received.append(False); ids.append(None); + # stall the job + j.stall(id) + do_wait() + self.assertEqual(notices_received[-1], "stalled", "Stalled notice not received") + self.assertEqual(ids[0], ids[-1]) + + notices_received.append(False); ids.append(None); + # retry the job + j.retry(id) + do_wait() + self.assertEqual(notices_received[-1], "retried", "Retried notice not received") + self.assertEqual(ids[0], ids[-1]) + + notices_received.append(False); ids.append(None); + # get the job again + id, job, cancelled = j.get() + self.assertEqual(job, 'b') + self.assertEqual(cancelled, 0) + self.assertEqual(ids[0], id) + do_wait() + self.assertEqual(notices_received[-1], "claimed", "Claimed notice not received") + self.assertEqual(ids[0], ids[-1]) + + notices_received.append(False); ids.append(None); + # finish the job + j.finish(id) + do_wait() + self.assertEqual(notices_received[-1], "finished", "Finished notice not received") + self.assertEqual(ids[0], ids[-1]) + + self.ps.remove_handler('publish_notice', publish_handler) + self.ps.remove_handler('claimed_notice', claimed_handler) + self.ps.remove_handler('cancelled_notice', cancelled_handler) + self.ps.remove_handler('stalled_notice', stalled_handler) + self.ps.remove_handler('retried_notice', retried_handler) + self.ps.remove_handler('finished_notice', finished_handler) + +suite = unittest.TestLoader().loadTestsFromTestCase(TestNotice) diff --git a/thoonk/feeds/feed.py b/thoonk/feeds/feed.py index ed89e04..ca32152 100644 --- a/thoonk/feeds/feed.py +++ b/thoonk/feeds/feed.py @@ -124,7 +124,7 @@ def config(self): """ with self.config_lock: if not self.config_valid: - conf = self.redis.get(self.feed_config) + conf = self.redis.get(self.feed_config) or "{}" self._config = json.loads(conf) self.config_valid = True return self._config diff --git a/thoonk/feeds/job.py b/thoonk/feeds/job.py index ae38c63..bb4303b 100644 --- a/thoonk/feeds/job.py +++ b/thoonk/feeds/job.py @@ -5,6 +5,7 @@ import time import uuid +import redis from thoonk.exceptions import * from thoonk.feeds import Queue @@ -56,7 +57,7 @@ class Job(Queue): feed.cancelled:[feed] -- A hash table of cancelled jobs. feed.claimed:[feed] -- A hash table of claimed jobs. feed.stalled:[feed] -- A hash table of stalled jobs. - feeed.funning:[feed] -- A hash table of running jobs. + feed.running:[feed] -- A hash table of running jobs. feed.finished:[feed]\x00[id] -- Temporary queue for receiving job result data. @@ -91,24 +92,30 @@ def __init__(self, thoonk, feed, config=None): """ Queue.__init__(self, thoonk, feed, config=None) - self.feed_published = 'feed.published:%s' % feed + #self.feed_publish = 'feed.publish:%s' % feed self.feed_cancelled = 'feed.cancelled:%s' % feed + self.feed_retried = 'feed.retried:%s' % feed + self.feed_finished = 'feed.finished:%s' % feed self.feed_job_claimed = 'feed.claimed:%s' % feed self.feed_job_stalled = 'feed.stalled:%s' % feed self.feed_job_finished = 'feed.finished:%s\x00%s' % (feed, '%s') self.feed_job_running = 'feed.running:%s' % feed + def get_channels(self): + return (self.feed_publish, self.feed_job_claimed, self.feed_job_stalled, + self.feed_finished, self.feed_cancelled, self.feed_retried) + def get_schemas(self): """Return the set of Redis keys used exclusively by this feed.""" schema = set((self.feed_job_claimed, self.feed_job_stalled, self.feed_job_running, - self.feed_published, + self.feed_publish, self.feed_cancelled)) for id in self.get_ids(): schema.add(self.feed_job_finished % id) - + return schema.union(Queue.get_schemas(self)) def get_ids(self): @@ -128,7 +135,7 @@ def retract(self, id): pipe = self.redis.pipeline() pipe.hdel(self.feed_items, id) pipe.hdel(self.feed_cancelled, id) - pipe.zrem(self.feed_published, id) + pipe.zrem(self.feed_publish, id) pipe.srem(self.feed_job_stalled, id) pipe.zrem(self.feed_job_claimed, id) pipe.lrem(self.feed_ids, 1, id) @@ -165,9 +172,16 @@ def put(self, item, priority=False): pipe.lpush(self.feed_ids, id) pipe.incr(self.feed_publishes) pipe.hset(self.feed_items, id, item) - pipe.zadd(self.feed_published, id, time.time()) + pipe.zadd(self.feed_publish, id, time.time()) results = pipe.execute() + + if results[-1]: + # If zadd was successful + self.thoonk._publish(self.feed_publish, (id, item)) + else: + self.thoonk._publish(self.feed_edit, (id, item)) + return id def get(self, timeout=0): @@ -195,6 +209,9 @@ def get(self, timeout=0): pipe.hget(self.feed_items, id) pipe.hget(self.feed_cancelled, id) result = pipe.execute() + + self.thoonk._publish(self.feed_job_claimed, (id,)) + return id, result[1], 0 if result[2] is None else int(result[2]) def finish(self, id, item=None, result=False, timeout=None): @@ -226,7 +243,8 @@ def finish(self, id, item=None, result=False, timeout=None): pipe.expire(self.feed_job_finished % id, timeout) pipe.hdel(self.feed_items, id) try: - result = pipe.execute() + pipe.execute() + self.thoonk._publish(self.feed_finished, (id, result if result else "")) break except redis.exceptions.WatchError: pass @@ -263,6 +281,7 @@ def cancel(self, id): pipe.zrem(self.feed_job_claimed, id) try: pipe.execute() + self.thoonk._publish(self.feed_cancelled, (id,)) break except redis.exceptions.WatchError: pass @@ -286,9 +305,10 @@ def stall(self, id): pipe.zrem(self.feed_job_claimed, id) pipe.hdel(self.feed_cancelled, id) pipe.sadd(self.feed_job_stalled, id) - pipe.zrem(self.feed_published, id) + pipe.zrem(self.feed_publish, id) try: pipe.execute() + self.thoonk._publish(self.feed_job_stalled, (id,)) break except redis.exceptions.WatchError: pass @@ -309,9 +329,10 @@ def retry(self, id): pipe = self.redis.pipeline() pipe.srem(self.feed_job_stalled, id) pipe.lpush(self.feed_ids, id) - pipe.zadd(self.feed_published, time.time(), id) + pipe.zadd(self.feed_publish, time.time(), id) try: results = pipe.execute() + self.thoonk._publish(self.feed_retried, (id,)) if not results[0]: return # raise exception? break diff --git a/thoonk/pubsub.py b/thoonk/pubsub.py index a060798..7c30566 100644 --- a/thoonk/pubsub.py +++ b/thoonk/pubsub.py @@ -95,7 +95,12 @@ def __init__(self, host='localhost', port=6379, db=0, listen=False): 'delete_notice': [], 'publish_notice': [], 'retract_notice': [], - 'position_notice': []} + 'position_notice': [], + 'stalled_notice': [], + 'retried_notice': [], + 'finished_notice': [], + 'claimed_notice': [], + 'cancelled_notice': []} self.listen_ready = threading.Event() self.listening = listen @@ -206,6 +211,20 @@ def register_handler(self, name, handler): self.handlers[name] = [] self.handlers[name].append(handler) + def remove_handler(self, name, handler): + """ + Unregister a function that was registered via register_handler + + Arguments: + name -- The name of the feed event. + handler -- The function for handling the event. + """ + try: + self.handlers[name].remove(handler) + except (KeyError, ValueError): + pass + + def create_feed(self, feed, config): """ Create a new feed with a given configuration. @@ -273,6 +292,12 @@ def set_config(self, feed, config): self.redis.set(self.feed_config % feed, jconfig) self._publish(self.conf_feed, (feed, self._feed_config.instance)) + def get_config(self, feed): + if not self.feed_exists(feed): + raise FeedDoesNotExist + config = self.redis.get(self.feed_config % feed) + return json.loads(config) + def get_feeds(self): """ Return the set of known feeds. @@ -341,12 +366,36 @@ def listen(self): elif event['channel'].startswith('feed.position'): self.position_notice(event['channel'].split(':', 1)[-1], event['data']) + elif event['channel'].startswith('feed.claimed'): + self.claimed_notice(event['channel'].split(':', 1)[-1], + event['data']) + elif event['channel'].startswith('feed.finished'): + id, data = event['data'].split(':', 1)[-1].split("\x00", 1) + self.finished_notice(event['channel'].split(':', 1)[-1], id, + data) + elif event['channel'].startswith('feed.cancelled'): + self.cancelled_notice(event['channel'].split(':', 1)[-1], + event['data']) + elif event['channel'].startswith('feed.stalled'): + self.stalled_notice(event['channel'].split(':', 1)[-1], + event['data']) + elif event['channel'].startswith('feed.retried'): + self.retried_notice(event['channel'].split(':', 1)[-1], + event['data']) elif event['channel'] == self.new_feed: #feed created event name, instance = event['data'].split('\x00') self.feeds.add(name) - self.lredis.subscribe((self.feed_publish % name, - self.feed_retract % name)) + config = self.get_config(name) + if config["type"] == "job": + self.lredis.subscribe((self.feed_publish % name, + "feed.cancelled:%s" % name, + "feed.claimed:%s" % name, + "feed.finished:%s" % name, + "feed.stalled:%s" % name,)) + else: + self.lredis.subscribe((self.feed_publish % name, + self.feed_retract % name)) self.create_notice(name) elif event['channel'] == self.del_feed: #feed destroyed event @@ -424,3 +473,64 @@ def position_notice(self, feed, id, rel_id): """ for handler in self.handlers['position_notice']: handler(feed, id, rel_id) + + def stalled_notice(self, feed, id): + """ + Generate a notice that a job has been stalled, and + execute any relevant event handlers. + + Arguments: + feed -- The name of the feed. + id -- The ID of the stalled item. + """ + for handler in self.handlers['stalled_notice']: + handler(feed, id) + + def retried_notice(self, feed, id): + """ + Generate a notice that a job has been retried, and + execute any relevant event handlers. + + Arguments: + feed -- The name of the feed. + id -- The ID of the retried item. + """ + for handler in self.handlers['retried_notice']: + handler(feed, id) + + def cancelled_notice(self, feed, id): + """ + Generate a notice that a job has been cancelled, and + execute any relevant event handlers. + + Arguments: + feed -- The name of the feed. + id -- The ID of the stalled item. + """ + for handler in self.handlers['cancelled_notice']: + handler(feed, id) + + def finished_notice(self, feed, id, result): + """ + Generate a notice that a job has finished, and + execute any relevant event handlers. + + Arguments: + feed -- The name of the feed. + id -- The ID of the stalled item. + """ + for handler in self.handlers['finished_notice']: + handler(feed, id, result) + + def claimed_notice(self, feed, id): + """ + Generate a notice that a job has been claimed, and + execute any relevant event handlers. + + Arguments: + feed -- The name of the feed. + id -- The ID of the stalled item. + """ + for handler in self.handlers['claimed_notice']: + handler(feed, id) + \ No newline at end of file From 332b4613b125e5323048c059f10104d6e5b9948a Mon Sep 17 00:00:00 2001 From: Simon Hewitt Date: Wed, 27 Jul 2011 00:21:37 +0100 Subject: [PATCH 06/17] updated get_feeds to update from redis --- thoonk/pubsub.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/thoonk/pubsub.py b/thoonk/pubsub.py index 7c30566..b8b0ded 100644 --- a/thoonk/pubsub.py +++ b/thoonk/pubsub.py @@ -304,6 +304,7 @@ def get_feeds(self): Returns: set """ + self.feeds.update(self.redis.smembers('feeds')) return self.feeds def feed_exists(self, feed): @@ -346,11 +347,10 @@ def listen(self): self.lredis.subscribe((self.new_feed, self.del_feed, self.conf_feed)) # get set of feeds - self.feeds.update(self.redis.smembers('feeds')) - if self.feeds: - # subscribe to exist feeds retract and publish - for feed in self.feeds: - self.lredis.subscribe(self[feed].get_channels()) + feeds = self.get_feeds() + # subscribe to exist feeds retract and publish + for feed in self.feeds: + self.lredis.subscribe(self[feed].get_channels()) self.listen_ready.set() for event in self.lredis.listen(): From 6defac447f6ea73802c29e08da565d3977839893 Mon Sep 17 00:00:00 2001 From: Simon Hewitt Date: Wed, 27 Jul 2011 13:10:41 +0100 Subject: [PATCH 07/17] updated gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 488838e..04712b9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ *.pyc .*.swp build/ +.project +.pydevproject +.settings From e4809d4b26b8b7b4cdf7cc5db6ce25c7437bb3ac Mon Sep 17 00:00:00 2001 From: Simon Hewitt Date: Wed, 3 Aug 2011 18:00:34 +0100 Subject: [PATCH 08/17] reverted job queue name to publishes to be inline with spec --- thoonk/feeds/job.py | 16 ++++++++-------- thoonk/pubsub.py | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/thoonk/feeds/job.py b/thoonk/feeds/job.py index bb4303b..c6d7c1d 100644 --- a/thoonk/feeds/job.py +++ b/thoonk/feeds/job.py @@ -92,7 +92,7 @@ def __init__(self, thoonk, feed, config=None): """ Queue.__init__(self, thoonk, feed, config=None) - #self.feed_publish = 'feed.publish:%s' % feed + self.feed_publishes = 'feed.publishes:%s' % feed self.feed_cancelled = 'feed.cancelled:%s' % feed self.feed_retried = 'feed.retried:%s' % feed self.feed_finished = 'feed.finished:%s' % feed @@ -102,7 +102,7 @@ def __init__(self, thoonk, feed, config=None): self.feed_job_running = 'feed.running:%s' % feed def get_channels(self): - return (self.feed_publish, self.feed_job_claimed, self.feed_job_stalled, + return (self.feed_publishes, self.feed_job_claimed, self.feed_job_stalled, self.feed_finished, self.feed_cancelled, self.feed_retried) def get_schemas(self): @@ -110,7 +110,7 @@ def get_schemas(self): schema = set((self.feed_job_claimed, self.feed_job_stalled, self.feed_job_running, - self.feed_publish, + self.feed_publishes, self.feed_cancelled)) for id in self.get_ids(): @@ -135,7 +135,7 @@ def retract(self, id): pipe = self.redis.pipeline() pipe.hdel(self.feed_items, id) pipe.hdel(self.feed_cancelled, id) - pipe.zrem(self.feed_publish, id) + pipe.zrem(self.feed_publishes, id) pipe.srem(self.feed_job_stalled, id) pipe.zrem(self.feed_job_claimed, id) pipe.lrem(self.feed_ids, 1, id) @@ -172,13 +172,13 @@ def put(self, item, priority=False): pipe.lpush(self.feed_ids, id) pipe.incr(self.feed_publishes) pipe.hset(self.feed_items, id, item) - pipe.zadd(self.feed_publish, id, time.time()) + pipe.zadd(self.feed_publishes, id, time.time()) results = pipe.execute() if results[-1]: # If zadd was successful - self.thoonk._publish(self.feed_publish, (id, item)) + self.thoonk._publish(self.feed_publishes, (id, item)) else: self.thoonk._publish(self.feed_edit, (id, item)) @@ -305,7 +305,7 @@ def stall(self, id): pipe.zrem(self.feed_job_claimed, id) pipe.hdel(self.feed_cancelled, id) pipe.sadd(self.feed_job_stalled, id) - pipe.zrem(self.feed_publish, id) + pipe.zrem(self.feed_publishes, id) try: pipe.execute() self.thoonk._publish(self.feed_job_stalled, (id,)) @@ -329,7 +329,7 @@ def retry(self, id): pipe = self.redis.pipeline() pipe.srem(self.feed_job_stalled, id) pipe.lpush(self.feed_ids, id) - pipe.zadd(self.feed_publish, time.time(), id) + pipe.zadd(self.feed_publishes, time.time(), id) try: results = pipe.execute() self.thoonk._publish(self.feed_retried, (id,)) diff --git a/thoonk/pubsub.py b/thoonk/pubsub.py index b8b0ded..84d0f27 100644 --- a/thoonk/pubsub.py +++ b/thoonk/pubsub.py @@ -388,7 +388,7 @@ def listen(self): self.feeds.add(name) config = self.get_config(name) if config["type"] == "job": - self.lredis.subscribe((self.feed_publish % name, + self.lredis.subscribe(("feed.publishes:%s" % name, "feed.cancelled:%s" % name, "feed.claimed:%s" % name, "feed.finished:%s" % name, From 2f4b9a2af40f4bace3a20f8192d8b4b92cb01e69 Mon Sep 17 00:00:00 2001 From: Simon Hewitt Date: Fri, 30 Sep 2011 18:14:21 +0100 Subject: [PATCH 09/17] made thoonk redis 2.4.9 compatible, fixed various unit tests --- test.cfg | 8 +- tests/test_feed.py | 14 ++- tests/test_job.py | 11 +-- tests/test_notice.py | 23 +++-- tests/test_queue.py | 6 +- tests/test_sorted_feed.py | 90 ++++++++++++------ thoonk/__init__.py | 4 +- thoonk/feeds/feed.py | 64 ++++++------- thoonk/feeds/job.py | 178 +++++++++++++++++------------------- thoonk/feeds/sorted_feed.py | 156 +++++++++++++++---------------- thoonk/pubsub.py | 58 +++++++----- 11 files changed, 322 insertions(+), 290 deletions(-) diff --git a/test.cfg b/test.cfg index ab55402..82c2ef0 100644 --- a/test.cfg +++ b/test.cfg @@ -4,7 +4,7 @@ # You were warned. # ===================================================================== -#[Test] -#host=localhost -#port=6379 -#db=10 +[Test] +host=localhost +port=6379 +db=10 diff --git a/tests/test_feed.py b/tests/test_feed.py index e83bcbc..3bdd948 100644 --- a/tests/test_feed.py +++ b/tests/test_feed.py @@ -1,13 +1,12 @@ import thoonk +from thoonk.feeds import Feed import unittest from ConfigParser import ConfigParser class TestLeaf(unittest.TestCase): - def __init__(self, *args, **kwargs): - unittest.TestCase.__init__(self, *args, **kwargs) - + def setUp(self, *args, **kwargs): conf = ConfigParser() conf.read('test.cfg') if conf.sections() == ['Test']: @@ -22,6 +21,7 @@ def __init__(self, *args, **kwargs): def test_05_basic_retract(self): """Test adding and retracting an item.""" l = self.ps.feed("testfeed") + self.assertTrue(isinstance(l, Feed)) l.publish('foo', id='1') r = l.get_ids() v = l.get_all() @@ -46,6 +46,10 @@ def test_10_basic_feed(self): def test_20_basic_feed_items(self): """Test items match completely.""" l = self.ps.feed("testfeed") + l.publish("hi", id='1') + l.publish("bye", id='2') + l.publish("thanks", id='3') + l.publish("you're welcome", id='4') r = l.get_ids() self.assertEqual(r, ['1', '2', '3', '4'], "Queue results did not match publish: %s" % r) c = {} @@ -56,6 +60,10 @@ def test_20_basic_feed_items(self): def test_30_basic_feed_retract(self): """Testing item retract items match.""" l = self.ps.feed("testfeed") + l.publish("hi", id='1') + l.publish("bye", id='2') + l.publish("thanks", id='3') + l.publish("you're welcome", id='4') l.retract('3') r = l.get_ids() self.assertEqual(r, ['1', '2','4'], "Queue results did not match publish: %s" % r) diff --git a/tests/test_job.py b/tests/test_job.py index 95967ef..e0b2350 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -6,9 +6,7 @@ class TestJob(unittest.TestCase): - def __init__(self, *args, **kwargs): - unittest.TestCase.__init__(self, *args, **kwargs) - + def setUp(self, *args, **kwargs): conf = ConfigParser() conf.read('test.cfg') if conf.sections() == ['Test']: @@ -20,11 +18,6 @@ def __init__(self, *args, **kwargs): print 'No test configuration found in test.cfg' exit() - - def setUp(self): - self.ps = thoonk.Pubsub(db=10, listen=True) - self.ps.redis.flushdb() - def tearDown(self): self.ps.close() @@ -38,7 +31,7 @@ def test_10_basic_job(self): testjobworker = self.ps.job("testjob") id_worker, query_worker, cancelled = testjobworker.get(timeout=3) result_worker = math.sqrt(float(query_worker)) - testjobworker.finish(id_worker, result_worker, True) + testjobworker.finish(id_worker, str(result_worker), True) #publisher gets result query_publisher, result_publisher = testjob.get_result(id, 1) diff --git a/tests/test_notice.py b/tests/test_notice.py index dac741a..d8e4be9 100644 --- a/tests/test_notice.py +++ b/tests/test_notice.py @@ -1,26 +1,27 @@ import thoonk +from thoonk.feeds import Feed, Job import unittest import time +import redis from ConfigParser import ConfigParser class TestNotice(unittest.TestCase): - def __init__(self, *args, **kwargs): - unittest.TestCase.__init__(self, *args, **kwargs) - + def setUp(self): conf = ConfigParser() conf.read('test.cfg') if conf.sections() == ['Test']: + redis.Redis(host=conf.get('Test', 'host'), + port=conf.getint('Test', 'port'), + db=conf.getint('Test', 'db')).flushdb() self.ps = thoonk.Thoonk(host=conf.get('Test', 'host'), port=conf.getint('Test', 'port'), db=conf.getint('Test', 'db'), listen=True) - self.ps.redis.flushdb() else: print 'No test configuration found in test.cfg' exit() - def tearDown(self): self.ps.close() @@ -36,9 +37,9 @@ def received_handler(feed, item, id): notice_received[0] = True self.ps.register_handler('publish_notice', received_handler) - j = self.ps.feed("testfeed") - + time.sleep(1) + self.assertEqual(j.__class__, Feed) self.assertFalse(notice_received[0]) #publisher @@ -48,12 +49,12 @@ def received_handler(feed, item, id): i = 0 while not notice_received[0] and i < 3: i += 1 - time.sleep(1) + time.sleep(3) - self.assertEqual(ids[0], ids[1]) - self.assertTrue(notice_received[0], "Notice not received") + self.assertEqual(ids[1], ids[0]) + self.ps.remove_handler('publish_notice', received_handler) def test_10_job_notices(self): @@ -104,6 +105,8 @@ def do_wait(): self.ps.register_handler('finished_notice', finished_handler) j = self.ps.job("testjob") + self.assertEqual(j.__class__, Job) + time.sleep(1) # wait for newfeed notice to propagate to listener self.assertFalse(notices_received[0]) diff --git a/tests/test_queue.py b/tests/test_queue.py index 417a767..49ee583 100644 --- a/tests/test_queue.py +++ b/tests/test_queue.py @@ -1,13 +1,12 @@ import thoonk +from thoonk.feeds import Queue import unittest from ConfigParser import ConfigParser class TestQueue(unittest.TestCase): - def __init__(self, *args, **kwargs): - unittest.TestCase.__init__(self, *args, **kwargs) - + def setUp(self): conf = ConfigParser() conf.read('test.cfg') if conf.sections() == ['Test']: @@ -22,6 +21,7 @@ def __init__(self, *args, **kwargs): def test_basic_queue(self): """Test basic QUEUE publish and retrieve.""" q = self.ps.queue("testqueue") + self.assertEqual(q.__class__, Queue) q.put("10") q.put("20") q.put("30") diff --git a/tests/test_sorted_feed.py b/tests/test_sorted_feed.py index 40e3e1d..264cc1a 100644 --- a/tests/test_sorted_feed.py +++ b/tests/test_sorted_feed.py @@ -1,13 +1,12 @@ import thoonk +from thoonk.feeds import SortedFeed import unittest from ConfigParser import ConfigParser class TestLeaf(unittest.TestCase): - - def __init__(self, *args, **kwargs): - unittest.TestCase.__init__(self, *args, **kwargs) - + + def setUp(self): conf = ConfigParser() conf.read('test.cfg') if conf.sections() == ['Test']: @@ -18,10 +17,11 @@ def __init__(self, *args, **kwargs): else: print 'No test configuration found in test.cfg' exit() - + def test_10_basic_sorted_feed(self): """Test basic sorted feed publish and retrieve.""" l = self.ps.sorted_feed("testfeed") + self.assertEqual(l.__class__, SortedFeed) l.publish("hi") l.publish("bye") l.publish("thanks") @@ -38,40 +38,43 @@ def test_10_basic_sorted_feed(self): def test_20_sorted_feed_before(self): """Test addding an item before another item""" l = self.ps.sorted_feed("testfeed") - l.publish_before('3', 'foo') + l.publish("hi") + l.publish("bye") + l.publish_before('2', 'foo') r = l.get_ids() - self.assertEqual(r, ['1', '2', '5', '3', '4'], "Sorted feed results did not match: %s." % r) + self.assertEqual(r, ['1', '3', '2'], "Sorted feed results did not match: %s." % r) def test_30_sorted_feed_after(self): """Test adding an item after another item""" l = self.ps.sorted_feed("testfeed") - l.publish_after('3', 'foo') + l.publish("hi") + l.publish("bye") + l.publish_after('1', 'foo') r = l.get_ids() - self.assertEqual(r, ['1', '2', '5', '3', '6', '4'], "Sorted feed results did not match: %s." % r) + self.assertEqual(r, ['1', '3', '2'], "Sorted feed results did not match: %s." % r) def test_40_sorted_feed_prepend(self): """Test addding an item to the front of the sorted feed""" l = self.ps.sorted_feed("testfeed") + l.publish("hi") + l.publish("bye") l.prepend('bar') r = l.get_ids() - self.assertEqual(r, ['7', '1', '2', '5', '3', '6', '4'], + self.assertEqual(r, ['3', '1', '2'], "Sorted feed results don't match: %s" % r) def test_50_sorted_feed_edit(self): """Test editing an item in a sorted feed""" l = self.ps.sorted_feed("testfeed") - l.edit('6', 'bar') + l.publish("hi") + l.publish("bye") + l.edit('1', 'bar') r = l.get_ids() - v = l.get_item('6') + v = l.get_item('1') vs = l.get_items() - items = {'1': 'hi', - '2': 'bye', - '3': 'thanks', - '4': "you're welcome", - '5': 'foo', - '6': 'bar', - '7': 'bar'} - self.assertEqual(r, ['7', '1', '2', '5', '3', '6', '4'], + items = {'1': 'bar', + '2': 'bye'} + self.assertEqual(r, ['1', '2'], "Sorted feed results don't match: %s" % r) self.assertEqual(v, 'bar', "Items don't match: %s" % v) self.assertEqual(vs, items, "Sorted feed items don't match: %s" % vs) @@ -79,33 +82,62 @@ def test_50_sorted_feed_edit(self): def test_60_sorted_feed_retract(self): """Test retracting an item from a sorted feed""" l = self.ps.sorted_feed("testfeed") + l.publish("hi") + l.publish("bye") + l.publish("thanks") + l.publish("you're welcome") l.retract('3') r = l.get_ids() - self.assertEqual(r, ['7', '1', '2', '5', '6', '4'], + self.assertEqual(r, ['1', '2', '4'], "Sorted feed results don't match: %s" % r) - def test_70_sorted_feed_move(self): + def test_70_sorted_feed_move_first(self): """Test moving items around in the feed.""" l = self.ps.sorted_feed('testfeed') - l.move_first('6') + l.publish("hi") + l.publish("bye") + l.publish("thanks") + l.publish("you're welcome") + l.move_first('4') r = l.get_ids() - self.assertEqual(r, ['6', '7', '1', '2', '5', '4'], + self.assertEqual(r, ['4', '1', '2', '3'], "Sorted feed results don't match: %s" % r) - l.move_last('7') + def test_71_sorted_feed_move_last(self): + """Test moving items around in the feed.""" + l = self.ps.sorted_feed('testfeed') + l.publish("hi") + l.publish("bye") + l.publish("thanks") + l.publish("you're welcome") + l.move_last('2') r = l.get_ids() - self.assertEqual(r, ['6', '1', '2', '5', '4', '7'], + self.assertEqual(r, ['1', '3', '4', '2'], "Sorted feed results don't match: %s" % r) - l.move_before('2', '5') + def test_72_sorted_feed_move_before(self): + """Test moving items around in the feed.""" + l = self.ps.sorted_feed('testfeed') + l.publish("hi") + l.publish("bye") + l.publish("thanks") + l.publish("you're welcome") + l.move_before('1', '2') r = l.get_ids() - self.assertEqual(r, ['6', '1', '5', '2', '4', '7'], + self.assertEqual(r, ['2', '1', '3', '4'], "Sorted feed results don't match: %s" % r) + def test_73_sorted_feed_move_after(self): + """Test moving items around in the feed.""" + l = self.ps.sorted_feed('testfeed') + l.publish("hi") + l.publish("bye") + l.publish("thanks") + l.publish("you're welcome") l.move_after('1', '4') r = l.get_ids() - self.assertEqual(r, ['6', '1', '4', '5', '2', '7'], + self.assertEqual(r, ['1', '4', '2', '3'], "Sorted feed results don't match: %s" % r) diff --git a/thoonk/__init__.py b/thoonk/__init__.py index 7a63f2b..c532d1f 100644 --- a/thoonk/__init__.py +++ b/thoonk/__init__.py @@ -6,5 +6,5 @@ from pubsub import Thoonk from pubsub import Thoonk as Pubsub -__version__ = '1.0.0rc1' -__version_info__ = (1, 0, 0, 'rc1', 0) +__version__ = '1.0.0rc1a' +__version_info__ = (1, 0, 0, 'rc1a', 0) diff --git a/thoonk/feeds/feed.py b/thoonk/feeds/feed.py index ca32152..54d7a24 100644 --- a/thoonk/feeds/feed.py +++ b/thoonk/feeds/feed.py @@ -13,7 +13,7 @@ import Queue as queue from thoonk.exceptions import * - +import redis.exceptions class Feed(object): @@ -199,26 +199,28 @@ def publish(self, item, id=None): publish_id = id if publish_id is None: publish_id = uuid.uuid4().hex - while True: - self.redis.watch(self.feed_ids) - - max = int(self.config.get('max_length', 0)) - pipe = self.redis.pipeline() - if max > 0: - delete_ids = self.redis.zrange(self.feed_ids, 0, -max) - for id in delete_ids: - if id != publish_id: - pipe.zrem(self.feed_ids, id) - pipe.hdel(self.feed_items, id) - pipe.publish(self.feed_retract, id) - pipe.zadd(self.feed_ids, publish_id, time.time()) - pipe.incr(self.feed_publishes) - pipe.hset(self.feed_items, publish_id, item) - try: - results = pipe.execute() - break - except redis.exceptions.WatchError: - pass + with self.redis.pipeline() as pipe: + while True: + try: + pipe.watch(self.feed_ids) + max = int(self.config.get('max_length', 0)) + if max > 0: + delete_ids = pipe.zrange(self.feed_ids, 0, -max) + pipe.multi() + for id in delete_ids: + if id != publish_id: + pipe.zrem(self.feed_ids, id) + pipe.hdel(self.feed_items, id) + self.thoonk._publish(self.feed_retract, (id,), pipe) + else: + pipe.multi() + pipe.zadd(self.feed_ids, **{publish_id: time.time()}) + pipe.incr(self.feed_publishes) + pipe.hset(self.feed_items, publish_id, item) + results = pipe.execute() + break + except redis.exceptions.WatchError: + pass if results[-3]: # If zadd was successful @@ -235,18 +237,16 @@ def retract(self, id): Arguments: id -- The ID value of the item to remove. """ - while True: - self.redis.watch(self.feed_ids) - if self.redis.zrank(self.feed_ids, id) is not None: - pipe = self.redis.pipeline() - pipe.zrem(self.feed_ids, id) - pipe.hdel(self.feed_items, id) - pipe.publish(self.feed_retract, id) + with self.redis.pipeline() as pipe: + while True: try: - pipe.execute() + pipe.watch(self.feed_ids) + if pipe.zrank(self.feed_ids, id) is not None: + pipe.multi() + pipe.zrem(self.feed_ids, id) + pipe.hdel(self.feed_items, id) + self.thoonk._publish(self.feed_retract, (id,), pipe) + pipe.execute() return except redis.exceptions.WatchError: pass - else: - self.redis.unwatch() - break diff --git a/thoonk/feeds/job.py b/thoonk/feeds/job.py index c6d7c1d..1f3d7f4 100644 --- a/thoonk/feeds/job.py +++ b/thoonk/feeds/job.py @@ -129,25 +129,23 @@ def retract(self, id): Arguments: id -- The ID of the job to remove. """ - while True: - self.redis.watch(self.feed_items) - if self.redis.hexists(self.feed_items, id): - pipe = self.redis.pipeline() - pipe.hdel(self.feed_items, id) - pipe.hdel(self.feed_cancelled, id) - pipe.zrem(self.feed_publishes, id) - pipe.srem(self.feed_job_stalled, id) - pipe.zrem(self.feed_job_claimed, id) - pipe.lrem(self.feed_ids, 1, id) - pipe.delete(self.feed_job_finished % id) + with self.redis.pipeline() as pipe: + while True: try: - pipe.execute() + pipe.watch(self.feed_items) + if pipe.hexists(self.feed_items, id): + pipe.multi() + pipe.hdel(self.feed_items, id) + pipe.hdel(self.feed_cancelled, id) + pipe.zrem(self.feed_publishes, id) + pipe.srem(self.feed_job_stalled, id) + pipe.zrem(self.feed_job_claimed, id) + pipe.lrem(self.feed_ids, 1, id) + pipe.delete(self.feed_job_finished % id) + pipe.execute() return except redis.exceptions.WatchError: pass - else: - self.redis.unwatch() - break def put(self, item, priority=False): """ @@ -166,13 +164,11 @@ def put(self, item, priority=False): if priority: pipe.rpush(self.feed_ids, id) - pipe.hset(self.feed_items, id, item) - pipe.zadd(self.feed_publishes, id, time.time()) else: pipe.lpush(self.feed_ids, id) pipe.incr(self.feed_publishes) - pipe.hset(self.feed_items, id, item) - pipe.zadd(self.feed_publishes, id, time.time()) + pipe.hset(self.feed_items, id, item) + pipe.zadd(self.feed_publishes, **{id: time.time()}) results = pipe.execute() @@ -205,7 +201,7 @@ def get(self, timeout=0): id = id[1] pipe = self.redis.pipeline() - pipe.zadd(self.feed_job_claimed, id, time.time()) + pipe.zadd(self.feed_job_claimed, **{id: time.time()}) pipe.hget(self.feed_items, id) pipe.hget(self.feed_cancelled, id) result = pipe.execute() @@ -226,28 +222,27 @@ def finish(self, id, item=None, result=False, timeout=None): timeout -- Time in seconds to keep the result data. The default is to store data indefinitely until retrieved. """ - while True: - self.redis.watch(self.feed_job_claimed) - if self.redis.zrank(self.feed_job_claimed, id) is None: - self.redis.unwatch() - return # raise exception? - - query = self.redis.hget(self.feed_items, id) - - pipe = self.redis.pipeline() - pipe.zrem(self.feed_job_claimed, id) - pipe.hdel(self.feed_cancelled, id) - if result: - pipe.lpush(self.feed_job_finished % id, item) - if timeout is not None: - pipe.expire(self.feed_job_finished % id, timeout) - pipe.hdel(self.feed_items, id) - try: - pipe.execute() - self.thoonk._publish(self.feed_finished, (id, result if result else "")) - break - except redis.exceptions.WatchError: - pass + with self.redis.pipeline() as pipe: + while True: + try: + pipe.watch(self.feed_job_claimed) + if pipe.zrank(self.feed_job_claimed, id) is None: + return # raise exception? + #query = pipe.hget(self.feed_items, id) + pipe.multi() + pipe.zrem(self.feed_job_claimed, id) + pipe.hdel(self.feed_cancelled, id) + if result: + pipe.lpush(self.feed_job_finished % id, item) + if timeout is not None: + pipe.expire(self.feed_job_finished % id, timeout) + pipe.hdel(self.feed_items, id) + self.thoonk._publish(self.feed_finished, + (id, item if result else ""), pipe) + pipe.execute() + break + except redis.exceptions.WatchError: + pass def get_result(self, id, timeout=0): """ @@ -269,22 +264,21 @@ def cancel(self, id): Arguments: id -- The ID of the job to cancel. """ - while True: - self.redis.watch(self.feed_job_claimed) - if self.redis.zrank(self.feed_job_claimed, id) is None: - self.redis.unwatch() - return # raise exception? - - pipe = self.redis.pipeline() - pipe.hincrby(self.feed_cancelled, id, 1) - pipe.lpush(self.feed_ids, id) - pipe.zrem(self.feed_job_claimed, id) - try: - pipe.execute() - self.thoonk._publish(self.feed_cancelled, (id,)) - break - except redis.exceptions.WatchError: - pass + with self.redis.pipeline() as pipe: + while True: + try: + pipe.watch(self.feed_job_claimed) + if self.redis.zrank(self.feed_job_claimed, id) is None: + return # raise exception? + pipe.multi() + pipe.hincrby(self.feed_cancelled, id, 1) + pipe.lpush(self.feed_ids, id) + pipe.zrem(self.feed_job_claimed, id) + self.thoonk._publish(self.feed_cancelled, (id,), pipe) + pipe.execute() + break + except redis.exceptions.WatchError: + pass def stall(self, id): """ @@ -295,23 +289,22 @@ def stall(self, id): Arguments: id -- The ID of the job to pause. """ - while True: - self.redis.watch(self.feed_job_claimed) - if self.redis.zrank(self.feed_job_claimed, id) is None: - self.redis.unwatch() - return # raise exception? - - pipe = self.redis.pipeline() - pipe.zrem(self.feed_job_claimed, id) - pipe.hdel(self.feed_cancelled, id) - pipe.sadd(self.feed_job_stalled, id) - pipe.zrem(self.feed_publishes, id) - try: - pipe.execute() - self.thoonk._publish(self.feed_job_stalled, (id,)) - break - except redis.exceptions.WatchError: - pass + with self.redis.pipeline() as pipe: + while True: + try: + pipe.watch(self.feed_job_claimed) + if pipe.zrank(self.feed_job_claimed, id) is None: + return # raise exception? + pipe.multi() + pipe.zrem(self.feed_job_claimed, id) + pipe.hdel(self.feed_cancelled, id) + pipe.sadd(self.feed_job_stalled, id) + pipe.zrem(self.feed_publishes, id) + self.thoonk._publish(self.feed_job_stalled, (id,), pipe) + pipe.execute() + break + except redis.exceptions.WatchError: + pass def retry(self, id): """ @@ -320,24 +313,23 @@ def retry(self, id): Arguments: id -- The ID of the job to resume. """ - while True: - self.redis.watch(self.feed_job_stalled) - if self.redis.sismember(self.feed_job_stalled, id) is None: - self.redis.unwatch() - return # raise exception? - - pipe = self.redis.pipeline() - pipe.srem(self.feed_job_stalled, id) - pipe.lpush(self.feed_ids, id) - pipe.zadd(self.feed_publishes, time.time(), id) - try: - results = pipe.execute() - self.thoonk._publish(self.feed_retried, (id,)) - if not results[0]: - return # raise exception? - break - except redis.exceptions.WatchError: - pass + with self.redis.pipeline() as pipe: + while True: + try: + pipe.watch(self.feed_job_stalled) + if pipe.sismember(self.feed_job_stalled, id) is None: + break # raise exception? + pipe.multi() + pipe.srem(self.feed_job_stalled, id) + pipe.lpush(self.feed_ids, id) + pipe.zadd(self.feed_publishes, **{id: time.time()}) + self.thoonk._publish(self.feed_retried, (id,), pipe) + results = pipe.execute() + if not results[0]: + return # raise exception? + break + except redis.exceptions.WatchError: + pass def maintenance(self): """ diff --git a/thoonk/feeds/sorted_feed.py b/thoonk/feeds/sorted_feed.py index 9eb6eee..3c9f206 100644 --- a/thoonk/feeds/sorted_feed.py +++ b/thoonk/feeds/sorted_feed.py @@ -3,9 +3,9 @@ Released under the terms of the MIT License """ -from thoonk.exceptions import * -from thoonk.feeds import * - +#from thoonk.exceptions import * +from thoonk.feeds import Feed +import redis.exceptions class SortedFeed(Feed): @@ -87,8 +87,8 @@ def prepend(self, item): pipe.lpush(self.feed_ids, id) pipe.incr(self.feed_publishes) pipe.hset(self.feed_items, id, item) - pipe.publish(self.feed_publish, '%s\x00%s' % (id, item)) - pipe.publish(self.feed_position, '%s\x00%s' % (id, 'begin:')) + self.thoonk._publish(self.feed_publish, (str(id), item), pipe) + self.thoonk._publish(self.feed_position, (str(id), 'begin:'), pipe) pipe.execute() return id @@ -105,27 +105,25 @@ def __insert(self, item, rel_id, method): to rel_id. """ id = self.redis.incr(self.feed_id_incr) - while True: - self.redis.watch(self.feed_items) - if not self.redis.hexists(self.feed_items, rel_id): - self.redis.unwatch() - return # raise exception? - - pipe = self.redis.pipeline() - pipe.linsert(self.feed_ids, method, rel_id, id) - pipe.hset(self.feed_items, id, item) - pipe.publish(self.feed_publish, '%s\x00%s' % (id, item)) - if method == 'BEFORE': - rel_id = ':%s' % rel_id - else: - rel_id = '%s:' % rel_id - pipe.publish(self.feed_position, '%s\x00%s' % (id, rel_id)) - - try: - pipe.execute() - break - except redis.exceptions.WatchError: - pass + if method == 'BEFORE': + pos_rel_id = ':%s' % rel_id + else: + pos_rel_id = '%s:' % rel_id + with self.redis.pipeline() as pipe: + while True: + try: + pipe.watch(self.feed_items) + if not pipe.hexists(self.feed_items, rel_id): + return # raise exception? + pipe.multi() + pipe.linsert(self.feed_ids, method, rel_id, id) + pipe.hset(self.feed_items, id, item) + self.thoonk._publish(self.feed_publish, (str(id), item), pipe) + self.thoonk._publish(self.feed_position, (str(id), pos_rel_id), pipe) + pipe.execute() + break + except redis.exceptions.WatchError: + pass return id def publish(self, item): @@ -142,8 +140,8 @@ def publish(self, item): pipe.rpush(self.feed_ids, id) pipe.incr(self.feed_publishes) pipe.hset(self.feed_items, id, item) - pipe.publish(self.feed_publish, '%s\x00%s' % (id, item)) - pipe.publish(self.feed_position, '%s\x00%s' % (id, ':end')) + self.thoonk._publish(self.feed_publish, (str(id), item), pipe) + self.thoonk._publish(self.feed_position, (str(id), ':end'), pipe) pipe.execute() return id @@ -155,22 +153,20 @@ def edit(self, id, item): id -- The ID value of the item to edit. item -- The new contents of the item. """ - while True: - self.redis.watch(self.feed_items) - if not self.redis.hexists(self.feed_items, id): - self.redis.unwatch() - return # raise exception? - - pipe = self.redis.pipeline() - pipe.hset(self.feed_items, id, item) - pipe.incr(self.feed_publishes) - pipe.publish(self.feed_publish, '%s\x00%s' % (id, item)) - - try: - pipe.execute() - break - except redis.exceptions.WatchError: - pass + with self.redis.pipeline() as pipe: + while True: + try: + pipe.watch(self.feed_items) + if not pipe.hexists(self.feed_items, id): + return # raise exception? + pipe.multi() + pipe.hset(self.feed_items, id, item) + pipe.incr(self.feed_publishes) + pipe.publish(self.feed_publish, '%s\x00%s' % (id, item)) + pipe.execute() + break + except redis.exceptions.WatchError: + pass def publish_before(self, before_id, item): """ @@ -216,33 +212,31 @@ def move(self, rel_position, id): else: raise ValueError('Relative ID formatted incorrectly') - while True: - self.redis.watch(self.feed_items) - if not self.redis.hexists(self.feed_items, id): - self.redis.unwatch() - break - if rel_id not in ['begin', 'end'] and \ - not self.redis.hexists(self.feed_items, rel_id): - self.redis.unwatch() - break - - pipe = self.redis.pipeline() - pipe.lrem(self.feed_ids, id, 1) - if rel_id == 'begin': - pipe.lpush(self.feed_ids, id) - elif rel_id == 'end': - pipe.rpush(self.feed_ids, id) - else: - pipe.linsert(self.feed_ids, dir, rel_id, id) - - pipe.publish(self.feed_position, - '%s\x00%s' % (id, rel_position)) - - try: - pipe.execute() - break - except redis.exceptions.WatchError: - pass + with self.redis.pipeline() as pipe: + while True: + try: + pipe.watch(self.feed_items) + if not pipe.hexists(self.feed_items, id): + break + if rel_id not in ['begin', 'end'] and \ + not pipe.hexists(self.feed_items, rel_id): + break + pipe.multi() + pipe.lrem(self.feed_ids, id, 1) + if rel_id == 'begin': + pipe.lpush(self.feed_ids, id) + elif rel_id == 'end': + pipe.rpush(self.feed_ids, id) + else: + pipe.linsert(self.feed_ids, dir, rel_id, id) + + pipe.publish(self.feed_position, + '%s\x00%s' % (id, rel_position)) + + pipe.execute() + break + except redis.exceptions.WatchError: + pass def move_before(self, rel_id, id): """ @@ -289,21 +283,19 @@ def retract(self, id): Arguments: id -- The ID value of the item to remove. """ - while True: - self.redis.watch(self.feed_items) - if self.redis.hexists(self.feed_items, id): - pipe = self.redis.pipeline() - pipe.lrem(self.feed_ids, id, 1) - pipe.hdel(self.feed_items, id) - pipe.publish(self.feed_retract, id) + with self.redis.pipeline() as pipe: + while True: try: - pipe.execute() + pipe.watch(self.feed_items) + if pipe.hexists(self.feed_items, id): + pipe.multi() + pipe.lrem(self.feed_ids, id, 1) + pipe.hdel(self.feed_items, id) + pipe.publish(self.feed_retract, id) + pipe.execute() break except redis.exceptions.WatchError: pass - else: - self.redis.unwatch() - return def get_ids(self): """Return the set of IDs used by items in the feed.""" diff --git a/thoonk/pubsub.py b/thoonk/pubsub.py index 84d0f27..ff08712 100644 --- a/thoonk/pubsub.py +++ b/thoonk/pubsub.py @@ -125,15 +125,21 @@ def __init__(self, host='localhost', port=6379, db=0, listen=False): self.lthread.start() self.listen_ready.wait() - def _publish(self, schema, items): + def _publish(self, schema, items, pipe=None): """ A shortcut method to publish items separated by \x00. Arguments: schema -- The key to publish the items to. items -- A tuple or list of items to publish. + pipe -- A redis pipeline to use to publish the item using. + Note: it is up to the caller to execute the pipe after + publishing """ - self.redis.publish(schema, "\x00".join(items)) + if pipe: + pipe.publish(schema, "\x00".join(items)) + else: + self.redis.publish(schema, "\x00".join(items)) def __getitem__(self, feed): """ @@ -253,21 +259,21 @@ def delete_feed(self, feed): feed -- The name of the feed. """ feed_instance = self._feed_config[feed] - deleted = False - while not deleted: - self.redis.watch('feeds') - if not self.feed_exists(feed): - return FeedDoesNotExist - pipe = self.redis.pipeline() - pipe.srem("feeds", feed) - for key in feed_instance.get_schemas(): - pipe.delete(key) - self._publish(self.del_feed, (feed, self._feed_config.instance)) - try: - pipe.execute() - deleted = True - except redis.exceptions.WatchError: - deleted = False + with self.redis.pipeline() as pipe: + while True: + try: + pipe.watch('feeds') + if not pipe.sismember('feeds', feed): + raise FeedDoesNotExist + pipe.multi() + pipe.srem("feeds", feed) + for key in feed_instance.get_schemas(): + pipe.delete(key) + self._publish(self.del_feed, (feed, self._feed_config.instance)) + pipe.execute() + break + except redis.exceptions.WatchError: + pass def set_config(self, feed, config): """ @@ -314,6 +320,7 @@ def feed_exists(self, feed): Arguments: feed -- The name of the feed. """ + return self.redis.sismember('feeds', feed) if not self.listening: if not feed in self.feeds: if self.redis.sismember('feeds', feed): @@ -326,9 +333,7 @@ def feed_exists(self, feed): def close(self): """Terminate the listening Redis connection.""" - self.redis.connection.disconnect() - if self.listening: - self.lredis.connection.disconnect() + self.redis.connection_pool.disconnect() def listen(self): """ @@ -341,7 +346,7 @@ def listen(self): - Item retractions. """ # listener redis object - self.lredis = redis.Redis(host=self.host, port=self.port, db=self.db) + self.lredis = self.redis.pubsub() # subscribe to feed activities channel self.lredis.subscribe((self.new_feed, self.del_feed, self.conf_feed)) @@ -353,7 +358,12 @@ def listen(self): self.lredis.subscribe(self[feed].get_channels()) self.listen_ready.set() - for event in self.lredis.listen(): + while True: + a = self.lredis.listen() + try: + event = a.next() + except: + break if event['type'] == 'message': if event['channel'].startswith('feed.publish'): #feed publish event @@ -364,8 +374,9 @@ def listen(self): self.retract_notice(event['channel'].split(':', 1)[-1], event['data']) elif event['channel'].startswith('feed.position'): + id, rel_id = event['data'].split('\x00', 1) self.position_notice(event['channel'].split(':', 1)[-1], - event['data']) + id, rel_id) elif event['channel'].startswith('feed.claimed'): self.claimed_notice(event['channel'].split(':', 1)[-1], event['data']) @@ -391,6 +402,7 @@ def listen(self): self.lredis.subscribe(("feed.publishes:%s" % name, "feed.cancelled:%s" % name, "feed.claimed:%s" % name, + "feed.retried:%s" % name, "feed.finished:%s" % name, "feed.stalled:%s" % name,)) else: From 72377fd8b335e674a8f785bf22fb8f7c86ba3caa Mon Sep 17 00:00:00 2001 From: Simon Hewitt Date: Thu, 17 Nov 2011 14:50:36 +0000 Subject: [PATCH 10/17] updated to use StrictRedis redis.transaction to remove boilerplate --- thoonk/feeds/feed.py | 64 +++++++--------- thoonk/feeds/job.py | 147 +++++++++++++++--------------------- thoonk/feeds/sorted_feed.py | 114 +++++++++++----------------- thoonk/pubsub.py | 28 +++---- 4 files changed, 147 insertions(+), 206 deletions(-) diff --git a/thoonk/feeds/feed.py b/thoonk/feeds/feed.py index 54d7a24..6034075 100644 --- a/thoonk/feeds/feed.py +++ b/thoonk/feeds/feed.py @@ -199,29 +199,26 @@ def publish(self, item, id=None): publish_id = id if publish_id is None: publish_id = uuid.uuid4().hex - with self.redis.pipeline() as pipe: - while True: - try: - pipe.watch(self.feed_ids) - max = int(self.config.get('max_length', 0)) - if max > 0: - delete_ids = pipe.zrange(self.feed_ids, 0, -max) - pipe.multi() - for id in delete_ids: - if id != publish_id: - pipe.zrem(self.feed_ids, id) - pipe.hdel(self.feed_items, id) - self.thoonk._publish(self.feed_retract, (id,), pipe) - else: - pipe.multi() - pipe.zadd(self.feed_ids, **{publish_id: time.time()}) - pipe.incr(self.feed_publishes) - pipe.hset(self.feed_items, publish_id, item) - results = pipe.execute() - break - except redis.exceptions.WatchError: - pass - + + max = int(self.config.get('max_length', 0)) + + def _publish(pipe): + if max > 0: + delete_ids = pipe.zrange(self.feed_ids, 0, -max) + pipe.multi() + for id in delete_ids: + if id != publish_id: + pipe.zrem(self.feed_ids, id) + pipe.hdel(self.feed_items, id) + self.thoonk._publish(self.feed_retract, (id,), pipe) + else: + pipe.multi() + pipe.zadd(self.feed_ids, **{publish_id: time.time()}) + pipe.incr(self.feed_publishes) + pipe.hset(self.feed_items, publish_id, item) + + results = self.redis.transaction(_publish, self.feed_ids) + if results[-3]: # If zadd was successful self.thoonk._publish(self.feed_publish, (publish_id, item)) @@ -237,16 +234,11 @@ def retract(self, id): Arguments: id -- The ID value of the item to remove. """ - with self.redis.pipeline() as pipe: - while True: - try: - pipe.watch(self.feed_ids) - if pipe.zrank(self.feed_ids, id) is not None: - pipe.multi() - pipe.zrem(self.feed_ids, id) - pipe.hdel(self.feed_items, id) - self.thoonk._publish(self.feed_retract, (id,), pipe) - pipe.execute() - return - except redis.exceptions.WatchError: - pass + def _retract(pipe): + if pipe.zrank(self.feed_ids, id) is not None: + pipe.multi() + pipe.zrem(self.feed_ids, id) + pipe.hdel(self.feed_items, id) + self.thoonk._publish(self.feed_retract, (id,), pipe) + + self.redis.transaction(_retract, self.feed_ids) \ No newline at end of file diff --git a/thoonk/feeds/job.py b/thoonk/feeds/job.py index 1f3d7f4..d7da695 100644 --- a/thoonk/feeds/job.py +++ b/thoonk/feeds/job.py @@ -129,23 +129,18 @@ def retract(self, id): Arguments: id -- The ID of the job to remove. """ - with self.redis.pipeline() as pipe: - while True: - try: - pipe.watch(self.feed_items) - if pipe.hexists(self.feed_items, id): - pipe.multi() - pipe.hdel(self.feed_items, id) - pipe.hdel(self.feed_cancelled, id) - pipe.zrem(self.feed_publishes, id) - pipe.srem(self.feed_job_stalled, id) - pipe.zrem(self.feed_job_claimed, id) - pipe.lrem(self.feed_ids, 1, id) - pipe.delete(self.feed_job_finished % id) - pipe.execute() - return - except redis.exceptions.WatchError: - pass + def _retract(pipe): + if pipe.hexists(self.feed_items, id): + pipe.multi() + pipe.hdel(self.feed_items, id) + pipe.hdel(self.feed_cancelled, id) + pipe.zrem(self.feed_publishes, id) + pipe.srem(self.feed_job_stalled, id) + pipe.zrem(self.feed_job_claimed, id) + pipe.lrem(self.feed_ids, 1, id) + pipe.delete(self.feed_job_finished % id) + + self.redis.transaction(_retract, self.feed_items) def put(self, item, priority=False): """ @@ -222,27 +217,22 @@ def finish(self, id, item=None, result=False, timeout=None): timeout -- Time in seconds to keep the result data. The default is to store data indefinitely until retrieved. """ - with self.redis.pipeline() as pipe: - while True: - try: - pipe.watch(self.feed_job_claimed) - if pipe.zrank(self.feed_job_claimed, id) is None: - return # raise exception? - #query = pipe.hget(self.feed_items, id) - pipe.multi() - pipe.zrem(self.feed_job_claimed, id) - pipe.hdel(self.feed_cancelled, id) - if result: - pipe.lpush(self.feed_job_finished % id, item) - if timeout is not None: - pipe.expire(self.feed_job_finished % id, timeout) - pipe.hdel(self.feed_items, id) - self.thoonk._publish(self.feed_finished, - (id, item if result else ""), pipe) - pipe.execute() - break - except redis.exceptions.WatchError: - pass + def _finish(pipe): + if pipe.zrank(self.feed_job_claimed, id) is None: + return # raise exception? + #query = pipe.hget(self.feed_items, id) + pipe.multi() + pipe.zrem(self.feed_job_claimed, id) + pipe.hdel(self.feed_cancelled, id) + if result: + pipe.lpush(self.feed_job_finished % id, item) + if timeout is not None: + pipe.expire(self.feed_job_finished % id, timeout) + pipe.hdel(self.feed_items, id) + self.thoonk._publish(self.feed_finished, + (id, item if result else ""), pipe) + + self.redis.transaction(_finish, self.feed_job_claimed) def get_result(self, id, timeout=0): """ @@ -264,21 +254,16 @@ def cancel(self, id): Arguments: id -- The ID of the job to cancel. """ - with self.redis.pipeline() as pipe: - while True: - try: - pipe.watch(self.feed_job_claimed) - if self.redis.zrank(self.feed_job_claimed, id) is None: - return # raise exception? - pipe.multi() - pipe.hincrby(self.feed_cancelled, id, 1) - pipe.lpush(self.feed_ids, id) - pipe.zrem(self.feed_job_claimed, id) - self.thoonk._publish(self.feed_cancelled, (id,), pipe) - pipe.execute() - break - except redis.exceptions.WatchError: - pass + def _cancel(pipe): + if self.redis.zrank(self.feed_job_claimed, id) is None: + return # raise exception? + pipe.multi() + pipe.hincrby(self.feed_cancelled, id, 1) + pipe.lpush(self.feed_ids, id) + pipe.zrem(self.feed_job_claimed, id) + self.thoonk._publish(self.feed_cancelled, (id,), pipe) + + self.redis.transaction(_cancel, self.feed_job_claimed) def stall(self, id): """ @@ -289,22 +274,17 @@ def stall(self, id): Arguments: id -- The ID of the job to pause. """ - with self.redis.pipeline() as pipe: - while True: - try: - pipe.watch(self.feed_job_claimed) - if pipe.zrank(self.feed_job_claimed, id) is None: - return # raise exception? - pipe.multi() - pipe.zrem(self.feed_job_claimed, id) - pipe.hdel(self.feed_cancelled, id) - pipe.sadd(self.feed_job_stalled, id) - pipe.zrem(self.feed_publishes, id) - self.thoonk._publish(self.feed_job_stalled, (id,), pipe) - pipe.execute() - break - except redis.exceptions.WatchError: - pass + def _stall(pipe): + if pipe.zrank(self.feed_job_claimed, id) is None: + return # raise exception? + pipe.multi() + pipe.zrem(self.feed_job_claimed, id) + pipe.hdel(self.feed_cancelled, id) + pipe.sadd(self.feed_job_stalled, id) + pipe.zrem(self.feed_publishes, id) + self.thoonk._publish(self.feed_job_stalled, (id,), pipe) + + self.redis.transaction(_stall, self.feed_job_claimed) def retry(self, id): """ @@ -313,23 +293,18 @@ def retry(self, id): Arguments: id -- The ID of the job to resume. """ - with self.redis.pipeline() as pipe: - while True: - try: - pipe.watch(self.feed_job_stalled) - if pipe.sismember(self.feed_job_stalled, id) is None: - break # raise exception? - pipe.multi() - pipe.srem(self.feed_job_stalled, id) - pipe.lpush(self.feed_ids, id) - pipe.zadd(self.feed_publishes, **{id: time.time()}) - self.thoonk._publish(self.feed_retried, (id,), pipe) - results = pipe.execute() - if not results[0]: - return # raise exception? - break - except redis.exceptions.WatchError: - pass + def _retry(pipe): + if pipe.sismember(self.feed_job_stalled, id) is None: + return # raise exception? + pipe.multi() + pipe.srem(self.feed_job_stalled, id) + pipe.lpush(self.feed_ids, id) + pipe.zadd(self.feed_publishes, **{id: time.time()}) + self.thoonk._publish(self.feed_retried, (id,), pipe) + + results = self.redis.transaction(_retry, self.feed_job_stalled) + if not results[0]: + return # raise exception? def maintenance(self): """ diff --git a/thoonk/feeds/sorted_feed.py b/thoonk/feeds/sorted_feed.py index 3c9f206..051816d 100644 --- a/thoonk/feeds/sorted_feed.py +++ b/thoonk/feeds/sorted_feed.py @@ -3,9 +3,7 @@ Released under the terms of the MIT License """ -#from thoonk.exceptions import * from thoonk.feeds import Feed -import redis.exceptions class SortedFeed(Feed): @@ -109,21 +107,17 @@ def __insert(self, item, rel_id, method): pos_rel_id = ':%s' % rel_id else: pos_rel_id = '%s:' % rel_id - with self.redis.pipeline() as pipe: - while True: - try: - pipe.watch(self.feed_items) - if not pipe.hexists(self.feed_items, rel_id): - return # raise exception? - pipe.multi() - pipe.linsert(self.feed_ids, method, rel_id, id) - pipe.hset(self.feed_items, id, item) - self.thoonk._publish(self.feed_publish, (str(id), item), pipe) - self.thoonk._publish(self.feed_position, (str(id), pos_rel_id), pipe) - pipe.execute() - break - except redis.exceptions.WatchError: - pass + + def _insert(pipe): + if not pipe.hexists(self.feed_items, rel_id): + return # raise exception? + pipe.multi() + pipe.linsert(self.feed_ids, method, rel_id, id) + pipe.hset(self.feed_items, id, item) + self.thoonk._publish(self.feed_publish, (str(id), item), pipe) + self.thoonk._publish(self.feed_position, (str(id), pos_rel_id), pipe) + + self.redis.transaction(_insert, self.feed_items) return id def publish(self, item): @@ -153,20 +147,15 @@ def edit(self, id, item): id -- The ID value of the item to edit. item -- The new contents of the item. """ - with self.redis.pipeline() as pipe: - while True: - try: - pipe.watch(self.feed_items) - if not pipe.hexists(self.feed_items, id): - return # raise exception? - pipe.multi() - pipe.hset(self.feed_items, id, item) - pipe.incr(self.feed_publishes) - pipe.publish(self.feed_publish, '%s\x00%s' % (id, item)) - pipe.execute() - break - except redis.exceptions.WatchError: - pass + def _edit(pipe): + if not pipe.hexists(self.feed_items, id): + return # raise exception? + pipe.multi() + pipe.hset(self.feed_items, id, item) + pipe.incr(self.feed_publishes) + pipe.publish(self.feed_publish, '%s\x00%s' % (id, item)) + + self.redis.transaction(_edit, self.feed_items) def publish_before(self, before_id, item): """ @@ -211,32 +200,26 @@ def move(self, rel_position, id): rel_id = rel_position[:-1] else: raise ValueError('Relative ID formatted incorrectly') - - with self.redis.pipeline() as pipe: - while True: - try: - pipe.watch(self.feed_items) - if not pipe.hexists(self.feed_items, id): - break - if rel_id not in ['begin', 'end'] and \ - not pipe.hexists(self.feed_items, rel_id): - break - pipe.multi() - pipe.lrem(self.feed_ids, id, 1) - if rel_id == 'begin': - pipe.lpush(self.feed_ids, id) - elif rel_id == 'end': - pipe.rpush(self.feed_ids, id) - else: - pipe.linsert(self.feed_ids, dir, rel_id, id) - pipe.publish(self.feed_position, - '%s\x00%s' % (id, rel_position)) + def _move(pipe): + if not pipe.hexists(self.feed_items, id): + return + if rel_id not in ['begin', 'end'] and \ + not pipe.hexists(self.feed_items, rel_id): + return + pipe.multi() + pipe.lrem(self.feed_ids, 1, id) + if rel_id == 'begin': + pipe.lpush(self.feed_ids, id) + elif rel_id == 'end': + pipe.rpush(self.feed_ids, id) + else: + pipe.linsert(self.feed_ids, dir, rel_id, id) + + pipe.publish(self.feed_position, + '%s\x00%s' % (id, rel_position)) - pipe.execute() - break - except redis.exceptions.WatchError: - pass + self.redis.transaction(_move, self.feed_items) def move_before(self, rel_id, id): """ @@ -283,19 +266,14 @@ def retract(self, id): Arguments: id -- The ID value of the item to remove. """ - with self.redis.pipeline() as pipe: - while True: - try: - pipe.watch(self.feed_items) - if pipe.hexists(self.feed_items, id): - pipe.multi() - pipe.lrem(self.feed_ids, id, 1) - pipe.hdel(self.feed_items, id) - pipe.publish(self.feed_retract, id) - pipe.execute() - break - except redis.exceptions.WatchError: - pass + def _retract(pipe): + if pipe.hexists(self.feed_items, id): + pipe.multi() + pipe.lrem(self.feed_ids, 1, id) + pipe.hdel(self.feed_items, id) + pipe.publish(self.feed_retract, id) + + self.redis.transaction(_retract, self.feed_items) def get_ids(self): """Return the set of IDs used by items in the feed.""" diff --git a/thoonk/pubsub.py b/thoonk/pubsub.py index ff08712..4f17c7f 100644 --- a/thoonk/pubsub.py +++ b/thoonk/pubsub.py @@ -84,7 +84,7 @@ def __init__(self, host='localhost', port=6379, db=0, listen=False): self.host = host self.port = port self.db = db - self.redis = redis.Redis(host=self.host, port=self.port, db=self.db) + self.redis = redis.StrictRedis(host=self.host, port=self.port, db=self.db) self.lredis = None self.feedtypes = {} @@ -259,21 +259,17 @@ def delete_feed(self, feed): feed -- The name of the feed. """ feed_instance = self._feed_config[feed] - with self.redis.pipeline() as pipe: - while True: - try: - pipe.watch('feeds') - if not pipe.sismember('feeds', feed): - raise FeedDoesNotExist - pipe.multi() - pipe.srem("feeds", feed) - for key in feed_instance.get_schemas(): - pipe.delete(key) - self._publish(self.del_feed, (feed, self._feed_config.instance)) - pipe.execute() - break - except redis.exceptions.WatchError: - pass + + def _delete_feed(pipe): + if not pipe.sismember('feeds', feed): + raise FeedDoesNotExist + pipe.multi() + pipe.srem("feeds", feed) + for key in feed_instance.get_schemas(): + pipe.delete(key) + self._publish(self.del_feed, (feed, self._feed_config.instance)) + + self.redis.transaction(_delete_feed, 'feeds') def set_config(self, feed, config): """ From 119ba6ebf0d1a17bc973c9ac6bd7ad0b362b5cb1 Mon Sep 17 00:00:00 2001 From: Simon Hewitt Date: Fri, 18 Nov 2011 16:08:00 +0000 Subject: [PATCH 11/17] Updated thoonk to the latest contracts Refactored the ConfigCache into FeedCache (it no longer references any config) Refactored the Listener thread into its own class Moved all exceptions into exceptions module --- contract.txt | 28 ++- tests/test_feed.py | 13 +- tests/test_job.py | 95 ++++++-- tests/test_notice.py | 95 +++++--- tests/test_sorted_feed.py | 20 +- thoonk/cache.py | 59 +++++ thoonk/config.py | 80 ------- thoonk/exceptions.py | 7 +- thoonk/feeds/feed.py | 48 +--- thoonk/feeds/job.py | 134 +++++------ thoonk/feeds/queue.py | 7 +- thoonk/feeds/sorted_feed.py | 4 +- thoonk/pubsub.py | 441 ++++++++++++------------------------ 13 files changed, 451 insertions(+), 580 deletions(-) create mode 100644 thoonk/cache.py delete mode 100644 thoonk/config.py diff --git a/contract.txt b/contract.txt index d1a133a..9b885cf 100644 --- a/contract.txt +++ b/contract.txt @@ -12,15 +12,19 @@ Delete Feed: PUBLISH delfeed [feed]\x00[instance uuid] EXEC // if nil: go back to WATCH -Set Config: - SET feed.config:[feed] //json(config) +Set Config Value: + HSET feed.config:[feed] name value PUBLISH conffeed [feed]\x00[instance uuid] +Get Config Value: + HGET feed.config:[feed] name + Feed: Publish: //id may be provided or generated. An id that has already been published will update that id + max = feed.config:[feed] max_length WATCH feed.ids:[feed] delete_ids = ZRANGE feed.ids:[feed] 0 [-max] // eg ZRANGE feed.ids:test 0 -5 if the max is 4 MULTI @@ -188,7 +192,7 @@ Job: LPUSH feed.ids:[feed] [id] INCR feed.publishes:[feed] HSET feed.items:[feed] [id] [item] - ZADD feed.published:[feed] [utc epoch] [id] + ZADD feed.published:[feed] [utc epoch milliseconds] [id] EXEC High Priority Put: @@ -196,14 +200,14 @@ Job: MULTI RPUSH feed.ids:[feed] [id] HSET feed.items:[feed] [id] [item] - ZADD feed.published:[feed] [utc epoch] [id] + ZADD feed.published:[feed] [utc epoch milliseconds] [id] EXEC Get: id = BRPOP feed.ids:[feed] [timeout] //if error/timeout, abort MULTI - ZADD feed.claimed:[feed] [utc epoch] [id] + ZADD feed.claimed:[feed] [utc epoch milliseconds] [id] item = HGET feed:items[feed] [id] EXEC //if the id fails to get from feed.ids to feed.claimed, the maintenance will notice eventually @@ -214,15 +218,12 @@ Job: MULTI ZREM feed.claimed:[feed] [id] HDEL feed.cancelled:[feed] [id] //just to make sure + INCR feed.finishes:[feed] //optionally if publishing a result: - LPUSH feed.jobfinished:[feed]\x00[id] [result] - EXPIRE feed.jobfinished:[feed]\x00[id] [timeout] + PUBLISH job.finish:[feed] [id]\x00[result] HDEL feed.items:[feed] [id] EXEC // if nil: go back to WATCH and try again - Get Result: - BRPOP feed:jobfinished:[feed]\x00[id] [timeout] - Get Ids: HKEYS feed.items:[feed] @@ -252,7 +253,7 @@ Job: SREM feed.stalled:[feed] [id] //if error, abort LPUSH feed.ids:[feed] [id] - ZADD feed.published:[feed] [utc epoch] [id] + ZADD feed.published:[feed] [utc epoch milliseconds] [id] EXEC // if nil retry Retract: @@ -267,6 +268,9 @@ Job: LREM feed.ids:[feed] 1 [id] EXEC // if fail, retry + getNumOfFailures: + HGET feed.cancelled:[feed] [id] + Maintenance: //maintain job queue -- only ran by one process per jobqueue on occassion -- still a bit hand-wavey MULTI keys = HKEYS feed.items:[feed] @@ -280,4 +284,4 @@ Job: LPUSH feed.ids:[feed] [key] check claimed jobs to see if any have been claimed too long and "Cancel" or "Stall" them - publish stats to a feed + publish stats to a feed \ No newline at end of file diff --git a/tests/test_feed.py b/tests/test_feed.py index 3bdd948..0a766e7 100644 --- a/tests/test_feed.py +++ b/tests/test_feed.py @@ -2,7 +2,7 @@ from thoonk.feeds import Feed import unittest from ConfigParser import ConfigParser - +import threading class TestLeaf(unittest.TestCase): @@ -12,16 +12,20 @@ def setUp(self, *args, **kwargs): if conf.sections() == ['Test']: self.ps = thoonk.Thoonk(host=conf.get('Test', 'host'), port=conf.getint('Test', 'port'), - db=conf.getint('Test', 'db')) + db=conf.getint('Test', 'db'), + listen=True) self.ps.redis.flushdb() else: print 'No test configuration found in test.cfg' exit() - + + def tearDown(self): + self.ps.close() + def test_05_basic_retract(self): """Test adding and retracting an item.""" l = self.ps.feed("testfeed") - self.assertTrue(isinstance(l, Feed)) + self.assertEqual(type(l), Feed) l.publish('foo', id='1') r = l.get_ids() v = l.get_all() @@ -76,6 +80,7 @@ def test_40_create_delete(self): """Testing feed delete""" l = self.ps.feed("test2") l.delete_feed() + def test_50_max_length(self): """Test feeds with a max length""" diff --git a/tests/test_job.py b/tests/test_job.py index e0b2350..83930c2 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -1,7 +1,7 @@ import thoonk import unittest -import math from ConfigParser import ConfigParser +import threading class TestJob(unittest.TestCase): @@ -22,48 +22,109 @@ def tearDown(self): self.ps.close() def test_10_basic_job(self): - """JOB publish, retrieve, finish, get result""" + """Test job publish, retrieve, finish flow""" #publisher testjob = self.ps.job("testjob") + self.assertEqual(testjob.get_ids(), []) + id = testjob.put('9.0') - + #worker - testjobworker = self.ps.job("testjob") - id_worker, query_worker, cancelled = testjobworker.get(timeout=3) - result_worker = math.sqrt(float(query_worker)) - testjobworker.finish(id_worker, str(result_worker), True) - - #publisher gets result - query_publisher, result_publisher = testjob.get_result(id, 1) - self.assertEqual(float(result_worker), float(result_publisher), "Job results did not match publish.") + id_worker, job_content, cancelled = testjob.get(timeout=3) + self.assertEqual(job_content, '9.0') + self.assertEqual(cancelled, 0) + self.assertEqual(id_worker, id) + testjob.finish(id_worker) + self.assertEqual(testjob.get_ids(), []) - + def test_20_cancel_job(self): """Test cancelling a job""" j = self.ps.job("testjob") #publisher id = j.put('9.0') #worker claims - id, query, cancelled = j.get() + id, job_content, cancelled = j.get() + self.assertEqual(job_content, '9.0') self.assertEqual(cancelled, 0) #publisher or worker cancels j.cancel(id) - id2, query2, cancelled2 = j.get() + id2, job_content2, cancelled2 = j.get() self.assertEqual(cancelled2, 1) + self.assertEqual(job_content2, '9.0') self.assertEqual(id, id2) #cancel the work again j.cancel(id) # check the cancelled increment again - id3, query3, cancelled3 = j.get() + id3, job_content3, cancelled3 = j.get() self.assertEqual(cancelled3, 2) + self.assertEqual(job_content3, '9.0') self.assertEqual(id, id3) #cleanup -- remove the job from the queue j.retract(id) self.assertEqual(j.get_ids(), []) def test_30_no_job(self): + """Test exception raise when job.get times out""" j = self.ps.job("testjob") - self.assertRaises(thoonk.feeds.queue.Empty, j.get, timeout=1) + self.assertEqual(j.get_ids(), []) + self.assertRaises(thoonk.exceptions.Empty, j.get, timeout=1) + +class TestJobResult(unittest.TestCase): + + def setUp(self, *args, **kwargs): + conf = ConfigParser() + conf.read('test.cfg') + if conf.sections() == ['Test']: + self.ps = thoonk.Thoonk(host=conf.get('Test', 'host'), + port=conf.getint('Test', 'port'), + db=conf.getint('Test', 'db'), + listen=True) + self.ps.redis.flushdb() + else: + print 'No test configuration found in test.cfg' + exit() + + def tearDown(self): + self.ps.close() + + def test_10_job_result(self): + """Test job result published""" + + create_event = threading.Event() + def create_handler(name): + self.assertEqual(name, "testjobresult") + create_event.set() + self.ps.register_handler("create", create_handler) + + #publisher + testjob = self.ps.job("testjobresult") + self.assertEqual(testjob.get_ids(), []) + + # Wait until the create event has been received by the ThoonkListener + create_event.wait() + + id = testjob.put('9.0') + + #worker + id_worker, job_content, cancelled = testjob.get(timeout=3) + self.assertEqual(job_content, '9.0') + self.assertEqual(cancelled, 0) + self.assertEqual(id_worker, id) + + result_event = threading.Event() + def result_handler(name, id, result): + self.assertEqual(name, "testjobresult") + self.assertEqual(id, id_worker) + self.assertEqual(result, "myresult") + result_event.set() + + self.ps.register_handler("finish", result_handler) + testjob.finish(id_worker, "myresult") + result_event.wait(1) + self.assertTrue(result_event.isSet(), "No result received!") + self.assertEqual(testjob.get_ids(), []) + self.ps.remove_handler("result", result_handler) -suite = unittest.TestLoader().loadTestsFromTestCase(TestJob) +#suite = unittest.TestLoader().loadTestsFromTestCase(TestJob) diff --git a/tests/test_notice.py b/tests/test_notice.py index d8e4be9..d7c715f 100644 --- a/tests/test_notice.py +++ b/tests/test_notice.py @@ -4,6 +4,7 @@ import time import redis from ConfigParser import ConfigParser +import threading class TestNotice(unittest.TestCase): @@ -26,38 +27,82 @@ def tearDown(self): self.ps.close() "claimed, cancelled, stalled, finished" - - def test_05_publish_notice(self): - notice_received = [False] + def test_01_feed_notices(self): + """Test for create, publish, edit, retract and delete notices from feeds""" + + """Feed Create Event""" + create_event = threading.Event() + def create_handler(feed): + self.assertEqual(feed, "test_notices") + create_event.set() + + self.ps.register_handler("create", create_handler) + l = self.ps.feed("test_notices") + create_event.wait(1) + self.assertTrue(create_event.isSet(), "Create notice not received") + self.ps.remove_handler("create", create_handler) + + """Feed Publish Event""" + publish_event = threading.Event() ids = [None, None] def received_handler(feed, item, id): - self.assertEqual(feed, "testfeed") + self.assertEqual(feed, "test_notices") ids[1] = id - notice_received[0] = True - - self.ps.register_handler('publish_notice', received_handler) - j = self.ps.feed("testfeed") - time.sleep(1) - self.assertEqual(j.__class__, Feed) - self.assertFalse(notice_received[0]) - - #publisher - ids[0] = j.publish('a') - - # block while waiting for notice - i = 0 - while not notice_received[0] and i < 3: - i += 1 - time.sleep(3) + publish_event.set() + + self.ps.register_handler('publish', received_handler) + ids[0] = l.publish('a') + publish_event.wait(1) - self.assertTrue(notice_received[0], "Notice not received") + self.assertTrue(publish_event.isSet(), "Publish notice not received") + self.assertEqual(ids[1], ids[0]) + self.ps.remove_handler('publish', received_handler) + """Feed Edit Event """ + edit_event = threading.Event() + def edit_handler(feed, item, id): + self.assertEqual(feed, "test_notices") + ids[1] = id + edit_event.set() + + self.ps.register_handler('edit', edit_handler) + l.publish('b', id=ids[0]) + edit_event.wait(1) + + self.assertTrue(edit_event.isSet(), "Edit notice not received") self.assertEqual(ids[1], ids[0]) + self.ps.remove_handler('edit', edit_handler) - self.ps.remove_handler('publish_notice', received_handler) + """Feed Retract Event""" + retract_event = threading.Event() + def retract_handler(feed, id): + self.assertEqual(feed, "test_notices") + ids[1] = id + retract_event.set() + + self.ps.register_handler('retract', retract_handler) + l.retract(ids[0]) + retract_event.wait(1) - def test_10_job_notices(self): + self.assertTrue(retract_event.isSet(), "Retract notice not received") + self.assertEqual(ids[1], ids[0]) + self.ps.remove_handler('retract', retract_handler) + + """Feed Delete Event""" + delete_event = threading.Event() + def delete_handler(feed): + self.assertEqual(feed, "test_notices") + delete_event.set() + + self.ps.register_handler("delete", delete_handler) + l.delete_feed() + delete_event.wait(1) + self.assertTrue(delete_event.isSet(), "Delete notice not received") + self.ps.remove_handler("delete", delete_handler) + + + def skiptest_10_job_notices(self): notices_received = [False] ids = [None, None] @@ -95,7 +140,7 @@ def do_wait(): i = 0 while not notices_received[-1] and i < 2: i += 1 - time.sleep(1) + time.sleep(0.2) self.ps.register_handler('publish_notice', publish_handler) self.ps.register_handler('claimed_notice', claimed_handler) @@ -106,8 +151,6 @@ def do_wait(): j = self.ps.job("testjob") self.assertEqual(j.__class__, Job) - time.sleep(1) # wait for newfeed notice to propagate to listener - self.assertFalse(notices_received[0]) # create the job diff --git a/tests/test_sorted_feed.py b/tests/test_sorted_feed.py index 264cc1a..20b8115 100644 --- a/tests/test_sorted_feed.py +++ b/tests/test_sorted_feed.py @@ -20,7 +20,7 @@ def setUp(self): def test_10_basic_sorted_feed(self): """Test basic sorted feed publish and retrieve.""" - l = self.ps.sorted_feed("testfeed") + l = self.ps.sorted_feed("sortedfeed") self.assertEqual(l.__class__, SortedFeed) l.publish("hi") l.publish("bye") @@ -37,7 +37,7 @@ def test_10_basic_sorted_feed(self): def test_20_sorted_feed_before(self): """Test addding an item before another item""" - l = self.ps.sorted_feed("testfeed") + l = self.ps.sorted_feed("sortedfeed") l.publish("hi") l.publish("bye") l.publish_before('2', 'foo') @@ -46,7 +46,7 @@ def test_20_sorted_feed_before(self): def test_30_sorted_feed_after(self): """Test adding an item after another item""" - l = self.ps.sorted_feed("testfeed") + l = self.ps.sorted_feed("sortedfeed") l.publish("hi") l.publish("bye") l.publish_after('1', 'foo') @@ -55,7 +55,7 @@ def test_30_sorted_feed_after(self): def test_40_sorted_feed_prepend(self): """Test addding an item to the front of the sorted feed""" - l = self.ps.sorted_feed("testfeed") + l = self.ps.sorted_feed("sortedfeed") l.publish("hi") l.publish("bye") l.prepend('bar') @@ -65,7 +65,7 @@ def test_40_sorted_feed_prepend(self): def test_50_sorted_feed_edit(self): """Test editing an item in a sorted feed""" - l = self.ps.sorted_feed("testfeed") + l = self.ps.sorted_feed("sortedfeed") l.publish("hi") l.publish("bye") l.edit('1', 'bar') @@ -81,7 +81,7 @@ def test_50_sorted_feed_edit(self): def test_60_sorted_feed_retract(self): """Test retracting an item from a sorted feed""" - l = self.ps.sorted_feed("testfeed") + l = self.ps.sorted_feed("sortedfeed") l.publish("hi") l.publish("bye") l.publish("thanks") @@ -93,7 +93,7 @@ def test_60_sorted_feed_retract(self): def test_70_sorted_feed_move_first(self): """Test moving items around in the feed.""" - l = self.ps.sorted_feed('testfeed') + l = self.ps.sorted_feed('sortedfeed') l.publish("hi") l.publish("bye") l.publish("thanks") @@ -105,7 +105,7 @@ def test_70_sorted_feed_move_first(self): def test_71_sorted_feed_move_last(self): """Test moving items around in the feed.""" - l = self.ps.sorted_feed('testfeed') + l = self.ps.sorted_feed('sortedfeed') l.publish("hi") l.publish("bye") l.publish("thanks") @@ -118,7 +118,7 @@ def test_71_sorted_feed_move_last(self): def test_72_sorted_feed_move_before(self): """Test moving items around in the feed.""" - l = self.ps.sorted_feed('testfeed') + l = self.ps.sorted_feed('sortedfeed') l.publish("hi") l.publish("bye") l.publish("thanks") @@ -130,7 +130,7 @@ def test_72_sorted_feed_move_before(self): def test_73_sorted_feed_move_after(self): """Test moving items around in the feed.""" - l = self.ps.sorted_feed('testfeed') + l = self.ps.sorted_feed('sortedfeed') l.publish("hi") l.publish("bye") l.publish("thanks") diff --git a/thoonk/cache.py b/thoonk/cache.py new file mode 100644 index 0000000..dd8e3f5 --- /dev/null +++ b/thoonk/cache.py @@ -0,0 +1,59 @@ +""" + Written by Nathan Fritz and Lance Stout. Copyright 2011 by &yet, LLC. + Released under the terms of the MIT License +""" + +import threading +import uuid +from thoonk.exceptions import FeedDoesNotExist + +class FeedCache(object): + + """ + The FeedCache class stores an in-memory version of each + feed. As there may be multiple systems using + Thoonk with the same Redis server, and each with its own + FeedCache instance, each FeedCache has a self.instance + field to uniquely identify itself. + + Attributes: + thoonk -- The main Thoonk object. + instance -- A hex string for uniquely identifying this + FeedCache instance. + + Methods: + invalidate -- Force a feed's config to be retrieved from + Redis instead of in-memory. + """ + + def __init__(self, thoonk): + """ + Create a new configuration cache. + + Arguments: + thoonk -- The main Thoonk object. + """ + self._feeds = {} + self.thoonk = thoonk + self.lock = threading.Lock() + + def __getitem__(self, feed): + """ + Return a feed object for a given feed name. + + Arguments: + feed -- The name of the requested feed. + """ + with self.lock: + if feed not in self._feeds: + feed_type = self.thoonk.redis.hget('feed.config:%s' % feed, "type") + if not feed_type: + raise FeedDoesNotExist + self._feeds[feed] = self.thoonk.feedtypes[feed_type](self.thoonk, feed) + return self._feeds[feed] + + def __delitem__(self, feed): + with self.lock: + if feed in self._feeds: + self._feeds[feed].delete() + del self._feeds[feed] diff --git a/thoonk/config.py b/thoonk/config.py deleted file mode 100644 index 7d1e4da..0000000 --- a/thoonk/config.py +++ /dev/null @@ -1,80 +0,0 @@ -""" - Written by Nathan Fritz and Lance Stout. Copyright 2011 by &yet, LLC. - Released under the terms of the MIT License -""" - -import json -import threading -import uuid - - -class ConfigCache(object): - - """ - The ConfigCache class stores an in-memory version of each - feed's configuration. As there may be multiple systems using - Thoonk with the same Redis server, and each with its own - ConfigCache instance, each ConfigCache has a self.instance - field to uniquely identify itself. - - Attributes: - thoonk -- The main Thoonk object. - instance -- A hex string for uniquely identifying this - ConfigCache instance. - - Methods: - invalidate -- Force a feed's config to be retrieved from - Redis instead of in-memory. - """ - - def __init__(self, thoonk): - """ - Create a new configuration cache. - - Arguments: - thoonk -- The main Thoonk object. - """ - self._feeds = {} - self.thoonk = thoonk - self.lock = threading.Lock() - self.instance = uuid.uuid4().hex - - def __getitem__(self, feed): - """ - Return a feed object for a given feed name. - - Arguments: - feed -- The name of the requested feed. - """ - with self.lock: - if feed in self._feeds: - return self._feeds[feed] - else: - if not self.thoonk.feed_exists(feed): - raise FeedDoesNotExist - config = self.thoonk.redis.get('feed.config:%s' % feed) - config = json.loads(config) - feed_type = config.get(u'type', u'feed') - feed_class = self.thoonk.feedtypes[feed_type] - self._feeds[feed] = feed_class(self.thoonk, feed, config) - return self._feeds[feed] - - def invalidate(self, feed, instance, delete=False): - """ - Delete a configuration so that it will be retrieved from Redis - instead of from the cache. - - Arguments: - feed -- The name of the feed to invalidate. - instance -- A UUID identifying the cache which made the - invalidation request. - delete -- Indicates if the entire feed object should be - invalidated, or just its configuration. - """ - if instance != self.instance: - with self.lock: - if feed in self._feeds: - if delete: - del self._feeds[feed] - else: - del self._feeds[feed].config diff --git a/thoonk/exceptions.py b/thoonk/exceptions.py index fe9da70..d808524 100644 --- a/thoonk/exceptions.py +++ b/thoonk/exceptions.py @@ -7,12 +7,11 @@ class FeedExists(Exception): pass -class NotAllowed(Exception): - pass - class FeedDoesNotExist(Exception): pass -class ItemDoesNotExist(Exception): +class Empty(Exception): pass +class NotListening(Exception): + pass \ No newline at end of file diff --git a/thoonk/feeds/feed.py b/thoonk/feeds/feed.py index 6034075..cde7556 100644 --- a/thoonk/feeds/feed.py +++ b/thoonk/feeds/feed.py @@ -3,8 +3,6 @@ Released under the terms of the MIT License """ -import json -import threading import time import uuid try: @@ -28,7 +26,6 @@ class Feed(object): thoonk -- The main Thoonk object. redis -- A Redis connection instance from the Thoonk object. feed -- The name of the feed. - config -- A dictionary of configuration values. Redis Keys Used: feed.ids:[feed] -- A sorted set of item IDs. @@ -54,7 +51,7 @@ class Feed(object): retract -- Remove an item from the feed. """ - def __init__(self, thoonk, feed, config=None): + def __init__(self, thoonk, feed): """ Create a new Feed object for a given Thoonk feed. @@ -67,12 +64,9 @@ def __init__(self, thoonk, feed, config=None): feed -- The name of the feed. config -- Optional dictionary of configuration values. """ - self.config_lock = threading.Lock() - self.config_valid = False self.thoonk = thoonk self.redis = thoonk.redis self.feed = feed - self._config = None self.feed_ids = 'feed.ids:%s' % feed self.feed_items = 'feed.items:%s' % feed @@ -114,39 +108,6 @@ def event_retract(self, id): """ pass - @property - def config(self): - """ - Return the feed's configuration. - - If the cached version is marked as invalid, then a new copy of - the config will be retrieved from Redis. - """ - with self.config_lock: - if not self.config_valid: - conf = self.redis.get(self.feed_config) or "{}" - self._config = json.loads(conf) - self.config_valid = True - return self._config - - @config.setter - def config(self, config): - """ - Set a new configuration for the feed. - - Arguments: - config -- A dictionary of configuration values. - """ - with self.config_lock: - self.thoonk.set_config(self.feed, config) - self.config_valid = False - - @config.deleter - def config(self): - """Mark the current configuration cache as invalid.""" - with self.config_lock: - self.config_valid = False - def delete_feed(self): """Delete the feed and its contents.""" self.thoonk.delete_feed(self.feed) @@ -156,7 +117,7 @@ def get_schemas(self): return set((self.feed_ids, self.feed_items, self.feed_publish, self.feed_publishes, self.feed_retract, self.feed_config, self.feed_edit)) - + # Thoonk Standard API # ================================================================= @@ -200,9 +161,8 @@ def publish(self, item, id=None): if publish_id is None: publish_id = uuid.uuid4().hex - max = int(self.config.get('max_length', 0)) - def _publish(pipe): + max = int(pipe.hget(self.feed_config, "max_length") or 0) if max > 0: delete_ids = pipe.zrange(self.feed_ids, 0, -max) pipe.multi() @@ -241,4 +201,4 @@ def _retract(pipe): pipe.hdel(self.feed_items, id) self.thoonk._publish(self.feed_retract, (id,), pipe) - self.redis.transaction(_retract, self.feed_ids) \ No newline at end of file + self.redis.transaction(_retract, self.feed_ids) diff --git a/thoonk/feeds/job.py b/thoonk/feeds/job.py index d7da695..93d9c61 100644 --- a/thoonk/feeds/job.py +++ b/thoonk/feeds/job.py @@ -5,21 +5,10 @@ import time import uuid -import redis -from thoonk.exceptions import * from thoonk.feeds import Queue from thoonk.feeds.queue import Empty - -class JobDoesNotExist(Exception): - pass - - -class JobNotPending(Exception): - pass - - class Job(Queue): """ @@ -58,8 +47,9 @@ class Job(Queue): feed.claimed:[feed] -- A hash table of claimed jobs. feed.stalled:[feed] -- A hash table of stalled jobs. feed.running:[feed] -- A hash table of running jobs. - feed.finished:[feed]\x00[id] -- Temporary queue for receiving job - result data. + feed.publishes:[feed] -- A count of the number of jobs published + feed.finishes:[feed] -- A count of the number of jobs finished + job.finish:[feed] -- A pubsub channel for job results Thoonk.py Implementation API: get_schemas -- Return the set of Redis keys used by this feed. @@ -77,7 +67,7 @@ class Job(Queue): stall -- Pause execution of a queued job. """ - def __init__(self, thoonk, feed, config=None): + def __init__(self, thoonk, feed): """ Create a new Job queue object for a given Thoonk feed. @@ -90,32 +80,30 @@ def __init__(self, thoonk, feed, config=None): feed -- The name of the feed. config -- Optional dictionary of configuration values. """ - Queue.__init__(self, thoonk, feed, config=None) + Queue.__init__(self, thoonk, feed) self.feed_publishes = 'feed.publishes:%s' % feed + self.feed_published = 'feed.published:%s' % feed self.feed_cancelled = 'feed.cancelled:%s' % feed self.feed_retried = 'feed.retried:%s' % feed - self.feed_finished = 'feed.finished:%s' % feed - self.feed_job_claimed = 'feed.claimed:%s' % feed - self.feed_job_stalled = 'feed.stalled:%s' % feed - self.feed_job_finished = 'feed.finished:%s\x00%s' % (feed, '%s') - self.feed_job_running = 'feed.running:%s' % feed + self.feed_finishes = 'feed.finishes:%s' % feed + self.feed_claimed = 'feed.claimed:%s' % feed + self.feed_stalled = 'feed.stalled:%s' % feed + self.feed_running = 'feed.running:%s' % feed + + self.job_finish = 'job.finish:%s' % feed def get_channels(self): - return (self.feed_publishes, self.feed_job_claimed, self.feed_job_stalled, - self.feed_finished, self.feed_cancelled, self.feed_retried) + return (self.feed_publishes, self.feed_claimed, self.feed_stalled, + self.feed_finishes, self.feed_cancelled, self.feed_retried) def get_schemas(self): """Return the set of Redis keys used exclusively by this feed.""" - schema = set((self.feed_job_claimed, - self.feed_job_stalled, - self.feed_job_running, + schema = set((self.feed_claimed, + self.feed_stalled, + self.feed_running, self.feed_publishes, self.feed_cancelled)) - - for id in self.get_ids(): - schema.add(self.feed_job_finished % id) - return schema.union(Queue.get_schemas(self)) def get_ids(self): @@ -134,11 +122,10 @@ def _retract(pipe): pipe.multi() pipe.hdel(self.feed_items, id) pipe.hdel(self.feed_cancelled, id) - pipe.zrem(self.feed_publishes, id) - pipe.srem(self.feed_job_stalled, id) - pipe.zrem(self.feed_job_claimed, id) + pipe.zrem(self.feed_published, id) + pipe.srem(self.feed_stalled, id) + pipe.zrem(self.feed_claimed, id) pipe.lrem(self.feed_ids, 1, id) - pipe.delete(self.feed_job_finished % id) self.redis.transaction(_retract, self.feed_items) @@ -161,9 +148,9 @@ def put(self, item, priority=False): pipe.rpush(self.feed_ids, id) else: pipe.lpush(self.feed_ids, id) - pipe.incr(self.feed_publishes) + pipe.incr(self.feed_publishes) pipe.hset(self.feed_items, id, item) - pipe.zadd(self.feed_publishes, **{id: time.time()}) + pipe.zadd(self.feed_published, **{id: int(time.time()*1000)}) results = pipe.execute() @@ -196,56 +183,40 @@ def get(self, timeout=0): id = id[1] pipe = self.redis.pipeline() - pipe.zadd(self.feed_job_claimed, **{id: time.time()}) + pipe.zadd(self.feed_claimed, **{id: int(time.time()*1000)}) pipe.hget(self.feed_items, id) pipe.hget(self.feed_cancelled, id) result = pipe.execute() - self.thoonk._publish(self.feed_job_claimed, (id,)) + self.thoonk._publish(self.feed_claimed, (id,)) return id, result[1], 0 if result[2] is None else int(result[2]) - def finish(self, id, item=None, result=False, timeout=None): + def get_failure_count(self, id): + return int(self.redis.hget(self.feed_cancelled, id) or 0) + + NO_RESULT = [] + def finish(self, id, result=NO_RESULT): """ Mark a job as completed, and store any results. Arguments: id -- The ID of the completed job. - item -- The result data from the job. - result -- Flag indicating that result data should be stored. - Defaults to False. - timeout -- Time in seconds to keep the result data. The default - is to store data indefinitely until retrieved. + result -- The result data from the job. (should be a string!) """ def _finish(pipe): - if pipe.zrank(self.feed_job_claimed, id) is None: + if pipe.zrank(self.feed_claimed, id) is None: return # raise exception? - #query = pipe.hget(self.feed_items, id) pipe.multi() - pipe.zrem(self.feed_job_claimed, id) + pipe.zrem(self.feed_claimed, id) pipe.hdel(self.feed_cancelled, id) - if result: - pipe.lpush(self.feed_job_finished % id, item) - if timeout is not None: - pipe.expire(self.feed_job_finished % id, timeout) + pipe.zrem(self.feed_published, id) + pipe.incr(self.feed_finishes) + if result is not self.NO_RESULT: + self.thoonk._publish(self.job_finish, (id, result), pipe) pipe.hdel(self.feed_items, id) - self.thoonk._publish(self.feed_finished, - (id, item if result else ""), pipe) - self.redis.transaction(_finish, self.feed_job_claimed) - - def get_result(self, id, timeout=0): - """ - Retrieve the result of a given job. - - Arguments: - id -- The ID of the job to check for results. - timeout -- Time in seconds to wait for results to arrive. - Default is to block indefinitely. - """ - result = self.redis.brpop(self.feed_job_finished % id, timeout) - if result is not None: - return result + self.redis.transaction(_finish, self.feed_claimed) def cancel(self, id): """ @@ -255,15 +226,14 @@ def cancel(self, id): id -- The ID of the job to cancel. """ def _cancel(pipe): - if self.redis.zrank(self.feed_job_claimed, id) is None: + if self.redis.zrank(self.feed_claimed, id) is None: return # raise exception? pipe.multi() pipe.hincrby(self.feed_cancelled, id, 1) pipe.lpush(self.feed_ids, id) - pipe.zrem(self.feed_job_claimed, id) - self.thoonk._publish(self.feed_cancelled, (id,), pipe) + pipe.zrem(self.feed_claimed, id) - self.redis.transaction(_cancel, self.feed_job_claimed) + self.redis.transaction(_cancel, self.feed_claimed) def stall(self, id): """ @@ -275,16 +245,15 @@ def stall(self, id): id -- The ID of the job to pause. """ def _stall(pipe): - if pipe.zrank(self.feed_job_claimed, id) is None: + if pipe.zrank(self.feed_claimed, id) is None: return # raise exception? pipe.multi() - pipe.zrem(self.feed_job_claimed, id) + pipe.zrem(self.feed_claimed, id) pipe.hdel(self.feed_cancelled, id) - pipe.sadd(self.feed_job_stalled, id) - pipe.zrem(self.feed_publishes, id) - self.thoonk._publish(self.feed_job_stalled, (id,), pipe) + pipe.sadd(self.feed_stalled, id) + pipe.zrem(self.feed_published, id) - self.redis.transaction(_stall, self.feed_job_claimed) + self.redis.transaction(_stall, self.feed_claimed) def retry(self, id): """ @@ -294,15 +263,14 @@ def retry(self, id): id -- The ID of the job to resume. """ def _retry(pipe): - if pipe.sismember(self.feed_job_stalled, id) is None: + if pipe.sismember(self.feed_stalled, id) is None: return # raise exception? pipe.multi() - pipe.srem(self.feed_job_stalled, id) + pipe.srem(self.feed_stalled, id) pipe.lpush(self.feed_ids, id) - pipe.zadd(self.feed_publishes, **{id: time.time()}) - self.thoonk._publish(self.feed_retried, (id,), pipe) + pipe.zadd(self.feed_published, **{id: time.time()}) - results = self.redis.transaction(_retry, self.feed_job_stalled) + results = self.redis.transaction(_retry, self.feed_stalled) if not results[0]: return # raise exception? @@ -319,8 +287,8 @@ def maintenance(self): pipe = self.redis.pipeline() pipe.hkeys(self.feed_items) pipe.lrange(self.feed_ids) - pipe.zrange(self.feed_job_claimed, 0, -1) - pipe.stall = pipe.smembers(self.feed_job_stalled) + pipe.zrange(self.feed_claimed, 0, -1) + pipe.stall = pipe.smembers(self.feed_stalled) keys, avail, claim, stall = pipe.execute() diff --git a/thoonk/feeds/queue.py b/thoonk/feeds/queue.py index 1c11791..d3716e4 100644 --- a/thoonk/feeds/queue.py +++ b/thoonk/feeds/queue.py @@ -5,14 +5,9 @@ import uuid -from thoonk.exceptions import * +from thoonk.exceptions import Empty from thoonk.feeds import Feed - -class Empty(Exception): - pass - - class Queue(Feed): """ diff --git a/thoonk/feeds/sorted_feed.py b/thoonk/feeds/sorted_feed.py index 051816d..58f5d30 100644 --- a/thoonk/feeds/sorted_feed.py +++ b/thoonk/feeds/sorted_feed.py @@ -32,7 +32,7 @@ class SortedFeed(Feed): publish_before -- Add an item immediately after an existing item. """ - def __init__(self, thoonk, feed, config=None): + def __init__(self, thoonk, feed): """ Create a new SortedFeed object for a given Thoonk feed. @@ -46,7 +46,7 @@ def __init__(self, thoonk, feed, config=None): config -- Optional dictionary of configuration values. """ - Feed.__init__(self, thoonk, feed, config) + Feed.__init__(self, thoonk, feed) self.feed_id_incr = 'feed.idincr:%s' % feed self.feed_position = 'feed.position:%s' % feed diff --git a/thoonk/pubsub.py b/thoonk/pubsub.py index 4f17c7f..53a553a 100644 --- a/thoonk/pubsub.py +++ b/thoonk/pubsub.py @@ -3,15 +3,12 @@ Released under the terms of the MIT License """ -import json import redis import threading import uuid -from thoonk import feeds -from thoonk.exceptions import * -from thoonk.config import ConfigCache - +from thoonk import feeds, cache +from thoonk.exceptions import FeedExists, FeedDoesNotExist, NotListening class Thoonk(object): @@ -40,7 +37,6 @@ class Thoonk(object): Attributes: db -- The Redis database number. feeds -- A set of known feed names. - _feed_config -- A cache of feed configurations. feedtypes -- A dictionary mapping feed type names to their implementation classes. handlers -- A dictionary mapping event names to event handlers. @@ -85,32 +81,16 @@ def __init__(self, host='localhost', port=6379, db=0, listen=False): self.port = port self.db = db self.redis = redis.StrictRedis(host=self.host, port=self.port, db=self.db) - self.lredis = None + self._feeds = cache.FeedCache(self) + self.instance = uuid.uuid4().hex self.feedtypes = {} - self.feeds = set() - self._feed_config = ConfigCache(self) - self.handlers = { - 'create_notice': [], - 'delete_notice': [], - 'publish_notice': [], - 'retract_notice': [], - 'position_notice': [], - 'stalled_notice': [], - 'retried_notice': [], - 'finished_notice': [], - 'claimed_notice': [], - 'cancelled_notice': []} - - self.listen_ready = threading.Event() + self.listening = listen self.feed_publish = 'feed.publish:%s' self.feed_retract = 'feed.retract:%s' self.feed_config = 'feed.config:%s' - self.conf_feed = 'conffeed' - self.new_feed = 'newfeed' - self.del_feed = 'delfeed' self.register_feedtype(u'feed', feeds.Feed) self.register_feedtype(u'queue', feeds.Queue) @@ -119,13 +99,11 @@ def __init__(self, host='localhost', port=6379, db=0, listen=False): self.register_feedtype(u'sorted_feed', feeds.SortedFeed) if listen: - #start listener thread - self.lthread = threading.Thread(target=self.listen) - self.lthread.daemon = True - self.lthread.start() - self.listen_ready.wait() + self.listener = ThoonkListener(self) + self.listener.start() + self.listener.ready.wait() - def _publish(self, schema, items, pipe=None): + def _publish(self, schema, items=[], pipe=None): """ A shortcut method to publish items separated by \x00. @@ -141,27 +119,6 @@ def _publish(self, schema, items, pipe=None): else: self.redis.publish(schema, "\x00".join(items)) - def __getitem__(self, feed): - """ - Return the configuration for a feed. - - Arguments: - feed -- The name of the feed. - - Returns: Dict - """ - return self._feed_config[feed] - - def __setitem__(self, feed, config): - """ - Set the configuration for a feed. - - Arguments: - feed -- The name of the feed. - config -- A dict of config values. - """ - self.set_config(feed, config) - def register_feedtype(self, feedtype, klass): """ Make a new feed type availabe for use. @@ -189,15 +146,16 @@ def startclass(feed, config=None): """ if config is None: config = {} - if self.feed_exists(feed): - return self[feed] - else: - if not config.get('type', False): - config['type'] = feedtype - return self.create_feed(feed, config) + config['type'] = feedtype + try: + self.create_feed(feed, config) + except FeedExists: + pass + return self._feeds[feed] + setattr(self, feedtype, startclass) - + def register_handler(self, name, handler): """ Register a function to respond to feed events. @@ -213,9 +171,10 @@ def register_handler(self, name, handler): name -- The name of the feed event. handler -- The function for handling the event. """ - if name not in self.handlers: - self.handlers[name] = [] - self.handlers[name].append(handler) + if self.listener: + self.listener.register_handler(name, handler) + else: + raise NotListening def remove_handler(self, name, handler): """ @@ -225,12 +184,11 @@ def remove_handler(self, name, handler): name -- The name of the feed event. handler -- The function for handling the event. """ - try: - self.handlers[name].remove(handler) - except (KeyError, ValueError): - pass - - + if self.listener: + self.listener.remove_handler(name, handler) + else: + raise NotListening + def create_feed(self, feed, config): """ Create a new feed with a given configuration. @@ -242,14 +200,9 @@ def create_feed(self, feed, config): feed -- The name of the new feed. config -- A dictionary of configuration values. """ - if config is None: - config = {} if not self.redis.sadd("feeds", feed): raise FeedExists - self.feeds.add(feed) - self.set_config(feed, config) - self._publish(self.new_feed, (feed, self._feed_config.instance)) - return self[feed] + self.set_config(feed, config, True) def delete_feed(self, feed): """ @@ -258,7 +211,7 @@ def delete_feed(self, feed): Arguments: feed -- The name of the feed. """ - feed_instance = self._feed_config[feed] + feed_instance = self._feeds[feed] def _delete_feed(pipe): if not pipe.sismember('feeds', feed): @@ -267,11 +220,11 @@ def _delete_feed(pipe): pipe.srem("feeds", feed) for key in feed_instance.get_schemas(): pipe.delete(key) - self._publish(self.del_feed, (feed, self._feed_config.instance)) + self._publish('delfeed', (feed, self.instance), pipe) self.redis.transaction(_delete_feed, 'feeds') - def set_config(self, feed, config): + def set_config(self, feed, config, new_feed=False): """ Set the configuration for a given feed. @@ -281,33 +234,23 @@ def set_config(self, feed, config): """ if not self.feed_exists(feed): raise FeedDoesNotExist - if type(config) == dict: - if u'type' not in config: - config[u'type'] = u'feed' - jconfig = json.dumps(config) - dconfig = config - else: - dconfig = json.loads(config) - if u'type' not in dconfig: - dconfig[u'type'] = u'feed' - jconfig = json.dumps(dconfig) - self.redis.set(self.feed_config % feed, jconfig) - self._publish(self.conf_feed, (feed, self._feed_config.instance)) - - def get_config(self, feed): - if not self.feed_exists(feed): - raise FeedDoesNotExist - config = self.redis.get(self.feed_config % feed) - return json.loads(config) - - def get_feeds(self): + if u'type' not in config: + config[u'type'] = u'feed' + pipe = self.redis.pipeline() + for k, v in config.iteritems(): + pipe.hset('feed.config:' + feed, k, v) + pipe.execute() + if new_feed: + self._publish('newfeed', (feed, self.instance)) + self._publish('conffeed', (feed, self.instance)) + + def get_feed_names(self): """ Return the set of known feeds. Returns: set """ - self.feeds.update(self.redis.smembers('feeds')) - return self.feeds + return self.redis.smembers('feeds') or set() def feed_exists(self, feed): """ @@ -317,21 +260,34 @@ def feed_exists(self, feed): feed -- The name of the feed. """ return self.redis.sismember('feeds', feed) - if not self.listening: - if not feed in self.feeds: - if self.redis.sismember('feeds', feed): - self.feeds.add(feed) - return True - return False - else: - return True - return feed in self.feeds def close(self): """Terminate the listening Redis connection.""" + if self.listening: + self.redis.publish(self.listener._finish_channel, "") + self.listener.finished.wait() self.redis.connection_pool.disconnect() - def listen(self): + +class ThoonkListener(threading.Thread): + + def __init__(self, thoonk, *args, **kwargs): + threading.Thread.__init__(self, *args, **kwargs) + self.lock = threading.Lock() + self.handlers = {} + self.thoonk = thoonk + self.ready = threading.Event() + self.redis = redis.StrictRedis(host=thoonk.host, port=thoonk.port, db=thoonk.db) + self.finished = threading.Event() + self.instance = thoonk.instance + self._finish_channel = "listenerclose_%s" % self.instance + self._pubsub = None + self.daemon = True + + def finish(self): + self.redis.publish(self._finish_channel, "") + + def run(self): """ Listen for feed creation and manipulation events and execute relevant event handlers. Specifically, listen for: @@ -342,203 +298,104 @@ def listen(self): - Item retractions. """ # listener redis object - self.lredis = self.redis.pubsub() - + self._pubsub = self.redis.pubsub() # subscribe to feed activities channel - self.lredis.subscribe((self.new_feed, self.del_feed, self.conf_feed)) + self._pubsub.subscribe((self._finish_channel, 'newfeed', 'delfeed', 'conffeed')) - # get set of feeds - feeds = self.get_feeds() # subscribe to exist feeds retract and publish - for feed in self.feeds: - self.lredis.subscribe(self[feed].get_channels()) - - self.listen_ready.set() - while True: - a = self.lredis.listen() + for feed in self.redis.smembers("feeds"): + self._pubsub.subscribe(self.thoonk._feeds[feed].get_channels()) + + self.ready.set() + for event in self._pubsub.listen(): + type = event.pop("type") + if event["channel"] == self._finish_channel: + if self._pubsub.subscription_count: + self._pubsub.unsubscribe() + elif type == 'message': + self._handle_message(**event) + elif type == 'pmessage': + self._handle_pmessage(**event) + + self.finished.set() + + def _handle_message(self, channel, data, pattern=None): + if channel == 'newfeed': + #feed created event + name, _ = data.split('\x00') + self._pubsub.subscribe(("feed.publish:"+name, "feed.edit:"+name, + "feed.retract:"+name, "feed.position:"+name, "job.finish:"+name)) + self.emit("create", name) + + elif channel == 'delfeed': + #feed destroyed event + name, _ = data.split('\x00') try: - event = a.next() + del self._feeds[name] except: - break - if event['type'] == 'message': - if event['channel'].startswith('feed.publish'): - #feed publish event - id, item = event['data'].split('\x00', 1) - self.publish_notice(event['channel'].split(':', 1)[-1], - item, id) - elif event['channel'].startswith('feed.retract'): - self.retract_notice(event['channel'].split(':', 1)[-1], - event['data']) - elif event['channel'].startswith('feed.position'): - id, rel_id = event['data'].split('\x00', 1) - self.position_notice(event['channel'].split(':', 1)[-1], - id, rel_id) - elif event['channel'].startswith('feed.claimed'): - self.claimed_notice(event['channel'].split(':', 1)[-1], - event['data']) - elif event['channel'].startswith('feed.finished'): - id, data = event['data'].split(':', 1)[-1].split("\x00", 1) - self.finished_notice(event['channel'].split(':', 1)[-1], id, - data) - elif event['channel'].startswith('feed.cancelled'): - self.cancelled_notice(event['channel'].split(':', 1)[-1], - event['data']) - elif event['channel'].startswith('feed.stalled'): - self.stalled_notice(event['channel'].split(':', 1)[-1], - event['data']) - elif event['channel'].startswith('feed.retried'): - self.retried_notice(event['channel'].split(':', 1)[-1], - event['data']) - elif event['channel'] == self.new_feed: - #feed created event - name, instance = event['data'].split('\x00') - self.feeds.add(name) - config = self.get_config(name) - if config["type"] == "job": - self.lredis.subscribe(("feed.publishes:%s" % name, - "feed.cancelled:%s" % name, - "feed.claimed:%s" % name, - "feed.retried:%s" % name, - "feed.finished:%s" % name, - "feed.stalled:%s" % name,)) - else: - self.lredis.subscribe((self.feed_publish % name, - self.feed_retract % name)) - self.create_notice(name) - elif event['channel'] == self.del_feed: - #feed destroyed event - name, instance = event['data'].split('\x00') - try: - self.feeds.remove(name) - except KeyError: - #already removed -- probably locally - pass - self._feed_config.invalidate(name, instance, delete=True) - self.delete_notice(name) - elif event['channel'] == self.conf_feed: - feed, instance = event['data'].split('\x00', 1) - self._feed_config.invalidate(feed, instance) - - def create_notice(self, feed): - """ - Generate a notice that a new feed has been created and - execute any relevant event handlers. - - Arguments: - feed -- The name of the created feed. - """ - for handler in self.handlers['create_notice']: - handler(feed) - - def delete_notice(self, feed): - """ - Generate a notice that a feed has been deleted, and - execute any relevant event handlers. - - Arguments: - feed -- The name of the deleted feed. - """ - for handler in self.handlers['delete_notice']: - handler(feed) - - def publish_notice(self, feed, item, id): - """ - Generate a notice that an item has been published to a feed, and - execute any relevant event handlers. - - Arguments: - feed -- The name of the feed. - item -- The content of the published item. - id -- The ID of the published item. - """ - self[feed].event_publish(id, item) - for handler in self.handlers['publish_notice']: - handler(feed, item, id) - - def retract_notice(self, feed, id): - """ - Generate a notice that an item has been retracted from a feed, and - execute any relevant event handlers. - - Arguments: - feed -- The name of the feed. - id -- The ID of the retracted item. - """ - self[feed].event_retract(id) - for handler in self.handlers['retract_notice']: - handler(feed, id) - - def position_notice(self, feed, id, rel_id): - """ - Generate a notice that an item has been moved, and - execute any relevant event handlers. - - Arguments: - feed -- The name of the feed. - id -- The ID of the moved item. - rel_id -- Where the item was moved, in relation to - existing items. - """ - for handler in self.handlers['position_notice']: - handler(feed, id, rel_id) - - def stalled_notice(self, feed, id): - """ - Generate a notice that a job has been stalled, and - execute any relevant event handlers. - - Arguments: - feed -- The name of the feed. - id -- The ID of the stalled item. - """ - for handler in self.handlers['stalled_notice']: - handler(feed, id) - - def retried_notice(self, feed, id): - """ - Generate a notice that a job has been retried, and - execute any relevant event handlers. - - Arguments: - feed -- The name of the feed. - id -- The ID of the retried item. - """ - for handler in self.handlers['retried_notice']: - handler(feed, id) + pass + self.emit("delete", name) + + elif channel == 'conffeed': + feed, _ = data.split('\x00', 1) + self.emit("config:"+feed, None) + + elif channel.startswith('feed.publish'): + #feed publish event + id, item = data.split('\x00', 1) + self.emit("publish", channel.split(':', 1)[-1], item, id) + + elif channel.startswith('feed.edit'): + #feed publish event + id, item = data.split('\x00', 1) + self.emit("edit", channel.split(':', 1)[-1], item, id) + + elif channel.startswith('feed.retract'): + self.emit("retract", channel.split(':', 1)[-1], data) + + elif channel.startswith('feed.position'): + id, rel_id = data.split('\x00', 1) + self.emit("position", channel.split(':', 1)[-1], id, rel_id) - def cancelled_notice(self, feed, id): - """ - Generate a notice that a job has been cancelled, and - execute any relevant event handlers. + elif channel.startswith('job.finish'): + id, result = data.split('\x00', 1) + self.emit("finish", channel.split(':', 1)[-1], id, result) + + def emit(self, event, *args): + with self.lock: + for handler in self.handlers.get(event, []): + handler(*args) - Arguments: - feed -- The name of the feed. - id -- The ID of the stalled item. + def register_handler(self, name, handler): """ - for handler in self.handlers['cancelled_notice']: - handler(feed, id) + Register a function to respond to feed events. - def finished_notice(self, feed, id, result): - """ - Generate a notice that a job has finished, and - execute any relevant event handlers. + Event types: + - create_notice + - delete_notice + - publish_notice + - retract_notice + - position_notice Arguments: - feed -- The name of the feed. - id -- The ID of the stalled item. + name -- The name of the feed event. + handler -- The function for handling the event. """ - for handler in self.handlers['finished_notice']: - handler(feed, id, result) + with self.lock: + if name not in self.handlers: + self.handlers[name] = [] + self.handlers[name].append(handler) - def claimed_notice(self, feed, id): + def remove_handler(self, name, handler): """ - Generate a notice that a job has been claimed, and - execute any relevant event handlers. + Unregister a function that was registered via register_handler Arguments: - feed -- The name of the feed. - id -- The ID of the stalled item. + name -- The name of the feed event. + handler -- The function for handling the event. """ - for handler in self.handlers['claimed_notice']: - handler(feed, id) - \ No newline at end of file + with self.lock: + try: + self.handlers[name].remove(handler) + except (KeyError, ValueError): + pass From 33b324d1c3944b9849827f45b954f3fe219f0f96 Mon Sep 17 00:00:00 2001 From: Simon Hewitt Date: Fri, 18 Nov 2011 16:57:43 +0000 Subject: [PATCH 12/17] moved publish feed names to helper function --- thoonk/pubsub.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/thoonk/pubsub.py b/thoonk/pubsub.py index 53a553a..a093ee6 100644 --- a/thoonk/pubsub.py +++ b/thoonk/pubsub.py @@ -277,7 +277,8 @@ def __init__(self, thoonk, *args, **kwargs): self.handlers = {} self.thoonk = thoonk self.ready = threading.Event() - self.redis = redis.StrictRedis(host=thoonk.host, port=thoonk.port, db=thoonk.db) + self.redis = redis.StrictRedis(host=thoonk.host, port=thoonk.port, + db=thoonk.db) self.finished = threading.Event() self.instance = thoonk.instance self._finish_channel = "listenerclose_%s" % self.instance @@ -286,6 +287,10 @@ def __init__(self, thoonk, *args, **kwargs): def finish(self): self.redis.publish(self._finish_channel, "") + + def _channels_for_feed(self, name): + return ("feed.publish:"+name, "feed.edit:"+name, "feed.retract:"+name, + "feed.position:"+name, "job.finish:"+name) def run(self): """ @@ -300,11 +305,13 @@ def run(self): # listener redis object self._pubsub = self.redis.pubsub() # subscribe to feed activities channel - self._pubsub.subscribe((self._finish_channel, 'newfeed', 'delfeed', 'conffeed')) - + self._pubsub.subscribe((self._finish_channel, 'newfeed', 'delfeed', + 'conffeed')) + + # subscribe to exist feeds retract and publish for feed in self.redis.smembers("feeds"): - self._pubsub.subscribe(self.thoonk._feeds[feed].get_channels()) + self._pubsub.subscribe(self._channels_for_feed(feed)) self.ready.set() for event in self._pubsub.listen(): @@ -323,8 +330,7 @@ def _handle_message(self, channel, data, pattern=None): if channel == 'newfeed': #feed created event name, _ = data.split('\x00') - self._pubsub.subscribe(("feed.publish:"+name, "feed.edit:"+name, - "feed.retract:"+name, "feed.position:"+name, "job.finish:"+name)) + self._pubsub.subscribe(self._channels_for_feed(name)) self.emit("create", name) elif channel == 'delfeed': From 9cf058c0501eddae4ca61b2bcceb95f4d5e3e5ab Mon Sep 17 00:00:00 2001 From: Simon Hewitt Date: Sat, 19 Nov 2011 14:44:03 +0000 Subject: [PATCH 13/17] updated Jobs to use Redis Lua Scripting --- scripts/jobs/cancel.lua | 7 +++ scripts/jobs/finish.lua | 12 +++++ scripts/jobs/get.lua | 4 ++ scripts/jobs/publish.lua | 9 ++++ scripts/jobs/retract.lua | 11 ++++ scripts/jobs/retry.lua | 7 +++ scripts/jobs/stall.lua | 8 +++ tests/test_job.py | 76 +++++++++++++++++++++++---- thoonk/exceptions.py | 9 ++++ thoonk/feeds/job.py | 107 ++++++++++----------------------------- thoonk/pubsub.py | 9 +++- 11 files changed, 167 insertions(+), 92 deletions(-) create mode 100644 scripts/jobs/cancel.lua create mode 100644 scripts/jobs/finish.lua create mode 100644 scripts/jobs/get.lua create mode 100644 scripts/jobs/publish.lua create mode 100644 scripts/jobs/retract.lua create mode 100644 scripts/jobs/retry.lua create mode 100644 scripts/jobs/stall.lua diff --git a/scripts/jobs/cancel.lua b/scripts/jobs/cancel.lua new file mode 100644 index 0000000..6424ce1 --- /dev/null +++ b/scripts/jobs/cancel.lua @@ -0,0 +1,7 @@ +-- ARGV: name, id +if redis.call('zrem', 'feed.claimed:'..ARGV[1], ARGV[2]) == 0 then + return false +end +redis.call('hincrby', 'feed.cancelled:'..ARGV[1], ARGV[2], 1) +redis.call('lpush', 'feed.ids:'..ARGV[1], ARGV[2]) +return true diff --git a/scripts/jobs/finish.lua b/scripts/jobs/finish.lua new file mode 100644 index 0000000..2a6c980 --- /dev/null +++ b/scripts/jobs/finish.lua @@ -0,0 +1,12 @@ +-- ARGV: name, id, result(optional) +if redis.call('zrem', 'feed.claimed:'..ARGV[1], ARGV[2]) == 0 then + return false; +end +redis.call('hdel', 'feed.cancelled:'..ARGV[1], ARGV[2]) +redis.call('zrem', 'feed.published:'..ARGV[1], ARGV[2]) +redis.call('incr', 'feed.finishes:'..ARGV[1]) +if table.getn(ARGV) == 3 then + redis.call('publish', 'job.finish:'..ARGV[1], ARGV[2].."\0"..ARGV[3]) +end +redis.call('hdel', 'feed.items:'..ARGV[1], ARGV[2]) +return true diff --git a/scripts/jobs/get.lua b/scripts/jobs/get.lua new file mode 100644 index 0000000..ebc99de --- /dev/null +++ b/scripts/jobs/get.lua @@ -0,0 +1,4 @@ +-- ARGV: name, id, time +r1 = redis.call('zadd', 'feed.claimed:'..ARGV[1], ARGV[3], ARGV[2]); +r2 = redis.call('hget', 'feed.items:'..ARGV[1], ARGV[2]); +return {r1, r2} diff --git a/scripts/jobs/publish.lua b/scripts/jobs/publish.lua new file mode 100644 index 0000000..ce76723 --- /dev/null +++ b/scripts/jobs/publish.lua @@ -0,0 +1,9 @@ +-- ARGV: name, id, item, time, priority +if ARGV[5] == nil then + redis.call('lpush', 'feed.ids:'..ARGV[1], ARGV[2]); +else + redis.call('rpush', 'feed.ids:'..ARGV[1], ARGV[2]); +end +redis.call('incr', 'feed.publishes:'..ARGV[1]); +redis.call('hset', 'feed.items:'..ARGV[1], ARGV[2], ARGV[3]); +return redis.call('zadd', 'feed.published:'..ARGV[1], ARGV[4], ARGV[2]); diff --git a/scripts/jobs/retract.lua b/scripts/jobs/retract.lua new file mode 100644 index 0000000..14f31c6 --- /dev/null +++ b/scripts/jobs/retract.lua @@ -0,0 +1,11 @@ +-- ARGV: name, id +if redis.call('hdel', 'feed.items:'..ARGV[1], ARGV[2]) == 0 then + return false +end +redis.call('hdel', 'feed.cancelled:'..ARGV[1], ARGV[2]) +redis.call('zrem', 'feed.published:'..ARGV[1], ARGV[2]) +redis.call('srem', 'feed.stalled:'..ARGV[1], ARGV[2]) +redis.call('zrem', 'feed.claimed:'..ARGV[1], ARGV[2]) +redis.call('lrem', 'feed.ids:'..ARGV[1], 1, ARGV[2]) +return true + diff --git a/scripts/jobs/retry.lua b/scripts/jobs/retry.lua new file mode 100644 index 0000000..10bbd53 --- /dev/null +++ b/scripts/jobs/retry.lua @@ -0,0 +1,7 @@ +-- ARGV: name, id, time +if redis.call('srem', 'feed.stalled:'..ARGV[1], ARGV[2]) == 0 then + return false; +end +redis.call('lpush', 'feed.ids:'..ARGV[1], ARGV[2]); +redis.call('zadd', 'feed.published:'..ARGV[1], ARGV[3], ARGV[2]); +return true diff --git a/scripts/jobs/stall.lua b/scripts/jobs/stall.lua new file mode 100644 index 0000000..679f277 --- /dev/null +++ b/scripts/jobs/stall.lua @@ -0,0 +1,8 @@ +-- ARGV: name, id +if redis.call('zrem', 'feed.claimed:'..ARGV[1], ARGV[2]) == 0 then + return false; +end +redis.call('hdel', 'feed.cancelled:'..ARGV[1], ARGV[2]); +redis.call('sadd', 'feed.stalled:'..ARGV[1], ARGV[2]); +redis.call('zrem', 'feed.published'..ARGV[1], ARGV[2]); +return true diff --git a/tests/test_job.py b/tests/test_job.py index 83930c2..b298b7c 100644 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -30,9 +30,9 @@ def test_10_basic_job(self): id = testjob.put('9.0') #worker - id_worker, job_content, cancelled = testjob.get(timeout=3) + id_worker, job_content = testjob.get(timeout=3) self.assertEqual(job_content, '9.0') - self.assertEqual(cancelled, 0) + self.assertEqual(testjob.get_failure_count(id), 0) self.assertEqual(id_worker, id) testjob.finish(id_worker) @@ -44,26 +44,82 @@ def test_20_cancel_job(self): #publisher id = j.put('9.0') #worker claims - id, job_content, cancelled = j.get() + id, job_content = j.get() self.assertEqual(job_content, '9.0') - self.assertEqual(cancelled, 0) + self.assertEqual(j.get_failure_count(id), 0) #publisher or worker cancels j.cancel(id) - id2, job_content2, cancelled2 = j.get() - self.assertEqual(cancelled2, 1) + id2, job_content2 = j.get() + self.assertEqual(j.get_failure_count(id), 1) self.assertEqual(job_content2, '9.0') self.assertEqual(id, id2) #cancel the work again j.cancel(id) # check the cancelled increment again - id3, job_content3, cancelled3 = j.get() - self.assertEqual(cancelled3, 2) + id3, job_content3 = j.get() + self.assertEqual(j.get_failure_count(id), 2) self.assertEqual(job_content3, '9.0') self.assertEqual(id, id3) #cleanup -- remove the job from the queue j.retract(id) self.assertEqual(j.get_ids(), []) + def test_25_stall_job(self): + """Test stalling a job""" + testjob = self.ps.job("testjob") + self.assertEqual(testjob.get_ids(), []) + + # put + id = testjob.put('9.0') + self.assertEqual(testjob.get_ids(), [id]) + + # invalid stall + self.assertRaises(thoonk.exceptions.JobNotClaimed, testjob.stall, id) + + # get + id_worker, job_content = testjob.get(timeout=3) + self.assertEqual(id_worker, id) + self.assertEqual(job_content, '9.0') + self.assertEqual(testjob.get_failure_count(id), 0) + + # invalid retry + self.assertRaises(thoonk.exceptions.JobNotStalled, testjob.retry, id) + + # stall + testjob.stall(id) + self.assertEqual(testjob.get_ids(), [id]) + self.assertRaises(thoonk.exceptions.Empty, testjob.get, timeout=1) + + # retry + testjob.retry(id) + self.assertEqual(testjob.get_ids(), [id]) + + # get + id_worker, job_content = testjob.get(timeout=3) + self.assertEqual(id_worker, id) + self.assertEqual(job_content, '9.0') + self.assertEqual(testjob.get_failure_count(id), 0) + + # finish + testjob.finish(id_worker) + self.assertEqual(testjob.get_ids(), []) + + def test_27_retract_job(self): + """Test retracting a job""" + testjob = self.ps.job("testjob") + self.assertEqual(testjob.get_ids(), []) + + # put + id = testjob.put('9.0') + self.assertEqual(testjob.get_ids(), [id]) + + # retract + testjob.retract(id) + self.assertEqual(testjob.get_ids(), []) + + # invalid retract + self.assertRaises(thoonk.exceptions.ItemDoesNotExist, testjob.retract, id) + def test_30_no_job(self): """Test exception raise when job.get times out""" j = self.ps.job("testjob") @@ -107,9 +163,9 @@ def create_handler(name): id = testjob.put('9.0') #worker - id_worker, job_content, cancelled = testjob.get(timeout=3) + id_worker, job_content = testjob.get(timeout=3) self.assertEqual(job_content, '9.0') - self.assertEqual(cancelled, 0) + self.assertEqual(testjob.get_failure_count(id), 0) self.assertEqual(id_worker, id) result_event = threading.Event() diff --git a/thoonk/exceptions.py b/thoonk/exceptions.py index d808524..440c37a 100644 --- a/thoonk/exceptions.py +++ b/thoonk/exceptions.py @@ -10,6 +10,15 @@ class FeedExists(Exception): class FeedDoesNotExist(Exception): pass +class ItemDoesNotExist(Exception): + pass + +class JobNotClaimed(Exception): + pass + +class JobNotStalled(Exception): + pass + class Empty(Exception): pass diff --git a/thoonk/feeds/job.py b/thoonk/feeds/job.py index 93d9c61..499f458 100644 --- a/thoonk/feeds/job.py +++ b/thoonk/feeds/job.py @@ -7,7 +7,8 @@ import uuid from thoonk.feeds import Queue -from thoonk.feeds.queue import Empty +from thoonk.exceptions import Empty, JobNotClaimed, JobNotStalled,\ + ItemDoesNotExist class Job(Queue): @@ -81,7 +82,7 @@ def __init__(self, thoonk, feed): config -- Optional dictionary of configuration values. """ Queue.__init__(self, thoonk, feed) - + self.feed_publishes = 'feed.publishes:%s' % feed self.feed_published = 'feed.published:%s' % feed self.feed_cancelled = 'feed.cancelled:%s' % feed @@ -117,17 +118,9 @@ def retract(self, id): Arguments: id -- The ID of the job to remove. """ - def _retract(pipe): - if pipe.hexists(self.feed_items, id): - pipe.multi() - pipe.hdel(self.feed_items, id) - pipe.hdel(self.feed_cancelled, id) - pipe.zrem(self.feed_published, id) - pipe.srem(self.feed_stalled, id) - pipe.zrem(self.feed_claimed, id) - pipe.lrem(self.feed_ids, 1, id) - - self.redis.transaction(_retract, self.feed_items) + success = self.redis.evalsha(self.thoonk.scripts["jobs/retract"], 0, self.feed, id) + if not success: + raise ItemDoesNotExist def put(self, item, priority=False): """ @@ -142,24 +135,13 @@ def put(self, item, priority=False): queue instead of the end. """ id = uuid.uuid4().hex - pipe = self.redis.pipeline() - - if priority: - pipe.rpush(self.feed_ids, id) - else: - pipe.lpush(self.feed_ids, id) - pipe.incr(self.feed_publishes) - pipe.hset(self.feed_items, id, item) - pipe.zadd(self.feed_published, **{id: int(time.time()*1000)}) - - results = pipe.execute() - - if results[-1]: + added = self.redis.evalsha(self.thoonk.scripts["jobs/publish"], 0, + self.feed, id, item, int(time.time()*1000), 1 if priority else None) + if added: # If zadd was successful self.thoonk._publish(self.feed_publishes, (id, item)) else: self.thoonk._publish(self.feed_edit, (id, item)) - return id def get(self, timeout=0): @@ -175,22 +157,14 @@ def get(self, timeout=0): Returns: id -- The id of the job job -- The job content - cancelled -- The number of times the job has been cancelled """ id = self.redis.brpop(self.feed_ids, timeout) if id is None: raise Empty id = id[1] - - pipe = self.redis.pipeline() - pipe.zadd(self.feed_claimed, **{id: int(time.time()*1000)}) - pipe.hget(self.feed_items, id) - pipe.hget(self.feed_cancelled, id) - result = pipe.execute() - - self.thoonk._publish(self.feed_claimed, (id,)) - - return id, result[1], 0 if result[2] is None else int(result[2]) + result = self.redis.evalsha(self.thoonk.scripts["jobs/get"], 0, self.feed, id, + int(time.time()*1000)) + return id, result[1] def get_failure_count(self, id): return int(self.redis.hget(self.feed_cancelled, id) or 0) @@ -204,19 +178,10 @@ def finish(self, id, result=NO_RESULT): id -- The ID of the completed job. result -- The result data from the job. (should be a string!) """ - def _finish(pipe): - if pipe.zrank(self.feed_claimed, id) is None: - return # raise exception? - pipe.multi() - pipe.zrem(self.feed_claimed, id) - pipe.hdel(self.feed_cancelled, id) - pipe.zrem(self.feed_published, id) - pipe.incr(self.feed_finishes) - if result is not self.NO_RESULT: - self.thoonk._publish(self.job_finish, (id, result), pipe) - pipe.hdel(self.feed_items, id) - - self.redis.transaction(_finish, self.feed_claimed) + success = self.redis.evalsha(self.thoonk.scripts["jobs/finish"], 0, self.feed, id, + *([result] if result is not self.NO_RESULT else [])) + if not success: + raise JobNotClaimed def cancel(self, id): """ @@ -225,15 +190,9 @@ def cancel(self, id): Arguments: id -- The ID of the job to cancel. """ - def _cancel(pipe): - if self.redis.zrank(self.feed_claimed, id) is None: - return # raise exception? - pipe.multi() - pipe.hincrby(self.feed_cancelled, id, 1) - pipe.lpush(self.feed_ids, id) - pipe.zrem(self.feed_claimed, id) - - self.redis.transaction(_cancel, self.feed_claimed) + success = self.redis.evalsha(self.thoonk.scripts["jobs/cancel"], 0, self.feed, id) + if not success: + raise JobNotClaimed def stall(self, id): """ @@ -244,16 +203,9 @@ def stall(self, id): Arguments: id -- The ID of the job to pause. """ - def _stall(pipe): - if pipe.zrank(self.feed_claimed, id) is None: - return # raise exception? - pipe.multi() - pipe.zrem(self.feed_claimed, id) - pipe.hdel(self.feed_cancelled, id) - pipe.sadd(self.feed_stalled, id) - pipe.zrem(self.feed_published, id) - - self.redis.transaction(_stall, self.feed_claimed) + success = self.redis.evalsha(self.thoonk.scripts["jobs/stall"], 0, self.feed, id) + if not success: + raise JobNotClaimed def retry(self, id): """ @@ -262,17 +214,10 @@ def retry(self, id): Arguments: id -- The ID of the job to resume. """ - def _retry(pipe): - if pipe.sismember(self.feed_stalled, id) is None: - return # raise exception? - pipe.multi() - pipe.srem(self.feed_stalled, id) - pipe.lpush(self.feed_ids, id) - pipe.zadd(self.feed_published, **{id: time.time()}) - - results = self.redis.transaction(_retry, self.feed_stalled) - if not results[0]: - return # raise exception? + success = self.redis.evalsha(self.thoonk.scripts["jobs/retry"], 0, self.feed, id, + int(time.time()*1000)) + if not success: + raise JobNotStalled def maintenance(self): """ diff --git a/thoonk/pubsub.py b/thoonk/pubsub.py index a093ee6..bc83d95 100644 --- a/thoonk/pubsub.py +++ b/thoonk/pubsub.py @@ -9,6 +9,7 @@ from thoonk import feeds, cache from thoonk.exceptions import FeedExists, FeedDoesNotExist, NotListening +import os class Thoonk(object): @@ -85,7 +86,13 @@ def __init__(self, host='localhost', port=6379, db=0, listen=False): self.instance = uuid.uuid4().hex self.feedtypes = {} - + + self.scripts = {} + for dirpath, _, filenames in os.walk("scripts/"): + for filename in filenames: + if filename.endswith(".lua"): + f = open(os.path.join(dirpath, filename), "r") + self.scripts[dirpath[8:]+"/"+filename[:-4]] = self.redis.script("LOAD", f.read()) self.listening = listen self.feed_publish = 'feed.publish:%s' From b21708987c619b465885fc055099c7493121b8f3 Mon Sep 17 00:00:00 2001 From: Simon Hewitt Date: Sun, 20 Nov 2011 16:10:31 +0000 Subject: [PATCH 14/17] added feed config/create/delete scripts --- scripts/config.lua | 11 +++++++++++ scripts/create.lua | 13 +++++++++++++ scripts/delete.lua | 9 +++++++++ tests/test_feed.py | 3 +-- thoonk/pubsub.py | 40 +++++++++++++++------------------------- 5 files changed, 49 insertions(+), 27 deletions(-) create mode 100644 scripts/config.lua create mode 100644 scripts/create.lua create mode 100644 scripts/delete.lua diff --git a/scripts/config.lua b/scripts/config.lua new file mode 100644 index 0000000..cd664c9 --- /dev/null +++ b/scripts/config.lua @@ -0,0 +1,11 @@ +-- ARGV: name, config(json), instance +if redis.call('sismember', 'feeds', name) then + return false +end +config = cjson.decode(ARGV[2]) +feed = 'feed.config:'..ARGV[1] +table.foreach(config, function(k, v) + redis.call('hset', feed, k, v) +end) +redis.call('publish', 'conffeed', ARGV[1]..'\0'..ARGV[3]) +return true diff --git a/scripts/create.lua b/scripts/create.lua new file mode 100644 index 0000000..038f835 --- /dev/null +++ b/scripts/create.lua @@ -0,0 +1,13 @@ +-- ARGV: name, config, instance +if redis.call('sadd', 'feeds', ARGV[1]) == 0 then + -- feed already exists + return false +end +feed = 'feed.config:'..ARGV[1] +config = cjson.decode(ARGV[2]) +-- TODO: check if config has a type key +for k, v in pairs(config) do + redis.call('hset', feed, k, v) +end +redis.call('publish', 'newfeed', ARGV[1]..'\0'..ARGV[3]) +return true diff --git a/scripts/delete.lua b/scripts/delete.lua new file mode 100644 index 0000000..f424d8e --- /dev/null +++ b/scripts/delete.lua @@ -0,0 +1,9 @@ +-- ARGS: feed, instance +if redis.call('srem', 'feeds', ARGV[1]) == 0 then + return false +end +feedtype = redis.call('hget', 'feed.config:'..ARGV[1], 'type') +-- TODO: delete schema keys! +redis.call('del', 'feed.config:'..ARGV[1]) +redis.call('publish', 'delfeed', ARGV[1]..'\0'..ARGV[2]) +return true diff --git a/tests/test_feed.py b/tests/test_feed.py index 0a766e7..cf13856 100644 --- a/tests/test_feed.py +++ b/tests/test_feed.py @@ -2,7 +2,6 @@ from thoonk.feeds import Feed import unittest from ConfigParser import ConfigParser -import threading class TestLeaf(unittest.TestCase): @@ -13,7 +12,7 @@ def setUp(self, *args, **kwargs): self.ps = thoonk.Thoonk(host=conf.get('Test', 'host'), port=conf.getint('Test', 'port'), db=conf.getint('Test', 'db'), - listen=True) + listen=False) self.ps.redis.flushdb() else: print 'No test configuration found in test.cfg' diff --git a/thoonk/pubsub.py b/thoonk/pubsub.py index bc83d95..600b002 100644 --- a/thoonk/pubsub.py +++ b/thoonk/pubsub.py @@ -10,6 +10,7 @@ from thoonk import feeds, cache from thoonk.exceptions import FeedExists, FeedDoesNotExist, NotListening import os +import json class Thoonk(object): @@ -92,7 +93,9 @@ def __init__(self, host='localhost', port=6379, db=0, listen=False): for filename in filenames: if filename.endswith(".lua"): f = open(os.path.join(dirpath, filename), "r") - self.scripts[dirpath[8:]+"/"+filename[:-4]] = self.redis.script("LOAD", f.read()) + if len(dirpath) > 8: + filename = dirpath[8:]+"/"+filename + self.scripts[filename[:-4]] = self.redis.script("LOAD", f.read()) self.listening = listen self.feed_publish = 'feed.publish:%s' @@ -207,9 +210,12 @@ def create_feed(self, feed, config): feed -- The name of the new feed. config -- A dictionary of configuration values. """ - if not self.redis.sadd("feeds", feed): + if 'type' not in config: + config['type'] = 'feed' + success = self.redis.evalsha(self.scripts["create"], 0, feed, + json.dumps(config), self.instance) + if not success: raise FeedExists - self.set_config(feed, config, True) def delete_feed(self, feed): """ @@ -218,18 +224,9 @@ def delete_feed(self, feed): Arguments: feed -- The name of the feed. """ - feed_instance = self._feeds[feed] - - def _delete_feed(pipe): - if not pipe.sismember('feeds', feed): - raise FeedDoesNotExist - pipe.multi() - pipe.srem("feeds", feed) - for key in feed_instance.get_schemas(): - pipe.delete(key) - self._publish('delfeed', (feed, self.instance), pipe) - - self.redis.transaction(_delete_feed, 'feeds') + success = self.redis.evalsha(self.scripts["delete"], 0, feed, self.instance) + if not success: + raise FeedDoesNotExist def set_config(self, feed, config, new_feed=False): """ @@ -239,17 +236,10 @@ def set_config(self, feed, config, new_feed=False): feed -- The name of the feed. config -- A dictionary of configuration values. """ - if not self.feed_exists(feed): + success = self.redis.evalsha(self.scripts["config"], 0, feed, + self.instance, json.dumps(config)) + if not success: raise FeedDoesNotExist - if u'type' not in config: - config[u'type'] = u'feed' - pipe = self.redis.pipeline() - for k, v in config.iteritems(): - pipe.hset('feed.config:' + feed, k, v) - pipe.execute() - if new_feed: - self._publish('newfeed', (feed, self.instance)) - self._publish('conffeed', (feed, self.instance)) def get_feed_names(self): """ From 4b632e58108d4df412acd167e747bdc21ca61ddd Mon Sep 17 00:00:00 2001 From: Simon Hewitt Date: Sun, 20 Nov 2011 16:34:54 +0000 Subject: [PATCH 15/17] delete all feed keys when deleting feed --- scripts/delete.lua | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/scripts/delete.lua b/scripts/delete.lua index f424d8e..c282fce 100644 --- a/scripts/delete.lua +++ b/scripts/delete.lua @@ -2,8 +2,39 @@ if redis.call('srem', 'feeds', ARGV[1]) == 0 then return false end +schema = { + feed = function(name) return { + 'feed.config:'..name, + 'feed.ids:'..name, + 'feed.items:'..name, + 'feed.publishes:'..name + } end, + sortedfeed = function(name) return { + 'feed.config:'..name, + 'feed.ids:'..name, + 'feed.items:'..name, + 'feed.publishes:'..name, + 'feed.idincr:'..name + } end, + queue = function(name) return { + 'feed.config:'..name, + 'feed.ids:'..name, + 'feed.items:'..name, + 'feed.publishes:'..name + } end, + job = function(name) return { + 'feed.config:'..name, + 'feed.ids:'..name, + 'feed.items:'..name, + 'feed.publishes:'..name, + 'feed.published:'..name, + 'feed.claimed:'..name, + 'feed.cancelled:'..name, + 'feed.finishes:'..name, + 'feed.stalled:'..name + } end +} feedtype = redis.call('hget', 'feed.config:'..ARGV[1], 'type') --- TODO: delete schema keys! -redis.call('del', 'feed.config:'..ARGV[1]) +redis.call('del', unpack(schema[feedtype](ARGV[1]))) redis.call('publish', 'delfeed', ARGV[1]..'\0'..ARGV[2]) return true From 9139bbeacdf3b13432626292482b973c6f1d83b0 Mon Sep 17 00:00:00 2001 From: Simon Hewitt Date: Sun, 20 Nov 2011 17:48:17 +0000 Subject: [PATCH 16/17] added scripts for feeds --- scripts/feed/publish.lua | 19 ++++++++++++++++ scripts/feed/retract.lua | 8 +++++++ thoonk/feeds/feed.py | 48 ++++++---------------------------------- 3 files changed, 34 insertions(+), 41 deletions(-) create mode 100644 scripts/feed/publish.lua create mode 100644 scripts/feed/retract.lua diff --git a/scripts/feed/publish.lua b/scripts/feed/publish.lua new file mode 100644 index 0000000..a9c18a1 --- /dev/null +++ b/scripts/feed/publish.lua @@ -0,0 +1,19 @@ +-- ARGV: feed, id, item, time +max = redis.call("hget", "feed.config:"..ARGV[1], "max_length") +if max and tonumber(max) > 0 then + ids = redis.call('zrange', 'feed.ids:'..ARGV[1], 0, -tonumber(max)) + table.foreach(ids, function(i, id) + redis.call('zrem', 'feed.ids:'..ARGV[1], id) + redis.call('hdel', 'feed.items:'..ARGV[1], id) + redis.call('publish', 'feed.retract:'..ARGV[1], id) + end) +end + +redis.call('incr', 'feed.publishes:'..ARGV[1]) +redis.call('hset', 'feed.items:'..ARGV[1], ARGV[2], ARGV[3]) +if redis.call('zadd', 'feed.ids:'..ARGV[1], ARGV[4], ARGV[2]) == 1 then + redis.call('publish', 'feed.edit:'..ARGV[1], ARGV[2]..'\0'..ARGV[3]) +else + redis.call('publish', 'feed.publish:'..ARGV[1], ARGV[2]..'\0'..ARGV[3]) +end +return zadd diff --git a/scripts/feed/retract.lua b/scripts/feed/retract.lua new file mode 100644 index 0000000..910cf60 --- /dev/null +++ b/scripts/feed/retract.lua @@ -0,0 +1,8 @@ +-- ARGV: feed, id +if redis.call('zrem', 'feed.ids:'..ARGV[1], ARGV[2]) == 0 then + return false +end +redis.call('hdel', 'feed.items:'..ARGV[1], ARGV[2]) +redis.call('publish', 'feed.retract:'..ARGV[1], ARGV[2]) +return true + diff --git a/thoonk/feeds/feed.py b/thoonk/feeds/feed.py index cde7556..cc11f36 100644 --- a/thoonk/feeds/feed.py +++ b/thoonk/feeds/feed.py @@ -5,13 +5,7 @@ import time import uuid -try: - import queue -except ImportError: - import Queue as queue - -from thoonk.exceptions import * -import redis.exceptions +from thoonk.exceptions import ItemDoesNotExist class Feed(object): @@ -160,32 +154,8 @@ def publish(self, item, id=None): publish_id = id if publish_id is None: publish_id = uuid.uuid4().hex - - def _publish(pipe): - max = int(pipe.hget(self.feed_config, "max_length") or 0) - if max > 0: - delete_ids = pipe.zrange(self.feed_ids, 0, -max) - pipe.multi() - for id in delete_ids: - if id != publish_id: - pipe.zrem(self.feed_ids, id) - pipe.hdel(self.feed_items, id) - self.thoonk._publish(self.feed_retract, (id,), pipe) - else: - pipe.multi() - pipe.zadd(self.feed_ids, **{publish_id: time.time()}) - pipe.incr(self.feed_publishes) - pipe.hset(self.feed_items, publish_id, item) - - results = self.redis.transaction(_publish, self.feed_ids) - - if results[-3]: - # If zadd was successful - self.thoonk._publish(self.feed_publish, (publish_id, item)) - else: - self.thoonk._publish(self.feed_edit, (publish_id, item)) - - return publish_id + return self.redis.evalsha(self.thoonk.scripts["feed/publish"], 0, + self.feed, publish_id, item, int(time.time()*1000)) def retract(self, id): """ @@ -194,11 +164,7 @@ def retract(self, id): Arguments: id -- The ID value of the item to remove. """ - def _retract(pipe): - if pipe.zrank(self.feed_ids, id) is not None: - pipe.multi() - pipe.zrem(self.feed_ids, id) - pipe.hdel(self.feed_items, id) - self.thoonk._publish(self.feed_retract, (id,), pipe) - - self.redis.transaction(_retract, self.feed_ids) + success = self.redis.evalsha(self.thoonk.scripts["feed/retract"], 0, + self.feed, id) + if not success: + raise ItemDoesNotExist \ No newline at end of file From 2a83a3a420afce4b1ececb18452b7be4e42c3e60 Mon Sep 17 00:00:00 2001 From: Simon Hewitt Date: Sun, 20 Nov 2011 17:53:34 +0000 Subject: [PATCH 17/17] tidied up feed publish return values --- thoonk/feeds/feed.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/thoonk/feeds/feed.py b/thoonk/feeds/feed.py index cc11f36..6741a7c 100644 --- a/thoonk/feeds/feed.py +++ b/thoonk/feeds/feed.py @@ -150,12 +150,15 @@ def publish(self, item, id=None): item -- The content of the item to add to the feed. id -- Optional ID to use for the item, if the ID already exists, the existing item will be replaced. + + Returns a tuple containing the ID of the item and a flag indicating if + the item was created or edited """ - publish_id = id - if publish_id is None: - publish_id = uuid.uuid4().hex - return self.redis.evalsha(self.thoonk.scripts["feed/publish"], 0, - self.feed, publish_id, item, int(time.time()*1000)) + if id is None: + id = uuid.uuid4().hex + created = self.redis.evalsha(self.thoonk.scripts["feed/publish"], 0, + self.feed, id, item, int(time.time()*1000)) + return id, created def retract(self, id): """