Merge pull request #1 from binux/master

update

README.md

@@ -50,7 +50,7 @@ Installation
 * `pip install pyspider`
 * run command `pyspider`, visit [http://localhost:5000/](http://localhost:5000/)
 
-**WARNING:** WebUI is opened to public by default, it can be used to execute any command which may harm to you system. Please use it in internal network or [enable `need-auth` for webui](http://docs.pyspider.org/en/latest/Command-Line/#-config).
+**WARNING:** WebUI is open to the public by default, it can be used to execute any command which may harm your system. Please use it in an internal network or [enable `need-auth` for webui](http://docs.pyspider.org/en/latest/Command-Line/#-config).
 
 Quickstart: [http://docs.pyspider.org/en/latest/Quickstart/](http://docs.pyspider.org/en/latest/Quickstart/)
 

docs/Command-Line.md

@@ -94,6 +94,7 @@ beanstalk:
     beanstalk://host:11300/
 redis:
     redis://host:6379/db
+    redis://host1:port1,host2:port2,...,hostn:portn (for redis 3.x in cluster mode)
 kombu:
     kombu+transport://userid:password@hostname:port/virtual_host
     see http://kombu.readthedocs.org/en/latest/userguide/connections.html#urls

docs/Deployment.md

@@ -85,6 +85,7 @@ beanstalk:
     beanstalk://host:11300/
 redis:
     redis://host:6379/db
+    redis://host1:port1,host2:port2,...,hostn:portn (for redis 3.x in cluster mode)
 builtin:
     None
 ```

pyspider/message_queue/__init__.py

@@ -27,6 +27,7 @@ def connect_message_queue(name, url=None, maxsize=0, lazy_limit=True):
         beanstalk://host:11300/
     redis:
         redis://host:6379/db
+        redis://host1:port1,host2:port2,...,hostn:portn (for redis 3.x in cluster mode)
     kombu:
         kombu+transport://userid:password@hostname:port/virtual_host
         see http://kombu.readthedocs.org/en/latest/userguide/connections.html#urls
@@ -47,19 +48,31 @@ def connect_message_queue(name, url=None, maxsize=0, lazy_limit=True):
         return Queue(name, host=parsed.netloc, maxsize=maxsize)
     elif parsed.scheme == 'redis':
         from .redis_queue import Queue
-        db = parsed.path.lstrip('/').split('/')
-        try:
-            db = int(db[0])
-        except:
-            logging.warning('redis DB must zero-based numeric index, using 0 instead')
-            db = 0
+        if ',' in parsed.netloc:
+            """
+            redis in cluster mode (there is no concept of 'db' in cluster mode)
+            ex. redis://host1:port1,host2:port2,...,hostn:portn
+            """
+            cluster_nodes = []
+            for netloc in parsed.netloc.split(','):
+                cluster_nodes.append({'host': netloc.split(':')[0], 'port': int(netloc.split(':')[1])})
 
-        password = parsed.password or None
+            return Queue(name=name, maxsize=maxsize, lazy_limit=lazy_limit, cluster_nodes=cluster_nodes)
 
-        return Queue(name, parsed.hostname, parsed.port, db=db, maxsize=maxsize, password=password, lazy_limit=lazy_limit)
+        else:
+            db = parsed.path.lstrip('/').split('/')
+            try:
+                db = int(db[0])
+            except:
+                logging.warning('redis DB must zero-based numeric index, using 0 instead')
+                db = 0
+
+            password = parsed.password or None
+
+            return Queue(name=name, host=parsed.hostname, port=parsed.port, db=db, maxsize=maxsize, password=password, lazy_limit=lazy_limit)
     elif url.startswith('kombu+'):
         url = url[len('kombu+'):]
         from .kombu_queue import Queue
         return Queue(name, url, maxsize=maxsize, lazy_limit=lazy_limit)
     else:
-        raise Exception('unknow connection url: %s', url)
+        raise Exception('unknown connection url: %s', url)

pyspider/message_queue/redis_queue.py

@@ -21,7 +21,7 @@ class RedisQueue(object):
     max_timeout = 0.3
 
     def __init__(self, name, host='localhost', port=6379, db=0,
-                 maxsize=0, lazy_limit=True, password=None):
+                 maxsize=0, lazy_limit=True, password=None, cluster_nodes=None):
         """
         Constructor for RedisQueue
 
@@ -31,7 +31,11 @@ def __init__(self, name, host='localhost', port=6379, db=0,
                     for better performance.
         """
         self.name = name
-        self.redis = redis.StrictRedis(host=host, port=port, db=db, password=password)
+        if(cluster_nodes is not None):
+            from rediscluster import StrictRedisCluster
+            self.redis = StrictRedisCluster(startup_nodes=cluster_nodes)
+        else:
+            self.redis = redis.StrictRedis(host=host, port=port, db=db, password=password)
         self.maxsize = maxsize
         self.lazy_limit = lazy_limit
         self.last_qsize = 0

requirements.txt

@@ -18,6 +18,7 @@ SQLAlchemy>=0.9.7
 six>=1.5.0
 amqp>=1.3.0,<2.0
 redis
+redis-py-cluster
 kombu
 psycopg2
 elasticsearch

setup.py

@@ -54,6 +54,7 @@
     'pymongo>=2.7.2',
     'SQLAlchemy>=0.9.7',
     'redis',
+    'redis-py-cluster',
     'psycopg2',
     'elasticsearch>=2.0.0,<2.4.0',
 ]