Merge pull request nusdbsystem#61 from airlovelq/redis_password

Using Redis password

scripts/docker_swarm/.env.sh

@@ -51,6 +51,7 @@ export ADMIN_HOST=singa_auto_admin
 export ADMIN_PORT=3000
 export REDIS_HOST=singa_auto_redis
 export REDIS_PORT=6379
+export REDIS_PASSWORD=singa_auto
 export PREDICTOR_PORT=3003
 export WEB_ADMIN_HOST=singa_auto_admin_web
 export ZOOKEEPER_HOST=singa_auto_zookeeper
@@ -81,3 +82,5 @@ export IMAGE_KAFKA=wurstmeister/kafka:2.12-2.1.1
 # Utility configuration
 export PYTHONPATH=$PWD # Ensures that `singa_auto` module can be imported at project root
 export PYTHONUNBUFFERED=1 # Ensures logs from Python appear instantly
+
+export CONTAINER_MODE=SWARM

scripts/docker_swarm/start_admin.sh

@@ -56,6 +56,7 @@ title "Starting SINGA-Auto's Admin..."
   -e ADMIN_PORT=$ADMIN_PORT \
   -e REDIS_HOST=$REDIS_HOST \
   -e REDIS_PORT=$REDIS_PORT \
+  -e REDIS_PASSWORD=$REDIS_PASSWORD \
   -e KAFKA_HOST=$KAFKA_HOST \
   -e KAFKA_PORT=$KAFKA_PORT \
   -e PREDICTOR_PORT=$PREDICTOR_PORT \
@@ -70,6 +71,7 @@ title "Starting SINGA-Auto's Admin..."
   -e PARAMS_DIR_PATH=$PARAMS_DIR_PATH \
   -e LOGS_DIR_PATH=$LOGS_DIR_PATH \
   -e APP_MODE=$APP_MODE \
+  -e CONTAINER_MODE=$CONTAINER_MODE \
   -v /var/run/docker.sock:/var/run/docker.sock \
   $VOLUME_MOUNTS \
   -p $ADMIN_EXT_PORT:$ADMIN_PORT \

scripts/docker_swarm/start_redis.sh

@@ -36,7 +36,7 @@ title "Starting SINGA-Auto's Redis..."
 (docker run --rm --name $REDIS_HOST \
   --network $DOCKER_NETWORK \
   -p $REDIS_EXT_PORT:$REDIS_PORT \
-  $IMAGE_REDIS \
+  $IMAGE_REDIS redis-server --appendonly yes --requirepass $REDIS_PASSWORD \
   &> $LOG_FILE_PATH) &
 
 ensure_stable "SINGA-Auto's Redis" $LOG_FILE_PATH 2

scripts/kubernetes/.env.sh

@@ -58,6 +58,7 @@ export ADMIN_HOST=singa-auto-admin
 export ADMIN_PORT=3000
 export REDIS_HOST=singa-auto-redis
 export REDIS_PORT=6379
+export REDIS_PASSWORD=singa_auto
 export PREDICTOR_PORT=3003
 export WEB_ADMIN_HOST=singa-auto-admin-web
 export ZOOKEEPER_HOST=singa-auto-zookeeper

scripts/kubernetes/create_config.py

@@ -83,6 +83,7 @@
     DB_DIR_PATH = sys.argv[51]
     INGRESS_NAME = sys.argv[52]
     INGRESS_EXT_PORT = sys.argv[53]
+    REDIS_PASSWORD = sys.argv[54]
 
     #zk service
     content = {}
@@ -275,9 +276,12 @@
     container = {}
     container.setdefault('name', REDIS_HOST)
     container.setdefault('image', IMAGE_REDIS)
-    template.setdefault('spec', {'containers': [container]})
+    container.setdefault('args', ['--appendonly','yes', "--requirepass", REDIS_PASSWORD])
+    # volumes = {}
+    # volumes.setdefault('name', 'redis-data')
+    # volumes.setdefault('nfs', {'server': RAFIKI_ADDR, 'path': HOST_WORKDIR_PATH + '/database/redis'})
+    template.setdefault('spec', {'containers': [container] })
     with open('{}/scripts/kubernetes/start_redis_deployment.json'.format(PYTHONPATH), 'w') as f:
-
         f.write(json.dumps(content, indent=4))
 
     #admin deployment
@@ -327,6 +331,7 @@
     env.append({'name': 'ADMIN_PORT', 'value': ADMIN_PORT})
     env.append({'name': 'REDIS_HOST', 'value': REDIS_HOST})
     env.append({'name': 'REDIS_PORT', 'value': REDIS_PORT})
+    env.append({'name': 'REDIS_PASSWORD', 'value': REDIS_PASSWORD})
     env.append({'name': 'KAFKA_HOST', 'value': KAFKA_HOST})
     env.append({'name': 'KAFKA_PORT', 'value': KAFKA_PORT})
     env.append({'name': 'PREDICTOR_PORT', 'value': PREDICTOR_PORT})

scripts/kubernetes/generate_config.sh

@@ -72,4 +72,5 @@ $CONTAINER_MODE \
 $CLUSTER_MODE \
 $DB_DIR_PATH \
 $INGRESS_NAME \
-$INGRESS_EXT_PORT
+$INGRESS_EXT_PORT \
+$REDIS_PASSWORD

singa_auto/admin/services_manager.py

@@ -41,7 +41,7 @@ class ServiceDeploymentError(Exception):
 ENVIRONMENT_VARIABLES_AUTOFORWARD = [
     'POSTGRES_HOST', 'POSTGRES_PORT', 'POSTGRES_USER', 'POSTGRES_PASSWORD', 'POSTGRES_DB',
     'SUPERADMIN_PASSWORD',
-    'REDIS_HOST', 'REDIS_PORT',
+    'REDIS_HOST', 'REDIS_PORT', 'REDIS_PASSWORD',
     'ADMIN_HOST', 'ADMIN_PORT',
     'DATA_DIR_PATH', 'LOGS_DIR_PATH', 'PARAMS_DIR_PATH',
     'KAFKA_HOST', 'KAFKA_PORT',

singa_auto/redis/redis.py

@@ -21,6 +21,7 @@
 import time
 import logging
 import msgpack
+import os
 
 logger = logging.getLogger(__name__)
 
@@ -47,7 +48,7 @@ def __init__(
         self._uid = str(
             uuid.uuid4())  # Process identifier for distributed locking
         self._namespace = namespace
-        self._redis = self._make_redis_client(redis_host, redis_port)
+        self._redis = self._make_redis_client(redis_host, redis_port, os.environ.get('REDIS_PASSWORD', None))
 
     def acquire_lock(self):
         lock_value = self._uid
@@ -141,12 +142,12 @@ def _decode_value(self, value):
     def _get_redis_name(self, name):
         return '{}:{}'.format(self._namespace, name)
 
-    def _make_redis_client(self, host, port):
+    def _make_redis_client(self, host, port, passwd):
         if host is not None and port is not None:
             import redis
             from redis import ConnectionPool
             from redis import StrictRedis
-            cache_connection_url = 'redis://{}:{}'.format(host, port)
+            cache_connection_url = 'redis://:{}@{}:{}'.format(passwd, host, port)
             connection_pool = ConnectionPool.from_url(cache_connection_url)
             client = StrictRedis(connection_pool=connection_pool,
                                  decode_responses=True)