feat(config): support starting pd nodes & store nodes (#16)

subtask of apache/incubator-hugegraph#2575
* Update src/deploy_start.py

---------

Co-authored-by: imbajin <[email protected]>

README.md

@@ -23,7 +23,12 @@ Note: modify the configs in basic_config.py before run the test script
 2. modify server/toolchain_git_branch_commit
 3. modify auth/https/serice ports if need..
 '''
-python deploy_start.py all # or 'server' or 'toolchain'
+python deploy_start.py all # or 'server' or 'toolchain' or 'hugegraph'
+
+'''
+Or you can set backend with followed command
+'''
+python deploy_start.py hugegraph hstore # or other backend supported by hugegraph
 
 # 2. decompress dataset
 unzip config/dataset.zip -d config/

src/common/deploy_graph.py

@@ -5,8 +5,10 @@
 create_time: 2020/4/22 5:17 下午
 """
 import os
+import shutil
 import subprocess
 import sys
+import time
 
 from config.basic_config import admin_password
 
@@ -118,6 +120,50 @@ def set_server_properties(package_dir_path, host, server_port, gremlin_port):
 }
 ''')
 
+def set_pd_properties(package_dir_path, host, grpc_port, rest_port, store_list, raft_port, raft_list):
+    """
+    修改 pd 组件配置
+    :return:
+    """
+    # 修改 application 文件
+    application_conf = package_dir_path + '/conf/application.yml'
+    alter_properties(application_conf,
+                     '8686',
+                     '%d' % grpc_port)
+    alter_properties(application_conf,
+                     '8620',
+                     '%d' % rest_port)
+    new_store_list = ','.join([f"{host}:{port}" for port in store_list])
+    alter_properties(application_conf,
+                     'initial-store-list: 127.0.0.1:8500',
+                     'initial-store-list: %s' % new_store_list)
+    alter_properties(application_conf,
+                     'address: 127.0.0.1:8610',
+                     'address: %s:%d' % (host, raft_port))
+    new_raft_list = ','.join([f"{host}:{port}" for port in raft_list])
+    alter_properties(application_conf,
+                     'peers-list: 127.0.0.1:8610',
+                     'peers-list: %s' % new_raft_list)
+
+def set_store_properties(package_dir_path, host, pd_list, grpc_port, raft_port, rest_port):
+    application_conf = package_dir_path + '/conf/application.yml'
+    new_pd_list = ','.join([f"{host}:{port}" for port in pd_list])
+    alter_properties(application_conf,
+                     'address: localhost:8686',
+                     'address: %s' % new_pd_list)
+    alter_properties(application_conf,
+                     'port: 8500',
+                     'port: %d' % grpc_port)
+    alter_properties(application_conf,
+                     'address: 127.0.0.1:8510',
+                     'address: %s:%d' % (host, raft_port))
+    alter_properties(application_conf,
+                     'address: 127.0.0.1:8610',
+                     'address: %s:%d' % (host, raft_port))
+    alter_properties(application_conf,
+                     'port: 8520',
+                     'port: %d' % rest_port)
+
 
 def set_hubble_properties(package_dir_path, host, port):
     """
@@ -128,6 +174,21 @@ def set_hubble_properties(package_dir_path, host, port):
     alter_properties(hubble_conf, 'server.host=localhost', 'server.host=%s' % host)
     alter_properties(hubble_conf, 'server.port=8088', 'server.port=%d' % port)
 
+def ensure_start(log_file_path, target, interval = 1):
+    while not os.path.exists(log_file_path):
+        time.sleep(interval)
+    with open(log_file_path, 'r') as log_file:
+        log_file.seek(0, 2)
+        while True:
+            line = log_file.readline()
+            if not line:
+                time.sleep(interval)
+                continue
+            # 检查目标语句是否在行中
+            if target in line:
+                return
+
+
 
 def start_graph(package_dir_path, graph_type):
     """
@@ -140,6 +201,21 @@ def start_graph(package_dir_path, graph_type):
             f'&& echo "{pa}" | ./bin/init-store.sh '
             '&& ./bin/start-hugegraph.sh' % package_dir_path
         )
+    elif graph_type == 'pd':
+        os.system(
+            'cd %s '
+            '&& ./bin/start-hugegraph-pd.sh' % package_dir_path
+        )
+    elif graph_type == 'store':
+        os.system(
+            'cd %s '
+            '&& ./bin/start-hugegraph-store.sh' % package_dir_path
+        )
+    elif graph_type == 'hugegraph_server':
+        os.system(
+            'cd %s '
+            '&& ./bin/start-hugegraph.sh' % package_dir_path
+        )
     else:
         os.system(
             f'chmod -R 755 {package_dir_path}'
@@ -151,6 +227,11 @@ def unzip_targz(file_path, file_name):
     cmd = f'cd {file_path} && tar -zxvf {file_name}'
     subprocess.check_call(cmd, shell=True)
 
+def update_backend_properties(file_path, target_path):
+    if(os.path.exists(target_path)):
+        os.remove(target_path)
+    shutil.copy2(file_path, target_path)
+
 
 class Deploy:
     """
@@ -168,6 +249,16 @@ def __init__(self, obj):
         self.loader_git = obj.loader_git
         self.tools_git = obj.tools_git
         self.hubble_git = obj.hubble_git
+        self.host = obj.host
+        self.pd_grpc_port = obj.pd_grpc_port
+        self.pd_rest_port = obj.pd_rest_port
+        self.store_list = obj.store_list
+        self.pd_raft_port = obj.pd_raft_port
+        self.raft_list = obj.raft_list
+        self.pd_list = obj.pd_list
+        self.store_grpc_port = obj.store_grpc_port
+        self.store_raft_port = obj.store_raft_port
+        self.store_rest_port = obj.store_rest_port
 
     @staticmethod
     def server(conf):
@@ -189,6 +280,51 @@ def server(conf):
         )
         start_graph(gen_dir, 'server')
 
+    @staticmethod
+    def pd(conf):
+        """
+        :return:
+        """
+        is_exists_path(conf.codebase_path)
+        get_code(conf.codebase_path, conf.server_git, conf.pd_local_repo)
+        compile_package(conf.project_path)
+        unzip_targz(conf.pd_path, conf.pd_tar_path.split('/')[-1])
+
+        gen_dir = os.path.join(conf.codebase_path, conf.pd_gen_dir)
+        # start graph_server
+        set_pd_properties(
+            gen_dir,
+            conf.host,
+            conf.pd_grpc_port,
+            conf.pd_rest_port,
+            conf.store_list,
+            conf.pd_raft_port,
+            conf.raft_list
+        )
+        start_graph(gen_dir, 'pd')
+
+    @staticmethod
+    def store(conf):
+        """
+        :return:
+        """
+        is_exists_path(conf.codebase_path)
+        get_code(conf.codebase_path, conf.server_git, conf.store_local_repo)
+        compile_package(conf.project_path)
+        unzip_targz(conf.store_path, conf.store_tar_path.split('/')[-1])
+
+        gen_dir = os.path.join(conf.codebase_path, conf.store_gen_dir)
+        # start graph_server
+        set_store_properties(
+            gen_dir,
+            conf.host,
+            conf.pd_list,
+            conf.store_grpc_port,
+            conf.store_raft_port,
+            conf.store_rest_port
+        )
+        start_graph(gen_dir, 'store')
+
     @staticmethod
     def toolchain(conf):
         is_exists_path(conf.codebase_path)
@@ -201,6 +337,59 @@ def toolchain(conf):
         # set_hubble_properties(hubble_package_dir_name, conf.graph_host, conf.hubble_port)
         start_graph(conf.hubble_path, 'hubble')
 
+    @staticmethod
+    def hugegraph(conf):
+        is_exists_path(conf.codebase_path)
+        get_code(conf.codebase_path, conf.server_git, 'incubator-hugegraph')
+        compile_package(conf.project_path)
+
+        unzip_targz(conf.pd_path, conf.pd_tar_path.split('/')[-1])
+        unzip_targz(conf.store_path, conf.store_tar_path.split('/')[-1])
+        unzip_targz(conf.server_path, conf.server_tar_path.split('/')[-1])
+
+        pd_gen_dir = os.path.join(conf.codebase_path, conf.pd_gen_dir)
+        # start graph_server
+        set_pd_properties(
+            pd_gen_dir,
+            conf.host,
+            conf.pd_grpc_port,
+            conf.pd_rest_port,
+            conf.store_list,
+            conf.pd_raft_port,
+            conf.raft_list
+        )
+        start_graph(pd_gen_dir, 'pd')
+        ensure_start(pd_gen_dir + "/logs/hugegraph-pd-stdout.log",
+                     f'Hugegraph-pd started.')
+
+        store_gen_dir = os.path.join(conf.codebase_path, conf.store_gen_dir)
+        # start graph_server
+        set_store_properties(
+            store_gen_dir,
+            conf.host,
+            conf.pd_list,
+            conf.store_grpc_port,
+            conf.store_raft_port,
+            conf.store_rest_port
+        )
+        start_graph(store_gen_dir, 'store')
+        ensure_start(store_gen_dir + "/logs/hugegraph-store.log",
+                     f'o.a.h.s.n.StoreNodeApplication - Starting StoreNodeApplication')
+
+        server_gen_dir = os.path.join(conf.codebase_path, conf.server_gen_dir)
+        # start graph_server
+        set_server_properties(
+            server_gen_dir,
+            conf.graph_host,
+            conf.server_port,
+            conf.gremlin_port
+        )
+        config_file_path = os.path.dirname(os.path.realpath(__file__)) + "/../dist/" + conf.server_backend + ".properties"
+        target_path = os.path.join(conf.codebase_path, conf.server_gen_dir + '/conf/graphs/hugegraph.properties')
+        update_backend_properties(config_file_path, target_path)
+        start_graph(server_gen_dir, 'hugegraph_server')
+
+
 
 if __name__ == "__main__":
     pass

src/config/basic_config.py

@@ -12,17 +12,25 @@
 # apache release version
 is_incubating = 'incubating-'
 # TODO: consider user * instead of fixed version?
-server_release_version = '1.5.0'
+hugegraph_release_version = '1.5.0'
 toolchain_release_version = '1.3.0'
+pd_local_repo = 'incubator-hugegraph/hugegraph-pd'
+store_local_repo = 'incubator-hugegraph/hugegraph-store'
 server_local_repo = 'incubator-hugegraph/hugegraph-server'
 toolchain_local_repo = 'incubator-hugegraph-toolchain'
-server_gen_dir = f'incubator-hugegraph/hugegraph-server/apache-hugegraph-server-{is_incubating}{server_release_version}'
+pd_gen_dir = f'incubator-hugegraph/hugegraph-pd/apache-hugegraph-pd-{is_incubating}{hugegraph_release_version}'
+store_gen_dir = f'incubator-hugegraph/hugegraph-store/apache-hugegraph-store-{is_incubating}{hugegraph_release_version}'
+server_gen_dir = f'incubator-hugegraph/hugegraph-server/apache-hugegraph-server-{is_incubating}{hugegraph_release_version}'
 toolchain_gen_dir = f'incubator-hugegraph-toolchain/apache-hugegraph-toolchain-{is_incubating}{toolchain_release_version}'
 toolchain_obj_template = 'apache-hugegraph-{tool_name}-' + is_incubating + f'{toolchain_release_version}'
 
 project_path = os.path.join(codebase_path, 'incubator-hugegraph')
+pd_path = os.path.join(codebase_path, pd_local_repo)
+store_path = os.path.join(codebase_path, store_local_repo)
 server_path = os.path.join(codebase_path, server_local_repo)
-server_tar_path = os.path.join(codebase_path, 'incubator-hugegraph/hugegraph-server', f'apache-hugegraph-server-{is_incubating}{server_release_version}' + '.tar.gz')
+pd_tar_path = os.path.join(codebase_path, pd_gen_dir + '.tar.gz')
+store_tar_path = os.path.join(codebase_path, store_gen_dir + '.tar.gz')
+server_tar_path = os.path.join(codebase_path, server_gen_dir + '.tar.gz')
 toolchain_path = os.path.join(codebase_path, toolchain_local_repo)
 loader_path = os.path.join(codebase_path, toolchain_gen_dir, toolchain_obj_template.format(tool_name='loader'))
 hubble_path = os.path.join(codebase_path, toolchain_gen_dir, toolchain_obj_template.format(tool_name='hubble'))
@@ -49,6 +57,18 @@
 admin_password = {'admin': 'admin'}
 test_password = {'tester': '123456'}
 
+host = '127.0.0.1'
+pd_grpc_port = 8686
+pd_rest_port = 8620
+store_list = [8500]
+pd_raft_port = 8610
+raft_list = [8610]
+
+pd_list = [8686]
+store_grpc_port = 8500
+store_raft_port = 8510
+store_rest_port = 8520
+
 # toolchain (includes loader, hubble, tools)
 toolchain_git = {
     'branch': '3b58fc6',

src/deploy_start.py

@@ -24,6 +24,12 @@ def graph_deploy(param, conf_obj):
         Deploy.server(conf_obj)
     elif param == 'toolchain':
         Deploy.toolchain(conf_obj)
+    elif param == 'pd':
+        Deploy.pd(conf_obj)
+    elif param == 'store':
+        Deploy.store(conf_obj)
+    elif param == 'hugegraph':
+        Deploy.hugegraph(conf_obj)
     else:
         Deploy.server(conf_obj)
         Deploy.toolchain(conf_obj)
@@ -32,8 +38,13 @@ def graph_deploy(param, conf_obj):
 if __name__ == "__main__":
     param_size = len(sys.argv)
     if param_size == 2 \
-            and sys.argv[1] in ['all', 'server', 'toolchain']:
+            and sys.argv[1] in ['all', 'server', 'toolchain', 'pd', 'store', 'hugegraph']:
+        graph_deploy(sys.argv[1], basic_config)
+    elif param_size == 3 \
+            and sys.argv[1] in ['all', 'server', 'toolchain', 'pd', 'store', 'hugegraph']\
+            and sys.argv[2] in ['hbase', 'hstore', 'cassandra', 'mysql', 'rocksdb', 'scylladb']:
+        basic_config.server_backend = sys.argv[2]
         graph_deploy(sys.argv[1], basic_config)
     else:
-        print('failed: 执行脚本参数为[all,server,toolchain]')
+        print('failed: 执行脚本参数为[all, hugegraph, toolchain, server, pd, store]')
         exit(1)

src/dist/cassandra.properties

@@ -0,0 +1,44 @@
+gremlin.graph=org.apache.hugegraph.HugeFactory
+
+# cache config
+#schema.cache_capacity=100000
+# vertex-cache default is 1000w, 10min expired
+vertex.cache_type=l2
+#vertex.cache_capacity=10000000
+#vertex.cache_expire=600
+# edge-cache default is 100w, 10min expired
+edge.cache_type=l2
+#edge.cache_capacity=1000000
+#edge.cache_expire=600
+
+
+# schema illegal name template
+#schema.illegal_name_regex=\s+|~.*
+
+#vertex.default_label=vertex
+
+store=hugegraph
+
+# task config
+task.scheduler_type=local
+task.schedule_period=10
+task.retry=0
+task.wait_timeout=10
+
+# search config
+search.text_analyzer=jieba
+search.text_analyzer_mode=INDEX
+
+backend=cassandra
+serializer=cassandra
+
+# cassandra backend config
+cassandra.host=localhost
+cassandra.port=9042
+cassandra.username=
+cassandra.password=
+#cassandra.connect_timeout=5
+#cassandra.read_timeout=20
+
+#cassandra.keyspace.strategy=SimpleStrategy
+#cassandra.keyspace.replication=3