init setup

Signed-off-by: zhihuiwan <[email protected]>

.gitignore

@@ -39,3 +39,6 @@ venv
 /python/fate_flow/localfs
 /python/fate_flow/*.env
 /python/fate_flow/conf
+/python/build
+/python/dist
+/python/*.egg-info

python/fate_flow/commands/server_cli.py

@@ -0,0 +1,184 @@
+#
+#  Copyright 2019 The FATE Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+import os
+import subprocess
+
+import click
+from ruamel import yaml
+
+import fate_flow
+
+CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
+HOME = os.path.dirname(fate_flow.__file__)
+SERVER_CONF_PATH = os.path.join(HOME, "conf", "service_conf.yaml")
+SETTING_PATH = os.path.join(HOME, "settings.py")
+SERVICE_SH = os.path.join(HOME, "commands", "service.sh")
+
+
+@click.group(short_help='Fate Flow', context_settings=CONTEXT_SETTINGS)
+@click.pass_context
+def flow_server_cli(ctx):
+    '''
+    Fate Flow server cli
+    '''
+    ctx.ensure_object(dict)
+    if ctx.invoked_subcommand == 'init':
+        return
+    pass
+
+
+@flow_server_cli.command('init', short_help='Flow Server init command')
+@click.option('--ip', type=click.STRING, help='Fate flow server ip address.')
+@click.option('--port', type=click.INT, help='Fate flow server http port.')
+@click.option('--home', type=click.STRING, help="Service's home directory, used to store essential information "
+                                                "such as data, logs, and more.")
+def initialization(**kwargs):
+    """
+    \b
+    - DESCRIPTION:
+        Flow Init Command. provide ip and port of a valid fate flow server.
+
+    \b
+    - USAGE:
+        fate_flow init --ip 127.0.0.1 --port 9380 --home /data/projects/fate_flow
+
+    """
+    init_server(kwargs.get("ip"), kwargs.get("port"), kwargs.get("home"))
+
+
+@flow_server_cli.command('start', short_help='Start run flow server')
+def start(**kwargs):
+    """
+    \b
+    - DESCRIPTION:
+        Start FATE Flow Server Command.
+
+    \b
+    - USAGE:
+        fate_flow start
+
+    """
+    run_command("start")
+
+
+@flow_server_cli.command('status', short_help='Query fate flow server status')
+def status(**kwargs):
+    """
+    \b
+    - DESCRIPTION:
+        Query fate flow server status command
+
+    \b
+    - USAGE:
+        fate_flow status
+
+    """
+    run_command("status")
+
+
+@flow_server_cli.command('stop', short_help='Stop run flow server')
+def stop(**kwargs):
+    """
+    \b
+    - DESCRIPTION:
+        Stop FATE Flow Server Command.
+
+    \b
+    - USAGE:
+        fate_flow stop
+
+    """
+    run_command("stop")
+
+
+@flow_server_cli.command('restart', short_help='Restart fate flow server')
+def restart(**kwargs):
+    """
+    \b
+    - DESCRIPTION:
+        ReStart FATE Flow Server Command.
+
+    \b
+    - USAGE:
+        fate_flow restart
+
+    """
+    run_command("restart")
+
+
+@flow_server_cli.command('version', short_help='Flow Server Version Command')
+def get_version():
+    import fate_flow
+    print(fate_flow.__version__)
+
+
+def replace_settings(home_path):
+    import re
+    with open(SETTING_PATH, "r") as file:
+        content = file.read()
+    content = re.sub(r"DATA_DIR.*", f"DATA_DIR = \"{home_path}/data\"", content)
+    content = re.sub(r"MODEL_DIR.*", f"MODEL_DIR = \"{home_path}/model\"", content)
+    content = re.sub(r"JOB_DIR.*", f"JOB_DIR = \"{home_path}/jobs\"", content)
+    content = re.sub(r"LOG_DIR.*", f"LOG_DIR = \"{home_path}/logs\"", content)
+    content = re.sub(r"SQLITE_FILE_NAME.*", f"SQLITE_FILE_NAME = \"{home_path}/fate_flow_sqlite.db\"", content)
+    with open(SETTING_PATH, "w") as file:
+        file.write(content)
+
+    with open(SERVICE_SH, "r") as file:
+        content = file.read()
+        content = re.sub(r"LOG_DIR.*=.*", f"LOG_DIR=\"{home_path}/logs\"", content)
+    with open(SERVICE_SH, "w") as file:
+        file.write(content)
+
+
+def init_server(ip, port, home):
+    with open(SERVER_CONF_PATH, "r") as file:
+        config = yaml.safe_load(file)
+    if ip:
+        print(f"ip: {ip}")
+        config["fateflow"]["host"] = ip
+    if port:
+        print(f"port: {port}")
+        config["fateflow"]["http_port"] = port
+    if home:
+        print(f"home: {home}")
+        replace_settings(home)
+
+    if ip or port:
+        with open(SERVER_CONF_PATH, "w") as file:
+            yaml.dump(config, file)
+
+    print("Init server completed!")
+
+
+def run_command(command):
+    try:
+        command = f"sh {SERVICE_SH} {HOME} {command}"
+        result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, text=True)
+        if result.returncode == 0:
+            print(result.stdout)
+            return result.stdout
+        else:
+            print(result.stdout)
+            print(f"Error: {result.stderr}")
+            return None
+    except subprocess.CalledProcessError as e:
+        print(f"Error: {e}")
+        return command
+
+
+if __name__ == '__main__':
+    flow_server_cli()

python/fate_flow/commands/service.sh

@@ -0,0 +1,162 @@
+#!/bin/bash
+
+#
+#  Copyright 2019 The FATE Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+PROJECT_BASE=$1
+LOG_DIR="/Users/tonly/FATE/fate_flow/logs"
+
+
+parse_yaml() {
+   local prefix=$2
+   local s='[[:space:]]*' w='[a-zA-Z0-9_]*' fs=$(echo @|tr @ '\034')
+   sed -ne "s|^\($s\)\($w\)$s:$s\"\(.*\)\"$s\$|\1$fs\2$fs\3|p" \
+        -e "s|^\($s\)\($w\)$s:$s\(.*\)$s\$|\1$fs\2$fs\3|p"  $1 |
+   awk -F$fs '{
+      indent = length($1)/2;
+      vname[indent] = $2;
+      for (i in vname) {if (i > indent) {delete vname[i]}}
+      if (length($3) > 0) {
+         vn=""; for (i=0; i<indent; i++) {vn=(vn)(vname[i])("_")}
+         printf("%s%s%s=\"%s\"\n", "'$prefix'",vn, $2, $3);
+      }
+   }'
+}
+
+getport() {
+    service_conf_path=${PROJECT_BASE}/conf/service_conf.yaml
+    if test -f "${service_conf_path}"; then
+      eval $(parse_yaml ${service_conf_path} "service_config_")
+      echo "fate flow http port: ${service_config_fateflow_http_port}, grpc port: ${service_config_fateflow_grpc_port}"
+      echo
+    else
+      echo "service conf not found: ${service_conf_path}"
+      exit
+    fi
+}
+
+getport
+
+getpid() {
+    echo "check process by http port and grpc port"
+    pid1=`lsof -i:${service_config_fateflow_http_port} | grep 'LISTEN' | awk 'NR==1 {print $2}'`
+    pid2=`lsof -i:${service_config_fateflow_grpc_port} | grep 'LISTEN' | awk 'NR==1 {print $2}'`
+    if [[ -n ${pid1} && "x"${pid1} = "x"${pid2} ]];then
+        pid=$pid1
+    elif [[ -z ${pid1} && -z ${pid2} ]];then
+        pid=
+    fi
+}
+
+mklogsdir() {
+    if [[ ! -d $LOG_DIR ]]; then
+        mkdir -p $LOG_DIR
+    fi
+}
+
+status() {
+    getpid
+    if [[ -n ${pid} ]]; then
+        echo "status:`ps aux | grep ${pid} | grep -v grep`"
+        lsof -i:${service_config_fateflow_http_port} | grep 'LISTEN'
+        lsof -i:${service_config_fateflow_grpc_port} | grep 'LISTEN'
+    else
+        echo "service not running"
+    fi
+}
+
+start() {
+    getpid
+    if [[ ${pid} == "" ]]; then
+        mklogsdir
+        if [[ $1x == "front"x ]];then
+          exec python $PROJECT_BASE/fate_flow_server.py >> "${LOG_DIR}/console.log" 2>>"${LOG_DIR}/error.log"
+        else
+          nohup python $PROJECT_BASE/fate_flow_server.py  >> "${LOG_DIR}/console.log" 2>>"${LOG_DIR}/error.log" &
+        fi
+        for((i=1;i<=100;i++));
+        do
+            sleep 0.1
+            getpid
+            if [[ -n ${pid} ]]; then
+                echo "service start sucessfully. pid: ${pid}"
+                return
+            fi
+        done
+        if [[ -n ${pid} ]]; then
+           echo "service start sucessfully. pid: ${pid}"
+        else
+           echo "service start failed, please check ${LOG_DIR}/error.log and ${LOG_DIR}/console.log"
+        fi
+    else
+        echo "service already started. pid: ${pid}"
+    fi
+}
+
+stop() {
+    getpid
+    if [[ -n ${pid} ]]; then
+        echo "killing: `ps aux | grep ${pid} | grep -v grep`"
+        for((i=1;i<=100;i++));
+        do
+            sleep 0.1
+            kill ${pid}
+            getpid
+            if [[ ! -n ${pid} ]]; then
+                echo "killed by SIGTERM"
+                return
+            fi
+        done
+        kill -9 ${pid}
+        if [[ $? -eq 0 ]]; then
+            echo "killed by SIGKILL"
+        else
+            echo "kill error"
+        fi
+    else
+        echo "service not running"
+    fi
+}
+
+
+case "$2" in
+    start)
+        start
+        status
+        ;;
+
+    starting)
+        start front
+        ;;
+
+    stop)
+        stop
+        ;;
+
+    status)
+        status
+        ;;
+
+    restart)
+        stop
+        sleep 2
+        start
+        status
+        ;;
+    *)
+        echo "usage: $0 {start|stop|status|restart}"
+        exit -1
+esac

python/fate_flow/hook/common/__init__.py

@@ -0,0 +1,15 @@
+
+#
+#  Copyright 2019 The FATE Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.

python/fate_flow/hook/flow/__init__.py

@@ -0,0 +1,14 @@
+#
+#  Copyright 2019 The FATE Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.