diff --git a/README.md b/README.md index 2bd884e678..a5fe478c0e 100644 --- a/README.md +++ b/README.md @@ -25,8 +25,10 @@ FATE is an open source project hosted by Linux Foundation. The [Technical Charte FATE can be deployed on a single node or on multiple nodes. Choose the deployment approach which matches your environment. [Release version can be downloaded here.](https://github.com/FederatedAI/FATE/wiki/Download) + ### Version >= 2.0 ### Standalone deployment + - Deploying FATE on a single node via PyPI, pre-built docker images or installers. It is for simple testing purposes. Refer to this [guide](./deploy/standalone-deploy/). ### Cluster deployment @@ -34,7 +36,7 @@ Deploying FATE to multiple nodes to achieve scalability, reliability and managea - [Cluster deployment by CLI](./deploy/cluster-deploy): Using CLI to deploy a FATE cluster. ### Quick Start -- [Training Demo With Installing FATE AND FATE-Flow From Pypi](doc/2.0/quick_start.md) +- [Training Demo With Installing FATE AND FATE-Flow From Pypi](doc/2.0/fate/quick_start.md) - [Training Demo With Installing FATE Only From Pypi](doc/2.0/fate/ml) ## Related Repositories (Projects) diff --git a/README_zh.md b/README_zh.md index 2fee074553..f632fae216 100644 --- a/README_zh.md +++ b/README_zh.md @@ -23,7 +23,7 @@ FATE于2019年2月首次对外开源,并成立 ### 2.0以前的版本 FATE 2.0以前的版本在[发布页](https://github.com/FederatedAI/FATE/releases), 下载资源汇总页在[wiki](https://github.com/FederatedAI/FATE/wiki/Download) -### 2.0.0-beta 版本 +### 2.0.0 版本 #### 单机版部署 在单节点上部署FATE单机版,支持从 PyPI 直接安装,docker,主机安装包三种方式。 - [单机版部署教程](./deploy/standalone-deploy) @@ -31,7 +31,7 @@ FATE 2.0以前的版本在[发布页](https://github.com/FederatedAI/FATE/releas - [原生集群安装](./deploy/cluster-deploy): Using CLI to deploy a FATE cluster. ### 快速开始 -- [从 PyPI 下载安装 FATE 和 FATE-Flow 并启动训练任务示例](doc/2.0/quick_start.md) +- [从 PyPI 下载安装 FATE 和 FATE-Flow 并启动训练任务示例](doc/2.0/fate/quick_start.md) - [从 PyPI 下载安装 FATE,并启动训练任务示例](doc/2.0/fate/ml) ## 关联仓库 diff --git a/bin/service.sh b/bin/service.sh new file mode 100644 index 0000000000..c5f671bbe4 --- /dev/null +++ b/bin/service.sh @@ -0,0 +1,324 @@ +#!/bin/bash +# +# Copyright 2019 The fate Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# --------------- Color Definitions --------------- +esc_c="\033[0m" +error_c="\033[31m" +ok_c="\033[32m" + +# -----------------global params-------------- +cwd=$(cd `dirname $0`; pwd) +fate_home=/data/projects/fate +eggroll_modules=(nodemanager clustermanager dashboard) +fate_modules=(fate-flow fate-board osx eggroll mysql) +# --------------- Logging Functions --------------- +print_info() { + local timestamp=$(date '+%Y-%m-%d %H:%M:%S') + local overwrite=$2 + + # Check if we need to overwrite the current line + if [ "$overwrite" == "overwrite" ]; then + echo -ne "\r${ok_c}[${timestamp}][INFO]${esc_c} $1" + else + echo -e "${ok_c}[${timestamp}][INFO]${esc_c} $1" + fi +} +print_ok() { + local overwrite=$2 + local timestamp=$(date '+%Y-%m-%d %H:%M:%S') + if [ "$overwrite" == "overwrite" ]; then + echo -ne "\r${ok_c}[${timestamp}][ OK ]${esc_c} $1" + else + echo -e "${ok_c}[${timestamp}][ OK ]${esc_c} $1" + fi +} +print_error() { + local overwrite=$3 + local timestamp=$(date '+%Y-%m-%d %H:%M:%S') + if [ "$overwrite" == "overwrite" ]; then + echo -ne "\r${error_c}[${timestamp}][ ER ]${esc_c} $1: $2" + else + echo -e "${error_c}[${timestamp}][ ER ]${esc_c} $1: $2" + fi +} + +source_env() { + echo "================>>>source_env<<<===============" +} + +check_python_env(){ + echo "================>>>check_python_env<<<===============" +} + +check_python_env(){ + echo "================>>>check_python_env<<<===============" +} + +print_split(){ + name=$1 + padding_length=$((20 - ${#name})) + padded_string=$(printf "%s%*s" "$1" $padding_length " ") + print_info " -----------------------------------------------------" + print_info "|component : ${padded_string} |" + print_info " -----------------------------------------------------" +} + +print_end(){ + echo "" +} + +eggroll(){ + print_split $1 + cd ${fate_home}/eggroll || exit 1 + case "$1" in + eggroll) + case "$2" in + status) + ;; + *) + "`pwd`/bin/eggroll.sh" all $2 + ;; + esac + + declare -A array + array[0]=" ---------------------------" + pid=`ps aux | grep "${fate_home}/eggroll/lib" | grep "org.fedai.eggroll.clustermanager.Bootstrap" | grep -v grep | awk '{print $2}'` + if [[ -n ${pid} ]]; then + array[1]="| clustermanager running |" + else + array[1]="| clustermanager failed |" + fi + unset pid + + pid=`ps aux | grep "${fate_home}/eggroll/lib" | grep "org.fedai.eggroll.nodemanager.Bootstrap" | grep -v grep | awk '{print $2}'` + if [[ -n ${pid} ]]; then + array[2]="| nodemanager running |" + else + array[2]="| nodemanager failed |" + fi + unset pid + + pid=`ps aux | grep "${fate_home}/eggroll/lib" | grep "org.fedai.eggroll.webapp.JettyServer" | grep -v grep | awk '{print $2}'` + if [[ -n ${pid} ]]; then + array[3]="| dashboard running |" + else + array[3]="| dashboard failed |" + fi + unset pid + + array[4]=" ---------------------------" + for msg in "${array[@]}"; do + echo "$msg" + done + ;; + all) + case "$2" in + status) + ;; + *) + "`pwd`/bin/eggroll.sh" all $2 + ;; + esac + ;; + *) + "`pwd`/bin/eggroll.sh" $1 $2 + ;; + esac + cd "${fate_home}" || exit 1 + print_end +} + +osx(){ + print_split $1 + cd ${fate_home}/osx || exit 1 + "`pwd`/service.sh" $2 + cd "${fate_home}" || exit 1 + print_end +} + +flow(){ + print_split $1 + cd ${fate_home}/fate_flow || exit 1 + "`pwd`/bin/service.sh" $2 + cd "${fate_home}" || exit 1 + print_end +} + +mysql(){ + print_split $1 + cd ${fate_home}/common/mysql/mysql-8.0.28 || exit 1 + "`pwd`/service.sh" $2 + cd "${fate_home}" || exit 1 + print_end +} + +board(){ + print_split $1 + cd ${fate_home}/fateboard || exit 1 + "`pwd`/service.sh" $2 + cd "${fate_home}" || exit 1 + print_end +} + +all(){ + declare -A array + array[0]=" ---------------------------" + if [[ $2 != "status" ]]; then + mysql mysql $2 + flow fate-flow $2 + board fate-board $2 + osx osx $2 + eggroll all $2 + fi + + pid=`ps aux | grep "${fate_home}/common/mysql/mysql-8.0.28/bin/mysqld_safe" | grep -v grep | awk '{print $2}'` + if [[ -n ${pid} ]]; then + array[1]="| mysql running |" + else + array[1]="| mysql failed |" + fi + unset pid + + + pid=`ps aux | grep "${fate_home}/fate_flow/python/fate_flow/fate_flow_server.py" | grep -v grep | awk '{print $2}'` + if [[ -n ${pid} ]]; then + array[2]="| fate-flow running |" + else + array[2]="| fate-flow failed |" + fi + unset pid + + + pid=`ps aux | grep "${fate_home}/fateboard/fateboard-" | grep "org.fedai.fate.board.bootstrap.Bootstrap" | grep -v grep | awk '{print $2}'` + if [[ -n ${pid} ]]; then + array[3]="| fate-board running |" + else + array[3]="| fate-board failed |" + fi + unset pid + + + pid=`ps aux | grep "${fate_home}/osx/conf" | grep "org.fedai.osx.broker.Bootstrap" | grep -v grep | awk '{print $2}'` + if [[ -n ${pid} ]]; then + array[4]="| osx running |" + else + array[4]="| osx failed |" + fi + unset pid + + + pid=`ps aux | grep "${fate_home}/eggroll/lib" | grep "org.fedai.eggroll.clustermanager.Bootstrap" | grep -v grep | awk '{print $2}'` + if [[ -n ${pid} ]]; then + array[5]="| clustermanager running |" + else + array[5]="| clustermanager failed |" + fi + unset pid + + pid=`ps aux | grep "${fate_home}/eggroll/lib" | grep "org.fedai.eggroll.clustermanager.Bootstrap" | grep -v grep | awk '{print $2}'` + if [[ -n ${pid} ]]; then + array[5]="| clustermanager running |" + else + array[5]="| clustermanager failed |" + fi + unset pid + + pid=`ps aux | grep "${fate_home}/eggroll/lib" | grep "org.fedai.eggroll.nodemanager.Bootstrap" | grep -v grep | awk '{print $2}'` + if [[ -n ${pid} ]]; then + array[6]="| nodemanager running |" + else + array[6]="| nodemanager failed |" + fi + unset pid + + pid=`ps aux | grep "${fate_home}/eggroll/lib" | grep "org.fedai.eggroll.webapp.JettyServer" | grep -v grep | awk '{print $2}'` + if [[ -n ${pid} ]]; then + array[7]="| dashboard running |" + else + array[7]="| dashboard failed |" + fi + unset pid + + array[8]=" ---------------------------" + for msg in "${array[@]}"; do + echo "$msg" + done +} + +status(){ + echo "" +} + +# --------------- Functions for info--------------- +# Print usage information for the script +usage() { + + echo -e "${ok_c}FATE${esc_c}" + echo "------------------------------------" + echo -e "${ok_c}Usage:${esc_c}" + echo -e " `basename ${0}` [component] start - Start the server application." + echo -e " `basename ${0}` [component] stop - Stop the server application." + echo -e " `basename ${0}` [component] status - Check and report the status of the server application." + echo -e " `basename ${0}` [component] restart - Restart the server application. " + echo -e " The ${ok_c}component${esc_c} include: {fate-flow | fate-board | osx | eggroll |clustermanager | nodemanager | dashboard | mysql | all} " + echo "" + echo -e "${ok_c}Examples:${esc_c}" + echo " `basename ${0}` fate-flow stop" + echo " `basename ${0}` eggroll restart" +} + +dispatch(){ +case "$2" in + start|stop|status|restart) + ;; + *) + usage + exit 1 + ;; +esac + + +case "$1" in + all) + all "$@" + ;; + clustermanager|nodemanager|dashboard|eggroll) + eggroll "$@" + ;; + osx) + osx "$@" + ;; + fate-flow) + flow "$@" + ;; + fate-board) + board "$@" + ;; + mysql) + mysql "$@" + ;; + *) + usage + exit 1 + ;; +esac +} + + +# main func +# --------------- Main--------------- +# Main case for control +dispatch "$@" diff --git a/doc/2.0/fate/components/linear_regression.md b/doc/2.0/fate/components/linear_regression.md index 915be62760..386fe8ce98 100644 --- a/doc/2.0/fate/components/linear_regression.md +++ b/doc/2.0/fate/components/linear_regression.md @@ -19,7 +19,7 @@ Here we simplify participants of the federation process into three parties. Party A represents Guest, party B represents Host. Party C, which is also known as “Arbiter,” is a third party that works as coordinator. Party C is responsible for generating private and public -keys. +keys. (Security of this algorithm is lower than SSHE-LinR, use SSHE-LinR if possible) The process of HeteroLinR training is shown below: diff --git a/doc/2.0/fate/components/logistic_regression.md b/doc/2.0/fate/components/logistic_regression.md index e2249aff0f..ef5168b1d3 100644 --- a/doc/2.0/fate/components/logistic_regression.md +++ b/doc/2.0/fate/components/logistic_regression.md @@ -26,7 +26,7 @@ samples stored in databases of the two involved parties. The federated model is built based on those overlapping samples. The whole sample alignment process will **not** leak confidential information (e.g., sample ids) on the two parties since it is conducted in an encrypted -way. +way. (Security of this algorithm is lower than SSHE-LR, use SSHE-LR if possible) ![Figure 1 (Federated HeteroLR Principle)](../../images/HeteroLR.png) diff --git a/doc/2.0/fate/quick_start.md b/doc/2.0/fate/quick_start.md new file mode 100644 index 0000000000..de8a9c0914 --- /dev/null +++ b/doc/2.0/fate/quick_start.md @@ -0,0 +1,114 @@ +## Quick Start + +1. install `fate_client` with extra package `fate` + +```sh +python -m pip install -U pip && python -m pip install fate_client[fate,fate_flow]==2.0.0 +``` +after installing packages successfully, initialize fate_flow service and fate_client + +```sh +mkdir fate_workspace +fate_flow init --ip 127.0.0.1 --port 9380 --home $(pwd)/fate_workspace +pipeline init --ip 127.0.0.1 --port 9380 + +fate_flow start +fate_flow status # make sure fate_flow service is started +``` + + +2. download example data + +```sh +wget https://raw.githubusercontent.com/wiki/FederatedAI/FATE/example/data/breast_hetero_guest.csv && \ +wget https://raw.githubusercontent.com/wiki/FederatedAI/FATE/example/data/breast_hetero_host.csv +``` + +3. transform example data to dataframe using in fate +```python +import os +from fate_client.pipeline import FateFlowPipeline + + +base_path = os.path.abspath(os.path.join(__file__, os.path.pardir)) +guest_data_path = os.path.join(base_path, "breast_hetero_guest.csv") +host_data_path = os.path.join(base_path, "breast_hetero_host.csv") + +data_pipeline = FateFlowPipeline().set_parties(local="0") +guest_meta = { + "delimiter": ",", "dtype": "float64", "label_type": "int64","label_name": "y", "match_id_name": "id" +} +host_meta = { + "delimiter": ",", "input_format": "dense", "match_id_name": "id" +} +data_pipeline.transform_local_file_to_dataframe(file=guest_data_path, namespace="experiment", name="breast_hetero_guest", + meta=guest_meta, head=True, extend_sid=True) +data_pipeline.transform_local_file_to_dataframe(file=host_data_path, namespace="experiment", name="breast_hetero_host", + meta=host_meta, head=True, extend_sid=True) +``` +4. run example + +```python +from fate_client.pipeline.components.fate import ( + HeteroSecureBoost, + Reader, + PSI, + Evaluation +) +from fate_client.pipeline import FateFlowPipeline + + +# create pipeline for training +pipeline = FateFlowPipeline().set_parties(guest="9999", host="10000") + +# create reader task_desc +reader_0 = Reader("reader_0") +reader_0.guest.task_parameters(namespace="experiment", name="breast_hetero_guest") +reader_0.hosts[0].task_parameters(namespace="experiment", name="breast_hetero_host") + +# create psi component_desc +psi_0 = PSI("psi_0", input_data=reader_0.outputs["output_data"]) + +# create hetero secure_boost component_desc +hetero_secureboost_0 = HeteroSecureBoost( + "hetero_secureboost_0", num_trees=1, max_depth=5, + train_data=psi_0.outputs["output_data"], + validate_data=psi_0.outputs["output_data"] +) + +# create evaluation component_desc +evaluation_0 = Evaluation( + 'evaluation_0', runtime_parties=dict(guest="9999"), metrics=["auc"], input_data=[hetero_secureboost_0.outputs["train_data_output"]] +) + +# add training task +pipeline.add_tasks([reader_0, psi_0, hetero_secureboost_0, evaluation_0]) + +# compile and train +pipeline.compile() +pipeline.fit() + +# print metric and model info +print (pipeline.get_task_info("hetero_secureboost_0").get_output_model()) +print (pipeline.get_task_info("evaluation_0").get_output_metric()) + +# deploy task for inference +pipeline.deploy([psi_0, hetero_secureboost_0]) + +# create pipeline for predicting +predict_pipeline = FateFlowPipeline() + +# add input to deployed_pipeline +deployed_pipeline = pipeline.get_deployed_pipeline() +reader_1 = Reader("reader_1") +reader_1.guest.task_parameters(namespace="experiment", name="breast_hetero_guest") +reader_1.hosts[0].task_parameters(namespace="experiment", name="breast_hetero_host") +deployed_pipeline.psi_0.input_data = reader_1.outputs["output_data"] + +# add task to predict pipeline +predict_pipeline.add_tasks([reader_1, deployed_pipeline]) + +# compile and predict +predict_pipeline.compile() +predict_pipeline.predict() +```