Merge pull request #3733 from FederatedAI/develop-1.7.2

Merge fate-v1.7.2's update into master

.gitignore

@@ -37,8 +37,8 @@ venv
 /jobs/
 /audit/
 .vscode/*
-/eggroll/
-/fateboard/
+# /eggroll/
+# /fateboard/
 /doc/federatedml_component/params/
 
 # build dir
@@ -65,4 +65,4 @@ entry_points.txt
 dependency_links.txt
 requires.txt
 SOURCES.txt
-top_level.txt
+top_level.txt

.gitmodules

@@ -1,12 +1,12 @@
 [submodule "fateboard"]
 	path = fateboard
 	url = https://github.com/FederatedAI/FATE-Board.git
-	branch = v1.7.2
+	branch = v1.7.2.2
 [submodule "eggroll"]
 	path = eggroll
 	url = https://github.com/WeBankFinTech/eggroll.git
 	branch = v2.4.3
 [submodule "fateflow"]
 	path = fateflow
 	url = https://github.com/FederatedAI/FATE-Flow.git
-	branch = v1.7.1
+	branch = v1.7.2

RELEASE.md

@@ -1,3 +1,21 @@
+## Release 1.7.2
+### Major Features and Improvements
+> FederatedML
+* New batch strategy in coordinated Hetero LR: support masked batch data and batch shuffle
+* Model inference protection enhancement for Hetero SecureBoost with FED-EINI algorithm
+* Hetero SecureBoost supports split feature importance on host side, disables gain feature importance
+* Offline SBT Feature transform component 
+
+> Bug-Fix
+* Fixed Bug for HeteroPearson with changing default q_field value for spdz
+* Fix Data Transform's schema label name setting problem when `with_label` is False
+* Add testing examples for new algorithm features, and delete deprecated params in algorithm examples.
+
+> FATE-ARCH
+* Support the loading of custom password encryption modules through plug-ins
+* Separate the base connection address of the data storage table from the data table information, and compatible with historical versions
+
+
 ## Release 1.7.1.1
 ### Major Features and Improvements
 > Deploy

conf/service_conf.yaml

@@ -1,6 +1,9 @@
 use_registry: false
 use_deserialize_safe_module: false
 dependent_distribution: false
+encrypt_password: false
+encrypt_module: fate_arch.common.encrypt_utils#pwdecrypt
+private_key:
 fateflow:
   # you must set real ip address, 127.0.0.1 and 0.0.0.0 is not supported
   host: 127.0.0.1

deploy/cluster-deploy/doc/fate_on_eggroll/fate-allinone_deployment_guide.md

@@ -263,6 +263,8 @@ Note: Replace ${version} with specific FATE version number,can be viewed on the
 cd /data/projects/
 wget https://webank-ai-1251170195.cos.ap-guangzhou.myqcloud.com/fate_cluster_install_${version}_release-c7-u18.tar.gz
 tar xzf fate_cluster_install_${version}_release-c7-u18.tar.gz
+
+Note: version without character v, such as fate_cluster_install_1.x.x_release-c7-u18.tar.gz 
 ```
 
 ### 5.2. Pre-Deployment Check

deploy/cluster-deploy/doc/fate_on_eggroll/fate-allinone_deployment_guide.zh.md

@@ -265,6 +265,8 @@ Swap:        131071           0      131071
 cd /data/projects/
 wget https://webank-ai-1251170195.cos.ap-guangzhou.myqcloud.com/fate_cluster_install_${version}_release-c7-u18.tar.gz
 tar xzf fate_cluster_install_${version}_release-c7-u18.tar.gz
+
+注意：version不带字符v，如fate_cluster_install_1.x.x_release-c7-u18.tar.gz
 ```
 
 ### 5.2. 部署前检查

deploy/cluster-deploy/doc/fate_on_eggroll/fate-exchange_deployment_guide.md

@@ -139,6 +139,8 @@ Note: Replace ${version} with the specific FATE version number.
 cd /data/projects/install
 wget https://webank-ai-1251170195.cos.ap-guangzhou.myqcloud.com/jdk-8u192-linux-x64.tar.gz
 wget https://webank-ai-1251170195.cos.ap-guangzhou.myqcloud.com/FATE_install_${version}_release.tar.gz
+
+Note: version without character v, such as FATE_install_1.x.x_release.tar.gz
 ```
 
 ## 5.2 Check OS Parameters

deploy/cluster-deploy/doc/fate_on_eggroll/fate-exchange_deployment_guide.zh.md

@@ -147,6 +147,8 @@ fi
 cd /data/projects/install
 wget https://webank-ai-1251170195.cos.ap-guangzhou.myqcloud.com/jdk-8u192-linux-x64.tar.gz
 wget https://webank-ai-1251170195.cos.ap-guangzhou.myqcloud.com/FATE_install_${version}_release.tar.gz
+
+注意：version不带字符v，如FATE_install_1.x.x_release.tar.gz
 ```
 
 ## 5.2 操作系统参数检查

deploy/cluster-deploy/sql/create-eggroll-meta-tables.sql

@@ -0,0 +1,125 @@
+-- create database if not exists, default database is eggroll_meta
+-- CREATE DATABASE IF NOT EXISTS `eggroll_meta`;
+
+-- all operation under this database
+-- USE `eggroll_meta`;
+
+-- store_locator
+CREATE TABLE IF NOT EXISTS `store_locator` (
+  `store_locator_id` SERIAL PRIMARY KEY,
+  `store_type` VARCHAR(255) NOT NULL,
+  `namespace` VARCHAR(2000) NOT NULL DEFAULT 'DEFAULT',
+  `name` VARCHAR(2000) NOT NULL,
+  `path` VARCHAR(2000) NOT NULL DEFAULT '',
+  `total_partitions` INT UNSIGNED NOT NULL,
+  `partitioner` VARCHAR(2000) NOT NULL DEFAULT 'BYTESTRING_HASH',
+  `serdes` VARCHAR(2000) NOT NULL DEFAULT '',
+  `version` INT UNSIGNED NOT NULL DEFAULT 0,
+  `status` VARCHAR(255) NOT NULL,
+  `created_at` DATETIME DEFAULT CURRENT_TIMESTAMP,
+  `updated_at` DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
+) DEFAULT CHARACTER SET latin1 COLLATE latin1_swedish_ci;
+
+CREATE UNIQUE INDEX `idx_u_store_loinsert-node.sqlcator_ns_n` ON `store_locator` (`namespace`(120), `name`(640));
+CREATE INDEX `idx_store_locator_st` ON `store_locator` (`store_type`(255));
+CREATE INDEX `idx_store_locator_ns` ON `store_locator` (`namespace`(767));
+CREATE INDEX `idx_store_locator_n` ON `store_locator` (`name`(767));
+CREATE INDEX `idx_store_locator_s` ON `store_locator` (`status`(255));
+CREATE INDEX `idx_store_locator_v` ON `store_locator` (`version`);
+
+
+-- store (option)
+CREATE TABLE IF NOT EXISTS `store_option` (
+  `store_option_id` SERIAL PRIMARY KEY,
+  `store_locator_id` BIGINT UNSIGNED NOT NULL,
+  `name` VARCHAR(255) NOT NULL,
+  `data` VARCHAR(2000) NOT NULL DEFAULT '',
+  `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  `updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
+) DEFAULT CHARACTER SET latin1 COLLATE latin1_swedish_ci;
+
+CREATE INDEX `idx_store_option_si` ON `store_option` (`store_locator_id`);
+
+
+-- store_partition
+CREATE TABLE IF NOT EXISTS `store_partition` (
+  `store_partition_id` SERIAL PRIMARY KEY,          -- self-increment sequence
+  `store_locator_id` BIGINT UNSIGNED NOT NULL,
+  `node_id` BIGINT UNSIGNED NOT NULL,
+  `partition_id` INT UNSIGNED NOT NULL,             -- partition id of a store
+  `status` VARCHAR(255) NOT NULL,
+  `created_at` DATETIME DEFAULT CURRENT_TIMESTAMP,
+  `updated_at` DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
+) DEFAULT CHARACTER SET latin1 COLLATE latin1_swedish_ci;
+
+CREATE UNIQUE INDEX `idx_u_store_partition_si_spi_ni` ON `store_partition` (`store_locator_id`, `store_partition_id`, `node_id`);
+CREATE INDEX `idx_store_partition_sli` ON `store_partition` (`store_locator_id`);
+CREATE INDEX `idx_store_partition_ni` ON `store_partition` (`node_id`);
+CREATE INDEX `idx_store_partition_s` ON `store_partition` (`status`(255));
+
+
+-- node
+CREATE TABLE IF NOT EXISTS `server_node` (
+  `server_node_id` SERIAL PRIMARY KEY,
+  `name` VARCHAR(2000) NOT NULL DEFAULT '',
+  `server_cluster_id` BIGINT UNSIGNED NOT NULL DEFAULT 0,
+  `host` VARCHAR(1000) NOT NULL,
+  `port` INT NOT NULL,
+  `node_type` VARCHAR(255) NOT NULL,
+  `status` VARCHAR(255) NOT NULL,
+  `last_heartbeat_at` DATETIME DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP,
+  `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  `updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
+) DEFAULT CHARACTER SET latin1 COLLATE latin1_swedish_ci;
+
+CREATE INDEX `idx_server_node_h_p_nt` ON `server_node` (`host`(600), `port`, `node_type`(100));
+CREATE INDEX `idx_server_node_h` ON `server_node` (`host`(767));
+CREATE INDEX `idx_server_node_sci` ON `server_node` (`server_cluster_id`);
+CREATE INDEX `idx_server_node_nt` ON `server_node` (`node_type`(255));
+CREATE INDEX `idx_server_node_s` ON `server_node` (`status`(255));
+
+
+-- session (main)
+CREATE TABLE IF NOT EXISTS `session_main` (
+  `session_id` VARCHAR(767) PRIMARY KEY,
+  `name` VARCHAR(2000) NOT NULL DEFAULT '',
+  `status` VARCHAR(255) NOT NULL,
+  `tag` VARCHAR(255),
+  `total_proc_count` INT,
+  `active_proc_count` INT,
+  `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  `updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
+) DEFAULT CHARACTER SET latin1 COLLATE latin1_swedish_ci;
+
+CREATE INDEX `idx_session_main_s` ON `session_main` (`status`);
+
+
+-- session (option)
+CREATE TABLE IF NOT EXISTS `session_option` (
+  `session_option_id`  SERIAL PRIMARY KEY,
+  `session_id` VARCHAR(2000),
+  `name` VARCHAR(255) NOT NULL,
+  `data` VARCHAR(2000) NOT NULL DEFAULT '',
+  `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  `updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
+) DEFAULT CHARACTER SET latin1 COLLATE latin1_swedish_ci;
+
+CREATE INDEX `idx_session_option_si` ON `session_option` (`session_id`(767));
+
+
+-- session (processor)
+CREATE TABLE IF NOT EXISTS `session_processor` (
+  `processor_id` SERIAL PRIMARY KEY,
+  `session_id` VARCHAR(767),
+  `server_node_id` INT NOT NULL,
+  `processor_type` VARCHAR(255) NOT NULL,
+  `status` VARCHAR(255),
+  `tag` VARCHAR(255),
+  `command_endpoint` VARCHAR(255),
+  `transfer_endpoint` VARCHAR(255),
+  `pid` INT NOT NULL DEFAULT -1,
+  `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  `updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
+) DEFAULT CHARACTER SET latin1 COLLATE latin1_swedish_ci;
+
+CREATE INDEX `idx_session_processor_si` ON `session_processor` (`session_id`(767));

deploy/cluster-deploy/sql/insert-node.sql

@@ -0,0 +1,7 @@
+CREATE DATABASE IF NOT EXISTS fate_flow;
+CREATE USER fate@'localhost' IDENTIFIED BY 'fate';
+GRANT ALL ON fate_flow.* TO fate@'localhost';
+GRANT ALL ON eggroll_meta.* TO fate@'localhost;
+use eggroll_meta;
+INSERT INTO server_node (host, port, node_type, status) values ('127.0.0.1', '4670', 'CLUSTER_MANAGER', 'HEALTHY');
+INSERT INTO server_node (host, port, node_type, status) values ('127.0.0.1', '4671', 'NODE_MANAGER', 'HEALTHY');

doc/federatedml_component/README.md

@@ -42,7 +42,6 @@ provide:
 | [Homo-NN](homo_nn.md)                                                         | HomoNN                       | Build homo neural network model through multiple parties.                                                                             | Table, values are instances.                      | Table, values are instances.                                                                          |                                                      | Neural Network Model, consists of model-meta and model-param.           |
 | [Hetero Secure Boosting](ensemble.md)                                         | HeteroSecureBoost            | Build hetero secure boosting model through multiple parties                                                                           | Table, values are instances.                      | Table, values are instances.                                                                          |                                                      | SecureBoost Model, consists of model-meta and model-param.              |
 | [Hetero Fast Secure Boosting](ensemble.md)                                    | HeteroFastSecureBoost        | Build hetero secure boosting model through multiple parties in layered/mix manners.                                                   | Table, values are instances.                      | Table, values are instances.                                                                          |                                                      | FastSecureBoost Model, consists of model-meta and model-param.          |
-| [Hetero Secure Boost Feature Transformer](sbt_feature_transformer.md) | SBTFeatureTransformer      | This component can encode sample using Hetero SBT leaf indices.                                                                       | Table, values are instances.                      | Table, values are instances.                                                                          |                                                      | SBT Transformer Model                                                   |
 | [Evaluation](evaluation.md)                                                   | Evaluation                   | Output the model evaluation metrics for user.                                                                                         | Table(s), values are instances.                   |                                                                                                       |                                                      |                                                                         |
 | [Hetero Pearson](correlation.md)                                              | HeteroPearson                | Calculate hetero correlation of features from different parties.                                                                      | Table, values are instances.                      |                                                                                                       |                                                      |                                                                         |
 | [Hetero-NN](hetero_nn.md)                                                     | HeteroNN                     | Build hetero neural network model.                                                                                                    | Table, values are instances.                      | Table, values are instances.                                                                          |                                                      | Hetero Neural Network Model, consists of model-meta and model-param.    |

doc/federatedml_component/README.zh.md

@@ -32,7 +32,6 @@ Federatedml模块包括许多常见机器学习算法联邦化实现。所有模
 | [Homo-NN](homo_nn.md)                                                         | HomoNN                  | 通过多方构建横向神经网络模块。                                                                                     | Table, 值为Instance             |                                                                                                   |                                         | 神经网络模型，由模型本身和模型参数组成        |
 | [Hetero Secure Boosting](ensemble.md)                                         | HeteroSecureBoost       | 通过多方构建纵向Secure Boost模块。                                                                             | Table，值为Instance              |                                                                                                   |                                         | SecureBoost模型，由模型本身和模型参数组成 |
 | [Hetero Fast Secure Boosting](ensemble.md)                                    | HeteroFastSecureBoost   | 使用分层/混合模式快速构建树模型                                                                                    | Table，值为Instance              | Table，值为Instance                                                                                  |                                         | FastSecureBoost模型          |
-| [Hetero Secure Boost Feature Transformer](sbt_feature_transformer.md) | SBTFeatureTransformer | 利用SBT叶子为特征编码                                                                                        | Table，值为Instance              | Table，值为Instance                                                                                  |                                         | SBT Transformer模型          |
 | [Evaluation](evaluation.md)                                                   | Evaluation              | 为用户输出模型评估指标。                                                                                        | Table(s), 值为Instance          |                                                                                                   |                                         |                            |
 | [Hetero Pearson](correlation.md)                                              | HeteroPearson           | 计算来自不同方的特征的Pearson相关系数。                                                                             | Table, 值为Instance             |                                                                                                   |                                         |                            |
 | [Hetero-NN](hetero_nn.md)                                                     | HeteroNN                | 构建纵向神经网络模块。                                                                                         | Table, 值为Instance             |                                                                                                   |                                         | 纵向神经网络模型                   |

doc/federatedml_component/feldman_verifiable_sum.md

@@ -6,7 +6,7 @@ Verifiable secret sharing mechanism is an efficient and practical secret
 sharing mechanism. Feldman Verifiable sum is a multi-party private data
 summation module based on verifiable secret sharing.This component can
 sum the same feature of common users among different participants
-without exposing private data.
+without exposing private data.（This module is still in the research stage, has not yet been put into production）
 
 Here, three participants of the federation process is given, Party A
 represents Guest, party B and party C represent Host. The process of

doc/federatedml_component/logistic_regression.md

@@ -115,9 +115,6 @@ which HE and Secret-Sharing hybrid protocol is included.
 >     >   - nesterov\_momentum\_sgd  
 >     >     Nesterov Momentum
 >     > 
->     >   - sqn  
->     >     stochastic quansi-newton. More details is available in this
->     >     [A Quasi-Newton Method Based Vertical Federated Learning Framework for Logistic Regression](https://arxiv.org/abs/1912.00513v2).
 > 
 > 5.  Three converge criteria:
 >     
@@ -157,14 +154,15 @@ which HE and Secret-Sharing hybrid protocol is included.
 
   - Hetero-LR extra features
 
-> 1.  Support different encrypt-mode to balance speed and security
-> 2.  Support OneVeRest
-> 3.  When modeling a multi-host task, "weight\_diff" converge criteria
+1. Support different encrypt-mode to balance speed and security
+2. Support OneVeRest
+3. When modeling a multi-host task, "weight\_diff" converge criteria
 >     is supported only.
-> 4.  Support sparse format data
-> 5.  Support early-stopping mechanism
-> 6.  Support setting arbitrary metrics for validation during training
-> 7.  Support stepwise. For details on stepwise mode, please refer [stepwise](stepwise.md).
+4. Support sparse format data
+5. Support early-stopping mechanism
+6. Support setting arbitrary metrics for validation during training
+7. Support stepwise. For details on stepwise mode, please refer [stepwise](stepwise.md).
+8. Support batch shuffle and batch masked strategy.
 
  - Hetero-SSHE-LR extra features
   > 1. Support different encrypt-mode to balance speed and security

doc/federatedml_component/sir.md

@@ -6,7 +6,8 @@ Secure Information Retrieval(SIR) securely retrieves target value(s)
 from host. This module is based on [Pohlig
 Hellman](https://ee.stanford.edu/~hellman/publications/28.pdf)
 commutative encryption and [Hauck Oblivious
-Transfer(OT)](https://eprint.iacr.org/2017/1011).
+Transfer(OT)](https://eprint.iacr.org/2017/1011). 
+This module is still in the research stage, has not yet been put into production.
 
 ## How to Use