From c53820b918b370d7241002a18b82ac5c62eb3f61 Mon Sep 17 00:00:00 2001 From: zhihuiwan <15779896112@163.com> Date: Wed, 19 Jan 2022 16:20:24 +0800 Subject: [PATCH 01/99] storage connector --- python/fate_arch/common/address.py | 114 ++++++++++++++++++++---- python/fate_arch/metastore/db_models.py | 9 ++ python/fate_arch/storage/__init__.py | 1 + python/fate_arch/storage/_utils.py | 43 +++++++-- 4 files changed, 145 insertions(+), 22 deletions(-) diff --git a/python/fate_arch/common/address.py b/python/fate_arch/common/address.py index 316b0f427a..cd4b02513a 100644 --- a/python/fate_arch/common/address.py +++ b/python/fate_arch/common/address.py @@ -1,12 +1,33 @@ from fate_arch.abc import AddressABC +from fate_arch.storage import StorageEngine, StorageConnector -class StandaloneAddress(AddressABC): - def __init__(self, home=None, name=None, namespace=None, storage_type=None): +class AddressBase(AddressABC): + def __init__(self, connector_name=None): + self.connector_name = connector_name + if connector_name: + connector = StorageConnector(connector_name=connector_name, engine=self.get_name) + if connector.info: + for k, v in connector.info.items(): + if hasattr(self, k): + self.__setattr__(k, v) + + @property + def connector(self): + return {} + + @property + def get_name(self): + return + + +class StandaloneAddress(AddressBase): + def __init__(self, home=None, name=None, namespace=None, storage_type=None, connector_name=None): self.home = home self.name = name self.namespace = namespace self.storage_type = storage_type + super(StandaloneAddress, self).__init__(connector_name=connector_name) def __hash__(self): return (self.home, self.name, self.namespace, self.storage_type).__hash__() @@ -17,12 +38,21 @@ def __str__(self): def __repr__(self): return self.__str__() + @property + def connector(self): + return {"home": self.home} + + @property + def get_name(self): + return StorageEngine.STANDALONE -class EggRollAddress(AddressABC): - def __init__(self, home=None, name=None, namespace=None): + +class EggRollAddress(AddressBase): + def __init__(self, home=None, name=None, namespace=None, connector_name=None): self.name = name self.namespace = namespace self.home = home + super(EggRollAddress, self).__init__(connector_name=connector_name) def __hash__(self): return (self.home, self.name, self.namespace).__hash__() @@ -33,11 +63,20 @@ def __str__(self): def __repr__(self): return self.__str__() + @property + def connector(self): + return {"home": self.home} + + @property + def get_name(self): + return StorageEngine.EGGROLL -class HDFSAddress(AddressABC): - def __init__(self, name_node, path=None): + +class HDFSAddress(AddressBase): + def __init__(self, name_node=None, path=None, connector_name=None): self.name_node = name_node self.path = path + super(HDFSAddress, self).__init__(connector_name=connector_name) def __hash__(self): return (self.name_node, self.path).__hash__() @@ -48,10 +87,19 @@ def __str__(self): def __repr__(self): return self.__str__() + @property + def connector(self): + return {"name_node": self.name_node} + + @property + def get_name(self): + return StorageEngine.HDFS -class PathAddress(AddressABC): - def __init__(self, path=None): + +class PathAddress(AddressBase): + def __init__(self, path=None, connector_name=None): self.path = path + super(PathAddress, self).__init__(connector_name=connector_name) def __hash__(self): return self.path.__hash__() @@ -62,15 +110,21 @@ def __str__(self): def __repr__(self): return self.__str__() + @property + def get_name(self): + return StorageEngine.PATH + -class MysqlAddress(AddressABC): - def __init__(self, user, passwd, host, port, db, name): +class MysqlAddress(AddressBase): + def __init__(self, user=None, passwd=None, host=None, port=None, db=None, name=None, connector_name=None): self.user = user self.passwd = passwd self.host = host self.port = port self.db = db self.name = name + self.connector_name = connector_name + super(MysqlAddress, self).__init__(connector_name=connector_name) def __hash__(self): return (self.host, self.port, self.db, self.name).__hash__() @@ -81,9 +135,18 @@ def __str__(self): def __repr__(self): return self.__str__() + @property + def connector(self): + return {"user": self.user, "passwd": self.passwd, "host": self.host, "port": self.port} + + @property + def get_name(self): + return StorageEngine.MYSQL + -class HiveAddress(AddressABC): - def __init__(self, host, name, port=10000, username=None, database='default', auth_mechanism='PLAIN', password=None): +class HiveAddress(AddressBase): + def __init__(self, host=None, name=None, port=10000, username=None, database='default', auth_mechanism='PLAIN', + password=None, connector_name=None): self.host = host self.username = username self.port = port @@ -91,6 +154,7 @@ def __init__(self, host, name, port=10000, username=None, database='default', au self.auth_mechanism = auth_mechanism self.password = password self.name = name + super(HiveAddress, self).__init__(connector_name=connector_name) def __hash__(self): return (self.host, self.port, self.database, self.name).__hash__() @@ -101,10 +165,18 @@ def __str__(self): def __repr__(self): return self.__str__() + @property + def connector(self): + return {"host": self.host, "port": self.port, "username": self.username, "password": self.password, "auth_mechanism": self.auth_mechanism} -class LinkisHiveAddress(AddressABC): + @property + def get_name(self): + return StorageEngine.HIVE + + +class LinkisHiveAddress(AddressBase): def __init__(self, host="127.0.0.1", port=9001, username='', database='', name='', run_type='hql', - execute_application_name='hive', source={}, params={}): + execute_application_name='hive', source={}, params={}, connector_name=None): self.host = host self.port = port self.username = username @@ -114,6 +186,7 @@ def __init__(self, host="127.0.0.1", port=9001, username='', database='', name=' self.execute_application_name = execute_application_name self.source=source self.params = params + super(LinkisHiveAddress, self).__init__(connector_name=connector_name) def __hash__(self): return (self.host, self.port, self.database, self.name).__hash__() @@ -124,10 +197,15 @@ def __str__(self): def __repr__(self): return self.__str__() + @property + def get_name(self): + return StorageEngine.LINKIS_HIVE -class LocalFSAddress(AddressABC): - def __init__(self, path): + +class LocalFSAddress(AddressBase): + def __init__(self, path=None, connector_name=None): self.path = path + super(LocalFSAddress, self).__init__(connector_name=connector_name) def __hash__(self): return (self.path).__hash__() @@ -137,3 +215,7 @@ def __str__(self): def __repr__(self): return self.__str__() + + @property + def get_name(self): + return StorageEngine.LOCALFS \ No newline at end of file diff --git a/python/fate_arch/metastore/db_models.py b/python/fate_arch/metastore/db_models.py index cf5817aea1..9af120fd3b 100644 --- a/python/fate_arch/metastore/db_models.py +++ b/python/fate_arch/metastore/db_models.py @@ -83,6 +83,15 @@ def init_database_tables(): DB.create_tables(table_objs) +class StorageConnectorModel(DataBaseModel): + f_name = CharField(max_length=100, primary_key=True) + f_engine = CharField(max_length=100, index=True) # 'MYSQL' + f_connector_info = JSONField() + + class Meta: + db_table = "t_storage_connector" + + class StorageTableMetaModel(DataBaseModel): f_name = CharField(max_length=100, index=True) f_namespace = CharField(max_length=100, index=True) diff --git a/python/fate_arch/storage/__init__.py b/python/fate_arch/storage/__init__.py index 37225b0a7f..5b760204a1 100644 --- a/python/fate_arch/storage/__init__.py +++ b/python/fate_arch/storage/__init__.py @@ -5,3 +5,4 @@ from fate_arch.storage._types import DEFAULT_ID_DELIMITER from fate_arch.storage._session import StorageSessionBase from fate_arch.storage._table import StorageTableBase, StorageTableMeta +from fate_arch.storage._utils import StorageConnector diff --git a/python/fate_arch/storage/_utils.py b/python/fate_arch/storage/_utils.py index 6853b239c2..7e2241e759 100644 --- a/python/fate_arch/storage/_utils.py +++ b/python/fate_arch/storage/_utils.py @@ -1,8 +1,39 @@ -from fate_arch import storage +import operator +from fate_arch.common.base_utils import current_timestamp +from fate_arch.metastore.db_models import DB, StorageConnectorModel -def get_table_info(name, namespace): - data_table_meta = storage.StorageTableMeta(name=name, namespace=namespace) - address = data_table_meta.get_address() - schema = data_table_meta.get_schema() - return address, schema + +class StorageConnector(): + def __init__(self, connector_name, engine, connector_info=None): + self.name = connector_name + self.engine = engine + self.connector_info = connector_info + + @DB.connection_context() + def create_or_update(self): + defaults = { + "f_name": self.name, + "f_engine": self.engine, + "connector_info": self.connector_info, + "f_create_time": current_timestamp(), + + } + connector, status = StorageConnectorModel.get_or_create( + f_name=self.name, + f_engine=self.engine, + defaults=defaults) + if status is False: + for key in defaults: + setattr(connector, key, defaults[key]) + connector.save(force_insert=False) + + @DB.connection_context() + @property + def info(self): + connectors = StorageConnectorModel.select().where(operator.attrgetter("f_name")(StorageConnectorModel) == self.name, + operator.attrgetter("f_engine")(StorageConnectorModel) == self.engine) + if connectors: + return connectors[0].f_connector_info + else: + return None From 428c6420052f924f0b30213a85c713b6d65a4f7b Mon Sep 17 00:00:00 2001 From: zhihuiwan <15779896112@163.com> Date: Thu, 20 Jan 2022 16:46:31 +0800 Subject: [PATCH 02/99] update storage connector --- python/fate_arch/common/address.py | 7 +++-- python/fate_arch/metastore/db_utils.py | 36 ++++++++++++++++++++++++ python/fate_arch/storage/__init__.py | 1 - python/fate_arch/storage/_utils.py | 39 -------------------------- 4 files changed, 40 insertions(+), 43 deletions(-) create mode 100644 python/fate_arch/metastore/db_utils.py diff --git a/python/fate_arch/common/address.py b/python/fate_arch/common/address.py index cd4b02513a..280b9ad614 100644 --- a/python/fate_arch/common/address.py +++ b/python/fate_arch/common/address.py @@ -1,5 +1,6 @@ from fate_arch.abc import AddressABC -from fate_arch.storage import StorageEngine, StorageConnector +from fate_arch.metastore.db_utils import StorageConnector +from fate_arch.storage import StorageEngine class AddressBase(AddressABC): @@ -7,8 +8,8 @@ def __init__(self, connector_name=None): self.connector_name = connector_name if connector_name: connector = StorageConnector(connector_name=connector_name, engine=self.get_name) - if connector.info: - for k, v in connector.info.items(): + if connector.get_info(): + for k, v in connector.get_info().items(): if hasattr(self, k): self.__setattr__(k, v) diff --git a/python/fate_arch/metastore/db_utils.py b/python/fate_arch/metastore/db_utils.py new file mode 100644 index 0000000000..eb769e7904 --- /dev/null +++ b/python/fate_arch/metastore/db_utils.py @@ -0,0 +1,36 @@ +import operator + +from fate_arch.common.base_utils import current_timestamp +from fate_arch.metastore.db_models import DB, StorageConnectorModel + + +class StorageConnector(): + def __init__(self, connector_name, engine=None, connector_info=None): + self.name = connector_name + self.engine = engine + self.connector_info = connector_info + + @DB.connection_context() + def create_or_update(self): + defaults = { + "f_name": self.name, + "f_engine": self.engine, + "f_connector_info": self.connector_info, + "f_create_time": current_timestamp(), + + } + connector, status = StorageConnectorModel.get_or_create( + f_name=self.name, + defaults=defaults) + if status is False: + for key in defaults: + setattr(connector, key, defaults[key]) + connector.save(force_insert=False) + + @DB.connection_context() + def get_info(self): + connectors = [connector for connector in StorageConnectorModel.select().where(operator.attrgetter("f_name")(StorageConnectorModel) == self.name)] + if connectors: + return connectors[0].f_connector_info + else: + return {} diff --git a/python/fate_arch/storage/__init__.py b/python/fate_arch/storage/__init__.py index 5b760204a1..37225b0a7f 100644 --- a/python/fate_arch/storage/__init__.py +++ b/python/fate_arch/storage/__init__.py @@ -5,4 +5,3 @@ from fate_arch.storage._types import DEFAULT_ID_DELIMITER from fate_arch.storage._session import StorageSessionBase from fate_arch.storage._table import StorageTableBase, StorageTableMeta -from fate_arch.storage._utils import StorageConnector diff --git a/python/fate_arch/storage/_utils.py b/python/fate_arch/storage/_utils.py index 7e2241e759..e69de29bb2 100644 --- a/python/fate_arch/storage/_utils.py +++ b/python/fate_arch/storage/_utils.py @@ -1,39 +0,0 @@ -import operator - -from fate_arch.common.base_utils import current_timestamp -from fate_arch.metastore.db_models import DB, StorageConnectorModel - - -class StorageConnector(): - def __init__(self, connector_name, engine, connector_info=None): - self.name = connector_name - self.engine = engine - self.connector_info = connector_info - - @DB.connection_context() - def create_or_update(self): - defaults = { - "f_name": self.name, - "f_engine": self.engine, - "connector_info": self.connector_info, - "f_create_time": current_timestamp(), - - } - connector, status = StorageConnectorModel.get_or_create( - f_name=self.name, - f_engine=self.engine, - defaults=defaults) - if status is False: - for key in defaults: - setattr(connector, key, defaults[key]) - connector.save(force_insert=False) - - @DB.connection_context() - @property - def info(self): - connectors = StorageConnectorModel.select().where(operator.attrgetter("f_name")(StorageConnectorModel) == self.name, - operator.attrgetter("f_engine")(StorageConnectorModel) == self.engine) - if connectors: - return connectors[0].f_connector_info - else: - return None From 970649b935ddf14701305bb60eab9cbdd7ab9970 Mon Sep 17 00:00:00 2001 From: mgqa34 Date: Fri, 21 Jan 2022 14:01:34 +0800 Subject: [PATCH 03/99] fix schema's label setting error of data_transform Signed-off-by: mgqa34 --- python/federatedml/util/data_transform.py | 55 ++++++++++++----------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/python/federatedml/util/data_transform.py b/python/federatedml/util/data_transform.py index 5657602fb1..dec09241d8 100644 --- a/python/federatedml/util/data_transform.py +++ b/python/federatedml/util/data_transform.py @@ -59,8 +59,8 @@ def __init__(self, data_transform_param): self.outlier_impute = data_transform_param.outlier_impute self.outlier_replace_value = data_transform_param.outlier_replace_value self.with_label = data_transform_param.with_label - self.label_name = data_transform_param.label_name.lower() - self.label_type = data_transform_param.label_type + self.label_name = data_transform_param.label_name.lower() if self.with_label else None + self.label_type = data_transform_param.label_type if self.with_label else None self.output_format = data_transform_param.output_format self.missing_impute_rate = None self.outlier_replace_rate = None @@ -416,20 +416,20 @@ def save_model(self): def load_model(self, model_meta, model_param): self.delimitor, self.data_type, self.exclusive_data_type, _1, _2, self.with_label, \ - self.label_type, self.output_format, self.header, self.sid_name, self.label_name, self.with_match_id = \ + self.label_type, self.output_format, self.header, self.sid_name, self.label_name, self.with_match_id = \ load_data_transform_model("DenseFeatureTransformer", model_meta, model_param) self.missing_fill, self.missing_fill_method, \ - self.missing_impute, self.default_value = load_missing_imputer_model(self.header, - "Imputer", - model_meta.imputer_meta, - model_param.imputer_param) + self.missing_impute, self.default_value = load_missing_imputer_model(self.header, + "Imputer", + model_meta.imputer_meta, + model_param.imputer_param) self.outlier_replace, self.outlier_replace_method, \ - self.outlier_impute, self.outlier_replace_value = load_outlier_model(self.header, - "Outlier", - model_meta.outlier_meta, - model_param.outlier_param) + self.outlier_impute, self.outlier_replace_value = load_outlier_model(self.header, + "Outlier", + model_meta.outlier_meta, + model_param.outlier_param) # ============================================================================= @@ -443,10 +443,10 @@ def __init__(self, data_transform_param): self.output_format = data_transform_param.output_format self.header = None self.sid_name = "sid" - self.label_name = data_transform_param.label_name self.with_match_id = data_transform_param.with_match_id self.match_id_name = "match_id" if self.with_match_id else None self.with_label = data_transform_param.with_label + self.label_name = data_transform_param.label_name if self.with_label else None def get_max_feature_index(self, line, delimitor=' '): if line.strip() == '': @@ -604,7 +604,7 @@ def save_model(self): def load_model(self, model_meta, model_param): self.delimitor, self.data_type, _0, _1, _2, self.with_label, \ - self.label_type, self.output_format, self.header, self.sid_name, self.label_name, self.with_match_id = \ + self.label_type, self.output_format, self.header, self.sid_name, self.label_name, self.with_match_id = \ load_data_transform_model( "SparseFeatureTransformer", model_meta, @@ -621,7 +621,7 @@ def __init__(self, data_transform_param): self.tag_with_value = data_transform_param.tag_with_value self.tag_value_delimitor = data_transform_param.tag_value_delimitor self.with_label = data_transform_param.with_label - self.label_type = data_transform_param.label_type + self.label_type = data_transform_param.label_type if self.with_label else None self.output_format = data_transform_param.output_format self.header = None self.sid_name = "sid" @@ -918,16 +918,16 @@ def save_model(self): def load_model(self, model_meta, model_param): self.delimitor, self.data_type, _0, self.tag_with_value, self.tag_value_delimitor, self.with_label, \ - self.label_type, self.output_format, self.header, self.sid_name, self.label_name, self.with_match_id = load_data_transform_model( - "SparseTagTransformer", - model_meta, - model_param) + self.label_type, self.output_format, self.header, self.sid_name, self.label_name, self.with_match_id = load_data_transform_model( + "SparseTagTransformer", + model_meta, + model_param) self.missing_fill, self.missing_fill_method, \ - self.missing_impute, self.default_value = load_missing_imputer_model(self.header, - "Imputer", - model_meta.imputer_meta, - model_param.imputer_param) + self.missing_impute, self.default_value = load_missing_imputer_model(self.header, + "Imputer", + model_meta.imputer_meta, + model_param.imputer_param) class DataTransform(ModelBase): @@ -1030,8 +1030,9 @@ def save_data_transform_model(input_format="dense", model_meta.tag_with_value = tag_with_value model_meta.tag_value_delimitor = tag_value_delimitor model_meta.with_label = with_label - model_meta.label_name = label_name - model_meta.label_type = label_type + if with_label: + model_meta.label_name = label_name + model_meta.label_type = label_type model_meta.output_format = output_format model_meta.with_match_id = with_match_id @@ -1058,8 +1059,8 @@ def load_data_transform_model(model_name="DataTransform", tag_with_value = model_meta.tag_with_value tag_value_delimitor = model_meta.tag_value_delimitor with_label = model_meta.with_label - label_name = model_meta.label_name - label_type = model_meta.label_type + label_name = model_meta.label_name if with_label else None + label_type = model_meta.label_type if with_label else None with_match_id = model_meta.with_match_id output_format = model_meta.output_format @@ -1077,7 +1078,7 @@ def load_data_transform_model(model_name="DataTransform", exclusive_data_type[col_name] = model_meta.exclusive_data_type.get(col_name) return delimitor, data_type, exclusive_data_type, tag_with_value, tag_value_delimitor, with_label, \ - label_type, output_format, header, sid_name, label_name, with_match_id + label_type, output_format, header, sid_name, label_name, with_match_id def save_missing_imputer_model(missing_fill=False, From 9d683bd8d49486ca1319bc2ff1c0096d60e8f10f Mon Sep 17 00:00:00 2001 From: mgqa34 Date: Mon, 24 Jan 2022 17:33:33 +0800 Subject: [PATCH 04/99] add predict type to hetero kmeans's predict output data's header Signed-off-by: mgqa34 --- .../unsupervised_learning/kmeans/kmeans_model_base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/federatedml/unsupervised_learning/kmeans/kmeans_model_base.py b/python/federatedml/unsupervised_learning/kmeans/kmeans_model_base.py index 7d1ffd1d63..b0a03928fe 100644 --- a/python/federatedml/unsupervised_learning/kmeans/kmeans_model_base.py +++ b/python/federatedml/unsupervised_learning/kmeans/kmeans_model_base.py @@ -192,12 +192,12 @@ def set_predict_data_schema(self, predict_datas, schemas): data_output2 = predict_data[1] if data_output1 is not None: data_output1.schema = { - "header": ["cluster_sample_count", "cluster_inner_dist", "inter_cluster_dist"], + "header": ["cluster_sample_count", "cluster_inner_dist", "inter_cluster_dist", "type"], "sid_name": "cluster_index", "content_type": "cluster_result" } if data_output2 is not None: - data_output2.schema = {"header": ["predicted_cluster_index", "distance"], + data_output2.schema = {"header": ["predicted_cluster_index", "distance", "type"], "sid_name": "id", "content_type": "cluster_result"} @@ -205,7 +205,7 @@ def set_predict_data_schema(self, predict_datas, schemas): else: data_output = predict_data if data_output is not None: - data_output.schema = {"header": ["label", "predicted label"], + data_output.schema = {"header": ["label", "predicted_label", "type"], "sid_name": schema.get('sid_name'), "content_type": "cluster_result"} From 979f1be1dc8b10e84510f6008d53e6ea0949a926 Mon Sep 17 00:00:00 2001 From: dylanfan <289765648@qq.com> Date: Tue, 8 Feb 2022 10:12:20 +0800 Subject: [PATCH 05/99] fixed default q_filed for spdz --- python/federatedml/secureprotol/spdz/spdz.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/python/federatedml/secureprotol/spdz/spdz.py b/python/federatedml/secureprotol/spdz/spdz.py index 5c6a8153ed..c37124c024 100644 --- a/python/federatedml/secureprotol/spdz/spdz.py +++ b/python/federatedml/secureprotol/spdz/spdz.py @@ -20,6 +20,9 @@ from federatedml.secureprotol.spdz.utils import naming +Q = 293973345475167247070445277780365744413 ** 2 + + class SPDZ(object): __instance = None @@ -37,7 +40,7 @@ def set_instance(cls, instance): def has_instance(cls): return cls.__instance is not None - def __init__(self, name="ss", q_field=2 << 60, local_party=None, all_parties=None, use_mix_rand=False, n_length=1024): + def __init__(self, name="ss", q_field=Q, local_party=None, all_parties=None, use_mix_rand=False, n_length=1024): self.name_service = naming.NamingService(name) self._prev_name_service = None self._pre_instance = None @@ -49,7 +52,12 @@ def __init__(self, name="ss", q_field=2 << 60, local_party=None, all_parties=Non if len(self.other_parties) > 1: raise EnvironmentError("support 2-party secret share only") self.public_key, self.private_key = PaillierKeypair.generate_keypair(n_length=n_length) - self.q_field = q_field + + if q_field is None or q_field < self.public_key.n: + self.q_field = self.public_key.n + else: + self.q_field = q_field + self.use_mix_rand = use_mix_rand def __enter__(self): From 83a63cfecf58bbd899a23b615e3972e210467e6d Mon Sep 17 00:00:00 2001 From: mgqa34 Date: Wed, 9 Feb 2022 10:24:00 +0800 Subject: [PATCH 06/99] add batch shuffle in hetero logistic regression Signed-off-by: mgqa34 --- .../param/logistic_regression_param.py | 9 ++-- .../hetero/procedure/batch_generator.py | 40 +++++++++++++-- .../linear_model/linear_model_base.py | 2 + .../hetero_lr_guest.py | 11 ++-- .../hetero_lr_host.py | 9 ++-- python/federatedml/model_selection/indices.py | 3 +- .../federatedml/model_selection/mini_batch.py | 50 ++++++++++++++++--- .../param/logistic_regression_param.py | 9 ++-- 8 files changed, 105 insertions(+), 28 deletions(-) diff --git a/python/fate_client/pipeline/param/logistic_regression_param.py b/python/fate_client/pipeline/param/logistic_regression_param.py index bec667b2f3..4c95db6178 100644 --- a/python/fate_client/pipeline/param/logistic_regression_param.py +++ b/python/fate_client/pipeline/param/logistic_regression_param.py @@ -112,7 +112,8 @@ class LogisticParam(BaseParam): def __init__(self, penalty='L2', tol=1e-4, alpha=1.0, optimizer='rmsprop', - batch_size=-1, learning_rate=0.01, init_param=InitParam(), + batch_size=-1, shuffle=True, + learning_rate=0.01, init_param=InitParam(), max_iter=100, early_stop='diff', encrypt_param=EncryptParam(), predict_param=PredictParam(), cv_param=CrossValidationParam(), decay=1, decay_sqrt=True, @@ -128,6 +129,7 @@ def __init__(self, penalty='L2', self.alpha = alpha self.optimizer = optimizer self.batch_size = batch_size + self.shuffle = shuffle self.learning_rate = learning_rate self.init_param = copy.deepcopy(init_param) self.max_iter = max_iter @@ -327,7 +329,8 @@ def check(self): class HeteroLogisticParam(LogisticParam): def __init__(self, penalty='L2', tol=1e-4, alpha=1.0, optimizer='rmsprop', - batch_size=-1, learning_rate=0.01, init_param=InitParam(), + batch_size=-1, shuffle=True, + learning_rate=0.01, init_param=InitParam(), max_iter=100, early_stop='diff', encrypted_mode_calculator_param=EncryptedModeCalculatorParam(), predict_param=PredictParam(), cv_param=CrossValidationParam(), @@ -339,7 +342,7 @@ def __init__(self, penalty='L2', callback_param=CallbackParam() ): super(HeteroLogisticParam, self).__init__(penalty=penalty, tol=tol, alpha=alpha, optimizer=optimizer, - batch_size=batch_size, + batch_size=batch_size, shuffle=shuffle, learning_rate=learning_rate, init_param=init_param, max_iter=max_iter, early_stop=early_stop, predict_param=predict_param, cv_param=cv_param, diff --git a/python/federatedml/framework/hetero/procedure/batch_generator.py b/python/federatedml/framework/hetero/procedure/batch_generator.py index 422f369252..e8ed463ce7 100644 --- a/python/federatedml/framework/hetero/procedure/batch_generator.py +++ b/python/federatedml/framework/hetero/procedure/batch_generator.py @@ -32,11 +32,20 @@ def register_batch_generator(self, transfer_variables, has_arbiter=True): transfer_variables.batch_data_index, has_arbiter) - def initialize_batch_generator(self, data_instances, batch_size, suffix=tuple()): - self.mini_batch_obj = MiniBatch(data_instances, batch_size=batch_size) + def initialize_batch_generator(self, data_instances, batch_size, suffix=tuple(), shuffle=False): + self.mini_batch_obj = MiniBatch(data_instances, batch_size=batch_size, shuffle=shuffle) self.batch_nums = self.mini_batch_obj.batch_nums batch_info = {"batch_size": batch_size, "batch_num": self.batch_nums} self.sync_batch_info(batch_info, suffix) + + """if need shuffle, batch_data will be generated during every iteration""" + if shuffle is True: + return + + self.prepare_batch_data(suffix) + + def prepare_batch_data(self, suffix=tuple()): + self.mini_batch_obj.generate_batch_data() index_generator = self.mini_batch_obj.mini_batch_data_generator(result='index') batch_index = 0 for batch_data_index in index_generator: @@ -44,7 +53,10 @@ def initialize_batch_generator(self, data_instances, batch_size, suffix=tuple()) self.sync_batch_index(batch_data_index, batch_suffix) batch_index += 1 - def generate_batch_data(self): + def generate_batch_data(self, suffix=tuple()): + if self.mini_batch_obj.shuffle: + self.prepare_batch_data(suffix=suffix) + data_generator = self.mini_batch_obj.mini_batch_data_generator(result='data') for batch_data in data_generator: yield batch_data @@ -55,6 +67,8 @@ def __init__(self): self.finish_sycn = False self.batch_data_insts = [] self.batch_nums = None + self.data_inst = None + self.shuffle = False def register_batch_generator(self, transfer_variables, has_arbiter=None): self._register_batch_data_index_transfer(transfer_variables.batch_info, transfer_variables.batch_data_index) @@ -62,14 +76,30 @@ def register_batch_generator(self, transfer_variables, has_arbiter=None): def initialize_batch_generator(self, data_instances, suffix=tuple(), **kwargs): batch_info = self.sync_batch_info(suffix) self.batch_nums = batch_info.get('batch_num') + + """if need shuffle, batch_data will be generated during every iteration""" + self.shuffle = kwargs.get("shuffle", False) + if self.shuffle is True: + self.data_inst = data_instances + return + + self.prepare_batch_data(data_instances, suffix) + + def prepare_batch_data(self, data_inst, suffix=tuple()): + self.batch_data_insts = [] for batch_index in range(self.batch_nums): batch_suffix = suffix + (batch_index,) batch_data_index = self.sync_batch_index(suffix=batch_suffix) # batch_data_inst = batch_data_index.join(data_instances, lambda g, d: d) - batch_data_inst = data_instances.join(batch_data_index, lambda d, g: d) + batch_data_inst = data_inst.join(batch_data_index, lambda d, g: d) self.batch_data_insts.append(batch_data_inst) - def generate_batch_data(self): + LOGGER.debug(f"mgq-debug5: batch_data_index is {list(batch_data_index.collect())}") + + def generate_batch_data(self, suffix=tuple()): + if self.shuffle: + self.prepare_batch_data(data_inst=self.data_inst, suffix=suffix) + batch_index = 0 for batch_data_inst in self.batch_data_insts: LOGGER.info("batch_num: {}, batch_data_inst size:{}".format( diff --git a/python/federatedml/linear_model/linear_model_base.py b/python/federatedml/linear_model/linear_model_base.py index 862dc91b52..fea2a856ad 100644 --- a/python/federatedml/linear_model/linear_model_base.py +++ b/python/federatedml/linear_model/linear_model_base.py @@ -66,6 +66,8 @@ def _init_model(self, params): self.init_param_obj = params.init_param # self.fit_intercept = self.init_param_obj.fit_intercept self.batch_size = params.batch_size + if hasattr(params, "shuffle"): + self.shuffle = params.shuffle self.max_iter = params.max_iter self.optimizer = optimizer_factory(params) self.converge_func = converge_func_factory(params.early_stop, params.tol) diff --git a/python/federatedml/linear_model/logistic_regression/hetero_logistic_regression/hetero_lr_guest.py b/python/federatedml/linear_model/logistic_regression/hetero_logistic_regression/hetero_lr_guest.py index 4e18a8e1d3..7cac4a96c4 100644 --- a/python/federatedml/linear_model/logistic_regression/hetero_logistic_regression/hetero_lr_guest.py +++ b/python/federatedml/linear_model/logistic_regression/hetero_logistic_regression/hetero_lr_guest.py @@ -104,7 +104,7 @@ def fit_binary(self, data_instances, validate_data=None): self.transfer_variable.use_async.remote(use_async) LOGGER.info("Generate mini-batch from input data") - self.batch_generator.initialize_batch_generator(data_instances, self.batch_size) + self.batch_generator.initialize_batch_generator(data_instances, self.batch_size, shuffle=self.shuffle) self.gradient_loss_operator.set_total_batch_nums(self.batch_generator.batch_nums) self.encrypted_calculator = [EncryptModeCalculator(self.cipher_operator, @@ -123,16 +123,17 @@ def fit_binary(self, data_instances, validate_data=None): while self.n_iter_ < self.max_iter: self.callback_list.on_epoch_begin(self.n_iter_) - LOGGER.info("iter:{}".format(self.n_iter_)) - batch_data_generator = self.batch_generator.generate_batch_data() + LOGGER.info("iter: {}".format(self.n_iter_)) + batch_data_generator = self.batch_generator.generate_batch_data(suffix=(self.n_iter_, )) self.optimizer.set_iters(self.n_iter_) batch_index = 0 for batch_data in batch_data_generator: batch_feat_inst = batch_data # Start gradient procedure - LOGGER.debug("iter: {}, before compute gradient, data count: {}".format(self.n_iter_, - batch_feat_inst.count())) + LOGGER.debug("iter: {}, batch: {}, before compute gradient, data count: {}".format(self.n_iter_, + batch_index, + batch_feat_inst.count())) optim_guest_gradient = self.gradient_loss_operator.compute_gradient_procedure( batch_feat_inst, self.encrypted_calculator, diff --git a/python/federatedml/linear_model/logistic_regression/hetero_logistic_regression/hetero_lr_host.py b/python/federatedml/linear_model/logistic_regression/hetero_logistic_regression/hetero_lr_host.py index d714729720..78414b9094 100644 --- a/python/federatedml/linear_model/logistic_regression/hetero_logistic_regression/hetero_lr_host.py +++ b/python/federatedml/linear_model/logistic_regression/hetero_logistic_regression/hetero_lr_host.py @@ -105,7 +105,7 @@ def fit_binary(self, data_instances, validate_data): LOGGER.debug(f"set_use_async") self.gradient_loss_operator.set_use_async() - self.batch_generator.initialize_batch_generator(data_instances) + self.batch_generator.initialize_batch_generator(data_instances, shuffle=self.shuffle) self.gradient_loss_operator.set_total_batch_nums(self.batch_generator.batch_nums) self.encrypted_calculator = [EncryptModeCalculator(self.cipher_operator, @@ -127,8 +127,8 @@ def fit_binary(self, data_instances, validate_data): while self.n_iter_ < self.max_iter: self.callback_list.on_epoch_begin(self.n_iter_) - LOGGER.info("iter:" + str(self.n_iter_)) - batch_data_generator = self.batch_generator.generate_batch_data() + LOGGER.info("iter: " + str(self.n_iter_)) + batch_data_generator = self.batch_generator.generate_batch_data(suffix=(self.n_iter_, )) batch_index = 0 self.optimizer.set_iters(self.n_iter_) for batch_data in batch_data_generator: @@ -136,6 +136,9 @@ def fit_binary(self, data_instances, validate_data): batch_feat_inst = batch_data # LOGGER.debug(f"MODEL_STEP In Batch {batch_index}, batch data count: {batch_feat_inst.count()}") + LOGGER.debug("iter: {}, batch: {}, before compute gradient, data count: {}".format(self.n_iter_, + batch_index, + batch_feat_inst.count())) optim_host_gradient = self.gradient_loss_operator.compute_gradient_procedure( batch_feat_inst, self.encrypted_calculator, self.model_weights, self.optimizer, self.n_iter_, batch_index) diff --git a/python/federatedml/model_selection/indices.py b/python/federatedml/model_selection/indices.py index 461e580d4f..a334c3dc08 100644 --- a/python/federatedml/model_selection/indices.py +++ b/python/federatedml/model_selection/indices.py @@ -22,8 +22,9 @@ def collect_index(data_insts): data_sids = data_insts.mapValues(lambda data_inst: None) - data_size = data_sids.count() # Record data nums that left + # data_size = data_sids.count() # Record data nums that left data_sids_iter = data_sids.collect() data_sids_iter = sorted(data_sids_iter, key=lambda x: x[0]) + data_size = len(data_sids_iter) return data_sids_iter, data_size diff --git a/python/federatedml/model_selection/mini_batch.py b/python/federatedml/model_selection/mini_batch.py index a0f369cec6..ee3334665c 100644 --- a/python/federatedml/model_selection/mini_batch.py +++ b/python/federatedml/model_selection/mini_batch.py @@ -14,6 +14,7 @@ # limitations under the License. # +import random from fate_arch.session import computing_session as session from federatedml.model_selection import indices @@ -21,19 +22,21 @@ class MiniBatch: - def __init__(self, data_inst, batch_size=320): + def __init__(self, data_inst, batch_size=320, shuffle=False): self.batch_data_sids = None self.batch_nums = 0 self.data_inst = data_inst self.all_batch_data = None self.all_index_data = None + self.data_sids_iter = None + self.shuffle = shuffle if batch_size == -1: self.batch_size = data_inst.count() else: self.batch_size = batch_size - self.__mini_batch_data_seperator(data_inst, batch_size) + self.__mini_batch_data_seperator(data_inst, self.batch_size) # LOGGER.debug("In mini batch init, batch_num:{}".format(self.batch_nums)) def mini_batch_data_generator(self, result='data'): @@ -49,7 +52,7 @@ def mini_batch_data_generator(self, result='data'): ------- A generator that might generate data or index. """ - LOGGER.debug("Currently, len of all_batch_data: {}".format(len(self.all_batch_data))) + LOGGER.debug("Currently, batch_num is: {}".format(self.batch_nums)) if result == 'index': for index_table in self.all_index_data: yield index_table @@ -58,25 +61,55 @@ def mini_batch_data_generator(self, result='data'): yield batch_data def __mini_batch_data_seperator(self, data_insts, batch_size): - data_sids_iter, data_size = indices.collect_index(data_insts) + self.data_sids_iter, data_size = indices.collect_index(data_insts) if batch_size > data_size: batch_size = data_size self.batch_size = batch_size - batch_nums = (data_size + batch_size - 1) // batch_size + self.batch_nums = (data_size + batch_size - 1) // batch_size + if self.shuffle is False: + self.__generate_batch_data(data_insts) + + def generate_batch_data(self): + if self.shuffle: + self.__generate_batch_data(data_insts=self.data_inst) + + def __generate_batch_data(self, data_insts): + if self.shuffle: + random.SystemRandom().shuffle(self.data_sids_iter) + + self.all_batch_data = [] + self.all_index_data = [] + + for bid in range(self.batch_nums): + batch_ids = self.data_sids_iter[bid * self.batch_size:(bid + 1) * self.batch_size] + index_table = session.parallelize(batch_ids, + include_key=True, + partition=data_insts.partitions) + batch_data = index_table.join(data_insts, lambda x, y: y) + LOGGER.debug(f"mgq-debug, batch data size is {batch_data.count()}, " + f"data_inst size is {data_insts.count()}, " + f"index table size is {index_table.count()}, " + f"index table is {list(index_table.collect())}, " + f"batch_ids is : {batch_ids}") + self.all_index_data.append(index_table) + self.all_batch_data.append(batch_data) + + """ batch_data_sids = [] curt_data_num = 0 curt_batch = 0 curt_batch_ids = [] - for sid, values in data_sids_iter: + data_size = len(self.data_sids_iter) + for sid, values in self.data_sids_iter: # print('sid is {}, values is {}'.format(sid, values)) curt_batch_ids.append((sid, None)) curt_data_num += 1 - if curt_data_num % batch_size == 0: + if curt_data_num % self.batch_size == 0: curt_batch += 1 - if curt_batch < batch_nums: + if curt_batch < self.batch_nums: batch_data_sids.append(curt_batch_ids) curt_batch_ids = [] if curt_data_num == data_size and len(curt_batch_ids) != 0: @@ -96,3 +129,4 @@ def __mini_batch_data_seperator(self, data_insts, batch_size): all_index_data.append(index_table) self.all_batch_data = all_batch_data self.all_index_data = all_index_data + """ diff --git a/python/federatedml/param/logistic_regression_param.py b/python/federatedml/param/logistic_regression_param.py index 46349f8ed2..448b870cc7 100644 --- a/python/federatedml/param/logistic_regression_param.py +++ b/python/federatedml/param/logistic_regression_param.py @@ -116,7 +116,8 @@ class LogisticParam(BaseParam): def __init__(self, penalty='L2', tol=1e-4, alpha=1.0, optimizer='rmsprop', - batch_size=-1, learning_rate=0.01, init_param=InitParam(), + batch_size=-1, shuffle=True, + learning_rate=0.01, init_param=InitParam(), max_iter=100, early_stop='diff', encrypt_param=EncryptParam(), predict_param=PredictParam(), cv_param=CrossValidationParam(), decay=1, decay_sqrt=True, @@ -149,6 +150,7 @@ def __init__(self, penalty='L2', self.use_first_metric_only = use_first_metric_only self.floating_point_precision = floating_point_precision self.callback_param = copy.deepcopy(callback_param) + self.shuffle = shuffle def check(self): descr = "logistic_param's" @@ -356,7 +358,8 @@ def check(self): class HeteroLogisticParam(LogisticParam): def __init__(self, penalty='L2', tol=1e-4, alpha=1.0, optimizer='rmsprop', - batch_size=-1, learning_rate=0.01, init_param=InitParam(), + batch_size=-1, shuffle=True, + learning_rate=0.01, init_param=InitParam(), max_iter=100, early_stop='diff', encrypted_mode_calculator_param=EncryptedModeCalculatorParam(), predict_param=PredictParam(), cv_param=CrossValidationParam(), @@ -368,7 +371,7 @@ def __init__(self, penalty='L2', callback_param=CallbackParam() ): super(HeteroLogisticParam, self).__init__(penalty=penalty, tol=tol, alpha=alpha, optimizer=optimizer, - batch_size=batch_size, + batch_size=batch_size, shuffle=shuffle, learning_rate=learning_rate, init_param=init_param, max_iter=max_iter, early_stop=early_stop, predict_param=predict_param, cv_param=cv_param, From ee2abb85ce538bbd5b7c5867af8c5b0f966d71e8 Mon Sep 17 00:00:00 2001 From: mgqa34 Date: Wed, 9 Feb 2022 12:27:39 +0800 Subject: [PATCH 07/99] delete debug info Signed-off-by: mgqa34 --- .../framework/hetero/procedure/batch_generator.py | 2 -- python/federatedml/model_selection/mini_batch.py | 5 ----- python/federatedml/util/data_transform.py | 2 -- 3 files changed, 9 deletions(-) diff --git a/python/federatedml/framework/hetero/procedure/batch_generator.py b/python/federatedml/framework/hetero/procedure/batch_generator.py index e8ed463ce7..05b2bc8d43 100644 --- a/python/federatedml/framework/hetero/procedure/batch_generator.py +++ b/python/federatedml/framework/hetero/procedure/batch_generator.py @@ -94,8 +94,6 @@ def prepare_batch_data(self, data_inst, suffix=tuple()): batch_data_inst = data_inst.join(batch_data_index, lambda d, g: d) self.batch_data_insts.append(batch_data_inst) - LOGGER.debug(f"mgq-debug5: batch_data_index is {list(batch_data_index.collect())}") - def generate_batch_data(self, suffix=tuple()): if self.shuffle: self.prepare_batch_data(data_inst=self.data_inst, suffix=suffix) diff --git a/python/federatedml/model_selection/mini_batch.py b/python/federatedml/model_selection/mini_batch.py index ee3334665c..c5cfaaf92c 100644 --- a/python/federatedml/model_selection/mini_batch.py +++ b/python/federatedml/model_selection/mini_batch.py @@ -89,11 +89,6 @@ def __generate_batch_data(self, data_insts): include_key=True, partition=data_insts.partitions) batch_data = index_table.join(data_insts, lambda x, y: y) - LOGGER.debug(f"mgq-debug, batch data size is {batch_data.count()}, " - f"data_inst size is {data_insts.count()}, " - f"index table size is {index_table.count()}, " - f"index table is {list(index_table.collect())}, " - f"batch_ids is : {batch_ids}") self.all_index_data.append(index_table) self.all_batch_data.append(batch_data) diff --git a/python/federatedml/util/data_transform.py b/python/federatedml/util/data_transform.py index 5657602fb1..7a77027c99 100644 --- a/python/federatedml/util/data_transform.py +++ b/python/federatedml/util/data_transform.py @@ -672,8 +672,6 @@ def read_data(self, input_data, mode="fit"): schema = make_schema(self.header, self.sid_name, self.label_name, self.match_id_name) set_schema(data_instance, schema) - for k, inst in data_instance.collect(): - LOGGER.debug(f"mgq-debug : {inst.inst_id}") return data_instance @staticmethod From 1cb908a091336664b1e68b8a545011eddc9723ed Mon Sep 17 00:00:00 2001 From: mgqa34 Date: Thu, 10 Feb 2022 15:45:34 +0800 Subject: [PATCH 08/99] fix dataio label_name setting problem Signed-off-by: mgqa34 --- python/federatedml/util/data_io.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/python/federatedml/util/data_io.py b/python/federatedml/util/data_io.py index 9f6aebd3f5..81f4ae9c01 100644 --- a/python/federatedml/util/data_io.py +++ b/python/federatedml/util/data_io.py @@ -59,8 +59,8 @@ def __init__(self, data_io_param): self.outlier_impute = data_io_param.outlier_impute self.outlier_replace_value = data_io_param.outlier_replace_value self.with_label = data_io_param.with_label - self.label_name = data_io_param.label_name - self.label_type = data_io_param.label_type + self.label_name = data_io_param.label_name if self.with_label else None + self.label_type = data_io_param.label_type if self.with_label else None self.output_format = data_io_param.output_format self.missing_impute_rate = None self.outlier_replace_rate = None @@ -402,7 +402,7 @@ def __init__(self, data_io_param): self.output_format = data_io_param.output_format self.header = None self.sid_name = "sid" - self.label_name = self.label_name = data_io_param.label_name + self.label_name = data_io_param.label_name def get_max_feature_index(self, line, delimitor=' '): if line.strip() == '': @@ -558,11 +558,11 @@ def __init__(self, data_io_param): self.tag_with_value = data_io_param.tag_with_value self.tag_value_delimitor = data_io_param.tag_value_delimitor self.with_label = data_io_param.with_label - self.label_type = data_io_param.label_type + self.label_type = data_io_param.label_type if self.with_label else None self.output_format = data_io_param.output_format self.header = None self.sid_name = "sid" - self.label_name = self.label_name = data_io_param.label_name + self.label_name = data_io_param.label_name if self.with_label else None self.missing_fill = data_io_param.missing_fill self.missing_fill_method = data_io_param.missing_fill_method self.default_value = data_io_param.default_value @@ -940,8 +940,9 @@ def save_data_io_model(input_format="dense", model_meta.tag_with_value = tag_with_value model_meta.tag_value_delimitor = tag_value_delimitor model_meta.with_label = with_label - model_meta.label_name = label_name - model_meta.label_type = label_type + if with_label: + model_meta.label_name = label_name + model_meta.label_type = label_type model_meta.output_format = output_format if header is not None: @@ -967,8 +968,8 @@ def load_data_io_model(model_name="DataIO", tag_with_value = model_meta.tag_with_value tag_value_delimitor = model_meta.tag_value_delimitor with_label = model_meta.with_label - label_name = model_meta.label_name - label_type = model_meta.label_type + label_name = model_meta.label_name if with_label else None + label_type = model_meta.label_type if with_label else None output_format = model_meta.output_format header = list(model_param.header) or None From 1c1dbc0cd8c875502db964af4f38f11d5efa76e2 Mon Sep 17 00:00:00 2001 From: dylanfan <289765648@qq.com> Date: Thu, 10 Feb 2022 18:09:49 +0800 Subject: [PATCH 09/99] test --- .../hetero_sshe_logistic_regression/hetero_lr_base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/federatedml/linear_model/logistic_regression/hetero_sshe_logistic_regression/hetero_lr_base.py b/python/federatedml/linear_model/logistic_regression/hetero_sshe_logistic_regression/hetero_lr_base.py index 078845f829..80354a4666 100644 --- a/python/federatedml/linear_model/logistic_regression/hetero_sshe_logistic_regression/hetero_lr_base.py +++ b/python/federatedml/linear_model/logistic_regression/hetero_sshe_logistic_regression/hetero_lr_base.py @@ -16,6 +16,7 @@ # See the License for the specific language governing permissions and # limitations under the License. + import copy from abc import ABC From eff5f026b4f12af2c5b79a741ed623fb33c56aa3 Mon Sep 17 00:00:00 2001 From: dylanfan <289765648@qq.com> Date: Thu, 10 Feb 2022 18:13:17 +0800 Subject: [PATCH 10/99] test Signed off by: dylanfan <289765648@qq.com> --- .../hetero_sshe_logistic_regression/hetero_lr_base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/federatedml/linear_model/logistic_regression/hetero_sshe_logistic_regression/hetero_lr_base.py b/python/federatedml/linear_model/logistic_regression/hetero_sshe_logistic_regression/hetero_lr_base.py index 80354a4666..078845f829 100644 --- a/python/federatedml/linear_model/logistic_regression/hetero_sshe_logistic_regression/hetero_lr_base.py +++ b/python/federatedml/linear_model/logistic_regression/hetero_sshe_logistic_regression/hetero_lr_base.py @@ -16,7 +16,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - import copy from abc import ABC From 21e36c2a07b2098fabe9d06c973f67786804d274 Mon Sep 17 00:00:00 2001 From: zhihuiwan <15779896112@163.com> Date: Fri, 11 Feb 2022 11:29:02 +0800 Subject: [PATCH 11/99] update --- python/fate_arch/metastore/db_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/fate_arch/metastore/db_utils.py b/python/fate_arch/metastore/db_utils.py index eb769e7904..ff029db32b 100644 --- a/python/fate_arch/metastore/db_utils.py +++ b/python/fate_arch/metastore/db_utils.py @@ -4,7 +4,7 @@ from fate_arch.metastore.db_models import DB, StorageConnectorModel -class StorageConnector(): +class StorageConnector: def __init__(self, connector_name, engine=None, connector_info=None): self.name = connector_name self.engine = engine From 91821199a7fa1118ad5c2c791ae5496d3a8881cc Mon Sep 17 00:00:00 2001 From: zhihuiwan <15779896112@163.com> Date: Fri, 11 Feb 2022 11:29:58 +0800 Subject: [PATCH 12/99] add connector create cli --- .../flow_client/flow_cli/commands/table.py | 36 +++++++++++++++++++ .../flow_client/flow_cli/utils/cli_args.py | 3 +- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/python/fate_client/flow_client/flow_cli/commands/table.py b/python/fate_client/flow_client/flow_cli/commands/table.py index ffdcbf0af2..1e06cfe0b1 100644 --- a/python/fate_client/flow_client/flow_cli/commands/table.py +++ b/python/fate_client/flow_client/flow_cli/commands/table.py @@ -111,6 +111,42 @@ def bind(ctx, **kwargs): access_server('post', ctx, 'table/bind', config_data) +@table.command("connector-create", short_help="create connector") +@cli_args.CONF_PATH +@click.pass_context +def connector_create_or_update(ctx, **kwargs): + """ + - DESCRIPTION: + + \b + Create a connector to fate address. + + \b + - USAGE: + flow table connector-create -c fateflow/examples/connector/create_or_update.json + """ + config_data, _ = preprocess(**kwargs) + access_server('post', ctx, 'table/connector/create', config_data) + + +@table.command("connector-query", short_help="query connector info") +@cli_args.CONNECTOR_NAME +@click.pass_context +def connector_query(ctx, **kwargs): + """ + - DESCRIPTION: + + \b + query connector info. + + \b + - USAGE: + flow table connector-query --connector-name xxx + """ + config_data, _ = preprocess(**kwargs) + access_server('post', ctx, 'table/connector/query', config_data) + + @table.command("tracking-source", short_help="Tracking Source Table") @cli_args.NAMESPACE @cli_args.TABLE_NAME diff --git a/python/fate_client/flow_client/flow_cli/utils/cli_args.py b/python/fate_client/flow_client/flow_cli/utils/cli_args.py index 1cabb37265..7d434a2e32 100644 --- a/python/fate_client/flow_client/flow_cli/utils/cli_args.py +++ b/python/fate_client/flow_client/flow_cli/utils/cli_args.py @@ -117,4 +117,5 @@ PRIVILEGE_COMMAND = click.option("--privilege-command", type=click.STRING, help="privilege command.") PRIVILEGE_COMPONENT = click.option("--privilege-component", type=click.STRING, help="privilege component.") -MIN_DATA = click.option("--min-data", type=click.INT, help="min data") \ No newline at end of file +MIN_DATA = click.option("--min-data", type=click.INT, help="min data") +CONNECTOR_NAME = click.option("--connector-name", type=click.STRING, required=True, help="connector name") \ No newline at end of file From 12c5373615cf437a98992eb594bb9b0521929989 Mon Sep 17 00:00:00 2001 From: zhihuiwan <15779896112@163.com> Date: Fri, 11 Feb 2022 14:30:51 +0800 Subject: [PATCH 13/99] update connector create command short help --- python/fate_client/flow_client/flow_cli/commands/table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/fate_client/flow_client/flow_cli/commands/table.py b/python/fate_client/flow_client/flow_cli/commands/table.py index 1e06cfe0b1..00940375f7 100644 --- a/python/fate_client/flow_client/flow_cli/commands/table.py +++ b/python/fate_client/flow_client/flow_cli/commands/table.py @@ -111,7 +111,7 @@ def bind(ctx, **kwargs): access_server('post', ctx, 'table/bind', config_data) -@table.command("connector-create", short_help="create connector") +@table.command("connector-create", short_help="create or update connector") @cli_args.CONF_PATH @click.pass_context def connector_create_or_update(ctx, **kwargs): From 68e353b78a46d250c4cf7ec9312c1f3a26712353 Mon Sep 17 00:00:00 2001 From: weijingchen Date: Fri, 11 Feb 2022 15:10:58 +0800 Subject: [PATCH 14/99] add EINI inference Signed-off-by: cwj --- .../hetero/hetero_secureboost_guest.py | 110 ++++++++++++++++ .../hetero/hetero_secureboost_host.py | 118 ++++++++++++++++++ ...cure_boosting_predict_transfer_variable.py | 1 + 3 files changed, 229 insertions(+) diff --git a/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_guest.py b/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_guest.py index ff5675bd59..4ac8133068 100644 --- a/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_guest.py +++ b/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_guest.py @@ -4,6 +4,7 @@ from federatedml.util import LOGGER from typing import List import functools +from federatedml.secureprotol import PaillierEncrypt from federatedml.protobuf.generated.boosting_tree_model_meta_pb2 import BoostingTreeModelMeta from federatedml.protobuf.generated.boosting_tree_model_meta_pb2 import ObjectiveMeta from federatedml.protobuf.generated.boosting_tree_model_meta_pb2 import QuantileMeta @@ -345,6 +346,115 @@ def boosting_fast_predict(self, data_inst, trees: List[HeteroDecisionTreeGuest], multi_class_num=self.booster_dim, predict_cache=predict_cache) return predict_result + @staticmethod + def get_leaf_idx_map(trees): + id_pos_map_list = [] + + for tree in trees: + array_idx = 0 + id_pos_map = {} + for node in tree.tree_node: + if node.is_leaf: + id_pos_map[node.id] = array_idx + array_idx += 1 + id_pos_map_list.append(id_pos_map) + + return id_pos_map_list + + @staticmethod + def go_to_children_branches(data_inst, tree_node, tree, sitename: str, candidate_list: List): + if tree_node.is_leaf: + candidate_list.append(tree_node) + else: + tree_node_list = tree.tree_node + if tree_node.sitename != sitename: + HeteroSecureBoostingTreeGuest.go_to_children_branches(data_inst, tree_node_list[tree_node.left_nodeid], + tree, sitename, candidate_list) + HeteroSecureBoostingTreeGuest.go_to_children_branches(data_inst, tree_node_list[tree_node.right_nodeid], + tree, sitename, candidate_list) + else: + next_layer_node_id = tree.go_next_layer(tree_node, data_inst, use_missing=tree.use_missing, + zero_as_missing=tree.zero_as_missing, decoder=tree.decode, + split_maskdict=tree.split_maskdict, + missing_dir_maskdict=tree.missing_dir_maskdict, + bin_sparse_point=None + ) + HeteroSecureBoostingTreeGuest.go_to_children_branches(data_inst, tree_node_list[next_layer_node_id], + tree, sitename, candidate_list) + + @staticmethod + def generate_leaf_candidates_guest(data_inst, sitename, trees, node_pos_map_list, + init_score, learning_rate, booster_dim): + candidate_nodes_of_all_tree = [] + + if booster_dim > 2: + epoch_num = len(trees) // booster_dim + else: + epoch_num = len(trees) + init_score = init_score / epoch_num + score_idx = 0 + + for tree, node_pos_map in zip(trees, node_pos_map_list): + if booster_dim > 2: + tree_init_score = init_score[score_idx] + score_idx += 1 + if score_idx == booster_dim: + score_idx = 0 + else: + tree_init_score = init_score + result_vec = [0 for i in range(len(node_pos_map))] + candidate_list = [] + HeteroSecureBoostingTreeGuest.go_to_children_branches(data_inst, tree.tree_node[0], tree, + sitename, candidate_list) + for node in candidate_list: + result_vec[node_pos_map[node.id]] = float(node.weight * learning_rate + tree_init_score) + candidate_nodes_of_all_tree.extend(result_vec) + + return np.array(candidate_nodes_of_all_tree) + + def EINI_guest_predict(self, data_inst, trees: List[HeteroDecisionTreeGuest], learning_rate, init_score, booster_dim, + sitename=None, party_list=None, predict_cache=None, pred_leaf=False): + + if sitename is None: + raise ValueError('input sitename is None, not able to run EINI predict algorithm') + + if pred_leaf: + raise ValueError('EINI predict mode does not support leaf idx prediction') + + # EINI algorithms + id_pos_map_list = self.get_leaf_idx_map(trees) + map_func = functools.partial(self.generate_leaf_candidates_guest, sitename=sitename, trees=trees, + node_pos_map_list=id_pos_map_list, init_score=init_score, + learning_rate=learning_rate, booster_dim=booster_dim) + position_vec = data_inst.mapValues(map_func) + + # encryption + encrypter = PaillierEncrypt() + encrypter.generate_key(1024) + encrypter_vec_table = position_vec.mapValues(encrypter.recursive_encrypt) + + # federation part + self.predict_transfer_inst.guest_predict_data.remote(booster_dim, idx=-1, suffix='booster_dim') + # send to first host party + self.predict_transfer_inst.guest_predict_data.remote(encrypter_vec_table, idx=0, suffix='position_vec', role=consts.HOST) + # get from last host party + result_table = self.predict_transfer_inst.host_predict_data.get(idx=len(party_list) - 1, suffix='merge_result', + role=consts.HOST) + + # decode result + result = result_table.mapValues(encrypter.recursive_decrypt) + + # result = result_table + if booster_dim == 1: + result = result.mapValues(lambda x: x[0]) + else: + result = result.mapValues(lambda x: np.array(x)) + + if predict_cache: + result = result.join(predict_cache, lambda v1, v2: v1 + v2) + + return result + @assert_io_num_rows_equal def predict(self, data_inst, ret_format='std'): diff --git a/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py b/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py index 103ec7400c..1d8de0d79f 100644 --- a/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py +++ b/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py @@ -159,6 +159,124 @@ def boosting_fast_predict(self, data_inst, trees: List[HeteroDecisionTreeHost]): comm_round += 1 + @staticmethod + def go_to_children_branches(data_inst, tree_node, tree, sitename: str, candidate_list: List): + if tree_node.is_leaf: + candidate_list.append(tree_node) + else: + tree_node_list = tree.tree_node + if tree_node.sitename != sitename: + HeteroSecureBoostingTreeHost.go_to_children_branches(data_inst, tree_node_list[tree_node.left_nodeid], + tree, sitename, candidate_list) + HeteroSecureBoostingTreeHost.go_to_children_branches(data_inst, tree_node_list[tree_node.right_nodeid], + tree, sitename, candidate_list) + else: + next_layer_node_id = tree.go_next_layer(tree_node, data_inst, use_missing=tree.use_missing, + zero_as_missing=tree.zero_as_missing, decoder=tree.decode, + split_maskdict=tree.split_maskdict, + missing_dir_maskdict=tree.missing_dir_maskdict, + bin_sparse_point=None + ) + HeteroSecureBoostingTreeHost.go_to_children_branches(data_inst, tree_node_list[next_layer_node_id], + tree, sitename, candidate_list) + + @staticmethod + def generate_leaf_candidates_host(data_inst, sitename, trees, node_pos_map_list): + candidate_nodes_of_all_tree = [] + + for tree, node_pos_map in zip(trees, node_pos_map_list): + + result_vec = [0 for i in range(len(node_pos_map))] + candidate_list = [] + HeteroSecureBoostingTreeHost.go_to_children_branches(data_inst, tree.tree_node[0], tree, sitename, + candidate_list) + for node in candidate_list: + result_vec[node_pos_map[node.id]] = 1 # create 0-1 vector + candidate_nodes_of_all_tree.extend(result_vec) + + return np.array(candidate_nodes_of_all_tree) + + @staticmethod + def generate_leaf_idx_dimension_map(trees, booster_dim): + cur_dim = 0 + leaf_dim_map = {} + leaf_idx = 0 + for tree in trees: + for node in tree.tree_node: + if node.is_leaf: + leaf_dim_map[leaf_idx] = cur_dim + leaf_idx += 1 + cur_dim += 1 + if cur_dim == booster_dim: + cur_dim = 0 + return leaf_dim_map + + @staticmethod + def merge_position_vec(host_vec, guest_encrypt_vec, booster_dim=1, leaf_idx_dim_map=None): + leaf_idx = -1 + rs = [0 for i in range(booster_dim)] + for en_num, vec_value in zip(guest_encrypt_vec, host_vec): + leaf_idx += 1 + if vec_value == 0: + continue + else: + dim = leaf_idx_dim_map[leaf_idx] + rs[dim] += en_num + return rs + + @staticmethod + def position_vec_element_wise_mul(guest_encrypt_vec, host_vec): + new_vec = [] + for en_num, vec_value in zip(guest_encrypt_vec, host_vec): + new_vec.append(en_num * vec_value) + return new_vec + + @staticmethod + def get_leaf_idx_map(trees): + id_pos_map_list = [] + + for tree in trees: + array_idx = 0 + id_pos_map = {} + for node in tree.tree_node: + if node.is_leaf: + id_pos_map[node.id] = array_idx + array_idx += 1 + id_pos_map_list.append(id_pos_map) + + return id_pos_map_list + + def EINI_host_predict(self, data_inst, trees: List[HeteroDecisionTreeHost], sitename, self_party_id, party_list): + + id_pos_map_list = self.get_leaf_idx_map(trees) + map_func = functools.partial(self.generate_leaf_candidates_host, sitename=sitename, trees=trees, + node_pos_map_list=id_pos_map_list) + position_vec = data_inst.mapValues(map_func) + + booster_dim = self.predict_transfer_inst.guest_predict_data.get(idx=0, suffix='booster_dim') + + self_idx = party_list.index(self_party_id) + if len(party_list) == 1: + guest_position_vec = self.predict_transfer_inst.guest_predict_data.get(idx=0, suffix='position_vec') + leaf_idx_dim_map = self.generate_leaf_idx_dimension_map(trees, booster_dim) + merge_func = functools.partial(self.merge_position_vec, booster_dim=booster_dim, + leaf_idx_dim_map=leaf_idx_dim_map) + result_table = position_vec.join(guest_position_vec, merge_func) + self.predict_transfer_inst.inter_host_data.remote(result_table, idx=self_idx + 1, suffix='position_vec', role=consts.HOST) + else: + # multi host case + # if is first host party, get encrypt vec from guest, else from previous host party + if self_party_id == party_list[0]: + guest_position_vec = self.predict_transfer_inst.guest_predict_data.get(idx=0, suffix='position_vec') + else: + guest_position_vec = self.predict_transfer_inst.inter_host_data.get(idx=self_idx - 1, suffix='position_vec') + result_table = position_vec.join(guest_position_vec, self.position_vec_element_wise_mul) + if self_party_id == party_list[-1]: + self.predict_transfer_inst.host_predict_data.remote(result_table, suffix='merge_result') + else: + self.predict_transfer_inst.inter_host_data.remote(result_table, idx=self_idx + 1, suffix='position_vec', + role=consts.HOST) + @assert_io_num_rows_equal def predict(self, data_inst): diff --git a/python/federatedml/transfer_variable/transfer_class/hetero_secure_boosting_predict_transfer_variable.py b/python/federatedml/transfer_variable/transfer_class/hetero_secure_boosting_predict_transfer_variable.py index 7a055c8499..4ca419f95c 100644 --- a/python/federatedml/transfer_variable/transfer_class/hetero_secure_boosting_predict_transfer_variable.py +++ b/python/federatedml/transfer_variable/transfer_class/hetero_secure_boosting_predict_transfer_variable.py @@ -33,3 +33,4 @@ def __init__(self, flowid=0): self.predict_stop_flag = self._create_variable(name='predict_stop_flag', src=['guest'], dst=['host']) self.guest_predict_data = self._create_variable(name='guest_predict_data', src=['guest'], dst=['host']) self.host_predict_data = self._create_variable(name='host_predict_data', src=['host'], dst=['guest']) + self.inter_host_data = self._create_variable(name='inter_host_data', src=['host'], dst=['host']) From a0e226e2e51d17b12fb4c59f94f22f7b2a844c1b Mon Sep 17 00:00:00 2001 From: mgqa34 Date: Fri, 11 Feb 2022 15:37:06 +0800 Subject: [PATCH 15/99] update batch masked & shuffle strategy Signed-off-by: mgqa34 --- .../hetero/procedure/batch_generator.py | 50 +++-- .../linear_model/linear_model_base.py | 6 + .../hetero_lr_guest.py | 19 +- .../hetero_lr_host.py | 6 +- .../federatedml/model_selection/mini_batch.py | 176 +++++++++------ .../gradient/hetero_linear_model_gradient.py | 203 ++++++++++-------- .../gradient/hetero_lr_gradient_and_loss.py | 23 +- .../param/logistic_regression_param.py | 11 +- 8 files changed, 310 insertions(+), 184 deletions(-) diff --git a/python/federatedml/framework/hetero/procedure/batch_generator.py b/python/federatedml/framework/hetero/procedure/batch_generator.py index 05b2bc8d43..8fcf88909f 100644 --- a/python/federatedml/framework/hetero/procedure/batch_generator.py +++ b/python/federatedml/framework/hetero/procedure/batch_generator.py @@ -26,23 +26,24 @@ def __init__(self): self.mini_batch_obj = None self.finish_sycn = False self.batch_nums = None + self.batch_masked = False def register_batch_generator(self, transfer_variables, has_arbiter=True): self._register_batch_data_index_transfer(transfer_variables.batch_info, transfer_variables.batch_data_index, has_arbiter) - def initialize_batch_generator(self, data_instances, batch_size, suffix=tuple(), shuffle=False): - self.mini_batch_obj = MiniBatch(data_instances, batch_size=batch_size, shuffle=shuffle) + def initialize_batch_generator(self, data_instances, batch_size, suffix=tuple(), + shuffle=False, batch_strategy="full", masked_rate=0): + self.mini_batch_obj = MiniBatch(data_instances, batch_size=batch_size, shuffle=shuffle, + batch_strategy=batch_strategy, masked_rate=masked_rate) self.batch_nums = self.mini_batch_obj.batch_nums - batch_info = {"batch_size": batch_size, "batch_num": self.batch_nums} + self.batch_masked = self.mini_batch_obj.batch_masked + batch_info = {"batch_size": batch_size, "batch_num": self.batch_nums, "batch_mutable": True, "batch_masked": self.batch_masked} self.sync_batch_info(batch_info, suffix) - """if need shuffle, batch_data will be generated during every iteration""" - if shuffle is True: - return - - self.prepare_batch_data(suffix) + if not self.mini_batch_obj.batch_mutable: + self.prepare_batch_data(suffix) def prepare_batch_data(self, suffix=tuple()): self.mini_batch_obj.generate_batch_data() @@ -53,13 +54,18 @@ def prepare_batch_data(self, suffix=tuple()): self.sync_batch_index(batch_data_index, batch_suffix) batch_index += 1 - def generate_batch_data(self, suffix=tuple()): - if self.mini_batch_obj.shuffle: - self.prepare_batch_data(suffix=suffix) + def generate_batch_data(self, with_index=False, suffix=tuple()): + if self.mini_batch_obj.batch_mutable: + self.prepare_batch_data(suffix) - data_generator = self.mini_batch_obj.mini_batch_data_generator(result='data') - for batch_data in data_generator: - yield batch_data + if with_index: + data_generator = self.mini_batch_obj.mini_batch_data_generator(result='both') + for batch_data, index_data in data_generator: + yield batch_data, index_data + else: + data_generator = self.mini_batch_obj.mini_batch_data_generator(result='data') + for batch_data in data_generator: + yield batch_data class Host(batch_info_sync.Host): @@ -68,7 +74,8 @@ def __init__(self): self.batch_data_insts = [] self.batch_nums = None self.data_inst = None - self.shuffle = False + self.batch_mutable = False + self.batch_masked = False def register_batch_generator(self, transfer_variables, has_arbiter=None): self._register_batch_data_index_transfer(transfer_variables.batch_info, transfer_variables.batch_data_index) @@ -76,14 +83,13 @@ def register_batch_generator(self, transfer_variables, has_arbiter=None): def initialize_batch_generator(self, data_instances, suffix=tuple(), **kwargs): batch_info = self.sync_batch_info(suffix) self.batch_nums = batch_info.get('batch_num') + self.batch_mutable = batch_info.get("batch_mutable") + self.batch_masked = batch_info.get("batch_masked") - """if need shuffle, batch_data will be generated during every iteration""" - self.shuffle = kwargs.get("shuffle", False) - if self.shuffle is True: + if not self.batch_mutable: + self.prepare_batch_data(data_instances, suffix) + else: self.data_inst = data_instances - return - - self.prepare_batch_data(data_instances, suffix) def prepare_batch_data(self, data_inst, suffix=tuple()): self.batch_data_insts = [] @@ -95,7 +101,7 @@ def prepare_batch_data(self, data_inst, suffix=tuple()): self.batch_data_insts.append(batch_data_inst) def generate_batch_data(self, suffix=tuple()): - if self.shuffle: + if self.batch_mutable: self.prepare_batch_data(data_inst=self.data_inst, suffix=suffix) batch_index = 0 diff --git a/python/federatedml/linear_model/linear_model_base.py b/python/federatedml/linear_model/linear_model_base.py index fea2a856ad..74b59a3e64 100644 --- a/python/federatedml/linear_model/linear_model_base.py +++ b/python/federatedml/linear_model/linear_model_base.py @@ -66,8 +66,14 @@ def _init_model(self, params): self.init_param_obj = params.init_param # self.fit_intercept = self.init_param_obj.fit_intercept self.batch_size = params.batch_size + if hasattr(params, "shuffle"): self.shuffle = params.shuffle + if hasattr(params, "masked_rate"): + self.masked_rate = params.masked_rate + if hasattr(params, "batch_strategy"): + self.batch_strategy = params.batch_strategy + self.max_iter = params.max_iter self.optimizer = optimizer_factory(params) self.converge_func = converge_func_factory(params.early_stop, params.tol) diff --git a/python/federatedml/linear_model/logistic_regression/hetero_logistic_regression/hetero_lr_guest.py b/python/federatedml/linear_model/logistic_regression/hetero_logistic_regression/hetero_lr_guest.py index 7cac4a96c4..62682d0b6f 100644 --- a/python/federatedml/linear_model/logistic_regression/hetero_logistic_regression/hetero_lr_guest.py +++ b/python/federatedml/linear_model/logistic_regression/hetero_logistic_regression/hetero_lr_guest.py @@ -104,8 +104,12 @@ def fit_binary(self, data_instances, validate_data=None): self.transfer_variable.use_async.remote(use_async) LOGGER.info("Generate mini-batch from input data") - self.batch_generator.initialize_batch_generator(data_instances, self.batch_size, shuffle=self.shuffle) + self.batch_generator.initialize_batch_generator(data_instances, self.batch_size, + batch_strategy=self.batch_strategy, + masked_rate=self.masked_rate, shuffle=self.shuffle) self.gradient_loss_operator.set_total_batch_nums(self.batch_generator.batch_nums) + if self.batch_generator.batch_masked: + self.gradient_loss_operator.set_use_sync() self.encrypted_calculator = [EncryptModeCalculator(self.cipher_operator, self.encrypted_mode_calculator_param.mode, @@ -124,27 +128,32 @@ def fit_binary(self, data_instances, validate_data=None): while self.n_iter_ < self.max_iter: self.callback_list.on_epoch_begin(self.n_iter_) LOGGER.info("iter: {}".format(self.n_iter_)) - batch_data_generator = self.batch_generator.generate_batch_data(suffix=(self.n_iter_, )) + batch_data_generator = self.batch_generator.generate_batch_data(suffix=(self.n_iter_, ), with_index=True) self.optimizer.set_iters(self.n_iter_) batch_index = 0 - for batch_data in batch_data_generator: + for batch_data, index_data in batch_data_generator: batch_feat_inst = batch_data + if self.batch_generator.batch_masked: + index_data = None # Start gradient procedure LOGGER.debug("iter: {}, batch: {}, before compute gradient, data count: {}".format(self.n_iter_, batch_index, batch_feat_inst.count())) + optim_guest_gradient = self.gradient_loss_operator.compute_gradient_procedure( batch_feat_inst, self.encrypted_calculator, self.model_weights, self.optimizer, self.n_iter_, - batch_index) + batch_index, + masked_index=index_data + ) loss_norm = self.optimizer.loss_norm(self.model_weights) self.gradient_loss_operator.compute_loss(batch_feat_inst, self.model_weights, self.n_iter_, batch_index, - loss_norm) + loss_norm, batch_masked=self.batch_generator.batch_masked) self.model_weights = self.optimizer.update_model(self.model_weights, optim_guest_gradient) batch_index += 1 diff --git a/python/federatedml/linear_model/logistic_regression/hetero_logistic_regression/hetero_lr_host.py b/python/federatedml/linear_model/logistic_regression/hetero_logistic_regression/hetero_lr_host.py index 78414b9094..5f0b1e0a31 100644 --- a/python/federatedml/linear_model/logistic_regression/hetero_logistic_regression/hetero_lr_host.py +++ b/python/federatedml/linear_model/logistic_regression/hetero_logistic_regression/hetero_lr_host.py @@ -106,6 +106,9 @@ def fit_binary(self, data_instances, validate_data): self.gradient_loss_operator.set_use_async() self.batch_generator.initialize_batch_generator(data_instances, shuffle=self.shuffle) + if self.batch_generator.batch_masked: + self.gradient_loss_operator.set_use_sync() + self.gradient_loss_operator.set_total_batch_nums(self.batch_generator.batch_nums) self.encrypted_calculator = [EncryptModeCalculator(self.cipher_operator, @@ -145,7 +148,8 @@ def fit_binary(self, data_instances, validate_data): # LOGGER.debug('optim_host_gradient: {}'.format(optim_host_gradient)) self.gradient_loss_operator.compute_loss(self.model_weights, self.optimizer, - self.n_iter_, batch_index, self.cipher_operator) + self.n_iter_, batch_index, self.cipher_operator, + batch_masked=self.batch_generator.batch_masked) self.model_weights = self.optimizer.update_model(self.model_weights, optim_host_gradient) batch_index += 1 diff --git a/python/federatedml/model_selection/mini_batch.py b/python/federatedml/model_selection/mini_batch.py index c5cfaaf92c..70932f671c 100644 --- a/python/federatedml/model_selection/mini_batch.py +++ b/python/federatedml/model_selection/mini_batch.py @@ -22,22 +22,23 @@ class MiniBatch: - def __init__(self, data_inst, batch_size=320, shuffle=False): + def __init__(self, data_inst, batch_size=320, shuffle=False, batch_strategy="full", masked_rate=0): self.batch_data_sids = None self.batch_nums = 0 self.data_inst = data_inst self.all_batch_data = None self.all_index_data = None self.data_sids_iter = None - self.shuffle = shuffle + self.batch_data_generator = None + self.batch_mutable = False + self.batch_masked = False if batch_size == -1: self.batch_size = data_inst.count() else: self.batch_size = batch_size - self.__mini_batch_data_seperator(data_inst, self.batch_size) - # LOGGER.debug("In mini batch init, batch_num:{}".format(self.batch_nums)) + self.__init_mini_batch_data_seperator(data_inst, self.batch_size, batch_strategy, masked_rate, shuffle) def mini_batch_data_generator(self, result='data'): """ @@ -56,72 +57,125 @@ def mini_batch_data_generator(self, result='data'): if result == 'index': for index_table in self.all_index_data: yield index_table - else: + elif result == "data": for batch_data in self.all_batch_data: yield batch_data + else: + for batch_data, index_table in zip(self.all_batch_data, self.all_index_data): + yield batch_data, index_table - def __mini_batch_data_seperator(self, data_insts, batch_size): - self.data_sids_iter, data_size = indices.collect_index(data_insts) + if self.batch_mutable: + self.__generate_batch_data() - if batch_size > data_size: - batch_size = data_size - self.batch_size = batch_size + def __init_mini_batch_data_seperator(self, data_insts, batch_size, batch_strategy, masked_rate, shuffle): + self.data_sids_iter, data_size = indices.collect_index(data_insts) - self.batch_nums = (data_size + batch_size - 1) // batch_size + self.batch_data_generator = get_batch_generator(data_size, batch_size, batch_strategy, masked_rate, shuffle=shuffle) + self.batch_nums = self.batch_data_generator.batch_nums + self.batch_mutable = self.batch_data_generator.batch_mutable() + self.batch_masked = self.batch_data_generator.batch_masked() - if self.shuffle is False: - self.__generate_batch_data(data_insts) + if self.batch_mutable is False: + self.__generate_batch_data() def generate_batch_data(self): - if self.shuffle: - self.__generate_batch_data(data_insts=self.data_inst) + if self.batch_mutable: + self.__generate_batch_data() - def __generate_batch_data(self, data_insts): - if self.shuffle: - random.SystemRandom().shuffle(self.data_sids_iter) + def __generate_batch_data(self): + self.all_index_data, self.all_batch_data = self.batch_data_generator.generate_data(self.data_inst, self.data_sids_iter) - self.all_batch_data = [] - self.all_index_data = [] - for bid in range(self.batch_nums): - batch_ids = self.data_sids_iter[bid * self.batch_size:(bid + 1) * self.batch_size] - index_table = session.parallelize(batch_ids, - include_key=True, - partition=data_insts.partitions) - batch_data = index_table.join(data_insts, lambda x, y: y) - self.all_index_data.append(index_table) - self.all_batch_data.append(batch_data) +def get_batch_generator(data_size, batch_size, batch_strategy, masked_rate, shuffle): + if batch_size >= data_size: + LOGGER.warning("As batch_size >= data size, all batch strategy will be disabled") + return FullBatchDataGenerator(data_size, data_size, shuffle=False) - """ - batch_data_sids = [] - curt_data_num = 0 - curt_batch = 0 - curt_batch_ids = [] - data_size = len(self.data_sids_iter) - for sid, values in self.data_sids_iter: - # print('sid is {}, values is {}'.format(sid, values)) - curt_batch_ids.append((sid, None)) - curt_data_num += 1 - if curt_data_num % self.batch_size == 0: - curt_batch += 1 - if curt_batch < self.batch_nums: - batch_data_sids.append(curt_batch_ids) - curt_batch_ids = [] - if curt_data_num == data_size and len(curt_batch_ids) != 0: - batch_data_sids.append(curt_batch_ids) - - self.batch_nums = len(batch_data_sids) - - all_batch_data = [] - all_index_data = [] - for index_data in batch_data_sids: - # LOGGER.debug('in generator, index_data is {}'.format(index_data)) - index_table = session.parallelize(index_data, include_key=True, partition=data_insts.partitions) - batch_data = index_table.join(data_insts, lambda x, y: y) - - # yield batch_data - all_batch_data.append(batch_data) - all_index_data.append(index_table) - self.all_batch_data = all_batch_data - self.all_index_data = all_index_data - """ + if round((masked_rate + 1) * batch_size) >= data_size: + LOGGER.warning("Masked dataset's batch_size >= data size, all batch strategy will be disabled") + return FullBatchDataGenerator(data_size, data_size, shuffle=False, masked_rate=0) + + if batch_strategy == "full": + return FullBatchDataGenerator(data_size, batch_size, shuffle=shuffle, masked_rate=masked_rate) + else: + if shuffle: + LOGGER.warning("if use random select batch strategy, shuffle will not work") + return RandomBatchDataGenerator(batch_size, masked_rate) + + +class FullBatchDataGenerator(object): + def __init__(self, data_size, batch_size, shuffle=False, masked_rate=0): + self.batch_nums = (data_size + batch_size - 1) // batch_size + self.masked_dataset_size = round((1 + masked_rate) * self.batch_nums) + self.batch_size = batch_size + self.shuffle = shuffle + + def generate_data(self, data_insts, data_sids): + if self.shuffle: + random.SystemRandom().shuffle(data_sids) + + index_table = [] + batch_data = [] + if self.batch_size != self.masked_dataset_size: + for bid in range(self.batch_nums): + batch_ids = data_sids[bid * self.batch_size:(bid + 1) * self.batch_size] + masked_ids = set() + for sid, _ in batch_ids: + masked_ids.add(sid) + possible_ids = random.SystemRandom().sample(data_sids, self.masked_dataset_size) + for pid, _ in possible_ids: + if pid not in masked_ids: + masked_ids.add(pid) + if len(masked_ids) == self.masked_dataset_size: + break + + masked_index_table = session.parallelize(zip(list(masked_ids), [None] * len(masked_ids)), + include_key=True, + partition=data_insts.partitions) + batch_index_table = session.parallelize(batch_ids, + include_key=True, + partition=data_insts.partitions) + batch_data_table = batch_index_table.join(data_insts, lambda x, y: y) + index_table.append(masked_index_table) + batch_data.append(batch_data_table) + else: + for bid in range(self.batch_nums): + batch_ids = data_sids[bid * self.batch_size : (bid + 1) * self.batch_size] + batch_index_table = session.parallelize(batch_ids, + include_key=True, + partition=data_insts.partitions) + batch_data_table = batch_index_table.join(data_insts, lambda x, y: y) + index_table.append(batch_index_table) + batch_data.append(batch_data_table) + + return index_table, batch_data + + def batch_mutable(self): + return self.masked_dataset_size > self.batch_size or self.shuffle + + def batch_masked(self): + return self.masked_dataset_size != self.batch_size + + +class RandomBatchDataGenerator(object): + def __init__(self, batch_size, masked_rate=0): + self.batch_nums = 1 + self.batch_size = batch_size + self.masked_dataset_size = round((1 + masked_rate) * self.batch_size) + + def generate_data(self, data_insts, *args, **kwargs): + if self.masked_dataset_size == self.batch_size: + batch_data = data_insts.sample(num=self.batch_size) + index_data = batch_data.mapValues(lambda value: None) + return index_data, batch_data + else: + masked_data = data_insts.sample(num=self.masked_dataset_size) + batch_data = masked_data.sample(num=self.batch_size) + masked_index_table = masked_data.mapValues(lambda value: None) + return masked_index_table, batch_data + + def batch_mutable(self): + return True + + def batch_masked(self): + return self.masked_dataset_size != self.batch_size diff --git a/python/federatedml/optim/gradient/hetero_linear_model_gradient.py b/python/federatedml/optim/gradient/hetero_linear_model_gradient.py index 2e75b714c0..9422584041 100644 --- a/python/federatedml/optim/gradient/hetero_linear_model_gradient.py +++ b/python/federatedml/optim/gradient/hetero_linear_model_gradient.py @@ -47,6 +47,9 @@ def set_total_batch_nums(self, total_batch_nums): def set_use_async(self): self.use_async = True + def set_use_sync(self): + self.use_async = False + def set_use_sample_weight(self): self.use_sample_weight = True @@ -54,69 +57,69 @@ def set_fixed_float_precision(self, floating_point_precision): if floating_point_precision is not None: self.fixed_point_encoder = FixedPointEncoder(2**floating_point_precision) - @staticmethod - def __compute_partition_gradient(data, fit_intercept=True, is_sparse=False): - """ - Compute hetero regression gradient for: - gradient = ∑d*x, where d is fore_gradient which differ from different algorithm - Parameters - ---------- - data: Table, include fore_gradient and features - fit_intercept: bool, if model has interception or not. Default True - - Returns - ---------- - numpy.ndarray - hetero regression model gradient - """ - feature = [] - fore_gradient = [] - - if is_sparse: - row_indice = [] - col_indice = [] - data_value = [] - - row = 0 - feature_shape = None - for key, (sparse_features, d) in data: - fore_gradient.append(d) - assert isinstance(sparse_features, SparseVector) - if feature_shape is None: - feature_shape = sparse_features.get_shape() - for idx, v in sparse_features.get_all_data(): - col_indice.append(idx) - row_indice.append(row) - data_value.append(v) - row += 1 - if feature_shape is None or feature_shape == 0: - return 0 - sparse_matrix = sp.csr_matrix((data_value, (row_indice, col_indice)), shape=(row, feature_shape)) - fore_gradient = np.array(fore_gradient) - - # gradient = sparse_matrix.transpose().dot(fore_gradient).tolist() - gradient = fate_operator.dot(sparse_matrix.transpose(), fore_gradient).tolist() - if fit_intercept: - bias_grad = np.sum(fore_gradient) - gradient.append(bias_grad) - # LOGGER.debug("In first method, gradient: {}, bias_grad: {}".format(gradient, bias_grad)) - return np.array(gradient) - - else: - for key, value in data: - feature.append(value[0]) - fore_gradient.append(value[1]) - feature = np.array(feature) - fore_gradient = np.array(fore_gradient) - if feature.shape[0] <= 0: - return 0 - - gradient = fate_operator.dot(feature.transpose(), fore_gradient) - gradient = gradient.tolist() - if fit_intercept: - bias_grad = np.sum(fore_gradient) - gradient.append(bias_grad) - return np.array(gradient) + # @staticmethod + # def __compute_partition_gradient(data, fit_intercept=True, is_sparse=False): + # """ + # Compute hetero regression gradient for: + # gradient = ∑d*x, where d is fore_gradient which differ from different algorithm + # Parameters + # ---------- + # data: Table, include fore_gradient and features + # fit_intercept: bool, if model has interception or not. Default True + + # Returns + # ---------- + # numpy.ndarray + # hetero regression model gradient + # """ + # feature = [] + # fore_gradient = [] + + # if is_sparse: + # row_indice = [] + # col_indice = [] + # data_value = [] + + # row = 0 + # feature_shape = None + # for key, (sparse_features, d) in data: + # fore_gradient.append(d) + # assert isinstance(sparse_features, SparseVector) + # if feature_shape is None: + # feature_shape = sparse_features.get_shape() + # for idx, v in sparse_features.get_all_data(): + # col_indice.append(idx) + # row_indice.append(row) + # data_value.append(v) + # row += 1 + # if feature_shape is None or feature_shape == 0: + # return 0 + # sparse_matrix = sp.csr_matrix((data_value, (row_indice, col_indice)), shape=(row, feature_shape)) + # fore_gradient = np.array(fore_gradient) + + # # gradient = sparse_matrix.transpose().dot(fore_gradient).tolist() + # gradient = fate_operator.dot(sparse_matrix.transpose(), fore_gradient).tolist() + # if fit_intercept: + # bias_grad = np.sum(fore_gradient) + # gradient.append(bias_grad) + # # LOGGER.debug("In first method, gradient: {}, bias_grad: {}".format(gradient, bias_grad)) + # return np.array(gradient) + + # else: + # for key, value in data: + # feature.append(value[0]) + # fore_gradient.append(value[1]) + # feature = np.array(feature) + # fore_gradient = np.array(fore_gradient) + # if feature.shape[0] <= 0: + # return 0 + + # gradient = fate_operator.dot(feature.transpose(), fore_gradient) + # gradient = gradient.tolist() + # if fit_intercept: + # bias_grad = np.sum(fore_gradient) + # gradient.append(bias_grad) + # return np.array(gradient) @staticmethod def __apply_cal_gradient(data, fixed_point_encoder, is_sparse): @@ -142,7 +145,7 @@ def __apply_cal_gradient(data, fixed_point_encoder, is_sparse): all_g = fixed_point_encoder.decode(all_g) return all_g - def compute_gradient(self, data_instances, fore_gradient, fit_intercept): + def compute_gradient(self, data_instances, fore_gradient, fit_intercept, need_average=True): """ Compute hetero-regression gradient Parameters @@ -157,25 +160,29 @@ def compute_gradient(self, data_instances, fore_gradient, fit_intercept): the hetero regression model's gradient """ - feature_num = data_overview.get_features_shape(data_instances) - data_count = data_instances.count() + # feature_num = data_overview.get_features_shape(data_instances) + # data_count = data_instances.count() is_sparse = data_overview.is_sparse_data(data_instances) - if data_count * feature_num > 100: - LOGGER.debug("Use apply partitions") - feat_join_grad = data_instances.join(fore_gradient, - lambda d, g: (d.features, g)) - f = functools.partial(self.__apply_cal_gradient, - fixed_point_encoder=self.fixed_point_encoder, - is_sparse=is_sparse) - gradient_sum = feat_join_grad.applyPartitions(f) - gradient_sum = gradient_sum.reduce(lambda x, y: x + y) - if fit_intercept: - # bias_grad = np.sum(fore_gradient) - bias_grad = fore_gradient.reduce(lambda x, y: x + y) - gradient_sum = np.append(gradient_sum, bias_grad) - gradient = gradient_sum / data_count + LOGGER.debug("Use apply partitions") + feat_join_grad = data_instances.join(fore_gradient, + lambda d, g: (d.features, g)) + f = functools.partial(self.__apply_cal_gradient, + fixed_point_encoder=self.fixed_point_encoder, + is_sparse=is_sparse) + gradient_sum = feat_join_grad.applyPartitions(f) + gradient_sum = gradient_sum.reduce(lambda x, y: x + y) + if fit_intercept: + # bias_grad = np.sum(fore_gradient) + bias_grad = fore_gradient.reduce(lambda x, y: x + y) + gradient_sum = np.append(gradient_sum, bias_grad) + + if need_average: + gradient = gradient_sum / data_instances.count() + else: + gradient = gradient_sum + """ else: LOGGER.debug(f"Original_method") feat_join_grad = data_instances.join(fore_gradient, @@ -187,7 +194,7 @@ def compute_gradient(self, data_instances, fore_gradient, fit_intercept): gradient_partition = gradient_partition.reduce(lambda x, y: x + y) gradient = gradient_partition / data_count - + """ return gradient @@ -229,19 +236,40 @@ def _asynchronous_compute_gradient(self, data_instances, model_weights, cipher, unilateral_gradient = np.append(unilateral_gradient, intercept) return unilateral_gradient - def _centralized_compute_gradient(self, data_instances, model_weights, cipher, current_suffix): + def _centralized_compute_gradient(self, data_instances, model_weights, cipher, current_suffix, masked_index=None): self.host_forwards = self.get_host_forward(suffix=current_suffix) fore_gradient = self.half_d + + batch_size = data_instances.count() + partial_masked_index_enc = None + if masked_index: + masked_index = masked_index.mapValues(lambda value: 0) + masked_index_to_encrypt = masked_index.subtractByKey(self.half_d) + partial_masked_index_enc = cipher.encrypt(masked_index_to_encrypt) + for host_forward in self.host_forwards: if self.use_sample_weight: host_forward = host_forward.join(data_instances, lambda h, v: h * v.weight) fore_gradient = fore_gradient.join(host_forward, lambda x, y: x + y) - self.remote_fore_gradient(fore_gradient, suffix=current_suffix) - unilateral_gradient = self.compute_gradient(data_instances, fore_gradient, model_weights.fit_intercept) + + def _apply_obfuscate(val): + val.apply_obfuscator() + return val + fore_gradient = fore_gradient.mapValues(lambda val: _apply_obfuscate(val) / batch_size) + + if partial_masked_index_enc: + masked_fore_gradient = partial_masked_index_enc.union(fore_gradient) + self.remote_fore_gradient(masked_fore_gradient, suffix=current_suffix) + else: + self.remote_fore_gradient(fore_gradient, suffix=current_suffix) + + # self.remote_fore_gradient(fore_gradient, suffix=current_suffix) + unilateral_gradient = self.compute_gradient(data_instances, fore_gradient, + model_weights.fit_intercept, need_average=False) return unilateral_gradient def compute_gradient_procedure(self, data_instances, encrypted_calculator, model_weights, optimizer, - n_iter_, batch_index, offset=None): + n_iter_, batch_index, offset=None, masked_index=None): """ Linear model gradient procedure Step 1: get host forwards which differ from different algorithm @@ -267,7 +295,8 @@ def compute_gradient_procedure(self, data_instances, encrypted_calculator, model else: unilateral_gradient = self._centralized_compute_gradient(data_instances, model_weights, cipher=encrypted_calculator[batch_index], - current_suffix=current_suffix) + current_suffix=current_suffix, + masked_index=masked_index) if optimizer is not None: unilateral_gradient = optimizer.add_regular_to_grad(unilateral_gradient, model_weights) @@ -325,7 +354,7 @@ def _centralized_compute_gradient(self, data_instances, cipher, current_suffix): fore_gradient = self.fore_gradient_transfer.get(idx=0, suffix=current_suffix) # Host case, never fit-intercept - unilateral_gradient = self.compute_gradient(data_instances, fore_gradient, False) + unilateral_gradient = self.compute_gradient(data_instances, fore_gradient, False, need_average=False) return unilateral_gradient def compute_gradient_procedure(self, data_instances, encrypted_calculator, model_weights, diff --git a/python/federatedml/optim/gradient/hetero_lr_gradient_and_loss.py b/python/federatedml/optim/gradient/hetero_lr_gradient_and_loss.py index 4ca74d23c9..848de62585 100644 --- a/python/federatedml/optim/gradient/hetero_lr_gradient_and_loss.py +++ b/python/federatedml/optim/gradient/hetero_lr_gradient_and_loss.py @@ -61,7 +61,7 @@ def compute_and_aggregate_forwards(self, data_instances, half_g, encrypted_half_ # fore_gradient = self.aggregated_forwards.join(data_instances, lambda wx, d: 0.25 * wx - 0.5 * d.label) return self.host_forwards - def compute_loss(self, data_instances, w, n_iter_, batch_index, loss_norm=None): + def compute_loss(self, data_instances, w, n_iter_, batch_index, loss_norm=None, batch_masked=False): """ Compute hetero-lr loss for: loss = (1/N)*∑(log2 - 1/2*ywx + 1/8*(wx)^2), where y is label, w is model weight and x is features @@ -102,7 +102,15 @@ def _sum_ywx(wx_y): # lambda v: np.square(vec_dot(v.features, w.coef_) + w.intercept_)).reduce(reduce_add) loss_list = [] + wx_squares = self.get_host_loss_intermediate(suffix=current_suffix) + if batch_masked: + wx_squares_sum = [] + for square_table in wx_squares: + square_sum = data_instances.join(square_table, lambda inst, enc_h_squares: enc_h_squares).reduce(lambda x, y: x + y) + wx_squares_sum.append(square_sum) + + wx_squares = wx_squares_sum if loss_norm is not None: host_loss_regular = self.get_host_loss_regular(suffix=current_suffix) @@ -171,7 +179,7 @@ def compute_half_g(self, data_instances, w, cipher, batch_index): encrypt_half_g = cipher[batch_index].encrypt(half_g) return half_g, encrypt_half_g - def compute_loss(self, lr_weights, optimizer, n_iter_, batch_index, cipher_operator): + def compute_loss(self, lr_weights, optimizer, n_iter_, batch_index, cipher_operator, batch_masked=False): """ Compute hetero-lr loss for: loss = (1/N)*∑(log2 - 1/2*ywx + 1/8*(wx)^2), where y is label, w is model weight and x is features @@ -182,8 +190,15 @@ def compute_loss(self, lr_weights, optimizer, n_iter_, batch_index, cipher_opera where Wh*Xh is a table obtain from host and ∑(Wh*Xh)^2 is a sum number get from host. """ current_suffix = (n_iter_, batch_index) - self_wx_square = self.forwards.mapValues(lambda x: np.square(4 * x)).reduce(reduce_add) - en_wx_square = cipher_operator.encrypt(self_wx_square) + + # self_wx_square = self.forwards.mapValues(lambda x: np.square(4 * x)).reduce(reduce_add) + self_wx_square = self.forwards.mapValues(lambda x: np.square(4 * x)) + if not batch_masked: + self_wx_square = self_wx_square.reduce(reduce_add) + en_wx_square = cipher_operator.encrypt(self_wx_square) + else: + en_wx_square = self_wx_square.mapValues(lambda x: cipher_operator.encrypt(x)) + self.remote_loss_intermediate(en_wx_square, suffix=current_suffix) loss_regular = optimizer.loss_norm(lr_weights) diff --git a/python/federatedml/param/logistic_regression_param.py b/python/federatedml/param/logistic_regression_param.py index 448b870cc7..8408097d88 100644 --- a/python/federatedml/param/logistic_regression_param.py +++ b/python/federatedml/param/logistic_regression_param.py @@ -116,7 +116,7 @@ class LogisticParam(BaseParam): def __init__(self, penalty='L2', tol=1e-4, alpha=1.0, optimizer='rmsprop', - batch_size=-1, shuffle=True, + batch_size=-1, shuffle=True, batch_strategy="full", masked_rate=5, learning_rate=0.01, init_param=InitParam(), max_iter=100, early_stop='diff', encrypt_param=EncryptParam(), predict_param=PredictParam(), cv_param=CrossValidationParam(), @@ -133,6 +133,9 @@ def __init__(self, penalty='L2', self.alpha = alpha self.optimizer = optimizer self.batch_size = batch_size + self.shuffle = shuffle + self.batch_strategy = batch_strategy + self.masked_rate = masked_rate self.learning_rate = learning_rate self.init_param = copy.deepcopy(init_param) self.max_iter = max_iter @@ -150,7 +153,6 @@ def __init__(self, penalty='L2', self.use_first_metric_only = use_first_metric_only self.floating_point_precision = floating_point_precision self.callback_param = copy.deepcopy(callback_param) - self.shuffle = shuffle def check(self): descr = "logistic_param's" @@ -358,7 +360,7 @@ def check(self): class HeteroLogisticParam(LogisticParam): def __init__(self, penalty='L2', tol=1e-4, alpha=1.0, optimizer='rmsprop', - batch_size=-1, shuffle=True, + batch_size=-1, shuffle=True, batch_strategy="full", masked_rate=5, learning_rate=0.01, init_param=InitParam(), max_iter=100, early_stop='diff', encrypted_mode_calculator_param=EncryptedModeCalculatorParam(), @@ -371,7 +373,8 @@ def __init__(self, penalty='L2', callback_param=CallbackParam() ): super(HeteroLogisticParam, self).__init__(penalty=penalty, tol=tol, alpha=alpha, optimizer=optimizer, - batch_size=batch_size, shuffle=shuffle, + batch_size=batch_size, shuffle=shuffle, batch_strategy=batch_strategy, + masked_rate=masked_rate, learning_rate=learning_rate, init_param=init_param, max_iter=max_iter, early_stop=early_stop, predict_param=predict_param, cv_param=cv_param, From d67b239d5fb6e1b976818bfc1af6c2fc5d023fb9 Mon Sep 17 00:00:00 2001 From: mgqa34 Date: Fri, 11 Feb 2022 16:03:09 +0800 Subject: [PATCH 16/99] fix bug of mini-batch random strategy Signed-off-by: mgqa34 --- .../hetero_logistic_regression/hetero_lr_guest.py | 2 +- python/federatedml/model_selection/mini_batch.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/federatedml/linear_model/logistic_regression/hetero_logistic_regression/hetero_lr_guest.py b/python/federatedml/linear_model/logistic_regression/hetero_logistic_regression/hetero_lr_guest.py index 62682d0b6f..2602e12a0f 100644 --- a/python/federatedml/linear_model/logistic_regression/hetero_logistic_regression/hetero_lr_guest.py +++ b/python/federatedml/linear_model/logistic_regression/hetero_logistic_regression/hetero_lr_guest.py @@ -133,7 +133,7 @@ def fit_binary(self, data_instances, validate_data=None): batch_index = 0 for batch_data, index_data in batch_data_generator: batch_feat_inst = batch_data - if self.batch_generator.batch_masked: + if not self.batch_generator.batch_masked: index_data = None # Start gradient procedure diff --git a/python/federatedml/model_selection/mini_batch.py b/python/federatedml/model_selection/mini_batch.py index 70932f671c..684e3aa800 100644 --- a/python/federatedml/model_selection/mini_batch.py +++ b/python/federatedml/model_selection/mini_batch.py @@ -167,12 +167,12 @@ def generate_data(self, data_insts, *args, **kwargs): if self.masked_dataset_size == self.batch_size: batch_data = data_insts.sample(num=self.batch_size) index_data = batch_data.mapValues(lambda value: None) - return index_data, batch_data + return [index_data], [batch_data] else: masked_data = data_insts.sample(num=self.masked_dataset_size) batch_data = masked_data.sample(num=self.batch_size) masked_index_table = masked_data.mapValues(lambda value: None) - return masked_index_table, batch_data + return [masked_index_table], [batch_data] def batch_mutable(self): return True From 6444c6ae17f431204a692e4c7dbcb70caf5dcd07 Mon Sep 17 00:00:00 2001 From: weijingchen Date: Fri, 11 Feb 2022 16:10:19 +0800 Subject: [PATCH 17/99] adjust feature importance strategy Signed-off-by: cwj --- eggroll | 2 +- .../decision_tree/hetero/hetero_decision_tree_guest.py | 3 ++- .../decision_tree/hetero/hetero_decision_tree_host.py | 9 +++------ 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/eggroll b/eggroll index d620c322f0..a7a68dba78 160000 --- a/eggroll +++ b/eggroll @@ -1 +1 @@ -Subproject commit d620c322f0e7ad94ae903d87c023fcf1d452a65d +Subproject commit a7a68dba78b7739c771c4a6c59c343c13752e763 diff --git a/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_decision_tree_guest.py b/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_decision_tree_guest.py index d26d0eb04d..1ebdc18cc8 100644 --- a/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_decision_tree_guest.py +++ b/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_decision_tree_guest.py @@ -598,7 +598,8 @@ def update_tree(self, split_info, reach_max_depth): self.cur_layer_nodes[i].fid = split_info[i].best_fid self.cur_layer_nodes[i].bid = split_info[i].best_bid - self.update_feature_importance(split_info[i]) + if self.sitename == split_info[i].sitename: + self.update_feature_importance(split_info[i]) self.tree_node.append(self.cur_layer_nodes[i]) diff --git a/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_decision_tree_host.py b/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_decision_tree_host.py index 9e4f6e2ecc..55c96e9376 100644 --- a/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_decision_tree_host.py +++ b/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_decision_tree_host.py @@ -179,10 +179,10 @@ def encode_split_info(self, split_info_list): LOGGER.debug('sitename is {}, self.sitename is {}' .format(split_info.sitename, self.sitename)) assert split_info.sitename == self.sitename - split_info.best_fid = self.encode("feature_idx", split_info.best_fid) + self.encode("feature_idx", split_info.best_fid) assert split_info.best_fid is not None - split_info.best_bid = self.encode("feature_val", split_info.best_bid, self.cur_to_split_nodes[i].id) - split_info.missing_dir = self.encode("missing_dir", split_info.missing_dir, self.cur_to_split_nodes[i].id) + self.encode("feature_val", split_info.best_bid, self.cur_to_split_nodes[i].id) + self.encode("missing_dir", split_info.missing_dir, self.cur_to_split_nodes[i].id) split_info.mask_id = None else: LOGGER.debug('this node can not be further split by host feature: {}'.format(split_info)) @@ -417,18 +417,15 @@ def compute_best_splits(self, cur_to_split_nodes: list, node_map: dict, dep: int if not self.complete_secure_tree: data = self.data_with_node_assignments - acc_histograms = self.get_local_histograms(dep, data, self.grad_and_hess, None, cur_to_split_nodes, node_map, ret='tb', hist_sub=False) - splitinfo_host, encrypted_splitinfo_host = self.splitter.find_split_host(histograms=acc_histograms, node_map=node_map, use_missing=self.use_missing, zero_as_missing=self.zero_as_missing, valid_features=self.valid_features, sitename=self.sitename,) - self.sync_encrypted_splitinfo_host(encrypted_splitinfo_host, dep, batch) federated_best_splitinfo_host = self.sync_federated_best_splitinfo_host(dep, batch) self.sync_final_splitinfo_host(splitinfo_host, federated_best_splitinfo_host, dep, batch) From 256bbdd4ab08b6a81c7ff88baa3ef7387be3bbc2 Mon Sep 17 00:00:00 2001 From: weijingchen Date: Fri, 11 Feb 2022 17:02:42 +0800 Subject: [PATCH 18/99] adjust feature importance strategy and update codes Signed-off-by: cwj --- .../hetero/hetero_decision_tree_guest.py | 75 ++------ .../hetero/hetero_decision_tree_host.py | 180 +++++++----------- .../hetero/hetero_fast_decision_tree_guest.py | 105 +++------- .../hetero/hetero_fast_decision_tree_host.py | 122 +++--------- .../decision_tree/tree_core/decision_tree.py | 59 ++---- .../decision_tree/tree_core/splitter.py | 46 +++-- 6 files changed, 187 insertions(+), 400 deletions(-) diff --git a/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_decision_tree_guest.py b/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_decision_tree_guest.py index 1ebdc18cc8..e3c8b4f343 100644 --- a/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_decision_tree_guest.py +++ b/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_decision_tree_guest.py @@ -26,8 +26,8 @@ def __init__(self, tree_param): self.sitename = consts.GUEST # will be modified in self.set_runtime_idx() self.complete_secure_tree = False - self.split_maskdict = {} - self.missing_dir_maskdict = {} + self.split_maskdict = {} # save split value + self.missing_dir_maskdict = {} # save missing dir self.host_party_idlist = [] self.compressor = None @@ -214,7 +214,8 @@ def find_best_split_guest_and_host(self, splitinfo_guest_host, need_decrypt=True best_gain_host = gain_host_i best_gain_host_idx = i - # if merge_host_split_only is True, guest hists is None + # if merge_host_split_only is True, guest split-info is None + # first one at 0 index is the best split of guest if splitinfo_guest_host[0] is not None and \ splitinfo_guest_host[0].gain >= best_gain_host - consts.FLOAT_ZERO: best_splitinfo = splitinfo_guest_host[0] @@ -225,9 +226,9 @@ def find_best_split_guest_and_host(self, splitinfo_guest_host, need_decrypt=True # so need type checking here if need_decrypt: best_splitinfo.sum_grad = self.decrypt(best_splitinfo.sum_grad) \ - if type(best_splitinfo.sum_grad) != int else best_splitinfo.sum_grad + if not isinstance(best_splitinfo.sum_grad, int) else best_splitinfo.sum_grad best_splitinfo.sum_hess = self.decrypt(best_splitinfo.sum_hess) \ - if type(best_splitinfo.sum_hess) != int else best_splitinfo.sum_hess + if not isinstance(best_splitinfo.sum_hess, int) else best_splitinfo.sum_hess best_splitinfo.gain = best_gain_host return best_splitinfo @@ -281,7 +282,8 @@ def federated_find_split(self, dep=-1, batch=-1, idx=-1): max_nodes = max(len(encrypted_splitinfo_host[host_idx][j]) for j in range(len(self.cur_to_split_nodes))) # batch split point finding for every cur to split nodes for k in range(0, max_nodes, consts.MAX_SPLITINFO_TO_COMPUTE): - batch_splitinfo_host = [encrypted_splitinfo[k: k + consts.MAX_SPLITINFO_TO_COMPUTE] for encrypted_splitinfo + batch_splitinfo_host = [encrypted_splitinfo[k: k + consts.MAX_SPLITINFO_TO_COMPUTE] for + encrypted_splitinfo in encrypted_splitinfo_host[host_idx]] encrypted_splitinfo_host_table = session.parallelize(zip(self.cur_to_split_nodes, batch_splitinfo_host), @@ -314,7 +316,7 @@ def get_computing_inst2node_idx(self): inst2node_idx = self.inst2node_idx return inst2node_idx - def compute_best_splits2(self, cur_to_split_nodes, node_map, dep, batch_idx): + def compute_best_splits(self, cur_to_split_nodes, node_map, dep, batch_idx): LOGGER.info('solving node batch {}, node num is {}'.format(batch_idx, len(cur_to_split_nodes))) inst2node_idx = self.get_computing_inst2node_idx() @@ -336,7 +338,8 @@ def compute_best_splits2(self, cur_to_split_nodes, node_map, dep, batch_idx): for host_idx, split_info_table in enumerate(host_split_info_tables): - host_split_info = self.splitter.find_host_best_split_info(split_info_table, self.get_host_sitename(host_idx), + host_split_info = self.splitter.find_host_best_split_info(split_info_table, + self.get_host_sitename(host_idx), self.encrypter, gh_packer=self.packer) split_info_list = [None for i in range(len(host_split_info))] @@ -349,43 +352,10 @@ def compute_best_splits2(self, cur_to_split_nodes, node_map, dep, batch_idx): suffix=(dep, batch_idx), idx=host_idx, role=consts.HOST) best_splits_of_all_hosts.append(split_info_list) - - # get encoded split-info from hosts - final_host_split_info = self.sync_final_split_host(dep, batch_idx) - for masked_split_info, encoded_split_info in zip(best_splits_of_all_hosts, final_host_split_info): - for s1, s2 in zip(masked_split_info, encoded_split_info): - s2.gain = s1.gain - s2.sum_grad = s1.sum_grad - s2.sum_hess = s1.sum_hess - - final_best_splits = self.merge_splitinfo(best_split_info_guest, final_host_split_info, need_decrypt=False) + final_best_splits = self.merge_splitinfo(best_split_info_guest, best_splits_of_all_hosts, need_decrypt=False) return final_best_splits - def compute_best_splits(self, cur_to_split_nodes, node_map, dep, batch_idx): - - acc_histograms = self.get_local_histograms(dep, self.data_with_node_assignments, self.grad_and_hess, - None, cur_to_split_nodes, node_map, ret='tensor', - hist_sub=False) - - best_split_info_guest = self.splitter.find_split(acc_histograms, self.valid_features, - self.data_bin.partitions, self.sitename, - self.use_missing, self.zero_as_missing) - LOGGER.debug('computing local splits done') - - if self.complete_secure_tree: - return best_split_info_guest - - self.federated_find_split(dep, batch_idx) - host_split_info = self.sync_final_split_host(dep, batch_idx) - - # compare host best split points with guest split points - cur_best_split = self.merge_splitinfo(splitinfo_guest=best_split_info_guest, - splitinfo_host=host_split_info, - merge_host_split_only=False) - - return cur_best_split - """ Federation Functions """ @@ -424,7 +394,7 @@ def sync_cur_to_split_nodes(self, cur_to_split_node, dep=-1, idx=-1): for i in range(len(mask_tree_node_queue)): mask_tree_node_queue[i] = Node(id=mask_tree_node_queue[i].id, parent_nodeid=mask_tree_node_queue[i].parent_nodeid, - is_left_node=mask_tree_node_queue[i].is_left_node,) + is_left_node=mask_tree_node_queue[i].is_left_node) self.transfer_inst.tree_node_queue.remote(mask_tree_node_queue, role=consts.HOST, @@ -523,10 +493,12 @@ def remove_sensitive_info(self): node.weight = None node.sum_grad = None node.sum_hess = None + node.fid = -1 + node.bid = -1 return new_tree_ - def initialize_root_node(self,): + def initialize_root_node(self): LOGGER.info('initializing root node') root_sum_grad, root_sum_hess = self.get_grad_hess_sum(self.grad_and_hess) root_node = Node(id=0, sitename=self.sitename, sum_grad=root_sum_grad, sum_hess=root_sum_hess, @@ -594,11 +566,7 @@ def update_tree(self, split_info, reach_max_depth): self.cur_layer_nodes[i].missing_dir = self.encode("missing_dir", split_info[i].missing_dir, self.cur_layer_nodes[i].id) - else: - self.cur_layer_nodes[i].fid = split_info[i].best_fid - self.cur_layer_nodes[i].bid = split_info[i].best_bid - - if self.sitename == split_info[i].sitename: + if split_info[i].sitename == self.sitename: self.update_feature_importance(split_info[i]) self.tree_node.append(self.cur_layer_nodes[i]) @@ -709,15 +677,9 @@ def fit(self): split_info = [] for batch_idx, i in enumerate(range(0, len(self.cur_layer_nodes), self.max_split_nodes)): - self.cur_to_split_nodes = self.cur_layer_nodes[i: i + self.max_split_nodes] node_map = self.get_node_map(self.cur_to_split_nodes) - - if self.new_ver: - cur_splitinfos = self.compute_best_splits2(self.cur_to_split_nodes, node_map, dep, batch_idx) - else: - cur_splitinfos = self.compute_best_splits(self.cur_to_split_nodes, node_map, dep, batch_idx) - + cur_splitinfos = self.compute_best_splits(self.cur_to_split_nodes, node_map, dep, batch_idx) split_info.extend(cur_splitinfos) self.update_tree(split_info, False) @@ -814,7 +776,6 @@ def get_model_meta(self): model_meta.use_missing = self.use_missing model_meta.zero_as_missing = self.zero_as_missing - return model_meta def set_model_meta(self, model_meta): diff --git a/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_decision_tree_host.py b/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_decision_tree_host.py index 55c96e9376..ee6a68eb74 100644 --- a/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_decision_tree_host.py +++ b/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_decision_tree_host.py @@ -1,15 +1,15 @@ import numpy as np +import functools from federatedml.ensemble.basic_algorithms.decision_tree.tree_core.node import Node from federatedml.util import LOGGER from federatedml.protobuf.generated.boosting_tree_model_meta_pb2 import DecisionTreeModelMeta from federatedml.protobuf.generated.boosting_tree_model_param_pb2 import DecisionTreeModelParam from federatedml.ensemble.basic_algorithms.decision_tree.tree_core.decision_tree import DecisionTree -from federatedml.ensemble.basic_algorithms.decision_tree.tree_core.splitter import SplitInfo from federatedml.transfer_variable.transfer_class.hetero_decision_tree_transfer_variable import \ HeteroDecisionTreeTransferVariable from federatedml.util import consts from federatedml.ensemble.basic_algorithms.decision_tree.tree_core.g_h_optim import PackedGHCompressor -import functools +from federatedml.ensemble.basic_algorithms.decision_tree.tree_core.splitter import SplitInfo class HeteroDecisionTreeHost(DecisionTree): @@ -18,6 +18,8 @@ def __init__(self, tree_param): super(HeteroDecisionTreeHost, self).__init__(tree_param) + # add host side feature importance support + self.feature_importance_type = 'split' self.encrypted_grad_and_hess = None self.runtime_idx = 0 self.sitename = consts.HOST # will be modified in self.set_runtime_idx() @@ -28,8 +30,9 @@ def __init__(self, tree_param): self.feature_num = -1 self.missing_dir_mask_left = {} # mask for left direction self.missing_dir_mask_right = {} # mask for right direction - self.split_maskdict = {} # mask for split value - self.missing_dir_maskdict = {} + self.split_maskdict = {} # save split value + self.split_feature_dict = {} # save split feature idx + self.missing_dir_maskdict = {} # save missing dir self.fid_bid_random_mapping = {} self.inverse_fid_bid_random_mapping = {} self.bin_num = None @@ -47,9 +50,13 @@ def __init__(self, tree_param): # code version control self.new_ver = True + # multi mode + self.mo_tree = False + """ Setting """ + def report_init_status(self): LOGGER.info('reporting initialization status') @@ -67,7 +74,9 @@ def init(self, flowid, runtime_idx, data_bin, bin_split_points, bin_sparse_point complete_secure=False, goss_subsample=False, cipher_compressing=False, - new_ver=True): + new_ver=True, + mo_tree=False + ): super(HeteroDecisionTreeHost, self).init_data_and_variable(flowid, runtime_idx, data_bin, bin_split_points, bin_sparse_points, valid_features, None) @@ -78,8 +87,8 @@ def init(self, flowid, runtime_idx, data_bin, bin_split_points, bin_sparse_point self.bin_num = bin_num self.run_cipher_compressing = cipher_compressing self.feature_num = self.bin_split_points.shape[0] - self.new_ver = new_ver + self.mo_tree = mo_tree self.report_init_status() @@ -94,7 +103,7 @@ def generate_missing_dir(self, dep, left_num=3, right_num=3): """ randomly generate missing dir mask """ - rn = np.random.choice(range(left_num+right_num), left_num + right_num, replace=False) + rn = np.random.choice(range(left_num + right_num), left_num + right_num, replace=False) left_dir = rn[0:left_num] right_dir = rn[left_num:] self.missing_dir_mask_left[dep] = left_dir @@ -115,10 +124,11 @@ def generate_fid_bid_random_mapping(feature_num, bin_num): return mapping - def encode(self, etype="feature_idx", val=None, nid=None): + def save_split_info(self, etype="feature_idx", val=None, nid=None): if etype == "feature_idx": - return val + self.split_feature_dict[nid] = val + return None if etype == "feature_val": self.split_maskdict[nid] = val @@ -170,7 +180,7 @@ def unmask_split_info(self, split_info_list, inverse_mask_id_mapping, left_missi return split_info_list - def encode_split_info(self, split_info_list): + def record_split_info(self, split_info_list): final_split_info = [] for i, split_info in enumerate(split_info_list): @@ -179,10 +189,10 @@ def encode_split_info(self, split_info_list): LOGGER.debug('sitename is {}, self.sitename is {}' .format(split_info.sitename, self.sitename)) assert split_info.sitename == self.sitename - self.encode("feature_idx", split_info.best_fid) + self.save_split_info("feature_idx", split_info.best_fid, self.cur_to_split_nodes[i].id) assert split_info.best_fid is not None - self.encode("feature_val", split_info.best_bid, self.cur_to_split_nodes[i].id) - self.encode("missing_dir", split_info.missing_dir, self.cur_to_split_nodes[i].id) + self.save_split_info("feature_val", split_info.best_bid, self.cur_to_split_nodes[i].id) + self.save_split_info("missing_dir", split_info.missing_dir, self.cur_to_split_nodes[i].id) split_info.mask_id = None else: LOGGER.debug('this node can not be further split by host feature: {}'.format(split_info)) @@ -199,7 +209,7 @@ def init_compressor_and_sync_gh(self): LOGGER.info("get encrypted grad and hess") if self.run_cipher_compressing: - self.cipher_compressor = PackedGHCompressor() + self.cipher_compressor = PackedGHCompressor(mo_mode=self.mo_tree) self.grad_and_hess = self.transfer_inst.encrypted_grad_and_hess.get(idx=0) @@ -227,32 +237,6 @@ def sync_federated_best_splitinfo_host(self, dep=-1, batch=-1): suffix=(dep, batch,)) return federated_best_splitinfo_host - def sync_final_splitinfo_host(self, splitinfo_host, federated_best_splitinfo_host, dep=-1, batch=-1): - - LOGGER.info("send host final splitinfo of depth {}, batch {}".format(dep, batch)) - final_splitinfos = [] - for i in range(len(splitinfo_host)): - best_idx, best_gain = federated_best_splitinfo_host[i] - if best_idx != -1: - LOGGER.debug('sitename is {}, self.sitename is {}' - .format(splitinfo_host[i][best_idx].sitename, self.sitename)) - assert splitinfo_host[i][best_idx].sitename == self.sitename - splitinfo = splitinfo_host[i][best_idx] - splitinfo.best_fid = self.encode("feature_idx", splitinfo.best_fid) - assert splitinfo.best_fid is not None - splitinfo.best_bid = self.encode("feature_val", splitinfo.best_bid, self.cur_to_split_nodes[i].id) - splitinfo.missing_dir = self.encode("missing_dir", splitinfo.missing_dir, self.cur_to_split_nodes[i].id) - splitinfo.gain = best_gain - else: - splitinfo = SplitInfo(sitename=self.sitename, best_fid=-1, best_bid=-1, gain=best_gain) - - final_splitinfos.append(splitinfo) - - self.transfer_inst.final_splitinfo_host.remote(final_splitinfos, - role=consts.GUEST, - idx=-1, - suffix=(dep, batch,)) - def sync_dispatch_node_host(self, dep): LOGGER.info("get node from host to dispath, depth is {}".format(dep)) dispatch_node_host = self.transfer_inst.dispatch_node_host.get(idx=0, @@ -267,7 +251,7 @@ def sync_dispatch_node_host_result(self, dispatch_node_host_result, dep=-1): idx=-1, suffix=(dep,)) - def sync_tree(self,): + def sync_tree(self, ): LOGGER.info("sync tree from guest") self.tree_node = self.transfer_inst.tree.get(idx=0) @@ -297,6 +281,7 @@ def sync_data_predicted_by_host(self, predict_data, send_times): @staticmethod def assign_an_instance(value1, value2, sitename=None, decoder=None, + split_feature_dict=None, bin_sparse_points=None, use_missing=False, zero_as_missing=False, split_maskdict=None, @@ -306,7 +291,7 @@ def assign_an_instance(value1, value2, sitename=None, decoder=None, if node_sitename != sitename: return value1 - fid = decoder("feature_idx", fid, nodeid, split_maskdict=split_maskdict) + fid = split_feature_dict[nodeid] bid = decoder("feature_val", bid, nodeid, split_maskdict=split_maskdict) missing_dir = decoder("missing_dir", 1, nodeid, missing_dir_maskdict=missing_dir_maskdict) direction = HeteroDecisionTreeHost.make_decision(value2, fid, bid, missing_dir, use_missing, zero_as_missing, @@ -321,6 +306,7 @@ def assign_instances_to_new_node(self, dispatch_node_host, dep=-1): sitename=self.sitename, decoder=self.decode, split_maskdict=self.split_maskdict, + split_feature_dict=self.split_feature_dict, bin_sparse_points=self.bin_sparse_points, use_missing=self.use_missing, zero_as_missing=self.zero_as_missing, @@ -337,29 +323,39 @@ def update_instances_node_positions(self): Pre-Process / Post-Process """ - def remove_duplicated_split_nodes(self, split_nid_used): + def remove_redundant_splitinfo_in_split_maskdict(self, split_nid_used): LOGGER.info("remove duplicated nodes from split mask dict") duplicated_nodes = set(self.split_maskdict.keys()) - set(split_nid_used) for nid in duplicated_nodes: del self.split_maskdict[nid] - def convert_bin_to_real(self, decode_func, split_maskdict): + def convert_bin_to_real(self, split_maskdict): LOGGER.info("convert tree node bins to real value") split_nid_used = [] for i in range(len(self.tree_node)): - if self.tree_node[i].is_leaf is True: + if self.tree_node[i].is_leaf: continue if self.tree_node[i].sitename == self.sitename: - fid = decode_func("feature_idx", self.tree_node[i].fid, self.tree_node[i].id, split_maskdict) - bid = decode_func("feature_val", self.tree_node[i].bid, self.tree_node[i].id, split_maskdict) - LOGGER.debug("shape of bin_split_points is {}".format(len(self.bin_split_points[fid]))) - real_splitval = self.encode("feature_val", self.bin_split_points[fid][bid], self.tree_node[i].id) - self.tree_node[i].bid = real_splitval + fid = self.split_feature_dict[self.tree_node[i].id] + bid = self.decode("feature_val", self.tree_node[i].bid, self.tree_node[i].id, split_maskdict) + # recover real split value + real_splitval = self.bin_split_points[fid][bid] + self.split_maskdict[self.tree_node[i].id] = real_splitval self.tree_node[i].fid = fid split_nid_used.append(self.tree_node[i].id) - self.remove_duplicated_split_nodes(split_nid_used) + self.remove_redundant_splitinfo_in_split_maskdict(split_nid_used) + + def collect_host_split_feat_importance(self): + + for node in self.tree_node: + if node.is_leaf: + continue + elif node.sitename == self.sitename: + LOGGER.debug('sitename are {} {}'.format(node.sitename, self.sitename)) + fid = self.split_feature_dict[node.id] + self.update_feature_importance(SplitInfo(sitename=self.sitename, best_fid=fid), False) """ Split finding @@ -372,7 +368,7 @@ def get_computing_inst2node_idx(self): inst2node_idx = self.inst2node_idx return inst2node_idx - def compute_best_splits2(self, cur_to_split_nodes: list, node_map, dep, batch): + def compute_best_splits(self, cur_to_split_nodes: list, node_map, dep, batch): LOGGER.info('solving node batch {}, node num is {}'.format(batch, len(cur_to_split_nodes))) if not self.complete_secure_tree: @@ -384,17 +380,17 @@ def compute_best_splits2(self, cur_to_split_nodes: list, node_map, dep, batch): cur_to_split_nodes, node_map, ret='tb', hist_sub=True) - split_info_table = self.splitter.host_prepare_split_points(histograms=acc_histograms, - use_missing=self.use_missing, - valid_features=self.valid_features, - sitename=self.sitename, - left_missing_dir=self.missing_dir_mask_left[dep], - right_missing_dir=self.missing_dir_mask_right[dep], - mask_id_mapping=self.fid_bid_random_mapping, - batch_size=self.bin_num, - cipher_compressor=self.cipher_compressor, - shuffle_random_seed=np.abs(hash((dep, batch))) - ) + split_info_table = self.splitter.host_prepare_split_points( + histograms=acc_histograms, + use_missing=self.use_missing, + valid_features=self.valid_features, + sitename=self.sitename, + left_missing_dir=self.missing_dir_mask_left[dep], + right_missing_dir=self.missing_dir_mask_right[dep], + mask_id_mapping=self.fid_bid_random_mapping, + batch_size=self.bin_num, + cipher_compressor=self.cipher_compressor, + shuffle_random_seed=np.abs(hash((dep, batch)))) # test split info encryption self.transfer_inst.encrypted_splitinfo_host.remote(split_info_table, @@ -402,34 +398,12 @@ def compute_best_splits2(self, cur_to_split_nodes: list, node_map, dep, batch): idx=-1, suffix=(dep, batch)) best_split_info = self.transfer_inst.federated_best_splitinfo_host.get(suffix=(dep, batch), idx=0) - unmasked_split_info = self.unmask_split_info(best_split_info, self.inverse_fid_bid_random_mapping, - self.missing_dir_mask_left[dep], self.missing_dir_mask_right[dep]) - return_split_info = self.encode_split_info(unmasked_split_info) - self.transfer_inst.final_splitinfo_host.remote(return_split_info, - role=consts.GUEST, - idx=-1, - suffix=(dep, batch,)) - else: - LOGGER.debug('skip splits computation') - - def compute_best_splits(self, cur_to_split_nodes: list, node_map: dict, dep: int, batch: int): - - if not self.complete_secure_tree: - - data = self.data_with_node_assignments - acc_histograms = self.get_local_histograms(dep, data, self.grad_and_hess, - None, cur_to_split_nodes, node_map, ret='tb', - hist_sub=False) - splitinfo_host, encrypted_splitinfo_host = self.splitter.find_split_host(histograms=acc_histograms, - node_map=node_map, - use_missing=self.use_missing, - zero_as_missing=self.zero_as_missing, - valid_features=self.valid_features, - sitename=self.sitename,) - self.sync_encrypted_splitinfo_host(encrypted_splitinfo_host, dep, batch) - federated_best_splitinfo_host = self.sync_federated_best_splitinfo_host(dep, batch) - self.sync_final_splitinfo_host(splitinfo_host, federated_best_splitinfo_host, dep, batch) - LOGGER.debug('computing host splits done') + unmasked_split_info = self.unmask_split_info( + best_split_info, + self.inverse_fid_bid_random_mapping, + self.missing_dir_mask_left[dep], + self.missing_dir_mask_right[dep]) + self.record_split_info(unmasked_split_info) else: LOGGER.debug('skip splits computation') @@ -438,7 +412,7 @@ def compute_best_splits(self, cur_to_split_nodes: list, node_map: dict, dep: int """ def fit(self): - + LOGGER.info("begin to fit host decision tree") self.init_compressor_and_sync_gh() @@ -459,20 +433,18 @@ def fit(self): batch = 0 for i in range(0, len(self.cur_layer_nodes), self.max_split_nodes): self.cur_to_split_nodes = self.cur_layer_nodes[i: i + self.max_split_nodes] - if self.new_ver: - self.compute_best_splits2(self.cur_to_split_nodes, - node_map=self.get_node_map(self.cur_to_split_nodes), - dep=dep, batch=batch) - else: - self.compute_best_splits(self.cur_to_split_nodes, - node_map=self.get_node_map(self.cur_to_split_nodes), dep=dep, batch=batch) + self.compute_best_splits(self.cur_to_split_nodes, + node_map=self.get_node_map(self.cur_to_split_nodes), + dep=dep, batch=batch) batch += 1 - dispatch_node_host = self.sync_dispatch_node_host(dep) self.assign_instances_to_new_node(dispatch_node_host, dep=dep) self.sync_tree() - self.convert_bin_to_real(decode_func=self.decode, split_maskdict=self.split_maskdict) + self.print_leafs() + # convert bin index to real split-value, and remove redundant nid in split mask dict + self.convert_bin_to_real(split_maskdict=self.split_maskdict) + self.collect_host_split_feat_importance() LOGGER.info("fitting host decision tree done") @staticmethod @@ -486,7 +458,6 @@ def traverse_tree(predict_state, data_inst, tree_=None, return predict_state while tree_[nid].sitename == sitename: - nid = HeteroDecisionTreeHost.go_next_layer(tree_[nid], data_inst, use_missing, zero_as_missing, None, split_maskdict, missing_dir_maskdict, decoder) @@ -588,8 +559,3 @@ def initialize_root_node(self, *args): def update_tree(self, *args): pass - - - - - diff --git a/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_fast_decision_tree_guest.py b/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_fast_decision_tree_guest.py index 1e55b81038..d0a395d7c5 100644 --- a/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_fast_decision_tree_guest.py +++ b/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_fast_decision_tree_guest.py @@ -64,37 +64,6 @@ def compute_best_splits_with_node_plan(self, tree_action, target_host_idx, cur_t LOGGER.debug('node plan at dep {} is {}'.format(dep, (tree_action, target_host_idx))) - cur_best_split = [] - - if tree_action == plan.tree_actions['guest_only']: - acc_histograms = self.get_local_histograms(dep, self.data_with_node_assignments, self.grad_and_hess, - None, cur_to_split_nodes, node_map, ret='tensor', - hist_sub=False) - - cur_best_split = self.splitter.find_split(acc_histograms, self.valid_features, - self.data_bin.partitions, self.sitename, - self.use_missing, self.zero_as_missing) - LOGGER.debug('computing local splits done') - - if tree_action == plan.tree_actions['host_only']: - - self.federated_find_split(dep, batch_idx, idx=target_host_idx) - - if mode == consts.LAYERED_TREE: - host_split_info = self.sync_final_split_host(dep, batch_idx, idx=target_host_idx) - LOGGER.debug('get encrypted split value from host') - - cur_best_split = self.merge_splitinfo(splitinfo_guest=[], - splitinfo_host=host_split_info, - merge_host_split_only=True) - - return cur_best_split - - def compute_best_splits_with_node_plan2(self, tree_action, target_host_idx, cur_to_split_nodes, node_map: dict, - dep: int, batch_idx: int, mode=consts.MIX_TREE): - - LOGGER.debug('node plan at dep {} is {}'.format(dep, (tree_action, target_host_idx))) - # In layered mode, guest hist computation does not start from root node, so need to disable hist-sub hist_sub = True if mode == consts.MIX_TREE else False @@ -114,44 +83,35 @@ def compute_best_splits_with_node_plan2(self, tree_action, target_host_idx, cur_ if tree_action == plan.tree_actions['host_only']: - split_info_table = self.transfer_inst.encrypted_splitinfo_host.get(idx=target_host_idx, suffix=(dep, batch_idx)) + split_info_table = self.transfer_inst.encrypted_splitinfo_host.get( + idx=target_host_idx, suffix=(dep, batch_idx)) - host_split_info = self.splitter.find_host_best_split_info(split_info_table, self.get_host_sitename(target_host_idx), - self.encrypter, gh_packer=self.packer) + host_split_info = self.splitter.find_host_best_split_info( + split_info_table, self.get_host_sitename(target_host_idx), self.encrypter, gh_packer=self.packer) split_info_list = [None for i in range(len(host_split_info))] for key in host_split_info: split_info_list[node_map[key]] = host_split_info[key] - # MIX mode and Layered mode difference: if mode == consts.MIX_TREE: for split_info in split_info_list: split_info.sum_grad, split_info.sum_hess, split_info.gain = self.encrypt(split_info.sum_grad), \ - self.encrypt(split_info.sum_hess), \ - self.encrypt(split_info.gain) + self.encrypt(split_info.sum_hess), \ + self.encrypt(split_info.gain) return_split_info = split_info_list else: return_split_info = copy.deepcopy(split_info_list) for split_info in return_split_info: split_info.sum_grad, split_info.sum_hess, split_info.gain = None, None, None - self.transfer_inst.federated_best_splitinfo_host.remote(return_split_info, suffix=(dep, batch_idx), idx=target_host_idx, role=consts.HOST) - if mode == consts.MIX_TREE: return [] elif mode == consts.LAYERED_TREE: - - final_host_split_info = self.sync_final_split_host(dep, batch_idx, idx=target_host_idx) - for s1, s2 in zip(split_info_list, final_host_split_info[0]): - s2.gain = s1.gain - s2.sum_grad = s1.sum_grad - s2.sum_hess = s1.sum_hess - cur_best_split = self.merge_splitinfo(splitinfo_guest=[], - splitinfo_host=final_host_split_info, + splitinfo_host=[split_info_list], merge_host_split_only=True, need_decrypt=False) return cur_best_split @@ -200,7 +160,7 @@ def assign_instances_to_new_node_with_node_plan(self, dep, tree_action, mode=con else: self.inst2node_idx = self.inst2node_idx.join(dispatch_node_host_result[idx], lambda unleaf_state_nodeid1, - unleaf_state_nodeid2: + unleaf_state_nodeid2: unleaf_state_nodeid1 if len( unleaf_state_nodeid1) == 2 else unleaf_state_nodeid2) @@ -241,22 +201,18 @@ def layered_mode_fit(self): split_info = [] for batch_idx, i in enumerate(range(0, len(self.cur_layer_nodes), self.max_split_nodes)): self.cur_to_split_nodes = self.cur_layer_nodes[i: i + self.max_split_nodes] - if self.new_ver: - cur_splitinfos = self.compute_best_splits_with_node_plan2(tree_action, host_idx, - node_map=self.get_node_map(self.cur_to_split_nodes), - cur_to_split_nodes=self.cur_to_split_nodes, - dep=dep, batch_idx=batch_idx, - mode=consts.LAYERED_TREE) - else: - cur_splitinfos = self.compute_best_splits_with_node_plan(tree_action, host_idx, node_map= - self.get_node_map(self.cur_to_split_nodes), - cur_to_split_nodes=self.cur_to_split_nodes, - dep=dep, batch_idx=batch_idx, - mode=consts.LAYERED_TREE) + cur_splitinfos = self.compute_best_splits_with_node_plan( + tree_action, + host_idx, + node_map=self.get_node_map(self.cur_to_split_nodes), + cur_to_split_nodes=self.cur_to_split_nodes, + dep=dep, + batch_idx=batch_idx, + mode=consts.LAYERED_TREE) split_info.extend(cur_splitinfos) self.update_tree(split_info, False) - self.assign_instances_to_new_node_with_node_plan(dep, tree_action, mode=consts.LAYERED_TREE,) + self.assign_instances_to_new_node_with_node_plan(dep, tree_action, mode=consts.LAYERED_TREE, ) if self.cur_layer_nodes: self.assign_instance_to_leaves_and_update_weights() @@ -308,18 +264,12 @@ def mix_mode_fit(self): split_info = [] for batch_idx, i in enumerate(range(0, len(self.cur_layer_nodes), self.max_split_nodes)): self.cur_to_split_nodes = self.cur_layer_nodes[i: i + self.max_split_nodes] - if self.new_ver: - cur_splitinfos = self.compute_best_splits_with_node_plan2(tree_action, host_idx, - node_map=self.get_node_map(self.cur_to_split_nodes), - cur_to_split_nodes=self.cur_to_split_nodes, - dep=dep, batch_idx=batch_idx, - mode=consts.MIX_TREE) - else: - cur_splitinfos = self.compute_best_splits_with_node_plan(tree_action, host_idx, node_map= - self.get_node_map(self.cur_to_split_nodes), - cur_to_split_nodes=self.cur_to_split_nodes, - dep=dep, batch_idx=batch_idx, - mode=consts.MIX_TREE) + cur_splitinfos = self.compute_best_splits_with_node_plan(tree_action, host_idx, + node_map=self.get_node_map( + self.cur_to_split_nodes), + cur_to_split_nodes=self.cur_to_split_nodes, + dep=dep, batch_idx=batch_idx, + mode=consts.MIX_TREE) split_info.extend(cur_splitinfos) if self.tree_type == plan.tree_type_dict['guest_feat_only']: @@ -338,7 +288,7 @@ def mix_mode_fit(self): format(self.sample_leaf_pos.count(), self.data_bin.count()) else: if self.cur_layer_nodes: - self.assign_instance_to_leaves_and_update_weights() # guest local updates + self.assign_instance_to_leaves_and_update_weights() # guest local updates self.convert_bin_to_real() # convert bin id to real value features self.sample_weights_post_process() @@ -378,7 +328,7 @@ def sync_sample_leaf_pos(self, idx): return leaf_pos def sync_host_cur_layer_nodes(self, dep, host_idx): - nodes = self.transfer_inst.host_cur_to_split_node_num.get(idx=host_idx, suffix=(dep, )) + nodes = self.transfer_inst.host_cur_to_split_node_num.get(idx=host_idx, suffix=(dep,)) for n in nodes: n.sum_grad = self.decrypt(n.sum_grad) n.sum_hess = self.decrypt(n.sum_hess) @@ -415,7 +365,7 @@ def handle_leaf_nodes(self, nodes): n.sitename = self.sitename if n.id > max_node_id: max_node_id = n.id - new_nodes = [Node() for i in range(max_node_id+1)] + new_nodes = [Node() for i in range(max_node_id + 1)] for n in nodes: new_nodes[n.id] = n return new_nodes @@ -429,7 +379,7 @@ def fit(self): LOGGER.info('fitting a hetero decision tree') if self.tree_type == plan.tree_type_dict['host_feat_only'] or \ - self.tree_type == plan.tree_type_dict['guest_feat_only']: + self.tree_type == plan.tree_type_dict['guest_feat_only']: self.mix_mode_fit() @@ -455,4 +405,3 @@ def get_model_meta(self): def get_model_param(self): return super(HeteroFastDecisionTreeGuest, self).get_model_param() - diff --git a/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_fast_decision_tree_host.py b/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_fast_decision_tree_host.py index dce1b446de..6002f79492 100644 --- a/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_fast_decision_tree_host.py +++ b/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_fast_decision_tree_host.py @@ -2,7 +2,7 @@ import functools import copy from federatedml.ensemble.basic_algorithms import HeteroDecisionTreeHost -from federatedml.ensemble.boosting.hetero import hetero_fast_secureboost_plan as plan +from federatedml.ensemble.basic_algorithms.decision_tree.tree_core import tree_plan as plan from federatedml.util import consts from federatedml.ensemble.basic_algorithms.decision_tree.tree_core.splitter import SplitInfo from federatedml.ensemble.basic_algorithms.decision_tree.tree_core.node import Node @@ -88,44 +88,9 @@ def get_host_split_info(self, splitinfo_host, federated_best_splitinfo_host): return final_splitinfos def compute_best_splits_with_node_plan(self, tree_action, target_host_id, cur_to_split_nodes, - node_map: dict, dep: int, batch_idx: int, + node_map: dict, dep: int, batch: int, mode=consts.LAYERED_TREE): - LOGGER.debug('node plan at dep {} is {}'.format(dep, (tree_action, target_host_id))) - - if tree_action == plan.tree_actions['host_only'] and target_host_id == self.self_host_id: - - data = self.data_with_node_assignments - acc_histograms = self.get_local_histograms(dep, data, self.grad_and_hess, - None, cur_to_split_nodes, node_map, ret='tb', - hist_sub=False) - splitinfo_host, encrypted_splitinfo_host = self.splitter.find_split_host(histograms=acc_histograms, - node_map=node_map, - use_missing=self.use_missing, - zero_as_missing=self.zero_as_missing, - valid_features=self.valid_features, - sitename=self.sitename - ) - - self.sync_encrypted_splitinfo_host(encrypted_splitinfo_host, dep, batch_idx) - federated_best_splitinfo_host = self.sync_federated_best_splitinfo_host(dep, batch_idx) - - if mode == consts.LAYERED_TREE: - LOGGER.debug('sending split info to guest') - self.sync_final_splitinfo_host(splitinfo_host, federated_best_splitinfo_host, dep, batch_idx) - LOGGER.debug('computing host splits done') - - else: - host_split_info = self.get_host_split_info(splitinfo_host, federated_best_splitinfo_host) - return host_split_info - else: - LOGGER.debug('skip best split computation') - return None - - def compute_best_splits_with_node_plan2(self, tree_action, target_host_id, cur_to_split_nodes, - node_map: dict, dep: int, batch: int, - mode=consts.LAYERED_TREE): - if tree_action == plan.tree_actions['host_only'] and target_host_id == self.self_host_id: data = self.data_with_node_assignments inst2node_idx = self.get_computing_inst2node_idx() @@ -135,17 +100,17 @@ def compute_best_splits_with_node_plan2(self, tree_action, target_host_id, cur_t cur_to_split_nodes, node_map, ret='tb', hist_sub=True) - split_info_table = self.splitter.host_prepare_split_points(histograms=acc_histograms, - use_missing=self.use_missing, - valid_features=self.valid_features, - sitename=self.sitename, - left_missing_dir=self.missing_dir_mask_left[dep], - right_missing_dir=self.missing_dir_mask_right[dep], - mask_id_mapping=self.fid_bid_random_mapping, - batch_size=self.bin_num, - cipher_compressor=self.cipher_compressor, - shuffle_random_seed=np.abs(hash((dep, batch))) - ) + split_info_table = self.splitter.host_prepare_split_points( + histograms=acc_histograms, + use_missing=self.use_missing, + valid_features=self.valid_features, + sitename=self.sitename, + left_missing_dir=self.missing_dir_mask_left[dep], + right_missing_dir=self.missing_dir_mask_right[dep], + mask_id_mapping=self.fid_bid_random_mapping, + batch_size=self.bin_num, + cipher_compressor=self.cipher_compressor, + shuffle_random_seed=np.abs(hash((dep, batch)))) # test split info encryption self.transfer_inst.encrypted_splitinfo_host.remote(split_info_table, @@ -159,11 +124,7 @@ def compute_best_splits_with_node_plan2(self, tree_action, target_host_id, cur_t self.missing_dir_mask_right[dep]) if mode == consts.LAYERED_TREE: - return_split_info = self.encode_split_info(unmasked_split_info) - self.transfer_inst.final_splitinfo_host.remote(return_split_info, - role=consts.GUEST, - idx=-1, - suffix=(dep, batch,)) + self.record_split_info(unmasked_split_info) elif mode == consts.MIX_TREE: return unmasked_split_info else: @@ -283,7 +244,7 @@ def host_local_assign_instances_to_new_node(self): def sync_sample_leaf_pos(self, sample_leaf_pos): LOGGER.debug('final sample pos sent') self.transfer_inst.dispatch_node_host_result.remote(sample_leaf_pos, idx=0, - suffix=('final sample pos', ), role=consts.GUEST) + suffix=('final sample pos',), role=consts.GUEST) def sync_leaf_nodes(self): leaves = [] @@ -317,17 +278,6 @@ def mask_node_id(self, nodes): n.id = -1 return nodes - def convert_bin_to_real(self): - LOGGER.info("convert tree node bins to real value") - for i in range(len(self.tree_node)): - if self.tree_node[i].is_leaf is True: - continue - if self.tree_node[i].sitename == self.sitename: - fid = self.decode("feature_idx", self.tree_node[i].fid, split_maskdict=self.split_maskdict) - bid = self.decode("feature_val", self.tree_node[i].bid, self.tree_node[i].id, self.split_maskdict) - real_splitval = self.encode("feature_val", self.bin_split_points[fid][bid], self.tree_node[i].id) - self.tree_node[i].bid = real_splitval - def convert_bin_to_real2(self): """ convert current bid in tree nodes to real value @@ -364,7 +314,7 @@ def mix_mode_fit(self): self.inst2node_idx = self.assign_instance_to_root_node(self.data_bin, root_node_id=0) # root node id is 0 - self.cur_layer_nodes = [Node(id=0, sitename=self.sitename, sum_grad=root_sum_grad, sum_hess=root_sum_hess,)] + self.cur_layer_nodes = [Node(id=0, sitename=self.sitename, sum_grad=root_sum_grad, sum_hess=root_sum_hess, )] for dep in range(self.max_depth): @@ -381,19 +331,10 @@ def mix_mode_fit(self): split_info = [] for i in range(0, len(self.cur_layer_nodes), self.max_split_nodes): self.cur_to_split_nodes = self.cur_layer_nodes[i: i + self.max_split_nodes] - if self.new_ver: - batch_split_info = self.compute_best_splits_with_node_plan2(tree_action, layer_target_host_id, - cur_to_split_nodes=self.cur_to_split_nodes, - node_map=self.get_node_map(self.cur_to_split_nodes), - dep=dep, batch=batch, - mode=consts.MIX_TREE) - else: - batch_split_info = self.compute_best_splits_with_node_plan(tree_action, layer_target_host_id, - cur_to_split_nodes=self.cur_to_split_nodes, - node_map=self.get_node_map(self.cur_to_split_nodes), - dep=dep, batch_idx=batch, - mode=consts.MIX_TREE) - + batch_split_info = self.compute_best_splits_with_node_plan( + tree_action, layer_target_host_id, cur_to_split_nodes=self.cur_to_split_nodes, + node_map=self.get_node_map( + self.cur_to_split_nodes), dep=dep, batch=batch, mode=consts.MIX_TREE) batch += 1 split_info.extend(batch_split_info) @@ -490,26 +431,19 @@ def layered_mode_fit(self): batch = 0 for i in range(0, len(self.cur_layer_nodes), self.max_split_nodes): self.cur_to_split_nodes = self.cur_layer_nodes[i: i + self.max_split_nodes] - if self.new_ver: - self.compute_best_splits_with_node_plan2(tree_action, layer_target_host_id, - cur_to_split_nodes=self.cur_to_split_nodes, - node_map=self.get_node_map(self.cur_to_split_nodes), - dep=dep, batch=batch, - mode=consts.LAYERED_TREE) - else: - self.compute_best_splits_with_node_plan(tree_action, layer_target_host_id, - cur_to_split_nodes=self.cur_to_split_nodes, - node_map=self.get_node_map(self.cur_to_split_nodes), - dep=dep, batch_idx=batch, ) - + self.compute_best_splits_with_node_plan(tree_action, layer_target_host_id, + cur_to_split_nodes=self.cur_to_split_nodes, + node_map=self.get_node_map(self.cur_to_split_nodes), + dep=dep, batch=batch, + mode=consts.LAYERED_TREE) batch += 1 - if layer_target_host_id == self.self_host_id: dispatch_node_host = self.sync_dispatch_node_host(dep) self.assign_instances_to_new_node(dispatch_node_host, dep) self.sync_tree() - self.convert_bin_to_real() + self.convert_bin_to_real(self.split_maskdict) + self.collect_host_split_feat_importance() """ Fit & Predict @@ -548,4 +482,4 @@ def get_model_meta(self): return super(HeteroFastDecisionTreeHost, self).get_model_meta() def get_model_param(self): - return super(HeteroFastDecisionTreeHost, self).get_model_param() \ No newline at end of file + return super(HeteroFastDecisionTreeHost, self).get_model_param() diff --git a/python/federatedml/ensemble/basic_algorithms/decision_tree/tree_core/decision_tree.py b/python/federatedml/ensemble/basic_algorithms/decision_tree/tree_core/decision_tree.py index 9eb4eb145f..45e639031f 100644 --- a/python/federatedml/ensemble/basic_algorithms/decision_tree/tree_core/decision_tree.py +++ b/python/federatedml/ensemble/basic_algorithms/decision_tree/tree_core/decision_tree.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- +#!/usr/bin/env python +# -*- coding: utf-8 -*- # # Copyright 2019 The FATE Authors. All Rights Reserved. @@ -69,8 +69,13 @@ def __init__(self, tree_param): self.tree_node_num = 0 self.runtime_idx = None self.valid_features = None - self.splitter = Splitter(self.criterion_method, self.criterion_params, self.min_impurity_split, - self.min_sample_split, self.min_leaf_node, self.min_child_weight) # splitter for finding splits + self.splitter = Splitter( + self.criterion_method, + self.criterion_params, + self.min_impurity_split, + self.min_sample_split, + self.min_leaf_node, + self.min_child_weight) # splitter for finding splits self.inst2node_idx = None # record the internal node id an instance belongs to self.sample_leaf_pos = None # record the final leaf id of samples self.sample_weights = None # leaf weights of samples @@ -134,49 +139,11 @@ def check_max_split_nodes(self): def set_flowid(self, flowid=0): LOGGER.info("set flowid, flowid is {}".format(flowid)) self.transfer_inst.set_flowid(flowid) - + def set_runtime_idx(self, runtime_idx): self.runtime_idx = runtime_idx self.sitename = ":".join([self.sitename, str(self.runtime_idx)]) - """ - Node encode/ decode - """ - # add node split-val/missing-dir to mask dict, hetero tree only - def encode(self, etype="feature_idx", val=None, nid=None): - if etype == "feature_idx": - return val - - if etype == "feature_val": - self.split_maskdict[nid] = val - return None - - if etype == "missing_dir": - self.missing_dir_maskdict[nid] = val - return None - - raise TypeError("encode type %s is not support!" % (str(etype))) - - # recover node split-val/missing-dir from mask dict, hetero tree only - @staticmethod - def decode(dtype="feature_idx", val=None, nid=None, split_maskdict=None, missing_dir_maskdict=None): - if dtype == "feature_idx": - return val - - if dtype == "feature_val": - if nid in split_maskdict: - return split_maskdict[nid] - else: - raise ValueError("decode val %s cause error, can't recognize it!" % (str(val))) - - if dtype == "missing_dir": - if nid in missing_dir_maskdict: - return missing_dir_maskdict[nid] - else: - raise ValueError("decode val %s cause error, can't recognize it!" % (str(val))) - - return TypeError("decode type %s is not support!" % (str(dtype))) - """ Histogram interface """ @@ -234,7 +201,7 @@ def sample_count_map_func(kv, node_map): # record node sample number in count_arr count_arr = np.zeros(len(node_map)) for k, v in kv: - if type(v) == int: # leaf node format: (leaf_node_id) + if isinstance(v, int): # leaf node format: (leaf_node_id) key = v else: # internal node format: (1, node_id) key = v[1] @@ -264,7 +231,7 @@ def get_sample_weights(self): # return sample weights to boosting class return self.sample_weights - @ staticmethod + @staticmethod def assign_instance_to_root_node(data_bin, root_node_id): return data_bin.mapValues(lambda inst: (1, root_node_id)) @@ -273,7 +240,7 @@ def float_round(num): """ prevent float error """ - return round(num, consts.TREE_DECIMAL_ROUND) + return np.round(num, consts.TREE_DECIMAL_ROUND) def update_feature_importance(self, splitinfo, record_site_name=True): diff --git a/python/federatedml/ensemble/basic_algorithms/decision_tree/tree_core/splitter.py b/python/federatedml/ensemble/basic_algorithms/decision_tree/tree_core/splitter.py index 1c91f35777..01afb1a54e 100644 --- a/python/federatedml/ensemble/basic_algorithms/decision_tree/tree_core/splitter.py +++ b/python/federatedml/ensemble/basic_algorithms/decision_tree/tree_core/splitter.py @@ -24,7 +24,6 @@ # ============================================================================= # # ============================================================================= - import numpy as np import warnings import functools @@ -41,7 +40,6 @@ class SplitInfo(object): def __init__(self, sitename=consts.GUEST, best_fid=None, best_bid=None, sum_grad=0, sum_hess=0, gain=None, missing_dir=1, mask_id=None, sample_count=-1): - self.sitename = sitename self.best_fid = best_fid self.best_bid = best_bid @@ -55,8 +53,8 @@ def __init__(self, sitename=consts.GUEST, best_fid=None, best_bid=None, def __str__(self): return '(fid {} bid {}, sum_grad {}, sum_hess {}, gain {}, sitename {}, missing dir {}, mask_id {}, ' \ 'sample_count {})\n'.format( - self.best_fid, self.best_bid, self.sum_grad, self.sum_hess, self.gain, self.sitename, self.missing_dir, - self.mask_id, self.sample_count) + self.best_fid, self.best_bid, self.sum_grad, self.sum_hess, self.gain, self.sitename, self.missing_dir, + self.mask_id, self.sample_count) def __repr__(self): return self.__str__() @@ -77,11 +75,11 @@ def __init__(self, criterion_method, criterion_params=[0, 0], min_impurity_split else: try: reg_lambda, reg_alpha = 0, 0 - if type(criterion_params) is list: + if isinstance(criterion_params, list): reg_lambda = float(criterion_params[0]) reg_alpha = float(criterion_params[1]) self.criterion = XgboostCriterion(reg_lambda=reg_lambda, reg_alpha=reg_alpha) - except: + except BaseException: warnings.warn("criterion_params' first criterion_params should be numeric") self.criterion = XgboostCriterion() @@ -91,7 +89,11 @@ def __init__(self, criterion_method, criterion_params=[0, 0], min_impurity_split self.min_child_weight = min_child_weight def _check_min_child_weight(self, l_h, r_h): - return l_h >= self.min_child_weight and r_h >= self.min_child_weight + + if isinstance(l_h, np.ndarray): + l_h, r_h = np.sum(l_h), np.sum(r_h) + rs = l_h >= self.min_child_weight and r_h >= self.min_child_weight + return rs def _check_sample_num(self, l_cnt, r_cnt): return l_cnt >= self.min_leaf_node and r_cnt >= self.min_leaf_node @@ -131,6 +133,9 @@ def find_split_single_histogram_guest(self, histogram, valid_features, sitename, if node_cnt < self.min_sample_split: break + if node_cnt < 1: # avoid float error + break + # last bin will not participate in split find, so bin_num - 1 for bid in range(bin_num - missing_bin - 1): @@ -143,7 +148,8 @@ def find_split_single_histogram_guest(self, histogram, valid_features, sitename, sum_hess_r = sum_hess - sum_hess_l node_cnt_r = node_cnt - node_cnt_l - if self._check_sample_num(node_cnt_l, node_cnt_r) and self._check_min_child_weight(sum_hess_l, sum_hess_r): + if self._check_min_child_weight(sum_hess_l, sum_hess_r) and self._check_sample_num(node_cnt_l, + node_cnt_r): gain = self.criterion.split_gain([sum_grad, sum_hess], [sum_grad_l, sum_hess_l], [sum_grad_r, sum_hess_r]) @@ -168,7 +174,8 @@ def find_split_single_histogram_guest(self, histogram, valid_features, sitename, node_cnt_r -= histogram[fid][-1][2] - histogram[fid][-2][2] # if have a better gain value, missing dir is left - if self._check_sample_num(node_cnt_l, node_cnt_r) and self._check_min_child_weight(sum_hess_l, sum_hess_r): + if self._check_sample_num(node_cnt_l, node_cnt_r) and self._check_min_child_weight(sum_hess_l, + sum_hess_r): gain = self.criterion.split_gain([sum_grad, sum_hess], [sum_grad_l, sum_hess_l], [sum_grad_r, sum_hess_r]) @@ -319,8 +326,10 @@ def construct_feature_split_points_batches(self, kv_iter, valid_features, sitena if partition_key is None: partition_key = str((nid, fid)) - split_info_list, g_h_sum_info = self.construct_feature_split_points(value, valid_features, sitename, use_missing, - left_missing_dir, right_missing_dir, mask_id_mapping) + split_info_list, g_h_sum_info = self.construct_feature_split_points(value, valid_features, sitename, + use_missing, + left_missing_dir, right_missing_dir, + mask_id_mapping) # collect all splitinfo of a node if nid not in split_info_dict: split_info_dict[nid] = [] @@ -335,7 +344,8 @@ def construct_feature_split_points_batches(self, kv_iter, valid_features, sitena split_info_list = split_info_dict[nid] if len(split_info_list) == 0: - result_list.append(((nid, partition_key+'-empty'), [])) # add an empty split info list if no split info available + result_list.append( + ((nid, partition_key + '-empty'), [])) # add an empty split info list if no split info available continue if shuffle_random_seed: @@ -348,10 +358,10 @@ def construct_feature_split_points_batches(self, kv_iter, valid_features, sitena batch_start_idx = range(0, len(split_info_list), batch_size) batch_idx = 0 for i in batch_start_idx: - key = (nid, (partition_key+'-{}'.format(batch_idx))) # nid, batch_id + key = (nid, (partition_key + '-{}'.format(batch_idx))) # nid, batch_id batch_idx += 1 g_h_sum_info = g_h_sum_dict[nid] - batch_split_info_list = split_info_list[i: i+batch_size] + batch_split_info_list = split_info_list[i: i + batch_size] # compress ciphers if cipher_compressor is not None: compressed_packages = cipher_compressor.compress_split_info(batch_split_info_list, g_h_sum_info) @@ -376,18 +386,18 @@ def _find_host_best_splits_map_func(self, value, decrypter, gh_packer=None, if gh_packer is None: split_info_list, g_h_info = value for split_info in split_info_list: - split_info.sum_grad, split_info.sum_hess = decrypter.decrypt(split_info.sum_grad), decrypter.decrypt(split_info.sum_hess) + split_info.sum_grad, split_info.sum_hess = decrypter.decrypt(split_info.sum_grad), decrypter.decrypt( + split_info.sum_hess) g_sum, h_sum = decrypter.decrypt(g_h_info.sum_grad), decrypter.decrypt(g_h_info.sum_hess) else: nid, package = value split_info_list = gh_packer.decompress_and_unpack(package) - g_sum, h_sum = split_info_list[-1].sum_grad, split_info_list[-1].sum_hess # g/h is at last index + g_sum, h_sum = split_info_list[-1].sum_grad, split_info_list[-1].sum_hess # g/h sum is at last index split_info_list = split_info_list[:-1] for idx, split_info in enumerate(split_info_list): l_g, l_h = split_info.sum_grad, split_info.sum_hess - r_g, r_h = g_sum - l_g, h_sum - l_h gain = self.split_gain(g_sum, h_sum, l_g, l_h, r_g, r_h) @@ -496,7 +506,7 @@ def node_weight(self, grad, hess): return self.criterion.node_weight(grad, hess) def split_gain(self, sum_grad, sum_hess, sum_grad_l, sum_hess_l, sum_grad_r, sum_hess_r): - gain = self.criterion.split_gain([sum_grad, sum_hess], \ + gain = self.criterion.split_gain([sum_grad, sum_hess], [sum_grad_l, sum_hess_l], [sum_grad_r, sum_hess_r]) return gain From 9d6dca57fab043e1775438215096ab1e0f130ebe Mon Sep 17 00:00:00 2001 From: weijingchen Date: Fri, 11 Feb 2022 17:39:02 +0800 Subject: [PATCH 19/99] update feature importance strategy Signed-off-by: cwj --- .../hetero/hetero_secureboost_guest.py | 25 ++++++---- .../hetero/hetero_secureboost_host.py | 47 ++++++++++++++----- ...cure_boosting_predict_transfer_variable.py | 1 + 3 files changed, 52 insertions(+), 21 deletions(-) diff --git a/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_guest.py b/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_guest.py index 4ac8133068..cd95a89344 100644 --- a/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_guest.py +++ b/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_guest.py @@ -36,7 +36,7 @@ def __init__(self): self.model_param = HeteroSecureBoostParam() self.complete_secure = False self.data_alignment_map = {} - self.predict_transfer_inst = HeteroSecureBoostTransferVariable() + self.hetero_sbt_transfer_variable = HeteroSecureBoostTransferVariable() self.model_name = 'HeteroSecureBoost' self.max_sample_weight = 1 self.max_sample_weight_computed = False @@ -111,6 +111,13 @@ def update_feature_importance(self, tree_feature_importance): self.feature_importances_[fid] += tree_feature_importance[fid] LOGGER.debug('cur feature importance {}'.format(self.feature_importances_)) + def sync_feature_importance(self): + host_feature_importance_list = self.hetero_sbt_transfer_variable.host_feature_importance.get(idx=-1) + for i in host_feature_importance_list: + self.feature_importances_.update(i) + + LOGGER.debug('self feature importance is {}'.format(self.feature_importances_)) + def fit_a_booster(self, epoch_idx: int, booster_dim: int): if self.cur_epoch_idx != epoch_idx: @@ -322,13 +329,13 @@ def boosting_fast_predict(self, data_inst, trees: List[HeteroDecisionTreeGuest], node_pos_tb = node_pos_tb.subtractByKey(reach_leaf_samples) if node_pos_tb.count() == 0: - self.predict_transfer_inst.predict_stop_flag.remote(True, idx=-1, suffix=(comm_round, )) + self.hetero_sbt_transfer_variable.predict_stop_flag.remote(True, idx=-1, suffix=(comm_round,)) break LOGGER.info('cur predict round is {}'.format(comm_round)) - self.predict_transfer_inst.predict_stop_flag.remote(False, idx=-1, suffix=(comm_round, )) - self.predict_transfer_inst.guest_predict_data.remote(node_pos_tb, idx=-1, suffix=(comm_round, )) - host_pos_tbs = self.predict_transfer_inst.host_predict_data.get(idx=-1, suffix=(comm_round, )) + self.hetero_sbt_transfer_variable.predict_stop_flag.remote(False, idx=-1, suffix=(comm_round,)) + self.hetero_sbt_transfer_variable.guest_predict_data.remote(node_pos_tb, idx=-1, suffix=(comm_round,)) + host_pos_tbs = self.hetero_sbt_transfer_variable.host_predict_data.get(idx=-1, suffix=(comm_round,)) for host_pos_tb in host_pos_tbs: node_pos_tb = node_pos_tb.join(host_pos_tb, self.merge_predict_pos) @@ -434,12 +441,12 @@ def EINI_guest_predict(self, data_inst, trees: List[HeteroDecisionTreeGuest], le encrypter_vec_table = position_vec.mapValues(encrypter.recursive_encrypt) # federation part - self.predict_transfer_inst.guest_predict_data.remote(booster_dim, idx=-1, suffix='booster_dim') + self.hetero_sbt_transfer_variable.guest_predict_data.remote(booster_dim, idx=-1, suffix='booster_dim') # send to first host party - self.predict_transfer_inst.guest_predict_data.remote(encrypter_vec_table, idx=0, suffix='position_vec', role=consts.HOST) + self.hetero_sbt_transfer_variable.guest_predict_data.remote(encrypter_vec_table, idx=0, suffix='position_vec', role=consts.HOST) # get from last host party - result_table = self.predict_transfer_inst.host_predict_data.get(idx=len(party_list) - 1, suffix='merge_result', - role=consts.HOST) + result_table = self.hetero_sbt_transfer_variable.host_predict_data.get(idx=len(party_list) - 1, suffix='merge_result', + role=consts.HOST) # decode result result = result_table.mapValues(encrypter.recursive_decrypt) diff --git a/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py b/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py index 1d8de0d79f..4631a4718d 100644 --- a/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py +++ b/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py @@ -35,13 +35,14 @@ def __init__(self): self.max_sample_weight = None self.round_decimal = None self.new_ver = True + self.feature_importances_ = {} # for fast hist self.sparse_opt_para = False self.run_sparse_opt = False self.has_transformed_data = False self.data_bin_dense = None - self.predict_transfer_inst = HeteroSecureBoostTransferVariable() + self.hetero_sbt_transfer_variable = HeteroSecureBoostTransferVariable() def _init_model(self, param: HeteroSecureBoostParam): @@ -79,6 +80,22 @@ def sparse_to_array(data, feature_sparse_point_array, use_missing, zero_as_missi new_data.features = sp.csc_matrix(np.array(new_feature_sparse_point_array) + offset) return new_data + def update_feature_importance(self, tree_feature_importance): + for fid in tree_feature_importance: + if fid not in self.feature_importances_: + self.feature_importances_[fid] = tree_feature_importance[fid] + else: + self.feature_importances_[fid] += tree_feature_importance[fid] + LOGGER.debug('cur feature importance {}'.format(self.feature_importances_)) + + def sync_feature_importance(self): + # generate anonymous + new_feat_importance = {} + sitename = 'host:' + str(self.component_properties.local_partyid) + for key in self.feature_importances_: + new_feat_importance[(sitename, key)] = self.feature_importances_[key] + self.hetero_sbt_transfer_variable.host_feature_importance.remote(new_feat_importance) + def fit_a_booster(self, epoch_idx: int, booster_dim: int): tree = HeteroDecisionTreeHost(tree_param=self.tree_param) @@ -94,6 +111,8 @@ def fit_a_booster(self, epoch_idx: int, booster_dim: int): new_ver=self.new_ver ) tree.fit() + self.update_feature_importance(tree.get_feature_importance()) + return tree def load_booster(self, model_meta, model_param, epoch_idx, booster_idx): @@ -146,16 +165,16 @@ def boosting_fast_predict(self, data_inst, trees: List[HeteroDecisionTreeHost]): LOGGER.debug('cur predict round is {}'.format(comm_round)) - stop_flag = self.predict_transfer_inst.predict_stop_flag.get(idx=0, suffix=(comm_round, )) + stop_flag = self.hetero_sbt_transfer_variable.predict_stop_flag.get(idx=0, suffix=(comm_round,)) if stop_flag: break - guest_node_pos = self.predict_transfer_inst.guest_predict_data.get(idx=0, suffix=(comm_round, )) + guest_node_pos = self.hetero_sbt_transfer_variable.guest_predict_data.get(idx=0, suffix=(comm_round,)) host_node_pos = guest_node_pos.join(data_inst, traverse_func) if guest_node_pos.count() != host_node_pos.count(): raise ValueError('sample count mismatch: guest table {}, host table {}'.format(guest_node_pos.count(), host_node_pos.count())) - self.predict_transfer_inst.host_predict_data.remote(host_node_pos, idx=-1, suffix=(comm_round, )) + self.hetero_sbt_transfer_variable.host_predict_data.remote(host_node_pos, idx=-1, suffix=(comm_round,)) comm_round += 1 @@ -253,29 +272,33 @@ def EINI_host_predict(self, data_inst, trees: List[HeteroDecisionTreeHost], site node_pos_map_list=id_pos_map_list) position_vec = data_inst.mapValues(map_func) - booster_dim = self.predict_transfer_inst.guest_predict_data.get(idx=0, suffix='booster_dim') + booster_dim = self.hetero_sbt_transfer_variable.guest_predict_data.get(idx=0, suffix='booster_dim') self_idx = party_list.index(self_party_id) if len(party_list) == 1: - guest_position_vec = self.predict_transfer_inst.guest_predict_data.get(idx=0, suffix='position_vec') + guest_position_vec = self.hetero_sbt_transfer_variable.guest_predict_data.get(idx=0, suffix='position_vec') leaf_idx_dim_map = self.generate_leaf_idx_dimension_map(trees, booster_dim) merge_func = functools.partial(self.merge_position_vec, booster_dim=booster_dim, leaf_idx_dim_map=leaf_idx_dim_map) result_table = position_vec.join(guest_position_vec, merge_func) - self.predict_transfer_inst.inter_host_data.remote(result_table, idx=self_idx + 1, suffix='position_vec', role=consts.HOST) + self.hetero_sbt_transfer_variable.inter_host_data.remote(result_table, idx=self_idx + 1, + suffix='position_vec', role=consts.HOST) else: # multi host case # if is first host party, get encrypt vec from guest, else from previous host party if self_party_id == party_list[0]: - guest_position_vec = self.predict_transfer_inst.guest_predict_data.get(idx=0, suffix='position_vec') + guest_position_vec = self.hetero_sbt_transfer_variable.guest_predict_data.get(idx=0, + suffix='position_vec') else: - guest_position_vec = self.predict_transfer_inst.inter_host_data.get(idx=self_idx - 1, suffix='position_vec') + guest_position_vec = self.hetero_sbt_transfer_variable.inter_host_data.get(idx=self_idx - 1, + suffix='position_vec') result_table = position_vec.join(guest_position_vec, self.position_vec_element_wise_mul) if self_party_id == party_list[-1]: - self.predict_transfer_inst.host_predict_data.remote(result_table, suffix='merge_result') + self.hetero_sbt_transfer_variable.host_predict_data.remote(result_table, suffix='merge_result') else: - self.predict_transfer_inst.inter_host_data.remote(result_table, idx=self_idx + 1, suffix='position_vec', - role=consts.HOST) + self.hetero_sbt_transfer_variable.inter_host_data.remote(result_table, idx=self_idx + 1, + suffix='position_vec', + role=consts.HOST) @assert_io_num_rows_equal def predict(self, data_inst): diff --git a/python/federatedml/transfer_variable/transfer_class/hetero_secure_boosting_predict_transfer_variable.py b/python/federatedml/transfer_variable/transfer_class/hetero_secure_boosting_predict_transfer_variable.py index 4ca419f95c..feb280bee5 100644 --- a/python/federatedml/transfer_variable/transfer_class/hetero_secure_boosting_predict_transfer_variable.py +++ b/python/federatedml/transfer_variable/transfer_class/hetero_secure_boosting_predict_transfer_variable.py @@ -34,3 +34,4 @@ def __init__(self, flowid=0): self.guest_predict_data = self._create_variable(name='guest_predict_data', src=['guest'], dst=['host']) self.host_predict_data = self._create_variable(name='host_predict_data', src=['host'], dst=['guest']) self.inter_host_data = self._create_variable(name='inter_host_data', src=['host'], dst=['host']) + self.host_feature_importance = self._create_variable(name='host_feature_importance', src=['host'], dst=['guest']) From 1fa80b534737accff58791ce814f3c9f6cda3d90 Mon Sep 17 00:00:00 2001 From: weijingchen Date: Fri, 11 Feb 2022 17:43:54 +0800 Subject: [PATCH 20/99] EINI test codes Signed-off-by: cwj --- .../ensemble/boosting/hetero/hetero_secureboost_guest.py | 4 ++++ .../ensemble/boosting/hetero/hetero_secureboost_host.py | 3 +++ 2 files changed, 7 insertions(+) diff --git a/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_guest.py b/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_guest.py index cd95a89344..096f787fac 100644 --- a/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_guest.py +++ b/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_guest.py @@ -501,6 +501,10 @@ def predict(self, data_inst, ret_format='std'): predict_rs = self.boosting_fast_predict(processed_data, trees=trees, predict_cache=predict_cache, pred_leaf=(ret_format == 'leaf')) + sitename = self.role + ':' + str(self.component_properties.local_partyid) + predict_rs_2 = self.EINI_guest_predict(processed_data, trees, self.learning_rate, self.init_score, + self.booster_dim, sitename, self.component_properties.host_party_idlist, + predict_cache, False) if ret_format == 'leaf': return predict_rs # predict result is leaf position diff --git a/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py b/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py index 4631a4718d..670cec82d1 100644 --- a/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py +++ b/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py @@ -323,6 +323,9 @@ def predict(self, data_inst): return self.boosting_fast_predict(processed_data, trees=trees) + sitename = self.role + ':' + str(self.component_properties.local_partyid) + self.EINI_host_predict(processed_data, trees, sitename, self.component_properties.local_partyid, + self.component_properties.host_party_idlist) def get_model_meta(self): model_meta = BoostingTreeModelMeta() From 220309bc87a6ead26b3164c0ec13af4982ec1300 Mon Sep 17 00:00:00 2001 From: mgqa34 Date: Fri, 11 Feb 2022 19:42:38 +0800 Subject: [PATCH 21/99] fix batch sync block when batch size equals to data size Signed-off-by: mgqa34 --- .../framework/hetero/procedure/batch_generator.py | 3 ++- python/federatedml/model_selection/mini_batch.py | 11 +++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/python/federatedml/framework/hetero/procedure/batch_generator.py b/python/federatedml/framework/hetero/procedure/batch_generator.py index 8fcf88909f..aa71fa5d4b 100644 --- a/python/federatedml/framework/hetero/procedure/batch_generator.py +++ b/python/federatedml/framework/hetero/procedure/batch_generator.py @@ -39,7 +39,8 @@ def initialize_batch_generator(self, data_instances, batch_size, suffix=tuple(), batch_strategy=batch_strategy, masked_rate=masked_rate) self.batch_nums = self.mini_batch_obj.batch_nums self.batch_masked = self.mini_batch_obj.batch_masked - batch_info = {"batch_size": batch_size, "batch_num": self.batch_nums, "batch_mutable": True, "batch_masked": self.batch_masked} + batch_info = {"batch_size": batch_size, "batch_num": + self.batch_nums, "batch_mutable": self.mini_batch_obj.batch_mutable, "batch_masked": self.batch_masked} self.sync_batch_info(batch_info, suffix) if not self.mini_batch_obj.batch_mutable: diff --git a/python/federatedml/model_selection/mini_batch.py b/python/federatedml/model_selection/mini_batch.py index 684e3aa800..305a9b93f1 100644 --- a/python/federatedml/model_selection/mini_batch.py +++ b/python/federatedml/model_selection/mini_batch.py @@ -91,10 +91,9 @@ def get_batch_generator(data_size, batch_size, batch_strategy, masked_rate, shuf LOGGER.warning("As batch_size >= data size, all batch strategy will be disabled") return FullBatchDataGenerator(data_size, data_size, shuffle=False) - if round((masked_rate + 1) * batch_size) >= data_size: - LOGGER.warning("Masked dataset's batch_size >= data size, all batch strategy will be disabled") - return FullBatchDataGenerator(data_size, data_size, shuffle=False, masked_rate=0) - + # if round((masked_rate + 1) * batch_size) >= data_size: + # LOGGER.warning("Masked dataset's batch_size >= data size, batch shuffle will be disabled") + # return FullBatchDataGenerator(data_size, data_size, shuffle=False, masked_rate=masked_rate) if batch_strategy == "full": return FullBatchDataGenerator(data_size, batch_size, shuffle=shuffle, masked_rate=masked_rate) else: @@ -106,7 +105,7 @@ def get_batch_generator(data_size, batch_size, batch_strategy, masked_rate, shuf class FullBatchDataGenerator(object): def __init__(self, data_size, batch_size, shuffle=False, masked_rate=0): self.batch_nums = (data_size + batch_size - 1) // batch_size - self.masked_dataset_size = round((1 + masked_rate) * self.batch_nums) + self.masked_dataset_size = min(data_size, round((1 + masked_rate) * self.batch_nums)) self.batch_size = batch_size self.shuffle = shuffle @@ -161,7 +160,7 @@ class RandomBatchDataGenerator(object): def __init__(self, batch_size, masked_rate=0): self.batch_nums = 1 self.batch_size = batch_size - self.masked_dataset_size = round((1 + masked_rate) * self.batch_size) + self.masked_dataset_size = min(batch_size, round((1 + masked_rate) * self.batch_size)) def generate_data(self, data_insts, *args, **kwargs): if self.masked_dataset_size == self.batch_size: From 1a25da00cde7b44306ac645bc9679ad156b91b52 Mon Sep 17 00:00:00 2001 From: weijingchen Date: Mon, 14 Feb 2022 10:48:48 +0800 Subject: [PATCH 22/99] fix bug Signed-off-by: cwj --- python/federatedml/components/hetero_secure_boost.py | 4 ++-- .../decision_tree/hetero/hetero_decision_tree_host.py | 5 +---- .../decision_tree/hetero/hetero_fast_decision_tree_host.py | 2 +- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/python/federatedml/components/hetero_secure_boost.py b/python/federatedml/components/hetero_secure_boost.py index b33787dfd9..5dbfa35ecd 100644 --- a/python/federatedml/components/hetero_secure_boost.py +++ b/python/federatedml/components/hetero_secure_boost.py @@ -29,7 +29,7 @@ def hetero_secure_boost_param(): @hetero_secure_boost_cpn_meta.bind_runner.on_guest def hetero_secure_boost_guest_runner(): - from federatedml.ensemble.boosting.hetero.hetero_fast_secureboost_guest import ( + from federatedml.ensemble.boosting.hetero.hetero_secureboost_guest import ( HeteroSecureBoostingTreeGuest, ) @@ -38,7 +38,7 @@ def hetero_secure_boost_guest_runner(): @hetero_secure_boost_cpn_meta.bind_runner.on_host def hetero_secure_boost_host_runner(): - from federatedml.ensemble.boosting.hetero.hetero_fast_secureboost_host import ( + from federatedml.ensemble.boosting.hetero.hetero_secureboost_host import ( HeteroSecureBoostingTreeHost, ) diff --git a/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_decision_tree_host.py b/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_decision_tree_host.py index ee6a68eb74..4878effb28 100644 --- a/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_decision_tree_host.py +++ b/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_decision_tree_host.py @@ -50,9 +50,6 @@ def __init__(self, tree_param): # code version control self.new_ver = True - # multi mode - self.mo_tree = False - """ Setting """ @@ -209,7 +206,7 @@ def init_compressor_and_sync_gh(self): LOGGER.info("get encrypted grad and hess") if self.run_cipher_compressing: - self.cipher_compressor = PackedGHCompressor(mo_mode=self.mo_tree) + self.cipher_compressor = PackedGHCompressor() self.grad_and_hess = self.transfer_inst.encrypted_grad_and_hess.get(idx=0) diff --git a/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_fast_decision_tree_host.py b/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_fast_decision_tree_host.py index 6002f79492..98aed25e6a 100644 --- a/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_fast_decision_tree_host.py +++ b/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_fast_decision_tree_host.py @@ -2,7 +2,7 @@ import functools import copy from federatedml.ensemble.basic_algorithms import HeteroDecisionTreeHost -from federatedml.ensemble.basic_algorithms.decision_tree.tree_core import tree_plan as plan +from federatedml.ensemble.boosting.hetero import hetero_fast_secureboost_plan as plan from federatedml.util import consts from federatedml.ensemble.basic_algorithms.decision_tree.tree_core.splitter import SplitInfo from federatedml.ensemble.basic_algorithms.decision_tree.tree_core.node import Node From 3a8d4f97584dc8ecde2651171d42b291d19cffaa Mon Sep 17 00:00:00 2001 From: weijingchen Date: Mon, 14 Feb 2022 16:10:23 +0800 Subject: [PATCH 23/99] fix EINI bug Signed-off-by: cwj --- .../boosting/hetero/hetero_secureboost_guest.py | 13 ++++++------- .../boosting/hetero/hetero_secureboost_host.py | 10 +++++++--- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_guest.py b/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_guest.py index 096f787fac..e372d61bd7 100644 --- a/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_guest.py +++ b/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_guest.py @@ -443,19 +443,18 @@ def EINI_guest_predict(self, data_inst, trees: List[HeteroDecisionTreeGuest], le # federation part self.hetero_sbt_transfer_variable.guest_predict_data.remote(booster_dim, idx=-1, suffix='booster_dim') # send to first host party - self.hetero_sbt_transfer_variable.guest_predict_data.remote(encrypter_vec_table, idx=0, suffix='position_vec', role=consts.HOST) + self.hetero_sbt_transfer_variable.guest_predict_data.remote(encrypter_vec_table, idx=0, suffix='position_vec', + role=consts.HOST) # get from last host party - result_table = self.hetero_sbt_transfer_variable.host_predict_data.get(idx=len(party_list) - 1, suffix='merge_result', + result_table = self.hetero_sbt_transfer_variable.host_predict_data.get(idx=len(party_list) - 1, + suffix='merge_result', role=consts.HOST) # decode result result = result_table.mapValues(encrypter.recursive_decrypt) - # result = result_table - if booster_dim == 1: - result = result.mapValues(lambda x: x[0]) - else: - result = result.mapValues(lambda x: np.array(x)) + # reformat + result = result.mapValues(lambda x: np.array(x)) if predict_cache: result = result.join(predict_cache, lambda v1, v2: v1 + v2) diff --git a/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py b/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py index 670cec82d1..3f21e08565 100644 --- a/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py +++ b/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py @@ -281,8 +281,7 @@ def EINI_host_predict(self, data_inst, trees: List[HeteroDecisionTreeHost], site merge_func = functools.partial(self.merge_position_vec, booster_dim=booster_dim, leaf_idx_dim_map=leaf_idx_dim_map) result_table = position_vec.join(guest_position_vec, merge_func) - self.hetero_sbt_transfer_variable.inter_host_data.remote(result_table, idx=self_idx + 1, - suffix='position_vec', role=consts.HOST) + self.hetero_sbt_transfer_variable.host_predict_data.remote(result_table, suffix='merge_result') else: # multi host case # if is first host party, get encrypt vec from guest, else from previous host party @@ -292,10 +291,15 @@ def EINI_host_predict(self, data_inst, trees: List[HeteroDecisionTreeHost], site else: guest_position_vec = self.hetero_sbt_transfer_variable.inter_host_data.get(idx=self_idx - 1, suffix='position_vec') - result_table = position_vec.join(guest_position_vec, self.position_vec_element_wise_mul) + if self_party_id == party_list[-1]: + leaf_idx_dim_map = self.generate_leaf_idx_dimension_map(trees, booster_dim) + func = functools.partial(self.merge_position_vec, booster_dim=booster_dim, + leaf_idx_dim_map=leaf_idx_dim_map) + result_table = position_vec.join(guest_position_vec, func) self.hetero_sbt_transfer_variable.host_predict_data.remote(result_table, suffix='merge_result') else: + result_table = position_vec.join(guest_position_vec, self.position_vec_element_wise_mul) self.hetero_sbt_transfer_variable.inter_host_data.remote(result_table, idx=self_idx + 1, suffix='position_vec', role=consts.HOST) From 5deea8fddaaa55d596275d600ee698527236a924 Mon Sep 17 00:00:00 2001 From: weijingchen Date: Mon, 14 Feb 2022 16:12:09 +0800 Subject: [PATCH 24/99] feature importance update Signed-off-by: cwj --- .../ensemble/boosting/hetero/hetero_secureboost_guest.py | 7 ------- .../ensemble/boosting/hetero/hetero_secureboost_host.py | 8 -------- 2 files changed, 15 deletions(-) diff --git a/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_guest.py b/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_guest.py index e372d61bd7..b62ad237a3 100644 --- a/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_guest.py +++ b/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_guest.py @@ -111,13 +111,6 @@ def update_feature_importance(self, tree_feature_importance): self.feature_importances_[fid] += tree_feature_importance[fid] LOGGER.debug('cur feature importance {}'.format(self.feature_importances_)) - def sync_feature_importance(self): - host_feature_importance_list = self.hetero_sbt_transfer_variable.host_feature_importance.get(idx=-1) - for i in host_feature_importance_list: - self.feature_importances_.update(i) - - LOGGER.debug('self feature importance is {}'.format(self.feature_importances_)) - def fit_a_booster(self, epoch_idx: int, booster_dim: int): if self.cur_epoch_idx != epoch_idx: diff --git a/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py b/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py index 3f21e08565..a26c4cf0e0 100644 --- a/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py +++ b/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py @@ -88,14 +88,6 @@ def update_feature_importance(self, tree_feature_importance): self.feature_importances_[fid] += tree_feature_importance[fid] LOGGER.debug('cur feature importance {}'.format(self.feature_importances_)) - def sync_feature_importance(self): - # generate anonymous - new_feat_importance = {} - sitename = 'host:' + str(self.component_properties.local_partyid) - for key in self.feature_importances_: - new_feat_importance[(sitename, key)] = self.feature_importances_[key] - self.hetero_sbt_transfer_variable.host_feature_importance.remote(new_feat_importance) - def fit_a_booster(self, epoch_idx: int, booster_dim: int): tree = HeteroDecisionTreeHost(tree_param=self.tree_param) From d79fd691ef835982bfcaf19ee6e23cbb74b56a06 Mon Sep 17 00:00:00 2001 From: weijingchen Date: Mon, 14 Feb 2022 16:44:22 +0800 Subject: [PATCH 25/99] EINI predict update Signed-off-by: cwj --- .../boosting/hetero/hetero_secureboost_host.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py b/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py index a26c4cf0e0..1d69fedeb8 100644 --- a/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py +++ b/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py @@ -223,7 +223,8 @@ def generate_leaf_idx_dimension_map(trees, booster_dim): return leaf_dim_map @staticmethod - def merge_position_vec(host_vec, guest_encrypt_vec, booster_dim=1, leaf_idx_dim_map=None): + def merge_position_vec(host_vec, guest_encrypt_vec, booster_dim=1, leaf_idx_dim_map=None, confusion_number=None): + leaf_idx = -1 rs = [0 for i in range(booster_dim)] for en_num, vec_value in zip(guest_encrypt_vec, host_vec): @@ -233,6 +234,11 @@ def merge_position_vec(host_vec, guest_encrypt_vec, booster_dim=1, leaf_idx_dim_ else: dim = leaf_idx_dim_map[leaf_idx] rs[dim] += en_num + + if confusion_number: + for i in range(len(rs)): + rs[i] = rs[i] * confusion_number # a pos random mask btw 1 and 2 + return rs @staticmethod @@ -244,8 +250,8 @@ def position_vec_element_wise_mul(guest_encrypt_vec, host_vec): @staticmethod def get_leaf_idx_map(trees): - id_pos_map_list = [] + id_pos_map_list = [] for tree in trees: array_idx = 0 id_pos_map = {} @@ -266,12 +272,14 @@ def EINI_host_predict(self, data_inst, trees: List[HeteroDecisionTreeHost], site booster_dim = self.hetero_sbt_transfer_variable.guest_predict_data.get(idx=0, suffix='booster_dim') + random_mask = float(np.random.random()) + 1 # generate a random mask btw 1 and 2 + self_idx = party_list.index(self_party_id) if len(party_list) == 1: guest_position_vec = self.hetero_sbt_transfer_variable.guest_predict_data.get(idx=0, suffix='position_vec') leaf_idx_dim_map = self.generate_leaf_idx_dimension_map(trees, booster_dim) merge_func = functools.partial(self.merge_position_vec, booster_dim=booster_dim, - leaf_idx_dim_map=leaf_idx_dim_map) + leaf_idx_dim_map=leaf_idx_dim_map, random_mask=random_mask) result_table = position_vec.join(guest_position_vec, merge_func) self.hetero_sbt_transfer_variable.host_predict_data.remote(result_table, suffix='merge_result') else: @@ -287,7 +295,7 @@ def EINI_host_predict(self, data_inst, trees: List[HeteroDecisionTreeHost], site if self_party_id == party_list[-1]: leaf_idx_dim_map = self.generate_leaf_idx_dimension_map(trees, booster_dim) func = functools.partial(self.merge_position_vec, booster_dim=booster_dim, - leaf_idx_dim_map=leaf_idx_dim_map) + leaf_idx_dim_map=leaf_idx_dim_map, random_mask=random_mask) result_table = position_vec.join(guest_position_vec, func) self.hetero_sbt_transfer_variable.host_predict_data.remote(result_table, suffix='merge_result') else: From 9729b659cf56b186e9f39c399d035dbec3eb8e85 Mon Sep 17 00:00:00 2001 From: weijingchen Date: Tue, 15 Feb 2022 11:05:28 +0800 Subject: [PATCH 26/99] update EINI param Signed-off-by: cwj --- .../hetero/hetero_secureboost_guest.py | 24 ++++++++++------ .../hetero/hetero_secureboost_host.py | 28 +++++++++++++------ python/federatedml/param/boosting_param.py | 17 +++++++---- 3 files changed, 46 insertions(+), 23 deletions(-) diff --git a/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_guest.py b/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_guest.py index b62ad237a3..1eb5203d8c 100644 --- a/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_guest.py +++ b/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_guest.py @@ -47,6 +47,10 @@ def __init__(self): self.other_rate = None self.new_ver = True + # EINI predict param + self.EINI_inference = False + self.EINI_random_mask = False + def _init_model(self, param: HeteroSecureBoostParam): super(HeteroSecureBoostingTreeGuest, self)._init_model(param) @@ -59,6 +63,8 @@ def _init_model(self, param: HeteroSecureBoostParam): self.other_rate = param.other_rate self.cipher_compressing = param.cipher_compress self.new_ver = param.new_ver + self.EINI_inference = param.EINI_inference + self.EINI_random_mask = param.EINI_random_mask if self.use_missing: self.tree_param.use_missing = self.use_missing @@ -442,13 +448,10 @@ def EINI_guest_predict(self, data_inst, trees: List[HeteroDecisionTreeGuest], le result_table = self.hetero_sbt_transfer_variable.host_predict_data.get(idx=len(party_list) - 1, suffix='merge_result', role=consts.HOST) - # decode result result = result_table.mapValues(encrypter.recursive_decrypt) - # reformat result = result.mapValues(lambda x: np.array(x)) - if predict_cache: result = result.join(predict_cache, lambda v1, v2: v1 + v2) @@ -491,12 +494,15 @@ def predict(self, data_inst, ret_format='std'): if tree_num == 0 and predict_cache is not None and not (ret_format == 'leaf'): return self.score_to_predict_result(data_inst, predict_cache) - predict_rs = self.boosting_fast_predict(processed_data, trees=trees, predict_cache=predict_cache, - pred_leaf=(ret_format == 'leaf')) - sitename = self.role + ':' + str(self.component_properties.local_partyid) - predict_rs_2 = self.EINI_guest_predict(processed_data, trees, self.learning_rate, self.init_score, - self.booster_dim, sitename, self.component_properties.host_party_idlist, - predict_cache, False) + if self.EINI_inference: + sitename = self.role + ':' + str(self.component_properties.local_partyid) + predict_rs = self.EINI_guest_predict(processed_data, trees, self.learning_rate, self.init_score, + self.booster_dim, sitename, + self.component_properties.host_party_idlist, + predict_cache, False) + else: + predict_rs = self.boosting_fast_predict(processed_data, trees=trees, predict_cache=predict_cache, + pred_leaf=(ret_format == 'leaf')) if ret_format == 'leaf': return predict_rs # predict result is leaf position diff --git a/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py b/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py index 1d69fedeb8..2e3e2e2075 100644 --- a/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py +++ b/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py @@ -1,6 +1,7 @@ from typing import List import functools import copy +import random import numpy as np from scipy import sparse as sp from federatedml.util import LOGGER @@ -44,6 +45,10 @@ def __init__(self): self.data_bin_dense = None self.hetero_sbt_transfer_variable = HeteroSecureBoostTransferVariable() + # EINI predict param + self.EINI_inference = False + self.EINI_random_mask = False + def _init_model(self, param: HeteroSecureBoostParam): super(HeteroSecureBoostingTreeHost, self)._init_model(param) @@ -55,6 +60,8 @@ def _init_model(self, param: HeteroSecureBoostParam): self.sparse_opt_para = param.sparse_optimization self.cipher_compressing = param.cipher_compress self.new_ver = param.new_ver + self.EINI_inference = param.EINI_inference + self.EINI_random_mask = param.EINI_random_mask if self.use_missing: self.tree_param.use_missing = self.use_missing @@ -223,7 +230,7 @@ def generate_leaf_idx_dimension_map(trees, booster_dim): return leaf_dim_map @staticmethod - def merge_position_vec(host_vec, guest_encrypt_vec, booster_dim=1, leaf_idx_dim_map=None, confusion_number=None): + def merge_position_vec(host_vec, guest_encrypt_vec, booster_dim=1, leaf_idx_dim_map=None, random_mask=None): leaf_idx = -1 rs = [0 for i in range(booster_dim)] @@ -235,9 +242,9 @@ def merge_position_vec(host_vec, guest_encrypt_vec, booster_dim=1, leaf_idx_dim_ dim = leaf_idx_dim_map[leaf_idx] rs[dim] += en_num - if confusion_number: + if random_mask: for i in range(len(rs)): - rs[i] = rs[i] * confusion_number # a pos random mask btw 1 and 2 + rs[i] = rs[i] * random_mask # a pos random mask btw 1 and 2 return rs @@ -263,7 +270,8 @@ def get_leaf_idx_map(trees): return id_pos_map_list - def EINI_host_predict(self, data_inst, trees: List[HeteroDecisionTreeHost], sitename, self_party_id, party_list): + def EINI_host_predict(self, data_inst, trees: List[HeteroDecisionTreeHost], sitename, self_party_id, party_list, + random_mask=False): id_pos_map_list = self.get_leaf_idx_map(trees) map_func = functools.partial(self.generate_leaf_candidates_host, sitename=sitename, trees=trees, @@ -272,7 +280,7 @@ def EINI_host_predict(self, data_inst, trees: List[HeteroDecisionTreeHost], site booster_dim = self.hetero_sbt_transfer_variable.guest_predict_data.get(idx=0, suffix='booster_dim') - random_mask = float(np.random.random()) + 1 # generate a random mask btw 1 and 2 + random_mask = random.SystemRandom().random() + 1 if random_mask else 0 # generate a random mask btw 1 and 2 self_idx = party_list.index(self_party_id) if len(party_list) == 1: @@ -326,10 +334,12 @@ def predict(self, data_inst): LOGGER.info('no tree for predicting, prediction done') return - self.boosting_fast_predict(processed_data, trees=trees) - sitename = self.role + ':' + str(self.component_properties.local_partyid) - self.EINI_host_predict(processed_data, trees, sitename, self.component_properties.local_partyid, - self.component_properties.host_party_idlist) + if self.EINI_inference: + sitename = self.role + ':' + str(self.component_properties.local_partyid) + self.EINI_host_predict(processed_data, trees, sitename, self.component_properties.local_partyid, + self.component_properties.host_party_idlist) + else: + self.boosting_fast_predict(processed_data, trees=trees) def get_model_meta(self): model_meta = BoostingTreeModelMeta() diff --git a/python/federatedml/param/boosting_param.py b/python/federatedml/param/boosting_param.py index f2b0fea254..e8036a0e63 100644 --- a/python/federatedml/param/boosting_param.py +++ b/python/federatedml/param/boosting_param.py @@ -509,7 +509,7 @@ def __init__(self, tree_param: DecisionTreeParam = DecisionTreeParam(), task_typ binning_error=consts.DEFAULT_RELATIVE_ERROR, sparse_optimization=False, run_goss=False, top_rate=0.2, other_rate=0.1, cipher_compress_error=None, cipher_compress=True, new_ver=True, - callback_param=CallbackParam()): + callback_param=CallbackParam(), EINI_inference=True, EINI_random_mask=False): super(HeteroSecureBoostParam, self).__init__(task_type, objective_param, learning_rate, num_trees, subsample_feature_rate, n_iter_no_change, tol, encrypt_param, @@ -530,6 +530,8 @@ def __init__(self, tree_param: DecisionTreeParam = DecisionTreeParam(), task_typ self.cipher_compress_error = cipher_compress_error self.cipher_compress = cipher_compress self.new_ver = new_ver + self.EINI_inference = EINI_inference + self.EINI_random_mask = EINI_random_mask self.callback_param = copy.deepcopy(callback_param) def check(self): @@ -548,6 +550,8 @@ def check(self): self.check_positive_number(self.top_rate, 'top_rate') self.check_boolean(self.new_ver, 'code version switcher') self.check_boolean(self.cipher_compress, 'cipher compress') + self.check_boolean(self.EINI_inference, 'eini inference') + self.check_boolean(self.EINI_random_mask, 'eini random mask') for p in ["early_stopping_rounds", "validation_freqs", "metrics", "use_first_metric_only"]: @@ -593,18 +597,19 @@ def __init__(self, tree_param: DecisionTreeParam = DecisionTreeParam(), task_typ complete_secure=False, tree_num_per_party=1, guest_depth=1, host_depth=1, work_mode='mix', metrics=None, sparse_optimization=False, random_seed=100, binning_error=consts.DEFAULT_RELATIVE_ERROR, cipher_compress_error=None, new_ver=True, run_goss=False, top_rate=0.2, other_rate=0.1, - cipher_compress=True, callback_param=CallbackParam()): + cipher_compress=True, callback_param=CallbackParam(), EINI_inference=True, EINI_random_mask=False): """ Parameters ---------- work_mode: {"mix", "layered"} - mix: alternate using guest/host features to build trees. For example, the first 'tree_num_per_party' trees use guest features, - the second k trees use host features, and so on + mix: alternate using guest/host features to build trees. For example, the first 'tree_num_per_party' trees + use guest features, the second k trees use host features, and so on layered: only support 2 party, when running layered mode, first 'host_depth' layer will use host features, and then next 'guest_depth' will only use guest features tree_num_per_party: int - every party will alternate build 'tree_num_per_party' trees until reach max tree num, this param is valid when work_mode is mix + every party will alternate build 'tree_num_per_party' trees until reach max tree num, this param is valid + when work_mode is mix guest_depth: int guest will build last guest_depth of a decision tree using guest features, is valid when work mode is layered host depth: int @@ -624,6 +629,8 @@ def __init__(self, tree_param: DecisionTreeParam = DecisionTreeParam(), task_typ new_ver=new_ver, cipher_compress=cipher_compress, run_goss=run_goss, top_rate=top_rate, other_rate=other_rate, + EINI_inference=EINI_inference, + EINI_random_mask=EINI_random_mask ) self.tree_num_per_party = tree_num_per_party From 3fbf6666757754d320df2629151b9e3687ca584b Mon Sep 17 00:00:00 2001 From: weijingchen Date: Tue, 15 Feb 2022 11:23:55 +0800 Subject: [PATCH 27/99] remove sbt transformer Signed-off-by: cwj --- .../dsl/v2/sbt_feature_transformer/README.md | 38 --- .../sbt_feat_transformer_testsuite.json | 50 ---- .../test_sbt_feat_transformer_conf_0.json | 98 ------- .../test_sbt_feat_transformer_conf_1.json | 102 ------- .../test_sbt_feat_transformer_conf_2.json | 66 ----- .../test_sbt_feat_transformer_dsl_0.json | 140 --------- .../test_sbt_feat_transformer_dsl_1.json | 161 ----------- .../test_sbt_feat_transformer_dsl_2.json | 84 ------ .../sbt_feature_transformer/README.md | 32 --- .../pipeline-sbt-transformer-fast-sbt.py | 137 --------- .../pipeline-sbt-transformer-multi.py | 124 -------- .../pipeline-sbt-transformer.py | 131 --------- .../sbt_transformer_testsuite.json | 47 --- .../component/sbt_feature_transformer.py | 34 --- .../param/sbt_feature_transformer_param.py | 16 -- .../components/sbt_feature_transformer.py | 45 --- .../sbt_feature_transformer.py | 267 ------------------ python/federatedml/param/__init__.py | 4 +- .../param/sbt_feature_transformer_param.py | 18 -- 19 files changed, 1 insertion(+), 1593 deletions(-) delete mode 100644 examples/dsl/v2/sbt_feature_transformer/README.md delete mode 100644 examples/dsl/v2/sbt_feature_transformer/sbt_feat_transformer_testsuite.json delete mode 100644 examples/dsl/v2/sbt_feature_transformer/test_sbt_feat_transformer_conf_0.json delete mode 100644 examples/dsl/v2/sbt_feature_transformer/test_sbt_feat_transformer_conf_1.json delete mode 100644 examples/dsl/v2/sbt_feature_transformer/test_sbt_feat_transformer_conf_2.json delete mode 100644 examples/dsl/v2/sbt_feature_transformer/test_sbt_feat_transformer_dsl_0.json delete mode 100644 examples/dsl/v2/sbt_feature_transformer/test_sbt_feat_transformer_dsl_1.json delete mode 100644 examples/dsl/v2/sbt_feature_transformer/test_sbt_feat_transformer_dsl_2.json delete mode 100644 examples/pipeline/sbt_feature_transformer/README.md delete mode 100644 examples/pipeline/sbt_feature_transformer/pipeline-sbt-transformer-fast-sbt.py delete mode 100644 examples/pipeline/sbt_feature_transformer/pipeline-sbt-transformer-multi.py delete mode 100644 examples/pipeline/sbt_feature_transformer/pipeline-sbt-transformer.py delete mode 100644 examples/pipeline/sbt_feature_transformer/sbt_transformer_testsuite.json delete mode 100644 python/fate_client/pipeline/component/sbt_feature_transformer.py delete mode 100644 python/fate_client/pipeline/param/sbt_feature_transformer_param.py delete mode 100644 python/federatedml/components/sbt_feature_transformer.py delete mode 100644 python/federatedml/feature/sbt_feature_transformer/sbt_feature_transformer.py delete mode 100644 python/federatedml/param/sbt_feature_transformer_param.py diff --git a/examples/dsl/v2/sbt_feature_transformer/README.md b/examples/dsl/v2/sbt_feature_transformer/README.md deleted file mode 100644 index 801b54de7b..0000000000 --- a/examples/dsl/v2/sbt_feature_transformer/README.md +++ /dev/null @@ -1,38 +0,0 @@ -## Sample Weight Configuration Usage Guide. - -#### Example Tasks - -This section introduces the dsl and conf for different types of tasks. - -1. Hetero SBT + SBT transformer: - - dsl: test_sbt_feat_transformer_dsl_0.json - - runtime_config : test_sbt_feat_transformer_conf_0.json - - An Hetero-SBT + SBT transformer, with local baseline comparison. - - -2. Hetero Fast SBT + SBT transformer: - - dsl: test_sbt_feat_transformer_dsl_1.json - - runtime_config : test_sbt_feat_transformer_conf_1.json - - Hetero Fast-SBT + SBT transformer, with local base line comparison and - transformer model loading. - -3. Hetero SBT(Multi) + SBT transformer: - - dsl: test_sbt_feat_transformer_dsl_2.json - - runtime_config : test_sbt_feat_transformer_conf_2.json - - Encode samples using multi-sbt - - -Users can use following commands to run the task. - - flow job submit -c ${runtime_config} -d ${dsl} - -After having finished a successful training task, you can use FATE Board to check output. \ No newline at end of file diff --git a/examples/dsl/v2/sbt_feature_transformer/sbt_feat_transformer_testsuite.json b/examples/dsl/v2/sbt_feature_transformer/sbt_feat_transformer_testsuite.json deleted file mode 100644 index d20d222584..0000000000 --- a/examples/dsl/v2/sbt_feature_transformer/sbt_feat_transformer_testsuite.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "data": [ - { - "file": "examples/data/breast_hetero_guest.csv", - "head": 1, - "partition": 4, - "table_name": "breast_hetero_guest", - "namespace": "experiment", - "role": "guest_0" - }, - { - "file": "examples/data/breast_hetero_host.csv", - "head": 1, - "partition": 4, - "table_name": "breast_hetero_host", - "namespace": "experiment", - "role": "host_0" - }, - { - "file": "examples/data/vehicle_scale_hetero_guest.csv", - "head": 1, - "partition": 4, - "table_name": "vehicle_scale_hetero_guest", - "namespace": "experiment", - "role": "guest_0" - }, - { - "file": "examples/data/vehicle_scale_hetero_host.csv", - "head": 1, - "partition": 4, - "table_name": "vehicle_scale_hetero_host", - "namespace": "experiment", - "role": "host_0" - } - ], - "tasks": { - "sbt_transform_and_baseline_compare": { - "conf": "./test_sbt_feat_transformer_conf_0.json", - "dsl": "./test_sbt_feat_transformer_dsl_0.json" - }, - "fast_sbt_transform_and_baseline_compare": { - "conf": "./test_sbt_feat_transformer_conf_1.json", - "dsl": "./test_sbt_feat_transformer_dsl_1.json" - }, - "multi_classification_sbt_transform": { - "conf": "./test_sbt_feat_transformer_conf_2.json", - "dsl": "./test_sbt_feat_transformer_dsl_2.json" - } - } -} \ No newline at end of file diff --git a/examples/dsl/v2/sbt_feature_transformer/test_sbt_feat_transformer_conf_0.json b/examples/dsl/v2/sbt_feature_transformer/test_sbt_feat_transformer_conf_0.json deleted file mode 100644 index 3b94d30548..0000000000 --- a/examples/dsl/v2/sbt_feature_transformer/test_sbt_feat_transformer_conf_0.json +++ /dev/null @@ -1,98 +0,0 @@ -{ - "dsl_version": 2, - "initiator": { - "role": "guest", - "party_id": 9999 - }, - "role": { - "host": [ - 9998 - ], - "guest": [ - 9999 - ] - }, - "component_parameters": { - "common": { - "secureboost_0": { - "task_type": "classification", - "objective_param": { - "objective": "cross_entropy" - }, - "num_trees": 3, - "validation_freqs": 1, - "encrypt_param": { - "method": "paillier" - }, - "tree_param": { - "max_depth": 3 - } - }, - "evaluation_0": { - "eval_type": "binary" - }, - "sbt_feature_transformer_0": { - "dense_format": true - } - }, - "role": { - "guest": { - "0": { - "reader_0": { - "table": { - "name": "breast_hetero_guest", - "namespace": "experiment" - } - }, - "data_transform_0": { - "with_label": true, - "output_format": "dense" - }, - "local_baseline_0": { - "model_name": "LogisticRegression", - "model_opts": { - "penalty": "l2", - "tol": 0.0001, - "C": 1.0, - "fit_intercept": true, - "solver": "lbfgs", - "max_iter": 50 - }, - "need_run": true - }, - "local_baseline_1": { - "model_name": "LogisticRegression", - "model_opts": { - "penalty": "l2", - "tol": 0.0001, - "C": 1.0, - "fit_intercept": true, - "solver": "lbfgs", - "max_iter": 50 - }, - "need_run": true - } - } - }, - "host": { - "0": { - "reader_0": { - "table": { - "name": "breast_hetero_host", - "namespace": "experiment" - } - }, - "data_transform_0": { - "with_label": false - }, - "local_baseline_0": { - "need_run": false - }, - "local_baseline_1": { - "need_run": false - } - } - } - } - } -} diff --git a/examples/dsl/v2/sbt_feature_transformer/test_sbt_feat_transformer_conf_1.json b/examples/dsl/v2/sbt_feature_transformer/test_sbt_feat_transformer_conf_1.json deleted file mode 100644 index fdfd95b0d0..0000000000 --- a/examples/dsl/v2/sbt_feature_transformer/test_sbt_feat_transformer_conf_1.json +++ /dev/null @@ -1,102 +0,0 @@ -{ - "dsl_version": 2, - "initiator": { - "role": "guest", - "party_id": 9999 - }, - "role": { - "host": [ - 9998 - ], - "guest": [ - 9999 - ] - }, - "component_parameters": { - "common": { - "fast_secureboost_0": { - "task_type": "classification", - "objective_param": { - "objective": "cross_entropy" - }, - "num_trees": 3, - "validation_freqs": 1, - "encrypt_param": { - "method": "paillier" - }, - "tree_param": { - "max_depth": 3 - }, - "work_mode": "mix" - }, - "evaluation_0": { - "eval_type": "binary" - }, - "sbt_feature_transformer_0": { - "dense_format": true - }, - "sbt_feature_transformer_1": { - "dense_format": false - } - }, - "role": { - "guest": { - "0": { - "reader_0": { - "table": { - "name": "breast_hetero_guest", - "namespace": "experiment" - } - }, - "data_transform_0": { - "with_label": true, - "output_format": "dense" - }, - "local_baseline_0": { - "model_name": "LogisticRegression", - "model_opts": { - "penalty": "l2", - "tol": 0.0001, - "C": 1.0, - "fit_intercept": true, - "solver": "lbfgs", - "max_iter": 50 - }, - "need_run": true - }, - "local_baseline_1": { - "model_name": "LogisticRegression", - "model_opts": { - "penalty": "l2", - "tol": 0.0001, - "C": 1.0, - "fit_intercept": true, - "solver": "lbfgs", - "max_iter": 50 - }, - "need_run": true - } - } - }, - "host": { - "0": { - "reader_0": { - "table": { - "name": "breast_hetero_host", - "namespace": "experiment" - } - }, - "data_transform_0": { - "with_label": false - }, - "local_baseline_0": { - "need_run": false - }, - "local_baseline_1": { - "need_run": false - } - } - } - } - } -} diff --git a/examples/dsl/v2/sbt_feature_transformer/test_sbt_feat_transformer_conf_2.json b/examples/dsl/v2/sbt_feature_transformer/test_sbt_feat_transformer_conf_2.json deleted file mode 100644 index 02e92ce241..0000000000 --- a/examples/dsl/v2/sbt_feature_transformer/test_sbt_feat_transformer_conf_2.json +++ /dev/null @@ -1,66 +0,0 @@ -{ - "dsl_version": 2, - "initiator": { - "role": "guest", - "party_id": 9999 - }, - "role": { - "host": [ - 9998 - ], - "guest": [ - 9999 - ] - }, - "component_parameters": { - "common": { - "secureboost_0": { - "task_type": "classification", - "objective_param": { - "objective": "cross_entropy" - }, - "num_trees": 2, - "validation_freqs": 1, - "random_seed": 1000, - "encrypt_param": { - "method": "paillier" - }, - "tree_param": { - "max_depth": 3 - } - }, - "sbt_feature_transformer_0": { - "dense_format": true - } - }, - "role": { - "guest": { - "0": { - "reader_0": { - "table": { - "name": "vehicle_scale_hetero_guest", - "namespace": "experiment" - } - }, - "data_transform_0": { - "with_label": true, - "output_format": "dense" - } - } - }, - "host": { - "0": { - "reader_0": { - "table": { - "name": "vehicle_scale_hetero_host", - "namespace": "experiment" - } - }, - "data_transform_0": { - "with_label": false - } - } - } - } - } -} diff --git a/examples/dsl/v2/sbt_feature_transformer/test_sbt_feat_transformer_dsl_0.json b/examples/dsl/v2/sbt_feature_transformer/test_sbt_feat_transformer_dsl_0.json deleted file mode 100644 index 9625a126f9..0000000000 --- a/examples/dsl/v2/sbt_feature_transformer/test_sbt_feat_transformer_dsl_0.json +++ /dev/null @@ -1,140 +0,0 @@ -{ - "components": { - "reader_0": { - "module": "Reader", - "output": { - "data": [ - "data" - ] - } - }, - "data_transform_0": { - "module": "DataTransform", - "input": { - "data": { - "data": [ - "reader_0.data" - ] - } - }, - "output": { - "data": [ - "train" - ], - "model": [ - "data_transform" - ] - } - }, - "intersection_0": { - "module": "Intersection", - "input": { - "data": { - "data": [ - "data_transform_0.train" - ] - } - }, - "output": { - "data": [ - "train" - ] - } - }, - "secureboost_0": { - "module": "HeteroSecureBoost", - "input": { - "data": { - "train_data": [ - "intersection_0.train" - ] - } - }, - "output": { - "data": [ - "train" - ], - "model": [ - "train" - ] - } - }, - "sbt_feature_transformer_0": { - "module": "SBTFeatureTransformer", - "input": { - "data": { - "data": [ - "intersection_0.train" - ] - }, - "isometric_model": [ - "secureboost_0.train" - ] - }, - "output": { - "data": [ - "train" - ], - "model": [ - "model" - ] - } - }, - "local_baseline_0": { - "module": "LocalBaseline", - "input": { - "data": { - "train_data": [ - "sbt_feature_transformer_0.train" - ] - } - }, - "output": { - "data": [ - "train" - ], - "model": [ - "local_baseline" - ] - } - }, - "local_baseline_1": { - "module": "LocalBaseline", - "input": { - "data": { - "train_data": [ - "intersection_0.train" - ] - } - }, - "output": { - "data": [ - "train" - ], - "model": [ - "local_baseline" - ] - } - }, - "evaluation_0": { - "module": "Evaluation", - "input": { - "data": { - "data": [ - "local_baseline_0.train" - ] - } - } - }, - "evaluation_1": { - "module": "Evaluation", - "input": { - "data": { - "data": [ - "local_baseline_1.train" - ] - } - } - } - } -} \ No newline at end of file diff --git a/examples/dsl/v2/sbt_feature_transformer/test_sbt_feat_transformer_dsl_1.json b/examples/dsl/v2/sbt_feature_transformer/test_sbt_feat_transformer_dsl_1.json deleted file mode 100644 index 9739417aca..0000000000 --- a/examples/dsl/v2/sbt_feature_transformer/test_sbt_feat_transformer_dsl_1.json +++ /dev/null @@ -1,161 +0,0 @@ -{ - "components": { - "reader_0": { - "module": "Reader", - "output": { - "data": [ - "data" - ] - } - }, - "data_transform_0": { - "module": "DataTransform", - "input": { - "data": { - "data": [ - "reader_0.data" - ] - } - }, - "output": { - "data": [ - "train" - ], - "model": [ - "data_transform" - ] - } - }, - "intersection_0": { - "module": "Intersection", - "input": { - "data": { - "data": [ - "data_transform_0.train" - ] - } - }, - "output": { - "data": [ - "train" - ] - } - }, - "fast_secureboost_0": { - "module": "HeteroFastSecureBoost", - "input": { - "data": { - "train_data": [ - "intersection_0.train" - ] - } - }, - "output": { - "data": [ - "train" - ], - "model": [ - "train" - ] - } - }, - "sbt_feature_transformer_0": { - "module": "SBTFeatureTransformer", - "input": { - "data": { - "data": [ - "intersection_0.train" - ] - }, - "isometric_model": [ - "fast_secureboost_0.train" - ] - }, - "output": { - "data": [ - "train" - ], - "model": [ - "model" - ] - } - }, - "sbt_feature_transformer_1": { - "module": "SBTFeatureTransformer", - "input": { - "data": { - "data": [ - "intersection_0.train" - ] - }, - "model": [ - "sbt_feature_transformer_0.model" - ] - }, - "output": { - "data": [ - "train" - ], - "model": [ - "model" - ] - } - }, - "local_baseline_0": { - "module": "LocalBaseline", - "input": { - "data": { - "train_data": [ - "sbt_feature_transformer_0.train" - ] - } - }, - "output": { - "data": [ - "train" - ], - "model": [ - "local_baseline" - ] - } - }, - "local_baseline_1": { - "module": "LocalBaseline", - "input": { - "data": { - "train_data": [ - "intersection_0.train" - ] - } - }, - "output": { - "data": [ - "train" - ], - "model": [ - "local_baseline" - ] - } - }, - "evaluation_0": { - "module": "Evaluation", - "input": { - "data": { - "data": [ - "local_baseline_0.train" - ] - } - } - }, - "evaluation_1": { - "module": "Evaluation", - "input": { - "data": { - "data": [ - "local_baseline_1.train" - ] - } - } - } - } -} \ No newline at end of file diff --git a/examples/dsl/v2/sbt_feature_transformer/test_sbt_feat_transformer_dsl_2.json b/examples/dsl/v2/sbt_feature_transformer/test_sbt_feat_transformer_dsl_2.json deleted file mode 100644 index 68bc8c92d0..0000000000 --- a/examples/dsl/v2/sbt_feature_transformer/test_sbt_feat_transformer_dsl_2.json +++ /dev/null @@ -1,84 +0,0 @@ -{ - "components": { - "reader_0": { - "module": "Reader", - "output": { - "data": [ - "data" - ] - } - }, - "data_transform_0": { - "module": "DataTransform", - "input": { - "data": { - "data": [ - "reader_0.data" - ] - } - }, - "output": { - "data": [ - "train" - ], - "model": [ - "data_transform" - ] - } - }, - "intersection_0": { - "module": "Intersection", - "input": { - "data": { - "data": [ - "data_transform_0.train" - ] - } - }, - "output": { - "data": [ - "train" - ] - } - }, - "secureboost_0": { - "module": "HeteroSecureBoost", - "input": { - "data": { - "train_data": [ - "intersection_0.train" - ] - } - }, - "output": { - "data": [ - "train" - ], - "model": [ - "train" - ] - } - }, - "sbt_feature_transformer_0": { - "module": "SBTFeatureTransformer", - "input": { - "data": { - "data": [ - "intersection_0.train" - ] - }, - "isometric_model": [ - "secureboost_0.train" - ] - }, - "output": { - "data": [ - "train" - ], - "model": [ - "model" - ] - } - } - } -} \ No newline at end of file diff --git a/examples/pipeline/sbt_feature_transformer/README.md b/examples/pipeline/sbt_feature_transformer/README.md deleted file mode 100644 index b193c39e01..0000000000 --- a/examples/pipeline/sbt_feature_transformer/README.md +++ /dev/null @@ -1,32 +0,0 @@ -## Sample Weight Configuration Usage Guide. - -#### Example Tasks - -This section introduces the dsl and conf for different types of tasks. - -1. Hetero SBT + SBT transformer: - - script: pipeline-sbt-transformer.py - - An Hetero-SBT + SBT transformer, with local baseline comparison. - - -2. Hetero Fast SBT + SBT transformer: - - script: pipeline-sbt-transformer-fast-sbt.py - - Hetero Fast-SBT + SBT transformer, with local base line comparison and - transformer model loading. - -3. Hetero SBT(Multi) + SBT transformer: - - script: pipeline-sbt-transformer-multi.py - - Encode samples using multi-sbt - - -Users can use following commands to run pipeline job directly. - - python ${pipeline_script} - -After having finished a successful training task, you can use FATE Board to check output. \ No newline at end of file diff --git a/examples/pipeline/sbt_feature_transformer/pipeline-sbt-transformer-fast-sbt.py b/examples/pipeline/sbt_feature_transformer/pipeline-sbt-transformer-fast-sbt.py deleted file mode 100644 index 6f4a45679c..0000000000 --- a/examples/pipeline/sbt_feature_transformer/pipeline-sbt-transformer-fast-sbt.py +++ /dev/null @@ -1,137 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import argparse - -from pipeline.backend.pipeline import PipeLine -from pipeline.component import DataTransform -from pipeline.component import HeteroFastSecureBoost -from pipeline.component import Intersection -from pipeline.component import SBTTransformer -from pipeline.component import LocalBaseline -from pipeline.component import Reader -from pipeline.interface import Data -from pipeline.component import Evaluation -from pipeline.interface import Model - -from pipeline.utils.tools import load_job_config - - -def main(config="../../config.yaml", namespace=""): - # obtain config - if isinstance(config, str): - config = load_job_config(config) - parties = config.parties - guest = parties.guest[0] - host = parties.host[0] - - - # data sets - guest_train_data = {"name": "breast_hetero_guest", "namespace": f"experiment{namespace}"} - host_train_data = {"name": "breast_hetero_host", "namespace": f"experiment{namespace}"} - - guest_validate_data = {"name": "breast_hetero_guest", "namespace": f"experiment{namespace}"} - host_validate_data = {"name": "breast_hetero_host", "namespace": f"experiment{namespace}"} - - # init pipeline - pipeline = PipeLine().set_initiator(role="guest", party_id=guest).set_roles(guest=guest, host=host,) - - # set data reader and data-io - reader_0, reader_1 = Reader(name="reader_0"), Reader(name="reader_1") - reader_0.get_party_instance(role="guest", party_id=guest).component_param(table=guest_train_data) - reader_0.get_party_instance(role="host", party_id=host).component_param(table=host_train_data) - reader_1.get_party_instance(role="guest", party_id=guest).component_param(table=guest_validate_data) - reader_1.get_party_instance(role="host", party_id=host).component_param(table=host_validate_data) - - data_transform_0, data_transform_1 = DataTransform(name="data_transform_0"), DataTransform(name="data_transform_1") - - data_transform_0.get_party_instance(role="guest", party_id=guest).component_param(with_label=True, output_format="dense") - data_transform_0.get_party_instance(role="host", party_id=host).component_param(with_label=False) - data_transform_1.get_party_instance(role="guest", party_id=guest).component_param(with_label=True, output_format="dense") - data_transform_1.get_party_instance(role="host", party_id=host).component_param(with_label=False) - - # data intersect component - intersect_0 = Intersection(name="intersection_0") - intersect_1 = Intersection(name="intersection_1") - - # secure boost component - hetero_fast_sbt_0 = HeteroFastSecureBoost(name="hetero_fast_sbt_0", - num_trees=3, - task_type="classification", - objective_param={"objective": "cross_entropy"}, - encrypt_param={"method": "paillier"}, - tree_param={"max_depth": 3}, - validation_freqs=1, - work_mode='mix', - tree_num_per_party=1 - ) - - # evaluation component - evaluation_0 = Evaluation(name="evaluation_0", eval_type="binary") - evaluation_1 = Evaluation(name="evaluation_1", eval_type="binary") - - # transformer - transformer_0 = SBTTransformer(name='sbt_transformer_0', dense_format=True) - transformer_1 = SBTTransformer(name='sbt_transformer_1', dense_format=True) - - # local baseline - def get_local_baseline(idx): - return LocalBaseline(name="local_baseline_{}".format(idx), model_name="LogisticRegression", - model_opts={"penalty": "l2", "tol": 0.0001, "C": 1.0, "fit_intercept": True, - "solver": "lbfgs", "max_iter": 50}) - - local_baseline_0 = get_local_baseline(0) - local_baseline_0.get_party_instance(role='guest', party_id=guest).component_param(need_run=True) - local_baseline_0.get_party_instance(role='host', party_id=host).component_param(need_run=False) - - local_baseline_1 = get_local_baseline(1) - local_baseline_1.get_party_instance(role='guest', party_id=guest).component_param(need_run=True) - local_baseline_1.get_party_instance(role='host', party_id=host).component_param(need_run=False) - - evaluation_1.get_party_instance(role='host', party_id=host).component_param(need_run=False) - - pipeline.add_component(reader_0) - pipeline.add_component(reader_1) - pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data)) - pipeline.add_component(data_transform_1, data=Data(data=reader_1.output.data), model=Model(data_transform_0.output.model)) - pipeline.add_component(intersect_0, data=Data(data=data_transform_0.output.data)) - pipeline.add_component(intersect_1, data=Data(data=data_transform_1.output.data)) - pipeline.add_component(hetero_fast_sbt_0, data=Data(train_data=intersect_0.output.data, - validate_data=intersect_1.output.data)) - pipeline.add_component(transformer_0, data=Data(data=intersect_0.output.data), - model=Model(isometric_model=hetero_fast_sbt_0.output.model)) - pipeline.add_component(transformer_1, data=Data(data=intersect_0.output.data), - model=Model(model=transformer_0.output.model)) - - pipeline.add_component(local_baseline_0, data=Data(data=transformer_0.output.data)) - pipeline.add_component(local_baseline_1, data=Data(data=intersect_0.output.data)) - - pipeline.add_component(evaluation_0, data=Data(data=local_baseline_0.output.data)) - pipeline.add_component(evaluation_1, data=Data(data=local_baseline_1.output.data)) - - pipeline.compile() - pipeline.fit() - - -if __name__ == "__main__": - parser = argparse.ArgumentParser("PIPELINE DEMO") - parser.add_argument("-config", type=str, - help="config file") - args = parser.parse_args() - if args.config is not None: - main(args.config) - else: - main() diff --git a/examples/pipeline/sbt_feature_transformer/pipeline-sbt-transformer-multi.py b/examples/pipeline/sbt_feature_transformer/pipeline-sbt-transformer-multi.py deleted file mode 100644 index 38873da7b8..0000000000 --- a/examples/pipeline/sbt_feature_transformer/pipeline-sbt-transformer-multi.py +++ /dev/null @@ -1,124 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import argparse - -from pipeline.backend.pipeline import PipeLine -from pipeline.component import DataTransform -from pipeline.component.hetero_secureboost import HeteroSecureBoost -from pipeline.component.intersection import Intersection -from pipeline.component.sbt_feature_transformer import SBTTransformer -from pipeline.component import LocalBaseline -from pipeline.component.reader import Reader -from pipeline.interface.data import Data -from pipeline.component.evaluation import Evaluation -from pipeline.interface.model import Model - -from pipeline.utils.tools import load_job_config - - -def main(config="../../config.yaml", namespace=""): - # obtain config - if isinstance(config, str): - config = load_job_config(config) - parties = config.parties - guest = parties.guest[0] - host = parties.host[0] - - - # data sets - guest_train_data = {"name": "vehicle_scale_hetero_guest", "namespace": f"experiment{namespace}"} - host_train_data = {"name": "vehicle_scale_hetero_host", "namespace": f"experiment{namespace}"} - - guest_validate_data = {"name": "vehicle_scale_hetero_guest", "namespace": f"experiment{namespace}"} - host_validate_data = {"name": "vehicle_scale_hetero_host", "namespace": f"experiment{namespace}"} - - # init pipeline - pipeline = PipeLine().set_initiator(role="guest", party_id=guest).set_roles(guest=guest, host=host,) - - # set data reader and data-io - reader_0, reader_1 = Reader(name="reader_0"), Reader(name="reader_1") - reader_0.get_party_instance(role="guest", party_id=guest).component_param(table=guest_train_data) - reader_0.get_party_instance(role="host", party_id=host).component_param(table=host_train_data) - reader_1.get_party_instance(role="guest", party_id=guest).component_param(table=guest_validate_data) - reader_1.get_party_instance(role="host", party_id=host).component_param(table=host_validate_data) - - data_transform_0, data_transform_1 = DataTransform(name="data_transform_0"), DataTransform(name="data_transform_1") - - data_transform_0.get_party_instance(role="guest", party_id=guest).component_param(with_label=True, output_format="dense") - data_transform_0.get_party_instance(role="host", party_id=host).component_param(with_label=False) - data_transform_1.get_party_instance(role="guest", party_id=guest).component_param(with_label=True, output_format="dense") - data_transform_1.get_party_instance(role="host", party_id=host).component_param(with_label=False) - - # data intersect component - intersect_0 = Intersection(name="intersection_0") - intersect_1 = Intersection(name="intersection_1") - - # secure boost component - hetero_secure_boost_0 = HeteroSecureBoost(name="hetero_secure_boost_0", - num_trees=3, - task_type="classification", - objective_param={"objective": "cross_entropy"}, - encrypt_param={"method": "paillier"}, - tree_param={"max_depth": 3}, - validation_freqs=1) - - # evaluation component - evaluation_0 = Evaluation(name="evaluation_0", eval_type="multi") - evaluation_1 = Evaluation(name="evaluation_1", eval_type="multi") - - # transformer - transformer_0 = SBTTransformer(name='sbt_transformer_0', dense_format=True) - - # local baseline - def get_local_baseline(idx): - return LocalBaseline(name="local_baseline_{}".format(idx), model_name="LogisticRegression", - model_opts={"penalty": "l2", "tol": 0.0001, "C": 1.0, "fit_intercept": True, - "solver": "lbfgs", "max_iter": 50}) - - local_baseline_0 = get_local_baseline(0) - local_baseline_0.get_party_instance(role='guest', party_id=guest).component_param(need_run=True) - local_baseline_0.get_party_instance(role='host', party_id=host).component_param(need_run=False) - - local_baseline_1 = get_local_baseline(1) - local_baseline_1.get_party_instance(role='guest', party_id=guest).component_param(need_run=True) - local_baseline_1.get_party_instance(role='host', party_id=host).component_param(need_run=False) - - evaluation_1.get_party_instance(role='host', party_id=host).component_param(need_run=False) - - pipeline.add_component(reader_0) - pipeline.add_component(reader_1) - pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data)) - pipeline.add_component(data_transform_1, data=Data(data=reader_1.output.data), model=Model(data_transform_0.output.model)) - pipeline.add_component(intersect_0, data=Data(data=data_transform_0.output.data)) - pipeline.add_component(intersect_1, data=Data(data=data_transform_1.output.data)) - pipeline.add_component(hetero_secure_boost_0, data=Data(train_data=intersect_0.output.data, - validate_data=intersect_1.output.data)) - pipeline.add_component(transformer_0, data=Data(data=intersect_0.output.data), - model=Model(isometric_model=hetero_secure_boost_0.output.model)) - pipeline.compile() - pipeline.fit() - - -if __name__ == "__main__": - parser = argparse.ArgumentParser("PIPELINE DEMO") - parser.add_argument("-config", type=str, - help="config file") - args = parser.parse_args() - if args.config is not None: - main(args.config) - else: - main() diff --git a/examples/pipeline/sbt_feature_transformer/pipeline-sbt-transformer.py b/examples/pipeline/sbt_feature_transformer/pipeline-sbt-transformer.py deleted file mode 100644 index d70608a325..0000000000 --- a/examples/pipeline/sbt_feature_transformer/pipeline-sbt-transformer.py +++ /dev/null @@ -1,131 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import argparse - -from pipeline.backend.pipeline import PipeLine -from pipeline.component import DataTransform -from pipeline.component import HeteroSecureBoost -from pipeline.component import Intersection -from pipeline.component import SBTTransformer -from pipeline.component import LocalBaseline -from pipeline.component import Reader -from pipeline.interface import Data -from pipeline.component import Evaluation -from pipeline.interface import Model - -from pipeline.utils.tools import load_job_config - - -def main(config="../../config.yaml", namespace=""): - # obtain config - if isinstance(config, str): - config = load_job_config(config) - parties = config.parties - guest = parties.guest[0] - host = parties.host[0] - - - # data sets - guest_train_data = {"name": "breast_hetero_guest", "namespace": f"experiment{namespace}"} - host_train_data = {"name": "breast_hetero_host", "namespace": f"experiment{namespace}"} - - guest_validate_data = {"name": "breast_hetero_guest", "namespace": f"experiment{namespace}"} - host_validate_data = {"name": "breast_hetero_host", "namespace": f"experiment{namespace}"} - - # init pipeline - pipeline = PipeLine().set_initiator(role="guest", party_id=guest).set_roles(guest=guest, host=host,) - - # set data reader and data-io - reader_0, reader_1 = Reader(name="reader_0"), Reader(name="reader_1") - reader_0.get_party_instance(role="guest", party_id=guest).component_param(table=guest_train_data) - reader_0.get_party_instance(role="host", party_id=host).component_param(table=host_train_data) - reader_1.get_party_instance(role="guest", party_id=guest).component_param(table=guest_validate_data) - reader_1.get_party_instance(role="host", party_id=host).component_param(table=host_validate_data) - - data_transform_0, data_transform_1 = DataTransform(name="data_transform_0"), DataTransform(name="data_transform_1") - - data_transform_0.get_party_instance(role="guest", party_id=guest).component_param(with_label=True, output_format="dense") - data_transform_0.get_party_instance(role="host", party_id=host).component_param(with_label=False) - data_transform_1.get_party_instance(role="guest", party_id=guest).component_param(with_label=True, output_format="dense") - data_transform_1.get_party_instance(role="host", party_id=host).component_param(with_label=False) - - # data intersect component - intersect_0 = Intersection(name="intersection_0") - intersect_1 = Intersection(name="intersection_1") - - # secure boost component - hetero_secure_boost_0 = HeteroSecureBoost(name="hetero_secure_boost_0", - num_trees=3, - task_type="classification", - objective_param={"objective": "cross_entropy"}, - encrypt_param={"method": "paillier"}, - tree_param={"max_depth": 3}, - validation_freqs=1) - - # evaluation component - evaluation_0 = Evaluation(name="evaluation_0", eval_type="binary") - evaluation_1 = Evaluation(name="evaluation_1", eval_type="binary") - - # transformer - transformer_0 = SBTTransformer(name='sbt_transformer_0', dense_format=True) - - # local baseline - def get_local_baseline(idx): - return LocalBaseline(name="local_baseline_{}".format(idx), model_name="LogisticRegression", - model_opts={"penalty": "l2", "tol": 0.0001, "C": 1.0, "fit_intercept": True, - "solver": "lbfgs", "max_iter": 50}) - - local_baseline_0 = get_local_baseline(0) - local_baseline_0.get_party_instance(role='guest', party_id=guest).component_param(need_run=True) - local_baseline_0.get_party_instance(role='host', party_id=host).component_param(need_run=False) - - local_baseline_1 = get_local_baseline(1) - local_baseline_1.get_party_instance(role='guest', party_id=guest).component_param(need_run=True) - local_baseline_1.get_party_instance(role='host', party_id=host).component_param(need_run=False) - - evaluation_1.get_party_instance(role='host', party_id=host).component_param(need_run=False) - - pipeline.add_component(reader_0) - pipeline.add_component(reader_1) - pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data)) - pipeline.add_component(data_transform_1, data=Data(data=reader_1.output.data), model=Model(data_transform_0.output.model)) - pipeline.add_component(intersect_0, data=Data(data=data_transform_0.output.data)) - pipeline.add_component(intersect_1, data=Data(data=data_transform_1.output.data)) - pipeline.add_component(hetero_secure_boost_0, data=Data(train_data=intersect_0.output.data, - validate_data=intersect_1.output.data)) - pipeline.add_component(transformer_0, data=Data(data=intersect_0.output.data), - model=Model(isometric_model=hetero_secure_boost_0.output.model)) - - pipeline.add_component(local_baseline_0, data=Data(data=transformer_0.output.data)) - pipeline.add_component(local_baseline_1, data=Data(data=intersect_0.output.data)) - - pipeline.add_component(evaluation_0, data=Data(data=local_baseline_0.output.data)) - pipeline.add_component(evaluation_1, data=Data(data=local_baseline_1.output.data)) - - pipeline.compile() - pipeline.fit() - - -if __name__ == "__main__": - parser = argparse.ArgumentParser("PIPELINE DEMO") - parser.add_argument("-config", type=str, - help="config file") - args = parser.parse_args() - if args.config is not None: - main(args.config) - else: - main() diff --git a/examples/pipeline/sbt_feature_transformer/sbt_transformer_testsuite.json b/examples/pipeline/sbt_feature_transformer/sbt_transformer_testsuite.json deleted file mode 100644 index 00174d079e..0000000000 --- a/examples/pipeline/sbt_feature_transformer/sbt_transformer_testsuite.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "data": [ - { - "file": "examples/data/breast_hetero_guest.csv", - "head": 1, - "partition": 4, - "table_name": "breast_hetero_guest", - "namespace": "experiment", - "role": "guest_0" - }, - { - "file": "examples/data/breast_hetero_host.csv", - "head": 1, - "partition": 4, - "table_name": "breast_hetero_host", - "namespace": "experiment", - "role": "host_0" - }, - { - "file": "examples/data/vehicle_scale_hetero_guest.csv", - "head": 1, - "partition": 4, - "table_name": "vehicle_scale_hetero_guest", - "namespace": "experiment", - "role": "guest_0" - }, - { - "file": "examples/data/vehicle_scale_hetero_host.csv", - "head": 1, - "partition": 4, - "table_name": "vehicle_scale_hetero_host", - "namespace": "experiment", - "role": "host_0" - } - ], - "pipeline_tasks": { - "sbt_transformer": { - "script": "./pipeline-sbt-transformer.py" - }, - "fast_sbt_transformer": { - "script": "./pipeline-sbt-transformer-fast-sbt.py" - }, - "sbt_transformer_multi": { - "script": "./pipeline-sbt-transformer-multi.py" - } - } -} \ No newline at end of file diff --git a/python/fate_client/pipeline/component/sbt_feature_transformer.py b/python/fate_client/pipeline/component/sbt_feature_transformer.py deleted file mode 100644 index 1731f7b218..0000000000 --- a/python/fate_client/pipeline/component/sbt_feature_transformer.py +++ /dev/null @@ -1,34 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from pipeline.param.sbt_feature_transformer_param import SBTTransformerParam -from pipeline.component.component_base import FateComponent -from pipeline.interface import Input -from pipeline.interface import Output -from pipeline.utils.logger import LOGGER - - -class SBTTransformer(FateComponent, SBTTransformerParam): - def __init__(self, **kwargs): - FateComponent.__init__(self, **kwargs) - - # print(self.name) - LOGGER.debug(f"{self.name} component created") - new_kwargs = self.erase_component_base_param(**kwargs) - SBTTransformerParam.__init__(self, **new_kwargs) - self.input = Input(self.name, data_type="multi") - self.output = Output(self.name) - self._module_name = "SBTFeatureTransformer" \ No newline at end of file diff --git a/python/fate_client/pipeline/param/sbt_feature_transformer_param.py b/python/fate_client/pipeline/param/sbt_feature_transformer_param.py deleted file mode 100644 index 6382f8e361..0000000000 --- a/python/fate_client/pipeline/param/sbt_feature_transformer_param.py +++ /dev/null @@ -1,16 +0,0 @@ -from pipeline.param.base_param import BaseParam - - -class SBTTransformerParam(BaseParam): - - def __init__(self, dense_format=True): - - """ - Args: - dense_format: return data in dense vec, otherwise return in sparse vec - """ - super(SBTTransformerParam, self).__init__() - self.dense_format = dense_format - - def check(self): - self.check_boolean(self.dense_format, 'SBTTransformer') diff --git a/python/federatedml/components/sbt_feature_transformer.py b/python/federatedml/components/sbt_feature_transformer.py deleted file mode 100644 index 7c1eca93d1..0000000000 --- a/python/federatedml/components/sbt_feature_transformer.py +++ /dev/null @@ -1,45 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 - -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from .components import ComponentMeta - -sbt_feature_transformer_cpn_meta = ComponentMeta("SBTFeatureTransformer") - - -@sbt_feature_transformer_cpn_meta.bind_param -def sbt_feature_transformer_param(): - from federatedml.param.sbt_feature_transformer_param import SBTTransformerParam - - return SBTTransformerParam - - -@sbt_feature_transformer_cpn_meta.bind_runner.on_guest -def sbt_feature_transformer_guest_runner(): - from federatedml.feature.sbt_feature_transformer.sbt_feature_transformer import ( - HeteroSBTFeatureTransformerGuest, - ) - - return HeteroSBTFeatureTransformerGuest - - -@sbt_feature_transformer_cpn_meta.bind_runner.on_host -def sbt_feature_transformer_host_runner(): - from federatedml.feature.sbt_feature_transformer.sbt_feature_transformer import ( - HeteroSBTFeatureTransformerHost, - ) - - return HeteroSBTFeatureTransformerHost diff --git a/python/federatedml/feature/sbt_feature_transformer/sbt_feature_transformer.py b/python/federatedml/feature/sbt_feature_transformer/sbt_feature_transformer.py deleted file mode 100644 index c2352d0eb6..0000000000 --- a/python/federatedml/feature/sbt_feature_transformer/sbt_feature_transformer.py +++ /dev/null @@ -1,267 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import numpy as np -import copy -import functools -from federatedml.model_base import ModelBase -from federatedml.util import LOGGER -from federatedml.util import consts -from federatedml.ensemble import HeteroSecureBoostingTreeGuest, HeteroSecureBoostingTreeHost -from federatedml.ensemble import HeteroFastSecureBoostingTreeGuest, HeteroFastSecureBoostingTreeHost -from federatedml.model_base import MetricMeta -from federatedml.util import abnormal_detection -from federatedml.param.sbt_feature_transformer_param import SBTTransformerParam -from federatedml.feature.sparse_vector import SparseVector -from federatedml.feature.instance import Instance -from federatedml.protobuf.generated.boosting_tree_model_param_pb2 import TransformerParam -from federatedml.protobuf.generated.boosting_tree_model_meta_pb2 import TransformerMeta - - -class HeteroSBTFeatureTransformerBase(ModelBase): - - def __init__(self): - super(HeteroSBTFeatureTransformerBase, self).__init__() - self.tree_model = None - self.role = None - self.dense_format = True - self.model_param = SBTTransformerParam() - - def _init_model(self, param: SBTTransformerParam): - self.dense_format = param.dense_format - - def _abnormal_detection(self, data_instances): - """ - Make sure input data_instances is valid. - """ - abnormal_detection.empty_table_detection(data_instances) - abnormal_detection.empty_feature_detection(data_instances) - self.check_schema_content(data_instances.schema) - - @staticmethod - def _get_model_type(model_dict): - """ - fast-sbt or sbt ? - """ - sbt_key_prefix = consts.HETERO_SBT_GUEST_MODEL.replace('Guest', '') - fast_sbt_key_prefix = consts.HETERO_FAST_SBT_GUEST_MODEL.replace('Guest', '') - for key in model_dict: - for model_key in model_dict[key]: - if sbt_key_prefix in model_key: - return consts.HETERO_SBT - elif fast_sbt_key_prefix in model_key: - return consts.HETERO_FAST_SBT - - return None - - def _init_tree(self, tree_model_type): - - if tree_model_type == consts.HETERO_SBT: - self.tree_model = HeteroSecureBoostingTreeGuest() if self.role == consts.GUEST else HeteroSecureBoostingTreeHost() - elif tree_model_type == consts.HETERO_FAST_SBT: - self.tree_model = HeteroFastSecureBoostingTreeGuest() if self.role == consts.GUEST \ - else HeteroFastSecureBoostingTreeHost() - - def _load_tree_model(self, model_dict, key_name='isometric_model'): - """ - load model - """ - # judge input model type by key in model_dict - LOGGER.info('loading model') - tree_model_type = self._get_model_type(model_dict[key_name]) - LOGGER.info('model type is {}'.format(tree_model_type)) - if tree_model_type is None: - raise ValueError('key related to tree models is not detected in model dict,' - 'please check the input model') - self._init_tree(tree_model_type) - # initialize tree model - self.tree_model.load_model(model_dict, model_key=key_name) - self.tree_model.set_flowid(self.flowid) - self.tree_model.component_properties = copy.deepcopy(self.component_properties) - - LOGGER.info('loading {} model done'.format(tree_model_type)) - - def _make_mock_isometric(self, tran_param, tran_meta): - - tree_param = tran_param.tree_param - tree_meta = tran_meta.tree_meta - param_name = tran_param.model_name - meta_name = tran_meta.model_name - mock_dict = {'isometric_model': {'sbt': {param_name: tree_param, meta_name: tree_meta}}} - return mock_dict - - def load_model(self, model_dict): - - LOGGER.debug(f"In load_model, model_dict: {model_dict}") - - if 'isometric_model' in model_dict: - self._load_tree_model(model_dict, key_name='isometric_model') - elif 'model' in model_dict: - tran_param, tran_meta = None, None - model = model_dict['model'] - for key in model: - content = model[key] - for model_key in content: - if 'Param' in model_key: - tran_param = content[model_key] - if 'Meta' in model_key: - tran_meta = content[model_key] - mock_dict = self._make_mock_isometric(tran_param, tran_meta) - self._load_tree_model(mock_dict) - else: - raise ValueError('illegal model input') - - def export_model(self): - - tree_meta_name, model_meta = self.tree_model.get_model_meta() - tree_param_name, model_param = self.tree_model.get_model_param() - param, meta = TransformerParam(), TransformerMeta() - param.tree_param.CopyFrom(model_param) - param.model_name = tree_param_name - meta.tree_meta.CopyFrom(model_meta) - meta.model_name = tree_meta_name - param_name, meta_name = 'SBTTransformerParam', 'SBTTransformerMeta' - - return {param_name: param, meta_name: meta} - - -class HeteroSBTFeatureTransformerGuest(HeteroSBTFeatureTransformerBase): - - def __init__(self): - super(HeteroSBTFeatureTransformerGuest, self).__init__() - self.role = consts.GUEST - self.leaf_mapping_list = [] - self.vec_len = -1 - self.feature_title = consts.SECUREBOOST - - @staticmethod - def join_feature_with_label(inst, leaf_indices, leaf_mapping_list, vec_len, dense): - - label = inst.label - if dense: - vec = np.zeros(vec_len) - offset = 0 - for tree_idx, leaf_idx in enumerate(leaf_indices): - vec[leaf_mapping_list[tree_idx][leaf_idx] + offset] = 1 - offset += len(leaf_mapping_list[tree_idx]) - return Instance(features=vec, label=label) - - else: - indices, value = [], [] - offset = 0 - for tree_idx, leaf_idx in enumerate(leaf_indices): - indices.append(leaf_mapping_list[tree_idx][leaf_idx] + offset) - value.append(1) - offset += len(leaf_mapping_list[tree_idx]) - return Instance(features=SparseVector(indices=indices, data=value, shape=vec_len), label=label) - - def _generate_header(self, leaf_mapping): - - header = [] - for tree_idx, mapping in enumerate(leaf_mapping): - feat_name_prefix = self.feature_title + '_' + str(tree_idx) + '_' - sorted_leaf_ids = sorted(list(mapping.keys())) - for leaf_id in sorted_leaf_ids: - header.append(feat_name_prefix+str(leaf_id)) - - return header - - def _generate_callback_result(self, header): - - index = [] - for i in header: - split_list = i.split('_') - index.append(split_list[-1]) - return {'feat_name': header, 'index': index} - - def _transform_pred_result(self, data_inst, pred_result): - - self.leaf_mapping_list, self.vec_len = self._extract_leaf_mapping() - join_func = functools.partial(self.join_feature_with_label, vec_len=self.vec_len, leaf_mapping_list=self.leaf_mapping_list, - dense=self.dense_format) - rs = data_inst.join(pred_result, join_func) - # add schema for new data table - rs.schema['header'] = self._generate_header(self.leaf_mapping_list) - if 'label_name' in data_inst.schema: - rs.schema['label_name'] = data_inst.schema['label_name'] - - return rs - - def _extract_leaf_mapping(self): - - # one hot encoding - leaf_mapping_list = [] - for tree_param in self.tree_model.boosting_model_list: - leaf_mapping = {} - idx = 0 - for node_param in tree_param.tree_: - if node_param.is_leaf: - leaf_mapping[node_param.id] = idx - idx += 1 - leaf_mapping_list.append(leaf_mapping) - - vec_len = 0 - for map_ in leaf_mapping_list: - vec_len += len(map_) - - return leaf_mapping_list, vec_len - - def _callback_leaf_id_mapping(self, mapping): - - metric_namespace = 'sbt_transformer' - metric_name = 'leaf_mapping' - self.tracker.set_metric_meta(metric_namespace, metric_name, - MetricMeta(name=metric_name, metric_type=metric_name, extra_metas=mapping)) - - def fit(self, data_inst): - - self._abnormal_detection(data_inst) - # predict instances to get leaf indexes - LOGGER.info('tree model running prediction') - predict_rs = self.tree_model.predict(data_inst, ret_format='leaf') - LOGGER.info('tree model prediction done') - - # transform pred result to new data table - LOGGER.debug('use dense is {}'.format(self.dense_format)) - rs = self._transform_pred_result(data_inst, predict_rs) - - # display result callback - LOGGER.debug('header is {}'.format(rs.schema)) - LOGGER.debug('extra meta is {}'.format(self._generate_callback_result(rs.schema['header']))) - self._callback_leaf_id_mapping(self._generate_callback_result(rs.schema['header'])) - - return rs - - def transform(self, data_inst): - return self.fit(data_inst) - - -class HeteroSBTFeatureTransformerHost(HeteroSBTFeatureTransformerBase): - - def __init__(self): - super(HeteroSBTFeatureTransformerHost, self).__init__() - self.role = consts.HOST - - def fit(self, data_inst): - - self._abnormal_detection(data_inst) - self.tree_model.predict(data_inst) - - def transform(self, data_inst): - self.fit(data_inst) - diff --git a/python/federatedml/param/__init__.py b/python/federatedml/param/__init__.py index c6cd914d61..eab12763ed 100755 --- a/python/federatedml/param/__init__.py +++ b/python/federatedml/param/__init__.py @@ -48,7 +48,6 @@ from federatedml.param.rsa_param import RsaParam from federatedml.param.sample_param import SampleParam from federatedml.param.sample_weight_param import SampleWeightParam -from federatedml.param.sbt_feature_transformer_param import SBTTransformerParam from federatedml.param.scale_param import ScaleParam from federatedml.param.scorecard_param import ScorecardParam from federatedml.param.secure_add_example_param import SecureAddExampleParam @@ -99,6 +98,5 @@ "ScorecardParam", "SecureInformationRetrievalParam", "SampleWeightParam", - "FeldmanVerifiableSumParam", - "SBTTransformerParam" + "FeldmanVerifiableSumParam" ] diff --git a/python/federatedml/param/sbt_feature_transformer_param.py b/python/federatedml/param/sbt_feature_transformer_param.py deleted file mode 100644 index 9773689d7b..0000000000 --- a/python/federatedml/param/sbt_feature_transformer_param.py +++ /dev/null @@ -1,18 +0,0 @@ -from federatedml.param.base_param import BaseParam - - -class SBTTransformerParam(BaseParam): - - def __init__(self, dense_format=True): - - """ - Parameters - ---------- - dense_format: bool - return data in dense vec if True, otherwise return in sparse vec - """ - super(SBTTransformerParam, self).__init__() - self.dense_format = dense_format - - def check(self): - self.check_boolean(self.dense_format, 'SBTTransformer') From 97d113fc09e625bd6f8819957c3ba592374bc352 Mon Sep 17 00:00:00 2001 From: weijingchen Date: Tue, 15 Feb 2022 16:00:42 +0800 Subject: [PATCH 28/99] add EINI param Signed-off-by: cwj --- .../pipeline/param/boosting_param.py | 17 ++++++++++++++--- .../boosting/hetero/hetero_secureboost_host.py | 2 +- python/federatedml/param/boosting_param.py | 9 +++++++++ 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/python/fate_client/pipeline/param/boosting_param.py b/python/fate_client/pipeline/param/boosting_param.py index 220a952ad0..014807cd8d 100644 --- a/python/fate_client/pipeline/param/boosting_param.py +++ b/python/fate_client/pipeline/param/boosting_param.py @@ -447,6 +447,14 @@ class HeteroSecureBoostParam(HeteroBoostingParam): cipher_compress: bool, default is True, use cipher compressing to reduce computation cost and transfer cost + EINI_inference: bool + default is True, a secure prediction method that hides decision path to enhance security in the inference + step. This method is insprired by EINI inference algorithm. + + EINI_random_mask: bool + default is False + multiply predict result by a random float number to confuse original predict result. This operation further + enhances the security of naive EINI algorithm. """ def __init__(self, tree_param: DecisionTreeParam = DecisionTreeParam(), task_type=consts.CLASSIFICATION, @@ -461,7 +469,7 @@ def __init__(self, tree_param: DecisionTreeParam = DecisionTreeParam(), task_typ binning_error=consts.DEFAULT_RELATIVE_ERROR, sparse_optimization=False, run_goss=False, top_rate=0.2, other_rate=0.1, cipher_compress_error=None, cipher_compress=True, new_ver=True, - callback_param=CallbackParam()): + callback_param=CallbackParam(), EINI_inference=True, EINI_random_mask=False): super(HeteroSecureBoostParam, self).__init__(task_type, objective_param, learning_rate, num_trees, subsample_feature_rate, n_iter_no_change, tol, encrypt_param, @@ -482,6 +490,8 @@ def __init__(self, tree_param: DecisionTreeParam = DecisionTreeParam(), task_typ self.cipher_compress_error = cipher_compress_error self.cipher_compress = cipher_compress self.new_ver = new_ver + self.EINI_inference = EINI_inference + self.EINI_random_mask = EINI_random_mask self.callback_param = copy.deepcopy(callback_param) def check(self): @@ -524,7 +534,7 @@ def __init__(self, tree_param: DecisionTreeParam = DecisionTreeParam(), task_typ complete_secure=False, tree_num_per_party=1, guest_depth=1, host_depth=1, work_mode='mix', metrics=None, sparse_optimization=False, random_seed=100, binning_error=consts.DEFAULT_RELATIVE_ERROR, cipher_compress_error=None, new_ver=True, run_goss=False, top_rate=0.2, other_rate=0.1, - cipher_compress=True, callback_param=CallbackParam()): + cipher_compress=True, callback_param=CallbackParam(), EINI_inference=True, EINI_random_mask=False): """ work_mode： @@ -553,7 +563,8 @@ def __init__(self, tree_param: DecisionTreeParam = DecisionTreeParam(), task_typ cipher_compress_error=cipher_compress_error, new_ver=new_ver, cipher_compress=cipher_compress, - run_goss=run_goss, top_rate=top_rate, other_rate=other_rate + run_goss=run_goss, top_rate=top_rate, other_rate=other_rate, + EINI_inference=EINI_inference, EINI_random_mask=EINI_random_mask ) self.tree_num_per_party = tree_num_per_party diff --git a/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py b/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py index 2e3e2e2075..e16bda7542 100644 --- a/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py +++ b/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py @@ -337,7 +337,7 @@ def predict(self, data_inst): if self.EINI_inference: sitename = self.role + ':' + str(self.component_properties.local_partyid) self.EINI_host_predict(processed_data, trees, sitename, self.component_properties.local_partyid, - self.component_properties.host_party_idlist) + self.component_properties.host_party_idlist, self.EINI_random_mask) else: self.boosting_fast_predict(processed_data, trees=trees) diff --git a/python/federatedml/param/boosting_param.py b/python/federatedml/param/boosting_param.py index e8036a0e63..a2e0077584 100644 --- a/python/federatedml/param/boosting_param.py +++ b/python/federatedml/param/boosting_param.py @@ -495,6 +495,15 @@ class HeteroSecureBoostParam(HeteroBoostingParam): cipher_compress: bool default is True, use cipher compressing to reduce computation cost and transfer cost + EINI_inference: bool + default is True, a secure prediction method that hides decision path to enhance security in the inference + step. This method is insprired by EINI inference algorithm. + + EINI_random_mask: bool + default is False + multiply predict result by a random float number to confuse original predict result. This operation further + enhances the security of naive EINI algorithm. + """ def __init__(self, tree_param: DecisionTreeParam = DecisionTreeParam(), task_type=consts.CLASSIFICATION, From c0baab8bd03a28a86dde21dab745b9ca17d89342 Mon Sep 17 00:00:00 2001 From: mgqa34 Date: Tue, 15 Feb 2022 16:32:01 +0800 Subject: [PATCH 29/99] fix some error of batch strategy, update parameter doc Signed-off-by: mgqa34 --- .../param/logistic_regression_param.py | 8 +++++--- .../federatedml/model_selection/mini_batch.py | 16 +++++++++++----- .../param/logistic_regression_param.py | 19 +++++++++++++++++++ 3 files changed, 35 insertions(+), 8 deletions(-) diff --git a/python/fate_client/pipeline/param/logistic_regression_param.py b/python/fate_client/pipeline/param/logistic_regression_param.py index 4c95db6178..af38e5f324 100644 --- a/python/fate_client/pipeline/param/logistic_regression_param.py +++ b/python/fate_client/pipeline/param/logistic_regression_param.py @@ -112,7 +112,7 @@ class LogisticParam(BaseParam): def __init__(self, penalty='L2', tol=1e-4, alpha=1.0, optimizer='rmsprop', - batch_size=-1, shuffle=True, + batch_size=-1, shuffle=True, batch_strategy="full", masked_rate=5, learning_rate=0.01, init_param=InitParam(), max_iter=100, early_stop='diff', encrypt_param=EncryptParam(), predict_param=PredictParam(), cv_param=CrossValidationParam(), @@ -130,6 +130,8 @@ def __init__(self, penalty='L2', self.optimizer = optimizer self.batch_size = batch_size self.shuffle = shuffle + self.batch_strategy = batch_strategy + self.masked_rate = masked_rate self.learning_rate = learning_rate self.init_param = copy.deepcopy(init_param) self.max_iter = max_iter @@ -329,7 +331,7 @@ def check(self): class HeteroLogisticParam(LogisticParam): def __init__(self, penalty='L2', tol=1e-4, alpha=1.0, optimizer='rmsprop', - batch_size=-1, shuffle=True, + batch_size=-1, shuffle=True, batch_strategy="full", masked_rate=5, learning_rate=0.01, init_param=InitParam(), max_iter=100, early_stop='diff', encrypted_mode_calculator_param=EncryptedModeCalculatorParam(), @@ -342,7 +344,7 @@ def __init__(self, penalty='L2', callback_param=CallbackParam() ): super(HeteroLogisticParam, self).__init__(penalty=penalty, tol=tol, alpha=alpha, optimizer=optimizer, - batch_size=batch_size, shuffle=shuffle, + batch_size=batch_size, shuffle=shuffle, batch_strategy=batch_strategy, masked_rate=masked_rate, learning_rate=learning_rate, init_param=init_param, max_iter=max_iter, early_stop=early_stop, predict_param=predict_param, cv_param=cv_param, diff --git a/python/federatedml/model_selection/mini_batch.py b/python/federatedml/model_selection/mini_batch.py index 305a9b93f1..b30ad41caa 100644 --- a/python/federatedml/model_selection/mini_batch.py +++ b/python/federatedml/model_selection/mini_batch.py @@ -99,16 +99,19 @@ def get_batch_generator(data_size, batch_size, batch_strategy, masked_rate, shuf else: if shuffle: LOGGER.warning("if use random select batch strategy, shuffle will not work") - return RandomBatchDataGenerator(batch_size, masked_rate) + return RandomBatchDataGenerator(data_size, batch_size, masked_rate) class FullBatchDataGenerator(object): def __init__(self, data_size, batch_size, shuffle=False, masked_rate=0): - self.batch_nums = (data_size + batch_size - 1) // batch_size - self.masked_dataset_size = min(data_size, round((1 + masked_rate) * self.batch_nums)) + self.batch_nums = (data_size + batch_size - 1) // batch_size + self.masked_dataset_size = min(data_size, round((1 + masked_rate) * batch_size)) self.batch_size = batch_size self.shuffle = shuffle + LOGGER.debug(f"Init Full Batch Data Generator, batch_nums: {self.batch_nums}, batch_size: {self.batch_size}, " + f"masked_dataset_size: {self.masked_dataset_size}, shuffle: {self.shuffle}") + def generate_data(self, data_insts, data_sids): if self.shuffle: random.SystemRandom().shuffle(data_sids) @@ -157,10 +160,13 @@ def batch_masked(self): class RandomBatchDataGenerator(object): - def __init__(self, batch_size, masked_rate=0): + def __init__(self, data_size, batch_size, masked_rate=0): self.batch_nums = 1 self.batch_size = batch_size - self.masked_dataset_size = min(batch_size, round((1 + masked_rate) * self.batch_size)) + self.masked_dataset_size = min(data_size, round((1 + masked_rate) * self.batch_size)) + + LOGGER.debug(f"Init Random Batch Data Generator, batch_nums: {self.batch_nums}, batch_size: {self.batch_size}, " + f"masked_dataset_size: {self.masked_dataset_size}") def generate_data(self, data_insts, *args, **kwargs): if self.masked_dataset_size == self.batch_size: diff --git a/python/federatedml/param/logistic_regression_param.py b/python/federatedml/param/logistic_regression_param.py index 8408097d88..fcc66daa12 100644 --- a/python/federatedml/param/logistic_regression_param.py +++ b/python/federatedml/param/logistic_regression_param.py @@ -53,9 +53,20 @@ class LogisticParam(BaseParam): optimizer : {'rmsprop', 'sgd', 'adam', 'nesterov_momentum_sgd', 'sqn', 'adagrad'}, default: 'rmsprop' Optimize method, if 'sqn' has been set, sqn_param will take effect. Currently, 'sqn' support hetero mode only. + batch_strategy : str, {'full', 'random'}, default: "full" + Strategy to generate batch data. + a) full: use full data to generate batch_data, batch_nums every iteration is ceil(data_size / batch_size) + b) random: select data randomly from full data, batch_num will be 1 every iteration. + batch_size : int, default: -1 Batch size when updating model. -1 means use all data in a batch. i.e. Not to use mini-batch strategy. + shuffle : bool, default: True + Work only in hetero logistic regression, batch data will be shuffle in every iteration. + + masked_rate: int, float: default: 5 + Use masked data to enhance security of hetero logistic regression + learning_rate : float, default: 0.01 Learning rate @@ -192,6 +203,14 @@ def check(self): raise ValueError(descr + " {} not supported, should be larger than {} or " "-1 represent for all data".format(self.batch_size, consts.MIN_BATCH_SIZE)) + if not isinstance(self.masked_rate, (float, int)) or self.masked_rate < 0: + raise ValueError("masked rate should be non-negative numeric number") + if not isinstance(self.batch_strategy, str) or self.batch_strategy.lower() not in ["full", "random"]: + raise ValueError("batch strategy should be full or random") + self.batch_strategy = self.batch_strategy.lower() + if not isinstance(self.shuffle, bool): + raise ValueError("shuffle define in batch should be boolean type") + if not isinstance(self.learning_rate, (float, int)): raise ValueError( "logistic_param's learning_rate {} not supported, should be float or int type".format( From 9d8f649cae7e0472ffb51713bd0f745d2b775d5b Mon Sep 17 00:00:00 2001 From: weijingchen Date: Tue, 15 Feb 2022 18:33:57 +0800 Subject: [PATCH 30/99] update EINI Signed-off-by: cwj --- .../ensemble/boosting/hetero/hetero_secureboost_guest.py | 1 - .../ensemble/boosting/hetero/hetero_secureboost_host.py | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_guest.py b/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_guest.py index 1eb5203d8c..e8786501d8 100644 --- a/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_guest.py +++ b/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_guest.py @@ -440,7 +440,6 @@ def EINI_guest_predict(self, data_inst, trees: List[HeteroDecisionTreeGuest], le encrypter_vec_table = position_vec.mapValues(encrypter.recursive_encrypt) # federation part - self.hetero_sbt_transfer_variable.guest_predict_data.remote(booster_dim, idx=-1, suffix='booster_dim') # send to first host party self.hetero_sbt_transfer_variable.guest_predict_data.remote(encrypter_vec_table, idx=0, suffix='position_vec', role=consts.HOST) diff --git a/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py b/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py index e16bda7542..fa83f2d960 100644 --- a/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py +++ b/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py @@ -278,8 +278,7 @@ def EINI_host_predict(self, data_inst, trees: List[HeteroDecisionTreeHost], site node_pos_map_list=id_pos_map_list) position_vec = data_inst.mapValues(map_func) - booster_dim = self.hetero_sbt_transfer_variable.guest_predict_data.get(idx=0, suffix='booster_dim') - + booster_dim = self.booster_dim random_mask = random.SystemRandom().random() + 1 if random_mask else 0 # generate a random mask btw 1 and 2 self_idx = party_list.index(self_party_id) From 87910373aa9759e0bf6743a3f33a9820783cc3f3 Mon Sep 17 00:00:00 2001 From: mgqa34 Date: Tue, 15 Feb 2022 20:54:06 +0800 Subject: [PATCH 31/99] fix sqn compute gradient parameter Signed-off-by: mgqa34 --- python/federatedml/optim/gradient/hetero_sqn_gradient.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/federatedml/optim/gradient/hetero_sqn_gradient.py b/python/federatedml/optim/gradient/hetero_sqn_gradient.py index d1c20988ef..e97cc90acd 100644 --- a/python/federatedml/optim/gradient/hetero_sqn_gradient.py +++ b/python/federatedml/optim/gradient/hetero_sqn_gradient.py @@ -73,7 +73,7 @@ def _update_w_tilde(self, model_weights): else: self.this_w_tilde += model_weights - def compute_gradient_procedure(self, *args): + def compute_gradient_procedure(self, *args, **kwargs): data_instances = args[0] encrypted_calculator = args[1] model_weights = args[2] @@ -102,7 +102,7 @@ def compute_gradient_procedure(self, *args): return gradient_results - def compute_loss(self, *args): + def compute_loss(self, *args, **kwargs): loss = self.gradient_computer.compute_loss(*args) return loss From 1adff630e19f44e0ee29b92d1bd3a0f79e15f650 Mon Sep 17 00:00:00 2001 From: dylanfan <289765648@qq.com> Date: Tue, 15 Feb 2022 21:05:27 +0800 Subject: [PATCH 32/99] optim sshe-lr Signed off by: dylanfan <289765648@qq.com> --- .../hetero_lr_base.py | 29 ++++++++++++++----- .../hetero_lr_guest.py | 8 ++--- .../hetero_lr_host.py | 4 +-- 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/python/federatedml/linear_model/logistic_regression/hetero_sshe_logistic_regression/hetero_lr_base.py b/python/federatedml/linear_model/logistic_regression/hetero_sshe_logistic_regression/hetero_lr_base.py index 078845f829..c5b3770903 100644 --- a/python/federatedml/linear_model/logistic_regression/hetero_sshe_logistic_regression/hetero_lr_base.py +++ b/python/federatedml/linear_model/logistic_regression/hetero_sshe_logistic_regression/hetero_lr_base.py @@ -145,13 +145,13 @@ def share_model(self, w, suffix): ) return wa, wb - def forward(self, weights, features, suffix, cipher): + def forward(self, weights, features, labels, suffix, cipher): raise NotImplementedError("Should not call here") def backward(self, error, features, suffix, cipher): raise NotImplementedError("Should not call here") - def compute_loss(self, weights, suffix, cipher): + def compute_loss(self, weights, labels, suffix, cipher): raise NotImplementedError("Should not call here") def fit(self, data_instances, validate_data=None): @@ -201,8 +201,8 @@ def fit_binary(self, data_instances, validate_data=None): ) as spdz: spdz.set_flowid(self.flowid) self.secure_matrix_obj.set_flowid(self.flowid) - if self.role == consts.GUEST: - self.labels = data_instances.mapValues(lambda x: np.array([x.label], dtype=int)) + # if self.role == consts.GUEST: + # self.labels = data_instances.mapValues(lambda x: np.array([x.label], dtype=int)) w_self, w_remote = self.share_model(w, suffix="init") last_w_self, last_w_remote = w_self, w_remote @@ -212,18 +212,25 @@ def fit_binary(self, data_instances, validate_data=None): self.cipher_tool = [] encoded_batch_data = [] + batch_labels_list = [] + for batch_data in batch_data_generator: if self.fit_intercept: batch_features = batch_data.mapValues(lambda x: np.hstack((x.features, 1.0))) else: batch_features = batch_data.mapValues(lambda x: x.features) + + if self.role == consts.GUEST: + # add batch labels for replacing labels; + batch_labels = batch_data.mapValues(lambda x: np.array([x.label], dtype=int)) + batch_labels_list.append(batch_labels) + self.batch_num.append(batch_data.count()) encoded_batch_data.append( fixedpoint_table.FixedPointTensor(self.fixedpoint_encoder.encode(batch_features), q_field=self.fixedpoint_encoder.n, endec=self.fixedpoint_encoder)) - self.cipher_tool.append(EncryptModeCalculator(self.cipher, self.encrypted_mode_calculator_param.mode, self.encrypted_mode_calculator_param.re_encrypted_rate)) @@ -241,20 +248,26 @@ def fit_binary(self, data_instances, validate_data=None): for batch_idx, batch_data in enumerate(encoded_batch_data): current_suffix = (str(self.n_iter_), str(batch_idx)) + if self.role == consts.GUEST: + batch_labels = batch_labels_list[batch_idx] + else: + batch_labels = None if self.reveal_every_iter: y = self.forward(weights=self.model_weights, features=batch_data, + labels=batch_labels, suffix=current_suffix, cipher=self.cipher_tool[batch_idx]) else: y = self.forward(weights=(w_self, w_remote), features=batch_data, + labels=batch_labels, suffix=current_suffix, cipher=self.cipher_tool[batch_idx]) if self.role == consts.GUEST: - error = y - self.labels + error = y - batch_labels self_g, remote_g = self.backward(error=error, features=batch_data, @@ -269,9 +282,9 @@ def fit_binary(self, data_instances, validate_data=None): # loss computing; suffix = ("loss",) + current_suffix if self.reveal_every_iter: - batch_loss = self.compute_loss(weights=self.model_weights, suffix=suffix, cipher=self.cipher_tool[batch_idx]) + batch_loss = self.compute_loss(weights=self.model_weights, labels=batch_labels, suffix=suffix, cipher=self.cipher_tool[batch_idx]) else: - batch_loss = self.compute_loss(weights=(w_self, w_remote), suffix=suffix, cipher=self.cipher_tool[batch_idx]) + batch_loss = self.compute_loss(weights=(w_self, w_remote), labels=batch_labels, suffix=suffix, cipher=self.cipher_tool[batch_idx]) if batch_loss is not None: batch_loss = batch_loss * self.batch_num[batch_idx] diff --git a/python/federatedml/linear_model/logistic_regression/hetero_sshe_logistic_regression/hetero_lr_guest.py b/python/federatedml/linear_model/logistic_regression/hetero_sshe_logistic_regression/hetero_lr_guest.py index 6700b1f8bd..2c35c9c9ae 100644 --- a/python/federatedml/linear_model/logistic_regression/hetero_sshe_logistic_regression/hetero_lr_guest.py +++ b/python/federatedml/linear_model/logistic_regression/hetero_sshe_logistic_regression/hetero_lr_guest.py @@ -65,7 +65,7 @@ def _compute_sigmoid(self, z, remote_z): return sigmoid_z - def forward(self, weights, features, suffix, cipher): + def forward(self, weights, features, labels, suffix, cipher): if not self.reveal_every_iter: LOGGER.info(f"[forward]: Calculate z in share...") w_self, w_remote = weights @@ -87,7 +87,7 @@ def forward(self, weights, features, suffix, cipher): self.encrypted_wx = self.wx_self + self.wx_remote - self.encrypted_error = sigmoid_z - self.labels + self.encrypted_error = sigmoid_z - labels tensor_name = ".".join(("sigmoid_z",) + suffix) shared_sigmoid_z = SecureMatrix.from_source(tensor_name, @@ -126,7 +126,7 @@ def backward(self, error, features, suffix, cipher): return gb2, ga2_2 - def compute_loss(self, weights, suffix, cipher=None): + def compute_loss(self, weights, labels, suffix, cipher=None): """ Use Taylor series expand log loss: Loss = - y * log(h(x)) - (1-y) * log(1 - h(x)) where h(x) = 1/(1+exp(-wx)) @@ -134,7 +134,7 @@ def compute_loss(self, weights, suffix, cipher=None): """ LOGGER.info(f"[compute_loss]: Calculate loss ...") wx = (-0.5 * self.encrypted_wx).reduce(operator.add) - ywx = (self.encrypted_wx * self.labels).reduce(operator.add) + ywx = (self.encrypted_wx * labels).reduce(operator.add) wx_square = (2 * self.wx_remote * self.wx_self).reduce(operator.add) + \ (self.wx_self * self.wx_self).reduce(operator.add) diff --git a/python/federatedml/linear_model/logistic_regression/hetero_sshe_logistic_regression/hetero_lr_host.py b/python/federatedml/linear_model/logistic_regression/hetero_sshe_logistic_regression/hetero_lr_host.py index 9a437fd5a7..2bfd4c6ae6 100644 --- a/python/federatedml/linear_model/logistic_regression/hetero_sshe_logistic_regression/hetero_lr_host.py +++ b/python/federatedml/linear_model/logistic_regression/hetero_sshe_logistic_regression/hetero_lr_host.py @@ -52,7 +52,7 @@ def _cal_z_in_share(self, w_self, w_remote, features, suffix): z = z1 + za_share + zb_share return z - def forward(self, weights, features, suffix, cipher): + def forward(self, weights, features, labels, suffix, cipher): if not self.reveal_every_iter: LOGGER.info(f"[forward]: Calculate z in share...") w_self, w_remote = weights @@ -108,7 +108,7 @@ def backward(self, error: fixedpoint_table.FixedPointTensor, features, suffix, c return ga_new, gb1 - def compute_loss(self, weights=None, suffix=None, cipher=None): + def compute_loss(self, weights=None, labels=None, suffix=None, cipher=None): """ Use Taylor series expand log loss: Loss = - y * log(h(x)) - (1-y) * log(1 - h(x)) where h(x) = 1/(1+exp(-wx)) From 73fcea61b52672549229e97abc0e390df12b1440 Mon Sep 17 00:00:00 2001 From: dylanfan <289765648@qq.com> Date: Tue, 15 Feb 2022 21:24:23 +0800 Subject: [PATCH 33/99] add batch log for sshe-lr Signed off by: dylanfan <289765648@qq.com> --- .../hetero_sshe_logistic_regression/hetero_lr_base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/federatedml/linear_model/logistic_regression/hetero_sshe_logistic_regression/hetero_lr_base.py b/python/federatedml/linear_model/logistic_regression/hetero_sshe_logistic_regression/hetero_lr_base.py index c5b3770903..414767fb16 100644 --- a/python/federatedml/linear_model/logistic_regression/hetero_sshe_logistic_regression/hetero_lr_base.py +++ b/python/federatedml/linear_model/logistic_regression/hetero_sshe_logistic_regression/hetero_lr_base.py @@ -247,6 +247,7 @@ def fit_binary(self, data_instances, validate_data=None): self.remote_optimizer.set_iters(self.n_iter_) for batch_idx, batch_data in enumerate(encoded_batch_data): + LOGGER.info(f"start to n_iter: {self.n_iter_}, batch idx: {batch_idx}") current_suffix = (str(self.n_iter_), str(batch_idx)) if self.role == consts.GUEST: batch_labels = batch_labels_list[batch_idx] From 564a8be2ae32613df71ecf4184d667cd0c360094 Mon Sep 17 00:00:00 2001 From: dylanfan <289765648@qq.com> Date: Tue, 15 Feb 2022 21:28:16 +0800 Subject: [PATCH 34/99] remove some comments for sshe-lr Signed off by: dylanfan <289765648@qq.com> --- .../hetero_sshe_logistic_regression/hetero_lr_base.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/python/federatedml/linear_model/logistic_regression/hetero_sshe_logistic_regression/hetero_lr_base.py b/python/federatedml/linear_model/logistic_regression/hetero_sshe_logistic_regression/hetero_lr_base.py index 414767fb16..6b3b1e8b0a 100644 --- a/python/federatedml/linear_model/logistic_regression/hetero_sshe_logistic_regression/hetero_lr_base.py +++ b/python/federatedml/linear_model/logistic_regression/hetero_sshe_logistic_regression/hetero_lr_base.py @@ -201,9 +201,6 @@ def fit_binary(self, data_instances, validate_data=None): ) as spdz: spdz.set_flowid(self.flowid) self.secure_matrix_obj.set_flowid(self.flowid) - # if self.role == consts.GUEST: - # self.labels = data_instances.mapValues(lambda x: np.array([x.label], dtype=int)) - w_self, w_remote = self.share_model(w, suffix="init") last_w_self, last_w_remote = w_self, w_remote LOGGER.debug(f"first_w_self shape: {w_self.shape}, w_remote_shape: {w_remote.shape}") @@ -221,7 +218,6 @@ def fit_binary(self, data_instances, validate_data=None): batch_features = batch_data.mapValues(lambda x: x.features) if self.role == consts.GUEST: - # add batch labels for replacing labels; batch_labels = batch_data.mapValues(lambda x: np.array([x.label], dtype=int)) batch_labels_list.append(batch_labels) From e4beeba28fd9b2ea0c5eda5e253311d84e7b8888 Mon Sep 17 00:00:00 2001 From: weijingchen Date: Wed, 16 Feb 2022 10:54:46 +0800 Subject: [PATCH 35/99] update EINI Signed-off-by: cwj --- .../boosting/hetero/hetero_fast_secureboost_guest.py | 6 +++++- .../boosting/hetero/hetero_fast_secureboost_host.py | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/python/federatedml/ensemble/boosting/hetero/hetero_fast_secureboost_guest.py b/python/federatedml/ensemble/boosting/hetero/hetero_fast_secureboost_guest.py index 84382da4fa..5dfdca4320 100644 --- a/python/federatedml/ensemble/boosting/hetero/hetero_fast_secureboost_guest.py +++ b/python/federatedml/ensemble/boosting/hetero/hetero_fast_secureboost_guest.py @@ -30,6 +30,10 @@ def _init_model(self, param: HeteroFastSecureBoostParam): self.guest_depth = param.guest_depth self.host_depth = param.host_depth + if self.work_mode == consts.MIX_TREE and self.EINI_inference: + LOGGER.info('Mix mode of fast-sbt does not support EINI predict, reset to False') + self.EINI_inference = False + def get_tree_plan(self, idx): if not self.init_tree_plan: @@ -125,7 +129,7 @@ def boosting_fast_predict(self, data_inst, trees: List[HeteroFastDecisionTreeGue guest_leaf_pos = node_pos.join(data_inst, traverse_func) # get leaf node from other host parties - host_leaf_pos_list = self.predict_transfer_inst.host_predict_data.get(idx=-1) + host_leaf_pos_list = self.hetero_sbt_transfer_variable.host_predict_data.get(idx=-1) for host_leaf_pos in host_leaf_pos_list: guest_leaf_pos = guest_leaf_pos.join(host_leaf_pos, self.merge_leaf_pos) diff --git a/python/federatedml/ensemble/boosting/hetero/hetero_fast_secureboost_host.py b/python/federatedml/ensemble/boosting/hetero/hetero_fast_secureboost_host.py index 24ec0f63c3..9e31c6eb17 100644 --- a/python/federatedml/ensemble/boosting/hetero/hetero_fast_secureboost_host.py +++ b/python/federatedml/ensemble/boosting/hetero/hetero_fast_secureboost_host.py @@ -38,6 +38,10 @@ def _init_model(self, param: HeteroFastSecureBoostParam): self.guest_depth = param.guest_depth self.host_depth = param.host_depth + if self.work_mode == consts.MIX_TREE and self.EINI_inference: + LOGGER.info('Mix mode of fast-sbt does not support EINI predict, reset to False') + self.EINI_inference = False + def get_tree_plan(self, idx): if not self.init_tree_plan: @@ -155,7 +159,7 @@ def boosting_fast_predict(self, data_inst, trees: List[HeteroFastDecisionTreeHos node_pos = data_inst.mapValues(lambda x: np.zeros(tree_num, dtype=np.int64)) local_traverse_func = functools.partial(self.traverse_host_local_trees, trees=trees) leaf_pos = node_pos.join(data_inst, local_traverse_func) - self.predict_transfer_inst.host_predict_data.remote(leaf_pos, idx=0, role=consts.GUEST) + self.hetero_sbt_transfer_variable.host_predict_data.remote(leaf_pos, idx=0, role=consts.GUEST) else: From 292dc2fe5eace0f46644d0ba6ed2cce938f22b51 Mon Sep 17 00:00:00 2001 From: weijingchen Date: Wed, 16 Feb 2022 11:04:01 +0800 Subject: [PATCH 36/99] add feature importance sync Signed-off-by: cwj --- .../boosting/hetero/hetero_fast_secureboost_guest.py | 2 ++ .../boosting/hetero/hetero_fast_secureboost_host.py | 2 ++ .../ensemble/boosting/hetero/hetero_secureboost_guest.py | 8 ++++++++ .../ensemble/boosting/hetero/hetero_secureboost_host.py | 9 +++++++++ 4 files changed, 21 insertions(+) diff --git a/python/federatedml/ensemble/boosting/hetero/hetero_fast_secureboost_guest.py b/python/federatedml/ensemble/boosting/hetero/hetero_fast_secureboost_guest.py index 5dfdca4320..c3cac7c918 100644 --- a/python/federatedml/ensemble/boosting/hetero/hetero_fast_secureboost_guest.py +++ b/python/federatedml/ensemble/boosting/hetero/hetero_fast_secureboost_guest.py @@ -86,6 +86,8 @@ def fit_a_booster(self, epoch_idx: int, booster_dim: int): tree.set_layered_depth(self.guest_depth, self.host_depth) tree.fit() self.update_feature_importance(tree.get_feature_importance()) + if self.work_mode == consts.LAYERED_TREE: + self.sync_feature_importance() return tree @staticmethod diff --git a/python/federatedml/ensemble/boosting/hetero/hetero_fast_secureboost_host.py b/python/federatedml/ensemble/boosting/hetero/hetero_fast_secureboost_host.py index 9e31c6eb17..08fe629491 100644 --- a/python/federatedml/ensemble/boosting/hetero/hetero_fast_secureboost_host.py +++ b/python/federatedml/ensemble/boosting/hetero/hetero_fast_secureboost_host.py @@ -102,6 +102,8 @@ def fit_a_booster(self, epoch_idx: int, booster_dim: int): LOGGER.debug('tree work mode is {}'.format(tree_type)) tree.fit() self.update_feature_importance(tree.get_feature_importance()) + if self.work_mode == consts.LAYERED_TREE: + self.sync_feature_importance() return tree def load_booster(self, model_meta, model_param, epoch_idx, booster_idx): diff --git a/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_guest.py b/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_guest.py index e8786501d8..c7c0695b3b 100644 --- a/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_guest.py +++ b/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_guest.py @@ -117,6 +117,13 @@ def update_feature_importance(self, tree_feature_importance): self.feature_importances_[fid] += tree_feature_importance[fid] LOGGER.debug('cur feature importance {}'.format(self.feature_importances_)) + def sync_feature_importance(self): + host_feature_importance_list = self.hetero_sbt_transfer_variable.host_feature_importance.get(idx=-1) + for i in host_feature_importance_list: + self.feature_importances_.update(i) + + LOGGER.debug('self feature importance is {}'.format(self.feature_importances_)) + def fit_a_booster(self, epoch_idx: int, booster_dim: int): if self.cur_epoch_idx != epoch_idx: @@ -146,6 +153,7 @@ def fit_a_booster(self, epoch_idx: int, booster_dim: int): tree.fit() self.update_feature_importance(tree.get_feature_importance()) + self.sync_feature_importance() return tree diff --git a/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py b/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py index fa83f2d960..a9eae6e2ea 100644 --- a/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py +++ b/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py @@ -95,6 +95,14 @@ def update_feature_importance(self, tree_feature_importance): self.feature_importances_[fid] += tree_feature_importance[fid] LOGGER.debug('cur feature importance {}'.format(self.feature_importances_)) + def sync_feature_importance(self): + # generate anonymous + new_feat_importance = {} + sitename = 'host:' + str(self.component_properties.local_partyid) + for key in self.feature_importances_: + new_feat_importance[(sitename, key)] = self.feature_importances_[key] + self.hetero_sbt_transfer_variable.host_feature_importance.remote(new_feat_importance) + def fit_a_booster(self, epoch_idx: int, booster_dim: int): tree = HeteroDecisionTreeHost(tree_param=self.tree_param) @@ -111,6 +119,7 @@ def fit_a_booster(self, epoch_idx: int, booster_dim: int): ) tree.fit() self.update_feature_importance(tree.get_feature_importance()) + self.sync_feature_importance() return tree From 6d2080fc208401ecea2515174c8dd50df89e5ff3 Mon Sep 17 00:00:00 2001 From: weijingchen Date: Wed, 16 Feb 2022 12:40:16 +0800 Subject: [PATCH 37/99] remove code Signed-off-by: cwj --- .../decision_tree/hetero/hetero_decision_tree_host.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_decision_tree_host.py b/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_decision_tree_host.py index 4878effb28..55f28cc0c7 100644 --- a/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_decision_tree_host.py +++ b/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_decision_tree_host.py @@ -85,7 +85,6 @@ def init(self, flowid, runtime_idx, data_bin, bin_split_points, bin_sparse_point self.run_cipher_compressing = cipher_compressing self.feature_num = self.bin_split_points.shape[0] self.new_ver = new_ver - self.mo_tree = mo_tree self.report_init_status() From 77eafc8164c5d62208b5b724ac5029f53da99365 Mon Sep 17 00:00:00 2001 From: weijingchen Date: Wed, 16 Feb 2022 14:26:14 +0800 Subject: [PATCH 38/99] remove code Signed-off-by: cwj --- .../decision_tree/hetero/hetero_decision_tree_host.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_decision_tree_host.py b/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_decision_tree_host.py index 55f28cc0c7..ca37268edb 100644 --- a/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_decision_tree_host.py +++ b/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_decision_tree_host.py @@ -72,7 +72,6 @@ def init(self, flowid, runtime_idx, data_bin, bin_split_points, bin_sparse_point goss_subsample=False, cipher_compressing=False, new_ver=True, - mo_tree=False ): super(HeteroDecisionTreeHost, self).init_data_and_variable(flowid, runtime_idx, data_bin, bin_split_points, From 99285afc4f65ddc87bd845001bdbc018a7a14aea Mon Sep 17 00:00:00 2001 From: weijingchen Date: Wed, 16 Feb 2022 14:27:55 +0800 Subject: [PATCH 39/99] remove code Signed-off-by: cwj --- python/fate_client/pipeline/param/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/fate_client/pipeline/param/__init__.py b/python/fate_client/pipeline/param/__init__.py index 7f24c5a7f4..9660a0bcfd 100644 --- a/python/fate_client/pipeline/param/__init__.py +++ b/python/fate_client/pipeline/param/__init__.py @@ -45,7 +45,6 @@ from pipeline.param.boosting_param import ObjectiveParam from pipeline.param.boosting_param import DecisionTreeParam from pipeline.param.predict_param import PredictParam -from pipeline.param.sbt_feature_transformer_param import SBTTransformerParam from pipeline.param.feature_imputation_param import FeatureImputationParam from pipeline.param.label_transform_param import LabelTransformParam from pipeline.param.sir_param import SecureInformationRetrievalParam @@ -58,7 +57,7 @@ "IntersectParam", "LinearParam", "LocalBaselineParam", "HeteroLogisticParam", "HomoLogisticParam", "PearsonParam", "PoissonParam", "PSIParam", "SampleParam", "SampleWeightParam", "ScaleParam", "ScorecardParam", - "UnionParam", "ObjectiveParam", "DecisionTreeParam", "PredictParam", "SBTTransformerParam", + "UnionParam", "ObjectiveParam", "DecisionTreeParam", "PredictParam", "FeatureImputationParam", "LabelTransformParam", "SecureInformationRetrievalParam","CacheLoaderParam"] From 255b3b3ead143b8541a423a625a51103b156a81f Mon Sep 17 00:00:00 2001 From: dylanfan <289765648@qq.com> Date: Wed, 16 Feb 2022 15:11:19 +0800 Subject: [PATCH 40/99] fixed some doc bugs Signed off by: dylanfan <289765648@qq.com> --- doc/federatedml_component/sir.md | 3 ++- doc/images/sshe-lr_forward.png | Bin 135284 -> 137795 bytes 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/federatedml_component/sir.md b/doc/federatedml_component/sir.md index 12013367a1..d0ee3b65e3 100644 --- a/doc/federatedml_component/sir.md +++ b/doc/federatedml_component/sir.md @@ -6,7 +6,8 @@ Secure Information Retrieval(SIR) securely retrieves target value(s) from host. This module is based on [Pohlig Hellman](https://ee.stanford.edu/~hellman/publications/28.pdf) commutative encryption and [Hauck Oblivious -Transfer(OT)](https://eprint.iacr.org/2017/1011). +Transfer(OT)](https://eprint.iacr.org/2017/1011). +This module is still in the research stage, has not yet been put into production. ## How to Use diff --git a/doc/images/sshe-lr_forward.png b/doc/images/sshe-lr_forward.png index 612d0bc995d7341dae91748191c31d4826fd8de4..75ed334707b41caf3c3929f9e249f89368324faf 100644 GIT binary patch literal 137795 zcmeFZcUY6z^EZs^T6RU$1(nV!(wl@PRacsHsXv16Gt05Fzv;CA2^gBw#2( zMG^>2N(h3$DkOm*AYBOMy@R^DzwiE@-}5|wy#Kt{o9p5tcXHq7%*%>m1$S*~Cdke<#oLu}je`EKNVHoMwU)Ohke|jO@HRByg#u zwDrr>+-TH5{Xovwua`gc2Wt3h(Xb{a*y$n^E}ucjzcEiZ2O8cy2t1de8qQj!R3D>{ z`t8Pnh^KA}OBUT=Jv<}zJs$XeNmWOTE%PWZpP3CU*4N^u+`Vk?OT>>6-wu;8#ZQ?)kai@QB*1f__99@+6m9cTOFqPQ@i_fY&t2`1?yQ{ zbc412(0*)9fGxLD@+_*^qiDsAQ5+fUF!=Rp;pq>=!ADL5!BSH{vphS|4Q9?bB9>7t z@*XdX$#GS$1s_IDY-ZIb*Hr}mph%>jJ~R~g)cG;ycXJ~NM;9meHD@%;`Aniaj`%jd z`?UpVV=K{=CJYh{FzXG(_}1+SVV-tDGz0Sr1DaGyAL^bijqAOCDziGgILK!$um`gm zhkt>-DbODzo!P;{QuR&kUKziRRPR(wbe{>cG8{E-T%Us2>l!AxMKi7b&kOzfe`LCj zIb~B(6W~#&G<_@yUWG^Ld2dIU(;RX2+|SIy`4c)KNudT^hE);}?NpP)zKI>rt8Pdg zV|n+}Hy!<)3d^jv8Q;|P-6JgAkl7hqxU9jbP8_AXuI&vdjn~DMQ>ONA6Dr`NTag;H zfW5DBV2RagTZFA{cS4M;Ywb0{#Gs(a}j|x>r&nlb(b4& zS(QYpDCXJ<*go~wWt8kJ2*T1Ui#?fIcl)uH1e-xp-gv5aRyCHCB1AEKGj!X+ZtL3k zhd3kO4@U1J;oUBoMR`^$gA&}j$(vaz=+{qHN)crDuuenxE=z&)@J}qP2YwijMmIRK zxcm=pJ=wT<%@X?|4G}(w$7f?whN$ruPCm~v6U5Fr6FSi5GQ)IB0EtRQrpGOTYfpW-y6yWoc@ zFOU$6VlP|~Tsze=Wihb1tcAKp8s-_cQRdDg|EYq>sGh|Q<+aXixl2x8ORC{G#&Qu* zd|(bhxtbIq7&^$q{ORbln}UN;xqHYA6=gWYt*L6++nmrn_!)`PX z0}bI&&!Q>vE)8wDI^!TFF0OOf-q|pG3Qbb;a+wM1RS&)di(EcwKE|Pb?-B?SFi>+`IeK@gz2s>ocNJYK z6T@W9T-fykg7vjsc8*!$y0WIp&|BTZHjP;%cJ;Jjo@ix3(~=LusC%~EZ8kR5x_T-y z*(WK6otf2GM0;M^O6goCUgvQ5YRAWpx+5$je@QEwvrt)k+DfJ5R=Ht&&QdyeVt)ns zgQ~qH`g9g(yY3$7tjIvlKnbfX#6?mkrf7OHAWOAD#n{)FuBIzow=RWjk2IRZa<^+~ zk!3b!KF)A5_|2Jq<)Vu}-oEpsmjVMIHan|2InxHTQkYXNd$uZZZ9tP(M z5n$eqK3azn7NDPPdvK{`J0(=l-glm+S#mbYnJN645-at6S8@tBD_}BQB!2kW+_PL5 zS)r%;cWH!*`2mI6)j%c6S9LKq3)Q<$z(najy{^SJvPdzs)qGWO8WLlXMx zvtVKhw<=17qj;z&H(5s`p%ou-Dr#L5UyKgeN+ci6Oa<8$*}?Q<(U!AY5jK7K{HJp9 zj5+qI4D^k=fF1n&m%J`QTuP9#$sf~y;|c)T!9Pp|8-Zp%%eDy>DG3oVV$fXLIh4AK z^Bi|a>Tpr|U=D_g8XbO8pW?3t)g+Iz+tcKkTem+?#JOp-;~jq<6~CEm=ca*rOe zO`Dvz2;v&}QYp%&D5PyT?-)SW9WlY8cRbghvZAc&U2r-m_Q9`Zz6dxx;o7{3^G_vl zR@d2NPcTO6H@P!=RT}!2kbY)FbPw6B9gT)MpP3u|iN)*K4^f#bLOcbM$DCg2gEsUf zt`#X}4|w1_1cEA6>PJav6-cuCXz5IjPM8Ne^V*o6kaJn-(uAGSyu8>=6J(qg=ac`Onhtx_Ij4? z%9MbnFbGq(Wr??xQf3DsNs>cQ@4=Cw%6R`RLYGhW%!uI1#=CCv8FSU}){4|xPXHBE z$Xe*f)#NsA;Q6tifB5#)9eO}C@YGRKfm*8=uAj{YBoi=gSHp{USuQ3Zy*KUH|=~O&jj~l7Y2D|1|cC5Kpje ziB=MO+PSfN-MVY}8jZsoTw$gkO1uUm;l1VyrtRi-o8sq->~&<5cmYMdBXly;Ntny0 zyc6k-&@CwiYYpifE?qk(XF_$64sjl)tV1Y75$A0k33s9BNYjLMZR~cD`&s~cvZF4s zTT@OBlcdRdWj-X!ocn%OK*i~5N9is3=1Fc0cu;=2gn&4}TJOT*p5{I+1Hh$07&gg zXd%+5JA|n<<>t`aYkDXx1Jos3!jD$}phxJRWP5K?I zdAORd9*EylAQ$ygqi86hyRO-JxbqI2b?ll7eqidB3lURULW)8x$Koto%Wu3V)c1h- zk`g88ZN+nC0jJd`z>BtPUQW2U0-6lB?$dQX7RR$ctn58vQ@oJSF)R++Erj`+_8KN6 z=vD)}EU)l904f7-(>j5E<&NCyRN&c-P>&R@mjFpD23KxnS+Np}6IhKhZ-Lr4ibkx4 zbf`H>m)2fvtq7wR%`0T*h8!4aS*0p$h=z=uG>H(lK-Q{FGtI%})yxL5F+~WRq>+!4 zp14TXm>XrljQbis!GKi<#TmX^cv@zQLq9hCiVBAY>&;@%9j+Xm=Jc3J^Iy+w zl%f$vN;bs@LE**z7pkJ*-TkSeCBsbZ(3VIw!s=Qu!xWZRb$Z8b%iHCF#r^LkKoRoL z%7e;(ca9>EYkQS&J*98;!{y!XMwbLb9Bw12zSd~Pa;BdMJ6xHl=hr_48`M4E>xA?1 z->nlcZ8p5Y9GB&iTH)z8Hb?+;iN8Q^i#!dQasG*Du~Je{)t&msSnYsmjYkS~MLDQ4 zoaT0%SQ_ekehcR5GDbx`4|vu*{Am3K?QQmm%8{tc#etAWHuI`YaYg0mxRo7!=3WVS zId0ALx9u*qg$sDEMtxj`)&1?ESkA;7jtxbU>WNIZ!{RtPWB;bSpWD470@xtL+}mX4 z99UT5g>25*gZ3+g$j-5EVeL_tJL7*% z|7VrJwwc?@a53hp$$UAA{x#ZGH@GT+8H6!kKk#9(W;KcaNJ!#BGXu}@7m&41=;Hk! zcX9suo#<~50RaEMm|S>uZ}aXkjXNi=9zFNqbH|sm-?AZ45M@%W0dc%k)@k?qucjr zp9TMYbz`A2ZgnDuHuGG@6W`z=IfxueXy(dZwf@ALR72e|Q$@Hh2m zaWwq##`ymydq21GKS}%1n}d5>*ah51Ye^HMDm}2Fb>>l%ZT`yRBH4Sl$_B|`@(r>r z*`6Fdd-2E_Vd|CB-SH1&!ww)uGTgtb5!+O~>f$`Qk(x4oC>>W7@##={2u)+nVq=ki z&q*FS%ROf^mjwyxDYb$+c3wP^CPD?&bO-=Ch4%VLW7A~#Qh(@`E(HEAzU`AE+cV4= z)g$LK&bmoa4{42jb^AU^z26FWkuDuYQYetUnw_fLN-SJ=EUQKEOVL=Lk1DH;=&`<>y|qz$W3@8OB1!c}*Wc zHQyOo@Ekmfw0UC`Ol1*wFV|sT`3X@s6m~$QMz$YFF}%8$xeQMhTiOOgBAn&bK8aP& z^}?u4NxghFCGv;m&v@CnC3JI9S@bb=%78@HMnc*-xcmoyzV+t*iRTr+oJKCm&Jhej zN3JK`y3CYFnQnZ}*^^Gt!b9EQJBQ>bJc8!0vyGF&w=8vOtmpKgP774_02V~u1LkPn zBO!11N24BCT+&pqX`%U+Ts(3ePF33`CZU!QVq&Ng9wh|CrqRrKD)(;gfWH z3_Jy^9)(c8;~`ID4mdxu(Y8MzTmF&VwE$f*e}RU3bN6v7i$^~%jVWV{o7l=v%Oloq z1;ogm;Ralgp^%M}ty`x;@Z`lDS#&$Dx^%5|ap$qy`vy_PM%mxmwTJMS>)9_9eU7Fr zQmZF|R-?K?F_S@_NQ&DApfs#~P}H9ME&1S>=RJPvKVLq=-;#AOI7*+^K}h{nJUTM^ z@(hVu3HP)#K-VxP67d-a&0Q0)*u_onoyCaD{zD6sxKB1^JvVri9*jng&G5v?4r4|( z6flcu*rM)Rsn%USiM9D`(nfpi{#YIX!}#L`ij#_4XbNBr)RE&u$S&$17UaU3Wmd%6 z<@G4HX0JH-2F~wtaL!{VrObm;G^|j)wiFh>5E07_WjlKL@FS2;BCo>A8~h*#!LkcS z!YA^P3rBM>ewr3U8m(QCAXEnsQPv5C7CdxVA*mRjKGm+5J&@&^pmCEI&v_6!%^9~g zuW-5$mU6!EQm0QAk7+{cZ|&EBR6&YaA*VZQBsw0jB;vJqs>fSi-?kc5(o8F==`QbC z9%0BR1lue@-zT-foC5w-1qZz9g@eP{3OU8#p0RMJVtKT-sfN3o9m-}Up`Yec*U5Y7 zXM9FBBWH5Bft>ny-9$gusW??shxyOw@v@$nX@dM=fHyk3qHCq>MXS7XWe^5>j25Nk z2KlU$J8xSTjq8;-AGc2zdts!RK8GOu4*-fS-Z$gEEz?Z#c~|N?A?)RJeRb@=@Se?gpCm=h+XgIL3@xt|cn$ zUOlP?O}G^%Z{_vDjTQ5J_A4Jx_Q6Rj^&|c?X|R>K0}r_nNNB2_J&Pa{N6FNYy4G=z z6y@7T0aF8rhKiaqa+aoh4Qw3ZR#HDjhI!i^L+0d}*;!27qm*^Z=|o@ThuSB>9KdUy zlgG^8^y-BVUQh?m)+bA##H!(*1*wDPaq&dk=e!DG_!(bzpXmM*0`?I@(f6@dfDjA4 z*HKHL)DPAsT?T%@s0N?C4^pozY*C3eL{FD_&j7Dc0hJ;n>Kzd!VOvf?-9+4COnD;m z!=)smLCtKibZXn5Ha%<4KM*KX-q*xUpgUcU!DgHx_ z-kt$N_}2@ZQ(2!^8jXY0DC-g`VAIU&nd3tZ^N;Ib0B6{G_b};bG)!uEB9NqzZ1O7+u6NdC=M^<>Ic`<3{YvNBOa(M$sp+^Eo7(xDgJ$7qK%nADpDI6) zNqCGXSy8Hw(ub)|##Hr~@+e6i&pr6)4ouhU4X1Jp2-C-9;xjq0g7D!nwR;O_fxq|Q zvfZv7!=@5ul$XUhA_cWifjQU$GW`XRsqS}~ohu|b1& z3UFTVfFc`iiN2HGn90E@7g20YUhn-6&5u8wFI-2C*ygR|F4de#ZNiy`Erd_|4|R+V zQ_J9*r$-&xLJR#yiJZ#*t1uT=?($;HA+@P}t~eQHHSU)$YBV+Pj{W%#ZxILx!^9;R+Ny3iF0p}oC( zM_+14_5o|Dw+p}8vOAc6TulYlCZ|cQt}(3WS#j%j`wZykMf_L3lx5y00|5xt5bW|b zOpTPXIRzlJ5-w4WtS*Zj}d-)UUX8H z8g%vGRBhK>tJF|6OvEsGwM1_p9x%Tt&)8ErA#%N;8J zzn;OD9f+Q$8l%;aipHWo9h*e^NxQJ(w2>SSKp9V{pk>{-N|o^)$$~0A)tWr!{dTcL zeM&h#F+TO%>?R6f^$KThUV_4OhNf%$-d(NUqK?B}`U}^oJ zH*|C4SAPLl7abcK0`Q1pxDSLI?>VQJDO=aE;~)BxyE4sq(gn%&!iy!a2uU)#%*#j@ z{v1X^2p=FlK{+F{P0l)dZQ5ii{%bJO-`pEk@j{kcF5IFT{{${#u5Yu*9F-qJg;rKR z+~$2^%)im;4aq~TaH~p}+Kpv3XRmvE%t}~ts7kr-%byQ4%e(x8x|sIj<*y(CD3dy- zOWz;IOR_KrtKqR#9eWDnr|j?Tx$HCAP%4v7D!n&;DmOYsM4TP#Z(6}_l`r0&&bTBY zH=x3p(4W%c_S9dZX!H_eKKW*8ZTfF|GWb=cb!_%&{1pN)p_@GvnDX1VD)4nx@M&xFfsex&hJmk_WwKFN= zaoRy1?Rc9Dg5!$*ltK%UN)5|6nTGr#zfS}caRG2#PrJ7Fz(@g+(zRwf7;Thb!IN(O zeSUfuQiVs=;cskwhK9yCYewWJ(%<);s1ayp-FqPd?>a@9GuJ9N)|xK@jqvwl6VZQg zh!lbZ$@pt?0ZUeS!<0|j=nvP%sQx#?Y|`Teu1bTYTZ!|xq7v4;`k?K;JbCH! zaW%D1&sQ#dD9r;LOcHDugIWpDNj~r!qF`Q3h1{?3_*12t)Cv(Jhg=hxQn+QcsHd(T zqlWYo(GNQBEkJouYZrEGiT7fz+Cs`04v>I?d~dyh{uqOatQ=Cd-u|g?etkT>xC~~Y zXyo2qnZR&n;ZMbEu#T3WmDSUa z(d)&2r%V$N2o6(o+MKs4`7mBcg^cw)iHI%h?!d?nY}qvkCl~FRH5cXR_=U8;3*uGxs<<{iy0ldDlc5soep?K zaXEkQggTA0z7(7>;Rd$|Y)ZsCyUT^*ZZ946*dGh%+ZH-Dx$u&NL?>aZ9*~-AoKt1x z1%do}3DV{gJ6!A^fSRCl3siUn>T~e`nUkYLa#n`TmOq8g7=0aDXLsLC)x&d@q^q@r zRYPuLmw%IjO%rX!C}M~)`1WHnV-s(?SUU#p19Wx1CKAMUp~){$M|o=MB5QJ38HHPG z%+$6zE5kyBk-S;4G&ReE(!~3X3X^rw&012rc$jR$4|r(m!R5)Z~t$bX*UpVZ$_ znkoYZdmNk^2tPhhYnRgx7C6;TTTWd@eZZaYaGwl8VP#&GvNl;qPGLifswbwP zgV0LT1zuWjHrU9p>YJnrYHd2;U1e&ciKE=GwNs{%sk%&9H1Q@w`^;J*)sV|(ld!dt z(U(uZ6ATv1Cm~t6@-HTQz$Oxl=F6%|h4J|Nrh|P}-DKRBmEx>#_s$(PX0N$I#r^rlBxVWeqh2O^!*mo0Vl+CD`Zly6U#%RtJbx_3tMytaA8 zv75j3AnPRKmR3m-l8PSeh*&#dYF+C*QHX`GKcy;lIhFWmD9mGOi?db$x;D=>Iyu5P zAX`Nimg;Ngg0020Q$E>tJtWREo;{#+P&}lV)&fjKG}u{Z;`W1FSGQ>BqOL&j3omzi zQ>>kMyptK}?X@#!+jU(47hm4nwXC&S2$B$hCa`55gn9c1iMe})Cn&`A!VxrjF(4=J zfyoR;jXdTZ=+Kz4LzT2X zc58n9b-Fh|z5#el7zxP!UVF)s>5b~mEX9*p{~`;e2S!^w5s=@7L-L9|)6*!U+@@?v zB*sh8337x#QD4h$k6rAkF%Ho^uJalI)LGO}K5T5DJ)zbHi@RkhQJu(h|)ut4I9 z3xWod;rW_EN{~Wq>OK*P9Pvvv?mQ4$CnDYsOBiLSxvMfS8qA2Y#fxSdfJq8qVZfI) zTkEXzzC4VdM8xc~%cQ0HFiQSmtf&n?)8^`pX(AJw-`|XG@vc`Bm>gk7j8b?t8CBz@ zz~055ks@*S8?1`!$Oo-=8!xPlYe^$0qv6_N&6N`bMeUlSf z=ey5^DKY&izJy?1G_V;8!c|F*>KVGD3HH3*jOh?9)9c}3JjsWgqwKhmxc#V zeyyq3&>g~)$*B!x)jLmcjD4t5$I|VvByCD|*?&dL&ekNAP^7{mcqSdMtE{60F$mI} z?YdDRun*R8eagD8#iCgIG~FKvD9!^rk4-cE8q+W-h4}zk@fSRsd09Dyis%@o4H?xm zhfRsOM)Jm-^Z!b;zi$>+u3;%A4Ngyk`{JmhB`(O9WzBCpm9MKPnayrbk0R*J9#bBv zT_hV^?3RBToiLF;-+}Kd7*I*Clmsu%#&?oRna|iw6XAe!6q6SBEs)Vg<1oPf>idjh zlzG(s`IM%^3gi+F=qe?4pUkV2b=VU#V!BxzoakATEvy`~YFn`~geN^=&LxIyh6OcU zXo+p)cpvI3CPOL}Z>P`)YJtGShmPu^7&Zj>WheFzd_9ZZ8eV=Zs;G|VM)aCkOn{Xv zV9Ew-(8aE&Lsxww(})IZd1i)aX>eNF+@(i}`F2fqHw?n6JVt3wHeKc&7Ai~bi!K##A?*{Hw>jKfhWm6Dz!@i zWm*AvVj*^Vt}xsLVrT)e1^g0VmRjDPU0Ueow9$2UL$tjCF6-r)@6VbNb5RnQyMD2~ z7z71;Qzy8GdVMMo*H+TEBuRjOv$?iH*&5NA)Da6o zqla$Uj`Y+b#Tt{nT-?KK&N6|}4#F^96~x{KVtb2{ozOojn}?qsB~jV2BdeUAmJz*M zbVXChgdGf`YGHM(|K*EXH?^(5q-Batd?9bFKPBRk4nXZtlveL<_bq@pgWU9oc@_DZ zi#3pt(|Dc)lR$KpWaD9+bcKmn`;A=q5ePMwSkXQ>0J zd7JcaDj_}qVelAnq7dnAMx+}Hmb9%<#JxCVO~_RodCRFdn)oD zB*4NVA>VeS#D1`E>ZeL+Gks6&bu3Jv*t+SO7=}{M*B&m3_sQD~ekxKWht!gF(6OK? zq>83b$I@FAkuph(U7qBxJY@zc@Xc$;uVvDfMtRpvb7UkC*;{n~xKELp2!`J%C>Z7i z@GKKn7%pRC;TxCuLMJ%UjD*h|_r@F~*|i@cm%sEDcP`N%-U<|hIRyozFo7y4_yxqeVy z)}{C*KiHTKlM40W3BtNJk7*dk*_+-0ab@LQosa3wipW&L2D`ht$MVS^MYKax#W`6<*!laz4eXw zCn1o}^^biDrS%jSgj#d)Ya}E+9uqDw*f0+E5hv*Z1gv>8mIEt?6YVb#3u+GyQ2>aI z`u*xfL(qm+`Q}3(vSv&FXag;jhW=-+E|`rMa!Zi;2)ba3Kf+%M~0slExH z*%L~Y-J&H9(;U@gkGvDkYU3??xHri1O8Qb=A2&lI8$^PXCKuHxR{r5(2c$4CL2)Ay8W5q z7l3>AIOPd)Ic?oiFTZ1R5vr^}<5x?#xSC;XuxBraa}V_U!F4M$f%s z5`Z&A-xAu4pQnEDE>z|;={`^wOT7r>+Z`{h!!ByS468T%+$hk9+w6;O&?uDaNF{RV zY&bBE%$3QBDk`{}Is^!+;LF88@g8_Dk2B-Dg#v6-h7*;ivv{`dKNJn_wE3yuuTO8* z`fQerX(d?YEUCoTK}!4oYVN=lTb<+r%Sh9lV(Wte<@U@O#pt#hA8ID1mT|S!lsagI}0**9oz%H4+52NF2!%cjL#_Q0l*=}rjAY%_O z+znJQITK+5hNpb;q}ZDl0^aQ??W}E;5vi!wkBQhJG@uq!97qH@PZZykt)&CUI#W^I z6J`82alLxGN!+VHKLoz$Yg65up}omK{+ibF#P1HX2v`ycbXlx;#!}X>{LwA~Xl@pR3cB$FJrXrhf zzzfI6BrO9nfKtmv-oMq0_WStJ4K3N-+-fzC&-D@2RjE^cRx@j4v4l{?^{d_*ylnV&1m5t-Pq^neWzCIW zyT)Lcmswh3CE!k!998N|i?!dtc3cKoE(Wg!p>)}+eRIf_x3AXh6N*qWW9){$qF(Ka zyU}WsM)0FyelDEKLv3&9%r?DFuz@9g$s?!yH+0f9^P7WU4_x2r<3x?0{^H^&jd5qg z51#A2i$XVrwQRV)3r+geTqizk{^Fh`3VPh?J^?UK;OxXlrnP^hz6aIJX`Br!5^DiY zh(+JqSoJPpf@^`4^;QF{tc|nN>TXvhsb0z3poHGRdg6TC z$}$zjIw^W<_P-%%Q};yn(uMaC4ld2LPVRF1RxQ5w*y1wI{6-E}-4HlSj-SWV?2~MKf}y;lfqdpFZdo z=SMbT_aU>~a+vxMTCe;&yVY2fcBoH#r9W49N^RPezz+ddja!yuRt@3KJK<}W0t99L zLpo)Ebhazht2~V6=4C5EMXMU!b^L3d0B!}n2lhB`!as`QodfgouVOiERWq8=Kqw7|DgY0&+VFRcSMNIj3UGS3 z`Zk8(_u_oBtAw1jj&q0X|f2@5&?8?-8^R0tF*&dQ^8_up5GAC81 zQIgxj{^gkSQ?|v0+N@X#dlZxkp*gkPM}O-Vx3Ybmo>dlQthHEY(JZQ-#+lpC)Sm@? zaGIy|jZEr^q)M4%|J6b8RxDXa?9JTFG!{=OcP^1E8MwtHq$A);QkSAvq;BmcuDYkD zHnJ$}%`Y7voB!pQ`;5X}^IxlO;YlAJC;Aa<32yOjr#*cTc@R?iE&z1ydFUBFGLwp5 z^jlRw+dtnnz1r&d{+|m_=Q$n|g=L)-^>FLGU%6hmt9cCaCX{OC9o$Vi=8v-WX6v|v z0kxi20|d-((k=L?>)#qIQe}l})O|sxiLxCZ+$Ut<>w0n0VEySzcWm?`inXLHQJ0PE z{)-l&}Zml}y!%~ls;O`jA!^UbjP4z#K$G{lkke;{(uz;$ZT%lN`z5WaV|lST&B(%BvVr# zEhee_r{yT-d8`$fOf+hY%DGV_l1%w@fpwWJjP2`2ZJgU_)XwX>N~SvzkfPIn#+^nI zek+puwrb+x_?Vd-Tq($c3D_GkZcdw^8vU`X&f--%BYUz}Ki<~&n9W&&RE7fB6RR!-!sMc@8Y zfG{th>s3qOT5(&5`nRgy@Ae`Io_Ar{B7z7G8Q2Fdk&e*u?p zWqUtjG&vh?v1rvzAFmOYYg63lxJiAh$J{uipYJB${13zw@Pj9IH{{~zY=1-{^4}HN z6us00ice0JYJk)RlpSd$i@YSf9;OE4=OECuw7!;{6PZ(AMioe=}GT}w!G9v_rvxN$Z zDP$S8pghPcvs#;I=N;dhmzi#grT<4WE*KFxbHiN#JY0=5A-;#UY6FE3bPxHZ|vUXih<3hZvi)+ftq0%E(WmONZ@b zTGl|-{6{vxMQJ1DZa>^?kojAmq1Uxvl#EYUf^2&N^^aTGl0=+An~m2?X;m zxmC>6Bw4sxo~~t`ubE3_zR3Br4QARn#K4adu0`_S{^zrfGB1J0B-y4XMNBWr3UfLV zR5pN!GR-dctelgz$^-U-!K~PTr)lY*3Tv~LM_=Zsty_Cz1_d8J6j)5_t3FDm-#|4W z>Oel8TfLp@EsDAZpHt<3pm<;GP0tG#C-Tm9A5y0>q!PDBqvs-t=n-v>qW+sZnCypVqZSaKY{V1>vAn}!Lm zKiJHQRWsokv0XNmll=O7w2@HUa_~@2}AIhGi^J+bMQd^V3Bv# znX@)R4r2|e+3Xl)4@PF;zm^|?Lgz1>m?Xn_+mEH$T?)V=vTTX+X~RkQSoj**E`<27 zsz9)F*%C=Zzhx(9md4P+jsEq`mf1}Mc`)j)G($H$Kr*JvWT(q{#CdS`5n94x%;jIs3! zVM_C3C>ipsinXGmjty}7&iZ1ooIoNwmq{>tXYUQPSKNRg_+i4c4i3!5jkt43to7TV zI@Z?4U`cJ7O5|Y6c?>8fB)1I7oV>^j5Dr%tp-t<_;dC0^kXa#|o*E2($ds7IcZmt^OH6qRN{{KV zB#$?1RHUGG40rq6K&Zlur$hLOBzGOs6E&Cz_VAC=(okOx7)R|jU1+9!YVmL?9=#;9 zHv{3xRvM4oh`p4r3gc*a7wR+HBrmd+D9B89>ens-VO!T*ASQY|u65tVFK}h`%Jd|7 zHkQ@8%8FPMuI3pixEc69)-p=kSQ)m%3_O?Cg)y8W^{|0+eAL`kjhT=Q%6^2xb` z=ce@fEi{Lb-dgBKYyH_Tz{4Bgxk{n+$12`6?*0V~-dCaLD#HWTe4v;jMXUx;?h&Wx z5xozEBo>7Tk{4{f?ewtgFf3ta43+9@f20eK*F46|L*RX;+x*T?l{-fol1}!EgSj*c zeT6DSTnBVO0pLB4u~yd?LMCv1+T|*PuEGop1=*+V+DoH(s7-FZ>guljf=;-(x1IPN z;YbYLjXRxc;(nuZTnUf^(gNb4B=r940hT*1PC$t9*5}#l+DnaLjq;~`&gWmu8dge! zb+;_;v;}G5z4XVXo&8KBEv&?qfP}rINve#vZF(P^9Vki6e6S8f8`Dv~B;g%}&$^wF z3gZ#;0+oVP-&fLiNj3jt&8YA)fcu%haZi!U+pZ^b96#mT95>_F=D^>w0*Z?*wuAv? z0YT5jMh@uZ@R!clkU{P@&{g16LKhuo$6Y8A@+Xi<#pdynUMli3C!DK)6+6Hn~pqj<|VxZ}gUwAb{fFz;z z+!8x< zTmcdv;G|>o6AH?w66HV7ECgXH-1FnzrA#b&z(xC|A4_5r`o7CQ4^T2LDC8CAoHGOI zubJwHxjpF8MMWw-z#z_!EKhfj0RSS$nNq-_O(@?d3!D1S%DU z0m+`^r!R!j7q|*yqKsFxVPC_5^pOy$JGN^QM(uCIO-PL9n{9HqOaV!;v{+`UsHyyL z*QXik>;T93HHUPU9&q)b3!c7Hqmh(k(iQNJ!a}|DKAu(>T4l8l`x>Q(w1F|O@WgR{ z%*jYjdquZ4m@1Z!`d)PLaL>knJ4FQa+{us$id%{Z|LQr%_amX-&f@N8ua@vB>6rrM z122U-_>tV|h1N;73eF29YuTbIdnIO(7U>njCdKkT##GPn%JKnd(lk`xuVnb8l{U6G zi!&uj0n;68$7Vj7FBX?u_&0RS&6@w`!}NhV{+04t0JxmpP27prn|%F2liRdbRUY=Y zx8}}k`{640OIKoXD=u@(*i-zj@Xx%+3b-6{NvfjrBV#$vNv=W`5=6NBij8n?FWm+Ud+>10^tR~6^4rywgVpsEE6GbX>Tt&tu0>)bBP34b?NpjuNw z6Q33>=_#N+h{ho7X_m&|=uF-x6i)^kZ+f2n&Fw@*(9)lo}DEg;eWz#{L z#ou5hpHnjW^UE*3K<0YRuXSK{9%xcn89xZY`N4vXv#KGC7p~RB&*N#~-Sl~xt&cBc zrHB1vYX2j#-L%hZyg!ZTyr!(}?O~bSwgF~e45SI{*hLxV43j6mdIo)0T`71v&d=E! zX4jYtEOokm<(2lvK+e@-aZJ|(&QoE=KWS22W)v?D?AZRzb)X%{d~Ja^&~$MJYUMd0 z&?GhARtJQ9Z+YDY&-KLG{}o4kPuEOd(hi#1d#&2qZ1X;35fgoqn$g6Jg=Q)}tVeUK zf!wR0#c%(a{uJ*rECyz1*qr&uJs;5iCvSoX^-4C|P2(2w*G@eY%kh>Ia&ehOKpIQ-2)#Y$Qu)8Kwns@GK3asw z*Yym3-%s8D2m`()IXO*g>MBeYG4js*6stx1?Z!U-s>2)}n0 z#o-F`5bNJLWlj^IlJPp!iS3!uKf=Lpi^E3Cp2Gihfc++lI@iX({(RxwiLh%M+qBEk z_O%f7%TI9&+beyGy~?y4QdB1Y*1Jc+Nf95WtQcW~MoLvh2dI7DYBM50?c%S(%#Y0P zwv z^cHaHHzlZZBrv1}Bx}TwhhPRsiw0HB!xOjLj?YQHyQeg9b9CS1hXHLJ`LZYSme553 z53a+R=kn!a*nsv0?h5Sp@&#~w6mWBn!fIcygfNcrhsYm3sgGPL@B`Uq94ym)k$>=1 z_pzZO?cQTYfd+{;GYc84U4e*MTsL{>=i#Vam7t2Ig~qwzQ?BpCjj9q!ab)@3G+Un= zx_?OR^1i8K_ENz7Un6~Ofs=rdqlo_<-;po(Rs@u}fd-9%uNkTU-vhtxcY^%Yq4B0{ z!Fpi{#{fa*b6-52Sh0Z?(>b?skkF40=eWz-^Dk#8n&V_^O_+@>5-WdxG>PizSH4=N z^ZihB`I?)GAy6(Wqk_Jbsd6uB$9#~#8&;5AR_TA$f%}9(Bes|KN_me#<8KwxFn+3hXV~WRzFlB4 z&YrwERrC-(E}NMX^{_lu<4uSF)wA&)=8(F|`vxNcb5@sa?0t7;GYNzG32grYbdrB5 zmCW~T#{P`{k<5eL2Zk!JJ%MuL$InE_*Jqb*&ur%D{uQmOzEe>Y@bLNt_$l~lI2Sxu`PDZs0hr;h5>)Iuw zxE}DwjX$3LVF$f4Txb{NQc>)^CV{+FLf7B%bSY12P)w7dckxWT5qPl(5w0Z7y7qAr z3cTYtYOI72iG1sAfQK=BNNe316*Rv~iWWswt*6LaQIG!Dz8_|{+$F<>mOan$vp#!v z8zGQw$Bl{Tm$eZqlX-N9_1(SxmOs^7hjwOlAuoZxN|q5efWClGb{`1U(^*^}#J)CT zpP7p#EUp&>Yc%-l2-gv}>E;-{iuqZ)n?s-#5I%w^e=J?`V9o z z)821IYpgXc2;i0*P@x#Whm8Pzh6j`n4jaT285(BhDr?GWFAY1CIG5+roBcAmOWx27 zjt`~|-B~;A?|~at4Gl*KQ8 z%=W?c_yn`elIWeJMH{X%keLg5cW+|r0b+FpwMWNFOj#b5JPiD{l;MGHGArmf=EUq& zK!D0(^U51ie$4XbSF_3j&iHg})~6M12?z5)Nfp8goYj%}d3(ptRqfTU)^tjh~0zZ3d5LTg;si&2^#-0pGwF?sJM9pNSls2nZ#{89XXl-;6glhp{#ORJE#Z5# z>P4TXY1#4n4bgt~fPMuWTnSONT3QWV|RRrm+MZ`y| zJ&&sXxb!NlZfh+OtmfVVwCFM?)cm~kcET=ofWXrAWNl4zx-VAgkDv*~0!nidFjsZb z7e>Zl3GD!#*5jrtA_F`HFyr&W(LdKPj8 zHvL2TUGirg{75u1a_pRI?`XvhIaf=O8jmw9ENnm{9?Ko~lU$V`@LJPh6u99&@Pj_j zfqJ{lMF@GO1x<%6$|7P_ey$2$yY-A9kUep%NeuW+E5aqVHh=w7xXBzv^%K{R0Ijr= zT$xo$+iiT~o|Bs<X!ufv^e0m zlb`~RZYl!?2bl76dFP6#KiL}s(;fV#h+e=iWvr|PKwd^H#&}9$EnoaU%)Mt+lUeuh zjXI;FBI;nFb1RAz6$wp{G9wDoL|Q@*8fs|Lqy}_UP^9-NgLFa&NN)iPp#_ipJ`Fv-dvx?EO2nHNPg8)xJ3kyvX8S=#i_> z;P{j`h0IN@kxC{*Jc5j6+}ywDoT!*v$D|J8wE)=U!4_=h5cbj8QzAv=SF2fs;m}f=cG6D%<9*!G_m~)ds*>Lrrd)ScJ!Z?5C}LZST~?kv2`}^I$6Y zfEXh@XA0SeT1CR5lIKR%Z1Yu=3%bX2I{)&FFPNw?E!@mz%nD-1{3cqIIX+UuD^gBX zLap}*w!K=W?j7*I4*VByPN zw+{@D<6v^3;Mez;#sN*P8fLaV4`(0TjxA1+@fd)2(i=ArO2mt7O^VP=spe2tn?^Q* zoSxMjl4%WSnB|u+hn}7JXe$NMGmazg)2w~a8Cpc_8P!r*?^;cnSF#xbOx~r*DC%XR zB1ig4zD37QlMB>*UJy&DGYZBENRXzU96j2I9#9Dmhct}}^|9w_TbHz(lbPx!4Tnar zuVBbiA&onwRYj1NM>Pd3{hs@0m?~{butL1eCoYp)$-QbnPQZxK-(-lK?`uq~I7-6@ z%ms>7!P64pQ5^CsFH0j-XB8~=VC%0y@E$Al(9{zouq%jQY<7mpxPmKCIuA;b%O4s* z)gi{5RYvQ%ik5vHiex4~h;^dh6v}cHmFQSEO+08r)iP9qwrhU^djC`-8O#X=`h!EQ z#$+_Sv6d`wf&D+EG@9H8Qc`?5j@P~@1z1ZJXvI6fB4ce!=9$l8DwmFKo@kjjO4>VF zzf~r_56;}n*7Ol<3|X!EP0A%?+{V?r4UI-CZbsZ#v&{GbqcYnr@R2T%1GULMx*1HKj3U-c8GaC&}8$#b4x_>Gezga)^bm0=96^eo1v)Bku zpIc8&>??C0xfuSk=gyY=Fpvx2}WVd z(V%UVt4Sirb)JA!7?RGP@#}7HVdkv)w<&V@`Kp!+xhT184)QL0*f{t>ZQVN|J1|e; zzhV)4DqDb0F0EUNY4O|)+M# znWqoYI!6jmkH^`NFS0z;!;f)r zJVoGWkq@C|59QrB!m@wdYq)Q~46LZ1A*lnJ#Y3CBk(>kb@MIj!0usN;_@R>j>@K}9 zlY;;i06qfSRjH=e(OZ7q*DR|i_Od<1JpR5iNP^s0SHH>Ak-O4n%LXct+5kf~NQyg{ z$vrWVIm{*xp4G)wM2}bxTHIA9Hi3psqks8k41~;pkQk(+EPWbam{rsoL7MPT$KFE5 z5xDt_fTU%-w<7Yu#^m!PEZc{-|A!7NLaZ%F2do5I4as$;6@~F+99xM5n0~b)mm8yn zCT25P3T89!3QZnv0S)jtV^=bg^$6HAwox@AmVL-|;@dE!Op6c?4*H_Mt~~$OV%uX? z_&K-S#j1*B{I4orL4LdwrBS%TjhXza!l#MA=kCXjSe5K;D*j5$W4hZk&Fj4fXP$yT zB>nF9-1!5_Zd%@RC*_Js@Z{pk+LP~qwNLkiCcY9aD`V&Eb(~EbAS{!9ts!G!M$SKD zE0-8I-HI%MadzvL!J9Yi1A*4YyNS+0@#5-N*O=d^bsk;vdH$`c0`A!%%IlfqQv#dS z%b>(G(3_Re=@6M1!H#EUp_2Mmn-qZ-Cdopf1ytr1>Vuz{qliBpKO2&RW&cQ}H z@{d6oItg}eAZerBe5$Re;bSoGfXQv35G9Zi-{?+sX>hGbF~XX%-WqUxWG~3rdA+lI ztp4PmTBG+~Ci=?Y{E z?{Tk%XHxd@Sa^${D*2OZ%}jz@6M*_wd zT}@@_*0;6_OZKX<5K zw`12WTI-Vv+1{t8Evt*=yT-t>5{1Ud`(Gg^4QnQ8zK1PWs}?;#GBE>+Ulg{Nj3Vu| zB3w@Hoi}nVp{v-xG5?v(Uj5$+>lB-w0wAFCI{gIiYQL^)EA?3qOt*P}jwB+yUO^re#01V!^Ji9@(2w7M zuQvNHC2}q`pvqt-9mUaEMtGQs4KeoyKr0T;OE&wxFC~kHzjX|(i4W)B7!VcIi1>))^t@SJ^tGeH?qty8|B77i;lzx7TtC6B)qfFV=~3s0SfEb z*ruaa{h%dI@TJ-fo(E2JKd@U**%i4T^q|~?9B$^pEfQ{Zwkr1xw5U@u>E%iGIrf@J zWu!z_<etyqtDlK!Nbd88FosGz@hHm?(t;OGv!yO2MiVGe~ux-6{*pYo+X9ep31> zo0;UIQ!2r}Lx3x|nBg9lj0c<(35yUO%`GbeFqJVSEe9Jv6?Ws<5gICxRbcxwg0+CL z#>r(R7ZdCW3GDFB!ytqK-552D_7}Bvm;FclKujcS^DD(M6kgp<>PRj%jpq5-*+gjh z>9Ffnz`?ZHJ4!%jTtr+(Voiptf^aQn^OGw{r)On~A3;z|Cf$*<7n^Hu& z&?ewel(9Xp4DalUtx3W2V0p@RB9DT&oT~_mQwSS*r;9*fpDLw6C8HuQ+@3r>y4lss zBURF@SWd3l5cx##9#Gq=bRY~&i#JzMey!b)bN=hgn}DYUf5zvbxL*BOjPpvxei`Ce zGq8Q%2w~6fuNK3B9o4C+`7c)usTJjZU{MnDE4R6tAxVpRw1~g#9|t?jK`2^^YfHrk zPe{%m>HgePjY>`Z1h!ViffBgMMv+Ni%5ijO=Y1g%z-dqNE7NF5$4wH;GdExeo7cI! zCY_Dr@t9CJE5s{|HI}(%k^Ai$@pJ~2e?0H9@7IHmji$8j(E$65bPz;FD?wy`t39A^ zu@S#x=JXn&bZ>IhD?GDw6EA}lCCDUN%`Cvc z$T32X4TB4anec*vPl&3*hOklZ!0C=897sAz-4sq@m*U1xToK2CJ?!{V#y5Ud;Q}QA zV@d(>F|vWasKUrx_P)&A;oOa8*qZo!zidbpvp@c^#WEdE1{hrpP1Cj!%@iRUZ4U#6 zomUKe1bwnd;Bi_Oo+-JhTTT&gX4Ix^wuFA4ea3rUyfDrx{1yU$jctHKWn+7`>om%A z{#Ytmnv~RxJTOp^P>yr2o2c~jYSrNc`Dayi;sTUUVL*uuuw(e`Fp!9iX^Ng)f6-{p z*8D=bewB4C56s?U>wR1&ViK`_Yxb80rflL-N=Kz@7+A3mh4IC*`*mQ5BhB@57^EWb zl7bh2dtR4zpmi&Dn<7(wI0K97wCZ>kmIEaQWf1-QTlfGr^u%AXut zfMU51#4ZxzW2XXlir)#h!-Vb1$*p;hxmjh;i>Cx=9uIjhP*{xA^LLD%J<`nQTq6yOHg5DRVF$?D}+$z|*~(`;)L3C}r0a z^UuOtxCpae@ohO==U?nQ18s5@Ar}>og;_0O3NW9?3kkRhxfED#&0M%;!&Jo+cr)Dy zucs_&LpA#9$FJ*9fmczf3Amz*b_rt(dp^;TB4w!qGiUjQUB_??atvmVYSxX)HiBwn z-iR6af1W4nfM}8JVGgxIrNQdFDiK>r+s;?#1e+O+2WY~f2HpPD2Qk_z$pv$o#?Ign zYzyR%Bb)jA#+tI*KD7lOarc_#U9;Ld=qjoQRR5?;~urZR{eIN9` z9Diu>r}{6O-1wMJU{csP_HYMp*pH1Em3hGIY~tpVzO{cu1}t@Zb_YxaO?+N$8ijHZ zY=15+`!vVa;)511wS*x!BhT+=yV@2tDI^O+J`K}$9jnMMKU+o)Gt|CpiLtjSCa3m? zh~lVNWCcvxhP2Iz+d_ZBWC-&oj?kyd+!^wT-J1m&AHPVJ9kv>6y+hmYsBzk2BblY7 z58s{NyoK70#5BmdJxP~vmLLn^ILb?6t`0K2H>;UjFTH{tBtgXvnA5SSH-(#xnj7Qx zwNMH$3;b6}ck3xxS!6RfpX4v*>eNunXD9S#-8bvJE#ZlsPnK7OGTyp5V9@XS*;c!z|`w;mP{BYmVEJ*q$450N6FH7uo>?{ zrhSkc&3Xo^_dT{)ZUAQnn%9?#J+A$U zvJjB87Jl4V07{8rTR0u#FRoxfw8GXJn^Go$nf!U&b-TG+P^tH;{ECnJX5+-p`{lvO z9nNv4-zaC@qz@yuHnD!wMBUbT^1Dv6B?wCH@1fMQWn^I z#4Kx{r;N{(0`)^WTJ` zZQdr77&lxv1tn~uXMhBHG8Pkg03NX5Z{-mwT?wU`3cKKKmFpm$2lH*QKJ;-ExeoYZ z&!brab6cdA{!AK7jv-8fDHMwx;Kl3DZI}Ax;YlVl!M;%D2>g^>)7+2M?(R1qr%XNa z$k$!Ukh5jLJN-O%-;}-^T7dLG1cA5VzrGi&q}0L~rBLHFN>M?#Avr6!8|*OR9S&1L zLnrjrR_4}m?NRIET3T4d=G_vG^Hu19)H0RLknZ`H^me9iIU1gF==)SBL!0d7WpN1w zdW7`q1jwS74cVbU<%ARwUYINFm<_Vx#|*q4SrPufAQ_uX&>-F6+^DdAheXzimS=o@ z!OXw5@)UHE)ydhOcODfA&oU|-wcYhDIe!Uiko>_xSr-UTE<|M(ZkC~&(dhS}7Fv5S z)uq;f|A^K;YDM^uGp>W-2g?;5F4}8fJV;?%GyTNJ_HZUj%Gl*d~cRZgmiBp)1~4T|!Or<3RirW+G}Tvw&BJ zYEPh_4-nE-T5_G(+Uv7}#Vf&hptEqVS!L?D`YgvK1D<_6hh;BIt5MiJ*m6uqz!2V@ ztMW6-EyP)oD{!9PUp8yZrpwH=~3hP);mfTW8788t0eT z(H(d0W-MpPKcQT(Kw8*YxLSByFmvAV87F`q%R6tC#dixuM{6Zv7wYHM^Yf5io?XXc zq=hLi%8~~_ff4xF73aT`4FgN}CXd&hgOWW(P++zZY7jYB955_&0GIN!@u5jNeSBOg zXf+0v`;%$z^45zT}HH-6Eb1H`H|0hb-w%L!W7h`nk^mXQp#X z?@n2}L?UfC{^aTOtiPk_9O@Cw~xEvYp2!gp!xYwcMB@U3VD)~v^Ywx7Dcm3k? z7Se%oCkI8vFFD=C>aE}Ia{OtimAa4Z7%Jo7_d1;kuH-{+vCRE5m z+Ox})1j(cbaQ0T_N4sCL5-j@bxb&_``?z~H{M;pG+7e<9yOsi9^I9hGYYbV3e#upJ?rgT~kQ)9i3=ZII;s zeaQPQRPt-h>Hfn8-o8h^xcK;hE8|no*l+o|JQ5y{>%maa*{Dh-dUtqhf5OQ zaLLQQp-*lG-Y8!*JHb27_59OWK=NRLM1PvOwpow|r@TRBcrEqFD-%uf(IzI|4oRe}GDhWt0CWQ0ZW3SZ`3xl3)aPk!^7<76BUV z+n0$!4SODY?G2ORAPVT>hP(#}n?j!wkj#j9`bOtGj;!KvR8>KdgrCtr>Pp!nsSZu?zY_E`;5u)WydPBAd7I&8vZ zOhn$QV8}j5>;Kg(OFDz7#ldSf$2A(4V6I9RP&jbMMAr6It!a2PiqnJ{sG{j%l{jRK z=cZPaqFTN)Tr0LJd*s+rMUHy0Ey|}&Kg%F-Q-;yWrvc#f6<-DAqmJBof4cXC?*cBi zpTd2ZG}zhFWISd63&4*WKHJaWw|)@z*gfl*_V5pxLCQ!PyZN z(eLO{@Q>`7_6vXmqJIRY;cR67!<_X@X*x2VEcLF`p0<#NK_VI)YMh+36RdD6!SpUX7^MQRXz;xk z_1vY7CX<$WW6CF%Qp=aB2Znf_*1X(QT8Q@G<3$w6=E!tMDTj3JF&yYgIyjG(QH#iY zO=vJIUVZKAWSRZ_f&X_Z>m49z-d+BwQ9_Du&QN~+DMutqpzzz|x746A{Rn_}{9~(j zn06X~W!nk#B_Z$Iuh#U7#zm@@%+n>yW#qC%1z9~&si>IpO$Z|Y*XFQ0?7&O`**?bdM;t?|;tpb0#Zr`P}PJb%j z3&+}m?IrzUv^7bSZbB)N13mu}C1ZL!aU)j&e1lfv+au z5u4F!>s&SWy);0OZTYa%i@Gso|9ObYBh+#yo<4q|#!VnT@t#u7RV3)#yDplScj9z> zm6betEv!c{ti^aROSn5~OQmp&C%QxLaYZ~D+&*D>G8X(VK&5ju?hpZ&^ff+^(Yl22 zEPe&cgw++ejvjYljQkr;+AvqN>e*{})lkZMf@dKmEY|MXYI)n&@%?tQzbyx3&x(03 zzgh#NLyQH$6a8SC@|cW{@Kw0=^+o;fRN0M7O31m&tp2EcsO}JniwtbxK8`I?m~mGtc0nIk%4ta`x3&h7zoz)1PU+$wVcA;mkblS=EnGR zTTBlRRLZIYXsKVAJ>L<2e@mcdh5C(*b8=l-@=>X$9ehs+1rsZx98(i!*Jeis?=3m5 z^}la&;a*7TpXP=!52WHT*HA1b>RlpM`Xrq+SW^aAq(?T$D|cQ2u4xcJZZQC*&j3Aj z763E>p_z8q7f@9$=sX35`4k-Bp}MWH17s3cBTE0}zxxQ!+ZCFRLgeN173(MCb&;x+ zrZZQF-krNl*<>C-l)V;Ivldr6b+)(Cws^#!+XEbrt=RO}nR-yrmtDdF zA+}5X1n9B{49+z;#Jt3g)qryPeq!acb{pW;n(&@JuG~9>OQ`@SD%9HZc54XRLGeT;);Ns0WBv!<@jp zTha)3>IKA0*n_`8u#i?B4ePj!XGU#JIug|S1wacP;ISIIiX(GM=@W71lkpF_gt#G& z@aY_dyR-uRfSSlMm=KN%8I2Rj<*6bkc6UI7${%xCo^74cE(q@c_}5tPK7vw$xT|8a zLyTCs>ZrlL+Z7cAsLl(ksHwKNm;rxaCu3R?x46U|&!_mS0fS zz4wLA6efB5+cCzQjxQ?yz14i6;qYeM{1Y-G9uJ43s}3jJ16Rbb+9aQ~hne#V;kuj6 zAWka4VF~@s>ing~yylx&RqB?!ySxDl*U9>_XClOQ&CrtCA)iq8sC;(71BXt&0fk1fCC*k^KzeEq@=>~)c+$S)dC`GYEoN&fJ7A?wEKze z?lZ@0+;vBk+wA}&UveFO2+|wz>YFT(TER^6@!w|>O_o6K4okAF!hRj}sE1E|yKcCGgw zf00(3XGOdx+*d7SB=!I4^^`8h6CSW=lsPt+ZvwPy7G7E1Sp~wvD)&Bs+C)`FXvcd7 zy^#%|E}&(*{X~>+in4%l@WaY&G;VC3{7iDA^eGFtdL&G1LFQJ6yPFykbK^uthe10Y z0L1chaw37pf6!MRrvl0^Q@Hea5oZSp5}RBYf6Xm-uN>JH0fzv389bvdMyc0Wr6bLT z?8IL&_?yS8=BO*=_t}L@c@o|Qm-EVXiX+Xpj>Xe&brjY2gZDvbIc1xh#m`boqIvyQ zKX=4qR~%=bM{ttG*hwl=q)Fv!J1W;}h>C0HTf261@iUbhA z{luJI3F_)-D=;1D!=YP+^nOy^r%>_RTuBAi^W*LvmliJCiPG}MTqoC$m7Ohe&omqbQ(%-mL*TH}aMLj~kHkiUNxdWQc*#xMSHskwW_9x;E zRVaZ@y0U>B@nZqcDf$2l>qDPcI>q|~Kqtg%lm+BPxTde>jEvU~i1tJ#sR-ZGg2H*WN zu;f{+@_5H2=hi;J_X;~79<=0_3{gX%J#!A{}7_mj#`eAw_uJ7>c z(!-Ndj_jqNRB_pxl{_`EHRVAP{vx!!^l~4yYfR-Au@_K!9+<;$U)G?^Lr>3FTT#N4)a{eXrG zh>7b+#u?1!j5J^Y1FKnV4nVztK|DHi`l`m2YyHbwg_1v89tR#CQ$IigRK?1LK=o@+alRGAk3G{a7 z4t5kj%DhTW-YS%x_z6*TPhjUuh@g~RjwiEs?4`LyaZwIf@i#@0nxe8|_?%qKWi^`z zLX!ru27hy36XaYKXTl=iUAe)5tZ}>}_BZ);wR6yXw7dxHdJ?b7AJCjt6H5m$S5}L4 zxO;L4io%g($A6`t75L1EUwbkD!ZnM2Vj}ILR~M$_f@hI44db=LB#3 z;5|7VFfmmCJQ|KY!-B|a`ZDGpgp=SH@74edjJIJ?ijt>?toM;#cPUPieTnbA8Tt56 z?m*yxk}F3e&$#D~)&Sb>O%bs#>GcT_d@2 zYS5)L6;YH9T4^}pNC(Jn#kdFGY7_Z=N=#u8e#&AW@;gP=ZTK>2a0C!jCENlyZnSV0 zP`>r|`vSC(NN3^#iWj4W@%5l#mU4x7RYEkS?TRj@XaN%J=6drDT7?ZD?z+#Y^;cE` z`|r5Av_196nWpbTLw8x>#xuaK_Wv~jH#^~fBH%(fyIe{U!P_uMgt&OgJ815GO(k`u zW6Aba@-boY?t&Ai$>fCib{&kD1I8H`ad^d2Uy^y#|1 z;}zmO-tdHkJTDN*iwYM)-UPs}FW;OJm!p_=INJZy%zkE95WG2KHMRP@T#28}%}hLx z1Xvid@WiQ10*N!;C>bBEFC~H7q@U($n)p8|pV})U8 zl08tiVoJLOn4U6T4u)O7Cm08BF#l>*^M3k@MmynmE+ZnnqO&(cO3cD8oS-K%C zzspfX?{XU_+20uqelqd&Vgvxxx&wA=&Qbr7k*-A6 zq0Z6UD=0h82m7`!!qwuagfo`4^s7c+j6~pO&P#s+W$?iMUlW zG{RcZ_Vvm7=nkmU|7$fwH-;d;|HD{zVY`dex0T>_+M>Y#nOv>l`uc6tSOwc zma~1+=bf(eS@c@@2WZk=U#sjW6_{?D!n7ImRYpx4?X_c$zt+J)y$1;6Fy&*g?m zN-&EG2=frVWKYO7J1~ z9C;-fFZut2wR_PiWc}Nw#S4O8mTY9?^AWH~L$GMq#|H+gbgOOxn13n8xbAcAb7;5p z-@__sn_?`yP`ZJJT3?{zs8O|`^6Z`S+mB+AKc`wAb`=@oTeXp=HJdg>&W$dM1KP?t z$CW91rPhp|`8mf_g9058jAxeOG(ED1lG+_O(H>uS(4UlcjC=0qOkLuav&>Pz|3LQw z9)y=(&AC^rr@|XS6nl?R@<4t@gv{88h9=&Cg)o-Q5sqBJOPs1y{SH6wJ&jdh)L-e9 z)r&w=WpbKH3;qFkk!06bc$K>}D0h0^`u@B{4tk-p+ngX7A$~pb=gJ2#%Wa0C5S`9V z_1mKAD+sG_F*c4?s?HHxW8@tb`Q=XJ{wWbKGeNU_ISs&FGES&9=PbWDh+?|%Ea?K1 zLc3cxhu+L31Gr&9*P~x0f~Y!oc{Qds7AAe}pUm?;)_eH8q#5ZnscTo%ne!gJvPW zQD`#rT3K7{#h2jgZB9c^1*&H<2K)im8IXGK^2WOIlC`{ddb)8!_n@n$LY6+bK{$lX z%wIRnF%Cm&z17N2M+05|xtc23Odxq=DK=A;w;v*VNI>MwsM#)b`x1g4BkhQ0w9ID_N1=__U*={YAzGGGxl#b}wx%I&OEl88OT*$=xC zy5DyqHyA_|-6MjMbN({moSihBvSNa9&7)wm%^mzZ51MsK{;WEk$_)5+^bZ@c_^=Ba zG{D=<9&#P`Xi->WhrP5WsjYs_hR7C-C~d>=o4TDt{er?==cz`a?8con7TX`+#3?(< zf;#j>=j~-_>m3wfzd43R9*1B8s4e%kU?(u{sWqS2*OlR)7wkw2^#LR0LzxPDIxPLIc7b#@gSHt=HyKJ~s^R{LW>!hPAwzu_d;(wbqpA z%8shM)cJclUUfU`0GqLZDn*6jcpdGMVogK~(r#Je6yr@q=o|iFQ?T-12U8nB4eBM# zc9?I2smDP$VL5~)92BpvZEeb)?2N4HQ@&E*!jal}qZ@9PFwbzE&e!lcpD9rmwxDhIxns5^HeV9y?1iHY6X5j%Yl=zl_>wqb8cVB9FkLT?0X)<- z#45K|Edckz%oVHylpimE-uPMn`7_|LqF=NvGA`{j}X8+GNUvoukss?7O*9Z@Y zx-prKiz7eL82eL4yNmsc%cye~dc>AKPPsa%umm6V(3kC2qXiG^lfZ$+8jzi6!qN zet4J5f=-dFc~9lhmG3hWj$)6Og!htD@G$fflHsIqsLWPN?XqJ<`oN?ePGB zQ33ZFexNVbWj%EV*Uc3t&@moeN9|MfV{*^ZrKZ$Sr1?ZLxr9jMlCEdgF%F(0}y_A-Ud z>Ra!DmE^(CSH-hNKIhvL`svrX(5aN2yL>EiH@nQs2X_^c160ma$)EZgT{)XmNQ(u& ziHnu-*Yf->E)LClUIdU69@v2g4ktiY}QgPUa@#x*Pq$>7}!P(Xte+* z^)Teg$2Eu83?LTVFLF9}4}NxgA3ZFV{IQU!KEyoZWV!B;7@WD1XH76oqi;IDGLB7F zl39rvuY9ex0jQZBtrNkJ?JM}dKg)2r%%)xZ?H(&%_GD(64AMhfRm=C=ycVdcyiv;| z(|$o#>2kv$dH#F@uh%a?u`vw+4P6EfImId~H3233tnrqEsYY4z=Ek;dCQ2d6RV60_ z2s%8DG0{RBTg4r>YRK3l3bV2vs|QKrk^ z4lVJaQ|BKWMeYX;eQcyXENZ#*)AN5aRkCxj+{G|k11HPrI4QMCNWMWJ(}p?BeQ#vL zxz{*0jIXrY+d{3E@cC+=dTT^YijOwr8Fhdb=5AHOZ~MD@Xir_YS1EV47sxIZmZED( zG7RS;kLTLH-P^Rr1!LMd5-8g3`Y#RgZu57XnUaDXi8rGZMaI9{9+|EVNS6i(w4A{A zLxT&o3&qz86Rn$a#jPrqs7Yvgh#PB->pLp8Eq-V(i*ru7oy$Wd-y_8pNFqCLh~sj{ z^Q?byoa_Y`pf}P2yF|H&fcZnGJhTYu_vEMcjGXs9OEubN7~u&zUB*Ufr8t!oE-izM z0|2R%`X6vFkrIHfYJ7*0T zT9=60$>90g<>pLqEt)7>7RgLoyDcTVVo$V_xkVGEtk7c;ZUuwTQVf3^#C z?Q=Ju%ksC`pl|0#2PwMERX_ctMH7`-CFsU>C643{w!03lADy}akxH_=eL~^%rFKtP zfrP3`9t~Kxg6@EdG(u;b%+rNqfX8m&!V#z4Xi;3hj*^FII$s@lgrpx1W z5e=Han9Qs@=U(K`QV~X-rg&#ig830`>!UO6T0!GNS^dP=HrY>6-C<9}t^X989R`=sk_)1J|) zpzrJA1~E4Ic0pUIV_%y!CFVo7Lj$));G`x68_~^Pf#K^Gj@hC_>LhKIFP&_SCQ85D zxT$bp(XFaru8>NIG|&huv&>mGuXy=gZ*KDByk>O}CT-WHZhd=hwjq4U-ngK|V%P<-wy>CR}&s?%9+ zTu+v<(ME;bj_Hu?XbSIx2!lQ&6j~5_W83&PV-q!;`XzXT?$fqo{QKf)wP4}r$m66Z z<1LbZOXLuD$y}jkrGMzjwlt^B;l_|o+f;GdpOeK zG&flxaYN)Db7Jss8wr@IRlTmrTtF@-q(q-X4JG=ch>2PWx{-Rf7dUQ0QNTyW74KCp zdwgkF9_C()Rjy-uakoQZ%oZAD!uZ?~-WpI50I{Jty+U$DNso1AAF^vFr))&PaOe1W zX_-c6KsI}Bp4@6>#0&cR9>Tr{PxoWu=M;|jwHd;Yowqa*C4T>YEG#M49Y=`>8^WZ- z1zyY>93Z;a{z1NDhP-T|tW>LjyCyw%l*2tJtfa$nKg%%)IuZ7UJ(u&a33%tivH^fU zrybVsIbJnqEtZUj_@0G|7H?9dklUJXFg)!jjY0Ll+mY>f%unl?>nI!GN@Lu@>W=jP9K>gl8R(uP&i=rW`4K* z@1fiG@aG|nA)Bs!o=STpF64$6 zybB;v%BSb4p$P)prEljR7+kF^jbosmSm5>_U@d+W;q7PpW*mH9#~t$g0FNd%FCzN+cy4LwbD1$29)CO3)So?(OCk0X)w%-W=Pv*Kj29LW zkrtX62WzMmI=iYXiM;E%K4c4OksElco&=qHt!+HS&#DC^T}@hRAKVOB7hh0PaN!xA zQp?(`^xH>lCl#GExLPckJvB5Ks^ai$Ra&N;X<)g9#@0QR<6KH7^%%?6Z=r3S*IN#S z6l1y`56J&V)lk^Mx;DkS2vmcC0)>Q{!j^rYiy-jtiHbGZ@_R>SJt%pYtD%a_dj)Q5 zi%W{Bsef*~Rm(42vF6oKYvv=>Q|v39aKx36V!{iiC{t_px4{?jpo?Fe6^9NTnqydG zk&9ps4Xa|Sgu!tZX6${~zVy@r0`QHzwqR|gNX49`l1U@>IacBQRmaII`mp##AH_OZ zl3943=)IOdsj^Q!QraW0rH15y$@vJvCC4lk_?0bRgX?alHwoR;vpRPL`nw(x$iKG%11+oW-JpX&auVtCZe*XlWkq`;EuZ}p;?S8-=2XMLl6X2sbSU!0)|QcqU?L4vy$Y66?l zmRcqp|Hg&I5o56(+q8|P9NC={!&03^!4ke-x@1_eB_^cSYAoJFuQzx+1cp}G8!BZyexo={q7f1eiJ^7GZ_lqYZoF08>?CZ|%1 z*jafZ8E;!X^q{amKndo!BDhnmFd6Fg9~ck0dKioQ>`jn&}Y~GTsu~q?;yoIiLH!Zxwes$zAB`^NzQc3c7`#9LJt4 zfmyiL&MoRV4_#sz0C!^SAh?v^sHbbqGKDPG7H6S2Ati%!3^R1SZ0gT_NtGRyEPYny z&8s0vBN%gCU~B+YdNN%!hdPvJI_1!w?eW{xct^2n*E*x%1s>VFMU4^baG1vWV)(6847lJNH89%8=9PUM5 z)KpMThP6BRM$$mQIr)}L^jSx;4pyB&pwD(}olET2R&>3Z-0P7xAlNozHs?7!b}MqL z^~v~$bzZSLm+OOV8>%Ymt1+3Y=q1|n5Xj;R9zM_6n-Q76^JFZ^L9gwn&7d21#Q9_0 zZw8e|y<8~Y85;6#+53Hm1i{3!_#$F?bffFJW27z{%J$cPj$Pw{MLh{iikbVg*%K)6 z+EfqIIvu@Q#a7PKmk5eN~ zKXH~NOGKNF!sYNw0?bV4h)mn52briki=&<4xOYdy_|FVy?f8eC@W(s$k!K^CUcL zWE(4kbiVuyh}DU7!|PFb<3`9Sfpli3M~&{zkS@thj?#zP^W6jQy~w&&VwGr+o#uNs z>ust8QJ8q^_RjGtPM6#o_iml zOArrHtQ8s)W7w$A-j6vm6YymVp7tmX)RWF!sd?$NQ9 zyFFuFDnqr(x2~pp%tltMq*0_Pd|%WSahqx8zM#pf@7?ZhbkFU3)K|LloDX=3&w}0; zxRJQrgrA9y%Pk>z41}amMJiH{=_GvouaAYz2osl+by~C$866cXm&D^U5{?%~U7St@ z%xcLyT~El>ACjJru*WN{jr0wwhZ2oWoNnWd%5ifs?8dzvqPAlyTW=A-)n!HPl7>J= zil+4+B-k!Svb=@Wo1{H#-*0Rnb6uz*J8aw<7*PL%GCgNT6U(7=9z!*2BVEl+CL{qo zP*Xnx6w2Ji8`n{an<`ytD$me4H$P)`Q8nar+$%!Zu zEvIFh$5bh0z)a8qDKDBk@Ik;}$2er86eoXwWgBV61TBgN#D9qdZu0H}+#zE97Ch2j zu!A@0Y=N)T`(p-zu!Q=WY}Bu0ievE@*`fKqwQRv;{P6VXaJQufAq&>H)bUusRpBYc zH#fbVLVS+8C}MZwWzB=BZ^!>=lznBp=zE)#D6qBTj`3B!>KEpfKgMN>ZU3;`R$AZ` z_<~dX>EZ3&y)5X$ITgmucArm*W04q*2zR*Go?L=Tjsq8=P8`EZV$T*>pn*I}BeZ<= z*X@VXjvG4K(lq4RNrT`P^i?%=$xf3Uo=kuccCkb6f>&a}Mc` z35}kL_9tsQ_axVniP$(^)jS({CDTWkPsWGkW#dV3{~I1YU5EZR38ba1TIXsMN?(o- z)8rO~*}fMhzD(3Bn>SihOJdIw$vuZNJz@HBw&Rj3TJuP}T>7F?9Q<^8kO2>B6f+}C z=I)Vsy;0DcrjC-3t!}{f)K-&)f*MjFXCKjwgLcW5=D-0qVjeH1f$1 zvHgRI*>q2V>4}VMm4G9`W@P{w9gxlb>zs*4R`||_d7j!YF!-J zH{&6s*rk~x{>Cw^SNZ#UIGfPKfRe29;AIES=177&5U857y3`esF7eLj_IZPuPXj6o z(Gwd7ewk<9FhBP~{u1z#8zvuE7;VXYZ2#N{QFKn~t^sxJST-iHCfi#(*UPpj+<}mr zi#?IP@RF;5SiKEJa}}uV}4t)MJ_h)Z9_RMZ|JPy{4Bwt6*`#J=q+V1Q(8aK@K&X%6EQYE*KM5%$bW7eSXA<6QzmL zWd@LZD!kkMwkkKu2Ip7|?kv(1{&xZ66`HpA2uGUFUo=4kFu^sc5&TF>$9Sc(I6N2o zzo>i9uqK!8-`}#Wh=?dvKt(}{AkzD`AkvkNKmtUhgAh6iSU{8(mEN~hsUadgBnlFw zOAQ?%5~K?OLI_FzcTo3!p6BpCyaxvu}o8^WD7vu4ejxo2j5z5*Ky5TiULqr8pU z0CA1WOAKj(Tx7NUU5Ybs15w@`#tAegXY}~z2*ORAao5&R!nt>z@I~Ax-ch96g6L{+ z8Gja#1P>~lcn*lP-qnDDD5)xlyD5Osf!;qS>u8WzD8X8|kf*#yP&9dayA43Q^XW44 z*~&%#zD*rK-_}3q&J`Vz-~tF_Z}uhCnJ;5OczhRdCC1q3D zZvT|DxJZC-J^QTg@IbHYrAaOO03xrKV#qvgC7duH1KcduW~cv8<%ZqkW~LxOXCO$V z_?3*LE?gTF_4wcr~Fk!I zyh;Y@TjmFq?tjiy+`L4-_j~u@m)PjviVeqY==_NnIdDiAzsq3%rGJjr07v#97-Ado z0p?Z0xO*Xeulz=52pcGnK`PA)=MJ(dS|yOyrU)U$>hVP^HpHv{AM^YPmU zfc)Ifn=E}^e6y#u;UQj1ns7}Wi1(5UNn5)_&B#*u zL1JxC#(qEpZ(PS2yRZW#3`tAgks}so9TLImUR{n&$OG~)Ojb+*z3eAfRWJ`{$a5An z_GiS;sMDy0I>@NOuWI6NF*DnI*#bY`Fbe{b->{P70`<#miOtNtR}N3rz!b_Wv6!@{ z_b<$OHQn-}p+{ZEYk;ERe*|0l<+l$rP#?`si$v>G#R8B6HDssf3kD@koAS z0LKU$bo_J9!uww}Y2pDh1>E=u6#rei4$6#+sp_yRiEXgJ&2we*u+f`=hg$i1CtVR2 zIj?5iziv5xgn~Ig@M-WoqzfG9XFi_-Y;a$1MUy71i_q_j2 zi}3BXa|r?gXs>(U^!BP+fv0Z~A}=7g_DHZ%Va|I=mFxgAjN{kBO*x5B=_+$y@O2jKMTNH=xF}a=}KeI;3Udl^F($9RGzc8t8{-VKme)Z7j@?-w20r44kZtx&OCf2Qx!*53x(y*iKC=Hesfp%XLFN-?JVGgs!-f z1_X4pYI*>AU%9sCGrIq1ZBBY&iMv6bd|q98kiBbbDpI(;b|RCnCDzxjw{qnMF^~S| zHQ~h<2{|B-MgCC(!uC9Rd%9$zrN$jwR1I*@0CH2-Wgrt)p2?4~XWq!a;5p0NyQ zG^uNWDGVUco^y4e(w3SF->mh~E8-}9@OCLn!(hpEi9WFDNL?me?(gU+wKj)r@oo*<116z zSOV*Z!l@ocW%nR_i63`QWr%aAH0V*FFZMRdaTF9pCe(#wO2~#!JAes#(5pR(xuXSe$ceq9*Fm*XFn0c?XNSGh}W!GWNc4+dfg2L@I1+@$0yYBUknT&mvO@RIl zf97}EG;}~N>mDM%DNO0nGZ)EUPJYDrJ^)a7R12MqKq{v4{hJ=fp1%aC$DHhBNv026 zVtw))-QDhcYzNWK@Fd=<+hwgrp+0B=c2Rq+cSS1Q<4fD0M8DC#X zuC$;XJG;A&I_5XqAWr&InVhnsu55TXAs1y&xb?FU&!zVLY((;IjyTYWssW>1i2s^9 z&D{yRoDx0P)^1c1`o}|;lWlq%f!U|?MCXJeX+?vBu!+U^?1Z-%%mW6@fTvj$l7iff zwVsL6gIEYN1FENF$bg*q{wa3D!0{et`u#sh{?gP@W-083ewX6N7>P?tDOK?BOs>~e zb0Amz7Bq7=%l^HyfrGOe-gvB3Wj9(@M;uC{x6=NZEj^Hky*kXT1xzia*xz?@GfYOjMqp#TIJWEb^;fdDWX+h;6~sNzJ(W(eyNzNu z$@;TVF8(M*PL7YSz7rAXWDhfPhB>8ooF?v>?;D%lvnL+a{e($>$!VDh!U0pa94K{r z{o64;?D=Z>EYG;qHzAYm3C=NSN-EI=US_b>!kK20Ka`qKUGBzglK)-70H&Tp`Keuo zLN#L(f*>BbE*uke2(v!CaXUq-V{d$)5s>Qt4b}aWSr{SXF9dzWvO`a|yBy0+@SJrCScB61hyHnEV7=oVK7v7!A8wtAMhJAuS%D(dJV z=uqUM>5+9ry!AOQ>;MD^XZc|YS?dN(&@9Sw4m2WBG-huwcIKp>2yu^2<$I>Q>c)_N z^UqnX^LJD#nsHK_ywkjLc}=>wWAFWY^8i;PZfN#8OOzF*$n-DVOIQe_S6&_54tb^f zemQ4Cr5Ho2cQ($OJ)PJN#49~kHJ!eRP*F92jW8!bc`sfM%>p#UQQdW$->tI)b?_GW zdw6U71nx2JlaWA0+Zl^00G>$0Utw^w_Ydz77fwx3=3~azYph}`5)(=3-b`kwfIeP#pnC!@15pwPlfSSdS419oVz!k_7fYJ?5Lbm_ zfKvlW*`@55&(<_NFeT0GZ%_|ByBB!TMS71 z{$)CgXo}eT5xJ|D#;vf$;H2WWts+nRSHN>M1CIA$w_ZO^%^|949dx_EFNhNl_%F3= zg8|xiNP)RxMbGrM^?rz-Rm`pSS@U`^q<6~W1u}A_CVcbSlCh8(Rif|x$||;U)-o`@ zyG3^-G(i)gevkwA(p0N>ihOf&zB2Edxk zzW^W)czu`j%=`}r%&M4w_YnVJa&P{p`IxvcCd8Tfo`3mG<{R~Yo8%wc0B_voQ8VBD zFD(naaTlJ=y!;cp&wL^R7>9{aW?ucjQtWa*p^FWhK$=_ZvYCdKw6J~m^C6k@o6*nJ zg#OFZ<4W>HC|d9Jn}<#=}U*r&|LLo)wzk!dG8!3(>k-UGQZGb#O( z6b9JG;WyNo`QnHFznbO$RBM*s51G+NjNVqt9^#yNZf_MY6zP69_EMGR|1%fN0Fe-; z9s<+BALX5-H4YV>x$73?eG_%LjlH?*lGdrw?EhCTSc&g>RQuciJ{V0U-TdjHkEVVE z%k7b1YK!dl`oyEy3%ikw`SPEFzhi{N>0cd@dHMU_==Qwdag=$P30BP>YpFld42fj zhpwCnK^}aR_qP_n|CW`GMNAg&wH5uR?pqhxe}0{<;tq&)kM7=o36abf9+?3@{m%z} zzV{bd?4Ne{eHnn%0>1aVNWZWCDGpfUpE~@$>~sM5t$!;0`*P{7ng35&3Ha9kTP+P( zfvHckJI?4b>jFH!KCGc6hR&<6hH8w!zZK?)k}ooFcEwMs0U)Tq)}SOT z4~K45j3d$0+T$`Ij3a=q%_?J*w!^}>L6+%eJYH_aZZwP|G*;^CTfjbD04>I$$T-@5 z;Ff#BjoNtQ+JT5|qpW!=D)TqncO9I&9Y$-!PW0HU#sVApS^f^>Eysqk;B{kA53Rlh z%YE*>ma~A1x(mo9#3i%Q@|Vx&H}m^Xj=;CG)F&EdA?CiwV7zYqB)yNN&XG<>_!g!}1k7{&y6HJtOCcMlHBfbr zDN#~@(2o@QJ45N-8}A9vknWzNr)h5DCZ}>ND(_m&Y?b!ahA@1(H0V)W{+>Y$hw*y# zAP4ePA3S8v$O0pS~b4O6#4OoTOE2 zP#!Pyn5Pf&g_H*q`9jO7GfYOVWhni|IC5vDkHOHm3_$M(L~o&7a;n z-6^P7bpSZwQyhrC4@9BAV$wWmv&%W5p*G}1HVh_s#!70Aj&08zq%E?9TtU}5ZKEbt z3xaz|-6eM@g-IpmZ_(Zj*MZYO_24sh;gQ~#@GjwPb4+8Gn!Qb5wxXU`N}F?(qP_Xs z+zThC>${lF_LbOUX~%tD$2-wWt60W zjVwOKT0*zKIb(ybev(FFsV`tGhaSU*W(Bl1)n~z`{Czb@RK$D;oefh>ZN{tnZK!!N zv^0R7SwJOCm_IG{ZVPY5{tUhhp$zd1nMXH1;nd>;zk)UJ<77JC$3KgnLD=M?J@j9I z3V7ZBb~enfq>p4sx0|(6ZrL`XX&w; z?+jNb%U;DV0gCrMYUY~89T>Cz=Ewtzs~Ov9Xk8Yp$G?<*TPIZZG~8V3nzA3T?Eg+# zWG=9`woj?QHk~8cAo$Cl5WBZCzJzm^M}{&_}~-&@lr?P11BD#Q2}aYPRG;`lN{yv-!}9AnUJl&^)O8 zp@5iAvY(3gCAPFMrec@1yJ@!-1pt}9)xW&FF2M+Nq(2_ESUS$aSR5WV1g(9COeS6B z+_*H6#dzA{gIyPy6bxZxx0H6pbiL{_b-~fUd8$3|_3(}LEn_>WSD;;pv*3ObNV^I z@BMpAD@)V3>c2t3J3eGbK-GPB>}c`W)Yb^<@%J&TTURkJzUiPxUppqSe@KkKBGzX1 z&)I1$Ddk!nvsr`LANgq-73#$i2Y{w51DX>3>LaI(e^_oIUwM!!Lw91Gub5m&UAd`t z_d0@(7xK?pG3}SiP|3KFp&ROvwKcYKF9VuEy)hL%w_fvu(c=`_D<{gsa-)2erKKk@ zwruZpr|mYl!9Dmc-4Pe^vG}8aRsr8NdeV)sFH>zEi!LEjF@xZfwT~Yexf6)ZRXzIk z<_$FeI>z=igFK|)!0_(zFA;cLpf#0&{o1m3Q+pOC!1WeN?0(Z#*tN8A$Qu%mzkTKz z(+R)iI9RSmGRc)ALeCjfELrm)QWuv$_D42jb_nQ8irR*;$xU%r@`W>uGpsTopkV_b98RQCqNS`jB zWuN1|lC;`^+CinP`A7?pke9`@>0Z(ke{?plbGqJR>~_E0I21(IZNU&Sx2zD~Gq)qP z{25Dz($@YLGh6{%2ZvtvaRv2Lj82kR{?xK0F( zIjq?mFnowjb*poENf47AOl;2#Shv~Sn$4^Tl2SHo8N!CFuFHL?4A_BoVUAzJ>)`e9 z9+S4p{ITbm-Ccg_7~9UmY$GY=gtDr$dT+!|F{=P}?MXu|B&*+lw zk*r8(3>sPgFv&rAfZWl@b`*Y}1%XP$CtMIKFN0pdHj(KYH$u7WiweB<*lD;Z9hST-yR)Fj7YHg1sq<@dpJyx;F_0cFo~t#SoQZx>~L;EcMeH+Q8|5<&no{}j29xu!Ddzpj=?j$R9W z-t?+0Ayi?}kkJJ(1oz>Q$mQh^KFX?jxW*H1XBv5yLl!Rn*FpkQK~!B2iH2R9fM8FY z4?7M) z^hF%$uk4c>JB_|I{iC^f-|;kN3hhF|}w@e+}=hDNiAmu_?=(Y}CgqLaftpejOd?u$JOqUU0QS_bTox3-eI=n&63jNVH zuBA?$0P<7?{w^ouhlV;g$UN-4Gukbxd#8_e97&rpMqna;Jr6=H4#6=qwMp@M)j0R{^LYZL=1B5j@4jbbzosomc zQ1S|i@jUzzEfhDi5;z=m%I%O|!G05aiDMseeCg*@yAQv4njj*!08IF8Klam&9C6VM z%@gNj1&7aoPs;6%6>`pDX--*KSdS9W3(8dQS~BmsS@JiNlw}_5NIg0HhVq%X_4+O4Vh{ZQA01bjFr0R7`wo@ITFnpsy#((?!B_@n`GDt6WzI@` z0VaK*Og9PlDCSTMYs@RubGm1Az7zo4I3pBx+&uGb2?CG>wVK=>lumGZ;r`|#ehA+5 z5Ld%V#bgdL0IqGe!({dlATN7K5@YFaa~`Z>& z7&X$J>TX9{3D!98LN1jz=*ML?U}*SS&_6}Xtd^ewW(~pX$x|Y#E9uK~NDAf$_26Yg zUY5^-YSn#6!-Eby zE|wpk%|ut;FiG>|gXh=*4e)I(Ap4zqP$z!<2JRF)fYj|}GH&h8;JEh#%+1J+CArht z@VN8YilVeU$;ej*6D1YkbmI(~Cd%fny0g>R3BV%7hVN@}y!#!b(lu{ zg9#yFevnDU{pdZ^8DVXZFdT|O?}Ro!dn?z?4bS#6fi7)-j#22&!mbPyr`h!yV!qec z0|Icqs36>FjdI@o@&dB*E-!E+t&HvzKWfHPoq2}FpK~gsZz(0ioWB6SOd2Hsfz-J> zJiM@Q8a%74Ny&ZVb))~aYl(VN&Kuabij^hchD*lquHp)1vNcZT%MoWiCpV{P)z#X` zN?sZEIma~7MmjQGY+C~wb&nL?GyD3zz78QTQQG@h^Y-SvkaElD%g!j1v`@FBd%x6h z{>%6G5P4mX{j^~u(RKOhMe{);P*f$d6XhIwMG|6-fP?x9WAnz8UbN|==t*@(W7Yii zmPynD^)p;6CVA;|ev*=6p08%!w#wWJbyAJ-Roe`-A**D1D|qW>v0%`rl8hoZn0(x= zDnG)tT#7^&BSLp%UBj0Npv|Yf@iK*~ZU2B=nPvu=&rZ}~bJgu3{j2Y-oh629zHrYq z;VhL07c1yn{vXt_)Gyi_V8obzRl!QdgS5rsN%fxy-n@8wc&&RDIy0>U`-|UH{n;A; zspep3oDbs}G%5lk6Ec5sk|P8OP|>+jtJx@mwnlrjwdFeFn|^2(YO_*Fe#!I2h;XfR zemTi<4_Kl=In$eZI-~fl-rJ6ly;Vzx6S8E%hIIRyVX1x~t0Pb{ zv=>N&Xb@DH& z6^7?RI=9-E6#TpCO<6YMTS3gxmU)AX80Wrm8#1kx1tqdS<`oAVMIM)pdXTl@sKFpp z8Y1jBz;S>vMe6AF_F&rPt5JZ$ls9_IM4|ls`pqfI;)Bh;sslo&Vo#v}`Ys3({ z{%VSR*Ft*QPm)85erb^r6SmhOf+k9+JLAeDY{tx{Y zGIIksE%qCK+ctv#R`}vt{eT7jLQ)Gy!iIDzB{kDnda)s zG}iM;eSq`v5;T07t$ip4nd*wyk{#T=`$M94&;zVUK}M;T57vi57OX$_?4>s32IMA0 z6F|y~0Ph35aB-Fqf(^ctOJ_v&!yL8nq2_=SzS(6KGfAIAarg>G&}@RMRa%97nIY3j z98+ATJUsxB`5dl{itX5dqd0Mwcw96t7WWeO`ZGLnrY(oTho>(a8!@YDStQ?#yK~c{ zZzI9owxm^o5mn&i0B+fB9tDcH zbFgHw;pyh=rAx|;d|6zzctV=Xg0ZIbT{I$UzQF~OlYD8SCBvjM#=1$JOO;7pz7@@% z)KtBJy+B;sW;}GKYWRRSPXixtrm!3uq;It3gOKjr^mv<%`C6JwN7zHPEwfs|K+ zflNkuq!ii9O#a|qj5U1(qwAHoFjDr5E<2FaK5wjvx@3A8TuUG>i(*M|$7uXyQ-Eeu zEs_zrh1fX|yJjj<^li!WAi z3(*bf3Wg0cvKln(lJYL~HLR)&PSK9d?tdS-@A$FoDtH}L%`-|bGFt5IrAu@xsuOwd z{W~V7c;ufRy?Xfhle5{+qo*JJ^(g!7*-OBMMb_X4LzGGXQeS+K*dWI%c1ufB%LMWY zfP5&83^;#P9N@)XU2c=0GuU=oB8M24?W~U>G0K6tYmdw)bUEUhTwRfDXvMF2uPk^@ z5?iKI1c=Sk*CaB+93Z~ZY8_F{3g^tsOG0u{nb@DM6%VL;#`twy^ohq0`SNl`+8Yop(DbRgs<1yDEH0tJ} zg%#G>Wyfq^dptH!}%SXvAeg%u6lg0@lvoWCu3h6ZnF&|4U9=~IIAH+4%@2`%<wLRiclWxeGc5JYTh|HOcHG_vW^? z{%o`S2xl{t_oomHWh8{Yx6f_+(=x2{MQdJJnMH_=Qjfqz_}7Nbw%2tqHYs&8!y>vx zP~>a(PrbU@193XDn#hP!r~|62{*oJETFDGkwk%)eo=_x}m?S1tS^Qf42PoK~)Z700 zj7LG)vI$7fE@epGeolPa5q&(^x;()7ET?Ddi@mj8qHckp3r{S@)7_ed?etB|%_Xp9 z>HV2H+h<`m3I^`<(5%ocTQAqF?&Y^CpgeoK4%}RGJTpaYBytHA2GtRa2f*?j?cS6h~d58aLBTs1bp7sa%l4c9H0)=A`_m7D0y> zo)4I&g09=}qf~QA*$Nj*0BcL4&$=x%Dw$~&(W|Q5E9<&iGACr6o|fx>=nf?>dZ%Lr zC0Nj~?-0iwKcz-T24TggeZv?<2aS6c_o0*TS&E_!qh?x!lH_o)TppvqrPw>y+y* zx`mtMHM8EH2=^4^e^#_8)+ETK-j{NAv3>Pav5+alNTAHl{_A$LRQ(03&5jjyv|8un z1c?t6S=FkEXEa|HDmCNBq8W)FmG&s_@Ok=mpMGs37M+*pRoP@49sWsu6h&FBDJqsq zm?VZs6yd@NxF$Yvrj=Xez?Oz3m|mX~x@>m*FBORP5ZOdfcAYZNR^Gn89_USy(2B|@ z@f#?Kp*q^u6Io9yDcVeR&o&sgl~hR`x>=~au!de&|2`ukXuX^;;MOoZ@@i;fa{HTi zmy7&l$nfowg4dWMOpBwqSqO^qO{;Txz~Onjy+B_rL<>h(G*po_w?2&wFVtVTY?c_{ zYmw|pKnJ9GzQ}uOp=@Vq6=9AeTG`KC>`yRsPvdGhXSdU4Zxe8BXi&2vv`4_uF*8X$ zc4z6MYK8m0ICp2%$3(u%Phq;rp5Yb}cPuhN)xNeJL8~-V7m2VyyL%-i1M@*;bxWS@ zYVX!8>!!L2^u}~lw_QnzNm{rz8@4hV`7h>pwIUONPis|~<qbz|Ux-H#Z>7-B#O^U0z2p0gAjDVja$2{5Oq=;>6}q zxC_9dKbeESgM!DG%x`_FJ;WNzh)A!D3XXl5o8OrHgtFa=6hNzvm_6rbTiQuxRG;%a z?%_ZxAM!*|kHmdQd|K5|6R6~Wid(~OG26vj@qsEDbuvGka;_G^U$qhBI|9>{;Q!Qn zqULi&fJq#5Mt`piSQU@*-L(IL{o;{tVUstP)3|RhCc~!sW%+q5NJ3!9@9`UX-io^D zhK18%lsAjjpY1*wkYddxp4f((bKa;cvcMMf8dKivD87$bV)X3vY!wW8@Zjw3y2?sc zviSyrLXs0BNxkNux&w|;>1y_rLhoJQI9??g{EB<@mfMI6CM7o7B#P|e@jA1U7J{z|2I-~BU!l4WQaE+;s@}ieMwI!rTaHt!!N=Mjl}dT?^v^XXd>08t z2Yssuo>6lhQ_Qup_qwROG>AaB@vU@6rB!Y!@73@uZ?Y}1T*O!@3T_hH32)1~t_b?y zVPAH1^I}kt{B>=Qnkx-69>2fM%N5JzsrLzYw9lTB7*Dxpj=gw#ff)*(Cy;P1ZpT`B zp$WzlQ6G6f>V`5ZtmF#}4ogd^Z>P=pH{2Yb#g*wlil?;sjN%3ZHl|xlOF&ilXG@_? zs!k`VR}HUYM+ujX`AHqWE^&<~c?*%#v0Ksbank6TDF4}DPn9GGc= zG|oJI-T&-Le)(a9^g%Rav<+59p=7%c*MGl$D<*ZTd+#HPj48V6!Mzf;#5G|%3~pMt z@0tXEz)bQpP1Cmgh)=!H&zJddxorvkJS(HBAAR!d>ed6kmScvtR{I3$qtC9{*cM`X zV8uVm%*%Z`PmBr^zaWl}<}YtMzizB*MW>XD#;QVAK6)`4W|xVXTy=l3cxX7=CZ1d& zJ;8Uo!-8yY2l@Uv2s>~E+E8<{bbOusiUtf)cHF^U7lXbQ)*tfq8nuxFOIKa-Y_<@p z>b!0bo$)d?MA+uy|@w2LVjN+>D zEXJJ_s(3c{BU2PwX(=MZl@~S>MAnr=)h2>|JBOec=b^^#X#I)+XG>yn!3imx%Ng(a z^3iuE`z4*l=j`+@N#@$Je`?RlEurZASR(hR zDAb=HU4CP<3Ie?iSghNSI!+awoCd$JiEMdtYWoC57kh|rkR2JU?Hec^=p}zSvFm-_ zdNQSrd?v2BJ>oi=yEx8JJ@BDjIE%#2O7oxktMgAJ^eQ;y#8uea*`>UHgc|VsS(#0Z z7wTgzQ(rrb;KT(EJcE}6gZp0M=_X?8%~_cNw&jlw@O!mdX>nH9<%+L>jA?TKCVj(d zFb1anu;nNxi(&mmjH1roMA{NT!ylp(u5#aE#aL=*Am(v-)|j7I+0D?9P5Q^G7hE!~ zc6xT~0*w|BJ`1xB{0~)IhwjsHTwPhoPzxhK-xgbP@Zs{(auRO-TcyTK^9@~Mv42-p z0L?IYRy0=@7^L>L2fDam)+)JNQC^u6AoWwz%@*t!(0F$|9F;xb4Zc6I zMSBD~lWdRK8a!69Ug9t2+F8qeJolM9`3*|t-VM@e=sqb)^C9`KRoB{Jn#T?p-SUcc zx(%hecP0qGfveZ|*)9i>#_=UChgfpDF+brYKMSW+=^*SP$c}{s;f)wmUflGa)&!)=vgV^qGV>&3De^dRLf+$F z(-p5N*RQ}oZu}Bl#?eB*tbR$Y)Mg7J<)&N`8*`q1N*4Ronh5bb)pn@`5QF zNfCAQv;odOdrs$NqE*DW0AY0JL7@osaB_e3X_1S|D_{LJlx-WIEqOB7n$f4gOeB7k z`Jv3s{8yT#l1K7gw}yX!CRb#XNcv9;EOlUFPn6hEgTPjH4d)L7e{ zQ~VUQ@Z`AaQq0rlfM_{PC<|gEXLhf6zM*-#%B%h);t$!Q`{Lw7WAUcpOJF<~Ar1NH zKa9AQ#ZsI{6#p@O-9l0rVIojIqXu~G+~l2Ns`3K>=H>5+Ti_Y*dh6Y3zQ{su3h9w^ zz}PcBn`#8YlyHcQXNxvXoahzFv~p?n(BV+PV^`E1&B?GY2d0kSjSe0gNDb1vqJQs1 z{r-5Oc-N||Bvios3H_Qxcm>`1od-p5ZI zCGA(yI^4XIwewBnY2Bl08!6ULy}*h{c}WamtHOOTuerCO=8gmx2jQXP8`_zNV6Soh z5Mgvs3A*MaDFMH|$ucvs{l@~1G3Fm4{UlI&I4Jd!DI4Bzk;?BxRo`5V4K&;o-M8g# zp5}N}IuFN`|OX*Y8Wp=EWHkw;3`hYJ{d^0V6XY-cT^Z}m7N1|RRVKnEO{k79`TLQi! zvHTL&*90+(?oFDJlC2ZtFBo3the_p5NB74^h3$!o0M$hBYkIMty&S5Fsn2v*gwV=3xKdrn)N;4+{steaOg7%joax^$tDy7fv0 zk8XFJc|>$sWU(5A%?Je>Lm$U4CKWwABwi>3-itlCmfD>1jrF}k>D@^(s z>X~!|D(QT>VE4UE>Etbqj)yyIMwV zn!I|@9c(pQW_abe>xC*k7x)ubjsQWMdu$&t7W-4aHuz4le5;6WuskbkQ9JmQP^zEx zUR9>j1PDRLQ;_!aKdkm#1v?tfXLt2hHrEYl%Jd3vbUM^G3}v6KVs+{oz6z{@M8O^% zNRZ@7LwSxG+D|#2L3Q>F<^(Q<)@KSUeSIQTt+5jI!RFrQ3`dgOgSwze3HR)=D85VN zv^+C+I0vdQKk&^^;#OyiGHGLH0V8Ws!(14Pk#O?WgzX5twYrd}opvwaf~X z+?|#WXRZrZv*H3(&kFi?xbK%L6K~{hvS>M7}P}7h!8IU z`i0G(HYPh58>4xamvL7Ql)mDnj@fl}FYH}}#8M&^uNJyic*#X=dYng$uG9xQ8+dsu z%$_>+A|=%<+3+hYO6eIe#1V@#xQ3)6XClExM;F?p!K|?WnFX5g@@c)Lmlb|q-TQ6WorCDrN4psWReI7h z5sce|?j;oMST{q~6H3&3vvojzS;1M^FeUNI;1%MPO``xP!Ll7>_@h<5$u(~YuT~K- zVu%OU(L~}Jc`IzBEsO))Rkf!^_!o{(1^t0jFDq=g>MrUCZ!FUv!k?{Dx5Eq9-M7bm zE48tm^lE`~4VpQHJVWfu4BNW7_tDZd+e}3D&(3mvJ-J^q!(lMWSDM7e)z(*cwu*%n z=%ocmp40B%1134j?kSQiD?J#ItE?myVM$yNytnFyz!97v_VAS$cf)(kKEg%v<(%~C zsI7U-IhU3<6p9hlweshvzKqaNTm4}BaXzC3>v#WLv2L5RT&eBz^YVU45+9#jCJ_DAq3?#)@3APw9fLWT!)0 zOq05X^M358#}awg1#y8}7qTpQV89553|(~W25|&Jmo&G~)9`R;$Y#%bRtE$_arV(i zY5G&HhGwSCcN$N1HC;SdDB_vfsA8cT-TpB4%O_a4H|7{;TQFPhH?@xIKU-Q9UAtNm z?t1u%D}VCdzg)qJ8F@eex^MIu#(%nOWobP8zS+U7iybU@VK?3%>Jhj1S9PF8tNb3n z*Pf)k*ltp>9wu(N_Mu{XHqdgEKF5g6F$-y3G%p87nZeWiaX&$>A} z0{=kG+&n*=E4vhK?OHBBP=j;p(#+0ynAd@Qcv3~%61}6zcp$pTwF2QJ`p2qbb}|gv zI7=qTmE+F{Wv$0@g{x-kD(IFJ_i&Fq}r7Ht@WvZ|4&fRCL}#`jET-F??GF- z2kA#3VYY`i7X57R6fA4w&DeYWHtoY;v1U1~!@c<%bk*au#!;XzAgnj#$!ETto{G}l z*>{RQZ9UN+Hls9@$!L;t8fXnEjHLDe%W&Ag90-tXOShH8i|G|Urhm7P7uTtY7tdUk zrOJ@N%1fN~Ig+n-O&MFu)v^~}?yBulcdQ!G58e0alo}3IzHnAO@88?V`?}gsx>fE% znWbG?&8Xl5iP1BmtGv=T02cmw;;HTtcRvR^~f zT3Wt^M~O(21|%QoHRh&Q{RsIn(RASAj=^iLNfTLf_j>O%zp=BOsTJ;GJNjjs78XzP z8(yjy5*NOn*P4`A%6-Tl0qA>$Wegm6+X8Hh3;{7|KbH@Q3_4f;d?jY%yG6?fHW0~`GHaMN&fG%?ynO&4$?V8QI z_lNL?X0uUVhj|BKuX>!kngeD{h7Tm(Ss%c95>#NY_Ch26Q0&S}|DBc8AL}fchDQYE zz%HMHh{bB=B;9+W;ozbtB`DrBp&{Yh`BHb^Z8-d3{a=sZ ze++^7*ev6|MeR5oc^}?@NxKCOI<^9%nN_dA@@!vEpW#g{e{`$FysJ$He6GvJ zh_vrB)#$WrEJf$W>5W02)`_t{&f5jyWy$`SgW%kZW-h1tk}@HZ@A~r6kJOMsz^uHu zH`-^cvOdhdcI9}Ea^Tb+yq)C(KCVAG;}<>^SsABY_R69!sUtY_-oFKy%Y(j@F7ps7 zpG8y5gBOO{l$RQ>JVgyuBoaSCrFA^hOtg_t{NjYmtv>Ob_zCx;J|Xpa@9 z6$2BTdk{bCr*a3lIJ`epZ>)FrE=M80puS#jipq#ro$#GbJIG?j+;|mYv9vj*ud}b| z6iZs(EeRA$inU@VFt6m4egRJx#iVLtWzyW38V>NCGOG#%o=pThNH9oVdpb4Kd7K5iD*k_}E7@0&U79YH<9 zRVVKDn!nwTSgT}FS87zeRu387%34x>A&+7)w!V*IX(C9Vhk{aLUg^!h`AZ_Xu ztA~6He43)lhnhRjhgO_gX#oudNXNZC)QQi*jKfK%8_x@|>aoNOoguv*3fFUei{n%* z96B#f7T2q`*Qf%REETuh%KT1ZJ<1ONr|gtZEVQ&Z)-z zz^3BqPd;-+>WO6}w75Y?-jB--$jzDh|2TW^sHV1WZCK?X2vVelj(~#FLg*a@0g);l zDS<=;gwR7TD&0_Q6pGQ3<~G(gOEPCCIgLJ# zui~-wJX=@O6*S6FLv09Bcon~n-H8sx~f zuK5j$^%qQ7|9K0r$z1tjdR6SME4EkeeKFy1*xIlACL(PJboaa{QwNCb+Z+o)yB+T? z%(A+7m_{L+tEKRt(bd%{L<72WJCBN*zWKL8^0X5+p?ge?9H(&Ls=4A5@h*a zw@MW~ghnJ4+d|#@rx=62nbua}pZU-((8_rct!r4`Cg74XEln5v@Ihu0$)(;9Yxdog z1+*>esQ-Gmc|Vds{~M-(LnS3sJ9S@I#oHnkX1mOiiJp!4&A)azZ4^H zNA&Cj3kgL;c+QUJqbtgl6k;qQ=BunmE9=9EN*S%pc29mP@LA|A>-`VCf(NayT^F^HNB3*OLb@@sbdE0Q2{(xsgl`Mwa! z4q>*o0xf606@Cy_DLC$gYQxgevZ&nvv$OhP^?U&7T8KbbL#VFud>!R&LDG(^ zw|@w6gDK`Z$A`_#F_+1^I%n`S9=t?dGVh#f7B|$vJ54o1Eb4rQ{7SvXFDVFdfznw(M(8#E=Tby^bB zBnI?in>1Z6b`w7zesS?JxXzRvv|SBaH!=xn#VAhjE^8rbwJ;1^8(Q1_o@(;cAibYy zH_0WcY8h6i5iC1pI}sukqPu;|qO!m!(swLh2E{{3`wl4~!vqGmT0qo#;)!>_I>!Px zX@5!3TaNv(DDbA;t|8BOS3(gOOfyvslBA$ukhwHVSNz)2Yg)B8_1KfAeFH`!X3r4| z4#E!v1MemZLR7vEE%2e&GRy4uq zdgpS|L}kDq71xNQ5$=`7;qNlbSdF=7ro@WB4@PSJcKFFoW3nMq)-#5v(d;mbNYV;y z58LS;jMF@bx--Z=V?p8M@)!5XVb&2RY=^VPFvtLEDs|Et z#^?RBialFT_CK}gNV9*cSNL=ic44#T$~(&zkMDrn&o-mE;x|UT-!xaXnc#P%r;%vm zY^|R=ku~L4ZkeZ99q5U^N^_SB8|tVWeZOtnZ;>!Oae271VkqkVipf2ShL3BEslyz# zmqlVOJ1BR|3dv1~OwzE>rq=vu{Lh@5<$yv~d6$#hKYoLv%Smu~{ugL5m)xy3_BbsY z9$AG*DijOUC+D8!tDt^9bRTRQQPRv6BG}>ie$LOHs{%6N+-X&TQpzH4DM&+(s#Inp zWMgW|UX6+q;$~E|NO_9hrmmR&M6uKriR!I>qJv0-?%lTDA}>&1#ArPjwsrIyLupVS zJ@FlGyScT$TL|S4c*M35CwUb+UWaOQyDicc9}y_zcUwik;zQ5FeJ+|3sAn@&mgM%y7f`5MJP)CyC|3hO^NaZh53CE^0`1Vy zN68_i3PHzTY?gf6yMv@-v4MVQh^fAa#Co~AD>%qpE2IA5XN0WZN#fIicQF6cFK3Lb z_({L+qI1>8iI^+{)c{l7XR{LX6+9)NSk7+w6;OwXkXT4zgrx1_OX0TjM$Z!9-vbQz z%|yK4kc@dR$xWS_#dGfl4suikrpq_Q=<2m6PMH_AicZ`g7pENDBzEjUg|66!+PDg? zW7`2%zl9I+@glKxvvta)%f`H6$gX8+Qw1HOB)=M8NO zi-Hyv0q~*VTa!;yDK8n4(jlQoVJBNNdd)BXYXAAgd^<73`u(E_MyyA4Btk55uit=C z_J(-O$_uh@2$t51N2%_W(zlWJoR#?t{3^7b=7i$hl{s9#m_Nr>}Ble zxJeJw1U?!i1;OmehQ{g1>yH0vOY~z7SbKtmO`2u#EB>Dr*T>9=NTU2e2+)r#_ljFB z%RRd2YN@u38w)eL8k8TMw4J()Uj*#J;4>g0*pR9egG=K`DhOM|VXdG8Gy8-f>@4^d zF|p$J4pw3SeGN8C)5ebMuFtPnWcxTXQbXf5kH z-|XvOZEx)UwButgx=}81@h9>xJzx3Q9?s1PstO%A^}2K?hZ-7rrGP}M-y_s9<_3gKVPvcLPzyA}J^}Qad)l`|-iWjPokuMeHOV*V*yY+PuQ<}xwBbejS zp`-#YHT$p5NsRh~XCqKwF9fV$bUp79e}vXm>8>aqC5lJg6cXuzSy#o9S4)E;J)7zI zJRb%Ry-oHoBOo753S{Y~g7g#rdaD=w{%RW5-+ zd*XEVum-nlIOdeyqPPuofB_58zPeW)d;PKWx+|7?kIDPM;#wPH+sL)g^>80&<6l6Z zJ5{Z|I;>unvh98%lsDDP)v}z>s+XDLurw>}_i*D-lqU+>$t0~3-@97wL-C@<#e(+59&|7DLp%Fyk`p)Pwe5<$)I{Un*xE_K#c17R zd_AwJ_sakXIneQ59T40I8c)c(F3TIyW$GA}pyPUm9(RG(c}IBQ{4vAlUHFbBK!+NG z$C~eHc1K9(%u9d6soCbu-*MHb+4kblDJQ~GW7NU+)uRv3ntQJAVo(~9I(ZW|eQ7a+H%kibnbPID#q3KZY@&y1!03=!Zric0j3E7gYwkJoAJ2;f2QhS_)*l4O zyuZx-um_1IC8E3)uZzvDWP}4is8;w6&*rMLj$Oe)oCRxd^w<5zW=&=3d$=c@T z*gdTvGiVBWvf{An)d%<2cD&{gjHbym^<1y#3NcSMyYxuC+QV$r%|K2@o_dI;qqNrD z0`G_P+Pm&_v%ilj>ducUD6jx^PUWt^laX5SrN`_3J5hQxX5Y%@nOfACX0_i7r)Jzu zdgO1T`IM7(v;^W!bz|@{0rvOVUZ{t?g6+Cvi-n8j)yLnZv5%&mOfS$5F;{D?XRN+F z^zOE#t2-^w?+igWteN}P!-|!c7TQy(p(G2fi0;r;1zK~cR=YOMp|@?1?3B zUAH_(r{17{lzdn7OP#C>siyCNLb7g@ZZnyx6(}pq?Hkx;IXpVqLSn<1sD5H)(s^{@#)s9x8)b`HHlWKTUFzU_Cg^B420xzL z&JV-9H>WQp-oy>9$U19wyZx8n#bxGLR|P1I#9K24x|cXLU9Z!%+Y(gQjB(|pkH3ki zmvppEQmx=U>e}U5=WinlI$5>aBs<)&ab}5W6#B-To=g0I|ERXmQXx|CvC=em*eV;h zwE!mc$B!9)xE}fY>2cxd2JaaqzKyNg>rsL+qJIuHgu`c2oYB}AJx`%H5qtSisOL;| z?RSeZ3q6jWpQsl@X*a)kIai2#RivSl9WKb)I*CBbhuEgYDHodMB;Ey1=zlr73a>1z z4O@OZ=`gUIFQagP3nsxHD|e4zx{28W9n`&CcvI7^~2_+RX>NA?9N$q`{uP?v?p7?0c8#p4>v17g7u%<3) zA$nB}-L(o3uDo)UpJ%_h*4ZN0R6t^@Z? zp|YytZGI7BvFD0OGmP3lDIV+H8 z%^s_n&eH$r>Ra&<-Rp3s#y{Nc(B8d53mhIih#LR}$k~F)Rwk1JwXg69w27Yf6pBg~ zMY=63vqtOsr`9J*TuUJ%nV&WqmD!-6Y&n)b6kv@~u$*&Cb6jdrIrFs8a{oC;sQ zA8=9NNrT&wr((fo(xD#xk8GfQsX$!9G2IZ#<=H)hxm!_Wz-mCVd<7mkO9mH6X!1`a z%t!gSzfP`MK|YKC_4qWXd8fctYGA*UXVvfI3HjNRB)XvXct+xMioD30p^ah=bEn$% zW_!*+k#j-gv7#|%+WlLPZzht=wsx3j#Nn~vO>`_GkH2lXOsv1L+x6!v5GT3xVlf)| z$_x3_@;rMPLgodfpf#gTXRcy}9D8%xxF5Jd33+SlYveQ#Wj7iWO{rd5r|~WesH322 z)l~56o-1bp>e}tFw_twJEBUm%i;rJfKGLCf05Q1<@vO|1f0xuSo$!ldpjc1^iN_2Z zC2D4N+gqCdU@q2Goy^)FpryLnY5XffV?yT3H*t!eDpD_fs?368XNdO;D_pM8Jh{z4 zQXTQK5nZc(zuiC7?wwmY-Dvb`W;9ioF-w976u}B|t47oXfrX+-tSxVu#TpJ&v07`o zrqjW@xRZMxzLncE4Q=BuS+KixWCxo`QI>ZNwC;K&8?>~P8ScKo^+&-_{^D5oc^rv& zv_r<58tMemJ>J~e7P*{ghYD7q!_CFtvz&f&^FBI5jCg-|oIcf>XDIq5!QHNvi^69j zGpx{?QdMrC3c>$rRMu&^L3gm?;3-F?KQQ!JePpZBEUm$4ODJfr>NnA0RQ4YeZZo>l zQtfujw%hzuzwgNK!WU+yi|*O3)5E{jdTrDF+S6nc@a3n>#()-XMU*acqNR%X?@=ry za*cY#u}FsPe9S$@d#>twL&@(hiW>r_hbCSZff726T~>%Wn3IXz@F1-o@$`|r&yi2c z$7L`hH#BuHp~Z05+19q!gH~4%J6f)0%N759*0QI>--&J339QS_ZPk5F$$(ucjy5xP zO!iyM9X?-XHoh(HmZ9y&$VnaXe0A8v?e~w_vQu;1X4|L-Z{A6ZGQNfPuE`4;U5dRg z|4(ViXnAW0==tovcr`;@o?!gh#Km;7~ zf@Q_z=fUhwZFo!8#8F1U2T(RviDwF?j6}s17j$h6F!IgHEKrlp{UT? zggDpg1^!kn(TsMo0qc*jgQo-ajM>;N*v$GDwvB2@sy36W?8`nYX>&f#@iPf2a;3pg z7b2~|*r5LQMcR~7d_oBrdvLX_J!}zu2%mSyY)S73^I)S|x#PaqLGhXc{sB){XK~Nd zM&XO*Ksj22C(Gx-GwUF51NTZtoKj#%^c9BtGVyB|Vfjw}$Q^*(A(CctAh0mg6^*UQ zw*EOL<2Fn7;~QzBD1caE&k@TYJqvyUxY@Cq^n9t3k^Bz}91|N(f<(56U1-CX{buCr zJLo$;_-3?6GU?aNvgK5lF zW2@QsPE(L;V^3`XnSK!@VlI#?ZkZm{&@qi~Y;-LtAr6>&aXe|uVl0hvve2(GVKkGA z$_jrf+pS;V=#*0JO4mk>9q0)PYT3F)m`0~!Pt{WhPi~)jC{s#F`XM{@q}{Lmk+u7w zY_81xnsw=mQg2XZBu=b#|(;uLZQ>m zcHq8fhSXjoO96vQ#E1Adf5KNkZOt_Z&pVJT=k&F&*FA^$TV>&pgT1X+sNRsf8;TG2q3vSpga9>hEPERi9Q0^w zByDSE{}F!W;3lp(>t$S5uGbjC42Q!UXZ9Q;wo9?gmlkF_ULuG7vH*1&1%9Wq(w>}i zI?@Z=`HBd3zvMCXoFW*`T@8|zm5_3dA0?h zbbMIP5|67NeeLa~{wO$piZWKXDF4f(x6br<-*S2R5D7pnY!S;hk>=~%n|a^1J$ytl z$uXmkfGQ&{jn7$WPtp8=c#m$m^E^e=%=a!~pWP;95aw2eJM}}+E zUI|~2pR9AkiCOURG#lEIBc!-8Kjci|Y@>;5NFi+;efX`?=<2Y`>rOK<_#=Vw?4NSr zaXx`{9Jb7)MvFxajTj!1sCOEbhP(n{%icK_2s#YFecpju%5=}q7;z10qdqR-WNmts za)W+&axj8jg}`Go<}uyZa?b0#t~ns>7pT6&5B*FRUs;Us6^JE+@5|gYo8XJ;K*yy!m_?L69cqweWO@HnA)p$Zo%C@x|Ae2S_pXjQyw<;)(fVV0G z?}-c}gC8U{Gd7uDY93}Hk>zFo@1#$DUG%ly)?bSez`)1@f$5n4mCp(7o~6x)UFS&q z*|a0ukwxc3-u}Jxz;Yz9-KGzC8`$Z|pq*(k6)a#T%T%S^G!go*#;3S+ydk7nLRIec z{7_>rmeGm_I8cX1-0KlTbq??1`Y%S(kaV|7ePb^^aCV8NBMaJtUvTd5DzEu#brWtJ zj=rOqf9TByg|ol=2Y@PyxfBwOl7E?xYHq-bY@OCCkXj~`1P;_JDRxU+l}OdB<2{)Z z?adjK5b>+)cL9QynCsyB`d&cpf`M9of;A}e7b#DeeeADD$$61g8+1$0h7If zR`o1ZrGXzu;CS=xJ-_jII8_rL?1N3($_L3e-pHjgk%| zrjB@8G#CQhOxWbxgasPEpeOjIL?)0duhY)RA2Mup$jd19&&P81=EAAM-K{h#aMvh^ zOve8W%4#hJ~#fE;{zw8kX{_d2h!5Md@hZaI3y3A`QYgK!t>N*pZ_q#L!@ zueWHx*Yk46d&kE<*MmQ>?dUHMyWUkEBk8wgG&o#ByR*9@>rYur!SstPqTq4to~Lfz zvUs=v0rx5Hi&IM}uFtmKuVkZq@;g9smht-POKxlVh%d#*vW9nX7KhKu>eq>z#i1 zl>cpe+j?0`>RFQC$>UyPOoj>KYk)~_xpPZY;z&r*w+M26qe$P#L_nDD^ zmYY#c6+%hCYsAqW3qFgpIg7hIXmJ12tV5no@&TAlAuxXpkT9FQKP0@>&f2;jU%3-v z<7lZ}L$5d+@B&43TMGS!wir$SzLx?grt?$>?YVm>bl;)56f9>Eu-zFzLT&{RCE&}7@W0^m z-$ZxWqf%(jWlwu9d%8~jqtdJ1+b%1v%@GtFgxD_w5?1h4R$1FokL~o%p@WbB`=g3n zx3-rF_^jB>?9mUc4FVZ$HXW!B`C$L$x50$V9vknSnUi(u-1QRU-JQ0U(6EMR@>ztX zwoF`mu*t+AZ9LjXdNJz(uJ7?^?jbNh4!JDFE-WFeIs$tf*m*YkN|L1VX%a7mm=L2e zqciM^S0|>;MWFs3a{hy3ZISl5^`HYHvqXC|IMlQ%B_jh~8}Z{L0ZuRjzr#GP)PkYy z-SzWBxE9oe4AAWb|D_Ka$$AOUniY!lb_VBt7WHah?%d^*kK^E@2X&C2*9bC(37~vY ze>N1Mzi^*vo!0!>{c9imqqAyc^bZj0s5$;YlHBe2MlB!T)ycfG<0Z~uEYX_J*mB-p zP{7_amkE{5&|eYmX;#s6S4tN)$y>PsDE3Rk#=3mGrS3QyN#?( z^S0dJ=WcQ&9wkH;%e1=@C@aEXwwE(gLBAZF-0*ypkAPHA-*L``A>*=R^9||&7G#ZB zUPaY*c4=4`OP~WVBiL}$S9@kM*UK-Irm&XC>qm#h3)z*YJXO zi9jomUw$z}QIQo)-KhG5EvdYG(YVRo27!h{|lsM}gpaV!a$=S;g< zK0{}V)C8gOd=rk}tUWH*e`=P*F_etZAVdMlIc{JbfJSJ-Mh)`TOf2)FeiQ**wxl5v ziJlIsxR-=2w*era{LQGV1PJ&#E}UE{gMSqoAZZ)Vnf|%*KELT)niEyYfjBe~5d#Vq zUhvaKShb#53@jMeEeB57YG26C0%(Y43Q>`9{JCJ>qI>5LopWM=KSyPK4%kba@6)4< z-I+}WfAX9w7axEHd_A8$&?s>Jz*i6pZv}pPRQJ#N#PTGn@v)6L3N=p}o8*XO_M7jCdyETi|3JHrT5A9Vj?fNvFdP7<2`G5Z}=3dCeUzdYdX zY)t?A%k?%#c;>Wq3rdsH;sdCY^iuK`@l)j-kUzUBo*+S`>&eh5n{xS?1K3 zEQ}`BoP+LvNl|P;o7HW2zn)AX^V8NnBB2ML7_}ReettK}bz3eO?TAbmF&;k6Z7}I} zK&73mg`M+{MTEyDTstXoem&y>!j>%<@!F~1%5lPMNmQ&M`NmO%)-RB(_D@?dzG#O$ z19?x`)Cu6IG`IoF=-p%MHL1W8SSKfw1n)9VB`HW0($cX$8rmV=dfd;a~FG*r+qr~ zEd&S%&GF?L58xUUbX|15U&@2M89B70X0AwXU1HsEOHYvdd9J6@GTs(H^zXl_$=jWv z>_fnt&ktHxCeVTlMHpK53r_kQi`awEki{%N*K~ksRebebd*!@*T${LI9(yetKs}(D z{{gQw2JvgDS|5Er28-jyN-K$ z@FBMM69d@kq>CyqSn+IdT5062ZC$R+)>aU+spH!P=r8^4)snQzJ?X;+tqpG<*7Z$M znu&uuOGRPsNwL65KZNsNRxc;yA#Y)?Zmc#QKY0%RqE3_>>xIxx@zwaBNb4-Al0 z?p-{*6wrKnu(9Yzd3ZHIQsduse}IKjOv*MPoYmhfsO__c3m5;r?y>oeskAc%la=S5 z2VcdxLE*nQCcR(_PenuFWm}(Y%wG+7`fcUI;4)Yu9`QuhpVu zcAKdvdW?G~cvnr>>Yjj3C{O4@D^x#Zt|FixodY+ISQ7m5S-yEBarm?;(MoBDPRAwW zmqYh$E5~oDFEwl&VDM5Q2RDCDO>bNRCzJ{uVXT_)hJGnVt8V*yszNk?yKqbsP(6@y z6WV({hPnq>kMvWA&5U;}fdPg)=+X&s^AJChHBIT0=ttD6AA5O!v20&EA`b{xNM<-> ziaan%3*xitt2#;SPoZs(mMKpE-C_25-n_hNqS~>`ZHjuiKf~ z>6aCr2pW?HJZrd<=JDqF;VjX>b{wA+v|9Ga7lE*wY9fa$0uOt zw0K>Cb-CX$MBxr}xZy4|z30huz-|I@c))J%S(PkOntUglO-Hpkd@E+JDwNrId1iLb z8|21U01Re0vX{v@Pr>CmyVjV2oopHBNPorZMA}cE-UYiOpF7$5d-S3VNnbblje$%_g`S4W#B8rS%Fn{t{-1Vs7 zBDtqh358&%T&KEaKO6{r*0VIuyDH<>;D@)hG^@vVIOk0Gtrg+I8x9!u%x~8R^h(%q zm(h=OR!rCGjM?;7*ul$o>d=p67Rd`NXb(TFI)GwquL0JxBPOVxL`0GL#E6jP+ZO*f zmapg8xYnv6gWO;-aF$tFdb$^TT;|ass&O1*Zp!feSIj_K=WM6~)X(aP21g?iu*f zo}9lQa)PSSAYg90UbQ=zItt?zO}ssq&s`m1#HBlU`H=oo-0C;;*G?_$#J;ev#r2S!wEqiFxi{nr`|0QjPBPGADR0t#ba@{*7zNl& zKxSXHWZOvzVh-f_B$wXR>uIN4mW3GI3*1a2LBv$7^PRiVt3bXxq>$L-okg`ghoFPP zC5_2b1LSbRDPoRQeKlZa`EuCG-Or7=zU$Ss6ztVUubu0=m}}tevIxlvMW_l$sV}+qxXE94b(~kGIIZ@GBm~2XC)Vj%TlcRd^lxWRIn?z)ciDo zXfTRV;mr*#=sxQVA)@D|LfU@vOV8d*N$EwD6RWA71c?)gKFyH8=_g{M-chpgL>&kk1X8(tf8mEk9eINzuz-_qHSj z&`yh@Lg1uZem4iO2dDezd-z2QZm26iB)p|E1nxYF&3B$W+*#r`KAl{83ze_(1)v}l zSyVo<95O`F{r&vLw_si!R*$w1dCJ`rS-z9TUip-=XI;Ty^(H#`nGeo|0KE6K*sW+v z$vJOYYAmb*V+*sDAiQ&eB&AyrZMQy6Ue!w=&y1|%>y{W zM$;PZ`5e(bR}d|Fc)tj|V$SA0!jiAt^Al;M2}AcY#}jVVk<@|ec&3%VGSnrvENb1gb)Ll;&c)(^~o=0eXcq#Ka?Ya^DzcrIb;0c6+Gx948PxpXip8f&z7x@e384s(=uGG&W%8VGL* zd4%F4J>*Cn)_^yVkQvOYH`Uj`p>UECoM!UXxrD=G%Pb!NK7}KAQQ)d@xrK6~>7Y-X zRFDW2rgtiAIpoL;Cr6^uW(hQ6C72pU++7*^i(CYHg!-+7^F4;Axq0niu@VA}G)km> zYNxv^(94aszB@fadEJK-k+`6gz0;+SJ&ibVq1^hIAOryEHDXEpdvZHsN#q<3HlG23 zBj95wmMDYrhq$-g)NT!$!qH*F`|Ijxn}YX|>a_e+{XYgl#u-8lSbRT_>jRV*9D-gS z@P*t`TFC!4J!?B*Hcoi8_7(VdbP~k}Xb$`Say_nV&%d!|KTI^g2fE9FQ|lS4961t| zt$l(QI2R+u?vOPklJc)4s9891x<*~oazGZ1-F+-Ph3>X~_w|D7KHa1E-n0}4}IqmE2>_Ay1dv7)#!gg9Fp;EozGyi~{-hoD@Uq7RD zj6PVJH=666?+8ZM%`cp`zvkS{-g#3hoH|3}SIwhsiuc7QDnM15LHfR!E4&AKXoVbS ztNBKY3KgOWrA7o3_hAQ5(TBk0E-a*B{9VS3XxrmC#mmhwzSz5Gc)xYlI@KE6;S6@B z3P5c4&c0T25H85ZPO-jd6Tnl5Ji!$7BFlfV{5MN+DDwSOGc$+2_CW_KPm|J z0`_A^)}!N3_E)!Hvi=6SiWJqf#y#Jpmbs6Rj)S}HiSwWezy#(v3FVEzw=2;Bv-Wc? zO~_cv8?9Wx&)aMcWw47es%#9uO76e(5J-LM>fuUCVYUrnuxwG;Otf|YIdGm!1U+m; zdEuwp(tu{@iVsm3Us~r^!sTuz;F-xy)u9WZ()89uo3!(wa50v(Ei+u@3>X6bW6Br3 z>SxxBIT^0JFV4`hdQOY76?vp`dzX@@)`YyTq@29+)t<@Y_$w4<(bAkDi9%9qBXHT6 z4lx5CbFAF+Cwz|^Lt1=@?ui;=37G3*Px>YqspJ|v66uuyJqM`%tMKJD?=OePSyV!e zG%&~%*9&S?nE0dBOMIu;7q=NY9+$mTe}OS0oh|7u!3=jo93RG7%mMy^zSO-arLa|{ zAxb}E{*^gTYNB5kvl$=8MWRzH;1U{+RHH%=Grzj|)VwH{XD*ZWI5fagzX- zPF$OYJ)b>&*V`DaEUoui@8ioirw-a!=8!VvKt8tD`W^ew||0^ zmM&|lOJ{m|JDmQWD2_efB{}1L`z+m3lk0=9yk9bMI%e>Ayv=bI1hrX6SU*l*ev1B9 zxBo3gxqiD+@oL#Az;2Ia3$y_y5GuOBS8p6X|2Jgr$(0q!9KWmm#}Pc31Y}Gz7}>E=E2vkQ01J63)CAOdp-6q&*A(#L>$0Le5Y*HY??KR0PqwN+FPxEs8W8oIxC z)htOc$1Kg2v*zy7nt^}JrcZ^}XQdih!-((CdMNSVAb?Pex)N-|i}EpC!B}#$VDW6c z_L!m9*WC-dUVlo4^R%%nS#NN1^w}92c(WDu5p@qjE|(dHQZtw8zf?$#y14$b1wSI{ zXg@c__!aY*^vnzh4aXn-?S28#f65)udRj|Ot^Q`?nl^pLXm8>?zPdXRgE!K=Y+;O2 z;l-~F0l}i2Rgi({Kc6WA#c*W+k5XSz0y6}7CtL-&S5tq&m~n;>V?%mNFwJgu!~Fs? z)tyNyo=ra|o|Akp0ZdboSFQ(^Jn#&ib}cD)_n5p-Gt>^?yrsMiHMwfnIddHDZrz@! z!Wu)AAzf3%fYKfxFRocGS1jrIFCq6yT|f zj^RV~R1}JTo2>eE^v&Z79_dBC4WP!!-_2(gvG54$E~ zQV5kv%b?Ib<#cQEQgXazlI6Khe{i=LUDyuV&$_^z#|oA5a?_j!_heCb8H}g!g(Nbh zLs0N*F@|+cWqA44Z;2L4=MQdO3k_pdbx{U->Cz`P)DPwjh& zp=T-t5Xn_mN#zy>?2q1(dDjF~Rk|xldX)J3VOf>@m0?LSgP)R8RQ)D*IC?chhk1dO zL_Epsj#G+Icq8(}XUyCH9>1<1p<+8r)gT3{D>5rhTk~CI`X>WbpXUexG84MC3=&Xc z^;B%5`TfBeG!Ui$zhPkxhP1?I3$_mvudA4CQ#Mtjcy5h!SFns;<*SyZU(IsQ0Ki5t zUW66pb2?K^9%SD+Gb|g6zXim7@K?L?g?!*q|0X-+)Wa4Xx!UT({r7tSHaPU&8SqPE z`c)&F%G$u76j1~N0;s1;Iy;K}T;4GB*B9biYJQ+M78@0)32KV_pN5z!N~MggKI{hc zMeJ(Y`FZhC7`ATuhwnZ2^_ZYlig0>1>y`qjS!&BiXo(r{(6nKfc?c0IOb1?HoMOsdC?7DQ4xk{T&d12l2U_IUh7BNwgAgyt5{Ui(*SuF2PL~JIjx+jkI9cTg73kmBGu=A=O_t)Z2d`{nILHL0@du8I} zaE9AU=yQa^@2v3`g&3zmRVI$P0Mhg+)RaGjdY}!+72Z=^;tV%Sq@EMo|6T*n#csjD zlKR!@bwlF`cO&FMu})$4@ps-l@Pzk0WOFQK?&ly7a*m~FTmqllWh;s4b2|9gBr zJt(kKuUn8OJ`Fj_fUoN~9^m3|ht2`84dZR9KRf+&WqNZ{be#=iUN~sJ^UduYH6c=n z##YC!%ni)~I=K&|@epb*;%@H2VI7~q=PMSUfHkDLGc>)5Vye)!8_v)Y z=4gigjV75ZUY+BXOyyr7WsQqfkimR`+YHpw0WUFN%O~KfN?S z`=!nL`*^MiU3PkVsPzx=^(RYWA74jHl%m$DN8(T$cFqxmoZb?ZBX{eG-RIq#N~Dm+ zkT!vsy}`tJpJ&)0i_u7xsW{N7C8B0JYN?vmiwR<(%*o*z0YXt6)%|IwGTSS)?gI!b zCb6g&f*)^8=5IZ2CRgcGI1f#x4rNa{%}F^AFKWrBvXtCnsgtWV*ckG zpi7JsHsxhexJfp>XBC?0xZDF9|sv6Ay4E$D}|g?^NbaqkTmi`W;Gy?vbi`e>~O_y$~6GM2m0j= zg#QUfBj`Y@{B$&p8KVDzZ0Dl_+N31tPufd=&-1%a;V)%GsW`tjZ@{_0^1j$+IMh=h z#_eGWd3u#q8|Djx9I%$tuV`7ZR=XTvwi@dv@CJaC4xeUO?})~cf6fUH%cDkEVR^B) zx`u!Zz2C|j+op)M2h8SosL}eBg-(SJyG>6rR~*X$HCEzPskZ*a`7vUw ztwZOQhPcQR721l^-EbTMD(~GYkn*mC%~7XQa;4`_59e{iOID%AHA&o=-;c`tO16`? zOBoJprrPwdhSvu*e}VAT*EyIA@2fR1i~{1R=)Dhs0PYB^(}Z9%T#CZ5Zef$vmAbuG zu7A@a;-Op%L)v>Fx!Fb&)MdStSSc>Pn+GvIrUX1#763+@=fKEcjr7ZWOh-QQbEA7J zYf5~Hg(I|+cLn?Jav04j8Vmwv^JMcYId;pJIQEXW?#QxxPfvmSUxhPrF2pG$72nRJ z2YtRjq*MpgVpQx2oI+e%j0i3Eq}=o-Wzo0=%B7|kpJHZ}G(qdAYbM)RTe>tu|M)^_ zYUYZ;ThAU3lW{JD7Fie%Bw}34x)bNoMgj_>6^=O)S3#%ttyfdxN;rl&RuU39 zhJ*gJ^MU-z=>Q!zD6C!q;XY8Bh>BT2kBZ#$Ehrm(uOIh#v|eYh2}zW!@q9fjVd9Mq zy#d-R>}$vP{!aWtQrZ2w27pUTXe#0@lY^n(nWFVavZVFdroz2FD~`VxMys_QTe}`qci4J+D2DN~EStA=`O!5+ zW{8PSDv9{1)fbeYm=V>^(RcI7brMi1EK1h z_tmW22cq~r)P!HwU^?B>yN1q!e**8S8orTY~YqNbouaBC;P~iRqCSuW<5U z1&Z_WDXee?0(8!+D-jde;}`_oM2R@X+{D%Bp9QpjGWVE1nF*LZLH+#NaO=(LI#$otu%bOVrOY}bZ!?7w8c4}Xf1KQ_R+FI zt1jmauOAO=O|y!fxfv|M_43W3?;~%E%kvy@1oj7^Xrxz^R?hi5Ot7iC7&J z;=`cf2PcKkkB+JFS!!YU_mmqj>>JlkPtYc*z%=Mu zP=XJyf-CO*|9zkf^952Hq@X)7rB%OtWs~Mir~f!$zPizfC1@YB^b_vIngs3lL^xBG zk&mcpl2iozDHX-qT45#D1A|NXlLt2C*mV3=t!}nm+0VWM*2d9{>7{hL#7C_sEe)_V z^tw_ft-oWo_#>b678I*InLg01bP;HVps<1hKhA>?`IEeWF>s8H{}`CTskv5vrwaP; z|6}aEhRp&b{u2t z%PPt=!PL?0%eiYJmrup3IHyK8y_feFO9V{;Czw$NzfP-mKKj+;g?=*3m{ z4&PZesn}9w+r+pQL-@y>GNMs6u~6y%c|7ew7cAA#Uaw%pBb8dMu?w?d(IMdHY296C z0Hq~9Z}m+Wq*C{j^as)yJWDwgti^N!psqv*X>nAGe44KnIBc5eb^koWGuxobBvQzn zu%)mS(KJtyzZCKuHRy&66wwhm20RA1FaHUkP?g$?Du~Dbr3tS$BS7UqXy)xI+HP~y0`NU+5Oxs4JK4DxqjbDsV_S#4*{!W7jqwJeA1 zS@P6a*mY!p>ooPzn9*&-5eSwZPes@G1u*_ zNCi~Wxyv%TZ|U-$;n4zZcU!NJa&S>LbxBx{u*{r9#Zr7biHf3bkw`SDmwZYiUkxrM zHaoNQ1x7DveLJB(+7Pu58LzX-%X3|Z&{e*6qG`n~A!h2W0+s_rg<66ZIz7BOlgOQ} z?m1d=tm;u6iNJ@)Qo|)C2~9CMvrg2@GEh#6T$s$vesK8N)eO;#Hn*O-odl|qYq0y% z$Wew1W-0KYXMnS}IM8Flq=2Wm;j|ECINRW6_AXBao=ILd>xTCOqbSz8g)z5F+CR^i zeDWXn7f1Ct4nlXST;_Ks+U$Fex)-K0(j6@NMPuJ7`}2^V$j z-RPP$E}|!66_A!FhH)@E$YekTqQoH*e-{cT!CEB)zDphbnJ|XrjYs&L-pGzlX_>$ zum2>PWH<-Y1X5-{$ovw0JcL=IKXZ6At79y|Kr;(eVNs6XZ~Ni>_^Ru>K1I9kc%lVo z?`(1<(+1B<4fA7n^$|X!p7mIE_0?pfQoqD>6}Z!w`#Jl02gE@(i{nbb>(&6p1Ie65 zfd}MCS&le0R2(=1k$Y-(MzeJFv&8GmI!x)8F#edTHGKy2k7lR6Y253WP$CF(jIS^M zn2L~rgta_KN<5l5Q>Epuqgs*6a9XC!aHcYvlwucoL z>OnJ_!}|MdeMq;kE-QCw<`!g)4n_8FEKV-@d1UmAe9-fCFZ@~QTI1@`VVFvZ-H&*N zI(C3zqR4n9NQ>$81$pnE46A5ANBcR(Pc@fWI0=rn`G?bNNpI2>FLQhyl-?Bp(pyyH zccavu0l5ZhlzL6gAdqn+p|Qo%2ky)3v^MLRE#|TF180}hb#xrw#5$h3x1$Y_^@i+;FgV)A;=J-}73@7-=z z<5LVyKRtt63b^43L9X)*Geu@f3UqZ8Y@9h}b%&%ndA-fpn+8g*g(=d4q*42RyFB1; zu?UD*O0MLkgo;`UuPD5++2#hEohOrX6|SN~FLt?RP!6mihc;Cv<6Go@a^IrHX6_yQ z#bACtqjZdIM_dP%dZVXB->nhz0A}sB6JF)sl*;%_g5cF)V`mU|_bf2=0}=PPzaFG;Ot6XNJ%n@wVC0EH27Gwj*jt``5K zE0qdQumQCm9GM5R4q%;IZ~&kMr`-4CgF z>BGNu_q)u7rjM6D{%M0LFw?z@50~FbWUqPHeRc2j+-(J=4C5Jsv7Q!7lVvwpqy3QB zovDs&_SNkiBVwh6rBXgm4?Tjf}5+JKlks@D)~(!HSK7t0CKE|Ww0I*#81obT1M z4y1L)<*wMo7krrXTDzVo*NTK6LaFU>P)Cz(6+e(M(z)qmc0yhXGtS&$#lHS2M=trk zJaZC10-FS?ecr18T#icr@r;2t>1^9 z%eKqg-D_{N>%bRs78@Uma>bTfW`WNVU!Cl1IqrW#WyPd7%1M^%ZuoezH6iH@+%FjM zEDEll-0E}e=K6z$MH}078S1f*aQ?zY?N1rBO4;Y%naEwkks<<61&Cmj61)Ew8m6qx z6hVO#FS&pyeyONoiB8k@Pki&$dccmu4`jHs++>d_fxN@*YxmP>;4Bi{Q>WGF)j+n; z-<*~S6@?t>>4_N!X$rhsGz!GlLF!;+?kisX4l`0N%VlO&oND}z`z`iH82MJPhtkdp z9ZK;x^G^KsH;xtk@;7{Of@)Vm2~z>W6%-9!5jn!}u+!{)qG_;}K?7MU3uA)F<$0Yd zI2q*~x2B?&%I0{3yF-P;xC0cEpB`v{unAn(MqqDu@%zR$(Y|lZ11;?RefeLTcFO~8 zKJ^^DnGID3xy@D&>8FQI=B+*Sv3)Y%BEGu4rxZVgrBl(peH zzOQ4we#cJ2Z!+&LCWL(Cz{oqnId@{!T>GfcOXtyaW=|uW-C~VG*sQ2)!Y!Z{BVpES zKeMWqBkpr9?q477z;1YNd4ZDDC02{0L(j-ks3}i@6P$TC;Sk@MIlJ4Aj_F&{Cbsg9 zspK5=C!5NCsY3b83X4Q=1x|4==7a1^jp#8->|Dt;dyrS;eYJM!{2SyFh*KohfvfIr z{Qtp9&k25SMdt1$3-?T%ZSnhAuUN;FRn+~m^?dbYDe(V?FCJ)B9TmqEJ5@AOj@Ub< z)5t@q1yYXchw@eQT{nPob8Q@zp?_n0&b0h9`InqHX?Le-A=M7srfQ< zbU_y2#<)3pNLH3&c{u)l`quFSAXA_qa{E?-te`yUOJ98EuailW_sOt%KALKZaXFeQ z;3T+-Zyg4HtpPS1|38u+m;^l|S0Np2jtz{7#11Npv%Aa>8I?FRm$nT2bX@c_GSMUU z)Igglfd7^^m2q3JB&)}^yxlLQ&thP!;9+D{xwvJ!V%wVmcc(cFyQ@oBOgX9iKqB?` z1gWyA#mnsPkl2F^ltnZZiVD~kv-ia(2r)DPfok-#+L2G`u1RXiiNVY zAf_oBYqV?fK=DJJB4QeK<6BP^+&UO|Kf^NU)v=<=7 z3DPqa&Ubzt6#p9EuJ=VLi%0eJUE6Yi<-nh4D#K5jO9 zKY!-k9ec7MMPNwO30%1te)YVdw)uyJXLR>$K79?m^DRvnL{jnl$8QroA*ecp6rzv5 zD_6u47x~=0TfTou@N2h)@rQaqc`J^Ip`WDt&wCJDFag^NaseGV!Pg0SY9Q6u7>Y*v z%eC00)}oJ*SfzRiWIx`STdG1$d99lPf-M${SxFda8JQ)GKL(K%#J!dmij@{}PHhcP zmf)U<%4vV`s`Y#drQmr$Y0`1iHp823sy=hk1N+9hLA2%}3k65@aHX^6FAkHo(hn7g z9>jMT8YvKycyj_+b7oX`gu%$O9I7Yo`vqke@smD*p-fGs^ooI&n56O+>(f-&^gZ*Z z;Z@A6Q_-c(Zr-4w#}5nepCUD_Wfy8qVwRNM#lS5kQ@gMBK;5BQ(%S1sRj&%K9pz3w z!G-k)D&hYAS~|6x{c&F@WTVqMZ!Tft0xzIj6l988+MR$-__TI=k4#~eU8};Z`w?qE z*Zei)MN1Xr$Sqm@k)#}}e`r_Vm-p)-R+~I2$Blp$Bx2P1zgAkqv)>l4#orb$qC@G6 z8$by=P+Og`d>6{@y5Fs`;I2i53Lq6hQ`fL8C^f1OhAo7ei>CH)a{AJ9y)FtDSK*xdet z9XCdvY^oW%6=;gV+;3t3$?%dIORP_aoJrNNdgO-MQV1u^W5CYOy-U;DY1T2l&~2K# zN({OfvgvCN*M81ek+hSr67m&T8^U>!rHjuW_4$%c170h-LVJQiQs^p>0eP}9o<>dm zGp(vMipL2bZXy=4w1DB8bg(IMPxJfB4xU75-v$*v(FG3jER&~@nVu;<_Epi7#bJ+` zo4$-G#N!$7L zLxOkBQWoS7bD2-{rES)xK}VdlRaC-ga@VE+nx9;|e6F)t-b9VW;-vPGlSBUxp4MG@ z?@-p-clS0Hx=k1}Mh=8^X1Kn5jR$``jPmbvr=?kgZ}XH%(Y5uZfny2oxR||_g3_MO zi)aS{atk)?@y|M4|3B_$-j6XUwrnc7<$%QEme|+g0l$zFw`V$Ci(KtfP}fY@{_4PaxHsg8{!QHDYKJN@{ku!8 zZi)j*p@;<2qsDyshbS-KqUxbqn7QFIIddaeW(EdKOptSwjaC11=FBYPzC#wmVY*7r z6Z0GClmjx4B^<+s<)sqKs_&Y({x*gL7CxIcYjMZBj`(w|lx#I;Jl+~Cs6y#qS^DCp zXq2RmV1L)#x&r^$64X>I6`pdQSmr?^Z-1mI<1dNOOR3>)-YV!>trFB~W-A@BVdF-0 zEieC5`qsVL-WdDGjK6PeDE35I@;fM%5|}I-h5RYC$gyEpw{?RZ^hcFYu0Py2?5;jM zqj@qNx*!~fHq(FabKX5{ibF?AOmkYN#|%Gk_St<_LLdL*>OWB_ON=H%5fFTZubN&4 z(qlZbgUv7gq6qGB_iXH>nxBEFC04QJbPVz*@8Zb~>lLqNwnd(G6u&W3rKbMDDwnlOe(X7D z4WEn|C@)WRBNL7(BguQGSN8-?7Ts_%`G32cdzfKi)z}<+~^qbfaGkua!qSbvvZWxMU*1b;^j7Es@D2Wlh0Dm-fm!9 zR^Ia&-oSoG)0Yg({*0zu&N{az0hU9(21JXufJ{}1@s-edbF{qB+0^~01UpA4cuA?% zA9r2h4rK5G#dqQllb5vBzz{q^Jn2IrD-k=+{|r3ofVtvmJn-8_bN-!t;2Oy2UkeAJ z5`8V4?Kicm$B`+V`I0@`z*i8Pre366|9~*os;tgW;q^hMI60=a-SjcJj@Kkk3-7d{ z+VfK6{b%rP37!U7JDuFV_%i1c%j~lX_SDO__Li;l+f`~=^m;7&Go~X(?on0F*84A{ zh#=4|9}4pca%{GcFjK9Pz_r zoN1OBHuy^@Hh$mWd`yl5J5Ec_Z)dX#A8@=hSNU*-(?ewXNE&ORS}O{b)w=xC3-$Cn z*~p2p4-blT=dCoO!NunQ8Sg;3k~Mw#eUyYf(I01?KO70YkrltuFy&`gAwag_vsL90 zFjM8hyBE|8{7T}v?ds4Q%^u3=^?&f8!}islYq;HPwsg>IcjMr#L%o^5PY=Ex#m}6s zR_IX%xy4LPhaT`uA6Y)Kadzy;ve9oo+OxS%u1!*m59pS;AaDC!9hzF+aVeNOf{~kM z;88cbn9qtm2E1cMW_6~hG>1^;6dS(`Xkpb!Rg${D#gyKQ-%;N%OKK~wN^(fV2M6DP z%bLjjR>YW`WgoSU=z5P^WfM_Rt*qUeqc4Nl%7guE zCxZqaE(O7^fihJv)%w2ihkM3@w>IBiMqNTYHhC|j{;{D8fkg$MQ&hGjJL^zThG>ly zTZeaUeZe)m$Vp#cZVX(azlxR!?JYX~XsRLDLf*<9VeMMq8JTXa@9sUr9+V%Vl)LG#!l$+X`JNLb?w;MHb{WE|F6o}j_g@}3>#o1=8 z@*J-$*i62ZB>OZmBKtyGAhr(%<4$NwL9d8B}8BpgF;FrqDmNpE9Vyo0(hCssfzr@zF@Dm=+Vf$xbZ}aelRpGZ&lm;9Ojr>F(wsW7UnERw3R7KU@sU0PQtzXNnCe-%d0ehBo#jFjcEws4s-w`qB* zel$wxTm3PZTP>U-93M?%5hf%<$M7nkU;;={@??JUFnUp6TedL7wvfB$eegD@H-GHR zSJQJAK+x%E*ow)BXM1bLMQFxG8XlPXoR%7ld%kRzA$R{fvRh!>*5Dc9xInZwf)2x# zn;-jcQn`EHWq~2fA3I1BQSHExBh_Os(gn|eC!WVE0dQYXsZ6T?VzG{IBR|8Gr<2Rj z<$UL>fV@A*JFb0XaiKj_lNq@-ZPUbf-l2>4Ya%*|4Y4TqIn8fNU)~fisv3&Z&kSMF z7Ap+tl@ONc`5C6OYyqpgZCQ>+Ewt@xi@=0q;20P0QM-}r{s529bgAwum|k+}M|FN< zqocGwtH(5*0)C!-MY^D8ZBQj$oZWfE#@3?Md&PIff92~+u-HF(-uWHec|Y|oWZ*iF zVUammm0i^;E5rXV#xTR=0l*tuhMbn{k%-1DPVpP6p-^H}yG~w*6hO>>DE6m+LKf z1u4BJ43GXjKlPOJs82a8#!BLy%S^R_FJ7tHNl|u~MleOa+gj~QnxDI&yv3MB(5r4n zXp>l5W!tBBl!dHKzglEyZ*Vk5BZc9k3Xevl;b6*!;0yw2WBY5VnJtT@+%V+H=$9=w zju`0Uwuh$t0`>n)IGL*E?Qqxd)!m`3LB{J=mMJOltUVHxyt>b2Eq@Z9Nn zY9TpIf3-qeRZf%mJ3xW?blWr6<%{`uFklb2O*M3B#{pV|0qZ7N9kCH5B~k1Tdw$1S zy!Nt+L+~y%*+$5U@c=DIfo0TKcm{YQmHFa+jQM@+7GbUuQHipq1B#J8=DulEC%s`D z1WHnlD`L$3D7h3;iqQC&1+w5yIQ5rvqR-!ZX|e&JwLrBzVTi79KZYYZmeIi`x_V9y zwWybQTd|BUm>D>Ck?+zn0@o6&7w38s0NU=U2r3QdCu@oFmL8vA$a2U~gwF4#T4- z;Oz*O_MTI}pyBra#mX0jYfZcv449?op9aJV`4^01TnXcFORk~uZ*oo0Y9l*46`q;? zTLZ^yYvv|1!j#TaJB^l!2uq?-%j*?^B!*1S!#PYm_~Iu_4Eflq+myA%v>4i}(AsXa zaEF>nrS&t7u$1>#rgtKQnyLa#bg#2XI*Sdc#Bha5FIzec7mJ55Mu7AS8dQ53O8x8t zMkC#ZSEnJnlWm1cUiH_PLIZ&hbopeio~QO6EnAPGSFsN47L$Hd*e4PK zaO1p@&l#AH*UmXy2X8?S0{pn@A9FoDQzU{^PUdH{YLC}C-2*>a@qe#ZxnSt78u-3Y z(UO@>65I5AKR~F{IKAwt!Z!J7a^sdveYXa&X21Px8Sh9@tSEf_{eu^8&Px)0Xu7i>Jv zt5oSG7;m1t$e1Qg6R|Q0!0F2P;n=y)-yh#NN@lut=`4Jw(A!oRy*>%O%53N?sHRGV z%qo+e8-5ab6@Kqp6no$UXtpD{_QxZFJ8Bk%1nCF7wGDFD?ND*FBxs5M;4W=gm4yY8 z8Z9ySUI#u(y5KZ^(a3LJHE)8p6fern6gU0$ZYh)bKWg9mha)pegAsZfOEn7`_fl$ZH)kVnl>J0KOm&Qt6%jJp zm_gntW!Wwp8isXwyI09R@^tK-UHk;f5T4&^JJN4~%sk7@b;eQ3!69`4?tKrA^?xtU zQ9v0Qix{td(pz+57n!ZiUhglGOn+E$-|^%t3)zK%OGXXdD#Vqzy<(0$t%b>+5kn!A z37B-pa2h>FOmz7|zKyy%&F8vkEOEx+%k-gzH2aKMqP}*NhKWvVIP@AzLK3Y zbiKzfDa=ufP0Nr9XW%4ay^*CMoW3DS1qL*ZPaOl8kh4Wt)HedA9k`>DsnK1AIvSC) zLp61i${p+~>bE`yYrF8IxoAwJA-YDxelJNST{gLVb#<$wmbp3IP-1qgJ`FePlvFy$-Sf6zC(81aMlvKEF=eO0A)^Fc^ zr%EwLhhpQI`|vi-WA3{b=!@>U)h2Ed%|QE#mV|M<#uezR8ye2^1iZJWe=p<`+#z|= z+5(%LjnVXo+29yX?vVo+XlO+4A@VSM;(am^502+~aI3vsPdDoSje~~i z`b7S5ZM1fx_zmN*V&yjx#Z!|HanW1p57c`R#uyCfH{%-@XkGRoP0*Gq-J(x+7Uqx? zUcOWbF!vrdcIE8ze;=l4NL%|IfbyMogX5-`3;>jw3i=XX0|4?wHx}-`6nOb7$P z#KjF-l_Lm5Ofu?F^w{%d|AM?OM-$ylzWGT~lb~SA`fh0zKGy0sem?6dvik4fkFwXA ztZJyZ^k(QPw={xJyd~_O_3@3*)GW3pQIPbFlGzNK6+3v^wl)IAa!$U@CK}RDZoBdc{)&A?IEAHGhjz% zO-9`_X?+E73Vr7dS+io0Y3P)19UwvNYUHh+Nv$!lIW?-MA)JfW)>Xdb4@?4gjx=pS znaJ3MT*hSQF^d~nwVQcvI3G`u}IsXAI%CRoW00Qc;-mlpWbqO=Y9vd#bjyPS@B zp_v&OuX1BeTh-j2Pt88%f~Db z1iAb$d2IlP8`dqR+`$TU|MKGM3C6LR5x&7aONYV#Y`PfX|2S6`@h#c5pWFuwwFmKmG~;+bIKJyNle}-Z_|V zF%VX`3A5%a&yF>tMrOSkTYB1vU*LBTHY9xAOZ(D10K@}|RiJpZDoVTf)%v61>!eeG z2-D+OuCx+U=d=spAv@H4-Armr4?f>tYx?KXxHpGkF(g}(Z&`w1rDns&Ium;CpNCH9 z-zz(Y8wMzPXxSWRXvdoB>(xkd?ZJitSiIlU_OAb6$Upr9I%r@`<*fV&v4p!{7(KufH!3Ha2N>u@Z3_;lGTE?bA1}+7sA%<>#Arp!o=%C#)Y||j*w}p)aCd$o>wk$Vkr;C&8tw22?|U*jXc3wed;tS9+!TCv`#e0VcTBK3oyXAN1$g|L zDYC?68kI?~^aE~1s%zLb;7@2oQ@v;^HX*HdCjX)~ZXwuN$+%G@#H$je^uJTT&u06B zAOE6R75rGNy4K~R(;74x_(w}^ZW0RDWY~UAtFo%Uu1byE;tAD^KbuOW&hZ|oK`QO0 z+t;iv8M)@t1N=M?)ZzP^k^06(R%Np(XRDT8d$LL!GWuE&ApQt$j&t&Y{G-YFY)*od{L2LpN@;I&hf`E7!r0Caq$?f=2|wYE^(tfm6U|F;dx80siQ|q%Dxb zXo4|d%pH{h9D1m)vDA%oVv3A)nO>fRmp1gR$Qim1zqU{cYKx;n=k#p(vs2b7X5Tz~ z4`J|u85?zQ7dcLYq27)8_6qO>mCsY>K%jMM8WRpVma4##H?DU*Z5ZEClsuV>o&Ps! zH&{$9_4^>VC)b{$9@||)h%rcOi*bdK_G1oxebNF4v=%1 zys)nxTlDwLU=KQg&Ne9=F=ruy2r}SkiNA~h;)Z2J20``<;S#I$K|Te4JLq>EPCyAs|eq4bZZt*?~`5h8d~)y6ZS8-2U^mm$uBjLa-=*B z5e3CoLV|TnO?0K#R`t~6{cnk+9+OukVYR3hsbI}=%M>~a+mOnJ&zo;ZwEwsSkU^p@ zF|NLj;x{m4Jk~y%n;MVA*lcHY0OjdmmW;95LBdKs1~;l~KdMhh{D?{^nJdNS1m!eK@_rowkV+Ho<`kCLR%zM zMSQ=ihe7O)g}{d5QVL1uQP3cSnd$}QD@Zodor_z+vpN?6xC%%|fDrc+(8mdV5awUi z9$JcIk#Wit+NScTyW?0KKo3m|p__nLj;B;Q#IsBCTsBOMn4`14Vf$A)F0;W2LwM&9 zONG2?!nihBVKQ3Us>cbW>~)!yvpe5fLb8`VoGXXs^rjI8_`%S${>pP-d+PkkN605~ zE8)7(I0VFt1pw!w30kUG7I=wdy8ZeAs-iDbZ>Ge>7&R;a>&2a~x}yrZ`at6=m`wlI zmsoy%upOK&>Zv2QE}P{7WHlV!`{sQ&j=Su>9!7>OyZQX!BSEzPBcq;*scFD|%V%Q_)={z}!7Q{k% z{}a34I_L}_9;)idt88j^uF=Bd3H_)LCe5Mpm4E?vsEP~p!TJyv=9hw!UA|Q|<@a-Yc?Fh}8B=^%%Uy*%}o_jmAGnw0~-M;_J6>Uob8%;_kffTgv zk#lHliniGbATB|Ja(M4tZL*0M< zg2OFyE}}HM)6_?Cc8?sz5m3MnhW;^!3f-}PE3h!V(`Ryuu!3Vp9P<})D~0surLvKk z?rxGW|Lr~S5E3@=pgJnhMh&RC<+sb$H4?`>tL-)M42K7087?!)eHp>xaancb6uAHo zfk@wHAQZAfx&U9TtONAp%AH`7as$XNT3vnfccEtbG=fkJ$cF6}3gzg{))U4gno-57P7s{Yj)!m{3fHmM(IX2nd zio2&Fb2DVhj(bM8%x2rftB$5`)2G;m{$b;tP(CH1>P2N_%;rtj#( z31?ZM$^qY8&Qo>AKf>%0Da(1>F+kW+zratNx*j4)F_9VCYb7PvZd8W^*1}FMxkv+7Zortzh-55e zEKf2p&IbB3oS+iOg^xMu6Vt0;vDcnT=@&a)HL#r;sV^q5OFq*VU%Q(SE~!jmPY`}+ z1s?yax}I+tdoT4i!V>!sggMFtUm>-nT-^(|mQt9TwAosdcz`^wtH{X;>311DTCA|? zFvylyrDAI*R;z&nq%lYZ6NtYIRA}Lh3vox4B^;-T#Xay&OYFx)m+e@Rt7@515utWg zhR@+k$&>1%qIV49(l${miVFaw`^q~0LHPrrY{ip6T6P8lRYK`xcfj*K+WtJ1rrLXn z++%%%_()j^7H$A4VCvUyf)C^21)K2gV{eTmJQpo->LE2VJ}Jcz}vf2*E{$gt`O^je7l zxUU2Uk7Y?e&U$5Prg+a3yj=xeB)Mhc_WaRg4))^${QZ~D7M|yig({f`b_`l|@;e&H z2-*E%f6uR6U8rWwL8lL{8Fcpj0UUcGG#}6 zFg&H7p&^7@3RPG;chk4Qeua1luh!h@u0h#KNif zRk9Dwtdr;%Z>{sdlj5%fwmuW-1GaQY8*@>2_NrHU##(l^ms{RK-J(Lc7VpFQvdIzQEXgXrYIMU$5>(p-tTsI$s}*@`5;E{Ppp3wEO> zI{5}pwGbG%m46BYbI&}GWT*Ka3Fq}trik^^vdOD3$+7#&>pn;-x2iixt zJ8DP>7G1||5qYk$h@>HaHX+n>CsSH&b#Ritw=$P5g+-XZcTI!E+Zz^L3m8O1#9zF0#-LMV%N zL%#ZU#H7J=dgS6Mf)J`nuuq)Emwe{+{>e=wi;XA`i`pg}>xy9`;Xui|Ko!e9#!!V6 z<4w>ujGrsJuEFjt9GV;2`~;$?T4cyAIk2JqG?GT1$gAuFRnVnE*`Ot@&phZIfv}`Qw5^0GBp4=68|VY>`ZE%ikwJ0vXxAuYY(e|P!S^R`N!}ne62Lgn zHs?LDdnwTNUeS=l`-4|qrebTYg&%ukgs zeHrcTuynP>r>{mPxYcI_aqILvt&d1qtr-R#RZ> zw;jG<^j=nlc;VgOKYrTaZf77G7$yfobpp>s#R-Y>3#oR+F9=#mFD+aQ6;MiTgnc6k z6TJ)GfGU~4`9+qaafYH|3hDq#*XA*5_YS2q^!(BaH2#Pn!94Z8*7|>CPM$WzoRqn! z4!WV4+l;}6hR;DS>QqqIT*&v=Yu9x&YI%Jw@pq?ygrFV6Y)#-7I;==Rne4epU`OU} z8c70Sh0GZL6mDVAE;)Kd+AtwP&ZzIq~Y+|a;92Bc& zX9e>oKJ`K1jmIhg%<@-0h(gIfoYXs9LyB~FRcn#cD^ng&OLFOlQMJ6@I|{o7`>yRY3LU#EdOui@^8aVaAv6xlH;}XC zrEv7tklhXo{Z>#>?Y3>-cTtqZBcIwQ3rbWHJI+Ig2-gB9$=Lzm&u+*&wt=xGLN)1jv ztIXLz)~BFjb^A$O!tXA}!|p;xCP|eC3kDtHY}fDuGNmkWVn!UzPs(PS1&@LUTq9@N z4XCpXLUf1MC&|r(p&9nD^-|uRB^)W&aYWqOSM6zJBs*=8p^8&oW|?WBnw>opPKFv7 zCy+gNhvM)<8c4hl#Uk;Q{c%%$i$0t4W$9_f?tdYN? z_BA36`t+8VBGRmgtkjg-|oOK)>Emh4Z z4sb?Hpf84RTu}aWx5ky4y)dXN(6Vw@`1$@8{@V^~O|Xq3{gipz^8Q&=Z37IWv{^8( z=3Q!)_0nSH*v44+n*w@V6L3g>p8xZ%_X4j?imb}FJLn!xsJUpA>+af>!B2Zp3uk~F z=}ogxQ=dLEV}EFsztfyKAznCSt4dbwD`P_VpKQqo#!(cvQv;Kd&@miq&c2k>M;mmu z$1?i3Q)*^he5COm+hS8iV?PHSO(Q8)nIbtIUp;PUsNPVN()MC}Yk(~xaq2k^oQ%JW zmndIw20NI7_8m@eZV_`V%#G2$yeAGxWhn@>Zep-y%(3aI zq^O)fH5X(`Hqc7nFGze!w(-NOUKF{{%a-2J!56%es~Pn?o@e?EQbY+Cm!q}>dhDns zKRUMbAY2?rn&i#6Ey!t;hnftKM|~dGn1&mC-N@*@S#bX`pZz`GRIWD z^}59noh`IjTdb0wmidy=YM^wA?({i}5sq({YehK`2KzQ1Rr+|^n5ObCo07Jbd<<3l z)4)ms_Eoj?CsUxr-Z-{nOfQHSO$B{xam%`Bp;C>gdO|t%F%%Dqwz$)tI)1nJF(tW>U9z)L~*f<@=x2r3>fyp>X6=V^`>OkqsX&mDyOgru4dsB6*1EB z=&jqYr_gp0W-ET~(~ik@^5|APFR0;pWl^HxhAi3;ZJ@x4HzeeP;Mw2I>Jc-VXEdP* zy2SW6&&>@y&TdpzYRGXs8i~G5H`?Q5UF7xccyxuq@qXk=X^!~jq`5ez{!gvZ3QTBt zJTRYLJ^N0UhX^Mt-4bo%c3?;rHCmJS@p|GxFd1g!tD5=a*lGA3jv+xY1SqKv|izj z{}M1Y(Ah@)TDnIJHZllm>YF6Brb9;9Lmi;l9rtLT|XuOEeC; zkBq8ofMBGxVA3KI<}Mp^FBmILG9FEKu$)@|ERA~p)n ztcK}^f*E}q3!+LIJ@u);tHfrUclKzl5!NtIB!dS!UUPZ{hHq{<8KJ0lLwFZs1MQFT zF975;GPb~fWR{%H4H>*t8975&olnCDmfyOzb4 zmrdJ---Vb|+*og|6$ZhyuD?7-)i$^H#%O0p)yh@Nk(pL~P`d03`g|BIxO(~MquB^Mvvsu7FxnIL!g-0N;lQ+3z|+cJJZ@y#ZO6EAq#DsaL7I;twKdO8EC zcOD7VmKguXOnDcK(xS*H?5ui~Hhe2N-4rGPB8XjgVEr-GYyE4_SEg%jitt?in#>Bz z9k8p}4mK^R-4!yB8=#TTKNPC->RXD~fX8=ZH;0=4VBCEKMnmXw*3h+SPK&y5(5>@X zcprZeMSj}J-GhPRul;VN+HPj8-h2NBR$bx04!N?1UZr*YGvLY=*YXB`7jkd=*U|YI zuy(yh^;3zIrrn8Xq@;tc4fh9z+o>&1htf)>-#z~#;n?d-j@@s(!v&N8aSjgX9=)Om zg#-|!_oq+{;?()udI|P;TXDXi??Tr9F-i2=b)V2G1lFxlbcb|?6ydUBezf_?5?iq9 zoK%5{8?6tnP;Xmkky!to!!0K)Fj!U^* z^iy8%2G;&P@s20-&-wJ(Mq)hh0gS%l7H3fF5k;0BUgE^Y(Yb|fnQzbAKn$9Re5r}-1V_@^83rNFpswX8i5!$Ro2xsiKPH4WJow;ufa7R zfSIQ?|HqH+wud$|vX!1OZ_S^mOBD>Y2MZC~i=h5ze23_(bxH%AJKs7%W!)1d`k);f zU119Q!&~j6mUT6j1+5T`RV(L}-Dzs2Yh`6gpzDOD-5U>LJp??c`;DOY>2Ae!inBov zf-HbV&ET|2Om+~$8aXSi4Ae(I7*Or>|97a12z-_OL@9qkf=d~HUi!Koh@n%mRsFGV zEFe@(LNW=$w6gylb|b7ONyQSoC|>{qz@U?GH-S+n3f^QBx*=>RY$Xgj{@`pw!bDI> zqGg>`?sC`oA6(9W0`|HROEw&^*M%E=9JhJuxB5 zp+VTie|-g@trJ*%8eoz9F_V3*TiJ4tMFzFUoL9%rX2MS1x}WE&Ut<6Eav>~X{ab3f zAG?Q>|>+opFhIX4YSqkPcG7K9NiKa22&OP5*o?B=6x_WHYKytd%JA80Ld!M zXBv%q$%8MD`|Q>Ti2kp9x4Zl}9mt)!-G7LwblURVPS|-+oteCL$j9-Me38^+P;r2x zzF-7Fmpp8`~;Pi@aFt zVZVzRgYba~k_vwNgOD}4lF;I;2*f?%%^T##OzCXP^jGKCLJc~#3Ql+DlVAS7{A{_< z(2kw6!Reess-@l^E0qKfUt}coZ#4W}&hV4=5)z!*;fojXbcY!l8}&KE$F!B-*aDZ5 zeB`9#_VBH*z4Srmmdxc(1C<0)b5b}~j(O^^Ps(qdb!5U`O4QUd-8t!}q~JOT9?UUj zpWg7EW14U}>JZSoGJ)eQwBDed?-m$AZ28mL{<~P?kI4&8Fq(%8hn(w~5A%znD zbVl`LSRxM@Nn1o}=32m|+ubXa@B2&p*p4>QyslZcbQk_~Em-Epr4@JI4RX68s|`rP zu1-d-MT4a4T*ZhFM^sje40Tq9ER=c&_{*18cS1uEj6a`05Ql+hAoBIELkAyvpaha( zAL40g2wr>Gn0pDI!w*89mBx!lFI!TR&rk(-6!2&y{!u&$SwK0om z2oLf5(^3OTF^!%bVC9Yz`dqykpW4;@ttp%JqmZ2keV&iKaSE%jD%2Dv5p>hHnx@j+ zN7Bla<20M7%pCnnNZYgmJi|A=qLX#mVpeErx!p>tH`_fS7olUR=Q`%{W#izo``bPZ z9Lw_ahAt7P1}g7R?fE(-Rq%ko>Z5#!1Ku<$do>Eb^exEoGBKH^4agy4fG56e)zICS z8Zk%JHn-)Ma`)@qgO4{7Us9MYS2`6SX1C%qM18ADujd+>kyr|*HcxNN^;pHO=yWv= z8GO88)DGeQHPCO4@z(iq_bm~RxU_`P^l5@-%?~DSwtbHqWiV_uLeZD{^h_5h#hHCX)IT)uY?^j!Kg_;D55P2YIza7sNA72VyS3&?-O6z; zNGleE=K0(=K_7^+>Gcz9=+b!iTKo2ikFPt;XRRoMl{ci)(7avINy|43T*U=~>Q9_g z3ZvWb$9>G?o^^kXZsx*S&R%cgZ-)^`pl0Kp?%Q&KYJoV3R51G3{VH(qJ_S_}ISD9} z*@AtiKf*Sk5*4);AjPj>noZfB$0Ib<(;Lc_a*C84W@)aGDtZHFo!YioYXrwuL}zxs z4wmU;aKJ9`6gZDtgo(Xei&h7Q#1-+P!4f6Ek%MgpyBCpSBvP1&ct>+)dpVx+c~p*sSo@ktLTJ5{tG1ei!TTIXIS2W&;;O zTiPi|Fwp_u5&e|r=p|RBvCJoAb?(m+#RUvj8({#+FCy1{UtXm4?N^S~J$1D0Xv$ni zq5qG!_l|2a+xkbN4x@sC1q4xG92-(a=~YE6AiYY9$RIT!ARv$s8=xSdgA}O>(o2Yt zARs|OVFW{mKmtStlh8s72_*#X4vaINIcLuMyPw~Cum44!XYaMvUVF7?ukX&*F|9** z`Cb}iO9p4MYQJL3mEQQ1&?v4Z--=YS);c9}yeTbc%`a+W{{Z&`D3Cb?lrMk)_Or|} z&DNy$La>EUCl^8L$X!{K@@H$$U(TI4-FaPN@4?ko*j|QEAGzmadR{&Ohdr4d+-t)e z(p}V^t>*U}jrGJ`Upcw2*_~ERj~3#d;=8>7?gH63MiQ{@kkO;y@)dCF3sB?a8C57~ z=Parv?YehJgw1Kc+Kye~oy@-NKVA`3aPDN0tRddoj-PWzw`X-o8cn~3_mkh(SN&4` zoGu|$B4@g*H>s;NfFvxTf< z-dmp)X8g9reCl?-= z)KPj`IRec+i?FVs=TkQVjIH(Gq$g)sde~M0S~9#VQ;L@8>E7GnZ^_oON=Ic`-p0*c(x6NB#h5Pn331aDz$z|}kVYa~Z%ue???b^Ub&neyH);0f$Y3X&f z;2K!iMhp!7^5-H>jbJ zN3tiq{K*p2Sj^aP`fEe7tC7ZGGB%uR%T=Hfs}totOXS*e^~pZzLan&!G9u-j68G$k z%jI8lW#Fut{U2O+XI_RYudk=(dI~$r_l9j5R0|HL{C;{bqet8CO#yWfe)P7W9Pu#x zyd0LU6|4HoL7@$C2NHUbNqoL$nE z+2^f=FL<+=nT7YYnX-f*emzofpq`8UzrMiSYXSjE>+j&QTl-aG5Z+{++6yuQg}PfX zr?1a32a2+_Lz(RZ!OWpNYXAptifP$eS^$u};)=AiLt1WTf}49YX8IdkcV+6qkInXf zr3q$Ug$J%P1fdChJ81^uML`fefV8CIz;r3AC5QGYl&&Ojxji6%?-mgk|pbX7p@WCj;9+0b<}cC z3v*9N+$0~lJYG6;^`q;)%t32d(heIgt2Z;+H?kIyR2a%473q5!Gu5&ddfV|X2*ZBk z-koNnSQQI`TL98mU%ySk;>`2W?Yd`1=mG5)MBE>^;nASUUR+ zW#bOat{1d(FWF*PUn}|Bj%PfHQ~dxk44WOmylsGo@bMj9iLoBgYT`US;=cMp? zrBTV$8xh}!?Y;tIek^7`BV{tfZWoeO;1RP|8%H9Mb43?D;`pkXp8SEsA)N!DBr0W7 zx-I}b44o>@)#;0m2RNyg8(11jhbnQGz3ddTOK`~nnSQvF0O8^`wGMjPWDuuNC&&M_ z5#Vh+v2*pd-|IJ#`T?tVnm7)iFixyPVf;2&i5{n_xNf1c zBcp$}E~dalV0>aPY+f&w)Bl4vvX?z<`r4-o4Qu+hA=5%`Ys$X_)$9n}|B4gwQ_8Uo zsEYh&U>lU&U0RM<2n&64soeJEr;(3mob2}|bocZs=gjtOKvDdTEqVLq>Te_!w}62I zmB>UV-0$V->T-P}5!{~g|7%bThgP8*RTAR=OSl-&kjbE)23JjA;N|50R7vF{Ei4Nst4xmWd|6{{4uy9@25O*@g{>yDJXPIW2b(n zzta-8J3qrMNqN~h5kc%h3wrLA*Ag=5dVq3X2c?LOTw9(5@kD5*w*8C!N`8r-(|W+4 z977)2tYZWidl9VIrSX;T!CZ9cO0zMEw7#M#9 z&>#PV!r?bQJX`aik?NT&PO7pIQ!DfG7OV2T9#R1gPD~O?WApmLf?<@LO|Lp+hCSoT z;Gl<T7)BSXH27DTvxTK~657h&!Hi%1uYzK}K99zOEoyP1OuhE6qot3RJI%6tLF#O_d=;pK zU(I|*h1rM0Fms}r?r`kvet9*yuZ@Y_`}atva!pXwudqkn3gEFH7^=DTtSAgrM(Y(_ zHEsjL*Kg!t32Uo z_mL={DBc;EcCpyYI~{ble&sT8z6Jnu1izI7hK%_e!hHR|8m^Y znp_kIUi6r9tUGL1kvw+it1g*GyNgaAQE7cmGZp6Fkus>WGBO-i+#B_BueH&Y((&=@ zVUqGzDUU&STfK;#T!ImW8aa3U?d=0JiXEjfM8WZG%mHi1bZOM=^G4SLnX}lq?=Ui6 z{bKj8m=N{JChMRVfD&l|dH%oSNq{ES3&mbWwIkjnGizan?xV3`jE>8CdlYDIV(6{| zawzVh`j+l$yn8ZngAqe?^f5;SKq06c477i2)1Ma7G>FY9m7_-P>1y~tf!^@aIUe!s zL&S$>O0QZgR#uI9@|XpwuNqex4=as7Re}ofqp_S!-{fzfN)A3b@K}PL= z_`iW@p}sR-1g)+MfEPYiOx*SS@khM4M6Jski6TtJ%@LZZTH7o^D28^9PC!dAzFGo3PMzayU) zet*8tAPxXmI0d7$b3b;He#UMfJ3Kj=h%#Tt)evqdLK&^Yas+ib63t*ZI=F65c#ZI| zzu(eH2VK`UUAh;mbd&w=(fX@b57*Zahjj z{M-H6sFDBokQeC@k1iG(0rY}xpZB}D`yRWoBluXHr9)W32K2?NK+6q03rnLb`2+Os zp7iGE6B~#TkIYCm!T7eI*4cH#{|%wSFvYtFv<1g)08LzW2zxw0e>EPLL)e`d4Lh7T z^q^Z=a~$iCWb8L$_Mw^3GYHz=3zo01w*>36f4UO=@o6<&KwcX(T8F*z*M4H~N%=bH z+BVg&th?A6xUTV0e{^mfwS>yZTVALzxGZrSRr@+IE&0hGB<@ap^!;2uCmt~$Cg2N# z24HsM<585;2vP*C&`_ncAVb5wa#-zJ#sf~{>ImXiLbY}occ@o#gSmSTNO7IwHX-XBuQDexrLN2rG)PI1b`K+E`qKM ztl{{$0j2U^p)~A7CN}y#KXhvS4L_p%YhzExFFG4|1z^HqbE1k<2p3Ld1Aha6zDQhp z4}d8Wa%uN?_G-(0KAez2^mfqxYetxRbuD&5ck$W0VKVjOgRdVTjlr{ypjkZLA|w1? z!7<)G0U%L^3)uirCB=I>fhJz0yz(Wwr@ZNH;G1jO1mdc;vNuxaHq_y8uZ|-Z{Nizr zT;@r+eb5wsfXTE0nz20~;KQ65WqkYy@H>kAaR8(7?*x3YNn`gpnR((Oc+PN}R8vm2 z0$~ad@AY&=O>?mre(!MtFcOU4VYs7-VB`lbFfqsdTqFkoKcJ0)i5@#t#eQ~xh9<(9 zSJG|N8E@x(yKjZY_X)dW%n*C2;qK_yR*qU$0PVu*ExzS4Sd(2Xq>%bk%JRo+Eeg~x zRGMDLQBSs_8~2bhZPeX7Nk=fU?TW?~7mL$taeQ6ZT^?Cq+~(}%Wz$H#%58=9b_%vL zwrS=50Jt7*ujv+u4Im59-3x<}I@jk$r&y402RnrnYd-4HP4*didz(w~7O*wGynl`5 zU&Zvn4P>;0~!(`#ykIlAkL%2vCfcEk~ z8jnkk%Q!P+xY`Xw8VDj!uuf>iqnvmEG;?BbsKr>*1ome9bi)Xz{jL2^!<=D6V7SfN zZ3K6;Jt%&@KanxkyQC*(nz^*pKB%5E#5!0X=jsX<*y0~Z`K>|iW7wn9j~B+>2H}Pj zrJT}Mi?DTM5yTUH8dRSAYt$pkM-c3vl@a>}-thHV+{d3{Zag}gX9SR_oHZFriQ1)R z%w9_Hm|0B}z?aZUkdP7*ZsF?!czEiT0KnliTr3&@MOaujxrN1U?3w>h044+;e53Jy z3t-}xRfD)8hmy6m${>~2a+d(aG($qqS?&|&!_AIoD#^_yo6XTu!o^V?*BeQWIK0Lqs{Ime=^K&)-)-jFD z400unZAYu=b@bG^g_`1oSt2}*P#6|*Fm!sT?gp}NaC)`C{}81h0%a9a8!{M~NV=F4 z$oA=;PH6nP><5#j|LU342`}jE%dfS<&s8au&e=B)41TWBc2x)t8i$nQnG=b`=6}D0kTNh^pBV?yzoE1NJ3! zCEG;9rvBGs;ZKQb;PgtG095uZ8c7n(lcr+Ga>Q)+bkFiR)zS&_IJMGex*C<0GTM~ zQQ5Yo;19wp(i|SjU#7{Ily}p5Vm+U>CfPOZnNZ#!`6TL3o-?IlqL)3utlsMsD}TZe z3LUh?;_XTUwcYpI9ZoM&TpO5B)@5ijbk^J$i#1`_gG>jSSKE}`*(ZAUw$-jmFK}l2 z_x0DgNnEw7nEq!mQW8tj%LDDBZ{Y^i6KyI_;L|6hW$y>;k%+`Bz(>}qg?k#aUyV>1 zY-(q@C~7GsI^r3XqP9{EcS2^-eS}UhgO&WxCwRb;>}&e`OmjR)bYI0@#z>aFp0mYj z{iW{dmh%Xl$H!q$i(aUxGo=aK&m9cj)^jd!ks_{I<;4mL@+SXxG_RxouxCi?CGIedQB<>F(>l}%59-XsTsY?uw$AcZH*IH3`D*^# zd_zB=A}Uxf3&-wb?loGmjkzm*;J=|1Jr3?lH_fxyO)f#0$xiWb6R(ql_GP-U?I=5# zshz45(NaC@>-6IZJGnv&828-}Z9gp;`&ngQS@NHVo4)zUd?ea&C$!2;PIB|Wkk$wd^V5=-0*4Y$aDDtOz;X_=WXpS zZyKv&^4yZGK1r8E*|oB2ByBd%@Sr~AQd0#!LuTpsa0e@EI&R?UR?K88=*C47OY{*^c0+LoLhQRTusJR*R4K;`?~8`WPenc&dHPSE3deKrF;7i{)njArWu+Dp%eb zkf3DgE?$?BpIi963FYG^llato?Il8?+sA`8>zw711-3*qk$(E@heT#eUffZS_<_Dd zSo>QY=Yd8(f@(|cWiF)@D#k&LEzEm9j}TN8_qANAM7l$tEt}~&myqN3hj{tXbGwgY z-%6;=_noJAym=^_T6D<}S_;HW5Yz?qo>$USZ|Xv_aCqm+)h|2WXQz`(I#-nvuyNQf zTn<}vFvzi>?SQB&C+teamF{CzfuAr3?0Vx2Bn-e0w60f`J>}UlXU_C_ejC9eCwL_! zE2Z#;cpdH~xPP>oW5CYpP7)O@(q^--ce-S$6*gF3^B#NukR$CNa@AHFo|CU8ExgOj zAhEXYF6PVWop%tbiT*JN@zJLa$v^HIIlZ&`7KcckJMKf)WF;c~nL&vu-GMAZw-O;wrC5?yM5N%3wal2&!JkYBx%p^)%Yu6?Ae&P($hQ1!4MZJ@!_}&5ZQpcDrz!r2~KC zI~=(Ky7rYXE=?;sdAZ9a;?>9r54_$YFXUZFKkcq~OqJVaNZxPt)z z)xyE5#lDHotw>*Xb!pfHs(#g3j((gj4|&I!TYeD=)mAwOti{l&HDR)4G>q0*{!>MNr$ukLsE58Vpn2j!?doJ#YD?xK9v_~% z)-g2@>Vb)v>CtDKvd!5-klx=~Bf$>1E2Qx?e0M!z+3b>9lUT;;lts9IhX^!GNyw7m~nN0nrZZ`|F+LcPk zi(ipmyiO=6{1&78qR*h_a^~v$skG2SOfHuKOSUz8Kcx3V>;3O27&d$7c6aRHp%h#z zVFx~Gh$*FeHxuXB*X?ax_xQZCg?3;1;fJiIcM9QUTn&E_^lekAiY=#i3NHkW&%ME6_Dxo6KdRgG%=xaw(%sd;TM-H37G(A~pClC2@0SMHppdt!*ch8n-gx zFBD!<)#)!&#Twm`Y7jK^Hux)Tb!y5!bQ}YqLap2Au5=%I5X5GS#v0})IFHrj6X6Tn zpd)6Z1+-^BX8J#HTMZDv)0$FVzB4|QD|ym;N&On~qIU&9-o-hKd|%AGW1?$77d6|? z)n`bEYv$GvMQYaeo06uN@@;>XOxv?(N?NiaWF*otK$pYTaNKPr`qKqxi;lF8YReA) z7{_?25Kc#+Sq@2&$SQP$>ai7x3uCiSy)*OYmldI{J#|Z&snOG+BU3BYxwZj|_$wb& z>r$p=SQW64ole_$v6|&Xs>uSUGj*9;db7xlb@t`GqaxPC;VtzD7k2+clN4a ze$M+;Dc&&)(|qA@WTE0baL8F|*#SjaG|}4JQOrfsB_UXs8>uJ@Aw=)GMJ~#6J}(61 zg3dd4HRSVBCprs}Op0w=?OE&7T$;)xrdtzi9-wFjP4{E@AgX$SrJiROd5ilF$v@tO z(}wcOBHpLd(7sYa~p0?~z$clHw^$A_2lN+j{F ztW30q`>mE*Yc-|J^pKhAg1ppb^4-FdRW~GQ8cLU+_{pM*&~*J6E%=ofV2P*Ym$HVU^Q%I>fMWlu5!~ zmLAM4bR%74Urv~6a|0_g^2f?-zPWO1sGO128l+y=jjJR0(TeW%6bN=+@XxgMrAgm0 zzd@^BcXd}y{I+@SgbGFV6GlULKQ;cCgt6RCH+FEemuA}SjEqiyDT9|KPp<8=I;&HF*SMiqrR2t;_}(#oOgk=c*Mym+RlxLGB)~Rx}dA?=R5_$>{Jj(?jVHjS6q260xhUD&*I)p$F-C zZpaI4M$)v06S{|EX%pS~yP|pNC(a=teS_&74f)k7cF~p)2p{%5TCbetkXx z$mUyr2oJ=Ed{f7ddKED%exTyXlU#lih60c~s3ssKq>k(c`XN_ZzvLJ3*E$=PA ztmpi}`Gh*N8f0SOXpj?gW&SPy?)p8r&F|iJK4}DjV%E%Y-l!~-%R)RR`;2lDB5X}( za-#0NmNbd3SFsfA*1Mb092sVnw@WZr=r6)MW?x0c)|y}Y!&*Snq&P9D6FzO2ixPeI z)eI{Bu+BnI5RGV#vP9Bn{MlF8c5E?7@4HsyH=FuY63&TSW5JiSp_marOJ1lyBRIsq z#ydP{-;osE@F3mcBA9T^L{H^q!Nsd0qe=Vil#>e5{U?2&ShWB8&?q^UZQzS_#4nvR-`N>OQ^r^*|-FNMbFB zq`yU#kLJBP8aF*oTl*5}y=`7*lKos@pv zG(6(QCp9Hlcze$sPzjFl+O>7xGvx}g*{dOd?48jbj2F(#xC~C2%AKNDHPdt8x)i0CQG!g{f*$MIM6ql*1}U;9sjS`ES@& zqTB`!)e}am3iL;|K@XXl?B!7}l!Qdx%aXJ()7xYrXi?B<(h_+e*ZE{JXsXIZIBO|t zeYsCaUTHYT$J+&%H(gpP?hKI5uQ2I~OdXp*>qG0L);0M8$>CpAK~OAI3c_{*x^N!& z5VkL;KTjn|$CpH{$FGgr-zM0XIE`oAv+hZ$x2897T|PaG=Z?~AhSPeOCZnkKg9MJq zBO0o0dQl80cE_T+?;x*KXddc3D@AjqjQF?lN2SIQv4NEYD)u z?hVF?+P<1!k{CQcFkdZOrp)V_J*#V3GXxZ(3gmNnadZyiO4f^Gya5)b5J4V2)BIP< z-2`p8E<}jOGH8twssfPdg3|dgSziowoo%H1R{YICjRqIu7c;T>s(Zk!Ir z<8OX$iCz#@i4YVFQyutI0rZ+Ocp`uUn{LAqB@7uYvxStV7)$L(Kt0tWQzX`$79&Id0#oWrAZE|=kdK$ zYdv6BYVeY8Rc1_mF}PnX*=}%OhppeCcx2_%!b)ZhN)+R}?4zDJh)FKB6VkGl;3rO) zkMTFpGxIjRMDUC)zIdX+y9pA>U07~N0J$dqU2yxxM*DWjb_>rYyiMLgoQIA(auX!O zmDcnpW6t#i`<~cx{RV5G!y_P(52#R~UHjJlgpJ-I? z?ik~ufHhj7O9p0ODW&L^T~3_;aM5P1d*>h!qd!BqJHcAtMeET$dvF)%e7z?kP(*p6 z4&)AFQnP=B=S2g`7AHPAPM|@<=UXJUe$ygFpKhhZQK>CjNkxV6vnzefL5ZfKmWfy(pKNSy zw7psrj>38=URblLbAzjUdn_>*mDf~-xqiIMHB;yhq3ySxTAQbSB0(!PR3f>)C%GaE zeY*EVIEpAhlz`kx=D@(sVZ6o3JjWtu=&0Kh*N2NR6c3L@4_KIyPH)mZEQqJ6UY}?u zZg+*y7ORJPPwm6mK1r$3Jp~9qC|1q(%NMjFre|wEV@=Elja(q96SsD#dK#19ltl9} zjO~oOm(FrKk$Q>NF+yIRA++(dgs~~To-Y2q4^UEg*?qWN=R6BV=yEa$mhUiMw{6jS z9sL<{CrooaH6t^Y{hhv)M%=TQSa8l#mMV^}TPr86XK8gO)~$K@yRW-s(mIy?-(i@5`T*c0wcTiA1v*ua~!$1Dc597dGW# zXxM;majn=yrcg}B1oLje;>_C{T2+YY%HZlNrC#E)cD9wvdLsyScQZZ#`&ZH$h~5ur z&nzOZj8i+ehB*b54)vNmOqr0d^`_*|i|IQd`OdVq4TZesx6QnPU`95<@5dF|W<;Jt zv+I0ky>f*mk1(7D%v>sPDL?i26a@`3nS<$t&PY;s^SSdpWgTMgR=u4{tYKC3gY?P@ zS<(2!m<~qA1eV|-SReK^BT{8;AvHt5AzQ6WnbgRe3|%P7B;)p-J_;9GCJrC1Cvs{$vYiJs$6K<);vDf=nO>>aJ{UeMGjUX{qH5X&W)%X?@@o`+}u zgw>npH4_AswG)P@-*k?cgry`hb8c2t7<(D%1gWE@&BvJTY}=aftj;|+;v81MtJUrC zDs1Ap20VsnIi~)vNgE{6mos);#dwf&AeFCfQJ^jSA+1obY!A+Rq8_z~ai5ghHb9-yqMcJ(4Lu!?~`{b{O&s&GvC9I5&1=B||28fS9^ zPqCkvO-hGl!yG*1nRf0D`#eH;^XY2*B@H^QG3HBz93(qTXDBI?VKO|?e$ZX#PeHDm zqc87Dp(Oj&uk_TWp*dsk9l~v`6Dk#U&~DtXovhBlF4h*v6=z!Sx(q?Z4TaC7BB61F83qZ1gt*WcFZP|(nN$(JmmN1LzJpl9cjAq@P zj$UTMJIVe+u)T<_QjFhf_PS6(5w_&czgDhNDupeSI zU_YW3eGY%CxEXc4#4XK-K{v0r?y(2PB!}+4A zyhb^$juV+#C*pn9)7DkEnP5f~C(1$YEcKHUzlrXWD9#o{^TebgP=KWFfM%op6Y5U; zKbq-@L{`m_#t$!O9*o^zC^@Rwv8Z^P)d{g0F<}2ppk>AnUbRoT=)wv6+&)r)xUyLI zngUyMzv^kwG)e{mUW_rR^R;V4j4T|H`WsCaXIew>uG+)v$MnT&}vPibLXZ=wJ+ zJ4{*vcY$M&R9m+R@@|a(s8SMyW@zUB0g*rvboS~xN)hmc3u4(XH7*jo8l240{9YIXnm*5AY9)$CH5 zW29r8qcG&!+9Z77TU7|u*Lp#U34wg+{bMbAW>jEmq{;IX8J?_Ry|WS{42B|N9ZPIL}my;UKj`B z{$y(&IQpzJftS=XvT_sebsYpDqeJO#E}vySf3AxW=`_-Jf%<5TH9kmwh)C=$Vs3#f z-=NXIY4UG}8myzT68~PiRJX5M>qE73i+t_17Weqh7XcsPvitEN&pkl zzx4LMR!IYDSc|%#_`~U>cBq8Wrlv0$y1z~Angdkcpt1fdl|Kc>P*;f0gcdNky-nJC=O2Oq=47OS1P49#slG=vEi7t|9&j%k%FV24J&Ssbpz^Z(+r9a78(JlHW%tM*%43;E?hJDjb< zp$sx!NRBTWOuN@sGR3z7SMHp-eLd4j-hwm$mL9yVs5T$|RATvgkdJQ4^vW=qq$M1c z6MTGSKt+vMRWB|AuJ5SLRQ>?z4PU1WDzfw;6A^Hr8%`2fNx~aI8Id(KUBFht zP(z6P1fWqF0>f0Ib9y~xr)5)QqDO{#MV+A0G*d&RWLGz3KyLy=igm0si)@Z#X4|GB@vsO2DH z>c)s$V={u?jRzgBV?-i-7FHyq#!>U)81ijIe_ZYTFQ7#0)$bZE&JFCS@5p-~Ho|~k z!;qSjStw}?`Hm%my9{`8Xitr@d>C`9viy>Y#L(M$Owe@3X+2Z6bnU{tcsHnMU9MH3 zBMKK}ph`O+kjIFy(cQoTU*lhq-E=455S*v|6@l+?=3+U@J8aZ-sFp8igN~k1K)}1h zK-JRJR}&%cs)261buz+tSFItnUau?9y5dnOAg5&fZapX(aYj&2)-GgvOe=pOYbZX% zXEOwHo-4Av*t(;TGC8h#&x-?YfcfYCxMSY*Cs*9ZAdt_GfU4dc|8qsRb6o|{4z37{ zK9-Xg!A4KTu_HRkzvgqAt+oBJM<8FGaR~$K51zbcv_9-yUy05eHrurD_|WWOR|AkI z#iUQHm-8=t|5#QHlsrqN)^Fx^ARIzMmD}K8PA%@W+&7*Zu(Nuzl2*iRbYPbkIK#XRrRWBpj8t>5Tnd=#=p=0tY+iY`F^#! ze|yz(Y-84mlYbbMe<3_!h|}zVf|T()uLk|V{fzYetS*5I(m}PG--Y~}SZ)N>`4OZc z#ap?%`TJJ_lBV7+O+q1elz^XZf581X0C|_Tu_S4GxJvik@_;{`;A+kP%44dA))LEe z)VU^dRn_e#!&JV?w_c3j4mehX8k-90nOazCEmZ0vb>}P3BhBY4c~pN4+_COXAOA_? z-M#+waq914KEzt2WK@=l@$0_p`-kL^E9o^0yn5WBvgRwdjXn}f{hK;|dE@YQB=>(FX<+X4|M-x2 za6@$9v=ESIl=>yG2LE}!e)fZPs;{0gZakR8{e=2G$&-k4Tp35oyg2S0Yry?(`Q1`@ zv$S6VhVymzwZ|(o9sO4&KBy9!xi0xbe50R|M}Fm2b-xp7YOQey>bTF^Cwy0C?P0s| zxI~TI>ctE~Bnm(NC}^eSrWzR7T=E~L9jldF`{a|UHG-!uQ;sWJuTc$X`tMyT1{8X7 zG|2FVi8-TTB{M-i3hzJ7EiZx7X3&r#v1%Wy1BX#j*$&NI?jvEH{h6bNWHC9Fi(0HA9g)(mLghJj^fI{ncn`-37 zbo`K~$7110MbYv$v)dlqpeLfTT7~~%vF07wJD@1=Ikz>@W8rhH#15K;nE&GPXXpTe zCFA{yPH9TCU;ZMWalpXT0;f;=dCjr2&cAcPwe6pY)@~r6OSM+NQ0ryo`DhmD=N3aa z%@kPj$OQNXpsv#E7W&FL6TV=ae!w$e$ru^r&wS6uGEyg!#7UZLW?P$RpU0ZyPXzy2 z+g?lFYCmeoPGin6QY+OrelW4Bb9EPtmak(SmJYH!s29nLy;F=~Iu=X{>-80i>AyIJ z8STT+ug?>2J+8o&=68;GJp@N*m@D1X7HqmsZiQi&Zw-etP{&Er`Y$5kg(AvwpJWTx zFPi8(N7k2_Z-aW?J7~UF)q~_=)K-7kC-zBeWLOr`Q%mK6kHC=ij-eHW>umw~w)Z>D zo_9L*x}R`U3>oZ5wmHD>jHxKKE+iE{4-T7nCT^MB+M$w&c3w0Ck zV2Opug~`=MxP9)-p}B3EH%a!H_8nY0z$82?@f@vfHv4U%XZ;M&=f73r@3r?c^P`!nVz^ zZ79ZO3XeWJ1V_B|(x9HJt*~ukn|;asf>M^02r22+p~U`H;PO5I zFIK!yl{nF%EEpa=)>k1jDqU+OVMeY0GAQ=Ct)yli=_OKFa5szR9VTc98*AM1c&AVH zYXpdsWIX?~5waElWo1FW(pS_{UFZXpe783)qUI;O$*3a6qrn=SZj`W-b$Y&5O$IW# zXO4z+O#Lh#vbq}E|FR~YBlg<8-3?`6OFr#xxKq&3aHd#PeP2ZLd4#h;n_k4wR`bf9 zL;q+{QR4O1HGD=MQ>s@yKLP7x@0@ThC2?2tv-(4PPi!}TE(ki*o5+(|MNp?!5uVW- zQk+|+r(f1cb!j-zjrv|{+=Zoe50rS2KKiK068w^;OboXRa+IT&vs3(+o2|CFFD#pBbA<@=O#i6<<1o2dLy*(V38vpbV|W&I;F-244o z!rvNb39y%kh{3WJvf&*yGwp+iipqMQ-Luo6da|lK4pba#`*hE%1ldg3X$L5gy}}}0 z&(HlK-#k$^K@6F*kmS*fRkzrrwcpC`H*5n zm!A=OL|*+E{C|*>5Kijm?0e!fa`6#3>_B{P-%K%f1!uIy1iCLQxH{F0?w(nXk*r`G zLY-67C_O)6C6c#Bba>M-*c%pG(bk-3Tv@)^@)J*GNNX;0&FflFO-d)$tK`Jp-}^?I zDh?J&g_fSsF!CZN6Y0M+^|dpezb@go{72U+_A|W4;nAuQrjsz&D>&>)O0H;(a8;r# z1PI<*J|!wVSX*dT!&AlE->I;#)b`1-eT_?H;45L8A?;*GUq`q6f5J z%K2P=#WpCZ@s*OY<^d_5YudNdrDZJoZk#QQ&2f92P;VK^59}@sFP4YBGLPI*reWeg zfh|+v4K!fzCxqY6oyYuUn$r@uTg;lx54CH~^o(HDz%pObvcsOUIj?{DN6oqWtr0<_H;MpU>;}ascTsg>dwH$VuPRMw3 zeCk#?cratBZgm8ohYKW7r|RHs)Y{BY64hFJb zDgU6&CnY3`r7Sq@!af?p#jD^z5n7hRR?%LPP^l#QGq6zSwYlLoSW?eH*-1oaC|w%P ztgE%*m6iF)bGvfl4yaJZSk;vigTcc1k`xVik7nndSFzi@vR`ZLa2b_$q)G0D#W`F` z@gfjsEL@t31q@(pewegf?{p~NFLrfOHd*ibwkDfbE<8Kap%a&{Q>bv4-uE+{!Vq2l z_t4OZ_k7ni%a;Z9U1nv4;8Nx?+XfK9613~B`D~ByK80U3G2@l#vONW7zVEcjd2(ZF z2IHcgzzTmp$3$TrpNsz9oA?OZq^yXX!TCe?d*#5)_uiS>Tu+O=s>LD|-r7;*b(M{` zK_1*G>q$xaME566g~WDvmdM%>`p$HG&2Y8dCq;Ph#k`26^)hUK$j*OoEFXy{b|JoY z-dFjdh}VSfd8g}I&y-aU;Ya!X#l2;~$YO2*hyQtMG2X1~rAc!m_J`XB5hw$~z;F?^ zvS9mC;HHrW8oOZp?)^Tv3Nb;8-U#2-O6$A_u{m>2McAU+))WLKsi!W8T{Xvqk@;MM zjtFohb_YTYG!N}a6O|kfxJ+MjzdkwGOC*HEsvCE&X7XfdBxiTN{f`^y_h*!QmTT<7 zeWe%P7ba@_@`p%&1^)O|A~_fXx#)3UNN9Ls>Gd^Fx6mT5weH6;&5;wbLDG4YE%st6 z;OrlEaI&>C$D9dsQslV^P{!Gk4j2991Qg3QnH0)98{% zXm@N(q4F~;_b?6Z$f-aq0@N)2Jv8{;2GqSxEYG@&783^xd9A_Hx42#9Dc9Osq{P)k z;Gt}7j^?~LH~dt<@u+B$bUlGVxnE}X5LSm&SN_5L>NO>$hjWMe+r^@^y}WAuUtrG{p47*`-e;umzP9w5b^_C~m~z;pf^JSkk~jl1NdAY-juY*(C3A-Y z!MXET7m!3={kL3Ft8k|k(+9J3tp~Yt2K|jLA>6XYeml$%{?cNIGyPnibdt|DNufJS zWu><+zIb%+R9zfxaU3e4KE9Oa@140&D^bkvPLt#d0T?1`%FLCI89w;ewvCOI5%K%P z8cdKWB1q-jJY{Wuk5r| zq^#9DyI&Uddf~x&j+AVs_}&y1ha{GR;Y{IcnDKvF-|KRLc?AMiLMi+_MZD1MR!!Up z1Y@yS-3CrhBsEGsFx;XLo-~&J$n?47=khX-$DN+-j829@gHq@a#))W8snf0r7+jT( zj7-F!P9&mH9c>+cX`inKx?;#k_0?Mk`NKJd;9Q(OBd4hnYJK9XVf2hpU)R`bzFf#FS!-`Q+&MRph3*J{>Sq56G=gwjYGO<^$a^LlRsHGFy3^)% zYcDe|zG7AN#KsfRm=W9V)~gz&mM!-Fr~@N-v^;;U4vZ5SCa#8Qn>N|_tM_9fxQ$q) z@wq4TU`S@z^n^CLL{}&7x5_Zqw3p{-)z!6j)dDd)jZ!^&8fK*NA+`LP<)$$?o7x3f zDs*YOja*WA4YVJC_1{x0=Q^r7BaAa}na<1ikzjU9tg)srnw*ay9V030l14u*ybW6r zoe#-UkkTRaT$E4^{H@p_dX`+R<2D^tnViCTh9QO&GdB(%$ zjIMI>3XyftEClI3@R7#-ppmU0mD(jGnq4SHFzZO$t>J2mF9^gP?TzuJ5`~o*eQ4tK zlW>esp3tMlf?SVu*VusT;{FG>_srfVvO92=}j za_)K~oij=*tw})4NcY8*Sj?QWtyGFZ4o(ps(^$~$+V}t!mXYGI-$2%`ge_O7MisHc zdP6(j=srUl?ZZjg%f>K=i^OS-IZA={26~JFvq0Px&%Xp=aym{7B^x#D>77%BO=Q_B z`_K={I-`MJHXT}UM>Uywr=F+hfbV^>F$_9Tx@XjR{gEuzKIMyeAR+b6~p}QF6<)82TgdQ2LvFIcl|?Z}Mxy z@usKS4o)<{sYHp$0hF!JGID0}E-ZdN>vDBTXVWo)RhJ`BQgN4DWb~;j$PM9xG5W#V=zIV^^Ei<9f4IgjS=|sT5?+o z<}nRe0R+$tS&V?CyEX^Z-*V7WU83OZM8|+B=5bCU>QCOQ4JSP#g-UbV`s0{T4SP{* z4-`;HK=c5))IO9iHTy<}T5sgiXN-;q742O9adbur)}~c8a8SvspQz^1 zT;?{!s9tXB1ubfJ%CI(krX4Oq)=uLhU3IeUAp7eRHikBGVqu_@o}jJV@#eQaUs=DF zTzb`#e+%I?#gkgF|2`+hGxMMy*`u~L0`;q-;^luEbYYDJ1bPm{aNeYWur|C?Sz(Q3 zk1LcX?Z+Z)#x2Ct6oo$2OT_i*_?IbuXoQD*EjW86yVl(_6j&-=J66*`@sO_6>Rug{ zCD+-t<&<|al=p$ds##0m$eRR>1bP6QWnJl9&1EGNQODVw~jo0d^#N*s;-+vuDqm<|3WyIC0 z`%85HHTmJb*a9~@3O17nUrdR&)3(%*`_&E`*h=)(^`Jg?)=DO5FRYox(%%4idXp%` z!x(2f>Xkp*?Jl=ew0c zal=KIVG;TEwXT@S~GKEgxF z4mzj_@923JO*c1~5ZjB>=Jab|hh0i@BT!YH48)4qnqQ?jBA~1pW%tqxuiRrpitGT5 z=9Z^jEvhKRgY|?7YdN2WgHUklth)+L1r0qzCt2Jx4m&4iT>QBWeT^pU5+cHSecx+| zn}_)1OdqP<`Y{p}0xd9xDin9tF<19;Y65Fowd4v1UV9WR?NFFkDY4@2-mMHGZM72U z?X^YG0)#1MB%=GXvvX(t>Uk2m`Bj`@se?Y^I>$W+UL)ZOJ4R=4o8QcEgK_0pwXf8c z5?#73FdMBF^fze(uay`^4wWj3NzAE7fNtEvfQ3EA1iROvfL*05DeE*6 zseACx?Jm-nY2r!ocGw$31ry4$6Z-27bkgzK%2k0UnEMaJC(=kOBA%z+Gf|fmjlO>o zzwFs~=|9dXjJf=GlIxaI^qXuqIB;HyKGp%RTJj`lEvWKjEh8*yoO^IT+JIwTKUvll zg~-_iK(`d5cE;Z8AqEgcQd*X~zKMd4ICV>*7OPX@c13Y0E_#^*7HQa1y;n+fZ_L6- zu9c#zx0n*RHk9fKdMGI#qiVK`qL&-ee|=fw#L=vJt>FHx&2mhn&b|PVZ|uTxr!?xd zoFB=W8_{?6AIWm*MC;}(Se>W+u$GDnOe)f;OeM}>SZckoEtcL_F&Ep&=4ceXsj22==S`R>6yA2{Wxe$2L!2sV#>O_H1h-W3c~^|g0*!POwV$fL%q zb_I{uDwATP&pQ{FSUgS9CQD$P)H<-O^!xdSS<3NtjjuRWv%MKCN@Fz9APAS+LZ!ZD zRmiL05*^y(2UeaEnb^lQ^Pl&UxIxTRY)>c|%aOI@vnrX4wFKK!hcoU#GO=u_{IO_E zB-Fkz%?$aMfX1AW!=zQQ1cwKYmo40pibH@*96o7TpF7Vwql1-(1q13U&SdS1sT0FR zfLjUAHC6&x9!0Xr5~YihrELkuyC7@bfsJLIRcs}8DHjc#vT{dyzs2N%#_3*Rj9tz* zn!~P8sLgn2`BaKRp<5b|dmOW6IdHA=#ILKIJ)Y8_`Lb={BFx69q9hx!D<(c4-y-YU z?b=kRWz=Ms>I0o@C8Q`|BB;$w0ollVE z3co0QUe`v-Z#ChkYWZbm|M3!2>zX+Vr>tk}80SGclxG{Z@oX?7KyS!-(23Y6<25&- zWMP>0?}^d**YNKw)uBg~67}B6jG5iZ94bRvc zPf9%TGMLl{yWn+(9@NOIp^bhJTDpLf=00L;*|ZC6{Ke?rGRjg9yZ*rG({;{&`71c5`rDr~oW|EJp*7NCGV%?YEMg`B3!SCu zyn;WvDVIFel;pVzpO%|#BaKLi?8C{%6x1z+o)EE>gcT+(p*0K!j@oU&1l3$|Sjz|y zUk$Clu$#Xq9=EMCn|1rCmG2e=Dz>hI7i(}IR)DdF0W zOgq(;xY*NIy3jJS6AUXj1TnJbwb@FrrPfgt^d`Nw+>-DHPKjx0z8=4G57#g?ahT#) z*nVDhFJ*Y}YW0YNIWo|A&5^093h+51YF9UzkqTPRH?Ab8zszilN>uuaQhXY}AzckQ zW0-E%sjir0t!vMjN#O_hU`q*&zG<~|=pw_o2>H6EvdUta&tL4v+o+UJ6jV98%Oz|X z;YGRKVvWem0L@Y*z80#X#75PB;e^-8KOg{q_;xSwm`mTI>tcjC{uun?r5X;Ac%L*} z6_Pp8^FFTCu{M^cm(-TFsPCB_l{yogJM+a!Xr7vr7GE~PJf-#hf2ndoJbkArZ6h@bmw-rAcqB^+YR4jpT(Ap&6dsWfw5&uVVMpNgDN>OnW4kI5PK8{iJ;ttZTUcSEGS7K%o?>qdo3iUhxHI8F-6l7DVFA@wkTDyN9;7-GvX!6d zeuKf^>~T!r%o&XrQ|3P1;O)ZEZ9axkn2-(12r$u4C;+%xkKusuXH=0+0hqPDjU|#*UqId5a{7LHa5KL%!P_Jaym49=++` z1wO)!*F{`#Eh#846}`_f?;Um~WPtlJ!%qjxJ#X+Q7L)0y zzBTNqc~o~HSNk|9Ow?azf~{m=%E`8?N^#X>tGDI{{e`m`EWxCBNtO}>#H_v0ZVg@+4stw{l~wnYCANtP68(dSdm&5+uc0`jDV8b?zFA<6p6iE@YI z#tz-NaX&97d@zeedujzc7gbFQtVkn~qguZee6W9^V)6XM*?*&B;fe@&u%p%sQcYlH zX=o0t009gT@j&uw@QqRsWn()){ht)WJG7#h=8-~f(7^7W|Mo(q35_7*m&!{YIZ zzA%gq-E%AvoVtZHVhrVSk>tkglN-3nsmeXD&+^aIMH~fVPYkRtm(R*DiQSy6%KEu# zl;vJ7+=Va9I><34mh$SGOF*K8Sk-u@Sl^Dt%8J>|?96X?W{O#V3ZKddT+7_y*ea%3 zI)EU@dZ*p&VkTnM=ppP}Q0x#PIxDG$dKmdgnuTXUC)OklMpVU6xHtAqWe|gOrHz3h zbM8sTDs1zOF`SE=-6=etfB1pR!ix%L~V%FkFKg`ZO$o>e;qWI5o$ z7(SEz^oAOV*ZP{Ok%L>2^Lq$9NrQQLi;~aKxg1 z9rCRm}%=f=uG)5H3?4H zr({W9tqd`~m&;vWVUv+knVIE~{D#5_F{#QC6i#R*JMzrE2s~C}-O@OFH^*v$r+jG~ z#i}!J)MT;5>U#$iIn)Q%O7%4Zrw7#0hAou2 zMg3cZ;W=5v=jyd+1CkRom3-?i1+6_A;?(1sHFRR3*n^L|GsOLyDCU!Z@Tpy_&N z!|9gs1>ab0C!ln@@)Un+ZO}KtO4IjA!m)4FVRJ1VNjE!R`98Y!wY>!WH+23G8+y9e}hJcCb0D`q9F%>=pte?{p0+s z_u#kwE+gfj?UFQAj-#Ea+Ws(1>2C_3g`CFql)_PA@~(xXQJ~qASi)>4zj1@snGkkf z8`p_W=Y?r*Yx4o%Wg?#~Vlt9}@g^u83`hqZ2ErkK;Cb*vq;$-ie<#<5WE|#!*vw48 z>Y?%zG0F3`exJ*!YI>ip>k==`@L_#L1G$j7o(n5_3qAY29czzFjv~}AgG2kpT=qyt zrd{m2AS-}$lkV}eJCU~klCk8ta5w|DHZsSL8|og%B&dOg-cj^(HWHiTeY!gSZy!Ws zgshx}I)DhtSV90a3Kf{WW%upeneOA5Te_a%z{-cmN#5C9G#3*orRmU>R<~64o~CsF z6#s=efNG$MJ_SZSnhTNaOI~~t^CCD(ryX;L!~buYf!PWQ(k1_bGwGf&l1Mnof#dm-e1L7BsmhtZkHhq`VtZl~pr zk=_d8DMczytwz-k%i?@dcz*9QJBQJ#P;OOE@bM2kzjr{QhZ`Tagp-2?A%g!fQri10 zyxk7yBU)a_``Ao3oowB2HL$WC2Hf9Uo+y;UmkT?NnP_Jl2j5eS6(IeD`jGufhgrvo zTD0X{hWQzw2^{ulJ9J)qu?mqoXz{nwP%y_Zj}Ju}ayIw@WVbwG)HN=c)KimQi;SPe zQ(xC61HdM3>Vxcp4-$;m>EpgPqTPM)n(;~g%nMtaI%F+6 zBoqp}X;5ltr&p2UjCY!YvN4THe91acUK&cxGLzfJM=4tDBTbf?+5D)781EFJxMvsl zB;*4yh{c2C7Y5sA>07MlN^hMy#5mnDDmx_QmvdLR8C~i!IW^<4WefWyBeC}GE7o-3 zEeix{y7W1Wo9Hi$H4+oltIwC9k7G1ZL@=3R;=^#pwcEy5jE{OCeb@$rbnkZY$9UI@ zPA3-L1#0IK#d%pf>GVk@*?%FbMq4{tH~o|%@-83&7jJw77VjU@=PxF>#;*~pVh!iD zS9@jg5c8lZqa!JMksi7;xj&4_UFk_~{}Oe{!A|-DNI0Myahuj&06qb-=GjrCiyB1i z#3hg%xSQ0jI*ugBzja~UU~2OIE&8nn8N?#s36xEMYrIm^%3Y~VHwA4R-FjVXG8$5l zsS+CdCf_plU@1@^zV@7V5)Lusl4F>JhfVK6 zA4Cyfo0@M0?OIOr-#r8IJ9249t#mz^`N#Gg!=P^MhQh(07t?*+W@LS0ye9Mm#7ZGg zXe;}2nwQu=ts|t1*@KVxNtMs+(^I(;7XLIpMkWd8J~tJ7kg6v)W#SC~ASknV zCrmFIpHP)tXz9Ijw+?D(5@VS}S`^!cDf0C6%(&{ydu^Nw7>Cw&UqDqoMLsL-sim%G zWU29OW7OjXuhuAIIGC&6u2)eRIhf@&CXtlK4QbL-pWi!WZBCK5m&+Y5@1xX`)ZS{W z8mIQ-9W|J5y8#80$J?3{aM~6c!oI?s23tan$K5PHTzM;npW;z;b(ogqCTW}%=8=A-=rKW3WyKOxWJ?L((w>Y#$ zcgeyny(WiZsnT4=Gw7f?c}MzR4zOF#uGhLuv_f5?_%M}7R4*w)9eB5|LO9i)Yvs}X z712w6tL#qZvRmgZz`$oAb6zv`Oc)oO8R!PZw@MPnG+)eh?wT9v{r>Gb)O zm}m|)_e#UFnW)MeBu}l8Kf?L)dWm6dXDn-mC|c94j&A)vi|EWdlW1VDRsE0Xa^t#^ z#~7>Vqj{qX-t;R$2hg@<|FsF$HIOvgInZ`F+=!Tar~=U^yELuv2tf#if3n)4Vp`!v zdOBBh?I_(uYSE1pFguuq=UMNVBXw7ga+6%Do0gNmGU z!85vc{bdHon0b5AT{qR(}mUDicNmWwStCG1)UhwLF#7=V$$V zX`&vg`|{-*FS`aTFg=iv^Uolpa!p`@^Jg#vqfp%^D1N;YVvrl63aPoGnMWTbY{Y{_ z(2h^)RH`B2ZRU*p{zKr8YLk4V7c;{@=AWkMc`AJ9^}bB@L-u0~RUG~wrpoM~*AKAb zx=)ve#~W6MC*B?cmYC>y== zM+3+Z7zsTPFShQJe_NCVNEx;~__r_Adc_~!s72fV^|?7}ZEX#60mcVaS8gFSK?pWv zKv>@KB03|E0P0@fpnKBIKW3wuk`x3?Um*$&vOjwHNM3b~PBfh$Lyb7y1TlG&&u))F#QXksix(K=+_^EWFW zNHWKt??)ccE=`y3%S=6T&(RO%opCuhsnU${RIk?0t$aUsZ1liRV>48D`-MT!c_{kM z9uF$6<{dHyXLtR8eYnbwfzIEou?@)n&1*(z$pZ`Ry6cs3Q{$^^u+ZV!0{3Nq$zmRg zV-l)%n2tx@&)|GK3bL*iILV^d%m0s&wAsE2Z#mV@BbCOEmqzB6IOseuYBz}0{cPsT8-Pe<*j`lm{{8g|j@?I9pW7ggozeN*jKzZZV&kFM1e*~d_V0>zG z?Mm?c?>pPLgIFK-a%}4pkX<4FviwLQvYyEyRRe0kvlv1vk>#v(A+m(&SEamSNVI)gA-x)-_e)N6n2b)g&MQ`6GfFF zrB_}nw95y0bjM!T=mVS{mic9MnV1?cWip4JXW5;M)n$<0G;mCe?)>a$r;P0lZzPFh zLQN@}gGYD=!aE`hVK$xK+Jw7JmTRGoxNP@XT9w#SZW<)?y!o~!!p%V|&E=*Hj_yX|exxJV7)<+GtZRuKHD2+0c^P=i zYdL?_c%d$crp`>)O)B@pzIe2hEU2TFc8MS`Qs*Zr0klYDjfu5E6W%MAr^YNLfQMxX!uO5?o2JDJ?*BV^J#qX|n+@qBqE9FVHok=nlDR;0pKHKw3=*6P_b5SV$ z)!V=MYPaP+XGajSK9Y$3q4FeRtbdz7tuIFP-#=Nje#|Tc>PDm}edyR#`s@sW9~zZ)kXDK<~hv#xZDJPs0jl%dikrh3e#e?Fx11_B4mb z48s=NB^yBnV}j$Q7`#Ov&p7u?RYD%s50CCqT%>OGdu?J36?f1vW>?Qt%}bW(qocF7 zkrIt`GL3X9(I*Q%-%^vF`zOv5j@NY;im(u8x!Xp1!I|sl%?QU|q2Cv4%I`JVI=$v4 z!i-8xjJ_qM>YL*}7wNbY96vOk=JKHEccU^~JgEH)5tsCIbJZ(PMdVc2C_Lcey19ZuxYV)f=snvz9)&5mpvadt6!!-;$Q zlITrm$cV;zqsW5uZ`Sf>n(3=k^?zEtiA83upGs7&T_5+xW=t$HVTEujMtO14N3hp! zy1+8kyVk%QRcB|n|HW^U^ zKN;<|mD;HTlH}5S?S7@!5!$S*8V-4t5b4c6O69hw6(%i~Qi_DMpu ziYKEfStp%&vr$7eHG8y2lE|XY;rWnHf7{8x`0-P5vD0=*;03o5`bZuV+e;~hhnA6wxVo##4nxW}mUAk9Ps<{>4QZa&WFK%76pbO1AnfQ@`iS0gk z8!Qs1_NHo(y{TwqAL$Sob*s3DP5U8FgbB*f zykm(UK$QjB85ecH8O#8jpYdZKU*T=wlBAXC$Oh8}^LTz}x*qrM#xE~r@60JlVltSu z%z;SQXh8j@bqrwF8sN!^!@@~PsF(Y|Q))W!^#A&?<`W+w2lmL{+>P|9tys{5_2b5A zaLFZSLhIKxwaKaulllNAbDe<6y4a3{?9*R})n%?W4XjkAGr-i-Yhz3ib3dzMj}G6d2|N9WV!yp$Or8NH&bYEl~LOT+U4+{;Qxt* zzd)c zL;yi$o`6oSQcZiz)Q1z5m?yhiA!!C^E63gt&6fYlO09~a9^_O4z}Rrf5A?qu-I3&8 z+@Gk}>p+qxX>p&`t*d%|KR5I>y8jNku;3azicXg7FqBp~%1r}Ef(Pq)s@zdxq2}Np z8j`RJajPZ zQuqX;^^nJgx}aN@A`p)Qk&_l2U-IDLcZWvOSWcY~i#-=0p+wQa=FE-CKR+7|X7D`v z;kGSf;<*xg{S^cr^AFQ>mQ+$}imy&#T+G=wM;Tm5iTH!kDziwA_3L3d8>Z*l>#y!X zhQkZyswf~>nrm=MbCCed6<0~#)Ro~=RtW%`zTV>h8SMo%UGK_Y)CWSdKzp&82*$$| z4WU}v>)R4?b+bOZD@)n-oV}6Qe4R-WB;Fl+Q0zMS(P~XZo^I(^<@JmHS>{xwJftId zXLa)>q+h&0*eV^v-zkRMGFbORxwu$j|G~3u@Y=@2CGH5Hce_H#9#4%3;bF=OB3jLw zmgO0=uG?>18iK@nI%6A8aQYA+Yf)0(q1Y}KzR=!k_~y4^K*9y{=H?-tz~4{ZHjduN z>NIZmWQV2(CkYZmFRoAiSGbl3v-4^|eyUGl04Jq*Ba^hlcfO>wR>S>)V0SGmG`Tg7 ztrWC%e$IxL2QzRL0rHN=02h{Iv52#cnE3$bTDEW5l5?LbSHmS|t0-{UlzZZlgna&0 z3g`2Bo9PMvd7jz;Oo-=GSZTA9~ldH-Gvfu80NYj!nxp7 z_e}42tdY2%wdM}*cD1On-r>ip?Sg#~)fY-OTR%cDq{9#ke5j=jrv-eH z&Ze7K$Meg8^a7B^Ip8_B@(}I|eL^VpXTi|u*mSNahTXM8WSCIO!o}{)b8YC#9?dsx za!75n{hTRVW>>Q;^*-7#Jut-BfRKcKD->mJ@QEn_VgkP{Y)7XiNp7%kYFvcuM?rr1 zzBAu+0*X{M-iqcqW?3|P(!O=c97qxsT?+J(D!&q}a!dcyl+ZBNb~xBo9nlHlt_=Ys z(jEk`+Pp1n>|6N}UgC{WbQd+r#d1S(Vlj_;-$>z9{W{mvFcw(B>=KcjDZjo>MV(7z z@~G>^#D2DBtJA`BA(Zo{l!^SE)XVi(|I7Rf+n!s=3Ky>c^>~wHEf;n+@C-8_`bc>+f)B6SkdILEy&<(T4d9ivoF)^ z>nl5%a+f?w<;+n?1%*f^lyA#$VUUf5v+Tz?1+lqeL88f@rkgE?m&%mRAbjIo?eABh zULIPVuzsuN>=elJYYLD^KFN|KL4N7JGjB;^s$PC%si{nOX1lO(e_+2merMT^SF(W_ z`Q>sOz+!2XSm@?=T1_U6gS>d+UTqwT5vyh8>IlaVZa zFlITUMDq|h68+A(iX}_5H&U{d=lN&T>C8$5LC~@-X+V%vZA$w8h7$TU*tKOTn~iPt zIWJkmOl`D-d;fTI|Kz)+#TQ1#zJP%y@tb0V1cFfhYXT{#OpuYVVE7_N%%mPP^9+Bn zdXvPv)cQZ7ro6CWfjd3y!v44*wb0@)>FSu}jn~5)Sb;le`_44q3?vISEaF_74TJW) zs_MdZ4iFFDA+rAE2&wavoR)=E!}aglgcC1IXOe;BbqQ=NI4xtstZp_Nln? z2h6bWK2^$Lvzu-~;;WP(pRhgVX}lY8o6QUc5K>1QOD)}=`V5$#N=#5n1MJF^ z4A(+mV1$ec&{fL44%2e}wq?=g4Zc;wF$iH4%c{s+?OOvsz^K_+m0xa|>17x8Z9n^o zC_vkQrT0j;6P(qndeN<<>h!ivRVvNRz>0z4i7PboGgn}%5Xk3KY8S7t6uGN($*-{v z?5K;u?T?xjrq6H(vKu(t)xd>Z_p?kw*e2UYNX{O;pZGiizzCs@rT`;z;kpg05x{64 zP=H!w=c~1)m$NCzg#uRfOEVPegY}2S16|s%{XLtVO3om~Fg=-Kc%%QyMi=18*iyLq z^;M2&3f5^H;88m%nHTNJriuhB%l&!r{nBIJ2!SH!x^cbq&&jI6aYF;1I|0LBt*WI} zC9E7GPlzyR4w#@vKysOv&Ucn8#kp_`R;mo$3XBLsAEZAtNjNR?nZrLTGdJ}@Sd57@ zzfY1b-vM+(EFIs5z#+JJk*B^M6Aj&BQm9}n>50mx72F8o6A3XilV&-v%P=Li^?fV# zHOs0w{7)2JA}mzdj5_L0Eq047xY9+^P`jKFfKDIj`8mdXJHe^0F6CEw?PE(6InJFC z&`rRzg)3nrCLHF|S2jjy3gl-~idRlTIQnTf1e=FR(7+QF5}N=)^jbz~T_ghbWx+t-WZ_BR>E?cNgu%KGF z;ls$mkZ$D!;gTQ_iAv-JX;vI($A zgsC|-`pwi$xp53AXLUk2aue=1)m`w>8THpgY|SRz)PIL0{r4cvzq=0h|0zZjA^)!0 zF_{MX7xmt?NvDy!aAE0d4oHVSa4N`+yw~*8c#LX2t$%#h9Z%brfe<^|&f)cz3w)7V zSr^rB`tMky|D=h)oTmlutyPh%949ai3+^Fxs?za)f3<5ca$^O!wANII;x5h2E4*t0 z2kiy7g4J=OxyX&gw1!PUkh}ro5UaqJhjjyI5BB6nm!|m&ZxsTcgtXusiK>WcNDVD5 z=DF%*)pv_~l!hX`t(t35%QmmVt^ytxI(>o`JDIu~_e~yWT4LmOU_5d1R$rG%$`rYu z7uK43#Q`K>@~gFXCw~_>54n^#_wro^W8Wr8YS?%KXN@9eD*|)=l=2f?gFG?s9qb{% z57tpXnx@h(xpV)c3whTHJ50VT9rAzoDEnZ>jH3CMf+r9cbmyjb{0(iIP~M`B+V$!D b@%e+8?zWbvZLqh8LTz{4`Ay;1{>T0gu48;% literal 135284 zcmeFZcT|(v_cw~;j1HrM3P=Y9=}mf(qC=D31B507qy~fl(gKX4bdcUbfrJu5GX&|1 z5;{mAfFLl60TP-tksxxPfX;kpzVH35`>uQcxof?w<&yF|=j`*@fS+ma-m-?#&|Ko9{yU*#di4_x%^^ZZrx%ahrHKl8+Rahc!0Arqti>ur{wogy#uT-JEY zMmvA%r{4pkye?ne-c_7-$h<{`F|dvq4|H0OC)dmMn%WUS~Deh#?nZoqU8Y3 zz2<$ev#!98m<#TD)*M<5ciDXbEpo`Z&o<`Z5IX;?rRH!oZ$jvm@!odxw$22;dIq_= z9$!bHr@761yYr{N^z@={GF69}HHfd5@m0dAHgX5btma2>-M{8&2Hzoc+%=wn4tzUF zGxQmasdI$zr12niBIbkHdU=DNk``hHwkk4Va;GIZTmFln?@sphGCe`rvApd>Weczo80Eh31>- zsdOWWYIAhIS4{}(`Jgh0+LolT0)A} ztoP&cb?qGkXY++prBOo1$VS$fZuK{|;_x_1AffD`Sb~}6!EFuxDs1x*8d=?xX+IJ) zF1y0pJj5r>O7rBGO7N>Y_)wQdXSh}qGIV0fpc1#j8RRqI}XpZz)tQgk4 zlMF|SAuG~tRwq4Pz4ldbnd^eNgq;fr88fNj8M7aIafsm{>a$)oe9IY?Ic| z5j$N0vGW=SkInQBS=9Hf?|6a@g`qlC;e-}QNd`vO%ziEtr-a2+z1djGwaz7vHdXk* zy>u$CE6H)*<0iG-;j;?`77(d^75s5#)+?ND{(|Ci2Q(kT=kx zEV4~KtjnUAUU6n$W z8YI5FnBvZo6QBhRbH1dt&L=x6lemL)8e3pNJ=leIk~v&z7iZUnw1nr!-aFktFPkH=VU7tT2 za%0eER~O#}z9wP&)CZtu7+GKE&V^DKJ3b=Z24V&%0<@Q@Z$s6mP@427UEJ+zeM2FW zz5F@>j}N@OVsN3ZdO;RZ?48=-je*9kEh`KVwp})%L zWY4W+EKkxvC{3in4>zofmCQBM@Fqe$a{FutSR*SoSkU(e+qNu7;5wmsBN1VkFAv&m zD*Xz7)mdCmQ2N(UV;CH@wat&9C`y$CcnOK|dr=I>1^IcJ!<)onOJFP0?JR(>xBEVx zZB3P&^q?R#i1MM|}<6Dx}CTUZXKI zf{y@?{(X##e|q)G`u5h4tY!?*H_X;Baf)^V;x=AHPn)A^)aVZLTi1J2d+P;**5 zbkuf2WH1@j{}YYZZ$NMbzE%?LvTGHeiP_yGndG<;kONx0QVZrLe>T8do@KH>px_ygAl7~I~6wTzRAA1=OtQL1!7GH0wJN5`q zi2*hD-P+dBf*ML)uyDo9pap~hgUGE_`aFfsbODo55iL9iBtg9BIc_H`11teV1(GNac2s#fhEg@6y0rRW1}kOaqNH?`ccLo7J5So z`tGvl8fKtudAWfm1>w_gp$r9~&OdSI(oQ>hYp*+GmSun5-aMiUB2isXEMWvR0+L%pxf zY?IvjrxmNB63>ph@l{boWMvJ6fW%I3e$>3VorWgwuZ@P8-)oiJpYE3D$ofpIC(3-E zl!-<5yovuH=aR?oHD=T#mQeQmB1q#=%UKAP2e8sezVkT{epa7~w}@fg3anDT@K zLr-;Nq5gpFOp1Hf^>74A^~y;Fu(inER=ktHXl1{THL3`wjYbqU@Tbts@R*)4l+oALegB*D9`XgF;0I$1(J#XU9RzB^6ZZx}JPjMGu5St0a zf=Ip#34>sweph}4iO?r!F!@gR<0XWbQ>QmRg0EyU0q|((AAiqV6{D|`Q%nkM(!%)3 zs))1R1HY|+85inLr|RT+2a60k@_CIlj_6tycshG&cV8eh88Cphq1<9nr%vi|Nxy)t z1Z4!41%nB1oz^rLtcNgWZEz8oS;aRurqU#}xOKB?RTIb>wjOxzBZy8r&M6?Q&_I9E zfMO(qDhbj-1)re7(*70la%x9fZt=1DNSU7N(meGfy?QO-tR^d()MD(Ss)ej>yC=h! zm^LsbGvAnDmshJKfBnwR!|)`EUR)AyC5(Xqi#%jx$dGlfq8ijd0Uq1N*Be4R z{8cPd5hL7}mYRY6gZ;<$mY4g$-MZrSd0vo<-&xr}y@1a-En)`{z!5WoS)mOaIw(ip zG#S3Q3Ry>aQioety-;I-EEPB!{k*Z~8Q=jJP2|%bA!V|GRs`cG-6&g~HD}rwHS`pz zItn{n8eLA6l-j4v8chMFX7yAoYd^uNAnijjG-6cjzPuzbuo|Z}qXg{es6<1T?ABEa zTK$P;<)FKl7Gs^3{x8#!%9Ht?n?Q~{nS`j&tTGw76-;wLm&}6GlrB3?jRqYbA*0~Mm zEYHx?iXCraKn~8kB`52+V@<6{05KoBUHyLSZlcUb6&wwX`f>cGp;0ivI6dF~W`Fcd z_N5dOIbeb?Y@-dJ%Y7_3jTJ|@r`xoyqU>$NHoW0@Fgu=kg8w4ZFj)Hp&C0329t;`w zbZ!nI9RLvt_`0jG1h4s~QrqZ2c{O9{aBya3;_&m}@~sQ0H~`1#?S>h#Qh`-K0!ww=_q&;8-0 zEjxeGJZnBgzQRDo?a$ zwWrVg4an~h2<^4gnPLmTRAWxNNkVwar=J%dav`T!ZU_=Ca$%)U8|&!F=4UgPmm-GF zD8wel9%0j!&izrt-4As zQB}W%&(3b4?c%t>C+0{;Lvw>YFgcqCO>cBt^;4mmc?ltu#`QNC4Z&>)cKLqrA08M? z@CivdE)SsC8p&i9enaXPOcyfeO!Ak9@|-)&zuU91%NZ_U#YB!@0hT>gxBrUK9Uk=4 z)}P-Z_E$z4w9AdyBw!B|Bf&D=isxUp-Vg)#wMHt-UGiY{>I`4DnFAk=8R>b^66P(6 z`7Q;%VwrH7n;4y%+x6AJaM&=it6Yx#pgZ;&V`{UQ4V~W0rK(qZy+gU3vQE*+O!{0_ zSKaj~#7p)GE&(@m9xwUa=PSpTSprAxH=&)BWT7q{u{u)&eU!&C2MT)K(u@(}5j)d? z%RXu5n$Xm=z6V`h-&gP(fF)iTf*X$VMqi!G=T(uQ)ZBmpD0*Hp~3e1>yeX#7gK zS3P#mD|3!n0PLjsk4N9hHSNM{6hB`MAk_z-CpNfMg*@t;;@($~v6n*xzDwp@-0+1A z{E?~3wH)$gdD_xdPgK}5LGdye=b0o)?lF;;Kq^;*lhATlxgm3g(aI^MZm;Q~3C1qB z*2$oE3(uqEaLt!wcP6dMT^`d;`~5Cf$5&Noe!(RF=Y5xi<=SCScW9`=!==g67(v7O z=tV6?UgAN=N#>*v>i*i#^k+I?0=lOl@4Dw&$8c@Y_?* z?e6ZosQLM?js(%ybXU5@VBc2b`i(uh;X*@Zaq;+v#J#Tu3z2q9ZH( z$(UY_Pw6;!G00d-iI2U0z*kJdo~ai_k=arg7{2mI5F%M;bY~0tvUPIdOOz5I4q$^p z<%r&>hLh!rk25D#{fH&aHcpO>3BKn(2XmGwh5L0g?O3Gqr{AI9UWI9x&*F<)pN@@N zvs>z!cLQbLyErKd>&$b1}`xlR)lHCRcG3Ax*z-lOGd_(*5$bPC6S^9`ht^ziJsK~dR|4x7?>dBvQHNJbI>^^DEVy<##ba2 zWlC}*G)X22h_c|!OW)0)p%0j%gQA$tJk3#L+-)`S1j)rt-@q{t`1*pGB3(Ho!4vSE zAh&q|GRpQ)Xpn0ut0#jQK?SiX5u12M96sF_1^;MPqXa-W;mE?fHEuO=dn<&?H)NnE z63^n{XaflrR0=PM9uiO*PNSk{8&YcOrKi#WI5Rsm4@{n2@J%Rt3j6as%Y6H@!B9(X zv;D$~E2}lOheW)Yu5UvC#@J?2ZtAl>QQ=8Z9n~`sQw^b=j5^$frN`rG2w@S0V2|Z@ zTGNFU9NJNK_x!a;uZU0CP=!~f0$U(yN&D}Vdk|&YU5LFAMQk|{>;tm}hRb{+xx`*3 zn_!}ywA9*02mPu6QI)6nBUK#DT8yI}rto>>(q&m}Wotv@{jL?uYjWgQy&6GRUMDc?P&TuU)_7L2+iS$pY<3W|b z-4_}1dPPaN$EBhX7r=t-X?2Ek2_lqijZgCYY zKt$2MaCRhMg6Q&<*VBsNA>zX*iE3=&B?46qh|G}2fV1;@5G{x{Wa+Wa;^VB8a5+Cx zak{IOzHqY9mSpCg(qMZaK7ZVBSHreNl;8`51)4eMPjhMPLHdMv?b}JJTor=Fm#^T`KTw?<%S0ul+M)y$o`t%4F zr;CL+UZsx3B{uyF%J3vyEJAbB7cpsTYtEm@S!oMzlJl5akFTgGv1kbJE`gd=cqen4 z)v{L&^Rkj$d9%F7K2A^8PRh)dk>*ZY|895z(VA{sg~&25=Eu3kJ{*jtOyk2s7zK)D zcr9_awqP5Xg$u+Fn&LCJ`WMLc+cFZ&YEia!O_U68`W@IuvW)ItrlqNd(6E`AwhrGA zyssi7<@1uOT_d@um9Cp5C%X6>R;Gxn%Q_@?Aw$GfcgqOpIZ$=>>nNmWRDO+ua|Wwc z4&(huVoP6KO1ofxehe=9x-tq2wuk#Rh3QEMi9yKy^286KZKaq5OM`?8zQ7KAwl(XQ zUJiw;x?4SE+P*xx(ta+=?)v69V3-R{{zl(Ngrs=#79&^NvrXHH8H^@+8ueKB17uUhb;Y2q!ZrVmXw)puLfqskiHHk5Cp8<0iPen^3anW!QF`C?%8WS0qOi z7eDUvh%plpeO=%X9e386ToyAu(Ky2IGW=pdl6Sn21Q|7EACjukqBWZc93m)|hw8_V zw%UF@PbBvd97Y2j^o8a&b%itHH(Nc(foI0 zwZrbM2y4pC>ZqA}`^NkeizXr4io%}TL~V_YrV!noMZ6vclcMK?FgFsK;Ua6NlVv=d zz!lU`bA}+qV8DN0iBmlY`ln-B z8V=L&y<&c3#*t@C-rIjHFtLd6HxeIO>Q9dO0NY6?TQ<+od$_yvX4%K9XowKFO-9$O zB+*9!wX1!)=4JIhK0Wj?2SOW|!(LXh!eg3*YDv#B1M^Qj!sFObO}<4a0M2b9ySr^j z61s24qBL_7yxMZWSNZVW^8|d!u!(hvgc?s=GI=C86W9OPmocNa0RY9KqR=#?VzoW2 zf**P8Min*oke=sJgFQRJ>0I!nz7P+mV&6Y?mK78j0*j#PoK339)!Mn2x`HU1=i>Xj z8jJwjK-*>JQt+gw&MwM%ou z1S~XT-nUgBQcNk9(c^}+X5!#N@LTq}Y28uxCNWq0 z+%_z|dP_7hXWTc73Wz2=;Nzw)kwTUs;2PY>(D-*O4^~r`x0XUm-A(j$4H9sXxR@^F zhWpgb+}ehOx8h@I%e!u(rS47jTISK+n{Zpk^(~bq{}es>s;U}Z;d$VP?MsxttSM~t zzEDGVIx3U_*yBLcJaSK3>4q6|qWuoISS{Sa)m4_}$yvnx=Q82f`KvN^cS>Et3I($4 zc?wUGnnLzeD|F;DHd`rV)?y%!_ijN{OHR)e{0G{W<1yr0bu8R@nH#}{w#_|=(5luN z0kOp`<)&?rZopah7W1Mq^KRp#$z>B}#K%&Wci&mv8if%PZB-rt6AR8RhWe3yCk3j- z)YiAt?N1!Z4qkb$lk{|ToXgBXg_@DuKqsyRwxGY9F4r?%bM)JOqG-^cu@gt$s1M`L z!tJ-_`kM(E)o->JViJb+M!Q*_A(YaYiHms6iuQC&;Wmd&#xm-bSe>j#h37zER`n%a zh}@vwsGmz>IbHIx5bR3B_@GsGBhdK7g^Z0oS&|r8M6GnSUC98UL`@-~4V;M3Shuj3 zpL=&gWg15@w$?`G$=bnkYog+bM#~C<41o@-zq2>BOCY8@OxnvB)B5`n4W;qGmPp7l zjZ4IPRtOcvo8+qexjQ!Ij(I%}Y@U{+v8;7cd@r$#`hwj@ ze4=SVp{Z^9?C%boO<%{iV*n2aPK+(9@P;!xA1Uqr2BK|@kZl43k%$N5ffyhD*tAjt z!N`u7aH2QzJL}$8AO_w+h$rUoc4zBpr80 z_|Ni^BlO~{unRS?;N5>%h{Bk>3ydC;TGdiVSc1MbI|F(L50=g}cl()Q-i~?kD4BdH73BC8)i(0t)Oe# za^Z9tZbMdJnXEATbO7JjP0vvcaVos!UQi&QOqklJCiZ5a(8F$tH9&$UIX$bL_r`GS zFf+47Tvd%3lI~R8Y8&R+Yz)L~(_+7-X7H$|v0~^85D&Uz?2eE`v26Pz>ghO?8|r;u zMaCw#IJz%06Im8)5?%?Y&oU20MH#Wz;GWsu`y86XU1-}hMNxLo`OBR7R!a>0L1G$^ zF#y~5Tg72ULi~OGUD_nTmM4W0E6Q!D!HlgT-C*9&#fF>?0Ao(AJTzih6U8Rajc{Nw zOU++e1=SFYuEBDb!g2)+uK&s~PVVVtAcns5j-kvRX9*XC5VF9+A7;d-o91~Jl~^Sw z$EO3?<+l?$>FZ1;eeAka+@L!_y&7prd6)0ReRTG9Yl)-Gn#Uz?ni9qsNI~xaF7#HR zph3D*Ns#Y&RN~OY@|ULWO-Y1;8QXflMzA_y0xVD)PQ`PH1xUJ?D5mhF(l|k4Fy(`& zTB-mL{*Pjt*Hxo*e!TxaM;G!7bvTDjjXirHUjvR5ilAwbk)c-l;x8>%$f0IQ?O`Rr zqEbx3qem9*MC3t?A*UAw$03rRpA2TDDZ3X6@x^Tbb5KvJbyL_7KdEBn8oLkN#o`8X z3%nU0N?OfjcQoQYgaHwLEGx^5adA|1CV3>&cNxMPTpUKlhAp~Wm5x*n~Gy4W>mH7>{Z20pI`;6(Yo zE|HH{P{Xy}me+v$03spA#uyS$8s$EsJ!8YWx{H@RMq;j8!=+dzMUT13|5YF5`S!CI zyoNZIUL?ybNoG#%@&=I?m=`!sFIquuRP3{B%`B(Z5uV0CzRc{or@HWQu@2xqh;Z9E z*OK%?NH%RKmVPrhRf4<1oWGBqTB&iHCil3zt;oopW&lmTt84lYVJJil_V^%*>)wIO z;r)OR-`%%A**>h5&ef%9^){c!W7F?n5K2sPx_9i_JpdGd<>F?@hy`S{6P#H)eE~F{ zWuz7_Xzq2v;0(?h0ofNnsmM0^B!q9a1c1k~{QF#*l(b z&e-uKE!u*`RpRtqh<w-8|ts-_kC>GI}4xr~4%q?xg~&I>ZFBw^-oM)%Td?6<|ivxYYgD)eU=$;tJL@U1e?Xc`bq}^S? zIoUw9Y@W~8&SgHdyEi6r_$IhFmwiN$#~hE%={$5V-qI?+svwH{R_M-oI=i;kQs#ty ztpoCFq;t{j8-q9ZZ4?zwpNvzClpEn#xFPnn=oC%h%ct;($%ET*pUy*bCIhsC-;(L{ zC#8Q3nvMT4=vnNJ%BT_@mDZq^VpkZ>3SwdxZr_kR{f0mH6}1Ze{_kPboM#RZ>=UlA zRG)Wu2kRzl4oEdAAb2m%6B<8|ut`*$y4o4oUTyx#werE0u&Qe@5a-)LCm^z}F6#HvC+?RT!@e)75Y(#kcXtIpp4ci%B{O&{tNH#M4*ze?Ivev{;eF z7$|nfE?hee5C{rXh?F&K_ycF??dHB_bvP7_usRzO6G}p>_YVTbSQ+zj8w6{bE)>j< zG3IE=LgZul>yuc~)#8a_1ZO+qf8JTwxc0wKPxKbq0cYv=t9xj`;{Tka5LiF6HcaL76~=_ z4ob8dGfvc^)(hNyf?Cg~mcrpu@+Zk3Cg6AAcLeq*@>~iRB1Q-Ap<&5egqHSTo=XK| z9B0eb^G@>39;8S_yXdn`;Fjn1BpegAH|6IrNlLwo9braRcPjqNEQ%g%-+0H+_2ph? z{VyBWjzlcqhU4HO&sUQ#dRWHfoy|zz9FbZ}1F$e5VPC^>l{r}1h&Qz|FYAvCnx@vN zkN(!@-lwd_vZ)tDUJEEbTw<3OJ4E0M4U?QG8NI8Z_S^ zUT+_{erHt8_{&maW3(^-;5owbh;p1$R+Zua&|AZjG((U4m3|~M)^4|{#|Ke)uY=M` z$n07^y(BAAXk10L6m#M^Lc1DIZPT0j`{Ol*E_rLMM;4ON;gi00?#?7xwKMd#(Qlas z%L)KZd&C3O8_^9$RuWFD`Yuz>4nN6&o8(`W?t9_vF-nCm|D>fu#1>^~_{j!WSFn3V zRoNX)6sVuAiY{3;S`o0NS9SX@8Q))-lB!!)We$EN1kbLvh`hi11&Tb92O$Z42l&`4 zmR``qehk6YXEt+yP(pa-rD;2MF=i;VTDNg+2>qx*?SszYi_-=FvRsbh6BfMa4_hDg zt-9V;TB9lPp#I`kz@0hC9G_5ap8AviU5zmxT3cyF=H=t8oBzwR7I`}O>VTSV+iB1# zQITx9(RVRJzLlFcWt5GAz7GTCgbpqDU(h-FfH>n+afa&E@E1lrtXiVSyhv^4|JEm) zu`Iq4#OrGM_PS$3AN@(J9qif@x&XQ_yR6HLg1C=|f-mzL5lkOa9ZZJ4#{4%x0>Q6z zL^7{l8I2PI$xj@~qB|Fw&ps@7+8IZF=5VFZ<|eB7?1GN9-eskAMG?DmsNu}UslM>G z-2A2#=6||1MBK>`jc^e+ z=#E!WZ;SIFl`DF2cP0FqW>5@Pf2Da?C%oYasD0`c?9iC=+>uay^@K{E#!3RN6^bIC z8-Mp3SlZ)lb&CU4sHXnI6Uh!69@GBMes2}El0(l^Tev~`dnbBT70pjpg2D-NHLWY_ z$@qq!-aNiZ?0*t+EYXvV6`_MGf1rQM@?9RBPHiGv4%ajJ+QYJe4w!2kE!_GSxQfLX zTGn8x4h_zeaW^nc3tIk3W|^$N^MDB+A{0|;!f@X>75q!S4FQj1fj+9g}YX)y4IUo7q5j=_?riWhC_{ zEJQlMZ)SZfB1){rkiZrct$g~c2jN{MXgYlIaO4lxDxSvs;|XBwvbZN5m05iR*?l0? zsg7}Rnt&ThG4#FKzukT0H{^Ok@CWvy@L`kYIVD23-nX{einkspXX{nVGQRT#yQZaz z#P{D!1pZ|d&VW&He|e2wex~vO)LGv~R>*yDBA^HMqNCA50|j2Cr*F+6&z)gM|7uw! z2(?TYa{Dha#Ou*lT}(4T--zZ9&WfWVV%LAor*6)6vsdv}7=Ng=sNeid2lF_t+QFI~ zs%iTQzwnys&O)gqa;(!Eoqzf96?WqWN=8-z{B%^%UY^fy^TsXIm_p&GU2V{8t|a7P zzR}Zvm759Ab4r841~O+3{#!K<2K))*DN^^pVpB)@UfdF%pDQ3)vVz;?Uto&wx@)aa zotxa3qahbKdas>}*usPfX{^EzJRgK=4&*CT9N`s3dtLvZR=dfW9I7)ONqKtcYilxa zrKg6%Fx4p=cm3zY05M#pE8VH|f(Hd@I|dHi1l$oFnm{U%pKVu7OPltZjsqde0w}wk zWO^o1J70*Grq`lJoRhv-o!KY{m2L*MeLJ&{<8xo=MPAl_qBo|umP(Hr06|+kNVEQZ ztY+Q*A#_3NLbCb#Z78paEs_SM+IiLIy#mz_2R%shDb>*qgpl27hal0_DB5PO(<77B zZ{vc}6(xD8-e}~eQk}%uwW251o%#BX!Q|C7E6x<4h6tQ(j_BUv?9sncg@-EFqUU>} zGFmGx2dlppNz7{=h}z`X1T3wY4nmMVeX?)p<|5=ERaj;HFA~!n{C0IC7+btEl^nLG zQ2O_$>UrR6Obs$}ZTw+1-k1b#>m*?|cApeYr3@J|~c`TTDV5WrCt~~2q ze5j;y+T*N1N8h;5>Cn=`Xo5|~5m7k?sH{#$46Z!BqhP1qtYo28rBGK)m{JytzaZxg zU31gt7Fm?(%g;`s`djx$93iFnIb2Xlu0m>jm?0(JMhHDTmZhd~L_8BRgkv7KKp;;q zQBy!i`hoWfQuNN9A)~ym_dnvxgrQA9NOLwPg1&L40(9K>HD8$!a1R0M3skl-aT_w6g9ri#8khkQ6{jot|L3F55l|#POS!e{Sxdg z;xsA)n8?kz|8os@6#|}71 zS%)w#6SpX(gc%$)2N52e-LDKjPat*I&5C$R%K3n?Jk>c2Ba)?5^2W#GKRgkms{BXr z?TLf)eCiepDS|n-q@oh=nJAG1tpeKRMpQn~yl>-u&s$^BimwGHJJHOCHCo;bCsx!p z{nPbKU3?e9P&F0VlhU`nHM@lk|Xke|jMJDQ({CD)UO|qr)LU9%rC<(GdsI zqU-?8chsR0`h>naCHQe}ts-ZtB#)WS7ed&qVCu-nd7F=e?+k#dA;7s=?F_t(tcS%6 z8R_l|Puy$TzJ-{~4ABp_KtUU!?%^4p;GvRX^+xIv#TE${TYUawe+F|GzEG}R4v$G4 zfsA=0rW!vO(WiNy6`1}Fgu43*I~qOKGL|`A0H{^2f(TAl79Y@%mEkH;kWgz+#D7iE z%3+Yk15t248~as7y1BS1D<}DV;Oi7AV(8XUO%LGcD!BjJvOeG7PD%?E@Tydo(i}LS zdI7~$r#cMUqeU}r2g~B+aGgF-<mqGkNLZ6;bA_Cm1zSm-v@3($z1eTijE0H9 zJ6(kFc^8!ZImBDC720i(K(c&8PG;O*vwD9R^wdZeR!qzXJC$7qTq4p1vWxz{%N)G< zcy6-&UKdpFHAB~4nB<2TzpASKzC z!@W7KuV1NekKHtFcn#_qK56poCt2C-ki%>5F}nO-J4cSH01^{RO%~}|_)sk6zG6%y z_hV9Kr8^~O?j-=-4nRE!pzxT;j{r|C`R=JZ?-gLujtxwx$5$A0N)#hfQx8};U zj)k;<=y(}%?Bexb9D)0EqU=8sokN-(<8(G7#I6xVEdCt4{or2s_QwYwcTT?EJ`olL zx_rrUAHDqBRw*z=BgVGbXiJ!=hHruxZvjcGvFRV6-a_;1)XM7xuseV z=tL-rH!Dy*Y|s0)5EPu&r?Hcjj$0%z{a|@;Buta_(4{buFj>jcctphYlSjXKWPoPR<~yv+g;lIm|WQZT^%dm$=6&YK-H`6*Fu)-CH(l#9~N#usEtVw^gWS zrR}U=hS4{?#N4IrjsVL3!~Nt~ay)*V1e<@Yh!vi&q!yGhC0ITFDpKKccm$_p1aBbK z3S{npyX~k2(MvmolztI_5g0XRpD2!PJlB6mmd-Fa<*0q_p~}1Hd5u%)6IZf%W+RzK z;@zu{ zqa<1Ws&6|sATHt_ivxcKm6@4cB$Vdq2B#jgGxn*f@`m|;F-u|r&^O+Rv?5G@>l8lU z+ec9*^=!yck8VzhkQsO=CdC^RBquhK)KtS z+}}^g2_Jern8>g=S!wH_=@Gj+N;i#`b&NmSk5MqO#5{aZUj~&{nW#jDi6doIO!d6b zN8Cmq<)782tFlk@qi@H3*|kOj9yFVr^9laQH?sEnq;pZn3^ytJlY{c}VQ)+!Mz zJ}y`FNPku)fiWbubS&lmJmupS^4tyAOb1-sqBh`s%1 zk~h8NBus%yIM8v@-kM*mP=qxsViy?t&imx05mH9qOy2j zCfliX`;fiow;DENCHVo$F!rc7>!|du`{AGJ`6_9mHWEIKlYh42)%7q`Ob3(()MCaJ7uwD zzr%mN`JQ2JE%o}hrTgbKJ!nS=wd+g{*99(k3OMsNd`%DFJ+F$a&r?)ew*MobDq2$+ zioqpR#246}qA%ECf7Ah$_d}8S+lkyF%k%i*CX8(eV^&`ukuf7hcdtFH)HFxXr~IgG z6D3e~mF};U1L#7nHyXH6`|V~opP9uvc6)t7pV;Vt4(?dNV(3Rb=uR=ho)|>S2n0U; zJJXF((SP9LyM3MsZT{+{!vWp{F{d(d5N_4|j(SqpaJBe;_+x*X`Y%*ZR*FieechKe zg8~k~whANu$#GV2;YJ>vqv49_f7Kz%pX`v6GK#-GbO$oX7@*nrvXW1f?q4UE@73k< zdd!qj>(@c~hkN{O?4vB9dLM*5kL4bSo&S$y^?TvE90cQ4(vH%lX4t&u7ysH`z87kV zJCu9h`kS+cl8W*%!f)Q}OO<3R9^+s?5S#YDJzf`HSgmar|2Q_?X7Jv|vMLOtWL$1L zC;YO>?MO(&fmo~mTKTU8kN}CaE>;Kf)yiHvhL}`@btne;i*_ zURj#r9Yc-IyLewaZJc&T;WQ7>XsvT5hxu_g?sqZZmcns`mG%_llm<{#$v)}E@bAR= z`9VFmV)N%;cXWt2m$MNoe zs_;`Avbo*?t@i)ZD47L0we-O~>dmKBmTu0RvHx^Gpf9{>vrnXDYgn!0`eHr|)a?@T z=>hjUl?BOUYtBUj`58>oscf|)54!dQ;5Htd06M|*{-bnq)Uf{Y;PCU?63=AgA{qOI zQ!ttax1^(Pa4;r4_(U+!LtD?l0D6SX$={cAbZPH%+rfG2NI6SF*$@M3jQxy?NApUoI4kOm&Ip~OpdT9IRo}q zz94nX;usw_-P^z0ZKCH)JP~E;9yY+aKqKr;1pEomM=_ia+5^;2g6-PnhHdgLILuu21NO}y^Y2wF6Y3TU8Vms>78r)a(U5tVI4>pE)nM~{D3;mr(*8!x7 zf9wa-Uk1DH6n5}=OZRfz@DvNAV*Lg@a3-d9!=%4_D0{%L`DjtJ8D6W3N1^Z!!fk8?+$&6R@dNj zp3Q)5>D&W$EBp*4KbkY~f{c1WdwcUx^KA24^EbU{#oB9yUi|P-YDSwUA7b{VM z%-$WXG-->rSecG8m^(ck*S8er&n4xu|Coeg!puGf1tWxNm{7wtf}k1r&R%#{aFs>h zVo}656O~GJR3=R1L3E*Z^QOYZJD<0YSSQ$)<=RU7fv+pbUuDzXl6!~g_Y@f|84$Jm zbb;pB#i&(O$K6LqcRPUZfyG?puS!ro93sH;YDpgBW@UzG7*2DwROAm{AtY`fBiPog zXx5jVu()p*AT*REjd-+#T>OJJG@Gg%#^)b~)taT4O0~as<|R?iT#-mT$1cDo4ZE)x z=^?X7RnF?<$Y-N5R7^37BeTS(M+ar0MQ3!NfDb$V`YpGZSZ z22ypl&kAm0GyCPzw|w}%6hzEOEfl9f561>^d%ByF-Z`pKCpQmziK@9S#cXvh#`G;& zO$XwV1gJ*e*XR^)d2pavY)DcSS;nt`K>6!k-;a2ZNz~L8ShfNm>2q!P(UPg&M zmUe1$R_+@}C*??AGGp{1x)^5m2@SJnSNIBt!7F@oQ`zcfZy6d`g=+gG)eWm#wf}q+ zCyH`7-qt~Yy|`EV<)+?FJ1{NxCW)> z7IE*sW?XNHcL-Qb*^A5+OZ+7f!J0M~#o=ue))beeb1Tlz1QW)zO}8KcD;SaO~=d|_&u{j`lkGB#Mpuirbc zbz0fFCTeuw{Q>Rugz5GWqkB6IMJwXZx1lrh63>gT?+TUu{@|xU?Pgr_c=Lj#!+tD* z#Y$@659WJnA@83xJA|!9tcH{CMKp(Yo{6a5mtar#PY)qe8-etp{8!92x-fR6pyKHnkp>`)$>@$B!x;MQjWPFi6Wu3w2 zQRST6Q^Z`iD*dZP?Jtl4_tEf3CF)H|th&b=vYqy9DWN08?Ww;H9=Ny3p+voUIa+pS z`ZoM?M~&!PXY~7OlsI{<^=rk^90J!IwYAo*c7TFNZ-|HfYgL1nQJnkAXN#y^u*d_N z0IK)r6Fqmso`rTtw~yKR@0GP!+8#6okOO;iFxa?peK1j{s`wY+^Jh)~jNNTE4K0*D z=%D(g$N1`H2GAYg^D+j`=uiIKwrjDvtWqXr7X5@!Gt0RB1vVNAk$3rxe}!ey(W<~S z%16G;VE+V}${+Lbj_y2)Xe|j{z5cCf-zN)RwO@0%{6OR+NULh|b1YZnl4GpA+dgQ- z30UI`M(WMr$#YVAz$H8w%9^*t!sQyc@m1qm8>~TWV?vFR832TbffH0A776Sj;I~t{ z;AmIr-6i{7?SD^d#ZD3}^w2cpmW<)~vxZk$4)1L62C z?zL~-{c7RPtwR?MCq9LopSK**S`Tz$5~HC>pkB1!Y=02px_w5VAj+;$)0+vMgCPlnrQLykfg?vzFm#vo9CEWzjt~r< z1U}v&!3v!~o%Ixh+-197VO}|@v%Jn8l=>7Q@~{zEI@rui!7iZkUVY}K3Tb0+HPdDv zzOr?`y}b@J(HTfpdDEvZoAuS%?OBN`P?NZ0!!q`k0a%42)%lGs#hN}k-8JIj@T)5{ zG@qy?@{7H!Xsk=^K^-s3I;*RV`?=Fj67b<~3SM_0oybRjBHjA!9g0fjoGX!cN>uyb zZHwq5{F$nr{$HHEXF$_g)9{V!s>_PHE24DPMir1Q2th?ang~dM&DxSKMqh)i?&AkB(+vyc~7d0jMc9ujSw7ZQt)?ETsg zZ+#?O@L6**7uDx;CpT4S&imnQO6!?2YF{1pO6K^^3+ zz`gk8J|PzqTwy?&P}9PJnn5pK&5>ZgZ%z~9c?=HPOK2uibxYC3uw6yD_tsC?ts5o) z0<1vs5X0NlKN&~gNe!7WvpI41-2IiPrr|!^^D8+f z?W=E{p*6^jtsT{GB9ORjlRmzCSCmE>EiDNJEtx?r#XSp$BtO|TR3_owwP>Y;PBXcL35f!cYEC_N9y zBPa*ux#s~};U3)c3+~GK^G_V0S=4(+Vhd;_rx>#p#RS-vEmXG_q;=wmzO1}pG#gOw zFc(S(NGIZ-<G-yzyXYf*dZyxQUI@KxgZT2;IX~UWAna&g=wh}6!f}K1DBsA;K;G8WUHeN!F zGA-;E`gRlg43~Z_AQq-c#R75y=`i?p_@6R}C7L&eU?hh4_*;*SUR2pTyFMSRga;uL0IwxFC4=xg)-^$1H=kfuRhLz*p%@F^Mt!x|XUBb`$8Rj@?4 zV;*GX@fQh-9{Y?83|UM+`q6ZoF5}u!j+VGWadR}-TSA-{DheXY#4n0FEY+W9cpC#I z&+x3cCMEC>8ZEOA1rLWmDs2g>*(nn(r=vCig-77$j`^^@+AXz|-%r5Y;YwGR@!;y!0ohsAJBwTVA7=qvn@)$3ljn zyIBenTfadQ`0kubjaONvKEVWCMnFq2F}#BW$(Oiqg52qp5zO zF(1&lF{4KbOe#WeH-S0xW9{i2J^IpJ=pz^U)^n2c zpwZ5MB&>~?~Epbjd+^j2ILd5L6J__N(sGv9nI(mUg?Pk zKQl1w9{t{jtAW`j#&dBIjXu4!oaQ`zXzUB3&wLJY*O zZc+x@^SSSg3L=at>GH;u zcY=sVJ7D(Q&f5`3;_LHmk32$O1e(Pnq?6LW=*OY``|2}zl^IZY3{)XaG54-B_nBojJ64h2dGzn zKX_h2J&2m@GxeR05bD>T0T+Cu_mM|Re9!<33evQ+5XR`A8Tt=1pUt}MEJf(hH}1{r z_6*eFBp4mCK#gIRgbY@mf=hP)DjV}=n^4hWmm+O%IoYjij(g+PF|N7#QvnaQ5}W0- z{OXUJ^^CjsYb|^c4=rC4^!oCJbH*KZ^%Pj|tOzDJU6j>kd`C2eUE< zmZ6GQ9MNgrSzy~}5BfIsZ(XriW7D~z%^%x~;zg!d7jNQ9z~&e|1|}RaEd8$N4b@F( zO24mL9a$#Ca6{9X1)ZXHoM99wg-t-k#SCQ=jNC?#wp&j}xdJ5HzvPK&i2>xV=`?YQ z)*R(g6r_$@5wo{H(jz_W1t~O_!Vl9l<_rP=x)OuC>c3oeoPJf6v!2t;f3^b1+=1Y0#V56m98Llb<(JaAX zsC-=G7_;ZZ=WsY?Rt+2gpgm7+514zFn{TEQgaL-nqH3`*NXvjodgWD5g&Zu!>UVj;a{6uRV@AL2FS{C(s>0tyD#KgnihnCbPGO+0bGsJCI&1}hVLu4! zNnaMXEE|E3o^xD4w(D^rj5ydTWba4K-{%l?@lea#-COToDd^@&u_|LJZ8O4*#CZM0g>-#hLapLMNL4t*a5DC~*=o@8a(SISbWbu~?xlLv*XC^f%*|4=ogxRI3!=^#zo_NXoAbvc zWto^BD1iBeX5keN4#dFaLD~io83NDoL~;8}dzOMp!7~tocljtT6F-EX>_rW>*kfDU zynvRq*W+ht^FCld-Nh&y*f@!cok7@->}&}nmLj;@h-Z0$w_e|+^s5Eh&3_lRsZ%R+m`!K>|w_BJuUqEhhWNKZ=_%c@Q#9EFBBeSP0*gMIW5KF$j z(<#dntEr;kq5*2nPdIUL2V-Ezb-XpSXoysG7`5oMQA;uE1l#)5*ufLB!g6gl#`^`Z zWRWE@08$oiDC$Sc3Gb<0a*G&nz!&x2szC%ObdMxjOfa-&mpPG|wI=M`yg`*!HLxO+?|fIVJ2SAk z>HbUz`yR%nt42k4d&y@T!M{)X$yh3g zGuV7m9v#c-Lv<&A_@bq@KNbQnN9?ikT7@whyZPrKJE-c-v!i_;IjQ$~I?jO>Pm?;o z3KbrVwRvMVB*4AKz%buJSGGg3YRB8Hp7A=h+LNm-q=W$HGN!7qGN)>NExQo1DzY@& zg0y-=4RF4)dT^op7pnz^%Q3}Yo2Gta*0C+-B1E==?BO94Vw1~!VJCiMtJ&PmDQ#sN zGF2MJqU-a>1&C?7!e|K`F02Uy8Cb^4AK=Z@4@H|umL+){fss&>C$>QQdVGTTPl#yuGEtsYQ6A-iGK<6rWFj935V`ZIB<~5wE$hNR@dLr^w!X; zGA2$~V)0cL;%N^it8V%s}}GYT7q;=okbM(YSp-JiNgfI9+{Rp0FB z@^#q8WlaN+rGmR4q1l9EGIUACLf(_*5X?d3`R{hs>fZ*Xu-yp7F$G#E&oxV89#Dya z-y|An+YN2k-mJLsm;t$PL+HAU<$j);nvk)v+QJ>S{?7xeMlUn4

tSQnVZB&dlnw zL8|&HtY~Kc^^etutsc!+6GB`Bs;q5&QVZ17{&Zznuk(<^`6bi`seX824do2Qai6<}1tw1vDhW(eogr43RNUo$R@ zhLU_Edv;yjDE0myHV>Z$Csw;`jokD~jhB#`2-zsA1k-)#cc5{GGGGuIsNf-k5be<+ zIMP`{8$$mjgYYo5&KW_$9lnt2fp)`Ce4lu^gv2`towuGQupS?^uCpbLq`f&}9x%ER zMDduLg}IBIZ6}uv7j$=vK7blfrcR+iQo;d z0%LnC{Y|m`4BEv4Jp!C^N|hB#a}WluYzB+g|Cn!cSHx{s0(QeHtBNqwkbWu;T~XM4 zajPXv9_Gt9p*DX)6)&FEI(Ndn<}fJ;-b&Rk*1=e{ftzGUE~RhYd{VK3YVmSF{*Obf z%r)gwc5m_4)%26VkaYvq*Tx^0rcWeYaGN98FVWmDr|Zvb_K@?F==@8KS^5DCW-`PK z?&}zzJ^e&yw1?`af%GO@hZoMajz^V{@x& z7~a{eW5v&#||mm7f?V&MEq{3N=4 zY($^vChlO9IG?BI19fqp!T0yl)4nGBgnSq?KKXHd4D z_G%iQSL)~R78We%agw^s5*uc~fyi~-;VrXNU*AI+7M#&yTi?Fzh>=>Y)r#%Xu-V@+ zs@zW|beHF*eEcfbf81iU1$aUo)y|n7%y|H_5;(Ljmy=5hfTa&Vnz@#3-xCin9U;|I z7b9k98w<25zYd!JR%Nh5FQ9kg5Nc3 ziatWGLEbUw!decj-kWOf7>w~g20W__9b6JJHfH#WiYR@eY`T8)A7zEMWwdUs=)n*^ z@M|~Vj3sfCcGJ$SJUPN}?HNXOP`r}Su_YW_AmU$i-|4Aj!Q)Vlf5+^gi1-*O-n0?@ z-CAe0CIWmTYlTReeLfR`@h_m1y>}e7G(A^0qCccG>tuXpcdKD$ejPn;cg-I>0{lD^ zrfF2ev01jVJ&P6NKZZAHBgiDNy3A9Mia+@ z4b1SIPs+iZI&r*mk{XtxI5NL2>k_EDRd=wrI$lKE(e|V*prcj-rn*ylYCbD;%Za0R z;ib-W)Z*KXp0n@0>h_gYw;E!+MS-W@b#b@ktmZH-UB3#^D3pGmVCu+UK&99dY#Rp# zkM~2SF8KNT>QmB*!C&D8p%j7kDBMQZ{Gme+ezI;_zurW1 zsQ|m|3lY$7BzHc2(q3acp z-?F!+H*5=hhO@br$FV-hltJ9Irb6(;*~Sq;%K| znv>)8@WL!x$Bg5m)xWuU{Atl}a*9+m(Qk@b;;RHeBjc^@1~YV$t&6*kL^+EfD@q41eASAP0UZa5W87&j%MjK5hh~%qDsd8GKN$T znXXS9%>uCmK>EgfHD+0dHe9Ji!>zMy1m8dcyf`a3AO610zh*8gX|T6g@~Y+Nn;ju9 zPv}Lt=hcn&*rval-p5#+{G;e{*C7VOli<7sr@6NfjLNl^y~6SO#9nwa67pm)^hA*a zn}DjrBSp2#Ef2hhW6g(N>~?r@QMeFUl1C-aI+@sqcd-3075udEsXK_z3#*1nAEDR#|EBWwVQL}T%rfvp+eolQ^vX}n9*@K%=9l8?3z*SR$er)sFs(9&C zm7*r2#JtVhwx?XLL(f_;j?o$ezbt!yn2N&vX8)nj`wcBD#VG3e4v1ICjR)Zae>BxH ze;b{i{|#}L@2Ej*3eS9O!O;N_#(_KBacCe-7Hp7VE~(L@rFhDNV!wB1o^ikYW+g`` zG6Ls^^_%^29*m^+Z{iLnEy>B(I}XM5>wlH>;MB23}E4d>IBq>c7N z)cdv**Gc)HYuOduhZy*glSjTy8a*VY97B1DjsD}?PGP=v*&%=U&G&GWr!$hr7+$^j z=6nz~7c-iDHhvC@JdK4(1^~kiuXg;ZlsD~iUUXBd2?&p2@ANszaD#1^JBQAO{T2fP z9aPPr5ZIiQDkkV&1QgLwSi8x*(CeOf^;plnP8;wM%`SUv@1+tvOVBto{rk0A(H>5a zUsV4k@J$ULJ2kfXB)#I#e@7evT4%`$g#Y}WlpRZ_pWXHbhf8k$Vh}1rEWIRM%)RAD zBUe~(nc|PEf5tgnddzRyiDxemeBAqM@b?DM={px=2RVosg1wl7r4xvJ$Evy=PK7yC zcbu4EK=pFYGB9i)ClCE=;-FX}?6U{U|BqmzM2QM=zPJHenE*>#O$Uz)w~l*$3+vfb z1Mab|4D3yzL&S%(jwu0>GQZ8oHnopgoP2e*(-q8z;wIO{J{`!=rKNdqPt1hM9AZFRg)7MdVpm>3 zL<)ySa6iB8difiAse`oSpXrwJ5CuYzs76CvlwKp#6#h=tsv*N6t94&VTlmcMoY+7p z&*MIrtn9IT7b-PJ`Y!4j_FL8!2%E3Krva$cqB~#&Cdwf@Ix*cYj)9ZS(dnDpO_35m zTG1aGpL|ogJ2ZMwvRl<%M}yqi6nm+KV=qT*c7pZBJ!H(tP`jYhg zCIG(gdY_|fa~N%1>~ShwIwh^zw1rLx%~K$iOjV<%P&O2Y>J70$lkyUL>wy+}P72wXBzRr*d$It6tf33^fflu}MMrCjf}_tj*$tzWW8|qnR zoi~RpD0fgwy!mHvPtR_B=nfzOzuS5wEITgJkHezD8FscQ2gyH38oK{9QY-Z7r%QhJ z&VWQ0380zuZ-yt#jq>b%!d90&P{|drQlIK^mKyY6&E{sir+a*kHE#3Si+4D;*8ytS zmwsn1X1W%X=zgPCLPU-{Eo64~d5F-5bY6A~FdjA2yC4Sxh~|<1P2W_y|3B!P_wC+=M9?(v6Ia?$eet% zXy#VupQgV>cnMqxjr~^#OVI=&ef2XJ>jT&p?HFwKllmNpz(y_PM-a~Esm|21F%Kqj z%Q9a|i1Y;5*ZOM|Cb3sdtnEmv+2<>4t>6NhQ`KM62TP}Sjr)nisxRh`s zTj6{|q99uikTHcaL1r3YM4mAKFIY86`UImgsuYkLgDH-p2`8(N`2fY}2r#ArS1;#l z_l58|m}jO$n2bE%;CJCK^T=||RU@pNRy$gdkz&*5`&fGaVq~0Tjr2)RZQm14X2iCn z(^Z4DBTIUP+I<_kCIGB?E_1iqB!G}ceK;XG!<(%dNlAW?WJjfrtUq&t|QXGP}yT0|jQ}kzN7+j0Y`62sZGZqk0WfQm?4SjR{Me6h` zs27|}GN&AWsaVq)7cPdp|03CC29g$}W zaw?)XKB_^h)t;#uXJtH}O9%6uESK|yE?4r_i?w}0gxH|#+XsD)7VByHdQz%cIp5&4 z+T?Qv#@YpaRCOn}$nt^l(+k}!5!qr*9|4+HKX+WRko8G*=)bq^60e~ zo`c{}YXGE(k3T-Dgm|`%*U5jPS#fv6J@0w#Up!jzgVjR#qe@P!!Saf7b$c2yt8$Jw7Ub z^l1q!>N6d`6So#Lovpq)2kYtsU$AWKkbDQJMdyx=+}nE>n^SADUxwZS>(mcG%cJ(A zUp+>*Qke{C7;o#IfWJROhEl_u;j(AepEV<4>**lx@eFe#V7E;Yq@_cjeZq`oh0b@B z>GU5MiV3SP2t`L0mDuN&Q&{&Z(kx$#o&JU>#rVWbXSX#f+oZj@1G_tXP2(FTG~#Aq ztqF(J(pKZZKraAEf#wqdBsHAkbLrx~6TofZ{dYzFuXs+;qs*RxxA)Nya*|mGA1#E; z*s1WQ=lESnaL%xC^8k#^gfO04g@q>j>g6F+*@qtNnf~oU0%>ux00)`@Y<4~U@0dsp z?f&YA)0z$eX9Amou=1FxLKn@f>2xZ9t4%a~1j&sc$*^f4!KDqP>;6DhE*^M%;dOH0 z-r3;4d)q;!Fq|*qS(_n%3tdQ9R%=(K7$HP3S018o4-m)C>9q}^;ONp^ufjsoiG%kG za3Olb}4;;G>rwYQI&zF`fJc|DmLQyJ6o_~eS0R=0JvM>o$6l2bNiq16kYc9 zo^K_ajyjv0h)nirwDl&qe1>?SFx1pW&9+xcAnlnxu~`!V{6V!EmswTZO|Jl&=F!Jz zGJU4oaD99H6Mzc}>mmx{`Vlp*?%%d2B9sQbLc z%c+BJxrmyw;QS6hea7DzD`dytUCX@a#_r+}4{K8&?z^tSGTYg_P*fLy4d~aB@i=GZ zZRyn|JIgt(uw8#@nf_du$ca)~d8{?6CrfD0xE6->AJ3e++q){LGMhFGU95qgDF zScWl^+~r|K`lum2l&X_&P2Z{||59nn{$h!*JDM9jcIH#H(fl)*6~9Rxj=9z8d6DSr z@$rB$CfeHvvLUg$3YqSHIXpUf{W;GZfEYp(h3{tLr>Uo!hoNDfu zF$)QwCTbQ&nD{bE^hTzP;?~12a1OV7*JgN7<(`IV+o&!#!cQz-#Hz6gR^g?&P zVguIqumV7LmH5kk?;Y|quPyc}odE$vVz0UbDDX*Az0^|Z$Q$_PIP7yP#YbQ~HcW!L zb}?<+L?IJUXSW_B5P%>G!TU5QssqI9a0vYlK(}JHyK~^lVgl<#K<;0O1;G{`pQ-X% zL&^4HyqfYhmU)jqGl|SQyhL#odQsq=&k86d?FmM9#uxICYO|6nxv@GKa-P$~(-)$%lmQ0S>QY$Pr*g*z#CIBK!qeFs?zJ${gqQ8H)HN*t_#8196{bXt zZ#r9p1#0a~uoubv2J1wY=->detWQ})p@E)hdi&e{*uYoYk6M6-6GITmxf?!sI&*4m zpsB35o0OVjk}?a}r!S#p`o=yzo;kYzD(D;i^sK_QUL~>#SLTLp6=p{TZE{@0!FaNf zYE}q^S1iSVCH7TtGV^|SexEa!SLO8^eweyYCddFagKOV6kAthzVXV~W5DiiaNL#Tc z|1IKx3=s_-(Tb9@5$_-hDu4tVK{GYJmge4yoDz{aK=d1bAzVx_04cKBI{#2mxlNm3`~`issO?a~?^=otvTGwmpqp`Adq5?7{7k5qjO=74(``PnWT;WJp%-N%B> zcN|_L<1e`B&i$@lo|ps9Sj#VE(z|h+nQgw*E@KbFgXas=&B3Xjy6?#IR4>}{Bv+<@ zd2k`-Zj>mOpFsS|RGN3nN^2%SWLnx_xOCW{f0Xwa)DDmo#9+ROj12q9!J3ZKaAt+^*n=NnJLwe&&e`r~1re0pZv-fOKVFSx5D0Wy-g`uWLXpc=rxM#6p^`XpaobGrO)ZSfmPSOIdDH(yO9iIhUIC)=QW zsjQCW11~?GsN=*?TB?cu;KjINldJsW?s-(viZ|+c>te*m^GkF=n|f{fgOm=NORx}J z>=fpUQdA*MK%tiCtSus%Fc<@fooV;Y)di&0i!wn9uZqLwqe(rz?B4IZ(WQkjaCnD< zneq$@S2X4__wHxDF)_nS*D@vvC1qhhrEQ$%o4t|ilA%lc+=hj)2ZKNd6&rPixUOD8 z07Rg^VDg%1QkyNxn~(y10OCffd-$rm>i8bVc<^hF9tF`oo%Uoweh_XpH#F`|o>wT8 zA{aGaI{Wu4v$EJn-$bRxIN33 zn-z#!yHM6N1Q_`xBhI;Y`O;e#p>>nAyCRI&0EpJDjmFymdU_!N>=l*!BxxXf9apFK zN%bVWKsjE@j!5FxcJx01L1s2#BSD^N2j#>ukJX(D*7P%gY5U3e8}ak>x6VFp`*4_i z+F?IaTEpk}AHChRcS{0J&Me-7#gfLG-HIjDPYT{=!TZcC(&S?u~|!vpb%uN@7}yR*=`9JXPaf# zl~L;7nm~z*G`6Q?gSw)xr82v%m)cjZzdSv!H|au^9qPtF=Prn{)ePG#K@NR_v>ZXj z#!uw#+NF){D7PCWl0y?wZQ z>7bjhEB}soEO}^Q`oDVgGC7>T`?#mlz*ZKh%%URDMIQ=um;TY%rmUW;Ro<66z}KQr|&8trH6+dgI?K{#TiU2Bj+kUkM*-n>;$+wCf*U4F-L{Ib#oeSyKX z3(Bsr0y=#_Aavb9rxFusrMEMDN2&zMQC}b@Oy&ViXl|SRFP`jZd7WB=69*=}S3j^m zgV44BIR7**tU#;0n%^|E2``W-4FO)cdCo!pE$EML3qbA_I#x9H_i3LWNL`KG+~!(Q zp!ud`EsX_{yD=HInPqmmco3`ts`v%g=4ZQ)DSeSJ+Fg^8<)(4#7X%BlWz(0LiBb+* zVCRR3ca9d@3wrZ>Jm?o=K`a&VJ5PZh32LX>KicIiNOk2O6fe>DC|}VCM81BGeB_I56S8_>OQi{HW{^h@JtGp6&VT8!vN}A#12f>kFQ;=ke9h+K-q$# z+G}8jfC31%h>uR?r8L*G5mny_wzAXw)4TtxPscq~ze9KD!t`Ha%vk}idHoM&>CDD~ zh0#HSIL&LlLobW{1yr>ZE?c|Ln`Q|vX!V$2kjVx?q~oNA;uib6@Qc(Dwsqf~0D%{= zJaL%jW231XkE~mhrCvpH9M|gs(9UiuzxLXn+GFhTVmq&OYBz*LcXvH*cf(A4of$Lh9Y-&@R3zOa!EZIK4HQs?}mV&&W+P(ZQUU_^Mb<*t>3oQ_T zIrUh#@AE|-`4q#>N@;}6dqL@O5Nqhh)zVzwfv#5U%HG%7NzN% z4hE|^{61mGk2_h_vtPcqPR+Ty7NK3ocLlj-lkPtUVGp~ zQRGDX9rp6c%9RQIK;NG53Xb>OmtF3dCW?|4wDGBsKy`X_09!JeG z0erEzGz${E=zA5ZgwAtJRqNh;A?I;lC3HeLsc)z&afnbPqQwxK>c)@#bLlEXoVCt7 z8aJm$bDyQ&>Hfnl*ZPK9+tzI{-o^k*p-bV#?mrHzA4Q*0)LL?ZFbH6}4&idy!(QS` zaux4gRR`{Jwi3Ks$kwr6cKaa+Z!gVZ=KVAD<8c&7E6u4`*@cdi zJWLw`QflV3u0X~q%>b7CFA|oQ70sP1cM7lCj>>)Zxe79mFI|8#5<&bu15-U6ubchg z&S#RGv=SLgybE9{u&UXa&)lstXK`G&(rbMm&9)K2QOLONh%JQI01O-BRCON zrog&o`Qed#O$#oUloELx2taF`=_(}Q*1v$qXKa~l?OG~1F$ zcTi4pFGT{z?FOY@q!z&xU*3*jbR?jqx;M*y+I&7+&pGop`MD7a1d-n+DgmN-fb z@T{cbO8_~e8G!?h`zq9JX4jDl=a@jowj%#;fErXlBFm3X^ZenxGX=XXah4%o71|)3 z18%^T!bO+@!S_Z5p=3V`w+*f>*4fLthvCg|xxzU5Ec_Dycb)zW%GUl!y^{~g1Ufjd zKoB)OB9wMbbTrbNhrpoMulePgC;I3qNWzGL&0!$9+4Lszr7mVp(6HC80X4y%REU_o z9U^}Ua0pLE(DM#`SwZYtA3Bo>3VS!E)2F|liN6J~MFPu*MSZt>$Lw`bFg9djaX_Y- zIR-WeX5H!fCrtiN8@&1F^)!Y4NZ%HKh#bRwrofpRVVUQrPRS%#D5o$Id>oxBQ5fdT znUWrV&UBjRc?Lu7f^{BdX}zfK*pEiYhiy!qAmr_uC|8I}_U5oW{Z#lH%Pq5SOpKdg ziWTVG5hv7*=?(+jO>uaWp^<@OLtF9h;`%_b8T?%CH$F6folA*ohDdcwtYnlrb{G{a z`)Yw1S5*13vQP&H_fem!57$)W+0`@*Bd@CLR1KtdJ(lhGQ{YnsoGlko(??H>Vc7hW z4lmZW6u2GiD7a6%8JIyy5IuOAUZInoydx-ar>TuqXaS{VC?RQ+M-|t}Z*|$KEFyc! z9@d_9{lyl-X^Lf=nq1dzjg!3C-`xyj9<=hP);SQ#Jb116^y?SSEa#@5-mcldpd}XL zeD}n@%)*FrOi%l=V%)xmlVcTrQO^I7W=%LMM7{YpFz3#kp+YDdG9pu)@DkLqi2E~ zk$(D5sS}L!K*V^y)m5;{p+8pn5lJ9Y)1WonU|+G};FRThns$}sNzC=j%!8i>YDJkF z@f{nvK0}KntQ1k3XUZ^mQyRj&YL;od>xm_d7X3vb7<n@tWM>TO=Sx$4o;Ts0!m(DHxk|WWfEzQKW=%x3TYy}zx17Zg2l}JHl|%cq^HS!W z-q@Q-R_hnClW)F0vEKT5v>jKBGHzk*w=?IBkZSGueb@h-?Io_TE{So3X9u|v32kuu zHjv8OU0xP55}Sl%KXOWVQY=ON!% ziJbK#m6_D}c(ny&=9f3pvZ5(`C!{QtbGgJeDU~3LBx>z4TghCgA!d`3h#CoTrDGg) z9dER5^r#a-WpM2*cCec|l5@o@J2iz03dggpnHGlow*2ScP!75TSepT1h6>8=g$(#} zxwS9F$VW30HunV$&3WyL1-pg^d9=ebbGxBUul3wAFC-;!U3L$Hf!eeht-B#FZc_s+IzZ$YXc9 zS6a@TDtsrR>8fqrkcW|2L~F9jKGt__A{}^JU9q9@f83z-_&H^NRb`}0eVUWju)9s( ze4O>T|iU$cyrO{sM}bynqX) z8<^|lTd+3^f?+x7B4LB=UT<5SDNdWW%;w9iq#}@l6kIVpTU5eE>!aLlfSbVrviiD9 zf{@t0VI-3tM?2|PxHx#hB+s>T{*`s@R8#U{ao#eNyLucwbF*9AobtEcsas;;bg2Dv zI^b2CY(?Bj=dzL+-yC0W%^)%DC(mRPEHc6Ip5L&^y7D4jkq>!}*E?k<_=Q31-srTt zX5je6cLPnYK~_q5GNYphRXpUCfIpA&PmiX1I0Lh?^}T`u8ODtW>N0 z&~E$VQ>nAq!b@Cfh;^fIObyed^IgQpQ7`{uF8Y80^bV(;`=P{fz15mGwx`>nmQDR& z7$`zr-Qi`J>);X+0N*{l#&u zmp0t+F2OO1(k>=DhHA5RO(U=_JLc905hQ#4k)eZ)A)p-&jupf1b{a(FV)EH3NLln` zl#NRhhv350<=ylnB`${pbJ7&muS$6G%>K;Kp6+qbJs89yZq=_JAMgaFrGWUIQ1EPD zOfApA_>7;Mo_D`=Cx@?@o0#Pj5?AflhT~DD=hz14HAd5h>&}s~Fs$D+gNa9N1$(0j zJJy1@Vq5;!KQ`wkUaii>kh7Pt{nbjnjYYef+qE60PN$Zn;t!Q+FLHMHYklxY$pU{a z9q$ni8-J>4NLT@!;(!Y)J(5p+z|-FIXaTqW3Ck>OCNf=`sQ%uCBl%m*X}$ds5cLEB zsNd;5ip}pBdX}i21w?bOZe<#oLlS2Cno6vBi1z}VRY|O)lowEy`tvuCvNa^>_wJIwpuzC~!|J@J5OHT^&2E3nj z`6Yk>dXD$|pfY&#_j~_p)A~5p)CZfb=XHAvufrd5WViCq({vuSFz%f%hY~Xx-d_8@ z@UJ?)H%ilxDeMFNmIcZN=U8@$e7rlX_iTZ$f$f>P6P>>s>*pRsbooFHf#E?7E%x#V zCyU5DapTaaP*fi#&p-HnLk zegT~1X-&g|1Lj;=k1x9BaUGRt2j;^lK`^@Ao)4>vWhK#7h17S4qYJ?a+b|(gjkkRy z`IeB}Ot{11Fq;_E5fpf?sMx~$r0c8X&0qbV_zF_dz7OsXmEegp9R_+`L1eK|phSXZ z%c$BJGXtET4SvqDrASN3WJyZ{QPJV`pEedB`Y-BI@3v5eq>s=yF0!Ek-s{S}M7kqLN=eF& zUzz^-wtDtl^u@NxOly%3JKmn?58ICkB9W)NIbk=Jy^!FlNfPeU;lYli?Pu`e+Ra!{OJY2 zuZcoN#uXpRCcjLY7F2GsyKxWiFx8Nx+-~rqCg%-AYs#T~wZnAG z?#rK&Z@b+h(BMbSUsvj;XdLY#L1V5Sp1q&T=x04LA+u+iNWv0^6Br5WEx5ZF!h!n) zWM2ubUy&MhM`K|bE;GTD)%ypm=fy42&NGqCJz@)`IMSvpU(&7$dyzlc29qA>RW>r9 zF^jQcN`al_@=Mvt^MCQUuq37bR!T_zg=6%432XsAI!}CAOYSViF;DW}aH|hV{69j;!a9q>kB*Y^oVIjFD_)V5TklC-$?7Ij*4BlGf}5VFR;DMS?K^zFLP)`qN|Hmv zuORCk0$E&Wi2vq2TWFN({k=RyJ0RCv%-8($z8e{;}e9UB#U4MLY-8 zt19PPLwQaA?o+sdl836;Xb9~oo$TFjbY5$`b>HmR`69*fPzdxoyZYD&RWcVZmzK&I&XKdKhVfy*0=+r_XgP- zy>}Ab$3D2mHN8spv6!ZW{H)`YXJ9h!(x6_en6?-1Cej$GaB9HiV7H!CE<2>-PnZ&% zo@irfu~jJWaKU9S1k3REl?lVIU_A@jnN@lJ_Jt|i$Y5ojSd)T+^UCVt*@PR~+T9(b zSTDaw>0l!}P`Dkdv|CG_i0S^8An^ijTn`AH?G&FEkrEQ!9NJmLsQx|Xhf7@%$XOqR zs_kO%%;dcf9?X?@Jvo7yoWK-eAXMblu2Pb_2YgV`P$!umPE{&jmKdXn-7Ijwnw7@PV|6v zY4$wf967jyo2`fH`EEnxR1F7H@t;ps6ar`KG!$&e$w(R}VNBFfPm*;_YUi z3q}Yv#?RFEU|UF`hLpU3-NVPl-0!-`9XmKU9!OSR358fSZ!Jy>a@HGrqv$a~7p9Tzl`0ib zVX2`>XK9s;FP1)cJ1+1RcB&_1s!Z|C*Bn|%*EDR$O;N0QX}f07=ey4Z)FmoDHuHcJ zcTC)p^0kH9!p-|(mftk^iHv2q7E?^}-e9&$ifbZwQlwDmLM)qwZBOE%@@K9%5LjrF zVU#jcs1>p2p~H2QcM<2-LP6sTQ*44&ZSxCLWY$p&n*ErUaKEe#wqOBquJq07{BZ>H z;GJ2x?_TBxNK4~p!iF16dQm@oS?Es&y7$EM8p?94kQm5p^8H8KloqANNg&HDpKljv zM<9l~fxzjBO5%!^E2 zn+>pPU%gfJ?_N-6Qor~ z^2q6xQDlz9i@DDv-m~g)h$mLJX1JXu^E!!)PdglOwU)~r$Ig9j|IlrwhT-dT+qpa8 z--iE!qs}yyxGn2+U6prRYT7~m9EiT%IlP{nD5|iwG(ED`^nh67kz+lUkta^_n;*U} zUDx;w`|gS7AqyX3S0Km+0RjJ2yDp{!!hd3J#wd-Vpc{|(f?wldVdrC1%10}7bOPa&z%LkpUE9|d(VNC%F@3i zVI9fzM}jo53!5Ar$L{p0BeYJ2GfXH?v@ng^Z_W9 zwL({Uog&3f7z(^NCbd$;r9{sm74VEox*#uqNILm3hBObR` zn|kwVFVPb3CTs-{4^3<@I++xracvjI@k2?(c`UJN z7`Lh63TZ)8HZx@6X0CPUjJ3UuLpxC4$`jIu9+g4SfwKFhDF*Y3optHgiinZ-<5L9B zl^+-xV@7!{A`mJz)PG%$m9vC)3HgD_W z2#*neoeKfk+NSCglX47Gg8UWArIT*_aYmjo1!L1Bi!zdNC8{g>nhRQryZ=>`z&N{o zF{T&f!<*#8bJ$gFj6aV&xAuN<)m%%>DM!odZZ?;M#nOvXGY6c9Y>JFbC$8E6ot8HVuz(e%f)r5 z4nSQfUJ3e+tRD!O8dg{FdvwMjA*CO?D;mmq)n?-?-a3>$Fjnf>cf ziLRDr-iRl4USxVYF5VSw|HS-6bK6i$nd=1wtx^!q3WZ6&2jew{Yw*%fN97HyreW-4 zrNPv!VSsJ*lmA4L)fMtdTDXBa*xJYcbvs*tLL3sWG^?bQH-#l^9yVq4Meqk+nWoBD zayuZ-#HvHfT2`4RNAW4-b^7l(E|fzRh~M1k0yWAkNXZ%+i-rv#z3q_}R9AssT7Zgv z&k+BpGYlE5M3vR0SDE$)K~+$l>h-0Y<-GQVAy=W^JY;eGJ&rjm;190~Q;g81)3m-LBF+w zLr1cfA??F|$28?8f{TxK^Hyt6MBl{-OqbMuM(6;t(Tjg}GUJrs{B`DCbf0)Gy*2>| z9{TnI3PPnyyCmG!TJEpe+|>|bh_GdrR!@kL_U#9uLQ!!8NMpfM;{O>RXs52()ns>bibktcWP6ARr)!3J3&{E?q%Iq$xGj01@dRy#~CMBGN4M7AXOd8af1mQi4*Y zh87SIkQRCZgpj?0-p~EK`+d)Q?S0ONbM15TX|V{4HOHJ|j9GtwrlI+~*Nba_)-r1= zT)RB}huX6{XzsZOFsB#}pze1#iPRKd&iE)9&Dfxp0H0U+8;-`9D2 z$YBENwt)T6Ya31uel;`6DVN-?YxLwZv(66Y0{h2de)d6pT~&PXciu;C3t7ds&N5eC6!@(>>}Zau zGD(O@e(ERyfn9ki-6fyhVh@DwNy;%c3Gc#5G13zzHfX z$zL()9peW&2voV8mjc#|wnwM~j8a*5kZ_)E;q1MC>GSr>-M=5@tR+|>hyR+UZtaY+ z=Gjz#8aJE%##KmH%ZVzu>tD1|cNr;x!%+d4zHIb{TbazR5iH1zl>|>Wdu5$#6|ga^{Q9VJ0E<2B)-PusWtojH>274a9_+o}hSF zDmoH41ZxT$6UEVJf|Hp3ks}B%a85Sl1r|a62-^Y&sDJBRP6>QguSY%-zkE)dx-8q9 z;Pqot>QCSH-z@Xuh}X_2q5aFd|3Z0ojrqs*0zae$K#+`dpsx1ZHSNr39omh`#QE#%^p`Hc3x2D`v zNMg6TU)CUV<;>=MKD0;?JxW*}hSleI^sbLd9bG#tl`KFGHayvs4fn!5=H}7A7JVuD za&)F@nC$Z~2N5b57|<;Y`8{nsPN-j1O5ykq=zp3I+$4sNHbQ)bXYRQkAl|L}HlLL# zY!@6#&CL&ViUH7Lx!`d1Ia`Ri%1K-OiJU6W@ygtfqgxn9ZCIO8zr`dnRo|%S%h?Xg z!>t4W%Xm5}VlB#t>%Zs1(o5y52!}DG&vw_90=<9>YpT&^MK?oO-qO05E0DyfZXPEe8Wv7fUo%Q)) zoB8=dW3o#8JlFa_LCoH%Om_QDyJfpGm&H4eNa}ke|5n=M*F1zyX#D8LDv|A6t_<); zC)NTiXyKF@PMYUI%km!*dbPlM74gp;zNg40V;1Kn4a1yY8{+oZE~Pv+s~a^`9!qG- zs8C9dvrn_$XK4jGT?WRJ#WA9zr*9SM*<=}f;$<>iM+O+10larL_a?5*FKjwjTX)YV#2=WTOcVfkB${33tI z!#;5D->1_*)gHb@*xzq~ct$;4Zx?hW?*0FKh}#AT#mb3q7G&-h_Vf!5n*b1%#)~Dv za0hsVeVpOq9udc&h#sifCZ|;;rhM)7ZlU3&Cbr2#D!6O#c=AQ_*uc?2HCX7Q>(S?< zF9Ob$F8ldUo1l4aHQaEhwQvM{dU9{i9AFCs?Duls`K`7TqvVIZhCjj zc_T>kI*!cRi2Ax>lzi-K#Q;E$=SQxFny@rnFf34s2L9QMM#s*cNtK33uYVLkz#Ap3 z7Hh>5gg}YF-kxOkQS1JRSOLW?^^yf@XsGjmuXp2nd+P8XMSB$xj)jgzj>V3yGq<7E z0%IlOv6*^=ZmD?l8ux+6DD(4|)OH9lfyI z5IKNs9|61@?=mVY;iZ&S-kLVp6%aDOxo1>XYk@8XT=sk_wle$JJQ zkC^X&a*XYyt#Gmj*=d4+>fSJ!rdm7b4>7}18-xkj_A^;9j+D7#^N<|~d=AB{z@8pi z!S@-}i&YBtQO?0KVgO_92~62Zm$j2tbJ%K}Fc?q=h-p0W`GRm=;tKw95m&1tdy$>l zPU#;mN#Q?S5}HF20#I(6xfW~9VSsNrbwh``eE=)#rwi48nv)5K0o!xz1N|ppyXOt> z&lPveD2C}Ocic5ZpLyacGJj+6M8|RJpb_|B{=T3-3p_OzzY8{T7VIA?IT+7TLXd!o zE>A$URlV&|lPj_isx?O3My%|v#CbWve(i%SPKwX~JQC{XkR49F=#03RTxufVu=_5m z^%ptUNE4#3*a`sxpSWuHP4QHMC&T&!7{;vsaX{R z%hA)8et$4CpOgN+NCU^w3H1qw@m`92ov6EMAn1(c807ab@*?GlLB4<2ujbzu+MvFWi=)#ZuyGfi^2#O@E9A87hA2Y15DkLfkXGdh9>H&@jM8p$Vrd z98f?17?h4mVWA?U|9^WsMDdEi0u=`3G>7;7>yN) z`D<9>T);`|p4-cpUMU_!+Fl`?Y03Dvn?O}so+E@F@{DM1p42_rVf69%a(=?WNz@rWU)Ry^o z8*fz!{-5&h|H;b#Q(@LRu2NeefS7u=)4)!XKt+xnHo}C{jiBF+7x?9E>LBsvDe4rx zAa#oP&*wYsKi>wf{{C!H;a`Wqb;KcF>d$@HKPLPWo~Xb0-+1(Y^NMG}Cku{P3jFgQ z{#VNN|Jon@ZKy*87Wnkvw*C7yKyn3k?*6ML{=RKO6$wC4F^7j__CL3u8&V_9ze@i1 zm;cuu{cn~0f6EyETh#}=+W*s*5#HQYz3)L>r;y&_kHG_dh^*W}dc?*_A3tJ>&%QRX z50Hux*`0ylGsv{cW8@8?jPHswL@q43nw8Rs>uLO@?5(`{f_ph*%XAV;ZrMbm0+#R1 z7v{j@a}`=|7~1F?dVLfb%&!{^8`Vv9Fm0n+v*Q?08z;pQ=t#T4sBF~6ij3#H^6Do% zvd7UQ5z(LkI~tylT>dVjF4G^0_nw;rA|#YbQj0R#nTUn&b>P#3NppBDY-27dyU|yj z*e9c$hx!>Cj0hlbA2<<(N_M^tzU+hqehbD2O_fpR?ONOY8g~`dgbcUJi@=YgtPSql z7__;j7cUXDvAb%)>(h9xRaAYzyjt<1l5bs?PKL{GoAghiNsYba{zQP3RG?E|Kq@Dq z>qxkQplnj5vhPFU&&h*>?^}}OomR$`jAsizUOM&5C=@>hPxMsYEK~DV4pR#&@&aoMOm$oer{Fl7d)9k*HghN<%%8cb_D@)|W?~}5 zyY4vJ;_Gt)0M^_XAYA6U&<&eI)u~Z70%8{LpIT8L!B^F2I(^NZDJ*iz*YU@UMLu>rq>08%N0-9X(4y`?Zr7mD0w;xhVDFB{Ax4xzKui zm@PU1UsSqB;BzYd@8|O83cAd@W_X)LBWaw+1D6^-xp%{tP}Tc`ODH4s_B%WU zwV#jcL9c~3g-7LbQ@T9r`0IakzK)0cfAeo`7s3h|iYQl;TF)C6eq$R8josUsJfA{fTQR zIEB2r6trFmIMMtH!6?PmQYMwgaDdjN zr%(Dp`V65XC8QQi zn8Lm5EBxE(dyinU{5Ro*4}YC(KIasu8b*s(hFunkDPdBv6Yo3CP9)jRblMp2= zV=sJdJ|9$9a{*P$<9VxZU+S87zA2OIM#ojg3oN;pm0E^bt0#76@ruDdPg6hJHth3j zD=^1xF;u_aOTd;^rIaN46<5G{$+|{ZCD?{F^TR@K#3Zc zqDHfbfEz!?^;33en@FPmou6C>)D{Z{;$YLRiVlO-i_q=uJ1=%61qRv9sFtpu;jaZ? z&xQ^99SXjSa|`vM2+yR6*)6gafxB$*ejc*gn!NF`jf3!2)mVP-G!EAbxJ^$(R* z+vlK(fdqQ^;>1pd@-J%M2w2X9Z$VjMw!{#PM#oAA{gbnzU6ui#4gb_zYpl7a`(WXo zHq5ub3vNMNwED8dn=53&bJO)n#Xan09{WW=Mr9@VTWSz=8|@XWGr+*)&-=V;IvQW_ z5wa-1ow?UXT$ur6Kn&motBj!%bN$dJidr^V9h>vy=9w=TQBE~7r}?pv?_2StjEeo= z%e&+(qnFNld9*4AiFZT3PqlNcIRuGD4?$S`-aOZJp5eh%^){o7$h(^;3SpW;9Byu) z`1kr0S7G{D{N_b{Q$H)^yE<{)*-l>`yHYARHx!ij}0br$EaYq2H$*D@n#LW(TQVR-xs@dZpq((%Ax9 zWq-AJ%IrBF&mFlXtZZX~SB!o;QLs`LK2UBY*<)MKn9bi7aTu0t`Eqq4v+M8h>oN9c&hxC~o=v?4NL7W>p26i^itOrs*c9cGJt_`q@Y> zYo~VQpaUkH#^iolcDQ@rOoop(@n@ov_4fB}s+UQm0g|pb8AVqiY=7!0dJkpFKq*rmhn#;>`(tn?qaU}>35K(Q(m*k&n zYSr=8N!Q)?ZlHF#s_Vk)5=bo8vbT7)JqC+^M6mN9!vsbmcO=&(2pBKD( zzCO?SMW0k}<*ryY>_c&S@l^2-piPTf@H`tJz-Rh~H1#Ps}Fu z>7du9aH8mhbO{E7I`6tLZ$&TX$?a-a3&_gK%Uw_3hiAihhId{{j0cUxhQR)FtH=yj zd`Cf?a(y8Cs8bx*y? z2lK0dpj>88y)DeEnigoqf$OLLYMwhx=h%yx57A5Tgr>4fJEShBr{mcP|p)Z+&p#-7SNHn8v?Od5Vk@*q(qnx@eTJPv&!b|eR3?kQ?hs`B^UhI+zEg~1?&E&M%Yd^4_V%h)v?&tP@U}Xz z)!tk8!-dT|+LVaP(yeuXiYK&riI)LVukxcbqF6wMH~QOIIC%-52w4xw4(OuxVwwXk zq`*CkR_Ywj*nym0u2I_THb$8_Y@daFj0KvwkWQQ-6&P4Zk?>s6fa_xnJ&)>g>#Drd;qH5YwrVUL1)dt7;MX{Q;bwG|ANgng;G3yV4@XXckx0dUZ0O0x)w z@?4`llCHw*e-$gfxiLkV34Iie`AI0jjW+CPQ6_tv%iw`PrlB10EBh}0#SWAw+Uu}H z;|G?*1{U+@f^cdbnC^e-qA>!Gbnu3;0Cru~=IvP%NgEIzMye!_$Y*_0_Z2uu2wSv=>nS<=k z+8WP2YR;YG3~X9zb29bvyC+1Tvf(ms7uR1~YYHRnWIolhzy@N7je9MqI{4m-hY{n~pyjO9*n zY!^g1crS%Ae9yM(#sLr^{H6FDkZuVv)$b{AK*pMOb|6@I4?l9Xfq=rqHo~~n?yevQ zF%hG~}=yQ9OSxAe3$3jl6v zxwAY(kNMglCKG30==ipvjcUdxO+!IcYV3vIgwq#fag5jSd}T#dL#~RalkQz+enjK= zJ+Z!T;oiB<`&Z3P=aZaG&GW;6hd2~2mG&&zI|tlz8+*I}7%}|{0wBQxe=_^4gZqVI z$c;>jSu~luPCpfsh0C9l+j=aKKv`}mu~1}wpW!>=ZFr`ozs>T z6cm5!fKvMleJ#CS?SuIzid-CiB&-n+m<&ZV8SqK~67k+$H{F(G$C~I`!++lAf$Hve z+lhX~13z1DgIlX7CR?@He}zc$NlHuJ=c0^(b28KKAd2Z`^-4qY2wp)&J-5)Z)H5w3(f>l_}cE#9HI$A;;9J zI?C-V!S8aOT#!5L0wU3xEb4=(aaAUK@2-f3WI0k1u8)f|sDRWJ3xFUAv*;klWR4%h z^$}m_h_)i)tTm{pSN%lA=Qpe{WMupW@cv2GB8W9-tm`#}*}sL0)Sq|AEvF4jFS8Wt%D5`vZ>E05Cvn{S_XY zMDQ^Up_d)dTPD_|uko>qE)k(P@k5ko+TzZ)%SHo6k5e9{kT)FR*W!ZuLMkC?ZTlzc zs=7yOOlB7!ECFvxr8Zl_vF?q~8^qhyTMa)p0lZ}#S8Ie>A(jnnPWZtSZoRB*m3j~^ zVx;s5#mll{D24qHI}Sbore1t{4qn2|6=?#N0U5cBy+w~$#Ve{0pwn#`9#5BU0}Hmv zadu9096khdc;o!?rqZkKj}-Qw_>qippDvMMBNXgsjL=HKR{42^n%h;@0K#63R8Jrb zK{aB0!L_1BX@=Q>cC~}4y=$r~d62zVYe6p_2}WP!kYQKlA}K|pvAWBYb}5Q2UavS% z-$w%^m+!1A;j+gD@L12-%Ph_y2gT<+Hm_*KT7wk3$}4WTemrFT8WhJ=9E{$o2#EGN z>%Q#^1Y6qj%Wf_DF~=6!W&rZ39M|Yh#GW8F_+dFDNaKcx1)Y%d4C^ld&(#YMnPZM^ z18b82ySH{yOn>t>_I*%E4gl`;zbjCM$S_&XRHY-&I}}pB-#cWm{_-&AkNd9eK=4@y{7A1) zS218KRdtde)~9ysmDw`YDg6MalpZCdw7Yex3Gc5&2M>~mfw?pMc^%C2d(nNsYJD$T zANXPZE<*){(RGuUkWg3c$LZ0RJIwa_<0Tmwm986B!Lhww%>0MLDoAC*X8ngk{&vM% zU?%U=Z8Uy`J=ZP)9Nt8chki{PdWaoQZd?2NDf96cS|9x8ma_d^?5NE2B+bf%4<-lO zRYyZUI8wobxdNQWUfaO9eltgNNB8v~Ycd0EIe;($^z33`Q;yzeP|2&(``4E2RCNrUGN$E`>V>(&j%tWLF|0g zeLnD@gKShVicG)M>lK!=`niwtw(XHd!}Gq4OK9ddWTm z;LS2c%PVuO*utg5+yRc?A6%^A;81OJm6qPElgu6eg4NarHqTZ6oUd=oTbQIZR?$9> zsE;!%L?i-mQjA?hXjjDESYMz&Gk=!kH6zS}O^@}2jJerqfS>oEV{Wl*OH!0oImFDo z7GP2~(#)62Z8EMrY`kk~p^k8WoMEJOnfKl-fSm-AHM`p5VAB+x*lCcW)@FOW1Y-Ab zJz};GmSQBJ^YAw}6_bkgHji3@(<1p#mi4fF$4BiN`{d2In_;7|zv;o*+h~cG51*8%8!{J==6f3DP$prTu zu|x_bZ6LVhHdJuYRigs|-*v(w2`Ij+T|KR)RB$XnJi8{GTF}wgUvIX5&K+QDR++wz zES@($B*5xH&;qQs3%m#$-RR5VIVKsTItqG^(n1}MtfBvUR0Jf%WS~g=xLqM*&CCS7(u~-X_NeA)%y5cDieifpTpo;m6P`V|BCuJ|DM@<(?EV zt*AH;Ak7XQ1T@LEnDn!+H;R>=hBmX8YND@y2{V^AuDT# zJH1GPL+8WC8}4?WkA1y!oB8;~d&keOpE`Zx^otjAFJ2$Teb?&m5}-7c!3|)_y*_vE zKd~{mZ%{PpRa8KVsFAoTaaB-4+|VY0ynifL48{Wwxr_>W%QrML~sf^1}6CUPlp`gLV(BA&eB>=@6)Bv_qM zT&CwI7CP3l_zJOlL3ze}c=JW8#xbOC$0ISs+EcL;d`8HKVM##&50-5W`_|EM4Mqx{ zy+E4M*e(0X{HNSnbHFb;a6Xt#f=!-Hl}&?d$~SE_Q2U|K&|LmjIz^3pI#0#DN2=C$ z9qFz++t#NSH*j7}R8_ui1vgc!ah}i6Vd)OJ zT<-E#t+1=%;8cG-n}(5+l^F;vdAX%e6@0(2uy5-^UHhw9o^+WnJXXn@=D?8GL&zsLHvn!IWQfV#)zSme8 zj;5@@Qhg#PtH}8{CHWtON#!!rW&FcgVzFKM_!9d5gYrj(T~IPo=rZh999>5ZIM8116(YPe3T%v6;LT}&fXG{1X{WV`_4Y?E&+Q+_=&f3Sn^r||4^2<_1OG=0VSZ?_nQ%`a_tOiMT$mlaZ}X0 z(pU{yQMW5S!Z* zJCV^HJQzo#OUU{GZ*v5pknA6N=XAd?%5bIb}{y#X}A;)PhB0Yd786oLGVXxTyb&jpsPmu$u6K; zdVF86)mvJPg|zgk3-Ht_}GL7YMV@+J!xg($Z%x1RilCI(yT3m;^4oSr)hQiEXVP z4&u4E^3oZF&&=6goHmCm6)IA=-7ph)DV2A7r=h2zOE|b}{bfh<`P=7GxpfNOl~v)~ zXGbTN8O}Vv$#ws)HYN%=5@(%g(BC9BGZmeBv&hkG79BI3uoQ2(wsyHJadTrD;u_!lCVif%6ny5(!BFFk3rpGh z{({PJ+`zd&c;xo((#+~)jq`=@)}n2`>7 z*CWG9+;Xc}m+`_-^8EneX;w46K;n?ln1`~(n-vf_+?qcVIy4?xy#$Dr) zGV;_MT8k)&rOci7kuMlK*vj_iC?$V$c22f1i$D{{goa?=qA0JI9O0fCpyIoEW|VF( zubG|t8e}W>0<>_19Bnm^nMkXReb{cj3{^(^I>&k=VQpf*W0vd6lINbS&Rjax@yUM5 zcO9eVG=5V!!BsuE)O^-d7ij@RxtGJ<6TZ>WGhqK>_bQ_GO>{Lb$ee^Yk;*U|r|#S| z-&S#_yY7x%Q|PW08Qgs3=9AqK&`1xbSA4jrnTNYU2Iu;btvBlfH3Tsok*$fjK$}{tii|f%$x!`P zWyM*2))jAUgGbw`%}%T(-l*>MKmW^Yp4zhA04&w&nS?A_+5q&-pa5u()Wr3!o z0qystgU-C zwi6)D;or&Rn5%rQm0Nt-0uOSF)4lpERCjDPfzC-8KhcPLvHNg_*@Iy2M{QUrl8)(U z+2_3tZ=)S%P>f$5-BO#XHv2-Cw_w9g*U|b&Lrvp^QG{Bph7x@O4$6%lT&-Q7Y?ql*t{L^KlT$=s}My;{8IcF>`~$g z(Q6l=pJ%SP6vNs!Qpr&MP{kB%%?FyU$qn6fvFGq7w@8*yoH7!*1zBi!{D^%Zt5KDt zQqVQN$V6A(xtCZ%oSIHH+?`vq#f_pd*C8RAy^Abck8`kJvNdmFA8r`>A9Lz zT9e5DxqPNih797*O?L~fr}U}^?Ek2ueUW7Dc8*0%#R<(X^3hVPr2~#o=x3V@gHW#9 z3ZEXs30_>C4Sn6_GP=K$L-Bzv1sf@_JGS^4p4_otgaPU>9v*2I9=pl73ZwvU$y*t9@d>F?koF=O5 zar4J@N-cZW;#M)N>H?pggWV<>aY?#U5tkAIDA$5Lp;wmqm$ZSJB{EL*4r$dV0XT!! zb*~37bLl`2>B1Qb1b^O-9Ycma@o|+EN zb9rlqoEFdmqTlBgh4oR!k>NKt(}M$$EFAOSre{@oETRlfk#6GJmM#X>#+?CJ_#R~) zA(co4G;Y3Sse>}O%H=Y1Ggb-c)SVPVO+OVztsWErt9J`Rmbb!S{zJV)j zrDGYsg_n;d>HUZ>A{}3VN4AfYa0;1b;7%@a<~TO+ z?7JU71#Q~fk(*xgn$9N+p6`f=aJ!{F=6w^|n`>vvQT?gR)i5&p*a2T5EUQ-EiULMV zH`G5u4tv9~d+B1{AaIjpa;f#xkd=(Zm--n9p_pB1%oqk3GyKlyn9$&3qGwyIQ9r!J zT2F1Rbfx2z^jG+tIs;p+Pq@?}eUT&rb4P@|tywXhhALC!i=ikRG(Ra-6AF{yAgv^~ z-HO@k94^ISiC=)el1eKS`K#jqLXlfcA6rvDS|8&(f3<&aMbBncUyyN{AYHt)ng$Fs z=9>af#me%<>T{Kj1(@(LvB+%iW3IOVE!4R_P;fu`8Y(^ScMD~*dg0uVWc->G>8dtl zi5ab`4i0yd7H#E6Z)FT!k>84AEjAthq<$09+!ID5MmOpx|D|#+GW5P^?%m0{8G1)v z{PFt*thB~VEKOHSSrdgor^0>Rq)tKUp=Y37RYzJq$i+)4np4}>Ri_)zgw>|9(oTJo zJ6bNS*k*b>bY)_Ywm-G>a}52o^9qRqVg630w7 zD>RrZCpzq0E`057tLN?pbEiTJRN~Q-#tZ*uslb<;xs(=?izX3vj-F{faXKFs90op-UjWgP1^&mG(=VtL24cE{#@>+6&! za6$`ZJ?_+Isqg7)agm`kk*~Z13f|~D#nk-z7K#;N8WQhP5ePBjI!cneCt7^c%E>Lw zsxx#z4*qhvzf^V;WaiPf2m9vcrD}j(=0Vbb>Uceww{$TzavoM&p_$fvG_b0k?UmcG zC`zln{-@ZF@|D?G#4!09p&Tumu}56cQ6%eod?pb`W%Fu2cFUf)>pn*#X$*yom+^rh zv#BZcIZqnRLitV96Y1!85I2`?+rip&b3(UK&$=&2z#Y$F*f8uEPK?FtXF&>8%Z{22 zlb?R-8N)0hYc+wvUcByd%Cq*6E`y!8Ww5Wag6n$64<2wf+g|)wRR)kZXC4K&hg43d z0Fn-u^^n_KX?pddfrYPkK3Q2}pDm9vcbcHO3TKl)A@Pqg!yP z!&SvQxpdxaJHq?OQatyF2d>Fbc-nFIHr868Yjoh~^-MpxF2`dlNA0_tgjgkwWs>G9 z{|Y=SzI4*UOt^`y8Qh#ipYN<>+q!#hn4E&vBgj}8^*qN(mEvG@iJZ~dFSau zLek=8?amGk&og)gO)y$=duW1N-WualwF<;WQLzd^fg~^ad1n40*}M-AI>#%T0*_)% zJH_f9tE8+}>OTirfsFRu4Zwymbdim+qQ5p#6=|++46S7j(pEfCQc*N}5< zwiQF7>QTKDu)Fp3Je4jh_A7ZsqcRMIMXP7XC!WVOd8ODtS?|;R8gfykBF6;}ssxQr zg$*kS+UhgJi*JWN-VZY#l#8lCK;Mw-TOu-6vJ{Q0v7$O`)VP73LKnN8Chxpy353Z3>?i_o#rEQuUrsm z8WM~W1B_Dp#$?VT!*UDUBpNUxqYK)T@-Z3o?BcqBdQ(^EOOB`o1rIg7AiXl?NRs6P zL7z3+P);Jr*|nF1%K2K+zOMtgJIQSa)A?=55cy2Sp2LB#*9VrjXl!u7+MaTk<6riH z(;BvfQLeF`wLKjs8QX5=*N`VOLGRIZO?# zp+$+}|Acf}=X-V5r)%DG0l34eNp@ZR2> z57bdHHUQ$xR^$_BWK!T0Y4?YTS3HY4J`)Q$m86C%H>B2ubrYu3Xg_+(IM7rA8&WAF zTg&HEBqKW<9Ps}B8^i2NOY11*$q^B=hUu3*#?>DT-n(~d39)6ridK};D1G+rgc>Nx zA8DR7<=D|K1tcmO8(*aP^wkn!dS9f}Z$tEm69S7hhF+E37n<{zeBiLN5Qa49mkM~` zyAqyoaV%D!%C8lL6XxFR_6F7r^j*#lG&rR;u zO_aXPewJY!-TMxEbWGG}%GH$NfEyDowlfxV>FjLix|o%GQyxrxg%9*Qe4Tc6QK;IH z51}cM3^i^dRkL@#WtSA@v1c^E&S;Y_)+1zfAg8A;OekGwRH8 zr&lWLa+}!P?n%;1Y2rp$!sXcIdA~tLA>;ZAW^t>~bHe$l@*QlE-Zj0gXI*`-MK!kh z9&Hl2<(tdF`gj?Q&;+ACiec6@uYo?Uao|~Ux;D0-+euUy)SNonr1a(Jo9dgAMNO^Z zoaL<5cAxC;_*n2cb$4Z@`##=Jm|MGFtL?s&R6;v7uWWG%60S>s7LE$-#bYTcjH2j*n+Z4Iznd2vE=2EfUB;> z4QJp+oJP0iZMnO~z=R!QIfH%Ax~m+87ccdJWj#`_h9Qv+kx}n=8g3^ncC%Slg`0_` zeODKJW0mc}H?gR{qtwK>uhSIQiys!@IySBNNW;hdO>W{sPnwFcEx#n5va)sj%C0;b=LD^{gFF5q_e-OtLFWa;5A%Tv%3Gglvv> zZ$@ZFCUQ^w^h_>|^nNcJ?Y@2SOYIIQ={_#mIi8G>JB27l8DEfN1| zy3w1`ylCyDlY$-U!v5l6^eGC`aThVUParR&8jTrFK^>V6+i8YP#%3+L%;U5fbp5B$ zXDJx9W*jDf%)eedk1%=Su5^7uD4~9}QP?RM_A;8L|mW2w$0VHtbf}aTmbL zs=Sddnc4JRX*u4rSFhy5HR_Mc^O2{`WOPeYO?^+$;Qz?9bnW9YO88oE&U(iUOJLs` zhBH<5+cX04hE6ENP8~OHJXdKK&Vl>>J#}8o<)QbNBa~{tK$u0%s|ylVD*%3~v)zxr z?Io0vHDOKup+z91_eteSyWC36ofIkiaQ&TB^N!Nb$GZaV_`KBwYqGD)=kV0{F*aJ= zVl-}2Smg`1zfv#o!uLJX1e@@SXUF~`SACDVy!Z5~5f{a)`|V28AV*KpCee}ym2|5` z<;v6xKGugC8DUdddAQtII>%Vys?Y4Fx80_a#z61rB9UQY;YizLY@iFfaq~-TfZ2@A zXMA6M$%yK-d5=crna)M}NW{bB(xkRyfqc&X?IE+a()C6rp6guK>vhs+IiBBOOq=9c z-G`*ENuROMp|jYAIDGX72DOHp9VP;ykm47GU)WQx4YQ?o?}iDB=B`P7WzNTigx=Igh)^v*6jOT{ z+9O`FJ1$w~ap(CGNg*oc9@X@lO%^kW$?Tm*f;LSXkug)_qZ3Rc&@c6zpv^?S?<|A^ z7cSU>nBN6Lq?aci7s-|`Q+E?g^uf62=dih zR*5*v{n1VCJz9H<<#SDSnqS4Y5dR@2IrbLM@muXWKeLRMNr{;#Xi;EF$QAzOVdjm@ zNw}Bhr%1=~dM5P#GU)7MPVU>VDn^CPhP&Q8jx0L(xlg$#%pJdLJ67cW$Wmbltn1Tm z!MtnDg^HhTN=*aScsO!xYQH77RcoMru78?=3M(x0z?^*s#gt`B$R8 zJbrE9W^9{T3YaNjb7D;Zd;^#~@>s$#3D-UA37cV9)ob&=kkCj+nxkJl&;sk+6cH== z!CHRmpsHlyY^&olUvjl8>Oq%5y{OTh{-n}#u`{{KW++oG8={z`Bap6_uTuCxT9-9^ zbt)D=T~02ISEh5!2&{k$Pb3B`-N>NR=w|`Vu<~NS1)o{_))%b`&u11J7n6p?M%lZb z){FQDoBVrLR$yJR{N*Q83l%j4&qUL|kkp6XPqS_6tJr3da=+VrKB~c&$@I;Q1);dV zjP^%E;ha`!A8?Ol4tl)P@1+i(5EQ8zJ~3_2V(BCcYil_NWanyWrmCt`&q%FU+|L#u z-h`}NA8{%Fl`>zgq3ZWNL~%}IJGJy23y1v^6T7c-Nl8G%Hi1f2eOGT~F|CDI)m?-F z`GCE^oIc&m=`7r7O6dp(B%5U+gNuZcPWj~MWbnEe-5LpIN^ih&+%j$)Wb!ywaJ8>pXdO`&a7^E=miCa?`B zL9Z=Srbiv%Z=w}1a-_1}_Dx~Qnee_-E(w}sFGSWxMAL0rqL0RL&ppnsc6(D?`)#-h z7|k>fYfN=Tw=v@*Y4Co_ijQ{?g9eFgEG5z5=gvrt_n3One+F}G22Gf{e#m4e75edk zuo7z}(c3LO-lvmdZ_v+uFv8SJu9%{RS3h^?MCg3UiXDFHtz~z^io5pUs2-R7(6gr| zF%HY7{~miTL3?AKItfQ!ta0O$wDQyIZd48B1F>GV;|yCjpYE@bRJ0}M*lXU38GdoM zn8!X^?~y~(MzwM6t(P0$MPOsne4IjKV=?k5n$Vfjv7fK$4ME%#f49v?K;PE?g0yfz ze{02nR*a9{@%ZtdzSGeyphhj@Phx0u_IBuF3pLKuPlZ@mYisPks8}x!23$A${!`_8 zd&NR#4VYc&qC_-ZN1kQruw%cf8~2Lio6LLSR;=4zErb+}Fm$o#ONY9rjNgD0oxhYj zDa(n1M=-K;3D797rrKNcdC1z@^keeIEJ?ehj(0c`l}TqU{Lb zJfUZdj~V}X8MgcefOS$w8g0Lq+$?)Tc8{0fK9kCffI&20ezovs!%`zoI<4GS%!-SU z&R1r8Wh`)2-CP&-CpRTFrEkn;1D z&W#yo8cirFd&OzF;)THJ<_XtGdoG$A2FgLPbZd#I8s}=NrK1Q%Y-&(B`+^ z3D}HjL}`x2)!d8_H+F%$=f17Ja^#s=E&T;Mp=B9Xi&O2}(~7-O4AoY0XG~HjjE0Rw z-i6*5B60r!J6RN$Qic;E&1on^Yulm}7EecUtCg-JpO3S>|2(d~31Xfoy@ibHMG=&r34p9_AK(nz6XKqN_RK27N&n=<5+ufqw7zKRA2quqOYvZ`=YwQV@n9 zA_hozNlBR?DAFZEx|wu|G$T|}QdB@BCpj1)u%W;hCEd79VB}x}Mm!h#`F!vDevaSs zJD%hDhl67}uJe87>#X;cbaV^<9l&S__|-Z$HRm@GTACameBNBpO+&>Svxqu?PrmKQ z{aj$Oq-@Z(*Q8|}Co%oX&ME&wo5~xHBlaH=$Uo*$6|bE4@CR_yz3yy!S*!5~%Jh$W zG(<)eL}eH;@wWMUr{~9E_|>%29l2Q7sK8_?g!?JWLt*I-W&2Vqo0YK}Njx(l7J-SO z*w{cIpV7L2W8FWM=6mqyU8SDW#AtfSdq*AP7;GQ~rJ%FJ`8>{#Vv#I9ERGW=9RmT0 z2nrV`h;&Fprv9`A{XDJA5fPX$>|#>FWEX+9IumQXG45yRmC&qEVG^;-pb+J>7 zsyr8x+o7#hi$)9!dm!YgbD3z+M($j?mMy2mhFK^02;ukl62C@s@(>iWDQ%J~67Suz z(*Jky9w+`N`F)&xMki3(&8BthfQvJm$-g8p$k%ly1h48K*x0YZXkIQnhZi?QKXp+% zHKQ%&>YH-fQ!VwtsO{%(6BgQ3@Bh7NadJa$pwUf` zW;|K$%qX?1`d)R?zd)P;0OAc)Wm;@K#;+ppibeU+WlG<-nUI{J1k2!%?AE-fPwbvH z6%5at$)7}tw=;~gEL&{;0ACZQcNqTFWS6+(d>otxfq< zXA9LrZH1@N)mM8v@b;r7V;VBg8KPBYG2Fd9j4&xD8(q8Yx2r)*dDY-|=?qzD=au~r zh47WmK>bTx7Y8DOmllgMPfxcP1?Gl!Yl%TBPD5{Id}?h(5jih49(GRqyxH?&la8Qu zUZ%VysJmD5mtGl-%f+ zv}XJ!IwHJ0Lewd9^@KIkz>VBW?yikyd-8C#;RmgI zT!Z$6sOMEQt=#&&-3@}&66(;*p*di}?-b1aOSG%`)g&>CBe>7T(raDiP{~zxi+u*^ zSuCr&+MsAkldtKo2(Og*DfA7t%%b$@)Hr-Tq3X324HP%w(8&mZ4K5?b=3`*HAYm?S~ed{P2z)RaWj0bFpbb>^sUpNhqITc^>Dff`=q1 zKMYdL&BeiL8DGS?H{A7aDz?}Fzg8nUK4+IEca&x#Y`FeIi%zFMqb;~S#NSiE+WOSQ zm@9lof^I>w=88nZE+^}R1AFIy5r^V+nM6X{x$mxA9VY{O;NHCtj!oF9?TNHak<3X- z*P~4cy@BEe@=Wdoc^7@PnVmWrY*r3ZtbW3`)3@U0{^Egqu2KjYuQ-7$F$!8&NcODz z-3I30U>CXXqLyrDzf&z_wK%e!;^K0i9jpy}x3JB0 z=i$!voSdYs1MVQ_UoRc=zLco}veUX~85$LbMoL84OMvREd`?(Eyqw!u2vs01))v3> zjdE@9mlT;TguKrY4Ql|q_nUm3(Pg&-)4*I+VAc%~8Z6S8&+^-H(dAL8< zp%+rbU@Q=w!R)c`7=~Zp^k1=K0BY!LFxH9$yvSsG0B%-zSeQ^^L#twMDa^LYMe2q= zjJ1ky#ZPzR?OE{Q`Krk`v-sXq;Zp{-!$p5Ow0%&9UuJg>?RrZ=rb|$UlmAUO2D~A6 zoy@DNGDpCXRJOjUT+_2;_%2R8dMeFPgEJjQMJt@xGjQQ~j>hF~>3OjDZJt1h6bs{% zzd>T67buwK5C)rAsE&;=bH)5Cw?EaW$T*cZ@CElzMa{bk2?PeZ7RStnA22(CI>05L zwhp|+lbAp)&aro=UC`%rY_-WxXF#s}+Y*T?SnSHFVJ?i>6t^Gc9|0(QHk8_jHq#MY&~jh zHm4d>P;==8+aQACC`keB!5j0uPf7rs6$zMUS~y^w>7}%plJ5vj=fA`4t13`iJr*hb zwwyzyz+}oF=PQhC{;sr@85y}uWqFBhdM15(Ia!HQv4WJZN@w^0OET zndE3Y!iU)R@8+z0UlJ_x+gA+b6yLn^_Fr+ng+R_anN>pt>`)|=OY6^n2n>okZAo18 zRJtOKCCo?btO+&VzHPVR>Y?lMQDsG{y7VMnQ8q!hASUraXb)8v+= z?z|0lt-J{oV2?XU=J#}n3Y1S=xp74Bm504D!{>A{r!wHfeJhlwt|F#LXVTZ%ZTobv z9YGI*AE2-W@(jorks@|P#rZFLiMo*A%^o@#;)Gsjd5U%Z4v8oL5Xzs1%y5Q$ z$C93Hnrlv_7g{x6BGDdrdJ;btm^NAx=R8vFf%xNKYr%Fdy*rpUH;S~K+i)C^d;;hi zc*~z?^dR3e9)cD|8-3wpm6d*`=H!aX&5E_q-ITsFrqqv zlQ)x8XB$61Zt$S74jf_&8JxN9ZNie#Z zi29X5*HKdcJ_}Oiz;CG##~H22N6@$!?Zd*@;O5m98KokpAnq?Q##nblgL}ewMzrQ@ z@}Wv6+d{AE;RlP3SJPe_-Fyob)Fv9zHF}P=p(8)L`C(K#H-ok-gI`^bFQ|W?X&Pax z?W86Zbc=5iq-(AR_*I6s(Rumf#+I#0w8DkF2xwDc%2!BnipjFT{s3#mRB-N>vil`| zXiIj)q41mFOvq<=)j>RV^7N59FV|8OBd5BC1RQiZ3bE6NDGrQqXLt0y*=mZQX6li2t zevWqEdUgOCxuXB+4lDcdAR;iF=;A8YxPe1e8Pw_%~HzacQf@jb6I zxT_Q>@k@dEExATMtcNr@i|`tKS|;8#(Wt@<#+$R!ws3?sOvt?BrkwP?7;6A=LHqJYsaKg;*g@weZ zyArYthhIOOb2SqbKKJZo;o5ww^n4pen(n^FHnNm*;VYem0IumaB95-1{&7&J?NL1` z9rJvCJ2bCp`RJk(e{|x-i8LFmi{j(!!;jh|b_tpL(aBg9)K!pB8`t$$!Cx1Hfmd{$ zNOZTv2|aHaTYR(>-n^bN&uSH*OLXs4E>a*2ZLmL(z?Ee@Vr79w!;DVqf5%_7>8ZXh zz?!C;_d%Fq<}`iycsL>fX~H^;ittpT9zq}Gk&~G!ps*$9wy14X3Rn6-ei7omh4{g* z^qI!mqGeh<1hI-<*t=_Tn*9a3pWrP;e(1rBe@uF)OG;oRN?Jci9R&SRhSBK3km+YA zu+l{Jq-6AdDFpGvlK#1kL(DvpfMm2bk5bPeN?R54CKf>R2pQoU-Ewz>50R@i-;#YT z2p`M6o*_0lC$yHJ&m>sWwkHlMsa=--+HuG3h1QS=3VYnB0u*$1?CTRxL%3(Yc$$2< zILgaJPHZ(p${tE|x4-PR231GwDPYL=E~;HesJ6;Vb9(SdiRs%p0QKvH(by4`l>R&s zmZd$3(%pwpc0JMJtRsRwOsTB;UlJwZZZ9!;>8cW&lEV-AzIGCZk{a|x$3@xXstNwR z-Bz;OOB4BfSDz&Q$1w*&K%(Q1E#xMHfsyvt-c_Y~!3%|K>>M!h^HO~FFnNhI!ygg0 zEs;<==H<)IVSa3&7x0&FKIA;pk;o69b4)al2G~4fCjN<7ldu;knM}sHW@< zx3wlq=p9IJ$gddJNLrb$Y`prNqLXy*LSh7vYllI=tLfD5TI~#YLoJBRod`DK`WCfS zKi#^S+Z_11w!Scnmjv4pDp!r(&G-AfgU`EASXug@_UKhqh|Bc&oc8lwW{_uDdSeGZ>4 z3&leAtl28Yx=-!7v**J7oC>XxCl8d;SjnI5@=ImOjOlQ+Nw z0%}aDM7&p!9|ep)TWQ3@4BDPQ%9gP2W$C7sadinJG^N-N&4)yinKhDd9`g#*7HV;xQC6RZ}lh)kfWN} z%ihvH3}@5DNBEcemKw1*tBu%8(q?(7lYh&p40FaeV*PD9m!Jh0V~dKJQyL=j+RYZX z@a}KyFIMW^;dud@E8%g``FcEm=}l)z)EiO1L~p>EVbNNtEt2sTm?)Tnn3n4qmF7vcwAI@R6q#8w^dY0;L^HrmDk{RRmp1jsEfKcmMu ze3KEsvtzcKFR!~vZ`RyZGi=0y-wG=qgVIrazhg|?-oZT;;!;@)13jAYn;8_$=3>P zo;@*V&}l*~oYH^RV&8i(FRNX(ff$SQKMFBhRlj73fsQPW@!Oot<1_YyUO@(8Ef8VS!P{agn#}6ZR50`26k@d={t?C(Yf_jiYV~9IZxv- zdh?dN?#4si+{JHV4Ph!RBdX5Wf0_On^G-zU_)Fzr+X!s5=WxoT2i*O^yg~Q#{xt~B zv2D6#qZ!wiL>RH8W8590n{kmsLt*O?>{XA0RcSrG8w8`L%%Qy*i z_i{;(jG1#Gh=dcikpiO>eNu;*F=8xxX-7#Tx%xLq|xgwgs}VpL4H{ z)^mGT^?Aan^kCL`@5#)5ZPfIKP>mpf{i1uQ#KB=_tDmF3-wL0a_H8@z8*AwNR!U7# z?mqC)ghAB5S4#6(5sdF=rnuR@9;|@J0;E&RHDBV@VKLpiE2TMm=d+QQ(zZ0 ziL}dtZ0TlqVu3}BLbrI{#3G)OUuM4{K-?GXZ! z>Oc;9tkVE}UrK@|>t2rpIgn?3Yb%5+wjH-w(1Q_Fs*THMSGdlbvaY05pI1bCcmBgz zil9r0@dR=}!yAcB)3EM&MtBA+A8lJp^5^_a3v~XQ zXV2+)fndiQTHpU{XcbG4PC^+;>%OMiQ-ty7 z61S|hx9l&1$7zT4SR3?iV0)c-SlUX$)RwlD5xg-vZ=FH>yA$qBUgUl5m4q3DE69BG z&{{PjbWKkbZxQ^Uv9IQ61LHa!u(3DezVN{WcrKJYuQXO)?g#u<=Q-FVBxQK_(Se18 zf^CXmT~D*O%^&VrYkQy1!l=PVpyOJ}XCd@rJo7@a8x`rQVq?oMZi*#_E@w8?`yD7m z{8`X?YBZLCL8$~H#Ga`%SC4w@>NP%V*z7Jm0oha0ok`yuRCIT!pzX_dz9U>Ak?W^S zd$jtX$cw1qFJXyDAa0g<*)(y8O`~*#Dte5$6}Fk!G^|TKCZEx7-;lX`S@Q;#v66}O zGsBOn5Y#SK(8lqScNsZU?l>pUqUhO~!=1F>qQj*Q+XQ?68k%8C*1uECVmpqxt`aEA zNX@-Em5zLg1fe{qzh$2JCLy-uU-+bm!1BC#tr%xl5xd0xo%Y5nBk&o}($uf>-bkhsm}Mdlvh{I>-F#_shwdF**bK=jvcxvH1_c zXI~%WFo@;Ne>lCq>CDdX%j4CI(_J%F(t=p+ZZiM#v9q;2Z7sf^BY=AMXCK&dE7Mv( zVjJ2RKkCunW2R|iDm+yT4=S9J+o z8=4T^rQm?6xRPKnm5o63Xb@u;+)((H!~7@uY);Gjgc`X)@Z!E1V1>wK-MB@8$O%1* z-KWBd$G=59K+X|Isa69$@}w&2T6-SA{U!pI+I4(G+`*r@H;X~FBk`l&H?*rfwV4J> z!4r?In?V?0TvVs(0ak5}Wa+OBEtsfwyin49=sMWAl*<>jcmYL&V_`=1N`JzAU;Zz4 zaDO{7m;NM7M#Uj@^wN{#J}urzw<{oU$3y(9 zx8^N;5iO17Itmi`UdxMuim4#|dY#fl;{cw5C7QvE+%Jtnc}sEa8GhJUMLf&dpl206 zJ=)z3yBmFEu^YCGda9vcuTyVY57-@ix{u9dkLQniLY-$XEz14s(tK~sA z`3s-IZF@hsIS}>=>cOt71%aV~6Q)|=f6*EH^ci0Xj8@~MWO|ufu^CL3ymw8T2v)~- z+>R&CHvx=E?WzJRSOPCk5uA$4+c0=V$0{&RYuNj|3~&St7QWHoncMUUqZzS_&BJx- z8v>)3z5?%3GVCf8VUq34-B|KDNqEmncA3U0oqs4Xg&X_B>KD4gokEz04-3)-4IT+L z>E;@^naA2amdnjV8Gp<0?~Do`mq>xi<#ogF2*>Rho?mn2G$a)(A#|R{kL04pm4MdRpxD zWHESCy#UyBv^+e_Y2gX#Sv*^7V@m@jE)*La?}=8tETjTbV;w>VcPPwLp~G)Q@LF2> zczKDopj^3TiD}(X*?;yWuLAp$*sI6$Qf*vwq6jTThl`-|h%G&h5h1L=@anWN%^-Nf z)E{dsF@uWSkQL+40gQ5TI*rxMhUhf_j7D4Cs26|pinX-&f7@GpJsf?OXV;lX8nhY{ zjo|icyNvF)@>_f!q;5W13AI1|Oa7J(Y;|?mh>0;o_STbvuHX8;UzAFoUIdo>?A#`^ z#Qg3n-VryP_8C99GAi&b3*&9U!>%U&cm*Rfo7w>zp}!^@bf8+BQ+J-rnLH{pQTUO` zwoQ}|SZ6Jml(aXfOTX;&JHSY5@axJ(XJ%6H#?*@U>0A3-fV*K`YhW$Jnx`YYN;BLS z+WfN8lOO%}?aYD)sr^N(PyO@3*k|+}O3z%L#nWrKgecVtU4D$WNw&Merc>3!!!#?`7C-e3eEzZ*Kg z7*m(ifvN3ru-i()T&EUxNhS<3GPTsv?%V4s0?mj(Kk=n)vGA@x-6)apRJ*9u& zrQECgQo|~2DRq<0nR2{Q`Sg14zrg876PymT^4Vlk4**Wjr)-747%N*;*gX*a57D*CJz&v()XW_{~XD+Rb73wTYNb@2DiQ>nv zpdVRMq(p|rdXi}y>qkTADZZRQAUGxq9>@9CER^yQx5CBfgumD~SF}4uth5IhgH@8XwoBvWW+_27I;UZIailIMu#ZNcfGALgO_uknl7SsL5ECyGwea3!;mb!mYpTBq3o5|k@;}7C6^lSkYFojvIb|T^vv37OcQM(NgzXg8ELM#%W1(M7b5&w)lni1F zu_btE{ALEj9KvtP9R5`JT46J6z>TGSa~4>bXv(p%kMr-~4@+#nGrkh#G#dn?{@QC0O}|Bd+s zw%WiiOi%pt8^G_}Z5j>%Q2WH6xvtLBA{75=!78YHV>e`10$&z9Z**GJ{8P-Mjt59d z@Di60+R(bOXz!Vb`^}K`Zs*M772w8aB%t=!=s#6Nwy5hvZ8(D_o-Fnd2g>T35?1yU zQ&*Ax^R*HMwfC{elyMZ00ZvLb)sN)V=uVJ1PFL)%H!Gq#>PGI|4F&GIDyp1n+&ke9 z47h;yNghT#WI+8jexkj=f(gk%)`*@s#X}EgB2S(&7~I>mc`Z=-Zf+B?h}o;cLeEH; z=&C(3W3l+3HLa%&AbjiuDrKB66_i}Zu(t_BSEmG@t+hr~h~EUhiW>UU#}Y9S1lRn_ z`2Z^F@QmDqydC^hr-h4fz@nx3O}FE3rgYU!1U_(Gz-}A-7ddtnl~ek`ue<259d1gC zdBZAWiq_hqex?#j7l3NQa+6rQ02PKK@u=>D)^E!Ab}^^>@3%d zA7;ceA!1;j#M5Ygu>;JHQ;Q`k%|?(vBZ}YoMDqGNe>;;DQLQL6Q+Yz%;(OX<3%XB8 zW}FP+F3c}`zhI`}K=6L9xTPJsE|ipt)8Rh~ZNzc%Oq3in8L#bK|0%Y=Dzx|dU>B8q zO_O3;6uKO)rbj(Xk)cm4y!M@&Ci-1ia5n(H^v+HP0iFyy%~+h~%n==QCYO2J`DqADPva0kkR25nZeU{#B&z zIaQV21fPEUiuUHpRh-`2Pe?Wu%#Qg2vB~Mzq_7jOnqWUOl*3|{j8O4AG#e2~Y(A6{ z%Z#^oghe!0L5p}0v##ismKNKxy6C&AVz3?4pLxv2d|MB|0(XgiY@YG_c>y1G_mae% zrUS-LCNB?m&yWeYK7GqR#0KJEa`~OKYLnw#p7&yX{Di!^?7hueOrNjKRifj@1y1_l z+0hG)qnkgO;432c2j*$=7MOAWfVwQirPaNInQ<*1hc&qn*D0M3*Y895gA#Y&q40q} zzr#(2)L%XQz{@V-1>YE%ah={?)ZmA@Q$l~k(dX?6Wg{2FUiRJnEwpu!vNnUc^=bD9 z_ek0HDxnYaP`KG}oBvE&EI@dJ!tVpQh6ADd%&4=d`waWPNzPg195U|5_PJZL5z3kE za)h#(;=GT2!w!(@rWLv^bW(`-imCh0u%0618+-Asu!W}_xdC6tc3`0|CM@25+Ka@9 zsdV!8V6={hk|X5;;MtxVNx)>LJps7Yi~QKi!nf+?pOabzC2shm@BwT;nvRc>@-d%X z;sG3IpDwp1zhIDztPgEZ7A$JMP4|)QT=1)AH6B|NOjkW0$f`K=HZi_U)1< zI^UkL4jMwauO1jcdJXbiEJFv5q~K9%JKO^!xiz}PG^q3%hpB^pb=g6g5l!&R=G%Es z?>nm;gezL5EuG){F2%9eDj_sX+)(jWi=DjiYVj=*uIZ(1`IBk-^POT?(frEktA@!iMlNY&kEa#&Na=@k%~@CCS~nylJ-t-6^)4 zgaZt*YBhiy$D~x`kLJ&tL97l^q}>nO9!@$JY%#e`yie#ebdZ{es!$18t99*1IV@}0 z6zzTz`#Q|G^`nn0dI@v2O<#9u#q@3U38xvi;SFp63imhuW;Ls<|Ljoo_M&hZh$(Hx z+0E32H2VLTXlYz6eDe0wu5$}_o<`8+4fy0PmzScIHBH%)P>e`{9MtsQYjrgEyvd&G;92r8m+FK6@nPo)bj`7tk@5R)pSG;~|74X` zT=YKHK1ZD@(~b-(-=oDu9C7BBf8C7d#8EiL5vV^CfrNCrP65{)s=BLv&tE+&f2A+V z;o(GW1!YII7SN;$wgW^5_?{~7l?Y?a!<~`9Org+CB<{Eo3Hil;XgekI_SWnUz7|EW zE^4j>po!H`a~!$MD(GDx+T!{ghlj8{-zBcZ^PNjIC&#H>BWl&NM1&;s?&#?ryJLzv z4UDjUDGRD?%&SQU@+-tnQ{-eo!3)E<_=jFRg!kh=ZUeH zVh(DN>!lgE`WOjkw+3PquCyZblfqK+(b`^Q>q4%KzdBYYz}54qLl%iPjoUR+w#wD- z7ynU#>x9;BlPn;|&qjN+CwV(0H`1e5otp!fQ_;s0x&D->@L>y8ji&trjGN`|8|4Y3 zt;@~UYoxlLGeO@TX4*7A#eh(iVx#;7F4B2QTrjFRuv2qxw4d66t~!>v`-Cb;z4V-} zE3mKNLu7LfjWu}XtoheRc;w%WxsYxdYE|XG%<3FU%Nt=#y`F-*F>{~|z$^uDomN|@ z;_4V$jc2MVxM}rqUZ^Lr_J*Hi!Bcs=ON!$sT*G8(CujjX-(Lx?tEgF;2OQjIjX#GF z+}KH;$eKCA%AuJ;c2{r@M?%YYhirlUn&S? zs0r78tG#*JHeWj>;g%j~NpouStozL8aNW_O1A$YQSQ8mKV{DKTbg+1jRpE{=l?S9X z=`1lNpA|9KgX8k))+;(y^-nt^FsW>g;DDXO-H0xUBA(x(g5drQ0{~bl z0PFhF70S+e!|yvj6v&8z9?Mf-qD!RhjIxOrsr=wN&Ts5f%{K000z|4IfrE&b%wV>$ zYV0>0Np*6-974pRfeW+-f_B<$CT1Q4BPjzoyi#QFHx=ud%A`>vf)vRj#vmNx#A`^U zMAeA>x-fjmQW9~jhGM{a@+HQ@J{@!(Pms%s7;gk~cW--=%FzB!;&LV?bUAq?S)jsi z=JM**>s#(?fj*Ew&iJr{I9_yTW_>KDf+5IEtAe4Uxw?X366QX7mmoyCNLl4(tv;fV z9g?_RU)J8v5?0*vkKOo`$IPrA<94{!%9A+}LLS6Lw=@*)4(p`LS{+W0p%sjRKPUjS zWiGU=lSm9~e%Lj$N5NW@LAsp;5|9x%Le^=;oEZ08iJ9p<FD6Cg7UEYD@Wh((r0t(i8Wyr3jQlKVfLj>vg?H4EIG#Y@!Kh3 zU&lwNS8sB8zb^->6^ctJ6+gmwC5kW22^}L-5a{Al>%tC!$Vy{wkdfss?7(A=%1=hrSPpl`??CU^0v^L^P`tg5H&E*ts+mJ&ue!28%k4KPgO!L6L z!`+;g$;Wae>&ro8xpWzx9b7)Tv)ms-g!44D{ESpR@FSvuSJ^M|x3o0uqOC%wm%iOS z8gK~?B95mWGmLq84snVYTd}|2<0JY_A%a;ZpW*~_`W|6n@x1LEZvb$oWO z`6Ne)u{3htG2N4{ogFWNsmJO|4%bKWmQtAjgoAhAt3mngB+kB86N+Fa&h8-$xrQw|!^z?zf!;&#^&azg0!Qdp%C8cvy=P+2gW( zuOu>bjyK{rU4?Ft(3x_PZ2fhpRz2ffw`M?NJK?=r_pVykdn~99P=sIqlWw@tL4GQ8 zH(=VZCF6EBI!|drrCv1h$tRoh2GCF48V-{ml3*veL=!vXfu%LN9{@{Yu+)$f-W~Bk zQ_5oeyW?smz0u>i;154U$JM3XR!)ijY7%`n5rHa0WizYyfUD(36c&;cRffc2r5m4daeTamL1Yvv~9+K+>bg5I`o4tuoyByScuOU(~bd zjBVKPW}GGRGXus2Fg_cDhzM$t3Ou4{emiT#HU64KMI>|g_q6Ur=h~?t~r< zuP%_?1K%sG5)=WS3043;PCLn2ZoEE!wM51oX3W=aAbg7BqrgI%?n0~lOFu@Y1he=+ zmxXtc=d9Eg+1To>6Ofe=`!wqR*g8xImudo^l zUo*eJ41V0&Ccs`_`BF=&i?iJPIU6{YCc%t@(fkBs!ns`XMkptnw7bTFAzg=hO#V0T zY`dGiCi3_`V=VmKhhis;fxK@2Q>_0v?5xra8-pRnioxOcSnrER1MiVL!`r?aeZe)x zc9&c5g|WFigB513&A=kMcgJxo_uqrvHI4mw`#4Bf9QJ-E@~^*&4Mv6OMp z*C-*-bND?Z(uh}6VJE2nOuj!9!$!|2$~ZvP80>91=m++CKPa*wW9|zTKQHQr-Q>8Ijp(HX zB6OMV5IJ*?9B-B>debb=lXvOLcD0~N=Z?}ukpZuAnP7)Yt9>-&SB6@*a|y0{;n>mi z%+|CIJ0vXTFOd6LVL~S;ivRUov{|0R;dYa%lrryaJAPcs z&GGl_rB*Flnxi_DNZyGyQaD_G@L!pOoHo$ zl}K7nc9Att6DhvN0oGI5^0yNQxFWA(T{}M|9A^Rs1;KXOi|pz@cT@3-%%9^GmgkQ^ ziLhsAETk2ludp4K{z``-#VXxHl^Ij-D9m}YFJ#||KZx&pS~Fr5Qz)njp~fy|3U_$j z4rd=xHK)_37Mb_B%zV%fD1lEyZq!fx%97I*rT?53l)wse`EB=@J`zyz%I-)j(H(Hu z*@`8OO@8)+Dj@n$!r-MkN3s?IEQTR!B?{POx>{-ln9Vu*x~ zW2qNmsX-`W{icM2zsh+>;IM{aLxYt8=X|`M<>CLNG{BEvulTJuG)tMgU%9bs@zmdo z*Sg4A?!(JLCu_RHrY?MHxW0yNK+j>aZL3Wt83+2;nV&e|2~l{766^$(=Umd?NgsI< zQ}`v<5TFs-_Z|tS!3$2yjW1sYKkr-BB>x1n#V9k-K@4@jl`|5^=Ew2G6u>^fFLAaS zfF98SMkdY3VDzcmgH64ZMS8e$op78ik{($ZtqI$Lqu|?XYXhGOfSKY0ofj054o@HZ46RZn`mU?cAy^ZjW4Q4f&h@ z14V%E*w4vsjv;|-X+VHs2E%9DzS`G=-)KdZFspB!#-sb{kqR=>ZvN9)u$u$0(u~UV zE2;jjOgAbgZ8#=Z&p7bxHc#Kv$WeY~9RuLd<*Dj12G?!-HWBbrYLO)U7+ZY5dr(l$ z$wtR^HGtX3|2VT_IMng=CDK|r#3TP5-O)}uU6-jQyLyag?8Ul#H4Z?9r@>r}>rF1u z8QyxIetNBmF&~>OwHvx6ADUk1Tu_e*SvQS>NK{xMeCDh;tL4d}^D+-pU_+}=%+da=4O&sRr3}8cN6B29A>A%1aog|y<<1-Nna#N#+#gOyGTwhZ z2LPsSD)w?mmGPptgtj`1{U`^`t4-bY+zX~*w^ls+Uz2(qDAj@q1|1s#(F?Z~8dhm} zHMwUIleT^HmlA_4p-vhA4L54deaxNPl)Xq?!%zMj8ah+B{Eerf_<+GUF)8I(@J?#S zjE;Fy`JTa(SIKs=ZMH|k?J*;RuKTx3BfOdUG05#$lwg+zygsV`sg{Mb!DCQ`qjaoE zQn@?e)s3(p~9^-NX|OB$%CJ`uVxqr(cQ5h=*5{pBjSd0 zlAFp_*|p50{XL;9U#pEH9rn*jO@uttCZf>MBm!STkvrP0F_PssKiIA?t^ zuy0NbM!I;*?dTc@-iQotX>(Z2vpmKRb8%EOB8GIl5gaU>W%bR!l*-O|E0+b3Pq(;) z>=yCVVR^ZCgW6~(Dz!yFK_qgp<67P^wi-6dCvQYBe-L%(o9FySDaufT%A0ExssT>} z8~o8kDzr4O`g*3dbA43Xa@_WXQsd_@s*L%Kw@~1h7UGpaZ21}q$cM!vILzS-U~Rn4 zF9)91P~TjrWB`@nH>j>SpV2UX+S%L(I0gdqArqFPu4+0IIO?_@xNkF{{}|8-!d8mX zWwKg!%1KtRfWOIDR3`J~HKl`EZ&b-o0O6THS-=wg==q%p;E7G1#3bp9ktmVzGeq8fXv>?`i*+}nQ6krmZMYRa zWhvz$(kc^1Mw7;$F_<0)GW;TctxQtz(d2PNla`pfvY+tc7xbUtDm@gjzC#R8DwDD- z`{N64^0Ko|3HO3TsTT^l<%s6=ooJr+ z+}C#}YfQ_JW~B@1S1s4!{}cdt2k_hOl-nbIK+bKh4k&7%bKmBzR$|NQN{8h7(6vzK zT1$v2v2P==Ldr^mm&Y>(1ccasoIj_NddJRhE#ERJtNexDW*ZTs7-XbeJlIdEoef$& z?D6weJ+pc#wKUn!wRPTw;Amhjw7w>&RMtT62O!s2x~2A~g_ZV0|Ub8O8?XqB`foCEq*udp0cqUO*A znVaS$>Lq$QgSO$76`p`#wqHV|)iw^&^rjN*y8C2I7TPKRb4$ObLXWYF zwVR3}_c!`wI;nrE2b>k3&9RF%dB_Hay`eRWvhe7U)^X%zENsx7xKxzy3H5qD zd5#xo32;FH1p4tpCVlbkO3dJ(|x#x92GY zvnno5Tojxtve29op+#r zW)|{BjW3K@o&Gd~X>ML4y=jkc#FYs*#HX0M^}OB&3W)+6D?L*+xmGvwnfv|AoZBtZ$!xsq-O5N zvM~*P8R-CrF9W~S6pS;5{LN!_=Y!QwL63Ig=d0o{Uu9_37lt{hU&ovAc6`vA2$o?m z&AtnelcX5gQ^Jf4?6aO4KafGr^$ znUvgfXicG&R->wfqR%`evwww01t%XyXC>u8f{8r`y15yp808%ZTE%^T?|HV~J&ntN zcLg$gb>VZ+7geKF>|+k~^M!Bd_UnsNj^p!S7^Fz%p~s^FSRC?JwzKDSLGc5dR62hp zCWr(R_WPCd>RVy>YuOJQ;?1(5;v*}PZjTd{<$y3v+wEn=%~_qju(9*JPn2Ev5L<57 z&BkMH?>8(;5F?ol(NP5dwLP#moJbCJo^F|&VAWt7Uy#Xjx7@sT@hlbKnWmStEobiSK7hzQ8l#)CUyAzPq^o$+?@kVCfX`!8MF6{ z=Wp9dT}r3)a^0k-6|p?I+}!^Cs{e0)bo1R%=bZk`YcAogt|mY&h~}{n)T1g!ONz6+ zzPNp!qvyorFKb?UIX}}KK=Um4PA3>-^8koAKOV1_S#s>ezPy%Ue-Q$pO+opyrVH%& zyBzd${s8P-fwOszqBvP4`6G-)+2@)AhCk7pW`jIcQq&a9F1IW+qOakb9dq8nf46PS z)@v$Px_kYZ-ph~WGmKuF*>A-CW(u~=v2GZ#wsNrCA9fzTSP~o|sh1|g1r*45oj%rP zX#qqkRN0+mFq$0waSV9R$&aZ0rO9Hy6aE$_2s(nDxC02-QYm^#36Ou)w~&jdOqSt{ zm+A{?d|K(ACB!=B?Z*?r-mbIzk;Hsurp-&=Av~KqTDLD&+qyJjozd;~&2~BdaF@1? zIeFwxrS3me`tzmbm4tfOLplP*-er8mOdh4+{O_7i5`24nhc6au(Aa(;*dylkllS!G zNNO>r97MbGboUVFKfZa!V3bJ(ATaIONqf+5Sp6fLROk~Wo+p0@H~*xrV44E38cB`! zyxCcg^gc=S{ZEj#qrMK{8k=b&v0Dn%BDj|C73lZKkfd5wUDcmTq>pr2K$Cm_%fRNwc= zK}^DIboy^YXb5rp>L&c?8yfusKhsC>>?>x45EUv8U5?w|Ia{8tZOy<%IxT_COG6}} zjix?y1`TB7(t`DsuZ~f9jQ=ZQg2e^YtFt-=qBIMaB9Z4gOLnsGB0wRjp!nRP5?|CS zV$0ymEeO60#PK-pFhA0Y6s=CIJ#>Mi7##SN|MvdfXkR4ioPhFpl?uL?_H7vXkK!|#Y>`q8yk6fc`1yFu< zAZ~j!p}e_9Jb*H?`TOpAWYA0Ma|vd>v=Pi#?*QD*)_|nqIeyLMhA-xNxT%{IxE%eg zL0_}yQFvE!{~*#wGceBDn(`M0Jt3ZojUjf=?8E)nipovx%~w6Y!_m;hW~YH7C?7#F z@!Mrqpd%^RA^d(WEQ_AAe0Zo8O|@zJjD~mbCFCqd%DnG=aU2zW<`;AA$aOi4TFgh7 z6X>z`BNpfhysylY2EVHj%r&nR)Ep+k)6AT8+3tN;U}UA3D%82I+y2J@=?o13j#Nt* zIayb2y+%n@-B~(ZmNM@?-fyQ}i6gPKxB8&}3_cX#s90c%a?_+v95;W6YtCah-WnuL zX~r(yw-RkQ?wtn41MR;?=HUGE>Zc><&YK6_Zs=j))dZOnKEdJuT7`V_lM8_|fT-8N zQ|u}JE;avKE3!uSB%^zr@oEYYrGA;iorq4QrsU z5ZSd@vz0w-WJ!q0zH4mR z*X;W;_H`IL-}9c4y7zuQzt8W#;qiXI&pEI2I+VM|8|G+d#@bD~ z(kWsewQjrG+}`@gd{fk=1%&KTLzWQ7s-EQp`4bAde>sEz5YS5Sx9Q=@mb;d=wUg%o9>>}DMuw1hH03@Za;lTJwY1&8HAMRQ#?Hk zKR>a!>&YkLpYpD?k_PzJJ42BaIQFKz%A%BXi@J^X)TrH|%roI%?$+BLh40R-HGI^O zw5VpBRS(P$6G)8yfhXC#HCB$V`?>W2D%gM&=Yj3sUYi|>=}evcSi9YslD?v`vx5;U z-Ji*z`T*?9BzVtJkT*Y>@={RCf?uipwD2UKmK#EQ@x41S6ad(?Y;@!-Fs8U|{mMqp z(#<2jP@`-`Tz<|Ici1Wu5S8(&fSKr8Ce~ljjCZYEXm}f5l}6o0&E1<+0XhBmupSK; z=otCmKfmG~?Kng=wl%yKB>Nm^BWEHE=3SO6ZaU6?i!sLA zjdrr_oF6AMb7h5h zils8UYaxAxU(4hSi1(F@yA_8KaKD|Aw>Pt8<`Ql6zk z;2D^effbWqey-DW2YqgshaQ9Z)vFaSv0^7dq08_zc|S}WTrc(?^)x#=|m;Z-LgLt z&b(=&r&R>jh*i~5I%oVWioEoaIi1Gc;}U^@2e#4IH299=ZlG;}j zSoUsGqpE8(DGh;ZdpS_4DVtTYN;^kHhANB%MwRlHzvRJ5rj7dLXt?~t(O{js+Hf0` z@9pqwFD1*md@ygmeV)>M=;>h@AaVef-w)p~YNN$7vFO@L>hRCQxm`x7Bj#7#&wGhI_( z3)?I+R5SMj{nrVzNZs9L(tu%(ius|;7e|jen-u^}6T`-(JZYo?s(i}x)qI1lMneg8 zlz0){n=`pZ%&+c|19jL((~?KI;C)m4yKDa?(8vkeV|}425a_geSG2)}8|X@$<#F7% zV>0U;#h*&!UUlkRGtx`G8Vl$%%e``TliDeN7R0fI#er1r+A#vEV%_I&0hJBPr&rG} zT9u1{%$h4T6aWWq+I0}_a$uv+eOo-StQ}e}0$`h(kJaQwiKi@HLxrWZ{rRzj)8T!5 z=ZOhI;Q}yZYR+fq&)~8AI=ix&gT_taVRlUtlc`v@+u9OFn#pFd>CFbSlAvhzl+$%i zLwx<=aJf9u*x@JUsHeE|AMbE+!aDQLB%x~=oE&Lq6(tl^w)Bq>by=L4lQyueC@0kS zrJ9uIQpTE9QBwIe_u{12z+i09G&$^6l{P&RVBEfF*HFsEjO2{n`MV{}t z={p6kYST1+U7EMi3{1z0D-JDgyE3I8_tZ}3+#&{9fYs1mze8narX;cF{@DMuQdHRU zCc9OGh+_ZE>3GFZ*(8_od3@f&&Wx(C%Nf@aY38qzYmPsXO_@KYdmIs_%3~>OC~t0l znox6!{>z=)C1!jTaj(#I7L5^4{hoiCF;eMs1i{~iTnQRrLyAS}>?;)zjPxc3rsvP- z5sC4yZD+! z7vtWPF`LZUmA=U0{{BJaAh9`A#6pCsYo|KJKxWke9Rqc+sIX&kYcM z9uBgYYoF?l_>q{taIo5T>Gjny6N<7adla3W3L3a+cf6cW7wYyBI}iYq$SQ6x3%i&N zY$d03oUw9Xv^2xTeM}F}8s;+&RzMtc@|MVP^5cDhzI%)n7v`z=N)GudnmnhvqMu*O z=x-405PXTg1C(=@8;Jhh9rxw*RT;=bG4$mvC!W@S;sYX=sIrX*KM#}@ZZ^;j_?e0z`tY%%J3J(=SNeT)9BfY%ODST8nj%EpHsb&Vu(?mG+xO+dl*IoV-VGC_3_G%ij?As z7US_s8h%qF!Zs6@*A@oOI+oJ>s!{4P_M~3x0HXB)u+&RW)T7CPHWUHryKy(%r=RlD z6&A2KK>v!&cp*JD%t9VV+q;E<$O5VwD_~eK^RpjJsItAD-#xQP9zj<+*CQK|7NFYgn$Qe7EMVk}bg&CB zPF^1N%8LS^OEc`*1sV4g*kK>e@K{HUU}0wgl{ECUlO|p7qC3_2kUM-7o>z7H*bMntP3caF-O3ExeAsJoV?H{~n^*kFwo-E>o4L<3 z=ja@qulhjwFPnBabMszMht=df?vC9^-?GF7Ja;s2l}X`xpHWDzcUW)iJxR308DW?- zoina_?&gn(7w(vN70*+7Msa7}M4W}i3#i(aAE60Oal*Gi5%N(lY1I^73#wjW5TU(QW3Cf$dskPbera3pvvm@g)A)(oIOsNu`p^oEf-A z=O##%YBiQ>Y_ioo-jGO{9$w!PKkRCAYwHbCxA4GSt_l0;*gZpi-66DaT-(NAE6odS zS|8UXf-5e&=+V);1x6FX2|1=NwGBxU&j!-_zlPqJbnrRkbJUOA=Y&u2DV`rZORgs- z6czNdevoXsR&gi*ZJ3;-GH3h(D)Rs-c@JFD!J(v{SyDix`sx-RnPGP7+D$QzxZNmW z$)^^$Nj~R^KUI#wA2`VY^m$7;l0pk2OU5*9ls*5D7u(lRD}tMSeusR>e0(L!g38T} z$}-9?p8hK;Oq}O$EA7Jy_fN-x$o%Q0w@@hrDNE~9CY6126fy_2+FmUtJWI8j!`BW& z0EO^ypz^Kw3<8(kY^1lmy31@D4;;4e^>OpvzM{4JYH9oa@qjLQYU<*!a zuAt|v06E-ZKk#0zz0rGPpzZVB&bH6!&OhoNKqDu5;F5n^{o5Bv?hOCgjWASAnO1nlb9tGCP$6x$U{CTdc1ww%W`bal3S53Hxnrj`w}F z?DedV;>}kLnR%JJ@CQb$GXI)nUcTtYHbc0$j`eiKCx^xq$=+F~b~QHB0tJnuG_+10 z5|3QwLDY(%sdZI+|7Kj{idGFq18YaBoqR&=;Lx#DO-!Xlgr{{z{V>`7F3*a~S2^{s zs3^hODho;Ljc9V*U~6y&b-$~cCE&8qjaLAlqOQ2k07n(gL>tbKzla4OUh3-nHX!y#^(ly+bNZW*J zIO!81@g9!L`mBAJ3p#*TBh3$FrR%3a(_F_iyY>1`1?=;>%K%65z7N&b#b0)L@UMt^ zu>sR>5p{TvLrWGHE7#^9PA?^pdmjK5r2oX7sJciYPoF_uUt{-T6Kr~R`k};@+9VaD z)M;Sxg5iTOzieJ9&jPz`zXc)x$gRX0G4nI7qu(sN>~#kZ42YUIt5vFb6F=L9OEj4a zXVTXn;EFC>CG>Fx+Js^+WQq3dzTb-Bo1$8w+%z=Q)xgweVL9E3N^BwJ;ASC5v(IoS zlV2_Xka;k%B~V}sl^oPKnxAm{M+yltZoe+>M7VS^?!=}m)JqzI&a394^|Vff9-VOC z<|;$uUi>6E=>J%pQfqQd(0vua5<|%S>3@cJ8w^GF)7t<;A;dO93&XFdIX>lE&^&^4 zK@Y}hcd7l_F`ur?9(1QAqP66t~RQ4PZPIGIjC(wJ--y*B+PA(6K36E zS*7WrHZ|&{HT4yWIm&Yf8E5aF!UjtN&N3vp>yA%6{&jR{Ba(7WjdHa2@*j^@?>h6* zG2ZW@nx(?aa~+>8J&+ITHuJ@M!&toutwAxXy+AhiVJoU#t;{BdQi&-#b~w0&=4|f? z;2qy$Xrd4M{4kv*aj^o@jiB(UTvX$ifEY*i`i2TbC^=O=QWAm3YPWOK(%|(D)OG4H z2IlS_lf3oY4dhhaJiI4jaG&AA4$Rh-@h%s23@>Dz?wvcD-rbyYsplERA_sCiB*mai zYTxxa_VmkdB;4rQ^)?rxa=|Xv^gRuc68{9}5}Iw&vX97sOYW@n>=m$kDZaE!RSv3n zv%zyy$EQCg)n#^P@p88?cZcS=r4i+^il6`pi4A9)49mAjFzMaBiQA|%f3iFLy-7)- z(x7;`9(&Y0&!iRJsuT+%d?rbu#k>61d*-WhD3g$%Br4m`gWEss3U`#av>!%d{8Cu( zdUq_>t_U9ws(1x#H$v|(6K8PTMb&x_mQB>`?xP*Oh1))TI}F$S{y8|xnc@eX-rAJB ze{t-OuC3vvG8gd_mCh30QqEr=`)bC-efom1QoVmH_?Uz~_?XA-`4Y;r1~tBQ3Gqd- z)7T|tb~#j~BP@=)Y98JpiFdXQn$5{bfOzDrT>vhVjJLzdi^1 z%2wXh_RQ}Cdu)OtlAIXkIn?gW9l^*`m zOwMOA^Suw$0I6*W^#S&9!|D4KtM3Kss-M8WPLA?Dko4(}`IM_lL%W5!#XldB>mt+y zy^ne}zQ&SU&oc8^Jyxm+cE_jg?hemmm#=0i8jE;J=v&}@xpy@8rNFi1pnF*gEbuKX zlw}iHU-GD5HG61|VQjea23X;#kDR4c`TEDzh3o9Msd9z<9jCSHgo5YCSHXt&`R zNc8Y}4Wn!wSCRG{jR9}|J#b{0r1a{yFefj2QU4UO9N9eElQ}Bu!d?lhvTMtm0RV18 z>o@`3PK4LRefl-;*4GZ+Q^tWj;-?K{EA6IU*rB(TjS1gf z$B&QA-jOw-L(dg|{Ri<(N{e!qTmNXaah7)qC7T&GZ4%&4=g!y1R*j$Jyuc+XY)1Rb z6qVZjhbj8?d$tJ%Jq)u*6*J2^)N}sPCyB7zI#nn%?Cc9pY1sAC0#V&g_@QgDY%K z^bT+q0S>S8irG1AGJcroi|GN}gD8Ao3IO7AP^&8k9I!#-1CyF*#@ ze)NWBq@hgdSsVNz9<8xk#?v z^ip+?2=8%eU$;n0%$Da>+78z=Bs0hw5SDOt*`bBnVI%~Ols3JWsWfIAA=Xx2gb?#g z2z=U$P*84X&oon;0O87?pIIyEeLf6f!#VZS;rdjhoi-i64|7W@cJT!K2b+8l)tpJBcWcNDVHVQ^GDWD@M5;)oG z?z(}tP4uus+fvK39Spcd&B>|ith5jGHukPIb1dkc@|#H5TLw1wZ^q_{-Osp>+Zu&* z*&MNRydt=FeDrds@xWX3*3E`G^~QD~S862gE#ph&?Rk)6Iw5ix| zaLeqYj44~&jbf4fM)Y%pA(PqIK)- z=kkv!08Z>J&3!rOFf;f(EcK)@#yu#wM0>knS-fZ+A)37Wu5m;A>Lu{JRV-Nq$KZve zfmoDW4J2Y-qi5(W4$#LeQBwPmXBYZdRc~_@Qzc&$E0~_IWy`n7C!7`C|C=rlJlhQ= zX!-!%>@CNd7X<#b(!(EqyV1G4e)=E7&4!BJP!lIN!1$@>e!tLx)pl_;()LNJ^asN&N9ABC^8w55 zUo^)~7+a-FVg$IWT7%7zsky$Pe*v)?GXKr8Xt_v$TAuq75g+Z6@E2E=G|gv#<_H(p zDYQsyFdRnb&8PK7*! z{wmidKlu~*418M3)LL?<>q0|LM7p7nsmLe0$#o|U8vN4MJgms91#nWoGkGh45)RD6 zl^#aqMi?yLqvG>1h(ZTF76orp`6529`dka`qK4dx)*D<=Pc=2&bA6#dJN>NUqRGS^1P#1VI9kvF`_v`FysJ7O(I>l5m==@Ol|!4( zNRBhrf|=vEpj|k;kFv&vUQt%oUrDY%BV=L`J-_=bR=#*%Egc~^OWYr0YYyZtQ*w9Q(TR2Xe2VAR)$ zYr{D?6y^-j6hfP`oni_AEB~CX*`CwvJM5B!*%PQ$C37j>#Vp3L@_xo0a~#PRHNc9` z&AD8S(56?kMQO)|nJ~`p`F*JB5CCT{)&ABQbPhZLr^7uM@WyMrvQ-Xj+S&M8a@k7z zsr)e$X73x}1{iFREZ`$>;vp=y=N($kbLruS{fl)`&Jws6ZQ&E8^8wgP=B3VG{}$qm zJ>KqiO&wJ7ZMAzKdXri=(nFa8fUz@lR!q_^^deABOsPR364#)v5^aff>{A_)$-I6TA8-#aq z3pLf{#-oag5ROLR!u!MKg$VV8DbXMkhANBRr!V8<74Fi<08gC7BOuTvI+wc7~!0^^nx2J#IYV%uKq!AofH z(gos65P%zd?NU#S5!xBOfrbUVSz1OdB~1M>6-nc6+gn-7q1be&_uGqXZ#E|6M)I}$ zc1-_%QM zam)aVu3?5QVq)AQek9C55y35?ph*huFRpmBw47GqxY!d-v`iLHDMLEsMX%kl8bt`& ztg^}uxXb4X{~j6F1hxbuEAConMuvHY8f0HSC|IDeI@L3i!7ae;JR=ym)T{=89=gBY zqnlkV0GSeuX@4V=3o}H_~?C- z5g`34>M~E_E|{!6UP~IeZD|J35GbckQ0HGb-4%caXqxC&*dEu zBKSyeL%AF{%a>L_OWgkhLUksGn?k_*d2zF;>N%GfA(D{RNYqa zyph^Zs~=mh(QGFJ>Qrdhorkns60WZh;A9qn&P@y6YfZi~hN!T>5Hh`>)9qNT(+XYDV`&C& zQ;;_RG3Y;tAzPM;qM;dgVI5nOtpemW`KSuK_zep*Bm@F*La!Ik7T%=ed>==3+t~(H zpv>*CW6&d3K}bH5BBAG?k3Cjgzn*U2pAmKm`Y6-|YneCpIYxMIGY1#p;=DCJw+BVo zSzJA#iWewxQU5AQXh3))O6^WM11*klX)%O(aJ97fx%__9J#m`;piT$+#e2%Oz`D0V zEh~{T+P<6=;t)wLE(7lp?9x;K25nDrM6WtJeTWo8w~J~P-p`ogoJ9kWG359 zOy#ESfL={#NpN{Mf-v>JmB!J5BPoqoD?8=)uLGG*Vu)F1PJDT{VLx_c!%b6JOFR}#gLbFMp|umBaJ2lcZ~0-ja>X+M)&yY;iT#y zA~V~qxxtnh_ey1e)j--ppyu;vXHBaabD@7z)APp-z?-Uv4hYG(9UBooI8dlhfZ72g zIPR~3OPEJ}<9a%e&T0V?Jxl0U1{)p0L} z1fb(&CU49YYj)%i7l2|D9Msz{XOrUM2kR~vZlEj9bn0Iua!|1u!gH1I{XYlq0;$He z-mi8w(wgjXnll^mkRe3WQcvv9vCw&96#zUKF^Num$6J^4X}fs>m>$A5V!*A2{etRZ z@?`KzB3~hgwa_xe4MM4WGcU*~#o1S-XbkT`YqI;fro}SzGRq|{)}k*6h%2Q$-+p$X zQ6-XHl~k<1fB*|6#MoY927{N@+MbS!B-vZY)Fd{SL2Zm>DelsvKaQ2|i;C^oqcicNXf6TbMO~Q>a)3ShP*@3K}D0&h`ZWt@1<=mtZMA$ii zqZ4^BZ*?QzPUEI9Si1ktA~{$t%(Hci6yir(AO@e zW_hz6O+P^+<=JCvKXYNO%``%(zZ4XpJF$6VX0q_LH-*#v6##WnH{|!#~otJFd`JOTl?RT7v^9k%CHG6!*cR%|lh+geY zN)ygUY4Df!6-jWgo6}l5Ohvn=LV(i!#psODg|yDd9HF>dI_t#!7u1G?wQO2eJOTy{xS_M}R8E`nx?pXF|4BR7B@( zfTFGrJQO+oH`z|-cZT_O+Wd^~$pEPdFccY0w(&y^d|71;W-YffMzHYJ<*V~(`^B=CN@`}@nMJoK zAvU`3%?fdlbo9LTV(AU9WncI6)C2-Ak)BAhr?K$>dDwepDVlckJ&GM&U#DxiRe;aa z-ay38RRWfrmu(SBbM|<&SwDbcOIzA(UfRkc@ES_XcA%CZVD*E+4UjJ2O094F=|;yL zC+NcIm@QOs^=gr`(+k?WHLBXtr$etIyQ{6#eTw z@lm`u0BD+Zo=i3FMMKGd(tG5PltjUz2R6Tt6^+;O=Y9azOmL9!&3+;Vt_cAcSb?c3 zziah+y5`mEW)NYWr8B|N00K$s!Jkc<#0^Z1Biq7V-)m>wbh$dvFcvUk4-!jxQ{Er0 zT3M;P^7^8$Km+m$sYGNLdzXS#IH}lXL!ixj)qpGoJ}g(&21n-nb8YZxTo)Dh+a->h zZi|t@gfw;C=l2-~50^*1jNI6+$L#gUhl8rjCz*r1or*nD8polV&&I~N3wR#$+VHy1 z-LHj>61Y?kvmr-J8drpwT(jP*v6Bx7k#hXABP;_pk@A;<>TV-vy&6!L)T`xIss}qs z(O|qcBTw^j8niYFh}y_%{DoAoa@0!&U<%YyK1V0m_8DJqxtVNBsZlc^D-|Q6b9B9c zAk=w$AjJqO5r6+8SJyX|SzA$D<{8lWYzKqnU8sr4uY36{%;RW)poK2}9D!YIfr;r%NzwEa5= z7ycMa2uXD2MWgO)lOZxSAhoJEclUsb?XYyC_P#T<0RTU!<3Kl0K&;5ho>m{dvi(tX z7RN}67cZV*-e4&wE((54^)3d5iLMh|vKls54%-&2qM5au>C>ssmijwSsCZubxZ-#T zlDtr91hv*)n}Y`bG~hcyQB_%uqsV%_*-`#;guRA?TMpmpYs>;H7)db$+19Vd_l6gW z%m8E&I7Wb-q`24;CO9FH2GuUQ54|?fZlk}~>Ur=;9B=I$Xv3lm!VZY)Jf^m3>>6HKF zQ|*LLVThkP6qX9iEy$8XWncb{+()*UoF5zCoO@nx{l9!B9(=}iNEC3^!|Xptt$~4I zq^JfgjXM468nAh{e&4(!JhA7Pk}jqNi0l%nO*U~}Pp4SC9h?n>>WmeTp&x|{=X!^Of z8+Ax44iwcAE5xONy`!NIF%d8)U{9|qUl6$##@+lhc|`D3Gy zVs2R|I>r+B`zHMPKBE;>&J(Pdsux!8=+8Ky)ks<7hD43uA8Dh(CLXPtstVk#@lAqS zWsF;lXlTdFOenjZXM)Yd9(;x0$=v#LajiVYY0eC-T!JOj5iZ&QIEkHOeEQFpuJ&aWc!hW`zF1OU09<;xHr}r=O zj>RCF^)aYYGHt=xO53mS$*Sogb4=XBL_uAag>~Ke5m?c>z6xEhSxMYZgK7oFNAYEn z<_W5{)9~VvDYY}{Ql3UN;yRGAMLdiEMl7o~fMC~n@(yYJCZNJ|QP`Qg@FyW*8IH$< zs?R0lEhMBL8?f(B>Ye+;rAF`W!yTIV{^IBGz>?wOirowg{Fbx+qPBckxJype}n!K;BLi(X!f%Qe&w| z9ok?!C~$MvJ+07n3RL@>J!hC4q8exiB|8roo-gR+7VtKY3vVa;F0Icb&dm#Pb&g6; z?ASk?6_nhW(P*468qLy4^$xDR_}lR$7{3)^i;;rzmGx@d;+&)0ov!0;u{^3)ieR=8 zZ)A=7X>xP5yAxw6yCiV7eAYe^XEd3357UmA?|Wv)Sg?OINgmXP@f}Ig#b6ZN&OG@x z`0y_*mCMGE#SLuy<-E8`(T`mfkzY5=}3?+Wc58^`4Jb|y`6%+IEe%sh2+^Ug2kiKi_T83X~ad$j`FB`72KkVcDfvy0Z1316`bpCiz}NswP> zTl$mW(ynEvMol5Ct|HUbPC#qpRI1|PXfkgB-P|e8kXwrvO|GQ>;rD*>PWkIM8%!s0l)OLz5mt4iba;CI z@Me^fT~9kiu1*tDMp0E_yHp42%ppw(JKT6{zkIsNbt2VhO#Fs=xw*Fnrf88JaT(mE znWUSmy*6cS=XB&_+pB`&JYF}ao7Wff$a=n1?aZBgamahnZgp7?b3YT<4OF%RH3wD0 zET1Y}ecz|A&$Y$Y-;l3$9lxJ}m+iYs6N2v2;g)2Dm4qykX>1)?c*{B^*5h#Ee`NF@ zYQr>vwsOyU)K-sQ@>ZNM|LP9UN(4tU|5`ie>)b(meNCq1ga)y1FRFICLEM({HJwh% zcYJSY+cbo{>6Y{m5(_O!ddyqHM~yWB;Eu_#-S4NW zsw0TZc(lOTxP!)7G6ZqBeZCGl1Gx=%kLheFG_&!1l)i7<{l82YPX=0f88vAmltx7a zB=lIhqDjZwK1holdc$TU?auey;EHiUXUE{f+N1CijgP9O0Tk&dhytTU;q{Y@Ve zo6A|n3|3Vd;FzAyNj~57LwM&-!N-_o*iyRh!UZumW6Fnx{$0Z65$J3aI52c>rv0vL zU)hCc2y9B>thHrgD*2K3N}Y~%n9=Yf0DaoT@{Jqdvn0`uN7vIj>=-^o-|*WhmuRRx z$1MLFw|mk$04woGlX@HJ<2a}Dv}IbZ??kcYm>wsT#w;OXVlce9(WqrA8!p0L{nLj>)YuulAj07f>@hhPH z%OvdNP^`0e;VH{T8fAPf+BhNIv}T!{22#AWdOf)7=YZ93LdNK=R}-l$?cr#beB-pS zhh>J0qwG!1t_iWn?zixJh|`3yCOh%vq=VK+Zc}x)OB^Muyd5~W&7T{ zhQKDU)Wbmf*a(C9^cqPQp3Z|$w0n}3o*E15 zSP(x-e!|3flqKj!qTnS{^*^?jV{y)NRgIdn)ztAnzWO`;ZjKoO^9Fqfi2HaH>p)S= z2MWcmtI~o3$U=qQlV~#-8mW#KEPcp2!Sy42@}vGYdw@%aH&E2@iD|bA;n3S zo=ktxzhd*d)%T4{p)1HpI89yDSD7_+?G)EdF`AI^VZmagHgmVyA9^lDVZ3f7xM)L} z>psZOR4o4!@zgyZo2*~c4D-XS_alriV_~0T$?E2koD?$zS;YfS!#z%JOV-C^7Y#a8 zmY{Sj8`R!Zzez%t7iBn=Z>#@jQz=X_BdZu%m<`W-m1Ek~U_NAVk}k{1@C--G)4h}y zCbn_yC&Rx9l2y;_m_hn6g+V9tH!XAmGCpa^fD&R4i{|OK;n#b&*f}fQ|le?$F`;3O(D%p)K`;ANxDQQ>w&6Qz+^|+4HB$PSWKU1hw zK+uDm(_}Gv8$d~$Zc@WjG3%5uSI^e=MP<*@fGa+VdqMl;f0@hf^5gB70crzSj%_E6 z_6R<@v@>#vcA$Kzco|{gs7;~o^*&qL6VB#QW)hvSA$MZMH&jv{U?k1`;|Rtoc7?~d z76WD1fqzm=R>yX&SnU#xF*w5LDBrmYS>A%r<|N{nzzI%ZBXnA_B)$e_aSQMOQ15_M zJg}_&l6=wiV*Oq72<2}NvWCpL{f;Si8AW$w{lzu8nWk396@9W2y)>m=SENlz*^Q4< zi&6zn{Pll`BSRY6SMQBt-d?XPnWv?En8@_(a zqFB3_S9YIjPk&zK^7IkCqBpRUApG;t zJ!I(px_FT*2Heb*Q(PsG#*Wt`ZC%vN84!wns+B`Q+$@AXFkTMWri#`_YAZgG?mmYx z#s(Zv^~|peGnV4zfwnRx>$Q*S@S5p;S4#Xl&UX3<;qiJMBb;xM6^)z0)x7`NF?G-z zO(C*CCI8ZI4f7zAZ+T1ZB7f?yv#swIOofA_XoRZo zUgDw5-^ofRlX0rxk8LX5EbX*>E^&zg+oaLOmhQheW;PX=f0<;&VtLJ6pAa)g1rGyb zJPY`Jh>MWlr~ZldFztT`juSwpiG8FB64O@)L|V>86xRR!C+;oWN-5LDb~<>xtbGM_ zPJW)#CF$O**AF8~OIWRqriA*wEtWz7e2qBBK|T-p+LUS43%jz{tDf-avH;M!sNX$H zo)9u%+akiAwUsOjpVFRVXjQpQs(|9YdYk9M#>LBi-QKZ=s;`37ucx6(*WTgU<{wSR7skGbN9~my00y^wOrX+Mrqb@_6z;dK*74#Sz zXK#2f*x;6)aZ&W}n0NeVSF+z{JXjC!17bpp=Isz!O9}jA)uIn#A-Lv9sUAbs+P$vh z(pjcoh@w&oasxeMq1b|ZGtJ(PclNtrYz@ioklFtrc{CIeZywKz`C|0xdMpR9ocme{ z%BM(C9Bon$5O5`!>ePk*(0RaSPZ$p%I+##s86>F)ahm@~eBfJS|GR8`kZSp-EC%U` zhWo@5!U8R$;;z^;Oi3UP1Z=>?>25%?*)|e`notyq<-d;~1nN{^6Od#RGav@r39%+< z7O~GG@!r^FhTSZoX&cK1_+Ag=_P2<>b-Ul?76A~+lhr-< z!4;6-Squd?X=Mp`;&NiLe^XKWj#6M!t^}W`Ifz^x8`T1)jmJdlFC+dx;SJ)b=FnKM6{reGQn5? z3{y~pmTTo8)nn3QjyVJcB|Ysh8~eV~O-iw1Q*R%HQL^ijj|GUhdjHv^^PBfb!su`V z#Iv9VxoQsClyg3IK2DB*LBTo}EZM|`pE3QHYPri|tg_x5AT|wp)G4FOe{=LCY_WoA zCswyRwfX+%h#vuV=94d=T`Lsg4OdK>A0)Ct_{AR_KKFgp0N%R}=z+UwtMm(Hh9z$| zX)Q~p9e`sQTc!qN9m9lGcU2qw4F$VU(N86m51JvtWTEh*Hxx4*Y2LH4FqRQt@bqS_ z=9I1D^09{2L?lbOsH`t^iUc`E_=?B67&Pqcl*KWy z3ES!}I_QNkVGvV=kpEq&rE!zl2B|x6s1fPcN5)d+o%_O-n`|qS9#!CjD7rS?(7wEW zXg|obB*bJRe!l;4<_)+*L=rj|#h-w!X{f!WqZ2O1@=1**RG!tOe}vPM#RuSlf~`ak zZO0Q6S1|gAYGCCXgv{(Mfb)>~0$w3SD5Bg*FOCODG3i2EXRZ35SIal8ANLN#Fai?Z3Aq z$-3#`vhINuU$BE)t@S@oztu?9Zn~D>uOzlqk==T;;X_Q9!Q}H3Anp1c@`)iP_zqzo z|C-xpUZi>PY6n)d$*%F*C|{x-LpZo3gyen32B2*I(3xD7Sr|>R1M5dpQ90C2KhWlEF(cQh@aj5%p(Pz)zC+ zm|?V^wcMg@?z*5qTzipJ&|ok!i9W`sa^VrULHq zjpp^-{IF~ zF%d5(Ibk0v%BK#D$)l%%C}_IJ*3Wl_;o&P@V%f`)VX^&m^G|7`ewl{0DqOGzzCMj>kmfr`;c65`mw6(}yJ5dA#e+)DTsF{Js-ZaqS?@ z?=C|WHng>_*^e*R3pCY79L*U}uJkEd6f8iqDIfB=XkzmHhhc(;WU0d_(u&4n_T%QRYOfsH>t@^Y%ICqCgw%9 z>B7!HilAqCk^)-C@Nj!*n*m;=C?^6dQrIKv_(YY}Bphg{io79Vq^N}06g#oPXZYL3 zzXzlS1D4st?=%L)Dk8Qnas+MNOn1^Z>-sBJwA;7n~ZV8D1RYV5D!v!M=lG*pnsX%pre(Vz( z+Qj#Ql^>dqM3O*mT{s_!m}2cRX7H}!JHE?>^>x%+5$VW?9x+%(M?J-wuzoZ4&-?VI zujUG)Z(b2qDm2fZGs&$ns#Zw^;DEdm=p^Kz-KMGW38MT<_1LfbqK@N+OX9zeU29DM zy<QtwOD+2C^$aSkMnB94h@|5>c4fnn)Ya}}L2nt-DTlloh z{?OR3hwb@XTRsoZeckKfX%djM)%PqToqD<@5+Vy5J_F;>3&vbz*5yEt?-(@E z@Fqw=0GbfVzh6B-itM)Rwg@zqmGcA$JZQ1S?dUo!1%#O08Tx0^VKGiG&j^n>uke!G zSf|&MkKs#u`NTyt7%RMcDTgw(`$lC5G%KT1Gn}d`IOFy zA0^UVx!uwK9w9{|g|H;t&H=`9Tx5)?t5-yb%^CTMe3~JvQ`e)c;Dt7ttLJ-blET*X zL$TOnv*tx3UIg5s~xDghzaLZAb$2zDHt9 zA2NER_5QxnxYe)*BZLrHZr#h;MbL)LaXdk#M9$QFqsjLmEq;n5Pu_-Y@?g>A4B|%nN0ci;waB(o{n@_Mn|)& za$yRe_VH86j^#s*U=T#;aa(lOKk-kWE7*} zrmLa38J=NteFY#mZ#iUCB&ErTkFuJkDXH}=8LZBXW2HkHG?WdPN`NK>`5GkFR zh@uom}5$^r-lPFo&5Qe zqTZSJN&#Wlqu5orx2|i|sAl%W&v7uJq0V14y?d*fvSF}zJ22p4gZldHFu!Nhd5OSv z6GDNHEzvUJ7}{s0x~5O8sZOUYbbyYRUwosLb$fkUt9`CGQRERlU>~->?^nYl1ZRqK zx?Wcs%zQ(izv3vKah7hlM8P4lk1FJwWgZobvn{ApuD#%I%kn1YPDXq*cfsSg7iFzU zW9~Tu6g;qWE&{DnPx=p9we{=)1<%F#F+2l+Tqool4ImMc=wZYSbF#8F@dNauZd@}i zqIhsU8{7)&SlGQVHyAaWc~KTeBjK=4F_P)obWtK=J932IyXgz`=_5nivHQhs;4e9% zgP=%$QF3oJCfr4IGcpQ01NC%+-gwq6Hd$pSjKRv8>k=vJ^}BTuO-7RC;4N3`NY0<` z$=5k>O>G3Y02L3MGU_@)%?Z)na6W3Yj-TiANnE~kFNDpoZn4TL7h?=g$EZ8n z#Gs}52-PRw?4(RSW(O(osKkGd#} z1ym3Pfn_a77o|$u6%nOM@69YCgwP@&C13$rrMV(Ss?vK15FlUzqLLL1NK0ZuQ4x?5 zkQPcvcqgFly3e!E=l9RMe{JyIJ7>;0b7tn;Ip6yLIe!_)wNGA7*Vr&tIGoW)s(I$q zToJ)FyL)t9a-?;H+s!su;gWZTqV{eMVnK3JEL2+~U`AN$`LfI~=EV?mTr+cQduY}x zwS-M3!=-?En__-YGOx9(3)~>Vx#9B1>7h4%MEy5pcG*NRt z2%adqjB>|JP*p~QUfz6QvsqB6Ln036U;YT{!WAq4f1+@MKk?^4-IHON&5A&a0Szb6 zBQH6CdJO`oS5OQO-S~Ef{jocrV#Jx%ykC$Nb^Vln)dw3@ir+o-4G0GNp)fSJ(#f|-cAu5bwmo&V=u$tRrpmn&*d z034!7-lU(|ex*Hs9rNY89;mH7jBN*VV)2!mK!>@{A2>hwuy#ZUT#70kYEb=L@(<{sDo3QQ10pL)6g z3WJ^Rd;-7(^yutmDZWgW0$fr)K7LT)X&YuSPpBA@7^-=^VoHKdS7%nIR&1 zm;Q4t&N>hbJNXTAfX(FcU3v=H%1P)KA*20l`UZYMJ_UjIH;4631%FzB?Nn0q3w)%8 z_aF8b*x)f3r!{>8M$k}|<$?-aEo2VhD+Y|tC{zNa1p1L}dS7wWUm zhnKhgTto zKGiaZ7=E#Ks04sF+U*FJyqY$z%FrV-zVIr(^v^4n9u!s={BhnV0~B3I_QtHH{P95R zcZG9@kcT$t5F?nRl_MUHfyNH(n(ZX4ddXR@EMG8bO+^?wb_xG2S^!d~vt209Y?`gb5wI<)Z!nhi` zYFBEvpSu|Gk~7X?Eq$Dy*51Q7%V-`2;DGa+-~aAR5;i8g00jBFW=-XsAWeOpD&#iZQBIFcjuNy|xiiTjUkkpw5tu?(|2)Rlfv zoeL-B5AcG@uYP5#101YEnAl~vmu-4y;t~&{PhFJC4pyKeKHVV|ah5^wCD#;<+XQo| z_+ZIknyK2xBpdtXk*T5uwl8aQTf*eCJkZzDuYe=YEp*&m>H&i~kDe1&gpT#$(*s%~ z_~FwXouPt`e(11*LcrI_e(@nSivt)i+b3N>_y!m_S{rvPcDB~lPG;apviFa5OKng) z8RUt}oLc_@fy6dHVTcQ0w@DQ7@1_LkkTVUg_ zsy#n*GRwoz`A-Q~LWOf4D2Pv)X3^mW-?#ScH1Q;Zk1x&4`XU{z8IHIP&p^ID@ za5=CTqX??$yKzl8_(kjR*!~c+5khw(cU?^1UV|B(l069(e@7Uc3}pRmTt+k>PqsNc z-nR7l19T)0IMmoS?8Dx@2Axqcv|?a`2@xsI(h6Wt4mO7Gl>jJ%hNZf?@?#RmkJe>X zRG9}Ww7|b3HarkcF#w14$-4`R*T-dfmF!UiZ@_DenB(Ldy89Ty%!J{hB%Af?{{ukr z9cbZ;l$&}mFy7VKo>!U$l6-`+?@$I`*aYYWzT~~qrHZ)7ic^<^FBJOi>jt`wn=k5# z6sh(uw*LwnI6LyB5mR5-!yWd~GueGF+1G700 z444*^2`7m$=xZbP2PZcRB7e^h{SLD*lL$~>(yo@12L@hHHArHvAu{pTM)tS9{9w?t zrCUwx!n0GC-I5B%Zbj|KpMQNMDXXMTI{Ra5Qe{a%j5pv^Eb66>zKs0%0@4~h%c*=! zw8)AYxjqsM;0aFpB^eeW*-!63pd3Dco-Pz^s0&^Y7V{C+m{v=SnvWvI4>yDFB6%Jm zX*&%?{X1sDj24h|5s)A_)CHAqitxYeE7yS(QLwqNUpYm5Qefo;bwGHhtDTqqha^{5 zeBAs8IR02Ns5XwIQBOUEdx0NLXUUA>29J!$kJIL5Q->#%TyYDRbjZ>plHb?@!6P;b zL#%JLGK`Xj%7VIlZ;8zf0Vks`;Ad+q7TfL#V|du`yD+iqLbpAuhh-G_?hYo$

Lo+H-kA5;K0IZ{_e4u54!Z>QR=P!Bh zcdXplQHOE+>>R^EJRktZC>FmsI&E&P#-Wp01&o~pweJaE;SjLdHuT92fjjpZNEB-{ z)8T6Gm6gE{z{5D7-Cz!?>f6S3E(TW@sSCc)S{YMMmraRq!QI6E#`)S4Oq`@QXv5ue znYItL3s4|31bqO4VY41O`8^!U;W6xMsGTtnvVL^b@;nFV^xtp{e?HfYWSw$9D3*-A zbbJah1Wo~lfc{vM6aJXM{`Q#sq&$FIxR50H;ND_4;?-NN1Gg2^jwffFx_Hs+0+Mrs z{t77Rk=5%{Fppt3EDY`Az%Gzdnd>1dfC6RXl$v~Ro>RA7_IG$fNNo4A94H8`nto6v zy3QgJk2r_b!R+Kp0T^~rg6-RGw=-6+EZb~+jm)7ETnot049*s4vQ!VW?i0tx^@YmALLbJlZ*pKqE@gT#S0E2-c8cmv+3?-k19py zHOMc3gB3Lxv>ht>4h-O^;1upH^to07eeDB0J$?IAy1}0VTCsQGDfXh0ah)3^Jirn# zjR&NI7{C%pczOS;Y!}ADetPkRCP3M1ul(l;sb7t(VU_R=CSV zJfB0TsLw3C-+zrwe`)8HQOJQ;?K}$V^Fi@?Oo*XP@_uuV3`P53xJmv8+|x^HBiH9X zc-cv1PPE0mim3(*mV)3}E9H(vv7OtL!6DD5d%sq%|9`>~{Q7q+!Sd0vfjw1`!4|&@ zLIL@J1I*)Wx-b9il=YP_@brF$zlJS`03hKQtibu7f_Wc#MmU1cV0jJt*ll^;q5> z@S*2NUjYCWs-v`nKEfPa!Up+9SnyjN|4aTOuj_Um4Gc9AEq*HZTZ=wxqjb$I_{o?0 z{b~I|vtYiuIzPvVvAk%HC-rbcw(?1r*%e0Rm;mP{uvb>&)V&>#N%8qsd2j3g;UILl zfeotv`3&lOLxpLJ8~i?AZ{b8eV<*Ee_gM_cNF4t#VtaQ5h-xi%Z6vYZ-z#<#O2mP6 zT=%taM1F%PjLByG0yN`Nw*-J!-)mpYwXtyZrzFAd&C)#pJWI3?eaBF)nDqIOAL=LC zuY5cq<7w(&RES+B39JS1LXo9v;-9pH>x|%6H@;BOuF4;HPdKBkd~4xZOSyWfyCR1P zf`(GH!w(hyFb#v|Z6%y|B-tkI4| z8NjaA;5$FPt?pDq8k;OFJbJWLIw48VK&&np2AM=ev)`Pa8Pf3Yk*N)3smI9PUbfnDNEz^>H6A_UpJYC<)UGEW zxd=WyKISl48|?3vqvcG=zSfw+__bJGN!bd?2zr^yA|(;tqHxAP63-raC*WN!3Bq0d{5xzUIaOUi{L z1DXNNh=x+hfI(#7)AO;J_OQJzmn}oK*|<8woPH9I%ug;yH!2k$i{U#^c&k$p!g&9t zLx~amSg%~rwG#`WU;}lrea1nqbIPJ;UGLF1TnCm(14Eh z+C^4Ewr@VMXjt~F4FqDG0wgn?e7PUe%ytZ4xJiskU{gomPvRp&2MYq_;U;ivxHIGz zVf7T!&yidwZRWTBC@!Nb;7 zLoN|IvN7DQTXvD2{&JSvz?5JQnulb_F*V-gl42wvH%2rOLn+E*L(CW&LJ`x{9Y|*E zfINH~Tzx=X=EgJdl9Jn1a^gDZG~`B)TlS8S-jLZJAn|W?1gEu`}@8%1af4+(^xo2ddk8uq^myBwun)d>v>WqU!iXRgiZ!nzIYT>16!UvGgV6dw3`> zG#pje3G@*C71)MR(i`l%`HkKN(j?5z+DLPx?^jdUAu3v*SN z2Af87TJmJhgnqv3dq}!z%6=%wNA>*Trxo{(^Ny1PVl0xgimaOY1$YEWiSF)$J2-LL^WnLE<; z!dW9U#AkG`LUg4S)rT4Z@tv4@Y6Fzp>;m4RMOHG)mm(%;?w#ylp0l}VtMv~l+Ci4f zwmhy8qbm68Mpuj64BBfnB#7DD(b-L7{htr#&Jr$#AgKj-FX3hSYHNCvKe2v?Sg_~8 z!U(3?lI8cd_I;hhN18}(^rlG00=RHudQ5f*d9;7``t{Ih&Dt|jd?bCQDbt)O05Sd} zllQCgBl&9a{8k-)gu*6!OL0fV)|4S*Gad`eagj$nJ97Q-{QE+>4odGlpi4U{WNse< zrxx%8<+7dHYpv<{pMWkP%q!pA#+Dynut>{hcN)1)CC&5tWG%liN{?P|*Y3|kBw&ms zBQf3KeeAyQ?KjLXk3IWqq=?9K#U04#*jq3T3nx;-#vLn99atNOMH|rr7uzo=o*&sZ zsJ7K3V%K4_g9h9?A&?OFFHJW4bAWu3&n3 zqTCg^+^*tmxoY>W=vuU=9glv`YMlD|bEU^5NT=XW|RFO5W?$@;IBjAEO%(YNQ?EONn@k+27~t*O*=`q zEG4-jLc$9cOK*Yt{qyU_O!^)tq==tEy}A|K7-~W*AfHSI0Rghmp~bsUwD{r230>DG zo+-^IcPEwQ@JaEMT6k3E-`jf9l0Q}VnVnE`aZaVNKvbwHcai>c)ZfLw=M-8KXVIk` zSBsOPT6bv~E#aijoVVyw5D*EKdYDhhkQb4rMq}yAubtT_;_%E0mL9ywAH&C(=$?H% zycH7vN9LsGw^+l#nq42iZJf(z(FyN(gI12j_=PxyKQjC5j(Rd$ccrJ>haeP{gYP7v z=;&VS?UMnv_2c81sa0AvLQq@=0{N7=F+*rvye(9EqW|vMpDcwA>OOlQ#q*^5ahnuR zT@IfN&uh@NzF;b@0x@LWjsozwzZaep?}g7ae|IjpIKd z?j;vy$&I!6JmwrEmoXpoY(k9`S;>~8pnXqTv*RtIsywTIGVLT#lgpGC7dc}V;v3Ta zxK)ZLD2ML^Pi&6MMS-YG9+%vbw%Z>F!OfAKwI6E6Z>>IIW3WxNRI#WU^glP=^!vNGa^LHWSW{u{+P%`!ZsZb*j|&t&(dPO04x)=OXKE>%lK_Zs%6;Pa@9 z0yy?n_2O>0Cv7!*$anY>LzQaM#g1hFH#6RJGbhfxt9pg8bZ%^7AEiO(THc){mbyIGBuJfk# zdG~Ie*je1I|GfMJ#9a5LEq|G=c!m_uRE~?u$aC#dDEF?OdkpiMO)DgRx^AXR7*weI z5-cDD!-(P0RT|gJi3xl38AFC@H$7^fi_`_y?i)>f78e%p=Z zBgs+o=pKFXF>O0yR_lmK$Lx{qwcgHM^-5Ersw> z+`>DF4ATRzFA86O7mx=`(i(ICq`>-B#2jl%cbPR z$(UOhMPTl2>X!Udx?3KXD4&cSwa}eR$Z}suv*?;ks7-8(jw%_eWA!brg@=Sk=rT`J zmr6dV-i9nG0w~s z+*sh)T&=pJ7>VJN*p28e5wQl|8rPaPADZtcg^K>%2dr%8o3T9`%g5lX0^gAQ37*AT z5uI39EtGP99OK!;v{MPSDO|ay0wTy2L2-P_;kg#zyPN3 z-qywwl$f)>AO}8m_OC{GOA8+K%oxC=4q^|M5ABGuVw~wFyP<4@tV$Mqc5)+}FM5i5 z@gFFu*u4w;oDWjD=i7RHMI&G-S~-DDDy<#+;5E|HbHWCGNF5=90PdW};(}a{&lLN6 z&-9;XH~>%f9OtVgtGfoI6wJQQK<@Tk5H16L?lgSZ8Ra!;{TTNC$yJfA<(eGi$$Yy} zH4xDeN#3OkesYnPm3l7RkY>NHOA|=yka>h3VHCYSJ3&{&HrU;=JWILwk2O0-t64gf ztKKNZ4*h4Bg<2xP_I&>lu^QK1msB9AN9ogmun(S^Rc)DIv{A_y? zwcK+R6%n+e;>rTwFc>Q6W|&_O?P1Q~uEZwAVX6Xmy29UHjb)dIcP(G*uNWZ@)7C)s z1z7UMXCtk?Y@+=CwkvTc>)2}3rw~>7WpBcQ7jMi=&t3`PmqvVSWg;^L4%<2NoNfj6MLh3!ZX>^{Z~Wn(|rl0zSa9d8*FY8)v2LZp2;M-G5o!>r7zEy-@?P z{ogijQcEj3X+y_kYJ@V>MRU1QX6_r zCJ&BvLrNZ%rSDJg>CEhyWLkhN7@p+)zNjX$vi2!`Tl#6C+z498=pNVHK+A?b|JdV) zRA{U$kKR=e4@FEju|He7UasYKdEU(k1&IS*+J4B%IF=Z4V^zx;MuTT5=#{6%lW|gl zt*fY~amnd+;{%yI<2im-;r5RleC^Bw-HRT!ToJRnhg|fjp&QWi!==mHTZ~M8#b^@H z0m}repVWE-aFrPXa}H4<&=ODlgYlOKZln&*`2H;&B6KHKCo^;CIexz^Tpq3h*M>BE zPxXU?vzxnA&8SKxM6l_4MG}+8!Sq6-!RK-5-NHz|gO)-q6Ml*n1!0g<@}W5NoNN9^ zMj}R+*}0_43eg&$!QHDhZE(y{^Q46}MmUCxbe#fhcUlBr(^{9z>k1oPpy_vdOk`rUZX}_x-Ho@EoSJlm;i08hvmCM`9M3z4mie9*DV>|V zC@nNb3uDTOmw8t3k*M8_nzi~Of2ct7KF%;ro2vdbVd|_V~gb*w_Sa#@kwh=iz0Wi7Ztdot0|`w&-UNW?afe1T^F zxS4!;Q&%{%GE9Uv&98fP&X6tmtVHcK8PnOKwc8*rYK)-#uX^zppVE@r?V79L0nGIv z|D@HguDyCm(t`Y+O`R)BotUJZuCVs#O4Ft!Ue{c9K1h~9Anz6^Aan7S3kE`d@TmMc zd31JZt>5zfM|6r4%1H&{5*#?qS(dw0f}_S~Z9HEIHQFP}H+HXtue(8@Du>ndb)4Ed zd$M+n36;TV;fz6Ui=tNlRWxqbVsjbJxbU6&mcQE5GZH;RbEs$CA?h?O1l5hPZ&$cG zM>{1r2|`@fr@|b*dSGN_uOxU7@RPu=NhL@$y=^60#U$)Q0rC0zh?w{K_QPi5!ILYK z0T2iYIyUlGE+O9-%}E_Kqrc5)s(Y-ng8dP*<}=ic#`j`A>jxGbLb5{eAt1snzFooo zU1({p+ygmnw-Yol!NWx79|@!PzS{^x-Ysjayw#>?%uUJh& z)@?v);%hN0G{ba{aF!9`=N1=1Dk&PR6=``tJ>I-odbKVa;mq0gG>?B<%&0n<46vRM zbDI`R=tFmu3_5i>cR=1qs5_ItqysO?i+dgAxkAVrmh>c>-AEhOv1y!CI?I_B8N@GJ zbRe1LqF|7hC{qkI$d049d*JM}A&-3m2P?jq^MR6%&ApuDf2JhIudfbvdW$xV1kjoU z43bH?i(4RjZf7?Al^^=g93P}`>=6@RqqaWi+|MSaRx6@LcR&swP*3OyJ$~qI--<6YiHcP4YDUallDJl zJZYU37B>Gv#AhgjiPC<`3f-5qs$@L81wwe%G5d9=^xtx(5QpKm5#mdep{n4I#Pw^; zAl-a|r?ocZU40FEhWm?|pMJJf*)2|taSkOgf-mt0EaxxP4AMdpz8lIn9&qyE>4f#B zY%%suh^iBqnvUB7bXvO;VxF(}>8By27gOWJy)$dwp>{KH90B<4IJ)x(&5Z#Ec>|oU z`6$RGuiRC6x`b^Uabi8A&p$pf@!4R;16Q{wx&s0!Bxvnl_#HUu$>^Qg%f#x4EfC`^ zAhmlD1gp7@C8RgEC!1Y5h(n6<>xzVrYnCt74i0a`0ry*?oOHry^UyvJk-tiYi&%Wa z*BdsY&0PHVrgpHCAxj)zR(1BT1(UJ*Alyzvj?>xO8Wb)+y8lM_j9fX z zaApP&Gow4ydggr!epTdosL)2zJnYR8a7XjuW&5p=!YGdNaSlJNv{>VDD_JSH;7~mo zXCp(?XwwiBRitugeolpV#tpPeokE=bk5;eUJolccy;9dXu(xz5X^kDRxPvyPXKE3V zzf{E9K4lOwC-MVG)w{T43xu2lLpgO@gaPDT4PVRF)I9JmYgNdc8E4bC#IAN8T4Pz` zLg!noBYfv`B9_pbPN!zh57POeCNsZ+bW>JuXwP6_Naviu3kal{G8X+$(~d80eO1Vw z5>6LmMlE^ih~CRtmB#5UI)SX@XO6xI@!!nTJktBg?SHh zJU+^mPuH|Pc`!7DcLGI=k^a&_H*ro5IQ?nHUbXk z76|0^UGN0ovhK}Bjddp1<3S?*lWlc6%GNW1T&5c zSQKz>-Ow&@eM8WQ;l@k^aWUR}YCH)3e{WpXI7M`sI{G1Vraz;JzDmRjJ2wP>+AUAP z7oND6bE2dqi$#0VB))2&(gXDYZij<&?BYs#-nv|Sa>ZBqKB zbL#rWTIH9GZgARfp1r<0nLUsZ+G9{VBLbd&&n{=7?tBE|hSOX0#uE20>)jlF9P^%$ z^#lk}gyjEKCd7?!^&cjTZyqt|-(mooq~jLqqEW}sCrlmuXOr)TRa_*&GccaZaW;=n z2Zn8An*OrpyGqB9cQD!#4@dNJ(^dN)7GA^?Hx?{^nc;A4Ob{NyblQ+cNk1ayMivEa z`@Or}d>+WIovfP?*yva8LLK*~yCUiTa4c1?G4HT(2JS3q%oO&JePD|w8oKea56lX{ za5vOI(2u4+QF-d}r8Ucmw@tdgm`qVNwt9clrEB%lj~g=E{$gU6U;qri?Jwi+NynLR z%;!nm8qyu&0M^n6{~9&Elquw<00-VAl7=6A=fF3gUzypz(S_<8Sv8_5KndIbrU>QG z>;lLkhr&im%x)awJmWn>#c{sjDfD_A&F3B}k@+Vk>UVy+IO%s0 z459B;d~rtUSS0d8qK|Ov=*0WUh?5TB8}1*=QQTw?OW~r8XG{I-v(xsqXwAoUX3C-u zu`{J- zTI{UsM$fXom)a~AF`qeeXjH{oPNOP#)ZRHmslJbGdZ}XU1E#E$|N6-5MWL+a=j-9h z!^azC5&0}}YKLh=#o8G$u#-DX^b2M!d___B zuvcs=b7XO?ceK4WQ@fzXsK<}SVof!%GX6&ooW;v{Dg1#CTMmjDdraA!2;3#^W%L;8 zc}ov&PaL=X{UdQNY2~xYf=~Ha8s9WK^xl>X?ekOEcuSazos@aQz6m?8q_!4xHeCM>1RjB z$I<;(rsie1=Aik=qZnFYf#;Y5Q#JG7y$u;vdB8Vr;EyZfalL~-UI4EoSAMDBzqazm z%hT8W*AJBWx_A2iokO?YV_Iaz49VLgY42+1mTTwU2h?aPCR}C4YYJ+oIO!-}8Sh5a z{1hV7Tenm{{X_gJtc|3^aJnja*2YEqzecDK6lj_!vGwe)JG0LHYNU8v>MW(S{g&xK z{-H%?)n{M*pRTw*?2W8H;6Fm@tCCGVf!l75&?}91gZ^4ni&RvW-5Iu+_4}$)T~rcP zuj`;Cy&g?fvgse(>oB{DysDwJrUIRbO0F74cy&0|^$Gp!IHg*1gLkVHhFs>d?{|q$ z&AZVQ1gG=+&J3p9QsKIRxw%4tVc3q-_xmw2%@x?(zFT^sn$$Zv_Z39alexn4$rI1p z98NsNK^boYQrNtEpcq)?)n9$-nf2wvJ*Uid`is1>LG*3+{O=<;gxggh&*sK4r6BRu zpCgg@+IgjZ0gu*}LdR0&$i9eNc5&anv=!8r_*JCjF9JLF(JMZn$7W_8i!|E59a2c& z9?xjVThvS>RXocn{_(LTl+a@HsL0229Q!AQ=|YBM&i|_;f|ltOs>RrC3gyRiY{ae^ zS@OW_T27er+?BcD`Qxb|oHV*t=G|)?NwO1H;3Y;nY#$zxFyQuRaK$sIHGbfb*KpqG zv?Wb0ywL-j+!#pkOmXS*sV>qkahz_>s=}vL(aTnH!%H1&`|SQz_jc-$n8qL7WS+M5 zJrFP3svITTXd!tt9sJeBduoq?vt8Y5k?;byPFX>Q+@i)53<(><_+t3=eRR$R5z5Zm zoAdb6Q&DfnO2mDQvJ$zdcI$G^I$juq%BxSRh7Le62KO7R+d( zH1V>gBPUlI`lxoNA{`q*{*Y*M|09ffSP0)zk6FWa(A{@FQ?zo;6}4YJv09e}m2|QC zshF*p`cN^lPx)SLBC_1qBh==8+V-hHYrk>urueCc?PrPFraZ=E1fgv-d!J10;pce6 zN!aI3OV@fvR8zXsf(^4^B;zu{`I_M})y=-#@?WO}%vl>Qc`X;(h>GlwuC&%XC8cwv zvQ4O-pL$cGT(YJsEJCS;_horz!YHM_9r4m#4FYZBO@Y?l*OoBZN9fg6BR%vo7fb&w z6^l#LK457J@30 z@D+6h++XF9Y(nKW_oQ^yfAY3UqRPIpq$J0q@Aqw0LSO2LW_!+y!_P$Dt7nY)oS!A#qj*w z42?O@4RaomGH)rW`6`OAG?%N^jZA4pi%(Z>mc-{@jZHdh6}rdnytp?}q^Q<#A%d?` z-3&2V`$mQ5y1{&WQdDHYLclB2{C1PQBGzD{wr|UZwK5vv+32y{@JCEm)o=egiWnj- ztEdpm6Uurg^|)!u$IChC$A=we?eq6=mi*0fABJL=Ji-sUGvjwk(O`SLXE;!ozC2@H{CG*eJRG= z<%&Ehfryfitb{|mtKL*DcC5w7;yBT~2WACk{LJme$IG?9L3z|<0!{+u^&iWL$2 zHw%PsoF@b4d*Jggo7lrdcl=5hQN$S+Fprmi3ZEAaVn-9xwE%ONj9j*KKQc)HYc3FP zOW(Ylu1%z>?}OdrW2{s)viX;FS*EtjiUh6Mpfd?b%g+^V{bU1 z3HxO59}7`cu3V4ilFt_6o{}z>+>Hj!cF#GBX)W-I%;6|cVwkOq8IN2K2wRI=dMxrQ zuT1mRx@jdbqTY_cT;aGXSgO7KajK}!_3(8~Yh+SoQmv5@U&C&mY|Sf+6ra*=g|aF) zx{4diPqDTv$@zF)C%kb|?K}!o>*Z*ETQ@N@%fq(bVgH>OJIu_Fi|rH`CBVMi^gkoj z%VP7Bt8b#1s2^3MEDARs=~=eC7RVOYIo7atN% z;(cpB?PT1(1AE3hM*;8E_%^e^pf8Zw&w2t+wPyA;9hexeuxAFrKV6;SqPL^z?{6;^ zGyTXeU5xDbDD+q<-Bew(*efnmS@zE@m|qcptX+Lcv*KSY3L??u>$1|SKpV*;UE&PhGDNOcQ?cN9 z>hVY}M(#g%byMlQHUeqk;83BDH+J&QNf*od68s65mxw83cc`pDi`{)iU0D5j?oha7 zWzA1kHQF_McA}07x~!Zs{ppjA9IG+k|A~#zp!8br_4TK2DJU&(*_|%=N(UD^8?V^n ztR$lzVyVZPGfVmNm`ix4Xs1M<9;irm*PK+S$%S52_uRO`li;2JkD<-zprP#%RIb%ESdrbX^g`uOw7kH_-I zQto&$p2Bo6*tUrBz{v)d_1*+on6tBtxpQuI?uhE6>M3Jb)B3fwan-{+S(ZxTWl5t3 z2|i5~W|VZW-K-S*4;$XOWy%Wo6snZEVEMRY$kAwU)@{$#|R4Z0agGT=Wm>n3cM*U8_DZ)e3{?z<7t2TOE^yqQwwXF_@*o z@(B9aALh)pQBil#)|!$`L`MK^64O>J63u!9b1uV~L@Wnn3whGV_O49JvooKxW~DDC z#3d%9%}y6sg2V3>G$&M&`|2>1<$pE{YWjmwYF=x4j;}GS_sKekkTgSN6)B-a*goS z)(lD}by)Mb_Mi)tA~9sIjl)a^^uke!AZ*6Oex2HRL@Ly{Gi)sVxmw-=ySskAlL5#GqnM^cr+xu#6t0W(!JIq|JK#Qq74;0 zc;thQmcc4Oo9y0@(A}y84Ap{Pzffv0gcPg>q$Z`vn z-45QdDo>=Fb%kO1l!ah+XKm6Y${h)2%u+SR^E4*-T!OSv-T9TV-^9z#lDcigz`Jd4 z2fK0AQqcdTjz`cjq8jBdsV|-Gr1(AZ+~Z2W*!%QxGWr-+Er$W%_$W2Tvot1}JS0JxVT9}&sl7;b5=xPZlf~#E&HFlR)x8b9 z>(P8n(%F*ztmE~9Plp363hjg6u*J~=yduP?)0>OEi3UCd+H>77h*PYiZJx|c2 z8=j=4jIEWVn+{oE5TtnXlb+7JW$I>h>07x&&X(y-xBIqQ?X*TrmIjO3kSJP}ORsEX z{kGtNqlq@ftJWEH2Zm;Zi}*EoHDu^J>v=Ix^0>kuUBeb9rOWfUBA`ZaZeijdFrPyP z^tw^X4%jroDIxn>I)%Jf`ZuHOYO%?t_g!4D_`Wt`Vl6rTIh%;# z^&FKHN%^CIkCgQbalgkQ_2YG6ib3r*Q{=-;7b1%?Ihz>j)Kz@c%2XUv@+hNtZS6~F!^-(HlxZNq?N&mclo#iU!9KB2RRotRdG&r$H==)Pvj5u)W z08@zol&sANTl<)0-+=2T92XM2n_O304(D&S@WkQnR6cOhxcHCfm9Mv$ zC+rq4X0a!to5JQt!C`vtZD2f4RNv@ZXM1`(j?Wi?P$5j)!O%KXij{A!dq`@8ahHx{ zFMp&}ltiP}pZQ{`ZE#+UcGz%&7a8%sY~g8KRAqSlRlFHDti5pFu)I6WgWI0tkrane zD*040)rW}YCG{uZM777~(}Y@9(c}JjO?ovD-E(Br@GZ8})B10!P@4Z3-=1-M<;;f* z@}1iPHm#zg9-aeFqJ@JUke9J{@McNPzDY(%u;{@}?-QjeJoGN@uWiTZ=|k;Di$VO{ zH>oU(kYCZbvY7{bibifJ= zPk|*oY^{B5Zt%XOZQ2FXXiWfNuhOHrA%;q>eWCnIIvVLRlZe)rPoAj;K}~!r5xjNd zRzP52-09MqLwZBAsJh86Yo5}z27wNemw6lpNqJ7*>m0oLfp5ZQywo^*m749O1U`nn zz%Feq5sm2VbX2OfkVTcc$zTjO6{abwZ8F4s1RK-|;l0P}x`;Q+9!!v8+0STqHNom& z9uD-wvS28mW(0XzW9H>q8y(pk$!92hwb5fYG{BjuM+@jz7=vPyGVSt*K1;)8#5^S4 z45WJ6>X?2c<>>w}Y(NoGCXz_>*o#XKkDFO4oA(&PkTb*MdU3=fcv(pz!1L zd4apF#ecukcAb)%re@d2xzmWZbBh9yG?-IGmES8t%C_1N?n<}TdU|E?wX!d=CN*8( zM@&g?3LJnAdyXmaoL^)XgO%M+mN!engOoOwddN)QkR7o02K(L)+(*POuWN%Xjw8K* zp>|Gze&|ATX2TtNXCn41?*>zlRaffOr($(5UBoEKw3o6;fs~P9uPTp#X487rUS#p;7MB=rTt6H#jOy{w>_zUy~LhHyX zXc0_t$5lG029{>0TWexpPUq0!aJ;6FglYOK+2MR>j$4^5S@abqbbPKWjFb$^Ht-ht zBP}2yPIA%Sx%lUNE!35GGq6zkgX)*6SA*lUbdPH2eJGJJ)@mP0O%ay&lppvpT`oPn zo!W!%U$$fCIC--#&_f8VE8$?-_AvZCUYol1UbD`7FldRU+%&754OUNEN-g;-r5yU^ z8g>Szj9sDU3!Pq4eSKs@EjEM7;X^pnZAVS4&-)?ssO8TE_ZmCeksK zb)p?UU!)mwSQgg0dZcT)Un747K|dxt`6s4MifHaBA61#JX0)-0lS6ykhO++rdi`Wy z?8UwccduzPXX6G#ZxJj*Qh)OI-m=gNg>sOHMhsNFST#-ldKxlzajdtZP%)wcY1rWK z_2Ss_&@$)E_21oqcIu>_lXrRw71ytNnk06&b!=1;IFK2)u!pT)1uogKSDGkRY>N`h zz0o)AWa;7=RB6}j8H75&vH84#8I`^T%S!K!hhZ42f24qR8zah=E{y+-wIrt2bkAbd zT1Ty%C&_yY=SN6?z_QKq1M=V`C%d{;{+2YQnMh(^|D;tpF1*A69F)Z9CFHu5x31>i z+mIp~CU4n(#-CnpnO4~AWVd9~RC2`-mX8?y$i~x40tfR+2~Yc#6R5d_t|}*D>H_$d zP3NEh(K;3JVr3O=pI$|#C z4{*~{ba-d$`Sx?*$`pst7o&GaW2i^TgC&Y4#Yp{ZYxRnk#TCNCK%zSBZU#rmK3)#q zVX(#`-0?_^nm+z2lTx;qt->uMc9gD%xUi0w^4iPBIBn^Vm;rNbKFJX}y_jR{ z-Ka-6OHH;X#f;9D4J8;5m|F5=Z++zDc4lz5TUd4q#~RRy1_R_EMT4^lUuv))O+(;{ zQfSj%==-%J$x2;^C549x1?|h=PC0B~XY$f~Ui%980Zc-87?3hLua7VU3jnjp3WcQ* zV8;OjXfn%w@Z+cgX@;OXjBHAQ31U*t+5l-?*B8UUPn4UWiT^?$p64Qc$y%Ut5ah31 zL2k+4L zlaY_SOwU)i;+0h-PbPKCafa-sV9bMDxh3+B>$iwvL!TqOnnbU2+swpnvs#pn6luJT z8BsGrvX6nbanl%5Ob8b4?3prye?MdGXk#LuOkT)`=lhv9_SralcPxNiPw%Fa+tUzz zXTrp3u6 zm(xMaxQgv}D5opgB^wdmR*iE#NLI@a2#W)|L5C%w$#ae0?#Y&li&^!g9VL6Wu6DDj zAV!f(9JNHu?J5F|d(Pm`gZ+7Vqgxi1bk`$??gQ@%pGBHBsVEy1H2)41f)-zplR5mG@CXd%vX?^%bG_b|vQ&zDyXyoKEN|3sh zTxC3F0!0%+*0L_Fh={Rh<))Q*!QCvOLezfa85g6mPEA3{MO`}rd1Kd$LL1k_)HZ=5 z(l4oE_}P3*L6U*syIefA){LWksx!^X6ID(f=~2Lrc>cfU&IKImy#4>9vb8#HB}Le_ z?L^o%he~3%q)3(w6O%);3S$sOISifK3fpc*IizTcnQ@pHlN?4WcGD0!&d_AV#2Csj zGGqAP-yu;w&;G9K|G)m%Z?EUN_IlR$^ZlIf&;7Z-@7Fzik%7l`JH|7kh6fFpH3k_U zJCId#Elm#$c#9+@2%U@d&8!#4d+#??k8{rR*ozjtkM9e-5i0hx-P_aKxR?>h&Y!3i zgxTpen*=#+wagP>n$Bb!T4WZc@l?bsIbw$^cFB$#E$q|l!75$SQ9)DkS~M(hCZ@W` z?Y&*^_Z_n0@5snSHWA-qXp_08mOoqi4alo^rR$+L+St3)uFenISRbdY=ly3*qh}$% z+`Qdf24%|}Z`T?jp%NOm+EboJspFw2#+8V*XSCtVFhlgtA8mQHFhvo_ z6m|AuW>4RcdN*Xza<|gAB6#(1@|pw1WRb$NA)A z+i%OxWDjH+BjdB18S15ZRbq$3W$$aD=DM z$;fSPQSN-pa2NI_I4Yt{43T<5p`fuCxnJMEpp!t|W$l{z*%J8EcaR?bL%p9j{~h?i zDX<4GaROJ+8t-Yr=Ge{h=V=l~AhA=+%ZKqqlecIEL5t()2t|>G^GT|HvIyF-hB%b= zrMK2>`$f8FRFu?!T^fUgb&{V-WY_5BIH3c1TdYnmv{9E~(Ux-0BM|L$r#6o2xvUNo zfrkxK{(umTwN|&%aqbKP?Y&)c7X1jEVmrVeAqOEYt32yOA~^UKZ8i7W1C(`AwFac^ zsaiUV2CQ{V`A>{_C&Y1`PGlBN$L9w2dRmo=cuh+WIsBI{tEeb+vU>9B1FOTgt~;aH z(J|kWmU{H$j7?u99!7Ofholcu|FwKoS?{H|;qG##JoGZarP!8GBB*Shd;}7^ztfDo zR=E+;z{yovj2&$Cm$v1OpabhZ*fT2pwfqpM&S;i?C~X?(y~;0&=0t|S!cKS73^8tK zR7bub_(7PNP$GHJJH$G7ufVa6857Ca30&(d``i5MB4*JMDe>0Yi}BR%y+8hpq1a2* zX6%V2-no8BeJnM^n4ssvscLKq#mu1SGe3{$OyD*BWzp3ig=!`5_T8uOY#@xQn66|$P(X!#9+7NmB^hHvM|ex+c{0TP z4Wn1)L#^cmFiF<^H#fM{4{I!^7#km z@S{PZzSg_5hNn6W)Bh~G?qls{tl@^~+CRXzl-c`VAf9;A#<618E8;+ShZLi$5~q{0 zzTL^6q6?(0oQwG|Db_4@{8u#T97jZ9Sl(GP-Ia?i6eT5ymIT6PFMmr*``i;yZoFlipmHU6x2f7irAz=^Y6u1WTxR_jxWPCNIubZTjym{l4 zZFrF%T-bi8YnTQft0c$b1RG}148~?;vZ%G{Z$(iK@eH#_NNg=mc6L)SvB-hACk!K9Yf$;M>olSG}7jQ8?O&jqOML1=8`zz<_0zfG$_u{S0?EookSxl0h9 z8VO;B;w78rV(PLAuwP^5YkoEyxqnUS;|~ggcPl}moA}2X)6tXc*h?`^6RhU#e!NND>Nz+Qu8^dUg4W=0{3>jyw9oD@toV<6P0z+1CL(~i! zMi~a-mHk%PKovnFv^*vm>7w>T7irrcrWL<5Qo?{wfj=2u0v(LS5ZDG&J9*k|(h`90S=M(+S=QW;w2 zqOG7kZ<)LS*x^5UO$d3DV*xT=1hhAbh~yvpuKkT0qI&wsWE!e|&kta!sO*XrD5u>~UG;_9R^(F>(h9A!=jDFsXrX zLn5-df%KSq-eoj=dmS=fa#`%%@8mmjNp@E>I@};zHbUX_b7@tIwB9zs$&+6-e+_e{ z%=aC>!j>yAXMtGy+yauZyF#vfftHudguO!fxfT|03$1vP9|GX+&171$*n2 z{y3;1*Q-9?sF*4V;jMLpk5E~P4keU6u+zvAJ$dY?W~pIV)KU~`vy!VOh|a4t6j0Io zC+o5CzfG$jjA&)%u~?MII^t<`3^Kjvh*2*ZzFw3)EgWf3;;F;MvRL*?qoRMhzX3J% zTvv-dvu58!<{pJPb;?%pwvwexowg(!KDpkk>v0R$bqnV$$oAf^m^*`3(%QM@459G& zG?ZE83)j)+4$!#1N0!%yHo*mjJ%|_|1uzgqgfg{$U>IX~A^fUC)9d>O9ianlb>XwP z{*bUIGVxMGpK-!$CeL3C!}SaXVN8`>NS$6ELy$Ar3Y3YPgaxkGIHDcYhR3rut8t;f-MelX8xL@5<@*Z%e!|It3kIzf~U=^`o z6}gTxq**gZ-p2VAcgbJ(?3Q4GxVv;S3Z;WAAQ$BllBMguIjBv#LD6zj2$9d4@dA+O zOZ;q%!CqDaOuicrv8m-9aI-&W>?-Wtl_}qEI*^%Yb)FKc~^{sqk0l1wd6 zSom=MkRp}MY~;n?lnRY}Mx%F-e=Fk zQjsPO{$5MA8aDc-ruGm zwY{=myR@!B3TM$YbhwVcYBt3+IDcXGG&2)0HYN0$mP|48@x`na(5J)v;SYU^#Qi=S zYN`Xj0M2puX%47s^AUS-Z7Az|;_Ew(Zw(%!{HIM?1vi5y4909nPc9FY;RCPjIk(q#0hV`{J0O z#z}{X!!R)#iNT|S(L zkt0)2-f1W?a76G#$#``>FT>|=y*E_E?R`1?`WVyKMI4`Cq!-q1oxqaaGumf?<&<+8 zIDF3FmaihoWCbeG*#IIeevD2gcs;TH%BbfE3IqR?LpKnRqu~KRf!8!VoZvTq`<3gRctS3xWp#RO_Y*6(%A7N zs|Z(6l-5Sm3<=W2yW*4aJi*fJwx7zsa{J;z4~w^QoAUsIprKG^^24n^?BuV!kr!Fv zzpCAroz|4L0I?+cTPVJ@H|OY_H<;|UpUY=OyXqqTQ=kI1tW$6X7CGu}0-C-jDi;jDx6!QvoB7y`JZXvGZJR`T<08tV7M$+yO!+aFF> zK$bMW&^Gb%JxRWuo1VWzq0X08;^dn$q}c49o>=+uc9hi5m+QOeVjmgz=oF?n{=Usr zcLRm$uH0rifTT=9xf$6+$Z`?_{X+g04_n!8(Kw1lm8_r#VkR2=$b>gsxqy;>e>zE3 zTYY6v;hV;XBe%aW<&m=Ns=e~WQK)Nse@aLSUZ(*ErTU14+k3*dx-vN2n#6CSxd3s;&AlYo5t!5xl7JQD}6Um2VYqOYsZa_$)LU) zn$;5k8xi%&|6>i0{7UQ$42E~EM=eJg`^jB^H$&6@t|9h{3(kKlRLlx%2r~Y~-n#Qz zH9ZyQIag_ovQx3RNN<_<+2*DgT6>3rmMyev4O)D7u3z*pvsSPx^Bn5Cc~R)pjSV_47oh2*Trj{6Joz0j*L!oY}h)<*= zi4svLVK>&wmZ69$!kf5)ZCr}j0tmF%>N0+vr7f{+<$UW9Tnr|&zjT}4z=caisO+uA z4R-bf1+!kKDo_HUzA|zBcE9S1^$aT=_0kZDrF%Hre{l(=Uz!Q_r6?O(`OiT;)cUGQ z#$vQH>UlxE2pp}k0o*yuN@xdq)hJ>EqPaZrUg!v)12%wJ((4x4#oTY4jjG5A)G@DH zg6hv42%A{z-;~-F)Pmrc&&s@pn6duH|Jt?;q-_uWS#yIp3_c&K1kyDAO9l2eh(8FN z)xSNsSpM9ze{F*NH0<`Q~{C< zWSX4gJV6q^`S0!GOpAQUxRC{Jz#PQV=HlkM-wkxZ16H$iVxj?%kQ`e7^-U=kkW3T_9cWr}He8twG+V(A2=y?s^4f_>(bcL{TmJQ?(`26GWUIrQ@9zN!AH(sE| z)uO6IlS_PaoU`DiU$SP0WY~mXO4JS?$X=%Ayl>kXgAiS%cqGRp>^szy0WLVYF42Bh zdUj2M_8G1GS*cY&(%)UPFyv zffn==m%3q}w`Nw>UBq2RU1;KyHIC3#P^80r4p~d3_$k(%-0?fz%+b}vr$6>_^|g9V zxj>KsmTPi{G*{3{e`#6O#ga78-Z6?gdrNfbBmH<%)HGK`M)i;9F`sq7ICx!{{jBSD znEbK^@g3H5wWub>aKrjg3FY-{O-Y^q_3i5+^_*Y`xlGF~oAZxI^FwIjnk0%c>eMsw-6}K;w_CM$tQE#N%y^1GV)6;}YOEI6k`MWcW zR~zQKhR)L9!-JfsiZz_onk>OX^t8^+Q2!^(EjVMv@xsv?Is22DL2uJa>LSo@m$Rlj zJLj7@aevJ)3yJ<@yKLUx*nA+lXjp?1;Tt$I&j)YLQY+|ad06ndA=#S&Pny7!#eK(r z@2>6M9xa+W1C%X*7CEFa#rebw|xbdhLOU)t!B$CY1X zrYu$he%;o7g}XHP_L|=pd`ox0I6FgG=1V(EFY;m1KWX!UpBwEiG`>(z@+%zDb_}2U zwuUvG>l8w)R)1`AwlV+nrNYrVd!hP0!?LQpyDiww*~;}&;>(|!oi8M5PaW$7)zUk+ z+~>0fRbxwEF^Dts7>n=kWT{bqH&o?q|IA_%tjVnc4r?%;TFw&%Qe`lBo%wg*alS^B zko?IS^qwW?jYRt|jq1D=GNj%}vRf)(`pswEO+fbDRO6PhF01@4x~`(PBoC4k&CgHW zP_%gJX0e{@KEHKuV}Gz)sN~}7wEK%-6}&p$IB9dlZ#iS^E;~SmLAiG0A_PC&h?94O z_bh4`WYK!TJ1HFUt=UXmKJ?C`Fh`2^jT zh$_a(_c;WqgyS)F6uGM)VHWCmFS9C{{l1X2;7Fxq#LT>gEGyl!jSQ}}LQlNwTx;aE zwIFBGd7MEmc@ZJZx_tJ6m4n1MyNGxoVBKR4uJ6v)43{iU?&ZAvYSxSP4&AABB_PSU z{Fk;=r~mGs9GoKI-#i9#RwcZ02&rtntt9@}@bH|5keNq(o#pYa{)dwe#=SaclRDLt zQnDJu`-P)FVh{D$E$E!yYPTSrZ)|v|=ibfQ^N(vnX42mr+Zj&O+#w76azi9JZ(PDJP5?}oO?MR=W&zy7+72xN( zcgqG7*^;eiPh0U~Ieo_2OB{OIsG-U~-(?MIh0N1w#aj;*xW~PGIn`Oo+c66tJ6mG_ zBh-0_vt|s>qtz2|adXZE2Ge>}cipW}mCxh0$x9+%OJ%F>Kmk4e#bwz%T8pB#)l0JR zjGCTG`i5!DgTg_P$3N=#S`8zDETu2+-YlX(qyt1c3$exM?u6RhgO3Ap}330 zUUdKRxLM(IECfmcv6?!lZ?zTF0B8B)YeY@sY_ldhsMY zJ__aViCqN;z+p+*&hYbgN@j!V#ZJpUpJM~dhqroEqa^dl{h?V>SbBKH>`z8O06ck+ z_UCKSN_}r9HM&8`4LxQwKr+=f<|R zU#enfli%}gkfN}ZD67vb*4W&YK%HtQqZWuaePN>QWO1eFzh%*xJ(pw2y}v6?arvOO zN0~-1hhy3N-p#>zMn(y1k{=YF7_?w1M;}O$H~k=T!@zbcMz%NQle!A}jk6iLVQ(BC zZSMGFbXJ}*dG?_9;hIXZTNAi|e)os5;OKv1s_c0l!4K;p813Kn>8<0lL*mQ#h0hPy zFJEH*|G%#2Z7gIa;vkucc=axF?=uk4_9R37w_OJpaEF+wvA)) zk((_@aOZ~KI&a$1Pj8u02#;x;%2g@sJ22@E|7m&naGU?-n{*Iwdd!@kI6PvLv!Nc5 zy@rW|M94Vv$>ZSr;X-aTKa3DsOirWMyqg9ox&s|P2572SL*f;Cq%%#$1zf^Uw&WUm z3Pcm;K|eA*k`u-wM-ZfXdEbO@XPYJKB*a zG`zLeb&KKA%Wp8%8z7+`+^mS>FFT4voyOk;NwUr+706Q!BG>R_BqaBG=v$KU!Svfl z_h3*|9Fmadgm@Wc?CxG*`@Bq@epOF>{)}wYyr`r{6GeIJFQ7YvOg1y0M&=y1|_)pK-R|1mn4e(M}BvR(Uv!t!s8VO-O$QBUZrE9~AD;$X*`tb${rdkxKnCt9`rlpvmZ+)|2>I!UoomvR^?6z7w`F#IQZ zJ|p<|hj=XV*#E1p}&Vhyp_UzFUl%VAd zR=gaXN_J+A^3;po%dshTR;4zWT+HqrKiPi?p+fH!ZtAX@)ClinZgX29=LiGjXZRLz zNQR|T*+J7u*i%g(w6Z>WqX7;?@nd~yuLa8F8AH#zqsi)D;U?uM+xE)?HnwtXEWgGQ z1_IN7mAbQKUIKC5B}n6pi2lD%?{-n6ma z)EovYliOjb4exe85GXuv0DKwOQLi7n{>*@fiaiY;J&F}z>M)~M$4E1*m$kYDkTjh6 z_#PW)W|n8OsXzC$w8j@F(N2c>>4OMZ7}GORsM@_!fhe>NM~E~{j`?Bc8N39 zqnJb|KV^TzU|uGRoddh!Ib2q`jc>vZA(@UmXBS#)1OiN;s`llds1b9H)Q;dY!=L|X zlL2_qNclzd?utgx(WT})mmOqHpi0f6i+SkU)n1|cJ5$;-bh;8KL6xw`0s?P~#u26Z z=C~q_?fHBp9WwnlPU?|A)CBl>2unr1;=C=sU<#UL!aN#EjI%Rp8=`w2{C26atj&eW7JrtgKzZF5zdiDR=)V*LTR5{arCB|3Xxfo97lWiJUZFa zxjoM{(*&*RXl!u_RJj;tLxrg1K0F>T?Ry@5>nQyFK&FYEDi=mj{&%nIp?~vFZjldt z+X`Tj#7_w3{A^8 zg}FROU>C=Do50^Xo@Jy4$q-D&3oODy0f}@XE@Vo5m!o!9hzP{XjO)2oj$jrhmpmlH z;k-LM#}F!$tur43)YFdW6aT-549p z7mp_J8_?|Knd($Ahc?uAkYxyb6wBuhFU2QV=^yr0HxMM__gn(o;KJ-|-TZCU7_t)} zwCh>~z)d?>wh6z{D>(~6F8hSULktc&ranA^71%K-c{?l~lBK+P4hzXz z>C13n?1QVtd4@dLGHLABq<1y&qft;e<~65iGM&Q)I_Ih+Bg=kFy_#<6UA9X&#=ns8 zHs*xY%*(r_fo&Nk!iuOkY{Bk0Hrqq! zuJN(n3Z26>Bc6rO(m>1bKXf(OhY9)O{MYs0r0*Zp#nFnOR#*5{;4d>lxSEmX<9His zKpXtf)kjLk+&8m%*g&A6wy%p>Y-f%rE|()+Udd?tA!at rFc% z#X?x!=D9!X&>l*V2`(^B5>Vq205pw_t&_#=LoI<(2r?ZnAg!v8x%Yt?IYcwP8)Slj zH@KSi4-fz9(}I(Y;m%ex=Xw{@z5s4vZBKjIb^6FI{cT1zUL)Ah-Eu6&Qe@#>Z`^>8 z*8d|&2jOmbwb9hNA9>9J5~E>COBO~|vG>KBHn`N>i^hVwb}Lz2^PpTpKD1|S71N4{ zuiNuZ)M}VSlLoxYZkN>u`mgAY7wk-$065AdQBf69egvRUo{r%g2YsTku3q`{&DO&E zOYS}x2S;ubs;}9zm(mex#2S&z@|(6+haq^Kb8N>tMqIrK2wck<7<`M4h+X7c`!_S&lRfLn>3Bq35cfhC&dX#HgljpgIa_Dm7ea|pN@w=}}Za>+ntbSA7vDCdEkdxy$Sy@?o88go+)5Ih2b%!^>Zv&{ivvQjo6aUkzIYGa# z@X6TR07L)+XLvdL?q9@i-Otj5wD2;QOaO$+##jm>kRk_thSmu*5?);GxCVChe;b?v zdz;U+ z0_h!$6fhor`|3>$@2+ES0HwQCMnPG#z&xorC?@)Bt`3u?!fu06jf)-(5BOz8x0Ap; z0YNFa5vjy1{nZU|c)S<1|7r&~TswsfZ&Fp1*XQU*kkqdhc!!FCF<4`npn}~QTV^iQ zDeL%aAdaLxfGi2QSIK;4z@J31sHy!au;E&^rGTJ2p(fD(ej5WQ0A+CT+EY8WnSy>JRREji zlIqcP6zkjczXcchNV0gxS(jfWQIlTeMrX-y94@#dPZdH0{*w4o?XCvf0Kl{o2C%_VgV5Zc1Vw6kB5!-gmg&@&2@=?Bx*|mNCa|Fv}xPPU3uh^ z_WJ3L@QDiT3`9HRo*2So^`#i@<_eGK8@8A4gjQ+6D8P$(6p!M~^@PCJPcKQr<+VZU zPZ$L5i1mhZqo6Usmh6p`s)yQ;TcS&C?|-e0OgNK`+4G>qN_w-F9~A`_W3|(2Z#i!S z7NeXTv&#F==Eei+=9Guxp6 From d6790b00f9ad63a90828917fccbe2c2465260c7d Mon Sep 17 00:00:00 2001 From: weijingchen Date: Wed, 16 Feb 2022 15:17:51 +0800 Subject: [PATCH 41/99] update conf Signed-off-by: cwj --- .../test_fast_sbt_mix_multiclass_conf.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/dsl/v2/hetero_fast_secureboost/test_fast_sbt_mix_multiclass_conf.json b/examples/dsl/v2/hetero_fast_secureboost/test_fast_sbt_mix_multiclass_conf.json index 821aedeed1..cf461b368d 100644 --- a/examples/dsl/v2/hetero_fast_secureboost/test_fast_sbt_mix_multiclass_conf.json +++ b/examples/dsl/v2/hetero_fast_secureboost/test_fast_sbt_mix_multiclass_conf.json @@ -39,7 +39,7 @@ "0": { "reader_0": { "table": { - "name": "breast_hetero_guest", + "name": "vehicle_scale_hetero_guest", "namespace": "experiment" } }, @@ -49,7 +49,7 @@ }, "reader_1": { "table": { - "name": "breast_hetero_guest", + "name": "vehicle_scale_hetero_guest", "namespace": "experiment" } }, @@ -63,7 +63,7 @@ "0": { "reader_0": { "table": { - "name": "breast_hetero_host", + "name": "vehicle_scale_hetero_host", "namespace": "experiment" } }, @@ -72,7 +72,7 @@ }, "reader_1": { "table": { - "name": "breast_hetero_host", + "name": "vehicle_scale_hetero_host", "namespace": "experiment" } }, From 64f50dca8418f5c70877b830c5a8a2e31bdd8a02 Mon Sep 17 00:00:00 2001 From: weijingchen Date: Wed, 16 Feb 2022 18:48:43 +0800 Subject: [PATCH 42/99] add adaption to selection Signed-off-by: cwj --- .../hetero/hetero_fast_secureboost_host.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/python/federatedml/ensemble/boosting/hetero/hetero_fast_secureboost_host.py b/python/federatedml/ensemble/boosting/hetero/hetero_fast_secureboost_host.py index 08fe629491..6cad5376be 100644 --- a/python/federatedml/ensemble/boosting/hetero/hetero_fast_secureboost_host.py +++ b/python/federatedml/ensemble/boosting/hetero/hetero_fast_secureboost_host.py @@ -188,15 +188,16 @@ def get_model_param(self): feature_importances = list(self.feature_importances_.items()) feature_importances = sorted(feature_importances, key=itemgetter(1), reverse=True) feature_importance_param = [] - LOGGER.debug('host feat importance is {}'.format(feature_importances)) - for fid, importance in feature_importances: - feature_importance_param.append(FeatureImportanceInfo(sitename=self.role, - fid=fid, - importance=importance.importance, - fullname=self.feature_name_fid_mapping[fid], - main=importance.main_type - )) - model_param.feature_importances.extend(feature_importance_param) + if self.work_mode == consts.MIX_TREE: + LOGGER.debug('host feat importance is {}'.format(feature_importances)) + for fid, importance in feature_importances: + feature_importance_param.append(FeatureImportanceInfo(sitename=self.role, + fid=fid, + importance=importance.importance, + fullname=self.feature_name_fid_mapping[fid], + main=importance.main_type + )) + model_param.feature_importances.extend(feature_importance_param) return param_name, model_param From 5b4b4707262a495ce24c77fb664084368d2ae1b5 Mon Sep 17 00:00:00 2001 From: mgqa34 Date: Thu, 17 Feb 2022 01:02:52 +0800 Subject: [PATCH 43/99] optimize mini batch generator logic and hetero-lr batch logic Signed-off-by: mgqa34 --- .../hetero_lr_guest.py | 13 ++- .../hetero_lr_host.py | 6 +- .../federatedml/model_selection/mini_batch.py | 99 +++++++++++-------- .../gradient/hetero_linear_model_gradient.py | 68 +------------ .../gradient/hetero_lr_gradient_and_loss.py | 3 +- 5 files changed, 71 insertions(+), 118 deletions(-) diff --git a/python/federatedml/linear_model/logistic_regression/hetero_logistic_regression/hetero_lr_guest.py b/python/federatedml/linear_model/logistic_regression/hetero_logistic_regression/hetero_lr_guest.py index 2602e12a0f..f287777810 100644 --- a/python/federatedml/linear_model/logistic_regression/hetero_logistic_regression/hetero_lr_guest.py +++ b/python/federatedml/linear_model/logistic_regression/hetero_logistic_regression/hetero_lr_guest.py @@ -89,6 +89,11 @@ def fit_binary(self, data_instances, validate_data=None): LOGGER.debug(f"MODEL_STEP After load data, data count: {data_instances.count()}") self.cipher_operator = self.cipher.gen_paillier_cipher_operator() + self.batch_generator.initialize_batch_generator(data_instances, self.batch_size, + batch_strategy=self.batch_strategy, + masked_rate=self.masked_rate, shuffle=self.shuffle) + self.gradient_loss_operator.set_total_batch_nums(self.batch_generator.batch_nums) + use_async = False if with_weight(data_instances): if self.model_param.early_stop == "diff": @@ -97,19 +102,13 @@ def fit_binary(self, data_instances, validate_data=None): self.gradient_loss_operator.set_use_sample_weight() LOGGER.debug(f"instance weight scaled; use weighted gradient loss operator") # LOGGER.debug(f"data_instances after scale: {[v[1].weight for v in list(data_instances.collect())]}") - elif len(self.component_properties.host_party_idlist) == 1: + elif len(self.component_properties.host_party_idlist) == 1 and not self.batch_generator.batch_masked: LOGGER.debug(f"set_use_async") self.gradient_loss_operator.set_use_async() use_async = True self.transfer_variable.use_async.remote(use_async) LOGGER.info("Generate mini-batch from input data") - self.batch_generator.initialize_batch_generator(data_instances, self.batch_size, - batch_strategy=self.batch_strategy, - masked_rate=self.masked_rate, shuffle=self.shuffle) - self.gradient_loss_operator.set_total_batch_nums(self.batch_generator.batch_nums) - if self.batch_generator.batch_masked: - self.gradient_loss_operator.set_use_sync() self.encrypted_calculator = [EncryptModeCalculator(self.cipher_operator, self.encrypted_mode_calculator_param.mode, diff --git a/python/federatedml/linear_model/logistic_regression/hetero_logistic_regression/hetero_lr_host.py b/python/federatedml/linear_model/logistic_regression/hetero_logistic_regression/hetero_lr_host.py index 5f0b1e0a31..8267436568 100644 --- a/python/federatedml/linear_model/logistic_regression/hetero_logistic_regression/hetero_lr_host.py +++ b/python/federatedml/linear_model/logistic_regression/hetero_logistic_regression/hetero_lr_host.py @@ -101,14 +101,12 @@ def fit_binary(self, data_instances, validate_data): self.header = self.get_header(data_instances) self.cipher_operator = self.cipher.gen_paillier_cipher_operator() + self.batch_generator.initialize_batch_generator(data_instances, shuffle=self.shuffle) + if self.transfer_variable.use_async.get(idx=0): LOGGER.debug(f"set_use_async") self.gradient_loss_operator.set_use_async() - self.batch_generator.initialize_batch_generator(data_instances, shuffle=self.shuffle) - if self.batch_generator.batch_masked: - self.gradient_loss_operator.set_use_sync() - self.gradient_loss_operator.set_total_batch_nums(self.batch_generator.batch_nums) self.encrypted_calculator = [EncryptModeCalculator(self.cipher_operator, diff --git a/python/federatedml/model_selection/mini_batch.py b/python/federatedml/model_selection/mini_batch.py index b30ad41caa..9f52cf9174 100644 --- a/python/federatedml/model_selection/mini_batch.py +++ b/python/federatedml/model_selection/mini_batch.py @@ -64,9 +64,8 @@ def mini_batch_data_generator(self, result='data'): for batch_data, index_table in zip(self.all_batch_data, self.all_index_data): yield batch_data, index_table - if self.batch_mutable: - self.__generate_batch_data() - + # if self.batch_mutable: + # self.__generate_batch_data() def __init_mini_batch_data_seperator(self, data_insts, batch_size, batch_strategy, masked_rate, shuffle): self.data_sids_iter, data_size = indices.collect_index(data_insts) @@ -95,6 +94,9 @@ def get_batch_generator(data_size, batch_size, batch_strategy, masked_rate, shuf # LOGGER.warning("Masked dataset's batch_size >= data size, batch shuffle will be disabled") # return FullBatchDataGenerator(data_size, data_size, shuffle=False, masked_rate=masked_rate) if batch_strategy == "full": + if masked_rate > 0: + LOGGER.warning("If using full batch strategy and masked rate > 0, shuffle will always be true") + shuffle = True return FullBatchDataGenerator(data_size, batch_size, shuffle=shuffle, masked_rate=masked_rate) else: if shuffle: @@ -102,13 +104,40 @@ def get_batch_generator(data_size, batch_size, batch_strategy, masked_rate, shuf return RandomBatchDataGenerator(data_size, batch_size, masked_rate) -class FullBatchDataGenerator(object): +class BatchDataGenerator(object): def __init__(self, data_size, batch_size, shuffle=False, masked_rate=0): - self.batch_nums = (data_size + batch_size - 1) // batch_size + self.batch_nums = None self.masked_dataset_size = min(data_size, round((1 + masked_rate) * batch_size)) self.batch_size = batch_size self.shuffle = shuffle + def batch_masked(self): + return self.masked_dataset_size != self.batch_size + + def batch_mutable(self): + return True + + @staticmethod + def _generate_batch_data_with_batch_ids(data_insts, batch_ids, masked_ids=None): + batch_index_table = session.parallelize(batch_ids, + include_key=True, + partition=data_insts.partitions) + batch_data_table = batch_index_table.join(data_insts, lambda x, y: y) + + if masked_ids: + masked_index_table = session.parallelize(masked_ids, + include_key=True, + partition=data_insts.partitions) + return masked_index_table, batch_data_table + else: + return batch_index_table, batch_data_table + + +class FullBatchDataGenerator(BatchDataGenerator): + def __init__(self, data_size, batch_size, shuffle=False, masked_rate=0): + super(FullBatchDataGenerator, self).__init__(data_size, batch_size, shuffle, masked_rate=masked_rate) + self.batch_nums = (data_size + batch_size - 1) // batch_size + LOGGER.debug(f"Init Full Batch Data Generator, batch_nums: {self.batch_nums}, batch_size: {self.batch_size}, " f"masked_dataset_size: {self.masked_dataset_size}, shuffle: {self.shuffle}") @@ -121,32 +150,26 @@ def generate_data(self, data_insts, data_sids): if self.batch_size != self.masked_dataset_size: for bid in range(self.batch_nums): batch_ids = data_sids[bid * self.batch_size:(bid + 1) * self.batch_size] - masked_ids = set() + masked_ids_set = set() for sid, _ in batch_ids: - masked_ids.add(sid) + masked_ids_set.add(sid) possible_ids = random.SystemRandom().sample(data_sids, self.masked_dataset_size) for pid, _ in possible_ids: - if pid not in masked_ids: - masked_ids.add(pid) - if len(masked_ids) == self.masked_dataset_size: + if pid not in masked_ids_set: + masked_ids_set.add(pid) + if len(masked_ids_set) == self.masked_dataset_size: break - masked_index_table = session.parallelize(zip(list(masked_ids), [None] * len(masked_ids)), - include_key=True, - partition=data_insts.partitions) - batch_index_table = session.parallelize(batch_ids, - include_key=True, - partition=data_insts.partitions) - batch_data_table = batch_index_table.join(data_insts, lambda x, y: y) + masked_ids = zip(list(masked_ids_set), [None] * len(masked_ids_set)) + masked_index_table, batch_data_table = self._generate_batch_data_with_batch_ids(data_insts, + batch_ids, + masked_ids) index_table.append(masked_index_table) batch_data.append(batch_data_table) else: for bid in range(self.batch_nums): - batch_ids = data_sids[bid * self.batch_size : (bid + 1) * self.batch_size] - batch_index_table = session.parallelize(batch_ids, - include_key=True, - partition=data_insts.partitions) - batch_data_table = batch_index_table.join(data_insts, lambda x, y: y) + batch_ids = data_sids[bid * self.batch_size: (bid + 1) * self.batch_size] + batch_index_table, batch_data_table = self._generate_batch_data_with_batch_ids(data_insts, batch_ids) index_table.append(batch_index_table) batch_data.append(batch_data_table) @@ -155,32 +178,26 @@ def generate_data(self, data_insts, data_sids): def batch_mutable(self): return self.masked_dataset_size > self.batch_size or self.shuffle - def batch_masked(self): - return self.masked_dataset_size != self.batch_size - -class RandomBatchDataGenerator(object): +class RandomBatchDataGenerator(BatchDataGenerator): def __init__(self, data_size, batch_size, masked_rate=0): + super(RandomBatchDataGenerator, self).__init__(data_size, batch_size, shuffle=False, masked_rate=masked_rate) self.batch_nums = 1 - self.batch_size = batch_size - self.masked_dataset_size = min(data_size, round((1 + masked_rate) * self.batch_size)) LOGGER.debug(f"Init Random Batch Data Generator, batch_nums: {self.batch_nums}, batch_size: {self.batch_size}, " f"masked_dataset_size: {self.masked_dataset_size}") - def generate_data(self, data_insts, *args, **kwargs): + def generate_data(self, data_insts, data_sids): if self.masked_dataset_size == self.batch_size: - batch_data = data_insts.sample(num=self.batch_size) - index_data = batch_data.mapValues(lambda value: None) - return [index_data], [batch_data] + batch_ids = random.SystemRandom().sample(data_sids, self.batch_size) + batch_index_table, batch_data_table = self._generate_batch_data_with_batch_ids(data_insts, batch_ids) + batch_data_table = batch_index_table.join(data_insts, lambda x, y: y) + return [batch_index_table], [batch_data_table] else: - masked_data = data_insts.sample(num=self.masked_dataset_size) - batch_data = masked_data.sample(num=self.batch_size) - masked_index_table = masked_data.mapValues(lambda value: None) - return [masked_index_table], [batch_data] - - def batch_mutable(self): - return True + masked_ids = random.SystemRandom().sample(data_sids, self.masked_dataset_size) + batch_ids = masked_ids[: self.batch_size] + masked_index_table, batch_data_table = self._generate_batch_data_with_batch_ids(data_insts, + batch_ids, + masked_ids) + return [masked_index_table], [batch_data_table] - def batch_masked(self): - return self.masked_dataset_size != self.batch_size diff --git a/python/federatedml/optim/gradient/hetero_linear_model_gradient.py b/python/federatedml/optim/gradient/hetero_linear_model_gradient.py index 9422584041..f0c5275157 100644 --- a/python/federatedml/optim/gradient/hetero_linear_model_gradient.py +++ b/python/federatedml/optim/gradient/hetero_linear_model_gradient.py @@ -57,70 +57,6 @@ def set_fixed_float_precision(self, floating_point_precision): if floating_point_precision is not None: self.fixed_point_encoder = FixedPointEncoder(2**floating_point_precision) - # @staticmethod - # def __compute_partition_gradient(data, fit_intercept=True, is_sparse=False): - # """ - # Compute hetero regression gradient for: - # gradient = ∑d*x, where d is fore_gradient which differ from different algorithm - # Parameters - # ---------- - # data: Table, include fore_gradient and features - # fit_intercept: bool, if model has interception or not. Default True - - # Returns - # ---------- - # numpy.ndarray - # hetero regression model gradient - # """ - # feature = [] - # fore_gradient = [] - - # if is_sparse: - # row_indice = [] - # col_indice = [] - # data_value = [] - - # row = 0 - # feature_shape = None - # for key, (sparse_features, d) in data: - # fore_gradient.append(d) - # assert isinstance(sparse_features, SparseVector) - # if feature_shape is None: - # feature_shape = sparse_features.get_shape() - # for idx, v in sparse_features.get_all_data(): - # col_indice.append(idx) - # row_indice.append(row) - # data_value.append(v) - # row += 1 - # if feature_shape is None or feature_shape == 0: - # return 0 - # sparse_matrix = sp.csr_matrix((data_value, (row_indice, col_indice)), shape=(row, feature_shape)) - # fore_gradient = np.array(fore_gradient) - - # # gradient = sparse_matrix.transpose().dot(fore_gradient).tolist() - # gradient = fate_operator.dot(sparse_matrix.transpose(), fore_gradient).tolist() - # if fit_intercept: - # bias_grad = np.sum(fore_gradient) - # gradient.append(bias_grad) - # # LOGGER.debug("In first method, gradient: {}, bias_grad: {}".format(gradient, bias_grad)) - # return np.array(gradient) - - # else: - # for key, value in data: - # feature.append(value[0]) - # fore_gradient.append(value[1]) - # feature = np.array(feature) - # fore_gradient = np.array(fore_gradient) - # if feature.shape[0] <= 0: - # return 0 - - # gradient = fate_operator.dot(feature.transpose(), fore_gradient) - # gradient = gradient.tolist() - # if fit_intercept: - # bias_grad = np.sum(fore_gradient) - # gradient.append(bias_grad) - # return np.array(gradient) - @staticmethod def __apply_cal_gradient(data, fixed_point_encoder, is_sparse): all_g = None @@ -153,6 +89,7 @@ def compute_gradient(self, data_instances, fore_gradient, fit_intercept, need_av data_instances: Table, input data fore_gradient: Table, fore_gradient fit_intercept: bool, if model has intercept or not + need_average: bool, gradient needs to be averaged or not Returns ---------- @@ -249,7 +186,8 @@ def _centralized_compute_gradient(self, data_instances, model_weights, cipher, c for host_forward in self.host_forwards: if self.use_sample_weight: - host_forward = host_forward.join(data_instances, lambda h, v: h * v.weight) + # host_forward = host_forward.join(data_instances, lambda h, v: h * v.weight) + host_forward = data_instances.join(host_forward, lambda v, h: h * v.weight) fore_gradient = fore_gradient.join(host_forward, lambda x, y: x + y) def _apply_obfuscate(val): diff --git a/python/federatedml/optim/gradient/hetero_lr_gradient_and_loss.py b/python/federatedml/optim/gradient/hetero_lr_gradient_and_loss.py index 848de62585..6d50bd4df9 100644 --- a/python/federatedml/optim/gradient/hetero_lr_gradient_and_loss.py +++ b/python/federatedml/optim/gradient/hetero_lr_gradient_and_loss.py @@ -74,7 +74,8 @@ def compute_loss(self, data_instances, w, n_iter_, batch_index, loss_norm=None, current_suffix = (n_iter_, batch_index) n = data_instances.count() - host_wx_y = self.host_forwards[0].join(data_instances, lambda x, y: (x, y.label)) + # host_wx_y = self.host_forwards[0].join(data_instances, lambda x, y: (x, y.label)) + host_wx_y = data_instances.join(self.host_forwards[0], lambda y, x: (x, y.label)) self_wx_y = self.half_d.join(data_instances, lambda x, y: (x, y.label)) def _sum_ywx(wx_y): From 9fb256d1916194af8a69dc3e98319157c7069f46 Mon Sep 17 00:00:00 2001 From: dylanfan <289765648@qq.com> Date: Thu, 17 Feb 2022 12:35:53 +0800 Subject: [PATCH 44/99] rename fixedpoin to fixedpoint_test.py Signed off by: dylanfan <289765648@qq.com> --- .../secureprotol/test/{fixedpoin_test.py => fixedpoint_test.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename python/federatedml/secureprotol/test/{fixedpoin_test.py => fixedpoint_test.py} (100%) diff --git a/python/federatedml/secureprotol/test/fixedpoin_test.py b/python/federatedml/secureprotol/test/fixedpoint_test.py similarity index 100% rename from python/federatedml/secureprotol/test/fixedpoin_test.py rename to python/federatedml/secureprotol/test/fixedpoint_test.py From 373eda3d1e28390132b13cff1b6e595a9f2cdf46 Mon Sep 17 00:00:00 2001 From: zhihuiwan <15779896112@163.com> Date: Thu, 17 Feb 2022 14:27:13 +0800 Subject: [PATCH 45/99] update connector key --- python/fate_arch/common/address.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/fate_arch/common/address.py b/python/fate_arch/common/address.py index 280b9ad614..26c8b6d9b4 100644 --- a/python/fate_arch/common/address.py +++ b/python/fate_arch/common/address.py @@ -10,7 +10,7 @@ def __init__(self, connector_name=None): connector = StorageConnector(connector_name=connector_name, engine=self.get_name) if connector.get_info(): for k, v in connector.get_info().items(): - if hasattr(self, k): + if hasattr(self, k) and v: self.__setattr__(k, v) @property @@ -138,7 +138,7 @@ def __repr__(self): @property def connector(self): - return {"user": self.user, "passwd": self.passwd, "host": self.host, "port": self.port} + return {"user": self.user, "passwd": self.passwd, "host": self.host, "port": self.port, "db": self.db} @property def get_name(self): @@ -168,7 +168,7 @@ def __repr__(self): @property def connector(self): - return {"host": self.host, "port": self.port, "username": self.username, "password": self.password, "auth_mechanism": self.auth_mechanism} + return {"host": self.host, "port": self.port, "username": self.username, "password": self.password, "auth_mechanism": self.auth_mechanism, "database": self.database} @property def get_name(self): From 5c6340bc6b7702bba545bc14822c7f6621bffa66 Mon Sep 17 00:00:00 2001 From: mgqa34 Date: Thu, 17 Feb 2022 17:49:11 +0800 Subject: [PATCH 46/99] enhance encrypt_mode tool Signed-off-by: mgqa34 --- .../pipeline/param/hetero_nn_param.py | 2 +- .../param/encrypted_mode_calculation_param.py | 12 ++-- python/federatedml/param/hetero_nn_param.py | 2 +- .../federatedml/secureprotol/encrypt_mode.py | 69 +++---------------- 4 files changed, 17 insertions(+), 68 deletions(-) diff --git a/python/fate_client/pipeline/param/hetero_nn_param.py b/python/fate_client/pipeline/param/hetero_nn_param.py index 5e54e1921a..d26181e65a 100644 --- a/python/fate_client/pipeline/param/hetero_nn_param.py +++ b/python/fate_client/pipeline/param/hetero_nn_param.py @@ -105,7 +105,7 @@ def __init__(self, early_stop="diff", tol=1e-5, encrypt_param=EncryptParam(), - encrypted_mode_calculator_param=EncryptedModeCalculatorParam(mode="confusion_opt"), + encrypted_mode_calculator_param=EncryptedModeCalculatorParam(), predict_param=PredictParam(), cv_param=CrossValidationParam(), validation_freqs=None, diff --git a/python/federatedml/param/encrypted_mode_calculation_param.py b/python/federatedml/param/encrypted_mode_calculation_param.py index 55d1d3ad36..4107a75259 100644 --- a/python/federatedml/param/encrypted_mode_calculation_param.py +++ b/python/federatedml/param/encrypted_mode_calculation_param.py @@ -17,6 +17,7 @@ # limitations under the License. # from federatedml.param.base_param import BaseParam +from federatedml.util import LOGGER class EncryptedModeCalculatorParam(BaseParam): @@ -42,11 +43,10 @@ def check(self): ["strict", "fast", "balance", "confusion_opt", "confusion_opt_balance"], descr) - if self.mode in ["balance", "confusion_opt_balance"]: - if type(self.re_encrypted_rate).__name__ not in ["int", "long", "float"]: - raise ValueError("re_encrypted_rate should be a numeric number") - - if not 0.0 <= self.re_encrypted_rate <= 1: - raise ValueError("re_encrypted_rate should in [0, 1]") + if self.mode != "strict": + LOGGER.warning("encrypted_mode_calculator will be remove in later version, " + "but in current version user can still use it, but it only supports strict mode, " + "other mode will be reset to strict for compatibility") + self.mode = "strict" return True diff --git a/python/federatedml/param/hetero_nn_param.py b/python/federatedml/param/hetero_nn_param.py index e6cfc82dbd..d440e02ca1 100644 --- a/python/federatedml/param/hetero_nn_param.py +++ b/python/federatedml/param/hetero_nn_param.py @@ -111,7 +111,7 @@ def __init__(self, early_stop="diff", tol=1e-5, encrypt_param=EncryptParam(), - encrypted_mode_calculator_param=EncryptedModeCalculatorParam(mode="confusion_opt"), + encrypted_mode_calculator_param=EncryptedModeCalculatorParam(), predict_param=PredictParam(), cv_param=CrossValidationParam(), validation_freqs=None, diff --git a/python/federatedml/secureprotol/encrypt_mode.py b/python/federatedml/secureprotol/encrypt_mode.py index bb9a61d913..677a697611 100644 --- a/python/federatedml/secureprotol/encrypt_mode.py +++ b/python/federatedml/secureprotol/encrypt_mode.py @@ -20,6 +20,7 @@ from collections import Iterable from federatedml.secureprotol import PaillierEncrypt from federatedml.util import consts +from federatedml.util import LOGGER class EncryptModeCalculator(object): @@ -32,10 +33,6 @@ class EncryptModeCalculator(object): mode: str, accpet 'strict', 'fast', 'balance'. "confusion_opt", "confusion_opt_balance" 'strict': means that re-encrypted every function call. - 'fast/confusion_opt": one record use only on confusion in encryption once during iteration. - 'balance/confusion_opt_balance": balance of 'confusion_opt', will use new confusion according to probability - decides by 're_encrypted_rate' - re_encrypted_rate: float or float, numeric, use if mode equals to "balance" or "confusion_opt_balance" """ @@ -48,67 +45,21 @@ def __init__(self, encrypter=None, mode="strict", re_encrypted_rate=1): self.enc_zeros = None self.align_to_input_data = True - self.soft_link_mode() - def soft_link_mode(self): - - if self.mode == "strict": - return - - if self.mode in ["confusion_opt", "fast"]: - self.mode = "fast" - - if self.mode in ["confusion_opt_balance", "balance"]: - self.mode = "balance" + if self.mode != "strict": + self.mode = "strict" + LOGGER.warning("encrypted_mode_calculator will be remove in later version, " + "but in current version user can still use it, but it only supports strict mode, " + "other mode will be reset to strict for compatibility") @staticmethod def add_enc_zero(obj, enc_zero): - if isinstance(obj, np.ndarray): - return obj + enc_zero - elif isinstance(obj, Iterable): - return type(obj)( - EncryptModeCalculator.add_enc_zero(o, enc_zero) if isinstance(o, Iterable) else o + enc_zero for o in - obj) - else: - return enc_zero + obj - - @staticmethod - def gen_random_number(): - return random.random() - - def should_re_encrypted(self): - return self.gen_random_number() <= self.re_encrypted_rate + consts.FLOAT_ZERO - - def set_enc_zeros(self, input_data, enc_func): - self.enc_zeros = input_data.mapValues(lambda val: enc_func(0)) - - def re_encrypt(self, input_data, enc_func): - if input_data is None: # no need to re-encrypt - return - self.set_enc_zeros(input_data, enc_func) + pass def encrypt_data(self, input_data, enc_func): - - if self.mode == "strict": - new_data = input_data.mapValues(enc_func) - return new_data - else: - target_data = input_data - if self.enc_zeros is None: - self.set_enc_zeros(target_data, enc_func) - elif self.mode == "balance" and self.should_re_encrypted(): - if not self.align_to_input_data: - target_data = self.enc_zeros - self.re_encrypt(target_data, enc_func) - elif self.enc_zeros.count() != input_data.count(): - if not self.align_to_input_data: - target_data = None - self.re_encrypt(target_data, enc_func) - new_data = input_data.join(self.enc_zeros, self.add_enc_zero) - return new_data + return input_data.mapValues(enc_func) def get_enc_func(self, encrypter, raw_enc=False, exponent=0): - if not raw_enc: return encrypter.recursive_encrypt else: @@ -137,14 +88,12 @@ def encrypt(self, input_data): return new_data def raw_encrypt(self, input_data, exponent=0): - raw_en_func = self.get_enc_func(self.encrypter, raw_enc=True, exponent=exponent) new_data = self.encrypt_data(input_data, raw_en_func) return new_data def init_enc_zero(self, input_data, raw_en=False, exponent=0): - en_func = self.get_enc_func(self.encrypter, raw_en, exponent) - self.set_enc_zeros(input_data, en_func) + pass def recursive_encrypt(self, input_data): return self.encrypter.recursive_encrypt(input_data) From 76e3de294de833256a0278ed2d5cbec6c7105e59 Mon Sep 17 00:00:00 2001 From: weijingchen Date: Fri, 18 Feb 2022 15:55:18 +0800 Subject: [PATCH 47/99] fix board display of fast-sbt host Signed-off-by: cwj --- .../decision_tree/hetero/hetero_fast_decision_tree_host.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_fast_decision_tree_host.py b/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_fast_decision_tree_host.py index 98aed25e6a..808f3d9c5d 100644 --- a/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_fast_decision_tree_host.py +++ b/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_fast_decision_tree_host.py @@ -266,12 +266,15 @@ def sync_cur_layer_nodes(self, nodes, dep): def process_leaves_info(self): # remove g/h info and rename leaves + # record node info for node in self.tree_node: node.sum_grad = None node.sum_hess = None if node.is_leaf: node.sitename = consts.GUEST + self.split_maskdict[node.id] = node.bid + self.missing_dir_maskdict[node.id] = node.missing_dir def mask_node_id(self, nodes): for n in nodes: @@ -286,6 +289,7 @@ def convert_bin_to_real2(self): if not node.is_leaf: node.bid = self.bin_split_points[node.fid][node.bid] + """ Mix Mode """ From 3550710090347d96b272c5bffea80be53751d8a3 Mon Sep 17 00:00:00 2001 From: weijingchen Date: Fri, 18 Feb 2022 16:15:39 +0800 Subject: [PATCH 48/99] fix board display of fast-sbt host Signed-off-by: cwj --- .../decision_tree/hetero/hetero_fast_decision_tree_host.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_fast_decision_tree_host.py b/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_fast_decision_tree_host.py index 808f3d9c5d..93a461af29 100644 --- a/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_fast_decision_tree_host.py +++ b/python/federatedml/ensemble/basic_algorithms/decision_tree/hetero/hetero_fast_decision_tree_host.py @@ -273,8 +273,9 @@ def process_leaves_info(self): node.sum_hess = None if node.is_leaf: node.sitename = consts.GUEST - self.split_maskdict[node.id] = node.bid - self.missing_dir_maskdict[node.id] = node.missing_dir + else: + self.split_maskdict[node.id] = node.bid + self.missing_dir_maskdict[node.id] = node.missing_dir def mask_node_id(self, nodes): for n in nodes: From 27ad2f04d07ad3e61ef88f6a0fcb37204efcffc7 Mon Sep 17 00:00:00 2001 From: weijingchen Date: Fri, 18 Feb 2022 17:40:27 +0800 Subject: [PATCH 49/99] update conf/dsl and waring Signed-off-by: cwj --- ...ecureboost_EINI_with_ramdom_mask_conf.json | 86 ++++++++++++++ ...peline-hetero-sbt-EINI-with-random-mask.py | 108 ++++++++++++++++++ .../pipeline/param/boosting_param.py | 2 + python/federatedml/param/boosting_param.py | 5 + 4 files changed, 201 insertions(+) create mode 100644 examples/dsl/v2/hetero_secureboost/test_secureboost_EINI_with_ramdom_mask_conf.json create mode 100644 examples/pipeline/hetero_sbt/pipeline-hetero-sbt-EINI-with-random-mask.py diff --git a/examples/dsl/v2/hetero_secureboost/test_secureboost_EINI_with_ramdom_mask_conf.json b/examples/dsl/v2/hetero_secureboost/test_secureboost_EINI_with_ramdom_mask_conf.json new file mode 100644 index 0000000000..a74c6a3197 --- /dev/null +++ b/examples/dsl/v2/hetero_secureboost/test_secureboost_EINI_with_ramdom_mask_conf.json @@ -0,0 +1,86 @@ +{ + "dsl_version": 2, + "initiator": { + "role": "guest", + "party_id": 9999 + }, + "role": { + "host": [ + 9998 + ], + "guest": [ + 9999 + ] + }, + "component_parameters": { + "common": { + "hetero_secure_boost_0": { + "task_type": "classification", + "objective_param": { + "objective": "cross_entropy" + }, + "num_trees": 3, + "validation_freqs": 1, + "encrypt_param": { + "method": "Paillier" + }, + "tree_param": { + "max_depth": 3 + }, + "EINI_inference": true, + "EINI_random_mask": true + }, + "evaluation_0": { + "eval_type": "binary" + } + }, + "role": { + "guest": { + "0": { + "reader_1": { + "table": { + "name": "breast_hetero_guest", + "namespace": "experiment" + } + }, + "reader_0": { + "table": { + "name": "breast_hetero_guest", + "namespace": "experiment" + } + }, + "data_transform_0": { + "with_label": true, + "output_format": "dense" + }, + "data_transform_1": { + "with_label": true, + "output_format": "dense" + } + } + }, + "host": { + "0": { + "reader_1": { + "table": { + "name": "breast_hetero_host", + "namespace": "experiment" + } + }, + "reader_0": { + "table": { + "name": "breast_hetero_host", + "namespace": "experiment" + } + }, + "data_transform_0": { + "with_label": false + }, + "data_transform_1": { + "with_label": false + } + } + } + } + } +} \ No newline at end of file diff --git a/examples/pipeline/hetero_sbt/pipeline-hetero-sbt-EINI-with-random-mask.py b/examples/pipeline/hetero_sbt/pipeline-hetero-sbt-EINI-with-random-mask.py new file mode 100644 index 0000000000..33f81dffa4 --- /dev/null +++ b/examples/pipeline/hetero_sbt/pipeline-hetero-sbt-EINI-with-random-mask.py @@ -0,0 +1,108 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import argparse + +from pipeline.backend.pipeline import PipeLine +from pipeline.component import DataTransform +from pipeline.component import HeteroSecureBoost +from pipeline.component import Intersection +from pipeline.component import Reader +from pipeline.interface import Data +from pipeline.component import Evaluation +from pipeline.interface import Model + +from pipeline.utils.tools import load_job_config + + +def main(config="../../config.yaml", namespace=""): + # obtain config + if isinstance(config, str): + config = load_job_config(config) + parties = config.parties + guest = parties.guest[0] + host = parties.host[0] + + + # data sets + guest_train_data = {"name": "breast_hetero_guest", "namespace": f"experiment{namespace}"} + host_train_data = {"name": "breast_hetero_host", "namespace": f"experiment{namespace}"} + + guest_validate_data = {"name": "breast_hetero_guest", "namespace": f"experiment{namespace}"} + host_validate_data = {"name": "breast_hetero_host", "namespace": f"experiment{namespace}"} + + # init pipeline + pipeline = PipeLine().set_initiator(role="guest", party_id=guest).set_roles(guest=guest, host=host,) + + # set data reader and data-io + + reader_0, reader_1 = Reader(name="reader_0"), Reader(name="reader_1") + reader_0.get_party_instance(role="guest", party_id=guest).component_param(table=guest_train_data) + reader_0.get_party_instance(role="host", party_id=host).component_param(table=host_train_data) + reader_1.get_party_instance(role="guest", party_id=guest).component_param(table=guest_validate_data) + reader_1.get_party_instance(role="host", party_id=host).component_param(table=host_validate_data) + + data_transform_0, data_transform_1 = DataTransform(name="data_transform_0"), DataTransform(name="data_transform_1") + + data_transform_0.get_party_instance(role="guest", party_id=guest).component_param(with_label=True, output_format="dense") + data_transform_0.get_party_instance(role="host", party_id=host).component_param(with_label=False) + data_transform_1.get_party_instance(role="guest", party_id=guest).component_param(with_label=True, output_format="dense") + data_transform_1.get_party_instance(role="host", party_id=host).component_param(with_label=False) + + # data intersect component + intersect_0 = Intersection(name="intersection_0") + intersect_1 = Intersection(name="intersection_1") + + # secure boost component + hetero_secure_boost_0 = HeteroSecureBoost(name="hetero_secure_boost_0", + num_trees=3, + task_type="classification", + objective_param={"objective": "cross_entropy"}, + encrypt_param={"method": "Paillier"}, + tree_param={"max_depth": 3}, + validation_freqs=1, + EINI_random_mask=True + ) + + # evaluation component + evaluation_0 = Evaluation(name="evaluation_0", eval_type="binary") + + pipeline.add_component(reader_0) + pipeline.add_component(reader_1) + pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data)) + pipeline.add_component(data_transform_1, data=Data(data=reader_1.output.data), model=Model(data_transform_0.output.model)) + pipeline.add_component(intersect_0, data=Data(data=data_transform_0.output.data)) + pipeline.add_component(intersect_1, data=Data(data=data_transform_1.output.data)) + pipeline.add_component(hetero_secure_boost_0, data=Data(train_data=intersect_0.output.data, + validate_data=intersect_1.output.data)) + pipeline.add_component(evaluation_0, data=Data(data=hetero_secure_boost_0.output.data)) + + pipeline.compile() + pipeline.fit() + + print("fitting hetero secureboost done, result:") + print(pipeline.get_component("hetero_secure_boost_0").get_summary()) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser("PIPELINE DEMO") + parser.add_argument("-config", type=str, + help="config file") + args = parser.parse_args() + if args.config is not None: + main(args.config) + else: + main() diff --git a/python/fate_client/pipeline/param/boosting_param.py b/python/fate_client/pipeline/param/boosting_param.py index 014807cd8d..32eed9c0a3 100644 --- a/python/fate_client/pipeline/param/boosting_param.py +++ b/python/fate_client/pipeline/param/boosting_param.py @@ -511,6 +511,8 @@ def check(self): self.check_positive_number(self.top_rate, 'top_rate') self.check_boolean(self.new_ver, 'code version switcher') self.check_boolean(self.cipher_compress, 'cipher compress') + self.check_boolean(self.EINI_inference, 'eini inference') + self.check_boolean(self.EINI_random_mask, 'eini random mask') if self.top_rate + self.other_rate >= 1: raise ValueError('sum of top rate and other rate should be smaller than 1') diff --git a/python/federatedml/param/boosting_param.py b/python/federatedml/param/boosting_param.py index a2e0077584..11a66829d4 100644 --- a/python/federatedml/param/boosting_param.py +++ b/python/federatedml/param/boosting_param.py @@ -562,6 +562,11 @@ def check(self): self.check_boolean(self.EINI_inference, 'eini inference') self.check_boolean(self.EINI_random_mask, 'eini random mask') + if self.EINI_inference and self.EINI_random_mask: + LOGGER.warning('To protect the inference decision path, notice that current setting will multiply' + ' predict result by a random number, hence SecureBoost will return confused predict scores' + ' that is not the same as the original predict scores') + for p in ["early_stopping_rounds", "validation_freqs", "metrics", "use_first_metric_only"]: # if self._warn_to_deprecate_param(p, "", ""): From 7ef33445ca82571b6f7c08e2888395fb304f8ddb Mon Sep 17 00:00:00 2001 From: mgqa34 Date: Mon, 21 Feb 2022 09:28:11 +0800 Subject: [PATCH 50/99] update submodule of develop-1.7.2 Signed-off-by: mgqa34 --- .gitignore | 6 +++--- .gitmodules | 4 ++-- fate.env | 6 +++--- fateboard | 2 +- fateflow | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index caf7b09c60..04184e6c30 100644 --- a/.gitignore +++ b/.gitignore @@ -37,8 +37,8 @@ venv /jobs/ /audit/ .vscode/* -/eggroll/ -/fateboard/ +# /eggroll/ +# /fateboard/ /doc/federatedml_component/params/ # build dir @@ -65,4 +65,4 @@ entry_points.txt dependency_links.txt requires.txt SOURCES.txt -top_level.txt \ No newline at end of file +top_level.txt diff --git a/.gitmodules b/.gitmodules index 24d22bf9f5..bfa4688abf 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,7 @@ [submodule "fateboard"] path = fateboard url = https://github.com/FederatedAI/FATE-Board.git - branch = v1.7.2 + branch = v1.7.2.1 [submodule "eggroll"] path = eggroll url = https://github.com/WeBankFinTech/eggroll.git @@ -9,4 +9,4 @@ [submodule "fateflow"] path = fateflow url = https://github.com/FederatedAI/FATE-Flow.git - branch = v1.7.1 + branch = develop-1.7.2 diff --git a/fate.env b/fate.env index bb3624688a..8c2a3f9a17 100755 --- a/fate.env +++ b/fate.env @@ -1,6 +1,6 @@ -FATE=1.7.1.1 -FATEFlow=1.7.1 -FATEBoard=1.7.2 +FATE=1.7.2 +FATEFlow=1.7.2 +FATEBoard=1.7.2.1 EGGROLL=2.4.3 CENTOS=7.2 UBUNTU=16.04 diff --git a/fateboard b/fateboard index 23b085fba9..5eb42605ee 160000 --- a/fateboard +++ b/fateboard @@ -1 +1 @@ -Subproject commit 23b085fba9c64ebff3f8cc31d1c95202864a3781 +Subproject commit 5eb42605ee8570424b7bc79ee322ce0e3e831ca7 diff --git a/fateflow b/fateflow index bfd5466115..610cc71331 160000 --- a/fateflow +++ b/fateflow @@ -1 +1 @@ -Subproject commit bfd5466115d915eb776664e6bae8364c4f9a5b83 +Subproject commit 610cc713311fbf37495e4f3b6afe957ce15c36f1 From cb55636859b14fc0d57dd8e417ab799e785959d5 Mon Sep 17 00:00:00 2001 From: mgqa34 Date: Mon, 21 Feb 2022 09:30:55 +0800 Subject: [PATCH 51/99] update submodule of develop-1.7.2 Signed-off-by: mgqa34 --- eggroll | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eggroll b/eggroll index a7a68dba78..d620c322f0 160000 --- a/eggroll +++ b/eggroll @@ -1 +1 @@ -Subproject commit a7a68dba78b7739c771c4a6c59c343c13752e763 +Subproject commit d620c322f0e7ad94ae903d87c023fcf1d452a65d From 0d02e07ba679a37810b1f241ff6a77b960454f3a Mon Sep 17 00:00:00 2001 From: zhihuiwan <15779896112@163.com> Date: Mon, 21 Feb 2022 12:55:37 +0800 Subject: [PATCH 52/99] update connector key --- conf/service_conf.yaml | 3 +++ python/fate_arch/common/conf_utils.py | 16 ++++++++++++++++ python/fate_arch/metastore/db_models.py | 4 ++-- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/conf/service_conf.yaml b/conf/service_conf.yaml index 2e5090c329..54f80a1807 100644 --- a/conf/service_conf.yaml +++ b/conf/service_conf.yaml @@ -1,6 +1,9 @@ use_registry: false use_deserialize_safe_module: false dependent_distribution: false +encrypt_password: false +encrypt_module: fate_arcm.common.encrypt_utils#pwdecrypt +private_key: fateflow: # you must set real ip address, 127.0.0.1 and 0.0.0.0 is not supported host: 127.0.0.1 diff --git a/python/fate_arch/common/conf_utils.py b/python/fate_arch/common/conf_utils.py index 1bde8926ad..010ef63aba 100644 --- a/python/fate_arch/common/conf_utils.py +++ b/python/fate_arch/common/conf_utils.py @@ -38,6 +38,22 @@ def get_base_config(key, default=None, conf_name=SERVICE_CONF): return config.get(key, default) +def decrypt_database_config(database=None, passwd_key="passwd"): + import importlib + if not database: + database = get_base_config("database", {}) + encrypt_password = get_base_config("encrypt_password", False) + encrypt_module = get_base_config("encrypt_module", False) + private_key = get_base_config("private_key", None) + if encrypt_password: + if not private_key: + raise Exception("private key is null") + module_fun = encrypt_module.split("#") + pwdecrypt_fun = getattr(importlib.import_module(module_fun[0]), module_fun[1]) + database[passwd_key] = pwdecrypt_fun(private_key, database.get(passwd_key)) + return database + + def update_config(key, value, conf_name=SERVICE_CONF): conf_path = conf_realpath(conf_name=conf_name) if not os.path.isabs(conf_path): diff --git a/python/fate_arch/metastore/db_models.py b/python/fate_arch/metastore/db_models.py index cf5817aea1..bc865e5827 100644 --- a/python/fate_arch/metastore/db_models.py +++ b/python/fate_arch/metastore/db_models.py @@ -22,13 +22,13 @@ from fate_arch.federation import FederationEngine from fate_arch.metastore.base_model import DateTimeField from fate_arch.common import file_utils, log, EngineType, conf_utils -from fate_arch.common.conf_utils import get_base_config +from fate_arch.common.conf_utils import decrypt_database_config from fate_arch.metastore.base_model import JSONField, SerializedField, BaseModel LOGGER = log.getLogger() -DATABASE = get_base_config("database", {}) +DATABASE = decrypt_database_config() is_standalone = conf_utils.get_base_config("default_engines", {}).get(EngineType.FEDERATION).upper() == \ FederationEngine.STANDALONE From 69766991f261a15a494feaabbda271fc4b292632 Mon Sep 17 00:00:00 2001 From: zhihuiwan <15779896112@163.com> Date: Mon, 21 Feb 2022 13:12:19 +0800 Subject: [PATCH 53/99] fix bug --- python/fate_arch/storage/mysql/_table.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/python/fate_arch/storage/mysql/_table.py b/python/fate_arch/storage/mysql/_table.py index 79160a6530..ee4867b519 100644 --- a/python/fate_arch/storage/mysql/_table.py +++ b/python/fate_arch/storage/mysql/_table.py @@ -129,15 +129,13 @@ def execute(self, sql, select=True): def _get_id_feature_name(self): id = self.meta.get_schema().get("sid", "id") - header = self.meta.get_schema().get("header") + header = self.meta.get_schema().get("header", []) id_delimiter = self.meta.get_id_delimiter() - if header: - if isinstance(header, str): - feature_list = header.split(id_delimiter) - elif isinstance(header, list): - feature_list = header - else: - feature_list = [header] + + if isinstance(header, str): + feature_list = header.split(id_delimiter) + elif isinstance(header, list): + feature_list = header else: - raise Exception("mysql table need data header") + feature_list = [header] return id, feature_list, id_delimiter From dcf404920f89a24b1e6a001f1ce4c22272bcd552 Mon Sep 17 00:00:00 2001 From: weijingchen Date: Mon, 21 Feb 2022 17:14:15 +0800 Subject: [PATCH 54/99] add key_length setting Signed-off-by: cwj --- .../ensemble/boosting/hetero/hetero_secureboost_guest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_guest.py b/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_guest.py index c7c0695b3b..00ea52b18d 100644 --- a/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_guest.py +++ b/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_guest.py @@ -444,7 +444,7 @@ def EINI_guest_predict(self, data_inst, trees: List[HeteroDecisionTreeGuest], le # encryption encrypter = PaillierEncrypt() - encrypter.generate_key(1024) + encrypter.generate_key(self.encrypt_param.key_length) encrypter_vec_table = position_vec.mapValues(encrypter.recursive_encrypt) # federation part From 5d019472452760d624df3cda37fa7e685ed90cc0 Mon Sep 17 00:00:00 2001 From: zhihuiwan <15779896112@163.com> Date: Mon, 21 Feb 2022 18:00:42 +0800 Subject: [PATCH 55/99] fix bug --- python/fate_arch/storage/mysql/_table.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/python/fate_arch/storage/mysql/_table.py b/python/fate_arch/storage/mysql/_table.py index ee4867b519..9f96e6d8a1 100644 --- a/python/fate_arch/storage/mysql/_table.py +++ b/python/fate_arch/storage/mysql/_table.py @@ -45,9 +45,14 @@ def __init__( def check_address(self): schema = self.meta.get_schema() if schema: - sql = "SELECT {},{} FROM {}".format( - schema.get("sid"), schema.get("header"), self._address.name - ) + if schema.get("sid") and schema.get("header"): + sql = "SELECT {},{} FROM {}".format( + schema.get("sid"), schema.get("header"), self._address.name + ) + else: + sql = "SELECT {} FROM {}".format( + schema.get("sid"), self._address.name + ) feature_data = self.execute(sql) for feature in feature_data: if feature: @@ -131,8 +136,9 @@ def _get_id_feature_name(self): id = self.meta.get_schema().get("sid", "id") header = self.meta.get_schema().get("header", []) id_delimiter = self.meta.get_id_delimiter() - - if isinstance(header, str): + if not header: + feature_list = [] + elif isinstance(header, str): feature_list = header.split(id_delimiter) elif isinstance(header, list): feature_list = header From dd01bbde85cb9b79ec37b0f3d1b3ddb9c3deff5c Mon Sep 17 00:00:00 2001 From: mgqa34 Date: Tue, 22 Feb 2022 01:01:07 +0800 Subject: [PATCH 56/99] enhance batch masked strategy Signed-off-by: mgqa34 --- .../hetero/procedure/batch_generator.py | 40 ++++++++++++++++--- .../hetero_lr_guest.py | 3 ++ .../hetero_lr_host.py | 5 ++- .../federatedml/model_selection/mini_batch.py | 23 +++++------ .../federatedml/secureprotol/encrypt_mode.py | 6 +-- .../hetero_lr_transfer_variable.py | 1 + 6 files changed, 54 insertions(+), 24 deletions(-) diff --git a/python/federatedml/framework/hetero/procedure/batch_generator.py b/python/federatedml/framework/hetero/procedure/batch_generator.py index aa71fa5d4b..51d2c4255b 100644 --- a/python/federatedml/framework/hetero/procedure/batch_generator.py +++ b/python/federatedml/framework/hetero/procedure/batch_generator.py @@ -31,6 +31,7 @@ def __init__(self): def register_batch_generator(self, transfer_variables, has_arbiter=True): self._register_batch_data_index_transfer(transfer_variables.batch_info, transfer_variables.batch_data_index, + getattr(transfer_variables, "batch_validate_info", None), has_arbiter) def initialize_batch_generator(self, data_instances, batch_size, suffix=tuple(), @@ -38,9 +39,10 @@ def initialize_batch_generator(self, data_instances, batch_size, suffix=tuple(), self.mini_batch_obj = MiniBatch(data_instances, batch_size=batch_size, shuffle=shuffle, batch_strategy=batch_strategy, masked_rate=masked_rate) self.batch_nums = self.mini_batch_obj.batch_nums - self.batch_masked = self.mini_batch_obj.batch_masked - batch_info = {"batch_size": batch_size, "batch_num": - self.batch_nums, "batch_mutable": self.mini_batch_obj.batch_mutable, "batch_masked": self.batch_masked} + self.batch_masked = self.mini_batch_obj.batch_size != self.mini_batch_obj.masked_batch_size + batch_info = {"batch_size": self.mini_batch_obj.batch_size, "batch_num": self.batch_nums, + "batch_mutable": self.mini_batch_obj.batch_mutable, + "masked_batch_size": self.mini_batch_obj.masked_batch_size} self.sync_batch_info(batch_info, suffix) if not self.mini_batch_obj.batch_mutable: @@ -68,6 +70,19 @@ def generate_batch_data(self, with_index=False, suffix=tuple()): for batch_data in data_generator: yield batch_data + def verify_batch_legality(self, suffix=tuple()): + validate_infos = self.sync_batch_validate_info(suffix) + least_batch_size = 0 + is_legal = True + for validate_info in validate_infos: + legality = validate_info.get("legality") + if not legality: + is_legal = False + least_batch_size = max(least_batch_size, validate_info.get("least_batch_size")) + + if not is_legal: + raise ValueError(f"To use batch masked strategy, (masked_rate + 1) * batch_size should >= {least_batch_size}") + class Host(batch_info_sync.Host): def __init__(self): @@ -77,15 +92,20 @@ def __init__(self): self.data_inst = None self.batch_mutable = False self.batch_masked = False + self.masked_batch_size = None def register_batch_generator(self, transfer_variables, has_arbiter=None): - self._register_batch_data_index_transfer(transfer_variables.batch_info, transfer_variables.batch_data_index) + self._register_batch_data_index_transfer(transfer_variables.batch_info, + transfer_variables.batch_data_index, + getattr(transfer_variables, "batch_validate_info", None)) def initialize_batch_generator(self, data_instances, suffix=tuple(), **kwargs): batch_info = self.sync_batch_info(suffix) + batch_size = batch_info.get("batch_size") self.batch_nums = batch_info.get('batch_num') self.batch_mutable = batch_info.get("batch_mutable") - self.batch_masked = batch_info.get("batch_masked") + self.masked_batch_size = batch_info.get("masked_batch_size") + self.batch_masked = self.masked_batch_size != batch_size if not self.batch_mutable: self.prepare_batch_data(data_instances, suffix) @@ -112,6 +132,16 @@ def generate_batch_data(self, suffix=tuple()): yield batch_data_inst batch_index += 1 + def verify_batch_legality(self, least_batch_size, suffix=tuple()): + if least_batch_size > self.masked_batch_size: + batch_validate_info = {"legality": False, + "least_batch_size": least_batch_size} + LOGGER.warning(f"masked_batch_size {self.masked_batch_size} is illegal, should >= {least_batch_size}") + else: + batch_validate_info = {"legality": True} + + self.sync_batch_validate_info(batch_validate_info, suffix) + class Arbiter(batch_info_sync.Arbiter): def __init__(self): diff --git a/python/federatedml/linear_model/logistic_regression/hetero_logistic_regression/hetero_lr_guest.py b/python/federatedml/linear_model/logistic_regression/hetero_logistic_regression/hetero_lr_guest.py index f287777810..c51e01b301 100644 --- a/python/federatedml/linear_model/logistic_regression/hetero_logistic_regression/hetero_lr_guest.py +++ b/python/federatedml/linear_model/logistic_regression/hetero_logistic_regression/hetero_lr_guest.py @@ -92,6 +92,9 @@ def fit_binary(self, data_instances, validate_data=None): self.batch_generator.initialize_batch_generator(data_instances, self.batch_size, batch_strategy=self.batch_strategy, masked_rate=self.masked_rate, shuffle=self.shuffle) + if self.batch_generator.batch_masked: + self.batch_generator.verify_batch_legality() + self.gradient_loss_operator.set_total_batch_nums(self.batch_generator.batch_nums) use_async = False diff --git a/python/federatedml/linear_model/logistic_regression/hetero_logistic_regression/hetero_lr_host.py b/python/federatedml/linear_model/logistic_regression/hetero_logistic_regression/hetero_lr_host.py index 8267436568..f55d6005c0 100644 --- a/python/federatedml/linear_model/logistic_regression/hetero_logistic_regression/hetero_lr_host.py +++ b/python/federatedml/linear_model/logistic_regression/hetero_logistic_regression/hetero_lr_host.py @@ -99,9 +99,12 @@ def fit_binary(self, data_instances, validate_data): LOGGER.debug(f"MODEL_STEP Start fin_binary, data count: {data_instances.count()}") self.header = self.get_header(data_instances) + model_shape = self.get_features_shape(data_instances) self.cipher_operator = self.cipher.gen_paillier_cipher_operator() self.batch_generator.initialize_batch_generator(data_instances, shuffle=self.shuffle) + if self.batch_generator.batch_masked: + self.batch_generator.verify_batch_legality(least_batch_size=model_shape + 1) if self.transfer_variable.use_async.get(idx=0): LOGGER.debug(f"set_use_async") @@ -115,7 +118,7 @@ def fit_binary(self, data_instances, validate_data): in range(self.batch_generator.batch_nums)] LOGGER.info("Start initialize model.") - model_shape = self.get_features_shape(data_instances) + # model_shape = self.get_features_shape(data_instances) if self.init_param_obj.fit_intercept: self.init_param_obj.fit_intercept = False diff --git a/python/federatedml/model_selection/mini_batch.py b/python/federatedml/model_selection/mini_batch.py index 9f52cf9174..7b43f71e09 100644 --- a/python/federatedml/model_selection/mini_batch.py +++ b/python/federatedml/model_selection/mini_batch.py @@ -72,7 +72,7 @@ def __init_mini_batch_data_seperator(self, data_insts, batch_size, batch_strateg self.batch_data_generator = get_batch_generator(data_size, batch_size, batch_strategy, masked_rate, shuffle=shuffle) self.batch_nums = self.batch_data_generator.batch_nums self.batch_mutable = self.batch_data_generator.batch_mutable() - self.batch_masked = self.batch_data_generator.batch_masked() + self.masked_batch_size = self.batch_data_generator.masked_batch_size if self.batch_mutable is False: self.__generate_batch_data() @@ -107,13 +107,10 @@ def get_batch_generator(data_size, batch_size, batch_strategy, masked_rate, shuf class BatchDataGenerator(object): def __init__(self, data_size, batch_size, shuffle=False, masked_rate=0): self.batch_nums = None - self.masked_dataset_size = min(data_size, round((1 + masked_rate) * batch_size)) + self.masked_batch_size = min(data_size, round((1 + masked_rate) * batch_size)) self.batch_size = batch_size self.shuffle = shuffle - def batch_masked(self): - return self.masked_dataset_size != self.batch_size - def batch_mutable(self): return True @@ -139,7 +136,7 @@ def __init__(self, data_size, batch_size, shuffle=False, masked_rate=0): self.batch_nums = (data_size + batch_size - 1) // batch_size LOGGER.debug(f"Init Full Batch Data Generator, batch_nums: {self.batch_nums}, batch_size: {self.batch_size}, " - f"masked_dataset_size: {self.masked_dataset_size}, shuffle: {self.shuffle}") + f"masked_batch_size: {self.masked_batch_size}, shuffle: {self.shuffle}") def generate_data(self, data_insts, data_sids): if self.shuffle: @@ -147,17 +144,17 @@ def generate_data(self, data_insts, data_sids): index_table = [] batch_data = [] - if self.batch_size != self.masked_dataset_size: + if self.batch_size != self.masked_batch_size: for bid in range(self.batch_nums): batch_ids = data_sids[bid * self.batch_size:(bid + 1) * self.batch_size] masked_ids_set = set() for sid, _ in batch_ids: masked_ids_set.add(sid) - possible_ids = random.SystemRandom().sample(data_sids, self.masked_dataset_size) + possible_ids = random.SystemRandom().sample(data_sids, self.masked_batch_size) for pid, _ in possible_ids: if pid not in masked_ids_set: masked_ids_set.add(pid) - if len(masked_ids_set) == self.masked_dataset_size: + if len(masked_ids_set) == self.masked_batch_size: break masked_ids = zip(list(masked_ids_set), [None] * len(masked_ids_set)) @@ -176,7 +173,7 @@ def generate_data(self, data_insts, data_sids): return index_table, batch_data def batch_mutable(self): - return self.masked_dataset_size > self.batch_size or self.shuffle + return self.masked_batch_size > self.batch_size or self.shuffle class RandomBatchDataGenerator(BatchDataGenerator): @@ -185,16 +182,16 @@ def __init__(self, data_size, batch_size, masked_rate=0): self.batch_nums = 1 LOGGER.debug(f"Init Random Batch Data Generator, batch_nums: {self.batch_nums}, batch_size: {self.batch_size}, " - f"masked_dataset_size: {self.masked_dataset_size}") + f"masked_batch_size: {self.masked_batch_size}") def generate_data(self, data_insts, data_sids): - if self.masked_dataset_size == self.batch_size: + if self.masked_batch_size == self.batch_size: batch_ids = random.SystemRandom().sample(data_sids, self.batch_size) batch_index_table, batch_data_table = self._generate_batch_data_with_batch_ids(data_insts, batch_ids) batch_data_table = batch_index_table.join(data_insts, lambda x, y: y) return [batch_index_table], [batch_data_table] else: - masked_ids = random.SystemRandom().sample(data_sids, self.masked_dataset_size) + masked_ids = random.SystemRandom().sample(data_sids, self.masked_batch_size) batch_ids = masked_ids[: self.batch_size] masked_index_table, batch_data_table = self._generate_batch_data_with_batch_ids(data_insts, batch_ids, diff --git a/python/federatedml/secureprotol/encrypt_mode.py b/python/federatedml/secureprotol/encrypt_mode.py index 677a697611..e9a885b121 100644 --- a/python/federatedml/secureprotol/encrypt_mode.py +++ b/python/federatedml/secureprotol/encrypt_mode.py @@ -14,12 +14,8 @@ # limitations under the License. # -import random import functools -import numpy as np -from collections import Iterable from federatedml.secureprotol import PaillierEncrypt -from federatedml.util import consts from federatedml.util import LOGGER @@ -33,6 +29,7 @@ class EncryptModeCalculator(object): mode: str, accpet 'strict', 'fast', 'balance'. "confusion_opt", "confusion_opt_balance" 'strict': means that re-encrypted every function call. + re_encrypted_rate: float or float, numeric, use if mode equals to "balance" or "confusion_opt_balance" """ @@ -47,7 +44,6 @@ def __init__(self, encrypter=None, mode="strict", re_encrypted_rate=1): self.align_to_input_data = True if self.mode != "strict": - self.mode = "strict" LOGGER.warning("encrypted_mode_calculator will be remove in later version, " "but in current version user can still use it, but it only supports strict mode, " "other mode will be reset to strict for compatibility") diff --git a/python/federatedml/transfer_variable/transfer_class/hetero_lr_transfer_variable.py b/python/federatedml/transfer_variable/transfer_class/hetero_lr_transfer_variable.py index 4e0e388fe8..f52a49af58 100644 --- a/python/federatedml/transfer_variable/transfer_class/hetero_lr_transfer_variable.py +++ b/python/federatedml/transfer_variable/transfer_class/hetero_lr_transfer_variable.py @@ -32,6 +32,7 @@ def __init__(self, flowid=0): super().__init__(flowid) self.batch_data_index = self._create_variable(name='batch_data_index', src=['guest'], dst=['host']) self.batch_info = self._create_variable(name='batch_info', src=['guest'], dst=['host', 'arbiter']) + self.batch_validate_info = self._create_variable(name="batch_validate_info", src=['host'], dst=['guest']) self.converge_flag = self._create_variable(name='converge_flag', src=['arbiter'], dst=['host', 'guest']) self.fore_gradient = self._create_variable(name='fore_gradient', src=['guest'], dst=['host']) self.forward_hess = self._create_variable(name='forward_hess', src=['guest'], dst=['host']) From 293926cb35b56a04cb39585abf94159e7dc0c1d7 Mon Sep 17 00:00:00 2001 From: mgqa34 Date: Tue, 22 Feb 2022 01:11:39 +0800 Subject: [PATCH 57/99] fix encrypt_mode Signed-off-by: mgqa34 --- python/federatedml/secureprotol/encrypt_mode.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/federatedml/secureprotol/encrypt_mode.py b/python/federatedml/secureprotol/encrypt_mode.py index e9a885b121..17e951ce1a 100644 --- a/python/federatedml/secureprotol/encrypt_mode.py +++ b/python/federatedml/secureprotol/encrypt_mode.py @@ -29,7 +29,6 @@ class EncryptModeCalculator(object): mode: str, accpet 'strict', 'fast', 'balance'. "confusion_opt", "confusion_opt_balance" 'strict': means that re-encrypted every function call. - re_encrypted_rate: float or float, numeric, use if mode equals to "balance" or "confusion_opt_balance" """ @@ -44,6 +43,7 @@ def __init__(self, encrypter=None, mode="strict", re_encrypted_rate=1): self.align_to_input_data = True if self.mode != "strict": + self.mode = "strict" LOGGER.warning("encrypted_mode_calculator will be remove in later version, " "but in current version user can still use it, but it only supports strict mode, " "other mode will be reset to strict for compatibility") From 066c170611b01126e5e9805416f4c9fc05795091 Mon Sep 17 00:00:00 2001 From: mgqa34 Date: Tue, 22 Feb 2022 01:19:16 +0800 Subject: [PATCH 58/99] Update python/federatedml/framework/hetero/procedure/batch_generator.py Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .../federatedml/framework/hetero/procedure/batch_generator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/federatedml/framework/hetero/procedure/batch_generator.py b/python/federatedml/framework/hetero/procedure/batch_generator.py index 51d2c4255b..be121e9293 100644 --- a/python/federatedml/framework/hetero/procedure/batch_generator.py +++ b/python/federatedml/framework/hetero/procedure/batch_generator.py @@ -81,7 +81,8 @@ def verify_batch_legality(self, suffix=tuple()): least_batch_size = max(least_batch_size, validate_info.get("least_batch_size")) if not is_legal: - raise ValueError(f"To use batch masked strategy, (masked_rate + 1) * batch_size should >= {least_batch_size}") + raise ValueError( + f"To use batch masked strategy, (masked_rate + 1) * batch_size should >= {least_batch_size}") class Host(batch_info_sync.Host): From a22adc16fb345ca5a830ceeb9523020ce039da1a Mon Sep 17 00:00:00 2001 From: mgqa34 Date: Tue, 22 Feb 2022 01:20:29 +0800 Subject: [PATCH 59/99] Update python/federatedml/model_selection/mini_batch.py Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- python/federatedml/model_selection/mini_batch.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/federatedml/model_selection/mini_batch.py b/python/federatedml/model_selection/mini_batch.py index 7b43f71e09..4fac9bd008 100644 --- a/python/federatedml/model_selection/mini_batch.py +++ b/python/federatedml/model_selection/mini_batch.py @@ -69,7 +69,8 @@ def mini_batch_data_generator(self, result='data'): def __init_mini_batch_data_seperator(self, data_insts, batch_size, batch_strategy, masked_rate, shuffle): self.data_sids_iter, data_size = indices.collect_index(data_insts) - self.batch_data_generator = get_batch_generator(data_size, batch_size, batch_strategy, masked_rate, shuffle=shuffle) + self.batch_data_generator = get_batch_generator( + data_size, batch_size, batch_strategy, masked_rate, shuffle=shuffle) self.batch_nums = self.batch_data_generator.batch_nums self.batch_mutable = self.batch_data_generator.batch_mutable() self.masked_batch_size = self.batch_data_generator.masked_batch_size From b46fe6198ed43c9ac8c698bdedb0122dc763ad35 Mon Sep 17 00:00:00 2001 From: mgqa34 Date: Tue, 22 Feb 2022 10:31:22 +0800 Subject: [PATCH 60/99] update batch_info_sync Signed-off-by: mgqa34 --- .../framework/hetero/sync/batch_info_sync.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/python/federatedml/framework/hetero/sync/batch_info_sync.py b/python/federatedml/framework/hetero/sync/batch_info_sync.py index a7cd708cc8..b5295b5260 100644 --- a/python/federatedml/framework/hetero/sync/batch_info_sync.py +++ b/python/federatedml/framework/hetero/sync/batch_info_sync.py @@ -23,9 +23,11 @@ class Guest(object): def _register_batch_data_index_transfer(self, batch_data_info_transfer, batch_data_index_transfer, + batch_validate_info_transfer, has_arbiter): self.batch_data_info_transfer = batch_data_info_transfer.disable_auto_clean() self.batch_data_index_transfer = batch_data_index_transfer.disable_auto_clean() + self.batch_validate_info_transfer = batch_validate_info_transfer self.has_arbiter = has_arbiter def sync_batch_info(self, batch_info, suffix=tuple()): @@ -43,11 +45,21 @@ def sync_batch_index(self, batch_index, suffix=tuple()): role=consts.HOST, suffix=suffix) + def sync_batch_validate_info(self, suffix): + if not self.batch_validate_info_transfer: + raise ValueError("batch_validate_info should be create in transfer variable") + + validate_info = self.batch_validate_info_transfer.get(idx=-1, + suffix=suffix) + return validate_info + class Host(object): - def _register_batch_data_index_transfer(self, batch_data_info_transfer, batch_data_index_transfer): + def _register_batch_data_index_transfer(self, batch_data_info_transfer, batch_data_index_transfer, + batch_validate_info_transfer): self.batch_data_info_transfer = batch_data_info_transfer.disable_auto_clean() self.batch_data_index_transfer = batch_data_index_transfer.disable_auto_clean() + self.batch_validate_info_transfer = batch_validate_info_transfer def sync_batch_info(self, suffix=tuple()): LOGGER.debug("In sync_batch_info, suffix is :{}".format(suffix)) @@ -65,6 +77,11 @@ def sync_batch_index(self, suffix=tuple()): suffix=suffix) return batch_index + def sync_batch_validate_info(self, validate_info, suffix=tuple()): + self.batch_validate_info_transfer.remote(obj=validate_info, + role=consts.GUEST, + suffix=suffix) + class Arbiter(object): def _register_batch_data_index_transfer(self, batch_data_info_transfer, batch_data_index_transfer): From 215a0d31dca04ecac4c3c1851341c68053213a47 Mon Sep 17 00:00:00 2001 From: paulbaogang Date: Tue, 22 Feb 2022 10:49:31 +0800 Subject: [PATCH 61/99] update --- .../doc/fate_on_eggroll/fate-allinone_deployment_guide.md | 2 ++ .../doc/fate_on_eggroll/fate-allinone_deployment_guide.zh.md | 2 ++ .../doc/fate_on_eggroll/fate-exchange_deployment_guide.md | 2 ++ .../doc/fate_on_eggroll/fate-exchange_deployment_guide.zh.md | 2 ++ 4 files changed, 8 insertions(+) diff --git a/deploy/cluster-deploy/doc/fate_on_eggroll/fate-allinone_deployment_guide.md b/deploy/cluster-deploy/doc/fate_on_eggroll/fate-allinone_deployment_guide.md index b4ff9aec94..f243750f5d 100644 --- a/deploy/cluster-deploy/doc/fate_on_eggroll/fate-allinone_deployment_guide.md +++ b/deploy/cluster-deploy/doc/fate_on_eggroll/fate-allinone_deployment_guide.md @@ -263,6 +263,8 @@ Note: Replace ${version} with specific FATE version number,can be viewed on the cd /data/projects/ wget https://webank-ai-1251170195.cos.ap-guangzhou.myqcloud.com/fate_cluster_install_${version}_release-c7-u18.tar.gz tar xzf fate_cluster_install_${version}_release-c7-u18.tar.gz + +Note: version without character v, such as fate_cluster_install_1.x.x_release-c7-u18.tar.gz ``` ### 5.2. Pre-Deployment Check diff --git a/deploy/cluster-deploy/doc/fate_on_eggroll/fate-allinone_deployment_guide.zh.md b/deploy/cluster-deploy/doc/fate_on_eggroll/fate-allinone_deployment_guide.zh.md index 5630cc981e..1f70d78793 100644 --- a/deploy/cluster-deploy/doc/fate_on_eggroll/fate-allinone_deployment_guide.zh.md +++ b/deploy/cluster-deploy/doc/fate_on_eggroll/fate-allinone_deployment_guide.zh.md @@ -265,6 +265,8 @@ Swap: 131071 0 131071 cd /data/projects/ wget https://webank-ai-1251170195.cos.ap-guangzhou.myqcloud.com/fate_cluster_install_${version}_release-c7-u18.tar.gz tar xzf fate_cluster_install_${version}_release-c7-u18.tar.gz + +注意：version不带字符v，如fate_cluster_install_1.x.x_release-c7-u18.tar.gz ``` ### 5.2. 部署前检查 diff --git a/deploy/cluster-deploy/doc/fate_on_eggroll/fate-exchange_deployment_guide.md b/deploy/cluster-deploy/doc/fate_on_eggroll/fate-exchange_deployment_guide.md index 694c657c15..6a8737f957 100644 --- a/deploy/cluster-deploy/doc/fate_on_eggroll/fate-exchange_deployment_guide.md +++ b/deploy/cluster-deploy/doc/fate_on_eggroll/fate-exchange_deployment_guide.md @@ -139,6 +139,8 @@ Note: Replace ${version} with the specific FATE version number. cd /data/projects/install wget https://webank-ai-1251170195.cos.ap-guangzhou.myqcloud.com/jdk-8u192-linux-x64.tar.gz wget https://webank-ai-1251170195.cos.ap-guangzhou.myqcloud.com/FATE_install_${version}_release.tar.gz + +Note: version without character v, such as FATE_install_1.x.x_release.tar.gz ``` ## 5.2 Check OS Parameters diff --git a/deploy/cluster-deploy/doc/fate_on_eggroll/fate-exchange_deployment_guide.zh.md b/deploy/cluster-deploy/doc/fate_on_eggroll/fate-exchange_deployment_guide.zh.md index f9e16da419..b78666102a 100644 --- a/deploy/cluster-deploy/doc/fate_on_eggroll/fate-exchange_deployment_guide.zh.md +++ b/deploy/cluster-deploy/doc/fate_on_eggroll/fate-exchange_deployment_guide.zh.md @@ -147,6 +147,8 @@ fi cd /data/projects/install wget https://webank-ai-1251170195.cos.ap-guangzhou.myqcloud.com/jdk-8u192-linux-x64.tar.gz wget https://webank-ai-1251170195.cos.ap-guangzhou.myqcloud.com/FATE_install_${version}_release.tar.gz + +注意：version不带字符v，如FATE_install_1.x.x_release.tar.gz ``` ## 5.2 操作系统参数检查 From 645c1846e7402cdaacf440e75c935ee2637ec316 Mon Sep 17 00:00:00 2001 From: weijingchen Date: Tue, 22 Feb 2022 10:55:58 +0800 Subject: [PATCH 62/99] add tree complexity computation Signed-off-by: cwj --- .../pipeline/param/boosting_param.py | 2 +- .../hetero/hetero_secureboost_host.py | 24 +++++++++++++++++++ python/federatedml/param/boosting_param.py | 4 ++-- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/python/fate_client/pipeline/param/boosting_param.py b/python/fate_client/pipeline/param/boosting_param.py index 32eed9c0a3..6c7205360f 100644 --- a/python/fate_client/pipeline/param/boosting_param.py +++ b/python/fate_client/pipeline/param/boosting_param.py @@ -469,7 +469,7 @@ def __init__(self, tree_param: DecisionTreeParam = DecisionTreeParam(), task_typ binning_error=consts.DEFAULT_RELATIVE_ERROR, sparse_optimization=False, run_goss=False, top_rate=0.2, other_rate=0.1, cipher_compress_error=None, cipher_compress=True, new_ver=True, - callback_param=CallbackParam(), EINI_inference=True, EINI_random_mask=False): + callback_param=CallbackParam(), EINI_inference=False, EINI_random_mask=False): super(HeteroSecureBoostParam, self).__init__(task_type, objective_param, learning_rate, num_trees, subsample_feature_rate, n_iter_no_change, tol, encrypt_param, diff --git a/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py b/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py index a9eae6e2ea..3e737b9659 100644 --- a/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py +++ b/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py @@ -279,6 +279,30 @@ def get_leaf_idx_map(trees): return id_pos_map_list + def count_complexity_helper(self, node, node_list, host_sitename, meet_host_node): + + if node.is_leaf: + return 1 if meet_host_node else 0 + if node.sitename == host_sitename: + meet_host_node = True + + return self.count_complexity_helper(node_list[node.left_nodeid], node_list, host_sitename, meet_host_node) + \ + self.count_complexity_helper(node_list[node.right_nodeid], node_list, host_sitename, meet_host_node) + + def count_complexity(self, trees): + + tree_valid_leaves_num = [] + sitename = self.role + ":" + str(self.component_properties.local_partyid) + for tree in trees: + valid_leaf_num = self.count_complexity_helper(tree[0], tree.tree_node, sitename, False) + tree_valid_leaves_num.append(valid_leaf_num) + + complexity = 1 + for num in tree_valid_leaves_num: + complexity *= num + + return complexity + def EINI_host_predict(self, data_inst, trees: List[HeteroDecisionTreeHost], sitename, self_party_id, party_list, random_mask=False): diff --git a/python/federatedml/param/boosting_param.py b/python/federatedml/param/boosting_param.py index 11a66829d4..ca9f7d9dfa 100644 --- a/python/federatedml/param/boosting_param.py +++ b/python/federatedml/param/boosting_param.py @@ -496,7 +496,7 @@ class HeteroSecureBoostParam(HeteroBoostingParam): default is True, use cipher compressing to reduce computation cost and transfer cost EINI_inference: bool - default is True, a secure prediction method that hides decision path to enhance security in the inference + default is False, a secure prediction method that hides decision path to enhance security in the inference step. This method is insprired by EINI inference algorithm. EINI_random_mask: bool @@ -518,7 +518,7 @@ def __init__(self, tree_param: DecisionTreeParam = DecisionTreeParam(), task_typ binning_error=consts.DEFAULT_RELATIVE_ERROR, sparse_optimization=False, run_goss=False, top_rate=0.2, other_rate=0.1, cipher_compress_error=None, cipher_compress=True, new_ver=True, - callback_param=CallbackParam(), EINI_inference=True, EINI_random_mask=False): + callback_param=CallbackParam(), EINI_inference=False, EINI_random_mask=False): super(HeteroSecureBoostParam, self).__init__(task_type, objective_param, learning_rate, num_trees, subsample_feature_rate, n_iter_no_change, tol, encrypt_param, From c0247a94f44bb75d55d0da5e5daf4ed128efff04 Mon Sep 17 00:00:00 2001 From: weijingchen Date: Tue, 22 Feb 2022 12:58:31 +0800 Subject: [PATCH 63/99] update check Signed-off-by: cwj --- .../pipeline/param/boosting_param.py | 204 +++++++++++------- .../boosting/boosting_core/boosting.py | 1 + .../boosting/boosting_core/hetero_boosting.py | 4 + .../hetero/hetero_secureboost_guest.py | 2 +- .../hetero/hetero_secureboost_host.py | 11 +- python/federatedml/param/boosting_param.py | 20 +- python/federatedml/util/consts.py | 2 + 7 files changed, 162 insertions(+), 82 deletions(-) diff --git a/python/fate_client/pipeline/param/boosting_param.py b/python/fate_client/pipeline/param/boosting_param.py index 6c7205360f..944e7cf1ae 100644 --- a/python/fate_client/pipeline/param/boosting_param.py +++ b/python/fate_client/pipeline/param/boosting_param.py @@ -374,88 +374,115 @@ def check(self): class HeteroSecureBoostParam(HeteroBoostingParam): """ - Define boosting tree parameters that used in federated ml. + Define boosting tree parameters that used in federated ml. - Parameters - ---------- - task_type : str, accepted 'classification', 'regression' only, default: 'classification' + Parameters + ---------- + task_type : {'classification', 'regression'}, default: 'classification' + task type - tree_param : DecisionTreeParam Object, default: DecisionTreeParam() + tree_param : DecisionTreeParam Object, default: DecisionTreeParam() + tree param - objective_param : ObjectiveParam Object, default: ObjectiveParam() + objective_param : ObjectiveParam Object, default: ObjectiveParam() + objective param - learning_rate : float, accepted float, int or long only, the learning rate of secure boost. default: 0.3 + learning_rate : float, int or long + the learning rate of secure boost. default: 0.3 - num_trees : int, accepted int, float only, the max number of trees to build. default: 5 + num_trees : int or float + the max number of trees to build. default: 5 - subsample_feature_rate : float, a float-number in [0, 1], default: 1.0 + subsample_feature_rate : float + a float-number in [0, 1], default: 1.0 - random_seed: seed that controls all random functions + random_seed: int + seed that controls all random functions - n_iter_no_change : bool, - when True and residual error less than tol, tree building process will stop. default: True + n_iter_no_change : bool, + when True and residual error less than tol, tree building process will stop. default: True - encrypt_param : EncodeParam Object, encrypt method use in secure boost, default: EncryptParam(), this parameter - is only for hetero-secureboost + encrypt_param : EncodeParam Object + encrypt method use in secure boost, default: EncryptParam(), this parameter + is only for hetero-secureboost - bin_num: int, positive integer greater than 1, bin number use in quantile. default: 32 + bin_num: positive integer greater than 1 + bin number use in quantile. default: 32 - encrypted_mode_calculator_param: EncryptedModeCalculatorParam object, the calculation mode use in secureboost, - default: EncryptedModeCalculatorParam(), only for hetero-secureboost + encrypted_mode_calculator_param: EncryptedModeCalculatorParam object + the calculation mode use in secureboost, default: EncryptedModeCalculatorParam(), only for hetero-secureboost - use_missing: bool, accepted True, False only, use missing value in training process or not. default: False + use_missing: bool + use missing value in training process or not. default: False - zero_as_missing: bool, accepted True, False only, regard 0 as missing value or not, - will be use only if use_missing=True, default: False + zero_as_missing: bool + regard 0 as missing value or not, will be use only if use_missing=True, default: False - validation_freqs: None or positive integer or container object in python. Do validation in training process or Not. - if equals None, will not do validation in train process; - if equals positive integer, will validate data every validation_freqs epochs passes; - if container object in python, will validate data if epochs belong to this container. - e.g. validation_freqs = [10, 15], will validate data when epoch equals to 10 and 15. - Default: None - The default value is None, 1 is suggested. You can set it to a number larger than 1 in order to - speed up training by skipping validation rounds. When it is larger than 1, a number which is - divisible by "num_trees" is recommended, otherwise, you will miss the validation scores - of last training iteration. + validation_freqs: None or positive integer or container object in python + Do validation in training process or Not. + if equals None, will not do validation in train process; + if equals positive integer, will validate data every validation_freqs epochs passes; + if container object in python, will validate data if epochs belong to this container. + e.g. validation_freqs = [10, 15], will validate data when epoch equals to 10 and 15. + Default: None + The default value is None, 1 is suggested. You can set it to a number larger than 1 in order to + speed up training by skipping validation rounds. When it is larger than 1, a number which is + divisible by "num_trees" is recommended, otherwise, you will miss the validation scores + of last training iteration. - early_stopping_rounds: should be a integer larger than 0，will stop training if one metric of one validation data - doesn’t improve in last early_stopping_round rounds， - need to set validation freqs and will check early_stopping every at every validation epoch, + early_stopping_rounds: integer larger than 0 + will stop training if one metric of one validation data + doesn’t improve in last early_stopping_round rounds， + need to set validation freqs and will check early_stopping every at every validation epoch, - metrics: list, default: [] - Specify which metrics to be used when performing evaluation during training process. - If set as empty, default metrics will be used. For regression tasks, default metrics are - ['root_mean_squared_error', 'mean_absolute_error']， For binary-classificatiin tasks, default metrics - are ['auc', 'ks']. For multi-classification tasks, default metrics are ['accuracy', 'precision', 'recall'] + metrics: list, default: [] + Specify which metrics to be used when performing evaluation during training process. + If set as empty, default metrics will be used. For regression tasks, default metrics are + ['root_mean_squared_error', 'mean_absolute_error']， For binary-classificatiin tasks, default metrics + are ['auc', 'ks']. For multi-classification tasks, default metrics are ['accuracy', 'precision', 'recall'] - use_first_metric_only: use only the first metric for early stopping + use_first_metric_only: bool + use only the first metric for early stopping - complete_secure: bool, if use complete_secure, when use complete secure, build first tree using only guest - features + complete_secure: bool + if use complete_secure, when use complete secure, build first tree using only guest features - sparse_optimization: this parameter is now abandoned + sparse_optimization: + this parameter is abandoned in FATE-1.7.1 - run_goss: bool, activate Gradient-based One-Side Sampling, which selects large gradient and small - gradient samples using top_rate and other_rate. + run_goss: bool + activate Gradient-based One-Side Sampling, which selects large gradient and small + gradient samples using top_rate and other_rate. - top_rate: float, the retain ratio of large gradient data, used when run_goss is True + top_rate: float + the retain ratio of large gradient data, used when run_goss is True - other_rate: float, the retain ratio of small gradient data, used when run_goss is True + other_rate: float + the retain ratio of small gradient data, used when run_goss is True - cipher_compress_error： This param is now abandoned + cipher_compress_error: {None} + This param is now abandoned - cipher_compress: bool, default is True, use cipher compressing to reduce computation cost and transfer cost + cipher_compress: bool + default is True, use cipher compressing to reduce computation cost and transfer cost - EINI_inference: bool - default is True, a secure prediction method that hides decision path to enhance security in the inference - step. This method is insprired by EINI inference algorithm. + EINI_inference: bool + default is False, this option changes the inference algorithm used in predict tasks. + a secure prediction method that hides decision path to enhance security in the inference + step. This method is insprired by EINI inference algorithm. - EINI_random_mask: bool - default is False - multiply predict result by a random float number to confuse original predict result. This operation further - enhances the security of naive EINI algorithm. - """ + EINI_random_mask: bool + default is False + multiply predict result by a random float number to confuse original predict result. This operation further + enhances the security of naive EINI algorithm. + + EINI_complexity_check: bool + default is False + check the complexity of tree models when running EINI algorithms. Complexity models are easy to hide their + decision path, while simple tree models are not, therefore if a tree model is too simple, it is not allowed + to run EINI predict algorithms. + + """ def __init__(self, tree_param: DecisionTreeParam = DecisionTreeParam(), task_type=consts.CLASSIFICATION, objective_param=ObjectiveParam(), @@ -469,7 +496,8 @@ def __init__(self, tree_param: DecisionTreeParam = DecisionTreeParam(), task_typ binning_error=consts.DEFAULT_RELATIVE_ERROR, sparse_optimization=False, run_goss=False, top_rate=0.2, other_rate=0.1, cipher_compress_error=None, cipher_compress=True, new_ver=True, - callback_param=CallbackParam(), EINI_inference=False, EINI_random_mask=False): + callback_param=CallbackParam(), EINI_inference=False, EINI_random_mask=False, + EINI_complexity_check=False): super(HeteroSecureBoostParam, self).__init__(task_type, objective_param, learning_rate, num_trees, subsample_feature_rate, n_iter_no_change, tol, encrypt_param, @@ -492,6 +520,7 @@ def __init__(self, tree_param: DecisionTreeParam = DecisionTreeParam(), task_typ self.new_ver = new_ver self.EINI_inference = EINI_inference self.EINI_random_mask = EINI_random_mask + self.EINI_complexity_check = EINI_complexity_check self.callback_param = copy.deepcopy(callback_param) def check(self): @@ -503,7 +532,6 @@ def check(self): if type(self.zero_as_missing) != bool: raise ValueError('zero as missing should be bool type') self.check_boolean(self.complete_secure, 'complete_secure') - self.check_boolean(self.sparse_optimization, 'sparse optimization') self.check_boolean(self.run_goss, 'run goss') self.check_decimal_float(self.top_rate, 'top rate') self.check_decimal_float(self.other_rate, 'other rate') @@ -513,13 +541,36 @@ def check(self): self.check_boolean(self.cipher_compress, 'cipher compress') self.check_boolean(self.EINI_inference, 'eini inference') self.check_boolean(self.EINI_random_mask, 'eini random mask') + self.check_boolean(self.EINI_complexity_check, 'eini complexity check') + + for p in ["early_stopping_rounds", "validation_freqs", "metrics", + "use_first_metric_only"]: + # if self._warn_to_deprecate_param(p, "", ""): + if self._deprecated_params_set.get(p): + if "callback_param" in self.get_user_feeded(): + raise ValueError(f"{p} and callback param should not be set simultaneously，" + f"{self._deprecated_params_set}, {self.get_user_feeded()}") + else: + self.callback_param.callbacks = ["PerformanceEvaluate"] + break + + descr = "boosting_param's" + + if self._warn_to_deprecate_param("validation_freqs", descr, "callback_param's 'validation_freqs'"): + self.callback_param.validation_freqs = self.validation_freqs + + if self._warn_to_deprecate_param("early_stopping_rounds", descr, "callback_param's 'early_stopping_rounds'"): + self.callback_param.early_stopping_rounds = self.early_stopping_rounds + + if self._warn_to_deprecate_param("metrics", descr, "callback_param's 'metrics'"): + self.callback_param.metrics = self.metrics + + if self._warn_to_deprecate_param("use_first_metric_only", descr, "callback_param's 'use_first_metric_only'"): + self.callback_param.use_first_metric_only = self.use_first_metric_only if self.top_rate + self.other_rate >= 1: raise ValueError('sum of top rate and other rate should be smaller than 1') - if self.sparse_optimization and self.cipher_compress: - raise ValueError('cipher compress is not supported in sparse optimization mode') - return True @@ -536,22 +587,25 @@ def __init__(self, tree_param: DecisionTreeParam = DecisionTreeParam(), task_typ complete_secure=False, tree_num_per_party=1, guest_depth=1, host_depth=1, work_mode='mix', metrics=None, sparse_optimization=False, random_seed=100, binning_error=consts.DEFAULT_RELATIVE_ERROR, cipher_compress_error=None, new_ver=True, run_goss=False, top_rate=0.2, other_rate=0.1, - cipher_compress=True, callback_param=CallbackParam(), EINI_inference=True, EINI_random_mask=False): + cipher_compress=True, callback_param=CallbackParam(), EINI_inference=True, EINI_random_mask=False, + EINI_complexity_check=False): """ - work_mode： - mix: alternate using guest/host features to build trees. For example, the first 'tree_num_per_party' trees use guest features, - the second k trees use host features, and so on + Parameters + ---------- + work_mode: {"mix", "layered"} + mix: alternate using guest/host features to build trees. For example, the first 'tree_num_per_party' trees + use guest features, the second k trees use host features, and so on layered: only support 2 party, when running layered mode, first 'host_depth' layer will use host features, and then next 'guest_depth' will only use guest features - tree_num_per_party: every party will alternate build 'tree_num_per_party' trees until reach max tree num, this param is valid when work_mode is - mix - guest_depth: guest will build last guest_depth of a decision tree using guest features, is valid when work mode - is layered - host depth: host will build first host_depth of a decision tree using host features, is valid when work mode is - layered - - other params are the same as HeteroSecureBoost + tree_num_per_party: int + every party will alternate build 'tree_num_per_party' trees until reach max tree num, this param is valid + when work_mode is mix + guest_depth: int + guest will build last guest_depth of a decision tree using guest features, is valid when work mode is layered + host depth: int + host will build first host_depth of a decision tree using host features, is valid when work mode is layered + """ super(HeteroFastSecureBoostParam, self).__init__(tree_param, task_type, objective_param, learning_rate, @@ -566,7 +620,9 @@ def __init__(self, tree_param: DecisionTreeParam = DecisionTreeParam(), task_typ new_ver=new_ver, cipher_compress=cipher_compress, run_goss=run_goss, top_rate=top_rate, other_rate=other_rate, - EINI_inference=EINI_inference, EINI_random_mask=EINI_random_mask + EINI_inference=EINI_inference, + EINI_random_mask=EINI_random_mask, + EINI_complexity_check=EINI_complexity_check ) self.tree_num_per_party = tree_num_per_party diff --git a/python/federatedml/ensemble/boosting/boosting_core/boosting.py b/python/federatedml/ensemble/boosting/boosting_core/boosting.py index c4f9665484..be9925dbd1 100644 --- a/python/federatedml/ensemble/boosting/boosting_core/boosting.py +++ b/python/federatedml/ensemble/boosting/boosting_core/boosting.py @@ -93,6 +93,7 @@ def __init__(self): self.metrics = None self.is_converged = False self.is_warm_start = False # warm start parameter + self.on_training = False # cache and header alignment self.predict_data_cache = PredictDataCache() diff --git a/python/federatedml/ensemble/boosting/boosting_core/hetero_boosting.py b/python/federatedml/ensemble/boosting/boosting_core/hetero_boosting.py index 943fb04c52..b5c3d246df 100644 --- a/python/federatedml/ensemble/boosting/boosting_core/hetero_boosting.py +++ b/python/federatedml/ensemble/boosting/boosting_core/hetero_boosting.py @@ -145,6 +145,8 @@ def fit(self, data_inst, validate_data=None): self.start_round = 0 + self.on_training = True + self.data_inst = data_inst self.data_bin, self.bin_split_points, self.bin_sparse_points = self.prepare_data(data_inst) @@ -294,6 +296,8 @@ def fit(self, data_inst, validate_data=None): self.start_round = 0 + self.on_training = True + self.data_bin, self.bin_split_points, self.bin_sparse_points = self.prepare_data(data_inst) if self.is_warm_start: diff --git a/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_guest.py b/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_guest.py index 00ea52b18d..95445a902e 100644 --- a/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_guest.py +++ b/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_guest.py @@ -501,7 +501,7 @@ def predict(self, data_inst, ret_format='std'): if tree_num == 0 and predict_cache is not None and not (ret_format == 'leaf'): return self.score_to_predict_result(data_inst, predict_cache) - if self.EINI_inference: + if self.EINI_inference and not self.on_training: # EINI is for inference stage sitename = self.role + ':' + str(self.component_properties.local_partyid) predict_rs = self.EINI_guest_predict(processed_data, trees, self.learning_rate, self.init_score, self.booster_dim, sitename, diff --git a/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py b/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py index 3e737b9659..d834041368 100644 --- a/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py +++ b/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py @@ -294,8 +294,9 @@ def count_complexity(self, trees): tree_valid_leaves_num = [] sitename = self.role + ":" + str(self.component_properties.local_partyid) for tree in trees: - valid_leaf_num = self.count_complexity_helper(tree[0], tree.tree_node, sitename, False) - tree_valid_leaves_num.append(valid_leaf_num) + valid_leaf_num = self.count_complexity_helper(tree.tree_node[0], tree.tree_node, sitename, False) + if valid_leaf_num != 0: + tree_valid_leaves_num.append(valid_leaf_num) complexity = 1 for num in tree_valid_leaves_num: @@ -306,6 +307,10 @@ def count_complexity(self, trees): def EINI_host_predict(self, data_inst, trees: List[HeteroDecisionTreeHost], sitename, self_party_id, party_list, random_mask=False): + complexity = self.count_complexity(trees) + if complexity < consts.EINI_TREE_COMPLEXITY: + raise ValueError('tree complexity: {}, is lower than safe ' + 'threshold, inference is not allowed.'.format(complexity)) id_pos_map_list = self.get_leaf_idx_map(trees) map_func = functools.partial(self.generate_leaf_candidates_host, sitename=sitename, trees=trees, node_pos_map_list=id_pos_map_list) @@ -366,7 +371,7 @@ def predict(self, data_inst): LOGGER.info('no tree for predicting, prediction done') return - if self.EINI_inference: + if self.EINI_inference and not self.on_training: # EINI is designed for inference stage sitename = self.role + ':' + str(self.component_properties.local_partyid) self.EINI_host_predict(processed_data, trees, sitename, self.component_properties.local_partyid, self.component_properties.host_party_idlist, self.EINI_random_mask) diff --git a/python/federatedml/param/boosting_param.py b/python/federatedml/param/boosting_param.py index ca9f7d9dfa..ed21506e10 100644 --- a/python/federatedml/param/boosting_param.py +++ b/python/federatedml/param/boosting_param.py @@ -496,7 +496,8 @@ class HeteroSecureBoostParam(HeteroBoostingParam): default is True, use cipher compressing to reduce computation cost and transfer cost EINI_inference: bool - default is False, a secure prediction method that hides decision path to enhance security in the inference + default is False, this option changes the inference algorithm used in predict tasks. + a secure prediction method that hides decision path to enhance security in the inference step. This method is insprired by EINI inference algorithm. EINI_random_mask: bool @@ -504,6 +505,12 @@ class HeteroSecureBoostParam(HeteroBoostingParam): multiply predict result by a random float number to confuse original predict result. This operation further enhances the security of naive EINI algorithm. + EINI_complexity_check: bool + default is False + check the complexity of tree models when running EINI algorithms. Complexity models are easy to hide their + decision path, while simple tree models are not, therefore if a tree model is too simple, it is not allowed + to run EINI predict algorithms. + """ def __init__(self, tree_param: DecisionTreeParam = DecisionTreeParam(), task_type=consts.CLASSIFICATION, @@ -518,7 +525,8 @@ def __init__(self, tree_param: DecisionTreeParam = DecisionTreeParam(), task_typ binning_error=consts.DEFAULT_RELATIVE_ERROR, sparse_optimization=False, run_goss=False, top_rate=0.2, other_rate=0.1, cipher_compress_error=None, cipher_compress=True, new_ver=True, - callback_param=CallbackParam(), EINI_inference=False, EINI_random_mask=False): + callback_param=CallbackParam(), EINI_inference=False, EINI_random_mask=False, + EINI_complexity_check=False): super(HeteroSecureBoostParam, self).__init__(task_type, objective_param, learning_rate, num_trees, subsample_feature_rate, n_iter_no_change, tol, encrypt_param, @@ -541,6 +549,7 @@ def __init__(self, tree_param: DecisionTreeParam = DecisionTreeParam(), task_typ self.new_ver = new_ver self.EINI_inference = EINI_inference self.EINI_random_mask = EINI_random_mask + self.EINI_complexity_check = EINI_complexity_check self.callback_param = copy.deepcopy(callback_param) def check(self): @@ -561,6 +570,7 @@ def check(self): self.check_boolean(self.cipher_compress, 'cipher compress') self.check_boolean(self.EINI_inference, 'eini inference') self.check_boolean(self.EINI_random_mask, 'eini random mask') + self.check_boolean(self.EINI_complexity_check, 'eini complexity check') if self.EINI_inference and self.EINI_random_mask: LOGGER.warning('To protect the inference decision path, notice that current setting will multiply' @@ -611,7 +621,8 @@ def __init__(self, tree_param: DecisionTreeParam = DecisionTreeParam(), task_typ complete_secure=False, tree_num_per_party=1, guest_depth=1, host_depth=1, work_mode='mix', metrics=None, sparse_optimization=False, random_seed=100, binning_error=consts.DEFAULT_RELATIVE_ERROR, cipher_compress_error=None, new_ver=True, run_goss=False, top_rate=0.2, other_rate=0.1, - cipher_compress=True, callback_param=CallbackParam(), EINI_inference=True, EINI_random_mask=False): + cipher_compress=True, callback_param=CallbackParam(), EINI_inference=True, EINI_random_mask=False, + EINI_complexity_check=False): """ Parameters @@ -644,7 +655,8 @@ def __init__(self, tree_param: DecisionTreeParam = DecisionTreeParam(), task_typ cipher_compress=cipher_compress, run_goss=run_goss, top_rate=top_rate, other_rate=other_rate, EINI_inference=EINI_inference, - EINI_random_mask=EINI_random_mask + EINI_random_mask=EINI_random_mask, + EINI_complexity_check=EINI_complexity_check ) self.tree_num_per_party = tree_num_per_party diff --git a/python/federatedml/util/consts.py b/python/federatedml/util/consts.py index b8411e97c1..e94602aa3c 100644 --- a/python/federatedml/util/consts.py +++ b/python/federatedml/util/consts.py @@ -335,3 +335,5 @@ MIN_HASH_FUNC_COUNT = 4 MAX_HASH_FUNC_COUNT = 32 + +EINI_TREE_COMPLEXITY = 1000000000 From 155401379ed2fa7799762e4535852f344e99a9ca Mon Sep 17 00:00:00 2001 From: weijingchen Date: Tue, 22 Feb 2022 13:03:48 +0800 Subject: [PATCH 64/99] update check Signed-off-by: cwj --- .../boosting/hetero/hetero_secureboost_host.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py b/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py index d834041368..ccedc4d08f 100644 --- a/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py +++ b/python/federatedml/ensemble/boosting/hetero/hetero_secureboost_host.py @@ -48,6 +48,7 @@ def __init__(self): # EINI predict param self.EINI_inference = False self.EINI_random_mask = False + self.EINI_complexity_check = False def _init_model(self, param: HeteroSecureBoostParam): @@ -62,6 +63,7 @@ def _init_model(self, param: HeteroSecureBoostParam): self.new_ver = param.new_ver self.EINI_inference = param.EINI_inference self.EINI_random_mask = param.EINI_random_mask + self.EINI_complexity_check = param.EINI_complexity_check if self.use_missing: self.tree_param.use_missing = self.use_missing @@ -307,10 +309,12 @@ def count_complexity(self, trees): def EINI_host_predict(self, data_inst, trees: List[HeteroDecisionTreeHost], sitename, self_party_id, party_list, random_mask=False): - complexity = self.count_complexity(trees) - if complexity < consts.EINI_TREE_COMPLEXITY: - raise ValueError('tree complexity: {}, is lower than safe ' - 'threshold, inference is not allowed.'.format(complexity)) + if self.EINI_complexity_check: + complexity = self.count_complexity(trees) + LOGGER.debug('checking EINI complexity: {}'.format(complexity)) + if complexity < consts.EINI_TREE_COMPLEXITY: + raise ValueError('tree complexity: {}, is lower than safe ' + 'threshold, inference is not allowed.'.format(complexity)) id_pos_map_list = self.get_leaf_idx_map(trees) map_func = functools.partial(self.generate_leaf_candidates_host, sitename=sitename, trees=trees, node_pos_map_list=id_pos_map_list) From 91f7b6407895a19a28ed19645211566306ba0ed8 Mon Sep 17 00:00:00 2001 From: mgqa34 Date: Tue, 22 Feb 2022 13:13:15 +0800 Subject: [PATCH 65/99] optimize least batch judge logic Signed-off-by: mgqa34 --- .../framework/hetero/procedure/batch_generator.py | 8 ++++---- .../hetero_logistic_regression/hetero_lr_host.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/python/federatedml/framework/hetero/procedure/batch_generator.py b/python/federatedml/framework/hetero/procedure/batch_generator.py index be121e9293..4bd4f4fb90 100644 --- a/python/federatedml/framework/hetero/procedure/batch_generator.py +++ b/python/federatedml/framework/hetero/procedure/batch_generator.py @@ -81,8 +81,8 @@ def verify_batch_legality(self, suffix=tuple()): least_batch_size = max(least_batch_size, validate_info.get("least_batch_size")) if not is_legal: - raise ValueError( - f"To use batch masked strategy, (masked_rate + 1) * batch_size should >= {least_batch_size}") + raise ValueError(f"To use batch masked strategy, " + f"(masked_rate + 1) * batch_size should > {least_batch_size}") class Host(batch_info_sync.Host): @@ -134,10 +134,10 @@ def generate_batch_data(self, suffix=tuple()): batch_index += 1 def verify_batch_legality(self, least_batch_size, suffix=tuple()): - if least_batch_size > self.masked_batch_size: + if self.masked_batch_size <= least_batch_size: batch_validate_info = {"legality": False, "least_batch_size": least_batch_size} - LOGGER.warning(f"masked_batch_size {self.masked_batch_size} is illegal, should >= {least_batch_size}") + LOGGER.warning(f"masked_batch_size {self.masked_batch_size} is illegal, should > {least_batch_size}") else: batch_validate_info = {"legality": True} diff --git a/python/federatedml/linear_model/logistic_regression/hetero_logistic_regression/hetero_lr_host.py b/python/federatedml/linear_model/logistic_regression/hetero_logistic_regression/hetero_lr_host.py index f55d6005c0..56b2fea0b2 100644 --- a/python/federatedml/linear_model/logistic_regression/hetero_logistic_regression/hetero_lr_host.py +++ b/python/federatedml/linear_model/logistic_regression/hetero_logistic_regression/hetero_lr_host.py @@ -104,7 +104,7 @@ def fit_binary(self, data_instances, validate_data): self.batch_generator.initialize_batch_generator(data_instances, shuffle=self.shuffle) if self.batch_generator.batch_masked: - self.batch_generator.verify_batch_legality(least_batch_size=model_shape + 1) + self.batch_generator.verify_batch_legality(least_batch_size=model_shape) if self.transfer_variable.use_async.get(idx=0): LOGGER.debug(f"set_use_async") From 3d8b88d7b48ace2918cde570fbc5a5441394c1e2 Mon Sep 17 00:00:00 2001 From: mgqa34 Date: Tue, 22 Feb 2022 13:55:49 +0800 Subject: [PATCH 66/99] modify fate-client\test's version Signed-off-by: mgqa34 --- python/fate_client/pyproject.toml | 2 +- python/fate_client/setup.py | 2 +- python/fate_test/pyproject.toml | 4 ++-- python/fate_test/setup.py | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/python/fate_client/pyproject.toml b/python/fate_client/pyproject.toml index f4187513c2..f1962fd397 100644 --- a/python/fate_client/pyproject.toml +++ b/python/fate_client/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "fate_client" -version = "0.1" +version = "1.7.2" description = "Clients for FATE, including flow_client and pipeline" authors = ["FederatedAI "] license = "Apache-2.0" diff --git a/python/fate_client/setup.py b/python/fate_client/setup.py index c061fd0a92..3d1481bda2 100644 --- a/python/fate_client/setup.py +++ b/python/fate_client/setup.py @@ -52,7 +52,7 @@ setup_kwargs = { "name": "fate-client", - "version": "0.1", + "version": "1.7.2", "description": "Clients for FATE, including flow_client and pipeline", "long_description": "FATE Client\n===========\n\nTools for interacting with FATE.\n\nquick start\n-----------\n\n1. (optional) create virtual env\n\n .. code-block:: bash\n\n python -m venv venv\n source venv/bin/activate\n\n\n2. install FATE Client\n\n .. code-block:: bash\n\n pip install fate-client\n\n\nPipeline\n========\n\nA high-level python API that allows user to design, start,\nand query FATE jobs in a sequential manner. For more information,\nplease refer to this `guide <./pipeline/README.rst>`__\n\nInitial Configuration\n---------------------\n\n1. Configure server information\n\n .. code-block:: bash\n\n # configure values in pipeline/config.yaml\n # use real ip address to configure pipeline\n pipeline init --ip 127.0.0.1 --port 9380 --log-directory ./logs\n\n\nFATE Flow Command Line Interface (CLI) v2\n=========================================\n\nA command line interface providing series of commands for user to design, start,\nand query FATE jobs. For more information, please refer to this `guide <./flow_client/README.rst>`__\n\nInitial Configuration\n---------------------\n\n1. Configure server information\n\n .. code-block:: bash\n\n # configure values in conf/service_conf.yaml\n flow init -c /data/projects/fate/conf/service_conf.yaml\n # use real ip address to initialize cli\n flow init --ip 127.0.0.1 --port 9380\n\n", "author": "FederatedAI", diff --git a/python/fate_test/pyproject.toml b/python/fate_test/pyproject.toml index c4bf948edb..22d5d28d58 100644 --- a/python/fate_test/pyproject.toml +++ b/python/fate_test/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "fate_test" -version = "0.1" +version = "1.7.2" description = "test tools for FATE" authors = ["FederatedAI "] license = "Apache-2.0" @@ -31,7 +31,7 @@ click = "^7.1.2" loguru = "^0.5.1" prettytable = "^1.0.0" sshtunnel = "^0.1.5" -fate_client = "^0.1" +fate_client = "^1.7" pandas = ">=1.1.5" colorama = "^0.4.4" diff --git a/python/fate_test/setup.py b/python/fate_test/setup.py index 5bbc0de648..6fdebacf54 100644 --- a/python/fate_test/setup.py +++ b/python/fate_test/setup.py @@ -7,7 +7,7 @@ install_requires = [ "click>=7.1.2,<8.0.0", - "fate_client>=0.1,<0.2", + "fate_client>=1.7,<2.0", "loguru>=0.5.1,<0.6.0", "pandas>=1.1.5", "poetry>=0.12", @@ -23,7 +23,7 @@ setup_kwargs = { "name": "fate-test", - "version": "0.1", + "version": "1.7.2", "description": "test tools for FATE", "long_description": 'FATE Test\n=========\n\nA collection of useful tools to running FATE\'s test.\n\n.. image:: images/tutorial.gif\n :align: center\n :alt: tutorial\n\nquick start\n-----------\n\n1. (optional) create virtual env\n\n .. code-block:: bash\n\n python -m venv venv\n source venv/bin/activate\n pip install -U pip\n\n\n2. install fate_test\n\n .. code-block:: bash\n\n pip install fate_test\n fate_test --help\n\n\n3. edit default fate_test_config.yaml\n\n .. code-block:: bash\n\n # edit priority config file with system default editor\n # filling some field according to comments\n fate_test config edit\n\n4. configure FATE-Pipeline and FATE-Flow Commandline server setting\n\n.. code-block:: bash\n\n # configure FATE-Pipeline server setting\n pipeline init --port 9380 --ip 127.0.0.1\n # configure FATE-Flow Commandline server setting\n flow init --port 9380 --ip 127.0.0.1\n\n5. run some fate_test suite\n\n .. code-block:: bash\n\n fate_test suite -i \n\n\n6. run some fate_test benchmark\n\n .. code-block:: bash\n\n fate_test benchmark-quality -i \n\n7. useful logs or exception will be saved to logs dir with namespace shown in last step\n\ndevelop install\n---------------\nIt is more convenient to use the editable mode during development: replace step 2 with flowing steps\n\n.. code-block:: bash\n\n pip install -e ${FATE}/python/fate_client && pip install -e ${FATE}/python/fate_test\n\n\n\ncommand types\n-------------\n\n- suite: used for running testsuites, collection of FATE jobs\n\n .. code-block:: bash\n\n fate_test suite -i \n\n\n- benchmark-quality used for comparing modeling quality between FATE and other machine learning systems\n\n .. code-block:: bash\n\n fate_test benchmark-quality -i \n\n\n\nconfiguration by examples\n--------------------------\n\n1. no need ssh tunnel:\n\n - 9999, service: service_a\n - 10000, service: service_b\n\n and both service_a, service_b can be requested directly:\n\n .. code-block:: yaml\n\n work_mode: 1 # 0 for standalone, 1 for cluster\n data_base_dir: \n parties:\n guest: [10000]\n host: [9999, 10000]\n arbiter: [9999]\n services:\n - flow_services:\n - {address: service_a, parties: [9999]}\n - {address: service_b, parties: [10000]}\n\n2. need ssh tunnel:\n\n - 9999, service: service_a\n - 10000, service: service_b\n\n service_a, can be requested directly while service_b don\'t,\n but you can request service_b in other node, say B:\n\n .. code-block:: yaml\n\n work_mode: 0 # 0 for standalone, 1 for cluster\n data_base_dir: \n parties:\n guest: [10000]\n host: [9999, 10000]\n arbiter: [9999]\n services:\n - flow_services:\n - {address: service_a, parties: [9999]}\n - flow_services:\n - {address: service_b, parties: [10000]}\n ssh_tunnel: # optional\n enable: true\n ssh_address: :\n ssh_username: \n ssh_password: # optional\n ssh_priv_key: "~/.ssh/id_rsa"\n\n\nTestsuite\n---------\n\nTestsuite is used for running a collection of jobs in sequence. Data used for jobs could be uploaded before jobs are\nsubmitted, and are cleaned when jobs finished. This tool is useful for FATE\'s release test.\n\ncommand options\n~~~~~~~~~~~~~~~\n\n.. code-block:: bash\n\n fate_test suite --help\n\n1. include:\n\n .. code-block:: bash\n\n fate_test suite -i \n\n will run testsuites in *path1*\n\n2. exclude:\n\n .. code-block:: bash\n\n fate_test suite -i -e -e ...\n\n will run testsuites in *path1* but not in *path2* and *path3*\n\n3. glob:\n\n .. code-block:: bash\n\n fate_test suite -i -g "hetero*"\n\n will run testsuites in sub directory start with *hetero* of *path1*\n\n4. replace:\n\n .. code-block:: bash\n\n fate_test suite -i -r \'{"maxIter": 5}\'\n\n will find all key-value pair with key "maxIter" in `data conf` or `conf` or `dsl` and replace the value with 5\n\n\n5. skip-data:\n\n .. code-block:: bash\n\n fate_test suite -i --skip-data\n\n will run testsuites in *path1* without uploading data specified in *benchmark.json*.\n\n\n6. yes:\n\n .. code-block:: bash\n\n fate_test suite -i --yes\n\n will run testsuites in *path1* directly, skipping double check\n\n7. skip-dsl-jobs:\n\n .. code-block:: bash\n\n fate_test suite -i --skip-dsl-jobs\n\n will run testsuites in *path1* but skip all *tasks* in testsuites. It\'s would be useful when only pipeline tasks needed.\n\n8. skip-pipeline-jobs:\n\n .. code-block:: bash\n\n fate_test suite -i --skip-pipeline-jobs\n\n will run testsuites in *path1* but skip all *pipeline tasks* in testsuites. It\'s would be useful when only dsl tasks needed.\n\n\nBenchmark Quality\n------------------\n\nBenchmark-quality is used for comparing modeling quality between FATE\nand other machine learning systems. Benchmark produces a metrics comparison\nsummary for each benchmark job group.\n\n.. code-block:: bash\n\n fate_test benchmark-quality -i examples/benchmark_quality/hetero_linear_regression\n\n.. code-block:: bash\n\n +-------+--------------------------------------------------------------+\n | Data | Name |\n +-------+--------------------------------------------------------------+\n | train | {\'guest\': \'motor_hetero_guest\', \'host\': \'motor_hetero_host\'} |\n | test | {\'guest\': \'motor_hetero_guest\', \'host\': \'motor_hetero_host\'} |\n +-------+--------------------------------------------------------------+\n +------------------------------------+--------------------+--------------------+-------------------------+---------------------+\n | Model Name | explained_variance | r2_score | root_mean_squared_error | mean_squared_error |\n +------------------------------------+--------------------+--------------------+-------------------------+---------------------+\n | local-linear_regression-regression | 0.9035168452250094 | 0.9035070863155368 | 0.31340413289880553 | 0.09822215051805216 |\n | FATE-linear_regression-regression | 0.903146386539082 | 0.9031411831961411 | 0.3139977881119483 | 0.09859461093919596 |\n +------------------------------------+--------------------+--------------------+-------------------------+---------------------+\n +-------------------------+-----------+\n | Metric | All Match |\n +-------------------------+-----------+\n | explained_variance | True |\n | r2_score | True |\n | root_mean_squared_error | True |\n | mean_squared_error | True |\n +-------------------------+-----------+\n\ncommand options\n~~~~~~~~~~~~~~~\n\nuse the following command to show help message\n\n.. code-block:: bash\n\n fate_test benchmark-quality --help\n\n1. include:\n\n .. code-block:: bash\n\n fate_test benchmark-quality -i \n\n will run benchmark testsuites in *path1*\n\n2. exclude:\n\n .. code-block:: bash\n\n fate_test benchmark-quality -i -e -e ...\n\n will run benchmark testsuites in *path1* but not in *path2* and *path3*\n\n3. glob:\n\n .. code-block:: bash\n\n fate_test benchmark-quality -i -g "hetero*"\n\n will run benchmark testsuites in sub directory start with *hetero* of *path1*\n\n4. tol:\n\n .. code-block:: bash\n\n fate_test benchmark-quality -i -t 1e-3\n\n will run benchmark testsuites in *path1* with absolute tolerance of difference between metrics set to 0.001.\n If absolute difference between metrics is smaller than *tol*, then metrics are considered\n almost equal. Check benchmark testsuite `writing guide <#benchmark-testsuite>`_ on setting alternative tolerance.\n\n5. skip-data:\n\n .. code-block:: bash\n\n fate_test benchmark-quality -i --skip-data\n\n will run benchmark testsuites in *path1* without uploading data specified in *benchmark.json*.\n\n\n6. yes:\n\n .. code-block:: bash\n\n fate_test benchmark-quality -i --yes\n\n will run benchmark testsuites in *path1* directly, skipping double check\n\n\nbenchmark testsuite\n~~~~~~~~~~~~~~~~~~~\n\nConfiguration of jobs should be specified in a benchmark testsuite whose file name ends\nwith "\\*benchmark.json". For benchmark testsuite example,\nplease refer `here <../../examples/benchmark_quality>`_.\n\nA benchmark testsuite includes the following elements:\n\n- data: list of local data to be uploaded before running FATE jobs\n\n - file: path to original data file to be uploaded, should be relative to testsuite or FATE installation path\n - head: whether file includes header\n - partition: number of partition for data storage\n - table_name: table name in storage\n - namespace: table namespace in storage\n - role: which role to upload the data, as specified in fate_test.config;\n naming format is: "{role_type}_{role_index}", index starts at 0\n\n .. code-block:: json\n\n "data": [\n {\n "file": "examples/data/motor_hetero_host.csv",\n "head": 1,\n "partition": 8,\n "table_name": "motor_hetero_host",\n "namespace": "experiment",\n "role": "host_0"\n }\n ]\n\n- job group: each group includes arbitrary number of jobs with paths to corresponding script and configuration\n\n - job: name of job to be run, must be unique within each group list\n\n - script: path to `testing script <#testing-script>`_, should be relative to testsuite\n - conf: path to job configuration file for script, should be relative to testsuite\n\n .. code-block:: json\n\n "local": {\n "script": "./local-linr.py",\n "conf": "./linr_config.yaml"\n }\n\n - compare_setting: additional setting for quality metrics comparison, currently only takes ``relative_tol``\n\n If metrics *a* and *b* satisfy *abs(a-b) <= max(relative_tol \\* max(abs(a), abs(b)), absolute_tol)*\n (from `math module `_),\n they are considered almost equal. In the below example, metrics from "local" and "FATE" jobs are\n considered almost equal if their relative difference is smaller than\n *0.05 \\* max(abs(local_metric), abs(pipeline_metric)*.\n\n .. code-block:: json\n\n "linear_regression-regression": {\n "local": {\n "script": "./local-linr.py",\n "conf": "./linr_config.yaml"\n },\n "FATE": {\n "script": "./fate-linr.py",\n "conf": "./linr_config.yaml"\n },\n "compare_setting": {\n "relative_tol": 0.01\n }\n }\n\n\ntesting script\n~~~~~~~~~~~~~~\n\nAll job scripts need to have ``Main`` function as an entry point for executing jobs; scripts should\nreturn two dictionaries: first with data information key-value pairs: {data_type}: {data_name_dictionary};\nthe second contains {metric_name}: {metric_value} key-value pairs for metric comparison.\n\nBy default, the final data summary shows the output from the job named "FATE"; if no such job exists,\ndata information returned by the first job is shown. For clear presentation, we suggest that user follow\nthis general `guideline <../../examples/data/README.md#data-set-naming-rule>`_ for data set naming. In the case of multi-host\ntask, consider numbering host as such:\n\n::\n\n {\'guest\': \'default_credit_homo_guest\',\n \'host_1\': \'default_credit_homo_host_1\',\n \'host_2\': \'default_credit_homo_host_2\'}\n\nReturned quality metrics of the same key are to be compared.\nNote that only **real-value** metrics can be compared.\n\n- FATE script: ``Main`` always has three inputs:\n\n - config: job configuration, `JobConfig <../fate_client/pipeline/utils/tools.py#L64>`_ object loaded from "fate_test_config.yaml"\n - param: job parameter setting, dictionary loaded from "conf" file specified in benchmark testsuite\n - namespace: namespace suffix, user-given *namespace* or generated timestamp string when using *namespace-mangling*\n\n- non-FATE script: ``Main`` always has one input:\n\n - param: job parameter setting, dictionary loaded from "conf" file specified in benchmark testsuite\n\n\ndata\n----\n\n`Data` sub-command is used for upload or delete dataset in suite\'s.\n\ncommand options\n~~~~~~~~~~~~~~~\n\n.. code-block:: bash\n\n fate_test data --help\n\n1. include:\n\n .. code-block:: bash\n\n fate_test data [upload|delete] -i \n\n will upload/delete dataset in testsuites in *path1*\n\n2. exclude:\n\n .. code-block:: bash\n\n fate_test data [upload|delete] -i -e -e ...\n\n will upload/delete dataset in testsuites in *path1* but not in *path2* and *path3*\n\n3. glob:\n\n .. code-block:: bash\n\n fate_test data [upload|delete] -i -g "hetero*"\n\n will upload/delete dataset in testsuites in sub directory start with *hetero* of *path1*\n\n\nfull command options\n---------------------\n\n.. click:: fate_test.scripts.cli:cli\n :prog: fate_test\n :show-nested:\n', "author": "FederatedAI", From bcfd419440737a433415fe2f16de95cbe008c488 Mon Sep 17 00:00:00 2001 From: mgqa34 Date: Tue, 22 Feb 2022 14:02:54 +0800 Subject: [PATCH 67/99] Update examples/pipeline/hetero_sbt/pipeline-hetero-sbt-EINI-with-random-mask.py Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .../hetero_sbt/pipeline-hetero-sbt-EINI-with-random-mask.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/pipeline/hetero_sbt/pipeline-hetero-sbt-EINI-with-random-mask.py b/examples/pipeline/hetero_sbt/pipeline-hetero-sbt-EINI-with-random-mask.py index 33f81dffa4..3da14852ae 100644 --- a/examples/pipeline/hetero_sbt/pipeline-hetero-sbt-EINI-with-random-mask.py +++ b/examples/pipeline/hetero_sbt/pipeline-hetero-sbt-EINI-with-random-mask.py @@ -83,7 +83,10 @@ def main(config="../../config.yaml", namespace=""): pipeline.add_component(reader_0) pipeline.add_component(reader_1) pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data)) - pipeline.add_component(data_transform_1, data=Data(data=reader_1.output.data), model=Model(data_transform_0.output.model)) + pipeline.add_component( + data_transform_1, data=Data( + data=reader_1.output.data), model=Model( + data_transform_0.output.model)) pipeline.add_component(intersect_0, data=Data(data=data_transform_0.output.data)) pipeline.add_component(intersect_1, data=Data(data=data_transform_1.output.data)) pipeline.add_component(hetero_secure_boost_0, data=Data(train_data=intersect_0.output.data, From be91d58e5ad840937daac32ed361d0fc0d0dae51 Mon Sep 17 00:00:00 2001 From: mgqa34 Date: Tue, 22 Feb 2022 14:03:27 +0800 Subject: [PATCH 68/99] Update examples/pipeline/hetero_sbt/pipeline-hetero-sbt-EINI-with-random-mask.py Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .../hetero_sbt/pipeline-hetero-sbt-EINI-with-random-mask.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/pipeline/hetero_sbt/pipeline-hetero-sbt-EINI-with-random-mask.py b/examples/pipeline/hetero_sbt/pipeline-hetero-sbt-EINI-with-random-mask.py index 3da14852ae..f34e0c6846 100644 --- a/examples/pipeline/hetero_sbt/pipeline-hetero-sbt-EINI-with-random-mask.py +++ b/examples/pipeline/hetero_sbt/pipeline-hetero-sbt-EINI-with-random-mask.py @@ -57,7 +57,9 @@ def main(config="../../config.yaml", namespace=""): data_transform_0, data_transform_1 = DataTransform(name="data_transform_0"), DataTransform(name="data_transform_1") - data_transform_0.get_party_instance(role="guest", party_id=guest).component_param(with_label=True, output_format="dense") + data_transform_0.get_party_instance( + role="guest", party_id=guest).component_param( + with_label=True, output_format="dense") data_transform_0.get_party_instance(role="host", party_id=host).component_param(with_label=False) data_transform_1.get_party_instance(role="guest", party_id=guest).component_param(with_label=True, output_format="dense") data_transform_1.get_party_instance(role="host", party_id=host).component_param(with_label=False) From 075da303cd8823faf8fb3d2a485b71d040904bfe Mon Sep 17 00:00:00 2001 From: mgqa34 Date: Tue, 22 Feb 2022 14:03:39 +0800 Subject: [PATCH 69/99] Update examples/pipeline/hetero_sbt/pipeline-hetero-sbt-EINI-with-random-mask.py Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .../hetero_sbt/pipeline-hetero-sbt-EINI-with-random-mask.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/pipeline/hetero_sbt/pipeline-hetero-sbt-EINI-with-random-mask.py b/examples/pipeline/hetero_sbt/pipeline-hetero-sbt-EINI-with-random-mask.py index f34e0c6846..2b89b59c1a 100644 --- a/examples/pipeline/hetero_sbt/pipeline-hetero-sbt-EINI-with-random-mask.py +++ b/examples/pipeline/hetero_sbt/pipeline-hetero-sbt-EINI-with-random-mask.py @@ -61,7 +61,9 @@ def main(config="../../config.yaml", namespace=""): role="guest", party_id=guest).component_param( with_label=True, output_format="dense") data_transform_0.get_party_instance(role="host", party_id=host).component_param(with_label=False) - data_transform_1.get_party_instance(role="guest", party_id=guest).component_param(with_label=True, output_format="dense") + data_transform_1.get_party_instance( + role="guest", party_id=guest).component_param( + with_label=True, output_format="dense") data_transform_1.get_party_instance(role="host", party_id=host).component_param(with_label=False) # data intersect component From f39bf043602a37441cf29fab85d730ebe30487b7 Mon Sep 17 00:00:00 2001 From: dylan-fan <289765648@qq.com> Date: Tue, 22 Feb 2022 15:41:25 +0800 Subject: [PATCH 70/99] add eggroll sql for deploy Signed Off: dylan-fan <289765648@qq.com> --- .../sql/create-eggroll-meta-tables.sql | 125 ++++++++++++++++++ deploy/cluster-deploy/sql/insert-node.sql | 7 + .../feldman_verifiable_sum.md | 2 +- 3 files changed, 133 insertions(+), 1 deletion(-) create mode 100644 deploy/cluster-deploy/sql/create-eggroll-meta-tables.sql create mode 100644 deploy/cluster-deploy/sql/insert-node.sql diff --git a/deploy/cluster-deploy/sql/create-eggroll-meta-tables.sql b/deploy/cluster-deploy/sql/create-eggroll-meta-tables.sql new file mode 100644 index 0000000000..dc8a685984 --- /dev/null +++ b/deploy/cluster-deploy/sql/create-eggroll-meta-tables.sql @@ -0,0 +1,125 @@ +-- create database if not exists, default database is eggroll_meta +-- CREATE DATABASE IF NOT EXISTS `eggroll_meta`; + +-- all operation under this database +-- USE `eggroll_meta`; + +-- store_locator +CREATE TABLE IF NOT EXISTS `store_locator` ( + `store_locator_id` SERIAL PRIMARY KEY, + `store_type` VARCHAR(255) NOT NULL, + `namespace` VARCHAR(2000) NOT NULL DEFAULT 'DEFAULT', + `name` VARCHAR(2000) NOT NULL, + `path` VARCHAR(2000) NOT NULL DEFAULT '', + `total_partitions` INT UNSIGNED NOT NULL, + `partitioner` VARCHAR(2000) NOT NULL DEFAULT 'BYTESTRING_HASH', + `serdes` VARCHAR(2000) NOT NULL DEFAULT '', + `version` INT UNSIGNED NOT NULL DEFAULT 0, + `status` VARCHAR(255) NOT NULL, + `created_at` DATETIME DEFAULT CURRENT_TIMESTAMP, + `updated_at` DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP +) DEFAULT CHARACTER SET latin1 COLLATE latin1_swedish_ci; + +CREATE UNIQUE INDEX `idx_u_store_loinsert-node.sqlcator_ns_n` ON `store_locator` (`namespace`(120), `name`(640)); +CREATE INDEX `idx_store_locator_st` ON `store_locator` (`store_type`(255)); +CREATE INDEX `idx_store_locator_ns` ON `store_locator` (`namespace`(767)); +CREATE INDEX `idx_store_locator_n` ON `store_locator` (`name`(767)); +CREATE INDEX `idx_store_locator_s` ON `store_locator` (`status`(255)); +CREATE INDEX `idx_store_locator_v` ON `store_locator` (`version`); + + +-- store (option) +CREATE TABLE IF NOT EXISTS `store_option` ( + `store_option_id` SERIAL PRIMARY KEY, + `store_locator_id` BIGINT UNSIGNED NOT NULL, + `name` VARCHAR(255) NOT NULL, + `data` VARCHAR(2000) NOT NULL DEFAULT '', + `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + `updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP +) DEFAULT CHARACTER SET latin1 COLLATE latin1_swedish_ci; + +CREATE INDEX `idx_store_option_si` ON `store_option` (`store_locator_id`); + + +-- store_partition +CREATE TABLE IF NOT EXISTS `store_partition` ( + `store_partition_id` SERIAL PRIMARY KEY, -- self-increment sequence + `store_locator_id` BIGINT UNSIGNED NOT NULL, + `node_id` BIGINT UNSIGNED NOT NULL, + `partition_id` INT UNSIGNED NOT NULL, -- partition id of a store + `status` VARCHAR(255) NOT NULL, + `created_at` DATETIME DEFAULT CURRENT_TIMESTAMP, + `updated_at` DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP +) DEFAULT CHARACTER SET latin1 COLLATE latin1_swedish_ci; + +CREATE UNIQUE INDEX `idx_u_store_partition_si_spi_ni` ON `store_partition` (`store_locator_id`, `store_partition_id`, `node_id`); +CREATE INDEX `idx_store_partition_sli` ON `store_partition` (`store_locator_id`); +CREATE INDEX `idx_store_partition_ni` ON `store_partition` (`node_id`); +CREATE INDEX `idx_store_partition_s` ON `store_partition` (`status`(255)); + + +-- node +CREATE TABLE IF NOT EXISTS `server_node` ( + `server_node_id` SERIAL PRIMARY KEY, + `name` VARCHAR(2000) NOT NULL DEFAULT '', + `server_cluster_id` BIGINT UNSIGNED NOT NULL DEFAULT 0, + `host` VARCHAR(1000) NOT NULL, + `port` INT NOT NULL, + `node_type` VARCHAR(255) NOT NULL, + `status` VARCHAR(255) NOT NULL, + `last_heartbeat_at` DATETIME DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP, + `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + `updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP +) DEFAULT CHARACTER SET latin1 COLLATE latin1_swedish_ci; + +CREATE INDEX `idx_server_node_h_p_nt` ON `server_node` (`host`(600), `port`, `node_type`(100)); +CREATE INDEX `idx_server_node_h` ON `server_node` (`host`(767)); +CREATE INDEX `idx_server_node_sci` ON `server_node` (`server_cluster_id`); +CREATE INDEX `idx_server_node_nt` ON `server_node` (`node_type`(255)); +CREATE INDEX `idx_server_node_s` ON `server_node` (`status`(255)); + + +-- session (main) +CREATE TABLE IF NOT EXISTS `session_main` ( + `session_id` VARCHAR(767) PRIMARY KEY, + `name` VARCHAR(2000) NOT NULL DEFAULT '', + `status` VARCHAR(255) NOT NULL, + `tag` VARCHAR(255), + `total_proc_count` INT, + `active_proc_count` INT, + `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + `updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP +) DEFAULT CHARACTER SET latin1 COLLATE latin1_swedish_ci; + +CREATE INDEX `idx_session_main_s` ON `session_main` (`status`); + + +-- session (option) +CREATE TABLE IF NOT EXISTS `session_option` ( + `session_option_id` SERIAL PRIMARY KEY, + `session_id` VARCHAR(2000), + `name` VARCHAR(255) NOT NULL, + `data` VARCHAR(2000) NOT NULL DEFAULT '', + `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + `updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP +) DEFAULT CHARACTER SET latin1 COLLATE latin1_swedish_ci; + +CREATE INDEX `idx_session_option_si` ON `session_option` (`session_id`(767)); + + +-- session (processor) +CREATE TABLE IF NOT EXISTS `session_processor` ( + `processor_id` SERIAL PRIMARY KEY, + `session_id` VARCHAR(767), + `server_node_id` INT NOT NULL, + `processor_type` VARCHAR(255) NOT NULL, + `status` VARCHAR(255), + `tag` VARCHAR(255), + `command_endpoint` VARCHAR(255), + `transfer_endpoint` VARCHAR(255), + `pid` INT NOT NULL DEFAULT -1, + `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + `updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP +) DEFAULT CHARACTER SET latin1 COLLATE latin1_swedish_ci; + +CREATE INDEX `idx_session_processor_si` ON `session_processor` (`session_id`(767)); diff --git a/deploy/cluster-deploy/sql/insert-node.sql b/deploy/cluster-deploy/sql/insert-node.sql new file mode 100644 index 0000000000..109b3847a2 --- /dev/null +++ b/deploy/cluster-deploy/sql/insert-node.sql @@ -0,0 +1,7 @@ +CREATE DATABASE IF NOT EXISTS fate_flow; +CREATE USER fate@'localhost' IDENTIFIED BY 'fate'; +GRANT ALL ON fate_flow.* TO fate@'localhost'; +GRANT ALL ON eggroll_meta.* TO fate@'localhost; +use eggroll_meta; +INSERT INTO server_node (host, port, node_type, status) values ('127.0.0.1', '4670', 'CLUSTER_MANAGER', 'HEALTHY'); +INSERT INTO server_node (host, port, node_type, status) values ('127.0.0.1', '4671', 'NODE_MANAGER', 'HEALTHY'); diff --git a/doc/federatedml_component/feldman_verifiable_sum.md b/doc/federatedml_component/feldman_verifiable_sum.md index 4e17bf2880..8a87239703 100644 --- a/doc/federatedml_component/feldman_verifiable_sum.md +++ b/doc/federatedml_component/feldman_verifiable_sum.md @@ -6,7 +6,7 @@ Verifiable secret sharing mechanism is an efficient and practical secret sharing mechanism. Feldman Verifiable sum is a multi-party private data summation module based on verifiable secret sharing.This component can sum the same feature of common users among different participants -without exposing private data. +without exposing private data.（This module is still in the research stage, has not yet been put into production） Here, three participants of the federation process is given, Party A represents Guest, party B and party C represent Host. The process of From fc826994d6ed4781173349b60f403326995f4cbc Mon Sep 17 00:00:00 2001 From: zhihuiwan <15779896112@163.com> Date: Tue, 22 Feb 2022 18:53:55 +0800 Subject: [PATCH 71/99] update address api --- python/fate_arch/common/address.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/python/fate_arch/common/address.py b/python/fate_arch/common/address.py index 26c8b6d9b4..8dec9cdf0a 100644 --- a/python/fate_arch/common/address.py +++ b/python/fate_arch/common/address.py @@ -7,7 +7,7 @@ class AddressBase(AddressABC): def __init__(self, connector_name=None): self.connector_name = connector_name if connector_name: - connector = StorageConnector(connector_name=connector_name, engine=self.get_name) + connector = StorageConnector(connector_name=connector_name, engine=self.storage_engine) if connector.get_info(): for k, v in connector.get_info().items(): if hasattr(self, k) and v: @@ -18,7 +18,7 @@ def connector(self): return {} @property - def get_name(self): + def storage_engine(self): return @@ -44,7 +44,7 @@ def connector(self): return {"home": self.home} @property - def get_name(self): + def storage_engine(self): return StorageEngine.STANDALONE @@ -69,7 +69,7 @@ def connector(self): return {"home": self.home} @property - def get_name(self): + def storage_engine(self): return StorageEngine.EGGROLL @@ -93,7 +93,7 @@ def connector(self): return {"name_node": self.name_node} @property - def get_name(self): + def storage_engine(self): return StorageEngine.HDFS @@ -112,7 +112,7 @@ def __repr__(self): return self.__str__() @property - def get_name(self): + def storage_engine(self): return StorageEngine.PATH @@ -141,7 +141,7 @@ def connector(self): return {"user": self.user, "passwd": self.passwd, "host": self.host, "port": self.port, "db": self.db} @property - def get_name(self): + def storage_engine(self): return StorageEngine.MYSQL @@ -171,7 +171,7 @@ def connector(self): return {"host": self.host, "port": self.port, "username": self.username, "password": self.password, "auth_mechanism": self.auth_mechanism, "database": self.database} @property - def get_name(self): + def storage_engine(self): return StorageEngine.HIVE @@ -199,7 +199,7 @@ def __repr__(self): return self.__str__() @property - def get_name(self): + def storage_engine(self): return StorageEngine.LINKIS_HIVE @@ -218,5 +218,5 @@ def __repr__(self): return self.__str__() @property - def get_name(self): + def storage_engine(self): return StorageEngine.LOCALFS \ No newline at end of file From 2e76cb0aa7a0899e5998e92f87337b357fa77f92 Mon Sep 17 00:00:00 2001 From: mgqa34 Date: Tue, 22 Feb 2022 19:10:22 +0800 Subject: [PATCH 72/99] move tensorflow-keras and pytorch dependencies of fate-client Signed-off-by: mgqa34 --- .../pipeline/component/__init__.py | 3 +-- .../nn/backend/keras/model_builder.py | 21 +++++++++++++++++-- .../nn/backend/pytorch/model_builder.py | 12 +++++++++-- .../component/nn/models/sequantial.py | 9 ++++++-- python/fate_client/pyproject.toml | 2 -- python/fate_client/setup.py | 2 -- 6 files changed, 37 insertions(+), 12 deletions(-) diff --git a/python/fate_client/pipeline/component/__init__.py b/python/fate_client/pipeline/component/__init__.py index 3333ef1c33..b9ba714bc2 100644 --- a/python/fate_client/pipeline/component/__init__.py +++ b/python/fate_client/pipeline/component/__init__.py @@ -30,7 +30,6 @@ from pipeline.component.union import Union from pipeline.component.feldman_verifiable_sum import FeldmanVerifiableSum from pipeline.component.sample_weight import SampleWeight -from pipeline.component.sbt_feature_transformer import SBTTransformer from pipeline.component.feature_imputation import FeatureImputation from pipeline.component.label_transform import LabelTransform from pipeline.component.hetero_sshe_lr import HeteroSSHELR @@ -47,7 +46,7 @@ "HomoLR", "HomoNN", "HomoSecureBoost", "HomoFeatureBinning", "Intersection", "LocalBaseline", "OneHotEncoder", "PSI", "Reader", "Scorecard", "FederatedSample", "FeatureScale", "Union", "ColumnExpand", "FeldmanVerifiableSum", - "SampleWeight", "DataTransform", "SBTTransformer", "FeatureImputation", + "SampleWeight", "DataTransform", "FeatureImputation", "LabelTransform", "SecureInformationRetrieval", "CacheLoader", "ModelLoader", "HeteroSSHELR", "HeteroKmeans", "HomoOneHotEncoder"] diff --git a/python/fate_client/pipeline/component/nn/backend/keras/model_builder.py b/python/fate_client/pipeline/component/nn/backend/keras/model_builder.py index f3518ae840..18169aa91d 100644 --- a/python/fate_client/pipeline/component/nn/backend/keras/model_builder.py +++ b/python/fate_client/pipeline/component/nn/backend/keras/model_builder.py @@ -13,9 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from tensorflow.keras.models import Sequential import json +_TF_KERAS_VALID = False +try: + from tensorflow.keras.models import Sequential + _TF_KERAS_VALID = True +except ImportError: + pass + def build_model(model_type="sequential"): if model_type != "sequential": @@ -26,9 +32,16 @@ def build_model(model_type="sequential"): class SequentialModel(object): def __init__(self): - self._model = Sequential() + if _TF_KERAS_VALID: + self._model = Sequential() + else: + self._model = None def add(self, layer): + if not _TF_KERAS_VALID: + raise ImportError("Please install tensorflow first, " + "can not import sequential model from tensorflow.keras.model !!!") + self._model.add(layer) @staticmethod @@ -54,6 +67,10 @@ def get_optimizer_config(optimizer): return opt_config def get_network_config(self): + if not _TF_KERAS_VALID: + raise ImportError("Please install tensorflow first, " + "can not import sequential model from tensorflow.keras.model !!!") + return json.loads(self._model.to_json()) diff --git a/python/fate_client/pipeline/component/nn/backend/pytorch/model_builder.py b/python/fate_client/pipeline/component/nn/backend/pytorch/model_builder.py index 242e1e8320..39b1c8568e 100644 --- a/python/fate_client/pipeline/component/nn/backend/pytorch/model_builder.py +++ b/python/fate_client/pipeline/component/nn/backend/pytorch/model_builder.py @@ -14,7 +14,12 @@ # limitations under the License. # -import torch.nn as nn +_TORCH_VALID = False +try: + import torch.nn as nn + _TORCH_VALID = True +except ImportError: + pass def build_model(model_type="sequential"): @@ -26,6 +31,9 @@ def build_model(model_type="sequential"): class SequentialModel(object): def __init__(self): - self._model = nn.Sequential() + if _TORCH_VALID: + self._model = nn.Sequential() + else: + self._model = None diff --git a/python/fate_client/pipeline/component/nn/models/sequantial.py b/python/fate_client/pipeline/component/nn/models/sequantial.py index d46376dbe5..6acfbf77bc 100644 --- a/python/fate_client/pipeline/component/nn/models/sequantial.py +++ b/python/fate_client/pipeline/component/nn/models/sequantial.py @@ -13,7 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from tensorflow.python.keras.engine import base_layer +_TF_KERAS_VALID = False +try: + from tensorflow.python.keras.engine import base_layer + _TF_KERAS_VALID = True +except ImportError: + pass class Sequential(object): @@ -25,7 +30,7 @@ def is_empty(self): return self._model is None def add(self, layer): - if isinstance(layer, base_layer.Layer): + if _TF_KERAS_VALID and isinstance(layer, base_layer.Layer): layer_type = "keras" elif isinstance(layer, dict): layer_type = "nn" diff --git a/python/fate_client/pyproject.toml b/python/fate_client/pyproject.toml index f1962fd397..7372a1046c 100644 --- a/python/fate_client/pyproject.toml +++ b/python/fate_client/pyproject.toml @@ -34,8 +34,6 @@ requests = ">=2.24.0" click = "^7.1.2" "ruamel.yaml" = "^0.16.10" loguru = "^0.5.1" -tensorflow = "==2.3.4" -torch = "1.4.0" flask = "^1.0.2" setuptools = "^50.0" diff --git a/python/fate_client/setup.py b/python/fate_client/setup.py index 3d1481bda2..60b7177208 100644 --- a/python/fate_client/setup.py +++ b/python/fate_client/setup.py @@ -39,8 +39,6 @@ "requests_toolbelt>=0.9.1,<0.10.0", "ruamel.yaml>=0.16.10,<0.17.0", "setuptools>=50.0,<51.0", - "tensorflow==2.3.4", - "torch", ] entry_points = { From dc9066c3cdd9353ed6a05ed0660f3ed16c29102a Mon Sep 17 00:00:00 2001 From: zhihuiwan <15779896112@163.com> Date: Wed, 23 Feb 2022 12:00:59 +0800 Subject: [PATCH 73/99] fix bug: specified key too long --- python/fate_arch/metastore/db_models.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/python/fate_arch/metastore/db_models.py b/python/fate_arch/metastore/db_models.py index bc865e5827..4c9d4f9fd5 100644 --- a/python/fate_arch/metastore/db_models.py +++ b/python/fate_arch/metastore/db_models.py @@ -77,18 +77,28 @@ class Meta: def init_database_tables(): members = inspect.getmembers(sys.modules[__name__], inspect.isclass) table_objs = [] + create_failed_list = [] for name, obj in members: if obj != DataBaseModel and issubclass(obj, DataBaseModel): table_objs.append(obj) - DB.create_tables(table_objs) + LOGGER.info(f"start create table {obj.__name__}") + try: + obj.create_table() + LOGGER.info(f"create table success: {obj.__name__}") + except Exception as e: + LOGGER.exception(e) + create_failed_list.append(obj.__name__) + if create_failed_list: + LOGGER.info(f"create tables failed: {create_failed_list}") + raise Exception(f"create tables failed: {create_failed_list}") class StorageTableMetaModel(DataBaseModel): f_name = CharField(max_length=100, index=True) f_namespace = CharField(max_length=100, index=True) f_address = JSONField() - f_engine = CharField(max_length=100, index=True) # 'EGGROLL', 'MYSQL' - f_store_type = CharField(max_length=50, index=True, null=True) # store type + f_engine = CharField(max_length=100) # 'EGGROLL', 'MYSQL' + f_store_type = CharField(max_length=50, null=True) # store type f_options = JSONField() f_partitions = IntegerField(null=True) @@ -114,8 +124,8 @@ class Meta: class SessionRecord(DataBaseModel): - f_engine_session_id = CharField(max_length=150, null=False, index=True) - f_manager_session_id = CharField(max_length=150, null=False, index=True) + f_engine_session_id = CharField(max_length=150, null=False) + f_manager_session_id = CharField(max_length=150, null=False) f_engine_type = CharField(max_length=10, index=True) f_engine_name = CharField(max_length=50, index=True) f_engine_address = JSONField() From 7a4a2d9c9054291cf0b6cb0b364d1b7335ebf82d Mon Sep 17 00:00:00 2001 From: zhihuiwan <15779896112@163.com> Date: Wed, 23 Feb 2022 16:42:52 +0800 Subject: [PATCH 74/99] git module --- fateflow | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fateflow b/fateflow index bfd5466115..cf505abe55 160000 --- a/fateflow +++ b/fateflow @@ -1 +1 @@ -Subproject commit bfd5466115d915eb776664e6bae8364c4f9a5b83 +Subproject commit cf505abe5562c448e6ad604ed49ab7ccc9d75466 From 464ab8f177585dcfe5cbd86bc3ecb0419645e013 Mon Sep 17 00:00:00 2001 From: zhihuiwan <15779896112@163.com> Date: Wed, 23 Feb 2022 18:48:47 +0800 Subject: [PATCH 75/99] update encrypt module --- conf/service_conf.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/service_conf.yaml b/conf/service_conf.yaml index 54f80a1807..a380541400 100644 --- a/conf/service_conf.yaml +++ b/conf/service_conf.yaml @@ -2,7 +2,7 @@ use_registry: false use_deserialize_safe_module: false dependent_distribution: false encrypt_password: false -encrypt_module: fate_arcm.common.encrypt_utils#pwdecrypt +encrypt_module: fate_arch.common.encrypt_utils#pwdecrypt private_key: fateflow: # you must set real ip address, 127.0.0.1 and 0.0.0.0 is not supported From df28754d508223e6c02aedab6b2bb42c732d7a70 Mon Sep 17 00:00:00 2001 From: mgqa34 Date: Wed, 23 Feb 2022 19:48:53 +0800 Subject: [PATCH 76/99] update examples of hetero-lr, fix homo-binning, modify default parameter of sshe-lr Signed-off-by: mgqa34 --- .../hetero_lr/pipeline-lr-binary.py | 2 + .../hetero_lr/pipeline-lr-multi.py | 2 + .../hetero_logistic_regression_testsuite.json | 4 + .../hetero_lr_batch_random_strategy_conf.json | 82 +++++++++++++++++ .../hetero_lr_batch_random_strategy_dsl.json | 78 ++++++++++++++++ .../hetero_lr_warm_start_conf.json | 2 - ...ogistic_regression_pipeline_testsuite.json | 3 + ...ipeline-hetero-lr-batch-random-strategy.py | 92 +++++++++++++++++++ .../pipeline-hetero-lr-warm-start.py | 2 - .../pipeline/param/hetero_sshe_lr_param.py | 4 +- .../homo_feature_binning/homo_binning_base.py | 2 +- .../federatedml/param/hetero_sshe_lr_param.py | 4 +- 12 files changed, 268 insertions(+), 9 deletions(-) create mode 100644 examples/dsl/v2/hetero_logistic_regression/hetero_lr_batch_random_strategy_conf.json create mode 100644 examples/dsl/v2/hetero_logistic_regression/hetero_lr_batch_random_strategy_dsl.json create mode 100644 examples/pipeline/hetero_logistic_regression/pipeline-hetero-lr-batch-random-strategy.py diff --git a/examples/benchmark_quality/hetero_lr/pipeline-lr-binary.py b/examples/benchmark_quality/hetero_lr/pipeline-lr-binary.py index fb7081bc69..3752dbe17e 100644 --- a/examples/benchmark_quality/hetero_lr/pipeline-lr-binary.py +++ b/examples/benchmark_quality/hetero_lr/pipeline-lr-binary.py @@ -101,6 +101,8 @@ def main(config="../../config.yaml", param="./lr_config.yaml", namespace=""): "learning_rate": param["learning_rate"], "optimizer": param["optimizer"], "batch_size": param["batch_size"], + "shuffle": False, + "masked_rate": 0, "early_stop": "diff", "tol": 1e-5, "floating_point_precision": param.get("floating_point_precision"), diff --git a/examples/benchmark_quality/hetero_lr/pipeline-lr-multi.py b/examples/benchmark_quality/hetero_lr/pipeline-lr-multi.py index 390657a203..3aa9aa2d1c 100644 --- a/examples/benchmark_quality/hetero_lr/pipeline-lr-multi.py +++ b/examples/benchmark_quality/hetero_lr/pipeline-lr-multi.py @@ -89,6 +89,8 @@ def main(config="../../config.yaml", param="./vehicle_config.yaml", namespace="" "learning_rate": param["learning_rate"], "optimizer": param["optimizer"], "batch_size": param["batch_size"], + "masked_rate": 0, + "shuffle": False, "early_stop": "diff", "init_param": { "init_method": param.get("init_method", 'random_uniform'), diff --git a/examples/dsl/v2/hetero_logistic_regression/hetero_logistic_regression_testsuite.json b/examples/dsl/v2/hetero_logistic_regression/hetero_logistic_regression_testsuite.json index adbcf9046e..5378ab86c7 100644 --- a/examples/dsl/v2/hetero_logistic_regression/hetero_logistic_regression_testsuite.json +++ b/examples/dsl/v2/hetero_logistic_regression/hetero_logistic_regression_testsuite.json @@ -106,6 +106,10 @@ "conf": "hetero_lr_warm_start_conf.json", "dsl": "hetero_lr_warm_start_dsl.json" }, + "hetero_lr_batch_random_strategy": { + "conf": "hetero_lr_batch_random_strategy_conf.json", + "dsl": "hetero_lr_batch_random_strategy_dsl.json" + }, "hetero_lr_normal_predict": { "deps": "hetero_lr_normal", "conf": "hetero_lr_normal_predict_conf.json", diff --git a/examples/dsl/v2/hetero_logistic_regression/hetero_lr_batch_random_strategy_conf.json b/examples/dsl/v2/hetero_logistic_regression/hetero_lr_batch_random_strategy_conf.json new file mode 100644 index 0000000000..263b644ea4 --- /dev/null +++ b/examples/dsl/v2/hetero_logistic_regression/hetero_lr_batch_random_strategy_conf.json @@ -0,0 +1,82 @@ +{ + "dsl_version": 2, + "initiator": { + "role": "guest", + "party_id": 9999 + }, + "role": { + "arbiter": [ + 10000 + ], + "host": [ + 10000 + ], + "guest": [ + 9999 + ] + }, + "component_parameters": { + "role": { + "host": { + "0": { + "reader_0": { + "table": { + "name": "breast_hetero_host", + "namespace": "experiment" + } + }, + "data_transform_0": { + "with_label": false + } + } + }, + "guest": { + "0": { + "reader_0": { + "table": { + "name": "breast_hetero_guest", + "namespace": "experiment" + } + }, + "data_transform_0": { + "with_label": true + } + } + } + }, + "common": { + "data_transform_0": { + "output_format": "dense" + }, + "hetero_lr_0": { + "penalty": "L2", + "tol": 0.0001, + "alpha": 0.01, + "optimizer": "rmsprop", + "batch_size": 320, + "batch_strategy": "random", + "learning_rate": 0.15, + "init_param": { + "init_method": "zeros" + }, + "max_iter": 30, + "early_stop": "diff", + "cv_param": { + "n_splits": 5, + "shuffle": false, + "random_seed": 103, + "need_cv": false + }, + "sqn_param": { + "update_interval_L": 3, + "memory_M": 5, + "sample_size": 5000, + "random_seed": null + } + }, + "evaluation_0": { + "eval_type": "binary" + } + } + } +} \ No newline at end of file diff --git a/examples/dsl/v2/hetero_logistic_regression/hetero_lr_batch_random_strategy_dsl.json b/examples/dsl/v2/hetero_logistic_regression/hetero_lr_batch_random_strategy_dsl.json new file mode 100644 index 0000000000..bdd74188d9 --- /dev/null +++ b/examples/dsl/v2/hetero_logistic_regression/hetero_lr_batch_random_strategy_dsl.json @@ -0,0 +1,78 @@ +{ + "components": { + "reader_0": { + "module": "Reader", + "output": { + "data": [ + "data" + ] + } + }, + "data_transform_0": { + "module": "DataTransform", + "input": { + "data": { + "data": [ + "reader_0.data" + ] + } + }, + "output": { + "data": [ + "data" + ], + "model": [ + "model" + ] + } + }, + "intersection_0": { + "module": "Intersection", + "input": { + "data": { + "data": [ + "data_transform_0.data" + ] + } + }, + "output": { + "data": [ + "data" + ] + } + }, + "hetero_lr_0": { + "module": "HeteroLR", + "input": { + "data": { + "train_data": [ + "intersection_0.data" + ] + } + }, + "output": { + "data": [ + "data" + ], + "model": [ + "model" + ] + } + }, + "evaluation_0": { + "module": "Evaluation", + "input": { + "data": { + "data": [ + "hetero_lr_0.data" + ] + } + }, + "output": { + "data": [ + "data" + ] + } + } + } +} \ No newline at end of file diff --git a/examples/dsl/v2/hetero_sshe_lr/hetero_lr_warm_start_conf.json b/examples/dsl/v2/hetero_sshe_lr/hetero_lr_warm_start_conf.json index 3928cd7ec8..2b09234930 100644 --- a/examples/dsl/v2/hetero_sshe_lr/hetero_lr_warm_start_conf.json +++ b/examples/dsl/v2/hetero_sshe_lr/hetero_lr_warm_start_conf.json @@ -65,8 +65,6 @@ "callbacks": [ "ModelCheckpoint" ], - "validation_freqs": 1, - "early_stopping_rounds": 1, "metrics": null, "use_first_metric_only": false, "save_freq": 1 diff --git a/examples/pipeline/hetero_logistic_regression/hetero_logistic_regression_pipeline_testsuite.json b/examples/pipeline/hetero_logistic_regression/hetero_logistic_regression_pipeline_testsuite.json index 4b521945e8..464c779e8b 100644 --- a/examples/pipeline/hetero_logistic_regression/hetero_logistic_regression_pipeline_testsuite.json +++ b/examples/pipeline/hetero_logistic_regression/hetero_logistic_regression_pipeline_testsuite.json @@ -89,6 +89,9 @@ }, "hetero-lr-warm-start": { "script": "pipeline-hetero-lr-warm-start.py" + }, + "hetero-lr-batch-random-strategy": { + "script": "pipeline-hetero-lr-batch-random-strategy.py" } } } \ No newline at end of file diff --git a/examples/pipeline/hetero_logistic_regression/pipeline-hetero-lr-batch-random-strategy.py b/examples/pipeline/hetero_logistic_regression/pipeline-hetero-lr-batch-random-strategy.py new file mode 100644 index 0000000000..bfee083090 --- /dev/null +++ b/examples/pipeline/hetero_logistic_regression/pipeline-hetero-lr-batch-random-strategy.py @@ -0,0 +1,92 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import argparse +import os +import sys + +cur_path = os.path.realpath(__file__) +for i in range(4): + cur_path = os.path.dirname(cur_path) +print(f'fate_path: {cur_path}') +sys.path.append(cur_path) + +from examples.pipeline.hetero_logistic_regression import common_tools + +from pipeline.utils.tools import load_job_config + + +def main(config="../../config.yaml", namespace=""): + # obtain config + if isinstance(config, str): + config = load_job_config(config) + + lr_param = { + "name": "hetero_lr_0", + "penalty": "L2", + "optimizer": "rmsprop", + "tol": 0.0001, + "alpha": 0.01, + "max_iter": 30, + "early_stop": "diff", + "batch_size": 320, + "batch_strategy": "random", + "learning_rate": 0.15, + "init_param": { + "init_method": "zeros" + }, + "sqn_param": { + "update_interval_L": 3, + "memory_M": 5, + "sample_size": 5000, + "random_seed": None + }, + "cv_param": { + "n_splits": 5, + "shuffle": False, + "random_seed": 103, + "need_cv": False + }, + "callback_param": { + "callbacks": ["ModelCheckpoint"], + "save_freq": "epoch" + } + } + + pipeline = common_tools.make_normal_dsl(config, namespace, lr_param) + # dsl_json = predict_pipeline.get_predict_dsl() + # conf_json = predict_pipeline.get_predict_conf() + # import json + # json.dump(dsl_json, open('./hetero-lr-normal-predict-dsl.json', 'w'), indent=4) + # json.dump(conf_json, open('./hetero-lr-normal-predict-conf.json', 'w'), indent=4) + + + # fit model + pipeline.fit() + # query component summary + common_tools.prettify(pipeline.get_component("hetero_lr_0").get_summary()) + common_tools.prettify(pipeline.get_component("evaluation_0").get_summary()) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser("PIPELINE DEMO") + parser.add_argument("-config", type=str, + help="config file") + args = parser.parse_args() + if args.config is not None: + main(args.config) + else: + main() diff --git a/examples/pipeline/hetero_sshe_lr/pipeline-hetero-lr-warm-start.py b/examples/pipeline/hetero_sshe_lr/pipeline-hetero-lr-warm-start.py index d70ee62f33..0e87ea5f67 100644 --- a/examples/pipeline/hetero_sshe_lr/pipeline-hetero-lr-warm-start.py +++ b/examples/pipeline/hetero_sshe_lr/pipeline-hetero-lr-warm-start.py @@ -97,8 +97,6 @@ def main(config="../../config.yaml", namespace=""): "reveal_every_iter": True, "callback_param": { "callbacks": ["ModelCheckpoint"], - "validation_freqs": 1, - "early_stopping_rounds": 1, "metrics": None, "use_first_metric_only": False, "save_freq": 1 diff --git a/python/fate_client/pipeline/param/hetero_sshe_lr_param.py b/python/fate_client/pipeline/param/hetero_sshe_lr_param.py index b83c26f9e6..84a5e86207 100644 --- a/python/fate_client/pipeline/param/hetero_sshe_lr_param.py +++ b/python/fate_client/pipeline/param/hetero_sshe_lr_param.py @@ -82,7 +82,7 @@ class LogisticRegressionParam(BaseParam): "respectively": Means guest and host can reveal their own part of weights only. "encrypted_reveal_in_host": Means host can be revealed his weights in encrypted mode, and guest can be revealed in normal mode. - reveal_every_iter: bool, default: True + reveal_every_iter: bool, default: False Whether reconstruct model weights every iteration. If so, Regularization is available. The performance will be better as well since the algorithm process is simplified. @@ -96,7 +96,7 @@ def __init__(self, penalty=None, decay=1, decay_sqrt=True, multi_class='ovr', use_mix_rand=True, reveal_strategy="respectively", - reveal_every_iter=True, + reveal_every_iter=False, callback_param=CallbackParam(), encrypted_mode_calculator_param=EncryptedModeCalculatorParam() ): diff --git a/python/federatedml/feature/homo_feature_binning/homo_binning_base.py b/python/federatedml/feature/homo_feature_binning/homo_binning_base.py index 36a5d8e695..22a3541cd2 100644 --- a/python/federatedml/feature/homo_feature_binning/homo_binning_base.py +++ b/python/federatedml/feature/homo_feature_binning/homo_binning_base.py @@ -53,7 +53,7 @@ def create_left_new(self): if np.fabs(value - self.value) <= consts.FLOAT_ZERO * 0.1: self.fixed = True return self - max_value = self.max_value + max_value = self.value return SplitPointNode(value, self.min_value, max_value, self.aim_rank, self.allow_error_rank) diff --git a/python/federatedml/param/hetero_sshe_lr_param.py b/python/federatedml/param/hetero_sshe_lr_param.py index 32d232efe1..adda7a1172 100644 --- a/python/federatedml/param/hetero_sshe_lr_param.py +++ b/python/federatedml/param/hetero_sshe_lr_param.py @@ -82,7 +82,7 @@ class LogisticRegressionParam(BaseParam): "respectively": Means guest and host can reveal their own part of weights only. "encrypted_reveal_in_host": Means host can be revealed his weights in encrypted mode, and guest can be revealed in normal mode. - reveal_every_iter: bool, default: True + reveal_every_iter: bool, default: False Whether reconstruct model weights every iteration. If so, Regularization is available. The performance will be better as well since the algorithm process is simplified. @@ -96,7 +96,7 @@ def __init__(self, penalty=None, decay=1, decay_sqrt=True, multi_class='ovr', use_mix_rand=True, reveal_strategy="respectively", - reveal_every_iter=True, + reveal_every_iter=False, callback_param=CallbackParam(), encrypted_mode_calculator_param=EncryptedModeCalculatorParam() ): From a602ce55b2fd08e61fd4b9f3e62c95230d676505 Mon Sep 17 00:00:00 2001 From: mgqa34 Date: Wed, 23 Feb 2022 23:50:04 +0800 Subject: [PATCH 77/99] recovery modify of homo binning base Signed-off-by: mgqa34 --- .../feature/homo_feature_binning/homo_binning_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/federatedml/feature/homo_feature_binning/homo_binning_base.py b/python/federatedml/feature/homo_feature_binning/homo_binning_base.py index 22a3541cd2..36a5d8e695 100644 --- a/python/federatedml/feature/homo_feature_binning/homo_binning_base.py +++ b/python/federatedml/feature/homo_feature_binning/homo_binning_base.py @@ -53,7 +53,7 @@ def create_left_new(self): if np.fabs(value - self.value) <= consts.FLOAT_ZERO * 0.1: self.fixed = True return self - max_value = self.value + max_value = self.max_value return SplitPointNode(value, self.min_value, max_value, self.aim_rank, self.allow_error_rank) From e7f8c175c9fa8c84a3d70b47b72b81b3dd6f53d2 Mon Sep 17 00:00:00 2001 From: mgqa34 Date: Thu, 24 Feb 2022 11:19:25 +0800 Subject: [PATCH 78/99] update gitmodule of fateboard Signed-off-by: mgqa34 --- .gitmodules | 2 +- fateboard | 2 +- fateflow | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index bfa4688abf..31e85fe3ce 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,7 @@ [submodule "fateboard"] path = fateboard url = https://github.com/FederatedAI/FATE-Board.git - branch = v1.7.2.1 + branch = develop-1.7.2.2 [submodule "eggroll"] path = eggroll url = https://github.com/WeBankFinTech/eggroll.git diff --git a/fateboard b/fateboard index 5eb42605ee..180a2b2471 160000 --- a/fateboard +++ b/fateboard @@ -1 +1 @@ -Subproject commit 5eb42605ee8570424b7bc79ee322ce0e3e831ca7 +Subproject commit 180a2b24717d9ab41f60d850418b8f829a6fe9db diff --git a/fateflow b/fateflow index 610cc71331..9bf35dd134 160000 --- a/fateflow +++ b/fateflow @@ -1 +1 @@ -Subproject commit 610cc713311fbf37495e4f3b6afe957ce15c36f1 +Subproject commit 9bf35dd13487bd5481c9fe1983f9a6419fad5e2e From 72aa5114746dd8308b83d612a3df10b82c67ec09 Mon Sep 17 00:00:00 2001 From: weijingchen Date: Thu, 24 Feb 2022 15:22:11 +0800 Subject: [PATCH 79/99] remove doc Signed-off-by: cwj --- doc/federatedml_component/README.md | 1 - doc/federatedml_component/README.zh.md | 1 - .../sbt_feature_transformer.md | 22 ------------------- 3 files changed, 24 deletions(-) delete mode 100644 doc/federatedml_component/sbt_feature_transformer.md diff --git a/doc/federatedml_component/README.md b/doc/federatedml_component/README.md index 0d0522f308..a15816404c 100644 --- a/doc/federatedml_component/README.md +++ b/doc/federatedml_component/README.md @@ -42,7 +42,6 @@ provide: | [Homo-NN](homo_nn.md) | HomoNN | Build homo neural network model through multiple parties. | Table, values are instances. | Table, values are instances. | | Neural Network Model, consists of model-meta and model-param. | | [Hetero Secure Boosting](ensemble.md) | HeteroSecureBoost | Build hetero secure boosting model through multiple parties | Table, values are instances. | Table, values are instances. | | SecureBoost Model, consists of model-meta and model-param. | | [Hetero Fast Secure Boosting](ensemble.md) | HeteroFastSecureBoost | Build hetero secure boosting model through multiple parties in layered/mix manners. | Table, values are instances. | Table, values are instances. | | FastSecureBoost Model, consists of model-meta and model-param. | -| [Hetero Secure Boost Feature Transformer](sbt_feature_transformer.md) | SBTFeatureTransformer | This component can encode sample using Hetero SBT leaf indices. | Table, values are instances. | Table, values are instances. | | SBT Transformer Model | | [Evaluation](evaluation.md) | Evaluation | Output the model evaluation metrics for user. | Table(s), values are instances. | | | | | [Hetero Pearson](correlation.md) | HeteroPearson | Calculate hetero correlation of features from different parties. | Table, values are instances. | | | | | [Hetero-NN](hetero_nn.md) | HeteroNN | Build hetero neural network model. | Table, values are instances. | Table, values are instances. | | Hetero Neural Network Model, consists of model-meta and model-param. | diff --git a/doc/federatedml_component/README.zh.md b/doc/federatedml_component/README.zh.md index 7d5a027f95..93ed1ca500 100644 --- a/doc/federatedml_component/README.zh.md +++ b/doc/federatedml_component/README.zh.md @@ -32,7 +32,6 @@ Federatedml模块包括许多常见机器学习算法联邦化实现。所有模 | [Homo-NN](homo_nn.md) | HomoNN | 通过多方构建横向神经网络模块。 | Table, 值为Instance | | | 神经网络模型，由模型本身和模型参数组成 | | [Hetero Secure Boosting](ensemble.md) | HeteroSecureBoost | 通过多方构建纵向Secure Boost模块。 | Table，值为Instance | | | SecureBoost模型，由模型本身和模型参数组成 | | [Hetero Fast Secure Boosting](ensemble.md) | HeteroFastSecureBoost | 使用分层/混合模式快速构建树模型 | Table，值为Instance | Table，值为Instance | | FastSecureBoost模型 | -| [Hetero Secure Boost Feature Transformer](sbt_feature_transformer.md) | SBTFeatureTransformer | 利用SBT叶子为特征编码 | Table，值为Instance | Table，值为Instance | | SBT Transformer模型 | | [Evaluation](evaluation.md) | Evaluation | 为用户输出模型评估指标。 | Table(s), 值为Instance | | | | | [Hetero Pearson](correlation.md) | HeteroPearson | 计算来自不同方的特征的Pearson相关系数。 | Table, 值为Instance | | | | | [Hetero-NN](hetero_nn.md) | HeteroNN | 构建纵向神经网络模块。 | Table, 值为Instance | | | 纵向神经网络模型 | diff --git a/doc/federatedml_component/sbt_feature_transformer.md b/doc/federatedml_component/sbt_feature_transformer.md deleted file mode 100644 index 79a27b14cf..0000000000 --- a/doc/federatedml_component/sbt_feature_transformer.md +++ /dev/null @@ -1,22 +0,0 @@ -# SBT Feature Transformer - -A feature engineering module that encodes sample using leaf indices -predicted by Hetero SBT/Fast-SBT. Samples will be transformed into -sparse 0-1 vectors after encoding. See [original -paper](https://research.fb.com/wp-content/uploads/2016/11/practical-lessons-from-predicting-clicks-on-ads-at-facebook.pdf) -for its details. - -![Figure 5: Encoding using leaf -indices\](../images/gbdt_lr.png) - - From 3b70d158898e854e586969b8b53fc7adc82a8781 Mon Sep 17 00:00:00 2001 From: dylan-fan <289765648@qq.com> Date: Thu, 24 Feb 2022 15:23:02 +0800 Subject: [PATCH 80/99] fixed benchmarck_performance training dsl Signed off by: dylan-fan <289765648@qq.com> --- .../hetero_lr/test_hetero_lr_train_job_dsl.json | 4 ++-- .../hetero_sbt/test_hetero_secureboost_train_job_dsl.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/benchmark_performance/hetero_lr/test_hetero_lr_train_job_dsl.json b/examples/benchmark_performance/hetero_lr/test_hetero_lr_train_job_dsl.json index d63fad8777..5171494bdc 100644 --- a/examples/benchmark_performance/hetero_lr/test_hetero_lr_train_job_dsl.json +++ b/examples/benchmark_performance/hetero_lr/test_hetero_lr_train_job_dsl.json @@ -169,7 +169,7 @@ "input": { "data": { "data": [ - "hetero_feature_selection_0.data" + "intersection_0.data" ] } }, @@ -187,7 +187,7 @@ "input": { "data": { "data": [ - "hetero_feature_selection_1.data" + "intersection_1.data" ] }, "model": [ diff --git a/examples/benchmark_performance/hetero_sbt/test_hetero_secureboost_train_job_dsl.json b/examples/benchmark_performance/hetero_sbt/test_hetero_secureboost_train_job_dsl.json index fa4a0c4934..e09d1a31bf 100644 --- a/examples/benchmark_performance/hetero_sbt/test_hetero_secureboost_train_job_dsl.json +++ b/examples/benchmark_performance/hetero_sbt/test_hetero_secureboost_train_job_dsl.json @@ -169,10 +169,10 @@ "input": { "data": { "train_data": [ - "hetero_feature_selection_0.data" + "intersection_0.data" ], "validate_data": [ - "hetero_feature_selection_1.data" + "intersection_1.data" ] } }, From 2389395ca5e127e7def619f289bf70f30e7f329c Mon Sep 17 00:00:00 2001 From: dylan-fan <289765648@qq.com> Date: Thu, 24 Feb 2022 15:29:41 +0800 Subject: [PATCH 81/99] fixed benchmarck_performance training dsl Signed-off-by: dylan-fan <289765648@qq.com> --- .../hetero_lr/test_hetero_lr_train_job_dsl.json | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/benchmark_performance/hetero_lr/test_hetero_lr_train_job_dsl.json b/examples/benchmark_performance/hetero_lr/test_hetero_lr_train_job_dsl.json index 5171494bdc..11440734dc 100644 --- a/examples/benchmark_performance/hetero_lr/test_hetero_lr_train_job_dsl.json +++ b/examples/benchmark_performance/hetero_lr/test_hetero_lr_train_job_dsl.json @@ -182,6 +182,7 @@ ] } }, + "hetero_scale_1": { "module": "FeatureScale", "input": { From be45beed38b0c8e5cd98a5a33b5d4c72de6543fb Mon Sep 17 00:00:00 2001 From: mgqa34 Date: Thu, 24 Feb 2022 15:33:45 +0800 Subject: [PATCH 82/99] update release, modify fate.env Signed-off-by: mgqa34 --- RELEASE.md | 13 +++++++++++++ doc/federatedml_component/logistic_regression.md | 15 ++++++++------- fate.env | 2 +- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index 8e1699f896..19691e8c7d 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,3 +1,16 @@ +## Release 1.7.2 +### Major Features and Improvements +> FederatedML +* New batch strategy in coordinated Hetero LR: support masked batch data and batch shuffle +* Model inference protection enhancement for Hetero SecureBoost with FED-EINI algorithm +* Hetero SecureBoost supports split feature importance on host side, disables gain feature importance +* Offline SBT Feature transform component + +> Bug-Fix +* Fixed Bug for HeteroPearson with changing default q_field value for spdz +* Fix Data Transform's schema label name setting problem when `with_label` is False + + ## Release 1.7.1.1 ### Major Features and Improvements > Deploy diff --git a/doc/federatedml_component/logistic_regression.md b/doc/federatedml_component/logistic_regression.md index 5c1157e104..899da6331d 100644 --- a/doc/federatedml_component/logistic_regression.md +++ b/doc/federatedml_component/logistic_regression.md @@ -157,14 +157,15 @@ which HE and Secret-Sharing hybrid protocol is included. - Hetero-LR extra features -> 1. Support different encrypt-mode to balance speed and security -> 2. Support OneVeRest -> 3. When modeling a multi-host task, "weight\_diff" converge criteria +1. Support different encrypt-mode to balance speed and security +2. Support OneVeRest +3. When modeling a multi-host task, "weight\_diff" converge criteria > is supported only. -> 4. Support sparse format data -> 5. Support early-stopping mechanism -> 6. Support setting arbitrary metrics for validation during training -> 7. Support stepwise. For details on stepwise mode, please refer [stepwise](stepwise.md). +4. Support sparse format data +5. Support early-stopping mechanism +6. Support setting arbitrary metrics for validation during training +7. Support stepwise. For details on stepwise mode, please refer [stepwise](stepwise.md). +8. Support batch shuffle and batch masked strategy. - Hetero-SSHE-LR extra features > 1. Support different encrypt-mode to balance speed and security diff --git a/fate.env b/fate.env index 8c2a3f9a17..af82091625 100755 --- a/fate.env +++ b/fate.env @@ -1,6 +1,6 @@ FATE=1.7.2 FATEFlow=1.7.2 -FATEBoard=1.7.2.1 +FATEBoard=1.7.2.2 EGGROLL=2.4.3 CENTOS=7.2 UBUNTU=16.04 From b48e0c0893ab7add95b415db2489af29e817e9c2 Mon Sep 17 00:00:00 2001 From: cwj Date: Thu, 24 Feb 2022 17:41:05 +0800 Subject: [PATCH 83/99] update conf & pipeline Signed-off-by: cwj --- .../hetero_secureboost_testsuite.json | 5 ++ .../test_EINI_predict_conf.json | 47 ++++++++++ ...ecureboost_EINI_with_ramdom_mask_conf.json | 86 ------------------- ...peline-hetero-sbt-EINI-with-random-mask.py | 35 +++++--- 4 files changed, 77 insertions(+), 96 deletions(-) create mode 100644 examples/dsl/v2/hetero_secureboost/test_EINI_predict_conf.json delete mode 100644 examples/dsl/v2/hetero_secureboost/test_secureboost_EINI_with_ramdom_mask_conf.json diff --git a/examples/dsl/v2/hetero_secureboost/hetero_secureboost_testsuite.json b/examples/dsl/v2/hetero_secureboost/hetero_secureboost_testsuite.json index 708832dbab..95833a6444 100644 --- a/examples/dsl/v2/hetero_secureboost/hetero_secureboost_testsuite.json +++ b/examples/dsl/v2/hetero_secureboost/hetero_secureboost_testsuite.json @@ -111,6 +111,11 @@ "dsl": "./test_predict_dsl.json", "deps": "train_binary" }, + "train_binary_EINI_predict": { + "conf": "./test_EINI_predict_conf.json", + "dsl": "./test_predict_dsl.json", + "deps": "train_binary" + }, "train_multi": { "conf": "./test_secureboost_train_multi_conf.json", "dsl": "./test_secureboost_train_dsl.json" diff --git a/examples/dsl/v2/hetero_secureboost/test_EINI_predict_conf.json b/examples/dsl/v2/hetero_secureboost/test_EINI_predict_conf.json new file mode 100644 index 0000000000..1760b10717 --- /dev/null +++ b/examples/dsl/v2/hetero_secureboost/test_EINI_predict_conf.json @@ -0,0 +1,47 @@ +{ + "dsl_version": 2, + "initiator": { + "role": "guest", + "party_id": 9999 + }, + "role": { + "host": [ + 10000 + ], + "guest": [ + 9999 + ] + }, + "job_parameters": { + "common": { + "model_id": "guest-10000#host-9999#model", + "model_version": "20200928174750711017114", + "job_type": "predict" + } + }, + "component_parameters": { + "common": {"hetero_secure_boost_0":{"EINI_inference": true, "EINI_random_mask": true}}, + "role": { + "guest": { + "0": { + "reader_0": { + "table": { + "name": "breast_hetero_guest", + "namespace": "experiment" + } + } + } + }, + "host": { + "0": { + "reader_0": { + "table": { + "name": "breast_hetero_host", + "namespace": "experiment" + } + } + } + } + } + } +} \ No newline at end of file diff --git a/examples/dsl/v2/hetero_secureboost/test_secureboost_EINI_with_ramdom_mask_conf.json b/examples/dsl/v2/hetero_secureboost/test_secureboost_EINI_with_ramdom_mask_conf.json deleted file mode 100644 index a74c6a3197..0000000000 --- a/examples/dsl/v2/hetero_secureboost/test_secureboost_EINI_with_ramdom_mask_conf.json +++ /dev/null @@ -1,86 +0,0 @@ -{ - "dsl_version": 2, - "initiator": { - "role": "guest", - "party_id": 9999 - }, - "role": { - "host": [ - 9998 - ], - "guest": [ - 9999 - ] - }, - "component_parameters": { - "common": { - "hetero_secure_boost_0": { - "task_type": "classification", - "objective_param": { - "objective": "cross_entropy" - }, - "num_trees": 3, - "validation_freqs": 1, - "encrypt_param": { - "method": "Paillier" - }, - "tree_param": { - "max_depth": 3 - }, - "EINI_inference": true, - "EINI_random_mask": true - }, - "evaluation_0": { - "eval_type": "binary" - } - }, - "role": { - "guest": { - "0": { - "reader_1": { - "table": { - "name": "breast_hetero_guest", - "namespace": "experiment" - } - }, - "reader_0": { - "table": { - "name": "breast_hetero_guest", - "namespace": "experiment" - } - }, - "data_transform_0": { - "with_label": true, - "output_format": "dense" - }, - "data_transform_1": { - "with_label": true, - "output_format": "dense" - } - } - }, - "host": { - "0": { - "reader_1": { - "table": { - "name": "breast_hetero_host", - "namespace": "experiment" - } - }, - "reader_0": { - "table": { - "name": "breast_hetero_host", - "namespace": "experiment" - } - }, - "data_transform_0": { - "with_label": false - }, - "data_transform_1": { - "with_label": false - } - } - } - } - } -} \ No newline at end of file diff --git a/examples/pipeline/hetero_sbt/pipeline-hetero-sbt-EINI-with-random-mask.py b/examples/pipeline/hetero_sbt/pipeline-hetero-sbt-EINI-with-random-mask.py index 2b89b59c1a..600e8510eb 100644 --- a/examples/pipeline/hetero_sbt/pipeline-hetero-sbt-EINI-with-random-mask.py +++ b/examples/pipeline/hetero_sbt/pipeline-hetero-sbt-EINI-with-random-mask.py @@ -57,13 +57,9 @@ def main(config="../../config.yaml", namespace=""): data_transform_0, data_transform_1 = DataTransform(name="data_transform_0"), DataTransform(name="data_transform_1") - data_transform_0.get_party_instance( - role="guest", party_id=guest).component_param( - with_label=True, output_format="dense") + data_transform_0.get_party_instance(role="guest", party_id=guest).component_param(with_label=True, output_format="dense") data_transform_0.get_party_instance(role="host", party_id=host).component_param(with_label=False) - data_transform_1.get_party_instance( - role="guest", party_id=guest).component_param( - with_label=True, output_format="dense") + data_transform_1.get_party_instance(role="guest", party_id=guest).component_param(with_label=True, output_format="dense") data_transform_1.get_party_instance(role="host", party_id=host).component_param(with_label=False) # data intersect component @@ -78,6 +74,7 @@ def main(config="../../config.yaml", namespace=""): encrypt_param={"method": "Paillier"}, tree_param={"max_depth": 3}, validation_freqs=1, + EINI_inference=True, EINI_random_mask=True ) @@ -87,10 +84,7 @@ def main(config="../../config.yaml", namespace=""): pipeline.add_component(reader_0) pipeline.add_component(reader_1) pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data)) - pipeline.add_component( - data_transform_1, data=Data( - data=reader_1.output.data), model=Model( - data_transform_0.output.model)) + pipeline.add_component(data_transform_1, data=Data(data=reader_1.output.data), model=Model(data_transform_0.output.model)) pipeline.add_component(intersect_0, data=Data(data=data_transform_0.output.data)) pipeline.add_component(intersect_1, data=Data(data=data_transform_1.output.data)) pipeline.add_component(hetero_secure_boost_0, data=Data(train_data=intersect_0.output.data, @@ -103,6 +97,27 @@ def main(config="../../config.yaml", namespace=""): print("fitting hetero secureboost done, result:") print(pipeline.get_component("hetero_secure_boost_0").get_summary()) + print('start to predict') + + # predict + # deploy required components + pipeline.deploy_component([data_transform_0, intersect_0, hetero_secure_boost_0, evaluation_0]) + + predict_pipeline = PipeLine() + # add data reader onto predict pipeline + predict_pipeline.add_component(reader_0) + # add selected components from train pipeline onto predict pipeline + # specify data source + predict_pipeline.add_component(pipeline, + data=Data(predict_input={pipeline.data_transform_0.input.data: reader_0.output.data})) + + # run predict model + predict_pipeline.predict() + predict_result = predict_pipeline.get_component("hetero_secure_boost_0").get_output_data() + print("Showing 10 data of predict result") + for ret in predict_result["data"][:10]: + print (ret) + if __name__ == "__main__": parser = argparse.ArgumentParser("PIPELINE DEMO") From 9e30fde7824284866ee3dee3edb13e51fe2ea9a6 Mon Sep 17 00:00:00 2001 From: weijingchen Date: Thu, 24 Feb 2022 17:44:57 +0800 Subject: [PATCH 84/99] update conf & pipeline Signed-off-by: cwj --- examples/pipeline/hetero_sbt/hetero_secureboost_testsuite.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/pipeline/hetero_sbt/hetero_secureboost_testsuite.json b/examples/pipeline/hetero_sbt/hetero_secureboost_testsuite.json index a1b0ff479c..e881c1c15e 100644 --- a/examples/pipeline/hetero_sbt/hetero_secureboost_testsuite.json +++ b/examples/pipeline/hetero_sbt/hetero_secureboost_testsuite.json @@ -102,6 +102,9 @@ "train_binary_predict": { "script": "./pipeline-hetero-sbt-binary-with-predict.py" }, + "train_binary_EINI_predict": { + "script": "./pipeline-hetero-sbt-EINI-with-random-mask.py" + }, "train_multi": { "script": "./pipeline-hetero-sbt-multi.py" }, From a36157111eaa97de9bc8bd0d163747b2bd02336d Mon Sep 17 00:00:00 2001 From: mgqa34 Date: Fri, 25 Feb 2022 02:57:08 +0800 Subject: [PATCH 85/99] update examples: remove deprecated param, fix linear regression using optimizer sqn error, add sqn examples of linr Signed-off-by: mgqa34 --- .../logistic_regression.md | 3 - .../hetero_linr_testsuite.json | 8 + .../test_hetero_linr_cv_job_conf.json | 3 - ...st_hetero_linr_multi_host_cv_job_conf.json | 3 - ...tero_linr_multi_host_predict_job_conf.json | 3 - ...hetero_linr_multi_host_train_job_conf.json | 3 - .../test_hetero_linr_train_job_conf.json | 3 - ...ero_linr_train_sample_weight_job_conf.json | 3 - ...est_hetero_linr_train_sparse_job_conf.json | 3 - ...etero_linr_train_sparse_sqn_job_conf.json} | 83 ++++--- ...hetero_linr_train_sparse_sqn_job_dsl.json} | 6 +- .../test_hetero_linr_train_sqn_job_conf.json} | 84 ++++--- .../test_hetero_linr_train_sqn_job_dsl.json} | 19 +- .../test_hetero_linr_validate_job_conf.json | 3 - .../test_hetero_linr_warm_start_job_conf.json | 3 - .../hetero_logistic_regression_testsuite.json | 12 - .../hetero_lr_ovr_sqn_conf.json | 83 ------- .../hetero_lr_sparse_sqn_dsl.json | 78 ------- .../test_hetero_poisson_cv_job_conf.json | 3 - .../test_hetero_poisson_train_job_conf.json | 3 - ..._hetero_poisson_train_sparse_job_conf.json | 3 - ...test_hetero_poisson_validate_job_conf.json | 3 - ...st_hetero_poisson_warm_start_job_conf.json | 3 - .../test_hetero_stepwise_linr_conf.json | 3 - .../test_hetero_stepwise_lr_conf.json | 3 - .../test_hetero_stepwise_poisson_conf.json | 3 - .../hetero_linr_testsuite.json | 6 + .../pipeline-hetero-linr-cv.py | 1 - .../pipeline-hetero-linr-multi-host-cv.py | 1 - .../pipeline-hetero-linr-multi-host.py | 3 +- .../pipeline-hetero-linr-sample-weight.py | 1 - .../pipeline-hetero-linr-sparse-sqn.py | 90 ++++++++ .../pipeline-hetero-linr-sparse.py | 3 +- .../pipeline-hetero-linr-sqn.py | 103 +++++++++ .../pipeline-hetero-linr-validate.py | 1 - .../pipeline-hetero-linr-warm-start.py | 2 - .../pipeline-hetero-linr.py | 1 - ...ogistic_regression_pipeline_testsuite.json | 9 - .../pipeline-hetero-poisson-cv.py | 1 - .../pipeline-hetero-poisson-sparse.py | 1 - .../pipeline-hetero-poisson-validate.py | 3 +- .../pipeline-hetero-poisson-warm-start.py | 6 +- .../hetero_poisson/pipeline-hetero-poisson.py | 3 +- .../hetero_stepwise/pipeline-stepwise-linr.py | 1 - .../hetero_stepwise/pipeline-stepwise-lr.py | 1 - .../pipeline-stepwise-poisson.py | 1 - .../param/logistic_regression_param.py | 217 +----------------- 47 files changed, 301 insertions(+), 581 deletions(-) rename examples/dsl/v2/{hetero_logistic_regression/hetero_lr_sparse_sqn_conf.json => hetero_linear_regression/test_hetero_linr_train_sparse_sqn_job_conf.json} (67%) rename examples/dsl/v2/{hetero_logistic_regression/hetero_lr_ovr_sqn_dsl.json => hetero_linear_regression/test_hetero_linr_train_sparse_sqn_job_dsl.json} (94%) rename examples/dsl/v2/{hetero_logistic_regression/hetero_lr_sqn_conf.json => hetero_linear_regression/test_hetero_linr_train_sqn_job_conf.json} (65%) rename examples/dsl/v2/{hetero_logistic_regression/hetero_lr_sqn_dsl.json => hetero_linear_regression/test_hetero_linr_train_sqn_job_dsl.json} (76%) delete mode 100644 examples/dsl/v2/hetero_logistic_regression/hetero_lr_ovr_sqn_conf.json delete mode 100644 examples/dsl/v2/hetero_logistic_regression/hetero_lr_sparse_sqn_dsl.json create mode 100644 examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-sparse-sqn.py create mode 100644 examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-sqn.py diff --git a/doc/federatedml_component/logistic_regression.md b/doc/federatedml_component/logistic_regression.md index 899da6331d..8207905775 100644 --- a/doc/federatedml_component/logistic_regression.md +++ b/doc/federatedml_component/logistic_regression.md @@ -115,9 +115,6 @@ which HE and Secret-Sharing hybrid protocol is included. > > - nesterov\_momentum\_sgd > > Nesterov Momentum > > -> > - sqn -> > stochastic quansi-newton. More details is available in this -> > [A Quasi-Newton Method Based Vertical Federated Learning Framework for Logistic Regression](https://arxiv.org/abs/1912.00513v2). > > 5. Three converge criteria: > diff --git a/examples/dsl/v2/hetero_linear_regression/hetero_linr_testsuite.json b/examples/dsl/v2/hetero_linear_regression/hetero_linr_testsuite.json index c35be84a36..c79ca01306 100644 --- a/examples/dsl/v2/hetero_linear_regression/hetero_linr_testsuite.json +++ b/examples/dsl/v2/hetero_linear_regression/hetero_linr_testsuite.json @@ -35,6 +35,10 @@ "conf": "./test_hetero_linr_predict_job_conf.json", "dsl": "./test_hetero_linr_predict_job_dsl.json" }, + "linr-train-sqn": { + "conf": "./test_hetero_linr_train_sqn_job_conf.json", + "dsl": "./test_hetero_linr_train_sqn_job_dsl.json" + }, "linr-warm-start": { "conf": "./test_hetero_linr_warm_start_job_conf.json", "dsl": "./test_hetero_linr_warm_start_job_dsl.json" @@ -64,6 +68,10 @@ "conf": "./test_hetero_linr_train_sparse_job_conf.json", "dsl": "./test_hetero_linr_train_sparse_job_dsl.json" }, + "linr_sparse_sqn": { + "conf": "./test_hetero_linr_train_sparse_sqn_job_conf.json", + "dsl": "./test_hetero_linr_train_sparse_sqn_job_dsl.json" + }, "linr_sample_weight": { "conf": "./test_hetero_linr_train_sample_weight_job_conf.json", "dsl": "./test_hetero_linr_train_sample_weight_job_dsl.json" diff --git a/examples/dsl/v2/hetero_linear_regression/test_hetero_linr_cv_job_conf.json b/examples/dsl/v2/hetero_linear_regression/test_hetero_linr_cv_job_conf.json index c90c5c5aac..b5ab8e6cc4 100644 --- a/examples/dsl/v2/hetero_linear_regression/test_hetero_linr_cv_job_conf.json +++ b/examples/dsl/v2/hetero_linear_regression/test_hetero_linr_cv_job_conf.json @@ -29,9 +29,6 @@ }, "max_iter": 20, "early_stop": "weight_diff", - "encrypted_mode_calculator_param": { - "mode": "fast" - }, "cv_param": { "n_splits": 5, "shuffle": false, diff --git a/examples/dsl/v2/hetero_linear_regression/test_hetero_linr_multi_host_cv_job_conf.json b/examples/dsl/v2/hetero_linear_regression/test_hetero_linr_multi_host_cv_job_conf.json index a84b530a12..ba019daabb 100644 --- a/examples/dsl/v2/hetero_linear_regression/test_hetero_linr_multi_host_cv_job_conf.json +++ b/examples/dsl/v2/hetero_linear_regression/test_hetero_linr_multi_host_cv_job_conf.json @@ -30,9 +30,6 @@ }, "max_iter": 20, "early_stop": "weight_diff", - "encrypted_mode_calculator_param": { - "mode": "fast" - }, "cv_param": { "n_splits": 5, "shuffle": false, diff --git a/examples/dsl/v2/hetero_linear_regression/test_hetero_linr_multi_host_predict_job_conf.json b/examples/dsl/v2/hetero_linear_regression/test_hetero_linr_multi_host_predict_job_conf.json index d35ce8272d..c5fb5bbc25 100644 --- a/examples/dsl/v2/hetero_linear_regression/test_hetero_linr_multi_host_predict_job_conf.json +++ b/examples/dsl/v2/hetero_linear_regression/test_hetero_linr_multi_host_predict_job_conf.json @@ -37,9 +37,6 @@ }, "max_iter": 20, "early_stop": "weight_diff", - "encrypted_mode_calculator_param": { - "mode": "fast" - }, "decay": 0.0, "decay_sqrt": false }, diff --git a/examples/dsl/v2/hetero_linear_regression/test_hetero_linr_multi_host_train_job_conf.json b/examples/dsl/v2/hetero_linear_regression/test_hetero_linr_multi_host_train_job_conf.json index 64e5094f71..8c3127d9b4 100644 --- a/examples/dsl/v2/hetero_linear_regression/test_hetero_linr_multi_host_train_job_conf.json +++ b/examples/dsl/v2/hetero_linear_regression/test_hetero_linr_multi_host_train_job_conf.json @@ -30,9 +30,6 @@ }, "max_iter": 20, "early_stop": "weight_diff", - "encrypted_mode_calculator_param": { - "mode": "fast" - }, "decay": 0.0, "decay_sqrt": false }, diff --git a/examples/dsl/v2/hetero_linear_regression/test_hetero_linr_train_job_conf.json b/examples/dsl/v2/hetero_linear_regression/test_hetero_linr_train_job_conf.json index 4f5f28c12e..18153126e7 100644 --- a/examples/dsl/v2/hetero_linear_regression/test_hetero_linr_train_job_conf.json +++ b/examples/dsl/v2/hetero_linear_regression/test_hetero_linr_train_job_conf.json @@ -29,9 +29,6 @@ }, "max_iter": 20, "early_stop": "weight_diff", - "encrypted_mode_calculator_param": { - "mode": "fast" - }, "decay": 0.0, "decay_sqrt": false, "floating_point_precision": 23 diff --git a/examples/dsl/v2/hetero_linear_regression/test_hetero_linr_train_sample_weight_job_conf.json b/examples/dsl/v2/hetero_linear_regression/test_hetero_linr_train_sample_weight_job_conf.json index 8c045d9d5d..a5610c7429 100644 --- a/examples/dsl/v2/hetero_linear_regression/test_hetero_linr_train_sample_weight_job_conf.json +++ b/examples/dsl/v2/hetero_linear_regression/test_hetero_linr_train_sample_weight_job_conf.json @@ -29,9 +29,6 @@ }, "max_iter": 20, "early_stop": "weight_diff", - "encrypted_mode_calculator_param": { - "mode": "fast" - }, "decay": 0.0, "decay_sqrt": false, "floating_point_precision": 23 diff --git a/examples/dsl/v2/hetero_linear_regression/test_hetero_linr_train_sparse_job_conf.json b/examples/dsl/v2/hetero_linear_regression/test_hetero_linr_train_sparse_job_conf.json index a6c57d756f..ec6b65e5f9 100644 --- a/examples/dsl/v2/hetero_linear_regression/test_hetero_linr_train_sparse_job_conf.json +++ b/examples/dsl/v2/hetero_linear_regression/test_hetero_linr_train_sparse_job_conf.json @@ -34,9 +34,6 @@ }, "max_iter": 2, "early_stop": "weight_diff", - "encrypted_mode_calculator_param": { - "mode": "fast" - }, "decay": 0.0, "decay_sqrt": false }, diff --git a/examples/dsl/v2/hetero_logistic_regression/hetero_lr_sparse_sqn_conf.json b/examples/dsl/v2/hetero_linear_regression/test_hetero_linr_train_sparse_sqn_job_conf.json similarity index 67% rename from examples/dsl/v2/hetero_logistic_regression/hetero_lr_sparse_sqn_conf.json rename to examples/dsl/v2/hetero_linear_regression/test_hetero_linr_train_sparse_sqn_job_conf.json index 851ad54098..f05e1f7186 100644 --- a/examples/dsl/v2/hetero_logistic_regression/hetero_lr_sparse_sqn_conf.json +++ b/examples/dsl/v2/hetero_linear_regression/test_hetero_linr_train_sparse_sqn_job_conf.json @@ -6,7 +6,7 @@ }, "role": { "arbiter": [ - 9999 + 10000 ], "host": [ 10000 @@ -16,68 +16,67 @@ ] }, "component_parameters": { + "common": { + "data_transform_0": { + "missing_fill": true, + "outlier_replace": false, + "output_format": "sparse" + }, + "hetero_linr_0": { + "penalty": "L2", + "tol": 0.001, + "alpha": 0.01, + "optimizer": "sgd", + "batch_size": 100, + "learning_rate": 0.15, + "init_param": { + "init_method": "zeros" + }, + "sqn_param": { + "update_interval_L": 3, + "memory_M": 5, + "sample_size": 5000, + "random_seed": null + }, + "max_iter": 2, + "early_stop": "weight_diff", + "decay": 0.0, + "decay_sqrt": false + }, + "evaluation_0": { + "eval_type": "regression", + "pos_label": 1 + } + }, "role": { - "host": { + "guest": { "0": { "data_transform_0": { - "with_label": false + "with_label": true, + "label_name": "motor_speed", + "label_type": "float" }, "reader_0": { "table": { - "name": "breast_hetero_host", + "name": "motor_hetero_guest", "namespace": "experiment" } } } }, - "guest": { + "host": { "0": { "data_transform_0": { - "with_label": true + "with_label": false }, "reader_0": { "table": { - "name": "breast_hetero_guest", + "name": "motor_hetero_host", "namespace": "experiment" } } } } - }, - "common": { - "data_transform_0": { - "output_format": "sparse" - }, - "hetero_lr_0": { - "penalty": "L2", - "tol": 0.0001, - "alpha": 1e-05, - "optimizer": "sqn", - "batch_size": 5000, - "learning_rate": 0.15, - "init_param": { - "init_method": "zeros" - }, - "max_iter": 30, - "early_stop": "diff", - "cv_param": { - "n_splits": 5, - "shuffle": false, - "random_seed": 103, - "need_cv": false - }, - "decay": 0.3, - "decay_sqrt": true, - "sqn_param": { - "update_interval_L": 3, - "memory_M": 5, - "sample_size": 5000, - "random_seed": null - } - }, - "evaluation_0": { - "eval_type": "binary" - } } } } \ No newline at end of file diff --git a/examples/dsl/v2/hetero_logistic_regression/hetero_lr_ovr_sqn_dsl.json b/examples/dsl/v2/hetero_linear_regression/test_hetero_linr_train_sparse_sqn_job_dsl.json similarity index 94% rename from examples/dsl/v2/hetero_logistic_regression/hetero_lr_ovr_sqn_dsl.json rename to examples/dsl/v2/hetero_linear_regression/test_hetero_linr_train_sparse_sqn_job_dsl.json index bdd74188d9..618e38e92e 100644 --- a/examples/dsl/v2/hetero_logistic_regression/hetero_lr_ovr_sqn_dsl.json +++ b/examples/dsl/v2/hetero_linear_regression/test_hetero_linr_train_sparse_sqn_job_dsl.json @@ -41,8 +41,8 @@ ] } }, - "hetero_lr_0": { - "module": "HeteroLR", + "hetero_linr_0": { + "module": "HeteroLinR", "input": { "data": { "train_data": [ @@ -64,7 +64,7 @@ "input": { "data": { "data": [ - "hetero_lr_0.data" + "hetero_linr_0.data" ] } }, diff --git a/examples/dsl/v2/hetero_logistic_regression/hetero_lr_sqn_conf.json b/examples/dsl/v2/hetero_linear_regression/test_hetero_linr_train_sqn_job_conf.json similarity index 65% rename from examples/dsl/v2/hetero_logistic_regression/hetero_lr_sqn_conf.json rename to examples/dsl/v2/hetero_linear_regression/test_hetero_linr_train_sqn_job_conf.json index 1fe33ba4d5..831a43d610 100644 --- a/examples/dsl/v2/hetero_logistic_regression/hetero_lr_sqn_conf.json +++ b/examples/dsl/v2/hetero_linear_regression/test_hetero_linr_train_sqn_job_conf.json @@ -6,7 +6,7 @@ }, "role": { "arbiter": [ - 9999 + 10000 ], "host": [ 10000 @@ -16,68 +16,64 @@ ] }, "component_parameters": { + "common": { + "hetero_linr_0": { + "penalty": "L2", + "tol": 0.001, + "alpha": 0.01, + "optimizer": "sqn", + "batch_size": -1, + "learning_rate": 0.15, + "init_param": { + "init_method": "zeros" + }, + "sqn_param": { + "update_interval_L": 3, + "memory_M": 5, + "sample_size": 5000, + "random_seed": null + }, + "max_iter": 20, + "early_stop": "weight_diff", + "decay": 0.0, + "decay_sqrt": false, + "floating_point_precision": 23 + }, + "evaluation_0": { + "eval_type": "regression", + "pos_label": 1 + } + }, "role": { "host": { "0": { - "data_transform_0": { - "with_label": false - }, "reader_0": { "table": { - "name": "breast_hetero_host", + "name": "motor_hetero_host", "namespace": "experiment" } + }, + "data_transform_0": { + "with_label": false } } }, "guest": { "0": { - "data_transform_0": { - "with_label": true - }, "reader_0": { "table": { - "name": "breast_hetero_guest", + "name": "motor_hetero_guest", "namespace": "experiment" } + }, + "data_transform_0": { + "with_label": true, + "label_name": "motor_speed", + "label_type": "float", + "output_format": "dense" } } } - }, - "common": { - "data_transform_0": { - "output_format": "dense" - }, - "hetero_lr_0": { - "penalty": "L2", - "tol": 0.0001, - "alpha": 1e-05, - "optimizer": "sqn", - "batch_size": 5000, - "learning_rate": 0.15, - "init_param": { - "init_method": "zeros" - }, - "max_iter": 10, - "early_stop": "diff", - "cv_param": { - "n_splits": 3, - "shuffle": false, - "random_seed": 103, - "need_cv": false - }, - "decay": 0.3, - "decay_sqrt": true, - "sqn_param": { - "update_interval_L": 3, - "memory_M": 5, - "sample_size": 5000, - "random_seed": null - } - }, - "evaluation_0": { - "eval_type": "binary" - } } } } \ No newline at end of file diff --git a/examples/dsl/v2/hetero_logistic_regression/hetero_lr_sqn_dsl.json b/examples/dsl/v2/hetero_linear_regression/test_hetero_linr_train_sqn_job_dsl.json similarity index 76% rename from examples/dsl/v2/hetero_logistic_regression/hetero_lr_sqn_dsl.json rename to examples/dsl/v2/hetero_linear_regression/test_hetero_linr_train_sqn_job_dsl.json index bdd74188d9..4b6a12b839 100644 --- a/examples/dsl/v2/hetero_logistic_regression/hetero_lr_sqn_dsl.json +++ b/examples/dsl/v2/hetero_linear_regression/test_hetero_linr_train_sqn_job_dsl.json @@ -41,8 +41,8 @@ ] } }, - "hetero_lr_0": { - "module": "HeteroLR", + "hetero_linr_0": { + "module": "HeteroLinR", "input": { "data": { "train_data": [ @@ -58,21 +58,6 @@ "model" ] } - }, - "evaluation_0": { - "module": "Evaluation", - "input": { - "data": { - "data": [ - "hetero_lr_0.data" - ] - } - }, - "output": { - "data": [ - "data" - ] - } } } } \ No newline at end of file diff --git a/examples/dsl/v2/hetero_linear_regression/test_hetero_linr_validate_job_conf.json b/examples/dsl/v2/hetero_linear_regression/test_hetero_linr_validate_job_conf.json index 2823037323..28c4359a3d 100644 --- a/examples/dsl/v2/hetero_linear_regression/test_hetero_linr_validate_job_conf.json +++ b/examples/dsl/v2/hetero_linear_regression/test_hetero_linr_validate_job_conf.json @@ -29,9 +29,6 @@ }, "max_iter": 20, "early_stop": "weight_diff", - "encrypted_mode_calculator_param": { - "mode": "fast" - }, "decay": 0.0, "decay_sqrt": false, "callback_param": { diff --git a/examples/dsl/v2/hetero_linear_regression/test_hetero_linr_warm_start_job_conf.json b/examples/dsl/v2/hetero_linear_regression/test_hetero_linr_warm_start_job_conf.json index 0c2aa995d2..ed210a5400 100644 --- a/examples/dsl/v2/hetero_linear_regression/test_hetero_linr_warm_start_job_conf.json +++ b/examples/dsl/v2/hetero_linear_regression/test_hetero_linr_warm_start_job_conf.json @@ -29,9 +29,6 @@ }, "max_iter": 5, "early_stop": "weight_diff", - "encrypted_mode_calculator_param": { - "mode": "fast" - }, "callback_param": { "callbacks": [ "ModelCheckpoint" diff --git a/examples/dsl/v2/hetero_logistic_regression/hetero_logistic_regression_testsuite.json b/examples/dsl/v2/hetero_logistic_regression/hetero_logistic_regression_testsuite.json index 5378ab86c7..06e5c3d72c 100644 --- a/examples/dsl/v2/hetero_logistic_regression/hetero_logistic_regression_testsuite.json +++ b/examples/dsl/v2/hetero_logistic_regression/hetero_logistic_regression_testsuite.json @@ -46,18 +46,10 @@ "conf": "hetero_lr_sample_weights_conf.json", "dsl": "hetero_lr_sample_weights_dsl.json" }, - "hetero_lr_sqn": { - "conf": "hetero_lr_sqn_conf.json", - "dsl": "hetero_lr_sqn_dsl.json" - }, "hetero_lr_sparse": { "conf": "hetero_lr_sparse_conf.json", "dsl": "hetero_lr_sparse_dsl.json" }, - "hetero_lr_ovr_sqn": { - "conf": "hetero_lr_ovr_sqn_conf.json", - "dsl": "hetero_lr_ovr_sqn_dsl.json" - }, "hetero_lr_feature_engineering": { "conf": "hetero_lr_feature_engineering_conf.json", "dsl": "hetero_lr_feature_engineering_dsl.json" @@ -98,10 +90,6 @@ "conf": "hetero_lr_early_stop_conf.json", "dsl": "hetero_lr_early_stop_dsl.json" }, - "hetero_lr_sparse_sqn": { - "conf": "hetero_lr_sparse_sqn_conf.json", - "dsl": "hetero_lr_sparse_sqn_dsl.json" - }, "hetero_lr_warm_start": { "conf": "hetero_lr_warm_start_conf.json", "dsl": "hetero_lr_warm_start_dsl.json" diff --git a/examples/dsl/v2/hetero_logistic_regression/hetero_lr_ovr_sqn_conf.json b/examples/dsl/v2/hetero_logistic_regression/hetero_lr_ovr_sqn_conf.json deleted file mode 100644 index 21c2b7a67c..0000000000 --- a/examples/dsl/v2/hetero_logistic_regression/hetero_lr_ovr_sqn_conf.json +++ /dev/null @@ -1,83 +0,0 @@ -{ - "dsl_version": 2, - "initiator": { - "role": "guest", - "party_id": 9999 - }, - "role": { - "arbiter": [ - 9999 - ], - "host": [ - 10000 - ], - "guest": [ - 9999 - ] - }, - "component_parameters": { - "role": { - "guest": { - "0": { - "reader_0": { - "table": { - "name": "vehicle_scale_hetero_guest", - "namespace": "experiment" - } - }, - "data_transform_0": { - "with_label": true - } - } - }, - "host": { - "0": { - "reader_0": { - "table": { - "name": "vehicle_scale_hetero_host", - "namespace": "experiment" - } - }, - "data_transform_0": { - "with_label": false - } - } - } - }, - "common": { - "data_transform_0": { - "output_format": "dense" - }, - "hetero_lr_0": { - "penalty": "L2", - "tol": 0.0001, - "alpha": 1e-05, - "optimizer": "sqn", - "batch_size": 5000, - "learning_rate": 0.15, - "init_param": { - "init_method": "zeros" - }, - "max_iter": 10, - "early_stop": "diff", - "cv_param": { - "n_splits": 3, - "shuffle": false, - "random_seed": 103, - "need_cv": false - }, - "decay": 0.3, - "decay_sqrt": true, - "sqn_param": { - "update_interval_L": 3, - "memory_M": 5, - "sample_size": 5000, - "random_seed": null - } - }, - "evaluation_0": { - "eval_type": "binary" - } - } - } -} \ No newline at end of file diff --git a/examples/dsl/v2/hetero_logistic_regression/hetero_lr_sparse_sqn_dsl.json b/examples/dsl/v2/hetero_logistic_regression/hetero_lr_sparse_sqn_dsl.json deleted file mode 100644 index bdd74188d9..0000000000 --- a/examples/dsl/v2/hetero_logistic_regression/hetero_lr_sparse_sqn_dsl.json +++ /dev/null @@ -1,78 +0,0 @@ -{ - "components": { - "reader_0": { - "module": "Reader", - "output": { - "data": [ - "data" - ] - } - }, - "data_transform_0": { - "module": "DataTransform", - "input": { - "data": { - "data": [ - "reader_0.data" - ] - } - }, - "output": { - "data": [ - "data" - ], - "model": [ - "model" - ] - } - }, - "intersection_0": { - "module": "Intersection", - "input": { - "data": { - "data": [ - "data_transform_0.data" - ] - } - }, - "output": { - "data": [ - "data" - ] - } - }, - "hetero_lr_0": { - "module": "HeteroLR", - "input": { - "data": { - "train_data": [ - "intersection_0.data" - ] - } - }, - "output": { - "data": [ - "data" - ], - "model": [ - "model" - ] - } - }, - "evaluation_0": { - "module": "Evaluation", - "input": { - "data": { - "data": [ - "hetero_lr_0.data" - ] - } - }, - "output": { - "data": [ - "data" - ] - } - } - } -} \ No newline at end of file diff --git a/examples/dsl/v2/hetero_poisson_regression/test_hetero_poisson_cv_job_conf.json b/examples/dsl/v2/hetero_poisson_regression/test_hetero_poisson_cv_job_conf.json index 0a9440e349..5445386b85 100644 --- a/examples/dsl/v2/hetero_poisson_regression/test_hetero_poisson_cv_job_conf.json +++ b/examples/dsl/v2/hetero_poisson_regression/test_hetero_poisson_cv_job_conf.json @@ -29,9 +29,6 @@ }, "max_iter": 10, "early_stop": "weight_diff", - "encrypted_mode_calculator_param": { - "mode": "fast" - }, "cv_param": { "n_splits": 5, "shuffle": false, diff --git a/examples/dsl/v2/hetero_poisson_regression/test_hetero_poisson_train_job_conf.json b/examples/dsl/v2/hetero_poisson_regression/test_hetero_poisson_train_job_conf.json index e03a17beb2..10b52537f5 100644 --- a/examples/dsl/v2/hetero_poisson_regression/test_hetero_poisson_train_job_conf.json +++ b/examples/dsl/v2/hetero_poisson_regression/test_hetero_poisson_train_job_conf.json @@ -29,9 +29,6 @@ }, "max_iter": 10, "early_stop": "weight_diff", - "encrypted_mode_calculator_param": { - "mode": "fast" - }, "decay_sqrt": false, "exposure_colname": "exposure" }, diff --git a/examples/dsl/v2/hetero_poisson_regression/test_hetero_poisson_train_sparse_job_conf.json b/examples/dsl/v2/hetero_poisson_regression/test_hetero_poisson_train_sparse_job_conf.json index bd2dd48549..4450d5ec82 100644 --- a/examples/dsl/v2/hetero_poisson_regression/test_hetero_poisson_train_sparse_job_conf.json +++ b/examples/dsl/v2/hetero_poisson_regression/test_hetero_poisson_train_sparse_job_conf.json @@ -32,9 +32,6 @@ }, "max_iter": 20, "early_stop": "weight_diff", - "encrypted_mode_calculator_param": { - "mode": "fast" - }, "decay_sqrt": false, "exposure_colname": "exposure" }, diff --git a/examples/dsl/v2/hetero_poisson_regression/test_hetero_poisson_validate_job_conf.json b/examples/dsl/v2/hetero_poisson_regression/test_hetero_poisson_validate_job_conf.json index 58ba876be8..26fd626ec3 100644 --- a/examples/dsl/v2/hetero_poisson_regression/test_hetero_poisson_validate_job_conf.json +++ b/examples/dsl/v2/hetero_poisson_regression/test_hetero_poisson_validate_job_conf.json @@ -29,9 +29,6 @@ }, "max_iter": 20, "early_stop": "weight_diff", - "encrypted_mode_calculator_param": { - "mode": "fast" - }, "decay_sqrt": false, "exposure_colname": "exposure", "callback_param": { diff --git a/examples/dsl/v2/hetero_poisson_regression/test_hetero_poisson_warm_start_job_conf.json b/examples/dsl/v2/hetero_poisson_regression/test_hetero_poisson_warm_start_job_conf.json index dfa25f1c70..89f055ffbf 100644 --- a/examples/dsl/v2/hetero_poisson_regression/test_hetero_poisson_warm_start_job_conf.json +++ b/examples/dsl/v2/hetero_poisson_regression/test_hetero_poisson_warm_start_job_conf.json @@ -29,9 +29,6 @@ }, "max_iter": 3, "early_stop": "weight_diff", - "encrypted_mode_calculator_param": { - "mode": "fast" - }, "decay_sqrt": false, "exposure_colname": "exposure", "callback_param": { diff --git a/examples/dsl/v2/hetero_stepwise/test_hetero_stepwise_linr_conf.json b/examples/dsl/v2/hetero_stepwise/test_hetero_stepwise_linr_conf.json index 2478866851..66992f00e6 100644 --- a/examples/dsl/v2/hetero_stepwise/test_hetero_stepwise_linr_conf.json +++ b/examples/dsl/v2/hetero_stepwise/test_hetero_stepwise_linr_conf.json @@ -31,9 +31,6 @@ }, "max_iter": 3, "early_stop": "diff", - "encrypted_mode_calculator_param": { - "mode": "fast" - }, "stepwise_param": { "score_name": "AIC", "direction": "backward", diff --git a/examples/dsl/v2/hetero_stepwise/test_hetero_stepwise_lr_conf.json b/examples/dsl/v2/hetero_stepwise/test_hetero_stepwise_lr_conf.json index d29c86f992..e61db27004 100644 --- a/examples/dsl/v2/hetero_stepwise/test_hetero_stepwise_lr_conf.json +++ b/examples/dsl/v2/hetero_stepwise/test_hetero_stepwise_lr_conf.json @@ -36,9 +36,6 @@ "need_stepwise": true, "max_step": 2, "nvmin": 2 - }, - "encrypted_mode_calculator_param": { - "mode": "fast" } } }, diff --git a/examples/dsl/v2/hetero_stepwise/test_hetero_stepwise_poisson_conf.json b/examples/dsl/v2/hetero_stepwise/test_hetero_stepwise_poisson_conf.json index fbf7c438fd..35f06ccc65 100644 --- a/examples/dsl/v2/hetero_stepwise/test_hetero_stepwise_poisson_conf.json +++ b/examples/dsl/v2/hetero_stepwise/test_hetero_stepwise_poisson_conf.json @@ -29,9 +29,6 @@ }, "max_iter": 5, "early_stop": "diff", - "encrypted_mode_calculator_param": { - "mode": "fast" - }, "decay": 0.0, "decay_sqrt": false, "stepwise_param": { diff --git a/examples/pipeline/hetero_linear_regression/hetero_linr_testsuite.json b/examples/pipeline/hetero_linear_regression/hetero_linr_testsuite.json index 00d575412a..49a1425fba 100644 --- a/examples/pipeline/hetero_linear_regression/hetero_linr_testsuite.json +++ b/examples/pipeline/hetero_linear_regression/hetero_linr_testsuite.json @@ -29,6 +29,9 @@ "linr-train": { "script": "./pipeline-hetero-linr.py" }, + "linr-train-sqn": { + "script": "./pipeline-hetero-linr-sqn.py" + }, "linr-warm-start": { "script": "./pipeline-hetero-linr-warm-start.py" }, @@ -47,6 +50,9 @@ "linr_sparse": { "script": "./pipeline-hetero-linr-sparse.py" }, + "linr_sparse-sqn": { + "script": "./pipeline-hetero-linr-sparse-sqn.py" + }, "linr_sample-weight": { "script": "./pipeline-hetero-linr-sample-weight.py" } diff --git a/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-cv.py b/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-cv.py index 7f1ec582f9..b0f3a7dca0 100644 --- a/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-cv.py +++ b/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-cv.py @@ -54,7 +54,6 @@ def main(config="../../config.yaml", namespace=""): alpha=0.01, max_iter=20, early_stop="weight_diff", batch_size=-1, learning_rate=0.15, decay=0.0, decay_sqrt=False, init_param={"init_method": "zeros"}, - encrypted_mode_calculator_param={"mode": "fast"}, cv_param={"n_splits": 5, "shuffle": False, "random_seed": 42, diff --git a/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-multi-host-cv.py b/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-multi-host-cv.py index ca8c7ff082..4f6b2d7131 100644 --- a/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-multi-host-cv.py +++ b/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-multi-host-cv.py @@ -57,7 +57,6 @@ def main(config="../../config.yaml", namespace=""): alpha=0.01, max_iter=20, early_stop="weight_diff", batch_size=-1, learning_rate=0.15, decay=0.0, decay_sqrt=False, init_param={"init_method": "zeros"}, - encrypted_mode_calculator_param={"mode": "fast"}, cv_param={"n_splits": 5, "shuffle": False, "random_seed": 42, diff --git a/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-multi-host.py b/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-multi-host.py index b1660fcd10..7160d7ae05 100644 --- a/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-multi-host.py +++ b/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-multi-host.py @@ -53,8 +53,7 @@ def main(config="../../config.yaml", namespace=""): hetero_linr_0 = HeteroLinR(name="hetero_linr_0", penalty="L2", optimizer="sgd", tol=0.001, alpha=0.01, max_iter=20, early_stop="weight_diff", batch_size=-1, learning_rate=0.15, decay=0.0, decay_sqrt=False, - init_param={"init_method": "zeros"}, - encrypted_mode_calculator_param={"mode": "fast"}) + init_param={"init_method": "zeros"}) evaluation_0 = Evaluation(name="evaluation_0", eval_type="regression", pos_label=1) # evaluation_0.get_party_instance(role='host', party_id=hosts[0]).component_param(need_run=False) diff --git a/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-sample-weight.py b/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-sample-weight.py index a3d68cb8c4..1bdd56472d 100644 --- a/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-sample-weight.py +++ b/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-sample-weight.py @@ -60,7 +60,6 @@ def main(config="../../config.yaml", namespace=""): alpha=0.01, max_iter=20, early_stop="weight_diff", batch_size=-1, learning_rate=0.15, decay=0.0, decay_sqrt=False, init_param={"init_method": "zeros"}, - encrypted_mode_calculator_param={"mode": "fast"}, floating_point_precision=23) evaluation_0 = Evaluation(name="evaluation_0", eval_type="regression", pos_label=1) diff --git a/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-sparse-sqn.py b/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-sparse-sqn.py new file mode 100644 index 0000000000..896fe28731 --- /dev/null +++ b/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-sparse-sqn.py @@ -0,0 +1,90 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import argparse + +from pipeline.backend.pipeline import PipeLine +from pipeline.component import DataTransform +from pipeline.component import Evaluation +from pipeline.component import HeteroLinR +from pipeline.component import Intersection +from pipeline.component import Reader +from pipeline.interface import Data + +from pipeline.utils.tools import load_job_config + + +def main(config="../../config.yaml", namespace=""): + # obtain config + if isinstance(config, str): + config = load_job_config(config) + parties = config.parties + guest = parties.guest[0] + host = parties.host[0] + arbiter = parties.arbiter[0] + + guest_train_data = {"name": "motor_hetero_guest", "namespace": f"experiment{namespace}"} + host_train_data = {"name": "motor_hetero_host", "namespace": f"experiment{namespace}"} + + pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=host, arbiter=arbiter) + + reader_0 = Reader(name="reader_0") + reader_0.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data) + reader_0.get_party_instance(role='host', party_id=host).component_param(table=host_train_data) + reader_0.get_party_instance(role='host', party_id=host).component_param(table=host_train_data) + + data_transform_0 = DataTransform(name="data_transform_0", output_format="sparse", + missing_fill=True, outlier_replace=False) + data_transform_0.get_party_instance(role='guest', party_id=guest).component_param(with_label=True, + label_name="motor_speed", + label_type="float") + data_transform_0.get_party_instance(role='host', party_id=host).component_param(with_label=False) + + intersection_0 = Intersection(name="intersection_0") + hetero_linr_0 = HeteroLinR(name="hetero_linr_0", penalty="L2", optimizer="sqn", tol=0.001, + alpha=0.01, max_iter=2, early_stop="weight_diff", batch_size=100, + learning_rate=0.15, decay=0.0, decay_sqrt=False, + init_param={"init_method": "zeros"}, + sqn_param={ + "update_interval_L": 3, + "memory_M": 5, + "sample_size": 5000, + "random_seed": None + }) + + evaluation_0 = Evaluation(name="evaluation_0", eval_type="regression", pos_label=1) + # evaluation_0.get_party_instance(role='host', party_id=host).component_param(need_run=False) + + pipeline.add_component(reader_0) + pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data)) + pipeline.add_component(intersection_0, data=Data(data=data_transform_0.output.data)) + pipeline.add_component(hetero_linr_0, data=Data(train_data=intersection_0.output.data)) + pipeline.add_component(evaluation_0, data=Data(data=hetero_linr_0.output.data)) + + pipeline.compile() + + pipeline.fit() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser("PIPELINE DEMO") + parser.add_argument("-config", type=str, + help="config file") + args = parser.parse_args() + if args.config is not None: + main(args.config) + else: + main() diff --git a/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-sparse.py b/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-sparse.py index 7dbb8efdfa..4d5f9db29b 100644 --- a/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-sparse.py +++ b/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-sparse.py @@ -56,8 +56,7 @@ def main(config="../../config.yaml", namespace=""): hetero_linr_0 = HeteroLinR(name="hetero_linr_0", penalty="L2", optimizer="sgd", tol=0.001, alpha=0.01, max_iter=2, early_stop="weight_diff", batch_size=100, learning_rate=0.15, decay=0.0, decay_sqrt=False, - init_param={"init_method": "zeros"}, - encrypted_mode_calculator_param={"mode": "fast"}) + init_param={"init_method": "zeros"}) evaluation_0 = Evaluation(name="evaluation_0", eval_type="regression", pos_label=1) # evaluation_0.get_party_instance(role='host', party_id=host).component_param(need_run=False) diff --git a/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-sqn.py b/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-sqn.py new file mode 100644 index 0000000000..b20854e1dd --- /dev/null +++ b/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-sqn.py @@ -0,0 +1,103 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import argparse + +from pipeline.backend.pipeline import PipeLine +from pipeline.component import DataTransform +from pipeline.component import Evaluation +from pipeline.component import HeteroLinR +from pipeline.component import Intersection +from pipeline.component import Reader +from pipeline.interface import Data + +from pipeline.utils.tools import load_job_config + + +def main(config="../../config.yaml", namespace=""): + # obtain config + if isinstance(config, str): + config = load_job_config(config) + parties = config.parties + guest = parties.guest[0] + host = parties.host[0] + arbiter = parties.arbiter[0] + + guest_train_data = {"name": "motor_hetero_guest", "namespace": f"experiment{namespace}"} + host_train_data = {"name": "motor_hetero_host", "namespace": f"experiment{namespace}"} + + pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=host, arbiter=arbiter) + + reader_0 = Reader(name="reader_0") + reader_0.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data) + reader_0.get_party_instance(role='host', party_id=host).component_param(table=host_train_data) + + data_transform_0 = DataTransform(name="data_transform_0") + data_transform_0.get_party_instance(role='guest', party_id=guest).component_param(with_label=True, label_name="motor_speed", + label_type="float", output_format="dense") + data_transform_0.get_party_instance(role='host', party_id=host).component_param(with_label=False) + + intersection_0 = Intersection(name="intersection_0") + hetero_linr_0 = HeteroLinR(name="hetero_linr_0", penalty="L2", optimizer="sqn", tol=0.001, + alpha=0.01, max_iter=20, early_stop="weight_diff", batch_size=-1, + learning_rate=0.15, decay=0.0, decay_sqrt=False, + init_param={"init_method": "zeros"}, + floating_point_precision=23, + sqn_param={ + "update_interval_L": 3, + "memory_M": 5, + "sample_size": 5000, + "random_seed": None + }) + + evaluation_0 = Evaluation(name="evaluation_0", eval_type="regression", pos_label=1) + # evaluation_0.get_party_instance(role='host', party_id=host).component_param(need_run=False) + + pipeline.add_component(reader_0) + pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data)) + pipeline.add_component(intersection_0, data=Data(data=data_transform_0.output.data)) + pipeline.add_component(hetero_linr_0, data=Data(train_data=intersection_0.output.data)) + pipeline.add_component(evaluation_0, data=Data(data=hetero_linr_0.output.data)) + + pipeline.compile() + + pipeline.fit() + + + # predict + # deploy required components + pipeline.deploy_component([data_transform_0, intersection_0, hetero_linr_0]) + + predict_pipeline = PipeLine() + # add data reader onto predict pipeline + predict_pipeline.add_component(reader_0) + # add selected components from train pipeline onto predict pipeline + # specify data source + predict_pipeline.add_component(pipeline, + data=Data(predict_input={pipeline.data_transform_0.input.data: reader_0.output.data})) + # run predict model + predict_pipeline.predict() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser("PIPELINE DEMO") + parser.add_argument("-config", type=str, + help="config file") + args = parser.parse_args() + if args.config is not None: + main(args.config) + else: + main() diff --git a/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-validate.py b/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-validate.py index 752a6a1d30..caea2499e1 100644 --- a/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-validate.py +++ b/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-validate.py @@ -70,7 +70,6 @@ def main(config="../../config.yaml", namespace=""): alpha=0.01, max_iter=20, early_stop="weight_diff", batch_size=-1, learning_rate=0.15, decay=0.0, decay_sqrt=False, init_param={"init_method": "zeros"}, - encrypted_mode_calculator_param={"mode": "fast"}, callback_param={"callbacks": ["EarlyStopping", "PerformanceEvaluate"], "validation_freqs": 1, "early_stopping_rounds": 5, diff --git a/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-warm-start.py b/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-warm-start.py index 31c5674479..973efbfa2d 100644 --- a/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-warm-start.py +++ b/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-warm-start.py @@ -56,7 +56,6 @@ def main(config="../../config.yaml", namespace=""): learning_rate=0.15, decay=0.0, decay_sqrt=False, callback_param={"callbacks": ["ModelCheckpoint"]}, init_param={"init_method": "zeros"}, - encrypted_mode_calculator_param={"mode": "fast"}, floating_point_precision=23) evaluation_0 = Evaluation(name="evaluation_0", eval_type="regression", pos_label=1) @@ -65,7 +64,6 @@ def main(config="../../config.yaml", namespace=""): penalty="L2", optimizer="sgd", tol=0.001, alpha=0.01, early_stop="weight_diff", batch_size=-1, learning_rate=0.15, decay=0.0, decay_sqrt=False, - encrypted_mode_calculator_param={"mode": "fast"}, floating_point_precision=23 ) diff --git a/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr.py b/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr.py index cb5dd826ed..e539db833c 100644 --- a/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr.py +++ b/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr.py @@ -55,7 +55,6 @@ def main(config="../../config.yaml", namespace=""): alpha=0.01, max_iter=20, early_stop="weight_diff", batch_size=-1, learning_rate=0.15, decay=0.0, decay_sqrt=False, init_param={"init_method": "zeros"}, - encrypted_mode_calculator_param={"mode": "fast"}, floating_point_precision=23) evaluation_0 = Evaluation(name="evaluation_0", eval_type="regression", pos_label=1) diff --git a/examples/pipeline/hetero_logistic_regression/hetero_logistic_regression_pipeline_testsuite.json b/examples/pipeline/hetero_logistic_regression/hetero_logistic_regression_pipeline_testsuite.json index 464c779e8b..c3d40fdda2 100644 --- a/examples/pipeline/hetero_logistic_regression/hetero_logistic_regression_pipeline_testsuite.json +++ b/examples/pipeline/hetero_logistic_regression/hetero_logistic_regression_pipeline_testsuite.json @@ -45,15 +45,9 @@ "hetero-lr-sample-weights": { "script": "pipeline-hetero-lr-sample-weights.py" }, - "hetero-lr-sqn": { - "script": "pipeline-hetero-lr-sqn.py" - }, "hetero-lr-sparse": { "script": "pipeline-hetero-lr-sparse.py" }, - "hetero-lr-ovr-sqn": { - "script": "pipeline-hetero-lr-ovr-sqn.py" - }, "hetero-lr-feature-engineering": { "script": "pipeline-hetero-lr-feature-engineering.py" }, @@ -84,9 +78,6 @@ "hetero-lr-early-stop": { "script": "pipeline-hetero-lr-early-stop.py" }, - "hetero-lr-sparse-sqn": { - "script": "pipeline-hetero-lr-sparse-sqn.py" - }, "hetero-lr-warm-start": { "script": "pipeline-hetero-lr-warm-start.py" }, diff --git a/examples/pipeline/hetero_poisson/pipeline-hetero-poisson-cv.py b/examples/pipeline/hetero_poisson/pipeline-hetero-poisson-cv.py index 2f18810b39..dca8871f76 100644 --- a/examples/pipeline/hetero_poisson/pipeline-hetero-poisson-cv.py +++ b/examples/pipeline/hetero_poisson/pipeline-hetero-poisson-cv.py @@ -57,7 +57,6 @@ def main(config="../../config.yaml", namespace=""): alpha=100.0, batch_size=-1, learning_rate=0.01, exposure_colname="exposure", decay_sqrt=False, init_param={"init_method": "zeros"}, - encrypted_mode_calculator_param={"mode": "fast"}, cv_param={ "n_splits": 5, "shuffle": False, diff --git a/examples/pipeline/hetero_poisson/pipeline-hetero-poisson-sparse.py b/examples/pipeline/hetero_poisson/pipeline-hetero-poisson-sparse.py index a863a2c139..8991b1621d 100644 --- a/examples/pipeline/hetero_poisson/pipeline-hetero-poisson-sparse.py +++ b/examples/pipeline/hetero_poisson/pipeline-hetero-poisson-sparse.py @@ -58,7 +58,6 @@ def main(config="../../config.yaml", namespace=""): exposure_colname="exposure", optimizer="rmsprop", penalty="L2", decay_sqrt=False, tol=0.001, init_param={"init_method": "zeros"}, - encrypted_mode_calculator_param={"mode": "fast"} ) evaluation_0 = Evaluation(name="evaluation_0", eval_type="regression", pos_label=1) diff --git a/examples/pipeline/hetero_poisson/pipeline-hetero-poisson-validate.py b/examples/pipeline/hetero_poisson/pipeline-hetero-poisson-validate.py index fc0d6c6edb..a28bd96c66 100644 --- a/examples/pipeline/hetero_poisson/pipeline-hetero-poisson-validate.py +++ b/examples/pipeline/hetero_poisson/pipeline-hetero-poisson-validate.py @@ -76,8 +76,7 @@ def main(config="../../config.yaml", namespace=""): "use_first_metric_only": False, "save_freq": 1 }, - init_param={"init_method": "zeros"}, - encrypted_mode_calculator_param={"mode": "fast"}) + init_param={"init_method": "zeros"}) pipeline.add_component(reader_0) pipeline.add_component(reader_1) diff --git a/examples/pipeline/hetero_poisson/pipeline-hetero-poisson-warm-start.py b/examples/pipeline/hetero_poisson/pipeline-hetero-poisson-warm-start.py index 5010a28bc4..8a564ae821 100644 --- a/examples/pipeline/hetero_poisson/pipeline-hetero-poisson-warm-start.py +++ b/examples/pipeline/hetero_poisson/pipeline-hetero-poisson-warm-start.py @@ -56,13 +56,11 @@ def main(config="../../config.yaml", namespace=""): alpha=100.0, batch_size=-1, learning_rate=0.01, optimizer="rmsprop", exposure_colname="exposure", decay_sqrt=False, tol=0.001, callback_param={"callbacks": ["ModelCheckpoint"]}, - init_param={"init_method": "zeros"}, penalty="L2", - encrypted_mode_calculator_param={"mode": "fast"}) + init_param={"init_method": "zeros"}, penalty="L2") hetero_poisson_1 = HeteroPoisson(name="hetero_poisson_1", early_stop="weight_diff", max_iter=10, alpha=100.0, batch_size=-1, learning_rate=0.01, optimizer="rmsprop", - exposure_colname="exposure", decay_sqrt=False, tol=0.001, penalty="L2", - encrypted_mode_calculator_param={"mode": "fast"}) + exposure_colname="exposure", decay_sqrt=False, tol=0.001, penalty="L2") evaluation_0 = Evaluation(name="evaluation_0", eval_type="regression", pos_label=1) evaluation_0.get_party_instance(role='host', party_id=host).component_param(need_run=False) diff --git a/examples/pipeline/hetero_poisson/pipeline-hetero-poisson.py b/examples/pipeline/hetero_poisson/pipeline-hetero-poisson.py index 8333dde3bf..2516faf210 100644 --- a/examples/pipeline/hetero_poisson/pipeline-hetero-poisson.py +++ b/examples/pipeline/hetero_poisson/pipeline-hetero-poisson.py @@ -55,8 +55,7 @@ def main(config="../../config.yaml", namespace=""): hetero_poisson_0 = HeteroPoisson(name="hetero_poisson_0", early_stop="weight_diff", max_iter=10, alpha=100.0, batch_size=-1, learning_rate=0.01, optimizer="rmsprop", exposure_colname="exposure", decay_sqrt=False, tol=0.001, - init_param={"init_method": "zeros"}, penalty="L2", - encrypted_mode_calculator_param={"mode": "fast"}) + init_param={"init_method": "zeros"}, penalty="L2") evaluation_0 = Evaluation(name="evaluation_0", eval_type="regression", pos_label=1) evaluation_0.get_party_instance(role='host', party_id=host).component_param(need_run=False) diff --git a/examples/pipeline/hetero_stepwise/pipeline-stepwise-linr.py b/examples/pipeline/hetero_stepwise/pipeline-stepwise-linr.py index 4084ab726b..c3e5244232 100644 --- a/examples/pipeline/hetero_stepwise/pipeline-stepwise-linr.py +++ b/examples/pipeline/hetero_stepwise/pipeline-stepwise-linr.py @@ -55,7 +55,6 @@ def main(config="../../config.yaml", namespace=""): alpha=0.01, batch_size=-1, learning_rate=0.15, decay=0.0, decay_sqrt=False, init_param={"init_method": "zeros"}, - encrypted_mode_calculator_param={"mode": "fast"}, stepwise_param={"score_name": "AIC", "direction": "backward", "need_stepwise": True, "max_step": 3, "nvmin": 2 }) diff --git a/examples/pipeline/hetero_stepwise/pipeline-stepwise-lr.py b/examples/pipeline/hetero_stepwise/pipeline-stepwise-lr.py index bf97ecdc74..c9ad934e63 100644 --- a/examples/pipeline/hetero_stepwise/pipeline-stepwise-lr.py +++ b/examples/pipeline/hetero_stepwise/pipeline-stepwise-lr.py @@ -54,7 +54,6 @@ def main(config="../../config.yaml", namespace=""): batch_size=-1, learning_rate=0.15, decay=0.0, decay_sqrt=False, init_param={"init_method": "zeros"}, - encrypted_mode_calculator_param={"mode": "fast"}, stepwise_param={"score_name": "AIC", "direction": "backward", "need_stepwise": True, "max_step": 2, "nvmin": 2 }) diff --git a/examples/pipeline/hetero_stepwise/pipeline-stepwise-poisson.py b/examples/pipeline/hetero_stepwise/pipeline-stepwise-poisson.py index 05d26a020b..d000b66620 100644 --- a/examples/pipeline/hetero_stepwise/pipeline-stepwise-poisson.py +++ b/examples/pipeline/hetero_stepwise/pipeline-stepwise-poisson.py @@ -55,7 +55,6 @@ def main(config="../../config.yaml", namespace=""): batch_size=-1, learning_rate=0.15, decay=0.0, decay_sqrt=False, alpha=0.01, init_param={"init_method": "zeros"}, - encrypted_mode_calculator_param={"mode": "fast"}, stepwise_param={"score_name": "AIC", "direction": "both", "need_stepwise": True, "max_step": 1, "nvmin": 2 }) diff --git a/python/fate_client/pipeline/param/logistic_regression_param.py b/python/fate_client/pipeline/param/logistic_regression_param.py index af38e5f324..8840531e28 100644 --- a/python/fate_client/pipeline/param/logistic_regression_param.py +++ b/python/fate_client/pipeline/param/logistic_regression_param.py @@ -46,8 +46,8 @@ class LogisticParam(BaseParam): alpha : float, default: 1.0 Regularization strength coefficient. - optimizer : {'rmsprop', 'sgd', 'adam', 'nesterov_momentum_sgd', 'sqn', 'adagrad'}, default: 'rmsprop' - Optimize method, if 'sqn' has been set, sqn_param will take effect. Currently, 'sqn' support hetero mode only. + optimizer : {'rmsprop', 'sgd', 'adam', 'nesterov_momentum_sgd', 'adagrad'}, default: 'rmsprop' + Optimize method. batch_size : int, default: -1 Batch size when updating model. -1 means use all data in a batch. i.e. Not to use mini-batch strategy. @@ -151,217 +151,4 @@ def __init__(self, penalty='L2', self.callback_param = copy.deepcopy(callback_param) def check(self): - descr = "logistic_param's" - - if self.penalty is None: - pass - elif type(self.penalty).__name__ != "str": - raise ValueError( - "logistic_param's penalty {} not supported, should be str type".format(self.penalty)) - else: - self.penalty = self.penalty.upper() - if self.penalty not in [consts.L1_PENALTY, consts.L2_PENALTY, 'NONE']: - raise ValueError( - "logistic_param's penalty not supported, penalty should be 'L1', 'L2' or 'none'") - - if not isinstance(self.tol, (int, float)): - raise ValueError( - "logistic_param's tol {} not supported, should be float type".format(self.tol)) - - if type(self.alpha).__name__ not in ["float", 'int']: - raise ValueError( - "logistic_param's alpha {} not supported, should be float or int type".format(self.alpha)) - - if type(self.optimizer).__name__ != "str": - raise ValueError( - "logistic_param's optimizer {} not supported, should be str type".format(self.optimizer)) - else: - self.optimizer = self.optimizer.lower() - if self.optimizer not in ['sgd', 'rmsprop', 'adam', 'adagrad', 'nesterov_momentum_sgd', 'sqn']: - raise ValueError( - "logistic_param's optimizer not supported, optimizer should be" - " 'sgd', 'rmsprop', 'adam', 'nesterov_momentum_sgd', 'sqn' or 'adagrad'") - - if self.batch_size != -1: - if type(self.batch_size).__name__ not in ["int"] \ - or self.batch_size < consts.MIN_BATCH_SIZE: - raise ValueError(descr + " {} not supported, should be larger than {} or " - "-1 represent for all data".format(self.batch_size, consts.MIN_BATCH_SIZE)) - - if not isinstance(self.learning_rate, (float, int)): - raise ValueError( - "logistic_param's learning_rate {} not supported, should be float or int type".format( - self.learning_rate)) - - self.init_param.check() - - if type(self.max_iter).__name__ != "int": - raise ValueError( - "logistic_param's max_iter {} not supported, should be int type".format(self.max_iter)) - elif self.max_iter <= 0: - raise ValueError( - "logistic_param's max_iter must be greater or equal to 1") - - if type(self.early_stop).__name__ != "str": - raise ValueError( - "logistic_param's early_stop {} not supported, should be str type".format( - self.early_stop)) - else: - self.early_stop = self.early_stop.lower() - if self.early_stop not in ['diff', 'abs', 'weight_diff']: - raise ValueError( - "logistic_param's early_stop not supported, converge_func should be" - " 'diff', 'weight_diff' or 'abs'") - - self.encrypt_param.check() - self.predict_param.check() - if self.encrypt_param.method not in [consts.PAILLIER, None]: - raise ValueError( - "logistic_param's encrypted method support 'Paillier' or None only") - - if type(self.decay).__name__ not in ["int", 'float']: - raise ValueError( - "logistic_param's decay {} not supported, should be 'int' or 'float'".format( - self.decay)) - - if type(self.decay_sqrt).__name__ not in ['bool']: - raise ValueError( - "logistic_param's decay_sqrt {} not supported, should be 'bool'".format( - self.decay_sqrt)) - self.stepwise_param.check() - - if self.early_stopping_rounds is None: - pass - elif isinstance(self.early_stopping_rounds, int): - if self.early_stopping_rounds < 1: - raise ValueError("early stopping rounds should be larger than 0 when it's integer") - if self.validation_freqs is None: - raise ValueError("validation freqs must be set when early stopping is enabled") - if self.metrics is not None and not isinstance(self.metrics, list): - raise ValueError("metrics should be a list") - - if not isinstance(self.use_first_metric_only, bool): - raise ValueError("use_first_metric_only should be a boolean") - - if self.floating_point_precision is not None and \ - (not isinstance(self.floating_point_precision, int) or\ - self.floating_point_precision < 0 or self.floating_point_precision > 63): - raise ValueError("floating point precision should be null or a integer between 0 and 63") - return True - - -class HomoLogisticParam(LogisticParam): - """ - Parameters - ---------- - re_encrypt_batches : int, default: 2 - Required when using encrypted version HomoLR. Since multiple batch updating coefficient may cause - overflow error. The model need to be re-encrypt for every several batches. Please be careful when setting - this parameter. Too large batches may cause training failure. - - aggregate_iters : int, default: 1 - Indicate how many iterations are aggregated once. - - use_proximal: bool, default: False - Whether to turn on additional proximial term. For more details of FedProx, Please refer to - https://arxiv.org/abs/1812.06127 - - mu: float, default 0.1 - To scale the proximal term - - """ - def __init__(self, penalty='L2', - tol=1e-4, alpha=1.0, optimizer='rmsprop', - batch_size=-1, learning_rate=0.01, init_param=InitParam(), - max_iter=100, early_stop='diff', - encrypt_param=EncryptParam(method=None), re_encrypt_batches=2, - predict_param=PredictParam(), cv_param=CrossValidationParam(), - decay=1, decay_sqrt=True, - aggregate_iters=1, multi_class='ovr', validation_freqs=None, - early_stopping_rounds=None, - metrics=['auc', 'ks'], - use_first_metric_only=False, - use_proximal=False, - mu=0.1, callback_param=CallbackParam() - ): - super(HomoLogisticParam, self).__init__(penalty=penalty, tol=tol, alpha=alpha, optimizer=optimizer, - batch_size=batch_size, - learning_rate=learning_rate, - init_param=init_param, max_iter=max_iter, early_stop=early_stop, - encrypt_param=encrypt_param, predict_param=predict_param, - cv_param=cv_param, multi_class=multi_class, - validation_freqs=validation_freqs, - decay=decay, decay_sqrt=decay_sqrt, - early_stopping_rounds=early_stopping_rounds, - metrics=metrics, use_first_metric_only=use_first_metric_only, - callback_param=callback_param) - self.re_encrypt_batches = re_encrypt_batches - self.aggregate_iters = aggregate_iters - self.use_proximal = use_proximal - self.mu = mu - - def check(self): - super().check() - if type(self.re_encrypt_batches).__name__ != "int": - raise ValueError( - "logistic_param's re_encrypt_batches {} not supported, should be int type".format( - self.re_encrypt_batches)) - elif self.re_encrypt_batches < 0: - raise ValueError( - "logistic_param's re_encrypt_batches must be greater or equal to 0") - - if not isinstance(self.aggregate_iters, int): - raise ValueError( - "logistic_param's aggregate_iters {} not supported, should be int type".format( - self.aggregate_iters)) - - if self.encrypt_param.method == consts.PAILLIER: - if self.optimizer != 'sgd': - raise ValueError("Paillier encryption mode supports 'sgd' optimizer method only.") - - if self.penalty == consts.L1_PENALTY: - raise ValueError("Paillier encryption mode supports 'L2' penalty or None only.") - - if self.optimizer == 'sqn': - raise ValueError("'sqn' optimizer is supported for hetero mode only.") - - return True - - -class HeteroLogisticParam(LogisticParam): - def __init__(self, penalty='L2', - tol=1e-4, alpha=1.0, optimizer='rmsprop', - batch_size=-1, shuffle=True, batch_strategy="full", masked_rate=5, - learning_rate=0.01, init_param=InitParam(), - max_iter=100, early_stop='diff', - encrypted_mode_calculator_param=EncryptedModeCalculatorParam(), - predict_param=PredictParam(), cv_param=CrossValidationParam(), - decay=1, decay_sqrt=True, sqn_param=StochasticQuasiNewtonParam(), - multi_class='ovr', validation_freqs=None, early_stopping_rounds=None, - metrics=['auc', 'ks'], floating_point_precision=23, - encrypt_param=EncryptParam(), - use_first_metric_only=False, stepwise_param=StepwiseParam(), - callback_param=CallbackParam() - ): - super(HeteroLogisticParam, self).__init__(penalty=penalty, tol=tol, alpha=alpha, optimizer=optimizer, - batch_size=batch_size, shuffle=shuffle, batch_strategy=batch_strategy, masked_rate=masked_rate, - learning_rate=learning_rate, - init_param=init_param, max_iter=max_iter, early_stop=early_stop, - predict_param=predict_param, cv_param=cv_param, - decay=decay, - decay_sqrt=decay_sqrt, multi_class=multi_class, - validation_freqs=validation_freqs, - early_stopping_rounds=early_stopping_rounds, - metrics=metrics, floating_point_precision=floating_point_precision, - encrypt_param=encrypt_param, - use_first_metric_only=use_first_metric_only, - stepwise_param=stepwise_param, - callback_param=callback_param) - self.encrypted_mode_calculator_param = copy.deepcopy(encrypted_mode_calculator_param) - self.sqn_param = copy.deepcopy(sqn_param) - - def check(self): - super().check() - self.encrypted_mode_calculator_param.check() - self.sqn_param.check() return True From 1ea5082e21317e86cd0d8c1c487b772a40b8318d Mon Sep 17 00:00:00 2001 From: mgqa34 Date: Fri, 25 Feb 2022 03:46:01 +0800 Subject: [PATCH 86/99] update code of sqn, delete deprecated examples Signed-off-by: mgqa34 --- .../pipeline-hetero-lr-ovr-sqn.py | 81 ------------------ .../pipeline-hetero-lr-sparse-sqn.py | 82 ------------------- .../pipeline-hetero-lr-sqn.py | 81 ------------------ .../linear_model/linear_model_weight.py | 2 +- .../hetero_linr_base.py | 1 + .../hetero_lr_base.py | 9 -- .../gradient/hetero_linr_gradient_and_loss.py | 8 +- .../optim/gradient/hetero_sqn_gradient.py | 17 +++- .../param/logistic_regression_param.py | 16 ++-- 9 files changed, 28 insertions(+), 269 deletions(-) delete mode 100644 examples/pipeline/hetero_logistic_regression/pipeline-hetero-lr-ovr-sqn.py delete mode 100644 examples/pipeline/hetero_logistic_regression/pipeline-hetero-lr-sparse-sqn.py delete mode 100644 examples/pipeline/hetero_logistic_regression/pipeline-hetero-lr-sqn.py diff --git a/examples/pipeline/hetero_logistic_regression/pipeline-hetero-lr-ovr-sqn.py b/examples/pipeline/hetero_logistic_regression/pipeline-hetero-lr-ovr-sqn.py deleted file mode 100644 index 8ec1eb39b1..0000000000 --- a/examples/pipeline/hetero_logistic_regression/pipeline-hetero-lr-ovr-sqn.py +++ /dev/null @@ -1,81 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import argparse -import os -import sys - -cur_path = os.path.realpath(__file__) -for i in range(4): - cur_path = os.path.dirname(cur_path) -print(f'fate_path: {cur_path}') -sys.path.append(cur_path) - -from examples.pipeline.hetero_logistic_regression import common_tools - -from pipeline.utils.tools import load_job_config - - -def main(config="../../config.yaml", namespace=""): - # obtain config - if isinstance(config, str): - config = load_job_config(config) - - lr_param = { - "name": "hetero_lr_0", - "penalty": "L2", - "optimizer": "sqn", - "tol": 0.0001, - "alpha": 1e-05, - "max_iter": 10, - "early_stop": "diff", - "batch_size": 5000, - "learning_rate": 0.15, - "decay": 0.3, - "decay_sqrt": True, - "init_param": { - "init_method": "zeros" - }, - "sqn_param": { - "update_interval_L": 3, - "memory_M": 5, - "sample_size": 5000, - "random_seed": None - }, - "cv_param": { - "n_splits": 3, - "shuffle": False, - "random_seed": 103, - "need_cv": False - } - } - - pipeline = common_tools.make_normal_dsl(config, namespace, lr_param, is_ovr=True) - # fit model - pipeline.fit() - # query component summary - common_tools.prettify(pipeline.get_component("hetero_lr_0").get_summary()) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser("PIPELINE DEMO") - parser.add_argument("-config", type=str, - help="config file") - args = parser.parse_args() - if args.config is not None: - main(args.config) - else: - main() diff --git a/examples/pipeline/hetero_logistic_regression/pipeline-hetero-lr-sparse-sqn.py b/examples/pipeline/hetero_logistic_regression/pipeline-hetero-lr-sparse-sqn.py deleted file mode 100644 index 4fa67cedc3..0000000000 --- a/examples/pipeline/hetero_logistic_regression/pipeline-hetero-lr-sparse-sqn.py +++ /dev/null @@ -1,82 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import argparse -import os -import sys - -cur_path = os.path.realpath(__file__) -for i in range(4): - cur_path = os.path.dirname(cur_path) -print(f'fate_path: {cur_path}') -sys.path.append(cur_path) - -from examples.pipeline.hetero_logistic_regression import common_tools - -from pipeline.utils.tools import load_job_config - - -def main(config="../../config.yaml", namespace=""): - # obtain config - if isinstance(config, str): - config = load_job_config(config) - - lr_param = { - "name": "hetero_lr_0", - "penalty": "L2", - "optimizer": "sqn", - "tol": 0.0001, - "alpha": 1e-05, - "max_iter": 30, - "early_stop": "diff", - "batch_size": 5000, - "learning_rate": 0.15, - "decay": 0.3, - "decay_sqrt": True, - "init_param": { - "init_method": "zeros" - }, - "sqn_param": { - "update_interval_L": 3, - "memory_M": 5, - "sample_size": 5000, - "random_seed": None - }, - "cv_param": { - "n_splits": 5, - "shuffle": False, - "random_seed": 103, - "need_cv": False - } - } - - pipeline = common_tools.make_normal_dsl(config, namespace, lr_param, is_dense=False) - # fit model - pipeline.fit() - # query component summary - common_tools.prettify(pipeline.get_component("hetero_lr_0").get_summary()) - common_tools.prettify(pipeline.get_component("evaluation_0").get_summary()) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser("PIPELINE DEMO") - parser.add_argument("-config", type=str, - help="config file") - args = parser.parse_args() - if args.config is not None: - main(args.config) - else: - main() diff --git a/examples/pipeline/hetero_logistic_regression/pipeline-hetero-lr-sqn.py b/examples/pipeline/hetero_logistic_regression/pipeline-hetero-lr-sqn.py deleted file mode 100644 index c8b61dcc6f..0000000000 --- a/examples/pipeline/hetero_logistic_regression/pipeline-hetero-lr-sqn.py +++ /dev/null @@ -1,81 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import argparse -import os -import sys - -cur_path = os.path.realpath(__file__) -for i in range(4): - cur_path = os.path.dirname(cur_path) -print(f'fate_path: {cur_path}') -sys.path.append(cur_path) - -from examples.pipeline.hetero_logistic_regression import common_tools - -from pipeline.utils.tools import load_job_config - - -def main(config="../../config.yaml", namespace=""): - # obtain config - if isinstance(config, str): - config = load_job_config(config) - - lr_param = { - "name": "hetero_lr_0", - "penalty": "L2", - "optimizer": "sqn", - "tol": 0.0001, - "alpha": 1e-05, - "max_iter": 10, - "early_stop": "diff", - "batch_size": 5000, - "learning_rate": 0.15, - "decay": 0.3, - "decay_sqrt": True, - "init_param": { - "init_method": "zeros" - }, - "sqn_param": { - "update_interval_L": 3, - "memory_M": 5, - "sample_size": 5000, - "random_seed": None - }, - "cv_param": { - "n_splits": 3, - "shuffle": False, - "random_seed": 103, - "need_cv": False - } - } - - pipeline = common_tools.make_normal_dsl(config, namespace, lr_param) - # fit model - pipeline.fit() - # query component summary - common_tools.prettify(pipeline.get_component("hetero_lr_0").get_summary()) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser("PIPELINE DEMO") - parser.add_argument("-config", type=str, - help="config file") - args = parser.parse_args() - if args.config is not None: - main(args.config) - else: - main() diff --git a/python/federatedml/linear_model/linear_model_weight.py b/python/federatedml/linear_model/linear_model_weight.py index 1cbea1de58..f6799f1785 100644 --- a/python/federatedml/linear_model/linear_model_weight.py +++ b/python/federatedml/linear_model/linear_model_weight.py @@ -62,7 +62,7 @@ def binary_op(self, other: 'LinearModelWeights', func, inplace): _w = [] for k, v in enumerate(self._weights): _w.append(func(self._weights[k], other._weights[k])) - return LinearModelWeights(_w, self.fit_intercept) + return LinearModelWeights(_w, self.fit_intercept, self.raise_overflow_error) def __repr__(self): return f"weights: {self.coef_}, intercept: {self.intercept_}" \ No newline at end of file diff --git a/python/federatedml/linear_model/linear_regression/hetero_linear_regression/hetero_linr_base.py b/python/federatedml/linear_model/linear_regression/hetero_linear_regression/hetero_linr_base.py index bdcddb0420..3f0eb391b7 100644 --- a/python/federatedml/linear_model/linear_regression/hetero_linear_regression/hetero_linr_base.py +++ b/python/federatedml/linear_model/linear_regression/hetero_linear_regression/hetero_linr_base.py @@ -49,6 +49,7 @@ def _init_model(self, params): gradient_loss_operator = sqn_factory(self.role, params.sqn_param) gradient_loss_operator.register_gradient_computer(self.gradient_loss_operator) gradient_loss_operator.register_transfer_variable(self.transfer_variable) + gradient_loss_operator.unset_raise_weight_overflow_error() self.gradient_loss_operator = gradient_loss_operator LOGGER.debug("In _init_model, optimizer: {}, gradient_loss_operator: {}".format( params.optimizer, self.gradient_loss_operator diff --git a/python/federatedml/linear_model/logistic_regression/hetero_logistic_regression/hetero_lr_base.py b/python/federatedml/linear_model/logistic_regression/hetero_logistic_regression/hetero_lr_base.py index aa5ff2672f..383d312fbd 100644 --- a/python/federatedml/linear_model/logistic_regression/hetero_logistic_regression/hetero_lr_base.py +++ b/python/federatedml/linear_model/logistic_regression/hetero_logistic_regression/hetero_lr_base.py @@ -54,15 +54,6 @@ def _init_model(self, params): # self.gradient_loss_operator.set_use_async() self.gradient_loss_operator.set_fixed_float_precision(self.model_param.floating_point_precision) - if params.optimizer == 'sqn': - gradient_loss_operator = sqn_factory(self.role, params.sqn_param) - gradient_loss_operator.register_gradient_computer(self.gradient_loss_operator) - gradient_loss_operator.register_transfer_variable(self.transfer_variable) - self.gradient_loss_operator = gradient_loss_operator - LOGGER.debug("In _init_model, optimizer: {}, gradient_loss_operator: {}".format( - params.optimizer, self.gradient_loss_operator - )) - def _get_meta(self): meta_protobuf_obj = lr_model_meta_pb2.LRModelMeta(penalty=self.model_param.penalty, tol=self.model_param.tol, diff --git a/python/federatedml/optim/gradient/hetero_linr_gradient_and_loss.py b/python/federatedml/optim/gradient/hetero_linr_gradient_and_loss.py index c97c400bef..24c96691a6 100644 --- a/python/federatedml/optim/gradient/hetero_linr_gradient_and_loss.py +++ b/python/federatedml/optim/gradient/hetero_linr_gradient_and_loss.py @@ -93,14 +93,14 @@ def compute_forward_hess(self, data_instances, delta_s, host_forwards): define forward_hess = (1/N)*∑(x * s) """ forwards = data_instances.mapValues( - lambda v: (np.dot(v.features, delta_s.coef_) + delta_s.intercept_)) + lambda v: (vec_dot(v.features, delta_s.coef_) + delta_s.intercept_)) for host_forward in host_forwards: forwards = forwards.join(host_forward, lambda g, h: g + h) if self.use_sample_weight: forwards = forwards.join(data_instances, lambda h, d: h * d.weight) - hess_vector = hetero_linear_model_gradient.compute_gradient(data_instances, - forwards, - delta_s.fit_intercept) + hess_vector = self.compute_gradient(data_instances, + forwards, + delta_s.fit_intercept) return forwards, np.array(hess_vector) diff --git a/python/federatedml/optim/gradient/hetero_sqn_gradient.py b/python/federatedml/optim/gradient/hetero_sqn_gradient.py index e97cc90acd..24bee664c3 100644 --- a/python/federatedml/optim/gradient/hetero_sqn_gradient.py +++ b/python/federatedml/optim/gradient/hetero_sqn_gradient.py @@ -44,6 +44,10 @@ def __init__(self, sqn_param: StochasticQuasiNewtonParam): self.memory_M = sqn_param.memory_M self.sample_size = sqn_param.sample_size self.random_seed = sqn_param.random_seed + self.raise_weight_overflow_error = True + + def unset_raise_weight_overflow_error(self): + self.raise_weight_overflow_error = False @property def iter_k(self): @@ -62,7 +66,8 @@ def register_transfer_variable(self, transfer_variable): def _renew_w_tilde(self): self.last_w_tilde = self.this_w_tilde self.this_w_tilde = LinearModelWeights(np.zeros_like(self.last_w_tilde.unboxed), - self.last_w_tilde.fit_intercept) + self.last_w_tilde.fit_intercept, + raise_overflow_error=self.raise_weight_overflow_error) def _update_hessian(self, *args): raise NotImplementedError("Should not call here") @@ -96,7 +101,8 @@ def compute_gradient_procedure(self, *args, **kwargs): self._update_hessian(data_instances, optimizer, cipher_operator) self.last_w_tilde = self.this_w_tilde self.this_w_tilde = LinearModelWeights(np.zeros_like(self.last_w_tilde.unboxed), - self.last_w_tilde.fit_intercept) + self.last_w_tilde.fit_intercept, + raise_overflow_error=self.raise_weight_overflow_error) # LOGGER.debug("After replace, last_w_tilde: {}, this_w_tilde: {}".format(self.last_w_tilde.unboxed, # self.this_w_tilde.unboxed)) @@ -175,7 +181,9 @@ def compute_gradient_procedure(self, cipher_operator, optimizer, n_iter_, batch_ optimizer.set_hess_matrix(self.opt_Hess) delta_grad = self.gradient_computer.compute_gradient_procedure( cipher_operator, optimizer, n_iter_, batch_index) - self._update_w_tilde(LinearModelWeights(delta_grad, fit_intercept=False)) + self._update_w_tilde(LinearModelWeights(delta_grad, + fit_intercept=False, + raise_overflow_error=self.raise_weight_overflow_error)) if self.iter_k % self.update_interval_L == 0: self.count_t += 1 # LOGGER.debug("Before division, this_w_tilde: {}".format(self.this_w_tilde.unboxed)) @@ -187,7 +195,8 @@ def compute_gradient_procedure(self, cipher_operator, optimizer, n_iter_, batch_ self._update_hessian(cipher_operator) self.last_w_tilde = self.this_w_tilde self.this_w_tilde = LinearModelWeights(np.zeros_like(self.last_w_tilde.unboxed), - self.last_w_tilde.fit_intercept) + self.last_w_tilde.fit_intercept, + raise_overflow_error=self.raise_weight_overflow_error) return delta_grad # self._update_w_tilde(cipher_operator) diff --git a/python/federatedml/param/logistic_regression_param.py b/python/federatedml/param/logistic_regression_param.py index fcc66daa12..c009a86ff5 100644 --- a/python/federatedml/param/logistic_regression_param.py +++ b/python/federatedml/param/logistic_regression_param.py @@ -28,6 +28,7 @@ from federatedml.param.sqn_param import StochasticQuasiNewtonParam from federatedml.param.stepwise_param import StepwiseParam from federatedml.util import consts +from federatedml.util import LOGGER deprecated_param_list = ["early_stopping_rounds", "validation_freqs", "metrics", "use_first_metric_only"] @@ -50,8 +51,8 @@ class LogisticParam(BaseParam): alpha : float, default: 1.0 Regularization strength coefficient. - optimizer : {'rmsprop', 'sgd', 'adam', 'nesterov_momentum_sgd', 'sqn', 'adagrad'}, default: 'rmsprop' - Optimize method, if 'sqn' has been set, sqn_param will take effect. Currently, 'sqn' support hetero mode only. + optimizer : {'rmsprop', 'sgd', 'adam', 'nesterov_momentum_sgd', 'adagrad'}, default: 'rmsprop' + Optimize method. batch_strategy : str, {'full', 'random'}, default: "full" Strategy to generate batch data. @@ -192,10 +193,14 @@ def check(self): "logistic_param's optimizer {} not supported, should be str type".format(self.optimizer)) else: self.optimizer = self.optimizer.lower() - if self.optimizer not in ['sgd', 'rmsprop', 'adam', 'adagrad', 'nesterov_momentum_sgd', 'sqn']: + if self.optimizer == "sqn": + LOGGER.warning("sqn is deprecated in fate-v1.7.x, optimizer will be reset to sgd for compatibility") + self.optimizer = "sgd" + + if self.optimizer not in ['sgd', 'rmsprop', 'adam', 'adagrad', 'nesterov_momentum_sgd']: raise ValueError( "logistic_param's optimizer not supported, optimizer should be" - " 'sgd', 'rmsprop', 'adam', 'nesterov_momentum_sgd', 'sqn' or 'adagrad'") + " 'sgd', 'rmsprop', 'adam', 'nesterov_momentum_sgd' or 'adagrad'") if self.batch_size != -1: if type(self.batch_size).__name__ not in ["int"] \ @@ -370,9 +375,6 @@ def check(self): if self.penalty == consts.L1_PENALTY: raise ValueError("Paillier encryption mode supports 'L2' penalty or None only.") - if self.optimizer == 'sqn': - raise ValueError("'sqn' optimizer is supported for hetero mode only.") - return True From e138d6315c00c3575fee499cc5f9737711412bff Mon Sep 17 00:00:00 2001 From: mgqa34 Date: Fri, 25 Feb 2022 04:08:11 +0800 Subject: [PATCH 87/99] recovery pipeline' lr param Signed-off-by: mgqa34 --- .../param/logistic_regression_param.py | 121 +++++++++++++++++- 1 file changed, 119 insertions(+), 2 deletions(-) diff --git a/python/fate_client/pipeline/param/logistic_regression_param.py b/python/fate_client/pipeline/param/logistic_regression_param.py index 8840531e28..299b41d594 100644 --- a/python/fate_client/pipeline/param/logistic_regression_param.py +++ b/python/fate_client/pipeline/param/logistic_regression_param.py @@ -46,8 +46,8 @@ class LogisticParam(BaseParam): alpha : float, default: 1.0 Regularization strength coefficient. - optimizer : {'rmsprop', 'sgd', 'adam', 'nesterov_momentum_sgd', 'adagrad'}, default: 'rmsprop' - Optimize method. + optimizer : {'rmsprop', 'sgd', 'adam', 'nesterov_momentum_sgd', 'sqn', 'adagrad'}, default: 'rmsprop' + Optimize method, if 'sqn' has been set, sqn_param will take effect. Currently, 'sqn' support hetero mode only. batch_size : int, default: -1 Batch size when updating model. -1 means use all data in a batch. i.e. Not to use mini-batch strategy. @@ -152,3 +152,120 @@ def __init__(self, penalty='L2', def check(self): return True + + +class HomoLogisticParam(LogisticParam): + """ + Parameters + ---------- + re_encrypt_batches : int, default: 2 + Required when using encrypted version HomoLR. Since multiple batch updating coefficient may cause + overflow error. The model need to be re-encrypt for every several batches. Please be careful when setting + this parameter. Too large batches may cause training failure. + + aggregate_iters : int, default: 1 + Indicate how many iterations are aggregated once. + + use_proximal: bool, default: False + Whether to turn on additional proximial term. For more details of FedProx, Please refer to + https://arxiv.org/abs/1812.06127 + + mu: float, default 0.1 + To scale the proximal term + + """ + def __init__(self, penalty='L2', + tol=1e-4, alpha=1.0, optimizer='rmsprop', + batch_size=-1, learning_rate=0.01, init_param=InitParam(), + max_iter=100, early_stop='diff', + encrypt_param=EncryptParam(method=None), re_encrypt_batches=2, + predict_param=PredictParam(), cv_param=CrossValidationParam(), + decay=1, decay_sqrt=True, + aggregate_iters=1, multi_class='ovr', validation_freqs=None, + early_stopping_rounds=None, + metrics=['auc', 'ks'], + use_first_metric_only=False, + use_proximal=False, + mu=0.1, callback_param=CallbackParam() + ): + super(HomoLogisticParam, self).__init__(penalty=penalty, tol=tol, alpha=alpha, optimizer=optimizer, + batch_size=batch_size, + learning_rate=learning_rate, + init_param=init_param, max_iter=max_iter, early_stop=early_stop, + encrypt_param=encrypt_param, predict_param=predict_param, + cv_param=cv_param, multi_class=multi_class, + validation_freqs=validation_freqs, + decay=decay, decay_sqrt=decay_sqrt, + early_stopping_rounds=early_stopping_rounds, + metrics=metrics, use_first_metric_only=use_first_metric_only, + callback_param=callback_param) + self.re_encrypt_batches = re_encrypt_batches + self.aggregate_iters = aggregate_iters + self.use_proximal = use_proximal + self.mu = mu + + def check(self): + super().check() + if type(self.re_encrypt_batches).__name__ != "int": + raise ValueError( + "logistic_param's re_encrypt_batches {} not supported, should be int type".format( + self.re_encrypt_batches)) + elif self.re_encrypt_batches < 0: + raise ValueError( + "logistic_param's re_encrypt_batches must be greater or equal to 0") + + if not isinstance(self.aggregate_iters, int): + raise ValueError( + "logistic_param's aggregate_iters {} not supported, should be int type".format( + self.aggregate_iters)) + + if self.encrypt_param.method == consts.PAILLIER: + if self.optimizer != 'sgd': + raise ValueError("Paillier encryption mode supports 'sgd' optimizer method only.") + + if self.penalty == consts.L1_PENALTY: + raise ValueError("Paillier encryption mode supports 'L2' penalty or None only.") + + if self.optimizer == 'sqn': + raise ValueError("'sqn' optimizer is supported for hetero mode only.") + + return True + + +class HeteroLogisticParam(LogisticParam): + def __init__(self, penalty='L2', + tol=1e-4, alpha=1.0, optimizer='rmsprop', + batch_size=-1, shuffle=True, batch_strategy="full", masked_rate=5, + learning_rate=0.01, init_param=InitParam(), + max_iter=100, early_stop='diff', + encrypted_mode_calculator_param=EncryptedModeCalculatorParam(), + predict_param=PredictParam(), cv_param=CrossValidationParam(), + decay=1, decay_sqrt=True, sqn_param=StochasticQuasiNewtonParam(), + multi_class='ovr', validation_freqs=None, early_stopping_rounds=None, + metrics=['auc', 'ks'], floating_point_precision=23, + encrypt_param=EncryptParam(), + use_first_metric_only=False, stepwise_param=StepwiseParam(), + callback_param=CallbackParam() + ): + super(HeteroLogisticParam, self).__init__(penalty=penalty, tol=tol, alpha=alpha, optimizer=optimizer, + batch_size=batch_size, shuffle=shuffle, batch_strategy=batch_strategy, masked_rate=masked_rate, + learning_rate=learning_rate, + init_param=init_param, max_iter=max_iter, early_stop=early_stop, + predict_param=predict_param, cv_param=cv_param, + decay=decay, + decay_sqrt=decay_sqrt, multi_class=multi_class, + validation_freqs=validation_freqs, + early_stopping_rounds=early_stopping_rounds, + metrics=metrics, floating_point_precision=floating_point_precision, + encrypt_param=encrypt_param, + use_first_metric_only=use_first_metric_only, + stepwise_param=stepwise_param, + callback_param=callback_param) + self.encrypted_mode_calculator_param = copy.deepcopy(encrypted_mode_calculator_param) + self.sqn_param = copy.deepcopy(sqn_param) + + def check(self): + super().check() + self.encrypted_mode_calculator_param.check() + self.sqn_param.check() + return True From 19b7aa34159fb3a7fe065e388ff961b862664388 Mon Sep 17 00:00:00 2001 From: mgqa34 Date: Fri, 25 Feb 2022 10:53:09 +0800 Subject: [PATCH 88/99] Update examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-sqn.py Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .../hetero_linear_regression/pipeline-hetero-linr-sqn.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-sqn.py b/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-sqn.py index b20854e1dd..db526c1c15 100644 --- a/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-sqn.py +++ b/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-sqn.py @@ -46,8 +46,13 @@ def main(config="../../config.yaml", namespace=""): reader_0.get_party_instance(role='host', party_id=host).component_param(table=host_train_data) data_transform_0 = DataTransform(name="data_transform_0") - data_transform_0.get_party_instance(role='guest', party_id=guest).component_param(with_label=True, label_name="motor_speed", - label_type="float", output_format="dense") + data_transform_0.get_party_instance( + role='guest', + party_id=guest).component_param( + with_label=True, + label_name="motor_speed", + label_type="float", + output_format="dense") data_transform_0.get_party_instance(role='host', party_id=host).component_param(with_label=False) intersection_0 = Intersection(name="intersection_0") From c1d9c4a95c81e00bf7fdc7dc6ce57c6f2388095c Mon Sep 17 00:00:00 2001 From: mgqa34 Date: Fri, 25 Feb 2022 10:53:39 +0800 Subject: [PATCH 89/99] Update examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-sqn.py Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .../hetero_linear_regression/pipeline-hetero-linr-sqn.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-sqn.py b/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-sqn.py index db526c1c15..a510d48bba 100644 --- a/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-sqn.py +++ b/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-sqn.py @@ -81,7 +81,6 @@ def main(config="../../config.yaml", namespace=""): pipeline.fit() - # predict # deploy required components pipeline.deploy_component([data_transform_0, intersection_0, hetero_linr_0]) From ee52f6ad72e6813fdc90a98dbe65055ddd20e13a Mon Sep 17 00:00:00 2001 From: mgqa34 Date: Fri, 25 Feb 2022 10:53:47 +0800 Subject: [PATCH 90/99] Update examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-sqn.py Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .../hetero_linear_regression/pipeline-hetero-linr-sqn.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-sqn.py b/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-sqn.py index a510d48bba..f6c16fa3f0 100644 --- a/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-sqn.py +++ b/examples/pipeline/hetero_linear_regression/pipeline-hetero-linr-sqn.py @@ -90,8 +90,10 @@ def main(config="../../config.yaml", namespace=""): predict_pipeline.add_component(reader_0) # add selected components from train pipeline onto predict pipeline # specify data source - predict_pipeline.add_component(pipeline, - data=Data(predict_input={pipeline.data_transform_0.input.data: reader_0.output.data})) + predict_pipeline.add_component( + pipeline, data=Data( + predict_input={ + pipeline.data_transform_0.input.data: reader_0.output.data})) # run predict model predict_pipeline.predict() From 7540b37d71093cbd9afaa105428f76ece23622c4 Mon Sep 17 00:00:00 2001 From: mgqa34 Date: Fri, 25 Feb 2022 11:08:04 +0800 Subject: [PATCH 91/99] Update examples/pipeline/hetero_sbt/pipeline-hetero-sbt-EINI-with-random-mask.py Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .../hetero_sbt/pipeline-hetero-sbt-EINI-with-random-mask.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/pipeline/hetero_sbt/pipeline-hetero-sbt-EINI-with-random-mask.py b/examples/pipeline/hetero_sbt/pipeline-hetero-sbt-EINI-with-random-mask.py index 600e8510eb..fd6fe3342e 100644 --- a/examples/pipeline/hetero_sbt/pipeline-hetero-sbt-EINI-with-random-mask.py +++ b/examples/pipeline/hetero_sbt/pipeline-hetero-sbt-EINI-with-random-mask.py @@ -108,8 +108,10 @@ def main(config="../../config.yaml", namespace=""): predict_pipeline.add_component(reader_0) # add selected components from train pipeline onto predict pipeline # specify data source - predict_pipeline.add_component(pipeline, - data=Data(predict_input={pipeline.data_transform_0.input.data: reader_0.output.data})) + predict_pipeline.add_component( + pipeline, data=Data( + predict_input={ + pipeline.data_transform_0.input.data: reader_0.output.data})) # run predict model predict_pipeline.predict() From a2fc9d922a510b50ce42cfd10f11641b1626175d Mon Sep 17 00:00:00 2001 From: mgqa34 Date: Fri, 25 Feb 2022 11:09:40 +0800 Subject: [PATCH 92/99] Update python/federatedml/transfer_variable/transfer_class/hetero_secure_boosting_predict_transfer_variable.py Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .../hetero_secure_boosting_predict_transfer_variable.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/federatedml/transfer_variable/transfer_class/hetero_secure_boosting_predict_transfer_variable.py b/python/federatedml/transfer_variable/transfer_class/hetero_secure_boosting_predict_transfer_variable.py index feb280bee5..8918fc20e2 100644 --- a/python/federatedml/transfer_variable/transfer_class/hetero_secure_boosting_predict_transfer_variable.py +++ b/python/federatedml/transfer_variable/transfer_class/hetero_secure_boosting_predict_transfer_variable.py @@ -34,4 +34,5 @@ def __init__(self, flowid=0): self.guest_predict_data = self._create_variable(name='guest_predict_data', src=['guest'], dst=['host']) self.host_predict_data = self._create_variable(name='host_predict_data', src=['host'], dst=['guest']) self.inter_host_data = self._create_variable(name='inter_host_data', src=['host'], dst=['host']) - self.host_feature_importance = self._create_variable(name='host_feature_importance', src=['host'], dst=['guest']) + self.host_feature_importance = self._create_variable( + name='host_feature_importance', src=['host'], dst=['guest']) From 9b9d09e87c60c3158217cb99868096c34990edfc Mon Sep 17 00:00:00 2001 From: mgqa34 Date: Fri, 25 Feb 2022 11:11:00 +0800 Subject: [PATCH 93/99] Update examples/pipeline/hetero_sbt/pipeline-hetero-sbt-EINI-with-random-mask.py Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .../hetero_sbt/pipeline-hetero-sbt-EINI-with-random-mask.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/pipeline/hetero_sbt/pipeline-hetero-sbt-EINI-with-random-mask.py b/examples/pipeline/hetero_sbt/pipeline-hetero-sbt-EINI-with-random-mask.py index fd6fe3342e..2e7804f317 100644 --- a/examples/pipeline/hetero_sbt/pipeline-hetero-sbt-EINI-with-random-mask.py +++ b/examples/pipeline/hetero_sbt/pipeline-hetero-sbt-EINI-with-random-mask.py @@ -118,7 +118,7 @@ def main(config="../../config.yaml", namespace=""): predict_result = predict_pipeline.get_component("hetero_secure_boost_0").get_output_data() print("Showing 10 data of predict result") for ret in predict_result["data"][:10]: - print (ret) + print(ret) if __name__ == "__main__": From 89814be54cf8465a69609a219b4da73501a34e9d Mon Sep 17 00:00:00 2001 From: weiwee Date: Fri, 25 Feb 2022 11:33:37 +0800 Subject: [PATCH 94/99] Update examples/pipeline/hetero_logistic_regression/pipeline-hetero-lr-batch-random-strategy.py Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .../pipeline-hetero-lr-batch-random-strategy.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/pipeline/hetero_logistic_regression/pipeline-hetero-lr-batch-random-strategy.py b/examples/pipeline/hetero_logistic_regression/pipeline-hetero-lr-batch-random-strategy.py index bfee083090..4856a259e3 100644 --- a/examples/pipeline/hetero_logistic_regression/pipeline-hetero-lr-batch-random-strategy.py +++ b/examples/pipeline/hetero_logistic_regression/pipeline-hetero-lr-batch-random-strategy.py @@ -73,7 +73,6 @@ def main(config="../../config.yaml", namespace=""): # json.dump(dsl_json, open('./hetero-lr-normal-predict-dsl.json', 'w'), indent=4) # json.dump(conf_json, open('./hetero-lr-normal-predict-conf.json', 'w'), indent=4) - # fit model pipeline.fit() # query component summary From 226c3d37e9b0d8013bf8520c3f7f3bc84cbc08d2 Mon Sep 17 00:00:00 2001 From: zhihuiwan <15779896112@163.com> Date: Fri, 25 Feb 2022 14:48:22 +0800 Subject: [PATCH 95/99] update release.md --- RELEASE.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/RELEASE.md b/RELEASE.md index 19691e8c7d..0615c45221 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -9,6 +9,11 @@ > Bug-Fix * Fixed Bug for HeteroPearson with changing default q_field value for spdz * Fix Data Transform's schema label name setting problem when `with_label` is False +* Add testing examples for new algorithm features, and delete deprecated params in algorithm examples. + +> FATE-ARCH +* Support loading of custom password encryption module +* Separate the base connection address of the data storage table from the data table information, and compatible with historical versions ## Release 1.7.1.1 From 273227e5a66e1c6381032479edcc62f081ed17cb Mon Sep 17 00:00:00 2001 From: mgqa34 Date: Fri, 25 Feb 2022 15:08:18 +0800 Subject: [PATCH 96/99] pep8 format for linear_model_weight Signed-off-by: mgqa34 --- python/federatedml/linear_model/linear_model_weight.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/federatedml/linear_model/linear_model_weight.py b/python/federatedml/linear_model/linear_model_weight.py index f6799f1785..d73e81cf10 100644 --- a/python/federatedml/linear_model/linear_model_weight.py +++ b/python/federatedml/linear_model/linear_model_weight.py @@ -65,4 +65,4 @@ def binary_op(self, other: 'LinearModelWeights', func, inplace): return LinearModelWeights(_w, self.fit_intercept, self.raise_overflow_error) def __repr__(self): - return f"weights: {self.coef_}, intercept: {self.intercept_}" \ No newline at end of file + return f"weights: {self.coef_}, intercept: {self.intercept_}" From 60626a8818f7cd8f6046f78d0976ee4302ba0d23 Mon Sep 17 00:00:00 2001 From: zhihuiwan <15779896112@163.com> Date: Fri, 25 Feb 2022 15:17:55 +0800 Subject: [PATCH 97/99] update release.md Signed-off-by: zhihuiwan <15779896112@163.com> --- RELEASE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE.md b/RELEASE.md index 0615c45221..5ee6c8e205 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -12,7 +12,7 @@ * Add testing examples for new algorithm features, and delete deprecated params in algorithm examples. > FATE-ARCH -* Support loading of custom password encryption module +* Support the loading of custom password encryption modules through plug-ins * Separate the base connection address of the data storage table from the data table information, and compatible with historical versions From e28b00b7302413fc366e78a0e5b681b6487ec60a Mon Sep 17 00:00:00 2001 From: code-review-doctor Date: Fri, 25 Feb 2022 07:27:19 +0000 Subject: [PATCH 98/99] Fix issue exception-handler-or found at https://codereview.doctor --- python/federatedml/util/anonymous_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/federatedml/util/anonymous_generator.py b/python/federatedml/util/anonymous_generator.py index c2b4de34ce..0040961828 100644 --- a/python/federatedml/util/anonymous_generator.py +++ b/python/federatedml/util/anonymous_generator.py @@ -33,6 +33,6 @@ def generate_anonymous(fid, party_id=None, role=None, model=None): def reconstruct_fid(encoded_name): try: col_index = int(encoded_name.split('_')[-1]) - except IndexError or ValueError: + except (IndexError, ValueError): raise RuntimeError(f"Decode name: {encoded_name} is not a valid value") return col_index From 882309a13e92c11214d10d440b04984eecacbd2b Mon Sep 17 00:00:00 2001 From: mgqa34 Date: Fri, 25 Feb 2022 16:21:09 +0800 Subject: [PATCH 99/99] update gitmodule and submodule-ref of fateboard-v1.7.2.2, fateflow-1.7.2 Signed-off-by: mgqa34 --- .gitmodules | 4 ++-- fateboard | 2 +- fateflow | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitmodules b/.gitmodules index 31e85fe3ce..2d2eff074f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,7 @@ [submodule "fateboard"] path = fateboard url = https://github.com/FederatedAI/FATE-Board.git - branch = develop-1.7.2.2 + branch = v1.7.2.2 [submodule "eggroll"] path = eggroll url = https://github.com/WeBankFinTech/eggroll.git @@ -9,4 +9,4 @@ [submodule "fateflow"] path = fateflow url = https://github.com/FederatedAI/FATE-Flow.git - branch = develop-1.7.2 + branch = v1.7.2 diff --git a/fateboard b/fateboard index 180a2b2471..0c4f24c860 160000 --- a/fateboard +++ b/fateboard @@ -1 +1 @@ -Subproject commit 180a2b24717d9ab41f60d850418b8f829a6fe9db +Subproject commit 0c4f24c860f2e814e563faffd10a15183d0eaede diff --git a/fateflow b/fateflow index 9bf35dd134..c51a0511de 160000 --- a/fateflow +++ b/fateflow @@ -1 +1 @@ -Subproject commit 9bf35dd13487bd5481c9fe1983f9a6419fad5e2e +Subproject commit c51a0511de1d2cc8e49d17cce1357ec3de03d2e8