From bffea2912637bbd3825b711bfd27a289f7290543 Mon Sep 17 00:00:00 2001 From: Chong Shen Ng Date: Tue, 11 Jun 2024 11:07:22 +0100 Subject: [PATCH 01/74] Add log cli --- src/py/flwr/cli/app.py | 2 ++ src/py/flwr/cli/log.py | 51 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 src/py/flwr/cli/log.py diff --git a/src/py/flwr/cli/app.py b/src/py/flwr/cli/app.py index e1417f1267ac..1b87ba222f3f 100644 --- a/src/py/flwr/cli/app.py +++ b/src/py/flwr/cli/app.py @@ -20,6 +20,7 @@ from .example import example from .new import new from .run import run +from .log import log app = typer.Typer( help=typer.style( @@ -34,6 +35,7 @@ app.command()(example) app.command()(run) app.command()(build) +app.command()(log) if __name__ == "__main__": app() diff --git a/src/py/flwr/cli/log.py b/src/py/flwr/cli/log.py new file mode 100644 index 000000000000..095ae784d449 --- /dev/null +++ b/src/py/flwr/cli/log.py @@ -0,0 +1,51 @@ +# Copyright 2024 Flower Labs GmbH. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Flower command line interface `log` command.""" + +import typer +from typing_extensions import Annotated + + +def log( + run_id: Annotated[ + int, + typer.Option(case_sensitive=False, help="The Flower run ID to query"), + ], + follow: Annotated[ + bool, + typer.Option(case_sensitive=False, help="Use this flag to follow logstream"), + ] = True, +) -> None: + """Get logs from Flower run.""" + from logging import DEBUG, INFO + + from flwr.common.grpc import GRPC_MAX_MESSAGE_LENGTH, create_channel + from flwr.common.logger import log + + def on_channel_state_change(channel_connectivity: str) -> None: + """Log channel connectivity.""" + log(DEBUG, channel_connectivity) + + def create_channel(): + """Create gRPC channel connection""" + pass + + channel = create_channel() + + try: + while True: + print('Log') + except KeyboardInterrupt: + log(INFO, "Exiting `flwr log`.") From aa7192910d7c8f4183bcab0b894ca78f6fa8fd23 Mon Sep 17 00:00:00 2001 From: Chong Shen Ng Date: Tue, 11 Jun 2024 11:27:07 +0100 Subject: [PATCH 02/74] Update --- src/py/flwr/cli/log.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/py/flwr/cli/log.py b/src/py/flwr/cli/log.py index 095ae784d449..256a3e7479b4 100644 --- a/src/py/flwr/cli/log.py +++ b/src/py/flwr/cli/log.py @@ -45,7 +45,6 @@ def create_channel(): channel = create_channel() try: - while True: - print('Log') + print('Log') except KeyboardInterrupt: log(INFO, "Exiting `flwr log`.") From 9d4f66610b40e6ffb6b7091840afe7a3da916d41 Mon Sep 17 00:00:00 2001 From: Chong Shen Ng Date: Tue, 11 Jun 2024 12:41:40 +0100 Subject: [PATCH 03/74] Refactor --- src/py/flwr/cli/log.py | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/src/py/flwr/cli/log.py b/src/py/flwr/cli/log.py index 256a3e7479b4..272c931dc66f 100644 --- a/src/py/flwr/cli/log.py +++ b/src/py/flwr/cli/log.py @@ -15,6 +15,7 @@ """Flower command line interface `log` command.""" import typer +import time from typing_extensions import Annotated @@ -38,13 +39,34 @@ def on_channel_state_change(channel_connectivity: str) -> None: """Log channel connectivity.""" log(DEBUG, channel_connectivity) - def create_channel(): - """Create gRPC channel connection""" - pass + def stream_logs(run_id, channel, duration): + """Stream logs with connection refresh""" + start_time = time.time() + # Set stub and req + # stub = ExecStub(channel) + # req = StreamLogsRequest(run_id=run_id) + for res in ['log']: + print(res) + if follow and time.time() - start_time < duration: + continue + else: + log(INFO, "Logstream exceeded duration.") + break - channel = create_channel() + channel = create_channel( + server_address="127.0.0.1:9093", + insecure=True, + root_certificates=None, + max_message_length=GRPC_MAX_MESSAGE_LENGTH, + interceptors=None, + ) + channel.subscribe(on_channel_state_change) + STREAM_DURATION = 60 try: - print('Log') + while True: + stream_logs(run_id, channel, STREAM_DURATION) + time.sleep(5) + log(INFO, "Reconnecting to logstream.") except KeyboardInterrupt: log(INFO, "Exiting `flwr log`.") From f46cc3029f09214e929a5c9e05ab0c4e9305cbe6 Mon Sep 17 00:00:00 2001 From: Chong Shen Ng Date: Tue, 11 Jun 2024 12:43:41 +0100 Subject: [PATCH 04/74] Lint --- src/py/flwr/cli/log.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/py/flwr/cli/log.py b/src/py/flwr/cli/log.py index 272c931dc66f..92103646b1ac 100644 --- a/src/py/flwr/cli/log.py +++ b/src/py/flwr/cli/log.py @@ -14,8 +14,9 @@ # ============================================================================== """Flower command line interface `log` command.""" -import typer import time + +import typer from typing_extensions import Annotated @@ -35,17 +36,20 @@ def log( from flwr.common.grpc import GRPC_MAX_MESSAGE_LENGTH, create_channel from flwr.common.logger import log + # from flwr.proto.exec_pb2 import StreamLogsRequest + # from flwr.proto.exec_pb2_grpc import ExecStub + def on_channel_state_change(channel_connectivity: str) -> None: """Log channel connectivity.""" log(DEBUG, channel_connectivity) def stream_logs(run_id, channel, duration): - """Stream logs with connection refresh""" + """Stream logs with connection refresh.""" start_time = time.time() # Set stub and req # stub = ExecStub(channel) # req = StreamLogsRequest(run_id=run_id) - for res in ['log']: + for res in ["log"]: print(res) if follow and time.time() - start_time < duration: continue From 2110415ef458f4895adaff1c3b6339c882918262 Mon Sep 17 00:00:00 2001 From: Chong Shen Ng Date: Tue, 11 Jun 2024 12:51:26 +0100 Subject: [PATCH 05/74] Fix isort --- src/py/flwr/cli/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/py/flwr/cli/app.py b/src/py/flwr/cli/app.py index 1b87ba222f3f..39144f6ffd1b 100644 --- a/src/py/flwr/cli/app.py +++ b/src/py/flwr/cli/app.py @@ -18,9 +18,9 @@ from .build import build from .example import example +from .log import log from .new import new from .run import run -from .log import log app = typer.Typer( help=typer.style( From a68dfcd6ec8da5f12dc3b4da35b39defe02dd2cb Mon Sep 17 00:00:00 2001 From: Chong Shen Ng Date: Tue, 11 Jun 2024 14:45:51 +0100 Subject: [PATCH 06/74] Restructure code --- src/py/flwr/cli/log.py | 32 +++++++++++++------------------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/src/py/flwr/cli/log.py b/src/py/flwr/cli/log.py index 92103646b1ac..c98feaac970a 100644 --- a/src/py/flwr/cli/log.py +++ b/src/py/flwr/cli/log.py @@ -36,26 +36,15 @@ def log( from flwr.common.grpc import GRPC_MAX_MESSAGE_LENGTH, create_channel from flwr.common.logger import log - # from flwr.proto.exec_pb2 import StreamLogsRequest - # from flwr.proto.exec_pb2_grpc import ExecStub - def on_channel_state_change(channel_connectivity: str) -> None: """Log channel connectivity.""" log(DEBUG, channel_connectivity) def stream_logs(run_id, channel, duration): """Stream logs with connection refresh.""" - start_time = time.time() - # Set stub and req - # stub = ExecStub(channel) - # req = StreamLogsRequest(run_id=run_id) - for res in ["log"]: - print(res) - if follow and time.time() - start_time < duration: - continue - else: - log(INFO, "Logstream exceeded duration.") - break + + def print_logs(run_id, channel, timeout): + """Print logs.""" channel = create_channel( server_address="127.0.0.1:9093", @@ -68,9 +57,14 @@ def stream_logs(run_id, channel, duration): STREAM_DURATION = 60 try: - while True: - stream_logs(run_id, channel, STREAM_DURATION) - time.sleep(5) - log(INFO, "Reconnecting to logstream.") + if follow: + while True: + log(INFO, "Streaming logs") + stream_logs(run_id, channel, STREAM_DURATION) + time.sleep(2) + log(INFO, "Reconnecting to logstream") + else: + print_logs(run_id, channel, timeout=1) except KeyboardInterrupt: - log(INFO, "Exiting `flwr log`.") + log(INFO, "Exiting logstream") + channel.close() From 90e2aa729cd329983aeee1c016ae5b5e7e1ede3d Mon Sep 17 00:00:00 2001 From: Chong Shen Ng Date: Tue, 11 Jun 2024 14:57:08 +0100 Subject: [PATCH 07/74] Fix untyped calls --- src/py/flwr/cli/log.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/py/flwr/cli/log.py b/src/py/flwr/cli/log.py index c98feaac970a..39e4f25eedeb 100644 --- a/src/py/flwr/cli/log.py +++ b/src/py/flwr/cli/log.py @@ -40,10 +40,10 @@ def on_channel_state_change(channel_connectivity: str) -> None: """Log channel connectivity.""" log(DEBUG, channel_connectivity) - def stream_logs(run_id, channel, duration): + def stream_logs(run_id, channel, duration) -> None: """Stream logs with connection refresh.""" - def print_logs(run_id, channel, timeout): + def print_logs(run_id, channel, timeout) -> None: """Print logs.""" channel = create_channel( From 96b6ba460c2b890c3e4bfb4012a337b9b24c2aa5 Mon Sep 17 00:00:00 2001 From: Chong Shen Ng Date: Tue, 11 Jun 2024 15:05:54 +0100 Subject: [PATCH 08/74] Add types --- src/py/flwr/cli/log.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/py/flwr/cli/log.py b/src/py/flwr/cli/log.py index 39e4f25eedeb..5ad2a174d6d2 100644 --- a/src/py/flwr/cli/log.py +++ b/src/py/flwr/cli/log.py @@ -15,6 +15,7 @@ """Flower command line interface `log` command.""" import time +import grpc import typer from typing_extensions import Annotated @@ -40,10 +41,10 @@ def on_channel_state_change(channel_connectivity: str) -> None: """Log channel connectivity.""" log(DEBUG, channel_connectivity) - def stream_logs(run_id, channel, duration) -> None: + def stream_logs(run_id, channel: grpc.Channel, duration: int) -> None: """Stream logs with connection refresh.""" - def print_logs(run_id, channel, timeout) -> None: + def print_logs(run_id, channel: grpc.Channel, timeout: int) -> None: """Print logs.""" channel = create_channel( From 3fe5fdc793c6a56dbb86c5e47f981f00014790a5 Mon Sep 17 00:00:00 2001 From: Chong Shen Ng Date: Tue, 11 Jun 2024 15:06:32 +0100 Subject: [PATCH 09/74] Add type --- src/py/flwr/cli/log.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/py/flwr/cli/log.py b/src/py/flwr/cli/log.py index 5ad2a174d6d2..3a8983b2fe06 100644 --- a/src/py/flwr/cli/log.py +++ b/src/py/flwr/cli/log.py @@ -41,10 +41,10 @@ def on_channel_state_change(channel_connectivity: str) -> None: """Log channel connectivity.""" log(DEBUG, channel_connectivity) - def stream_logs(run_id, channel: grpc.Channel, duration: int) -> None: + def stream_logs(run_id: int, channel: grpc.Channel, duration: int) -> None: """Stream logs with connection refresh.""" - def print_logs(run_id, channel: grpc.Channel, timeout: int) -> None: + def print_logs(run_id: int, channel: grpc.Channel, timeout: int) -> None: """Print logs.""" channel = create_channel( From 8dd95ae5dfb4e5c701a79ee066dd38f03e161d60 Mon Sep 17 00:00:00 2001 From: Chong Shen Ng Date: Tue, 11 Jun 2024 15:13:27 +0100 Subject: [PATCH 10/74] Fix imports --- src/py/flwr/cli/log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/py/flwr/cli/log.py b/src/py/flwr/cli/log.py index 3a8983b2fe06..5c0c5cae895e 100644 --- a/src/py/flwr/cli/log.py +++ b/src/py/flwr/cli/log.py @@ -15,8 +15,8 @@ """Flower command line interface `log` command.""" import time -import grpc +import grpc import typer from typing_extensions import Annotated From 6a57e95ac77f6fe5742af90afb36c13e90642796 Mon Sep 17 00:00:00 2001 From: Chong Shen Ng Date: Tue, 11 Jun 2024 15:25:23 +0100 Subject: [PATCH 11/74] Fix pylint error and warnings --- src/py/flwr/cli/log.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/py/flwr/cli/log.py b/src/py/flwr/cli/log.py index 5c0c5cae895e..284eaa497177 100644 --- a/src/py/flwr/cli/log.py +++ b/src/py/flwr/cli/log.py @@ -15,11 +15,18 @@ """Flower command line interface `log` command.""" import time +from logging import DEBUG, INFO import grpc import typer from typing_extensions import Annotated +from flwr.common.grpc import GRPC_MAX_MESSAGE_LENGTH, create_channel +# Use alias to avoid pylint error E0102: function already defined +from flwr.common.logger import log as logger + +STREAM_DURATION: int = 60 + def log( run_id: Annotated[ @@ -32,18 +39,16 @@ def log( ] = True, ) -> None: """Get logs from Flower run.""" - from logging import DEBUG, INFO - - from flwr.common.grpc import GRPC_MAX_MESSAGE_LENGTH, create_channel - from flwr.common.logger import log def on_channel_state_change(channel_connectivity: str) -> None: """Log channel connectivity.""" - log(DEBUG, channel_connectivity) + logger(DEBUG, channel_connectivity) + # pylint: disable=unused-argument def stream_logs(run_id: int, channel: grpc.Channel, duration: int) -> None: """Stream logs with connection refresh.""" + # pylint: disable=unused-argument def print_logs(run_id: int, channel: grpc.Channel, timeout: int) -> None: """Print logs.""" @@ -55,17 +60,16 @@ def print_logs(run_id: int, channel: grpc.Channel, timeout: int) -> None: interceptors=None, ) channel.subscribe(on_channel_state_change) - STREAM_DURATION = 60 try: if follow: while True: - log(INFO, "Streaming logs") + logger(INFO, "Streaming logs") stream_logs(run_id, channel, STREAM_DURATION) time.sleep(2) - log(INFO, "Reconnecting to logstream") + logger(INFO, "Reconnecting to logstream") else: print_logs(run_id, channel, timeout=1) except KeyboardInterrupt: - log(INFO, "Exiting logstream") + logger(INFO, "Exiting logstream") channel.close() From a4c64c225683f9a66602849205ab75d7e8156004 Mon Sep 17 00:00:00 2001 From: Chong Shen Ng Date: Tue, 11 Jun 2024 15:36:04 +0100 Subject: [PATCH 12/74] Move try-except inside if-else --- src/py/flwr/cli/log.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/py/flwr/cli/log.py b/src/py/flwr/cli/log.py index 284eaa497177..ff7cb537c402 100644 --- a/src/py/flwr/cli/log.py +++ b/src/py/flwr/cli/log.py @@ -61,15 +61,15 @@ def print_logs(run_id: int, channel: grpc.Channel, timeout: int) -> None: ) channel.subscribe(on_channel_state_change) - try: - if follow: + if follow: + try: while True: logger(INFO, "Streaming logs") stream_logs(run_id, channel, STREAM_DURATION) time.sleep(2) logger(INFO, "Reconnecting to logstream") - else: - print_logs(run_id, channel, timeout=1) - except KeyboardInterrupt: - logger(INFO, "Exiting logstream") - channel.close() + except KeyboardInterrupt: + logger(INFO, "Exiting logstream") + channel.close() + else: + print_logs(run_id, channel, timeout=1) From 7ee30a2b36f06d604cf1aca52f9e91ee00ce7d53 Mon Sep 17 00:00:00 2001 From: Chong Shen Ng Date: Tue, 11 Jun 2024 16:41:01 +0100 Subject: [PATCH 13/74] Black --- src/py/flwr/cli/log.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/py/flwr/cli/log.py b/src/py/flwr/cli/log.py index ff7cb537c402..be605a1f60d5 100644 --- a/src/py/flwr/cli/log.py +++ b/src/py/flwr/cli/log.py @@ -22,6 +22,7 @@ from typing_extensions import Annotated from flwr.common.grpc import GRPC_MAX_MESSAGE_LENGTH, create_channel + # Use alias to avoid pylint error E0102: function already defined from flwr.common.logger import log as logger From 3a69f754eaeaff2dcf3d895bf20c770c4be3f940 Mon Sep 17 00:00:00 2001 From: Chong Shen Ng Date: Tue, 11 Jun 2024 16:43:15 +0100 Subject: [PATCH 14/74] Remove comments --- src/py/flwr/cli/log.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/py/flwr/cli/log.py b/src/py/flwr/cli/log.py index be605a1f60d5..1cebf5129578 100644 --- a/src/py/flwr/cli/log.py +++ b/src/py/flwr/cli/log.py @@ -22,8 +22,6 @@ from typing_extensions import Annotated from flwr.common.grpc import GRPC_MAX_MESSAGE_LENGTH, create_channel - -# Use alias to avoid pylint error E0102: function already defined from flwr.common.logger import log as logger STREAM_DURATION: int = 60 From f2031d9ebd0d0cad5959fa9cef1997c230ec9a05 Mon Sep 17 00:00:00 2001 From: Chong Shen Ng Date: Wed, 12 Jun 2024 09:08:03 +0100 Subject: [PATCH 15/74] Update docstring --- src/py/flwr/cli/log.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/py/flwr/cli/log.py b/src/py/flwr/cli/log.py index 1cebf5129578..c49ec07884ea 100644 --- a/src/py/flwr/cli/log.py +++ b/src/py/flwr/cli/log.py @@ -45,11 +45,11 @@ def on_channel_state_change(channel_connectivity: str) -> None: # pylint: disable=unused-argument def stream_logs(run_id: int, channel: grpc.Channel, duration: int) -> None: - """Stream logs with connection refresh.""" + """Stream logs from the beginning of a run with connection refresh.""" # pylint: disable=unused-argument def print_logs(run_id: int, channel: grpc.Channel, timeout: int) -> None: - """Print logs.""" + """Print logs from the beginning of a run.""" channel = create_channel( server_address="127.0.0.1:9093", From a301049dea8cafa2c3dcaaa93c4810abdae80097 Mon Sep 17 00:00:00 2001 From: Chong Shen Ng Date: Wed, 12 Jun 2024 09:21:10 +0100 Subject: [PATCH 16/74] Add refresh period to input arg --- src/py/flwr/cli/log.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/py/flwr/cli/log.py b/src/py/flwr/cli/log.py index c49ec07884ea..c97c0c54a2e6 100644 --- a/src/py/flwr/cli/log.py +++ b/src/py/flwr/cli/log.py @@ -24,14 +24,19 @@ from flwr.common.grpc import GRPC_MAX_MESSAGE_LENGTH, create_channel from flwr.common.logger import log as logger -STREAM_DURATION: int = 60 - def log( run_id: Annotated[ int, typer.Option(case_sensitive=False, help="The Flower run ID to query"), ], + period: Annotated[ + int, + typer.Option( + case_sensitive=False, + help="Use this to set connection refresh time period (in seconds)", + ), + ] = 60, follow: Annotated[ bool, typer.Option(case_sensitive=False, help="Use this flag to follow logstream"), @@ -64,7 +69,7 @@ def print_logs(run_id: int, channel: grpc.Channel, timeout: int) -> None: try: while True: logger(INFO, "Streaming logs") - stream_logs(run_id, channel, STREAM_DURATION) + stream_logs(run_id, channel, period) time.sleep(2) logger(INFO, "Reconnecting to logstream") except KeyboardInterrupt: From 3d5bd50e73adcdc3e701a2c3e0ac64455fee905d Mon Sep 17 00:00:00 2001 From: Chong Shen Ng Date: Wed, 12 Jun 2024 09:36:14 +0100 Subject: [PATCH 17/74] Fix arg --- src/py/flwr/cli/log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/py/flwr/cli/log.py b/src/py/flwr/cli/log.py index c97c0c54a2e6..ce56189aed96 100644 --- a/src/py/flwr/cli/log.py +++ b/src/py/flwr/cli/log.py @@ -49,7 +49,7 @@ def on_channel_state_change(channel_connectivity: str) -> None: logger(DEBUG, channel_connectivity) # pylint: disable=unused-argument - def stream_logs(run_id: int, channel: grpc.Channel, duration: int) -> None: + def stream_logs(run_id: int, channel: grpc.Channel, period: int) -> None: """Stream logs from the beginning of a run with connection refresh.""" # pylint: disable=unused-argument From aa84fc761b96e2f522e61c767a07c27b0bf54c44 Mon Sep 17 00:00:00 2001 From: Chong Shen Ng Date: Wed, 12 Jun 2024 09:43:55 +0100 Subject: [PATCH 18/74] Add SuperExec address to input arg --- src/py/flwr/cli/log.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/py/flwr/cli/log.py b/src/py/flwr/cli/log.py index ce56189aed96..f15c2db84347 100644 --- a/src/py/flwr/cli/log.py +++ b/src/py/flwr/cli/log.py @@ -14,15 +14,18 @@ # ============================================================================== """Flower command line interface `log` command.""" +import sys import time from logging import DEBUG, INFO import grpc import typer from typing_extensions import Annotated +from typing import Optional from flwr.common.grpc import GRPC_MAX_MESSAGE_LENGTH, create_channel from flwr.common.logger import log as logger +from flwr.cli import config_utils def log( @@ -30,6 +33,10 @@ def log( int, typer.Option(case_sensitive=False, help="The Flower run ID to query"), ], + superexec_address: Annotated[ + Optional[str], + typer.Option(case_sensitive=False, help="The address of the SuperExec server"), + ] = None, period: Annotated[ int, typer.Option( @@ -56,8 +63,25 @@ def stream_logs(run_id: int, channel: grpc.Channel, period: int) -> None: def print_logs(run_id: int, channel: grpc.Channel, timeout: int) -> None: """Print logs from the beginning of a run.""" + if superexec_address is None: + global_config = config_utils.load( + config_utils.get_flower_home() / "config.toml" + ) + if global_config: + superexec_address = global_config["federation"]["default"] + else: + typer.secho( + "No SuperExec address was provided and no global config " + "was found.", + fg=typer.colors.RED, + bold=True, + ) + sys.exit() + + assert superexec_address is not None + channel = create_channel( - server_address="127.0.0.1:9093", + server_address=superexec_address, insecure=True, root_certificates=None, max_message_length=GRPC_MAX_MESSAGE_LENGTH, From f95fba666989e92e620af702259127ddc6d85f52 Mon Sep 17 00:00:00 2001 From: Chong Shen Ng Date: Wed, 12 Jun 2024 09:47:19 +0100 Subject: [PATCH 19/74] Lint --- src/py/flwr/cli/log.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/py/flwr/cli/log.py b/src/py/flwr/cli/log.py index f15c2db84347..b5d53967f109 100644 --- a/src/py/flwr/cli/log.py +++ b/src/py/flwr/cli/log.py @@ -17,15 +17,15 @@ import sys import time from logging import DEBUG, INFO +from typing import Optional import grpc import typer from typing_extensions import Annotated -from typing import Optional +from flwr.cli import config_utils from flwr.common.grpc import GRPC_MAX_MESSAGE_LENGTH, create_channel from flwr.common.logger import log as logger -from flwr.cli import config_utils def log( @@ -71,8 +71,7 @@ def print_logs(run_id: int, channel: grpc.Channel, timeout: int) -> None: superexec_address = global_config["federation"]["default"] else: typer.secho( - "No SuperExec address was provided and no global config " - "was found.", + "No SuperExec address was provided and no global config was found.", fg=typer.colors.RED, bold=True, ) From 55a68722aa4e88770cd12349f6d0a26b2c3c9711 Mon Sep 17 00:00:00 2001 From: Chong Shen Ng Date: Wed, 12 Jun 2024 09:53:03 +0100 Subject: [PATCH 20/74] Add config_utils --- src/py/flwr/cli/config_utils.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/py/flwr/cli/config_utils.py b/src/py/flwr/cli/config_utils.py index d943d87e3812..3203e75f7e44 100644 --- a/src/py/flwr/cli/config_utils.py +++ b/src/py/flwr/cli/config_utils.py @@ -14,6 +14,7 @@ # ============================================================================== """Utility to validate the `pyproject.toml` file.""" +import os from pathlib import Path from typing import Any, Dict, List, Optional, Tuple @@ -121,3 +122,13 @@ def validate(config: Dict[str, Any]) -> Tuple[bool, List[str], List[str]]: return False, [reason], [] return True, [], [] + + +def get_flower_home() -> Path: + """Return the Flower home directory based on env variables.""" + return Path( + os.getenv( + "FLWR_HOME", + f"{os.getenv('XDG_DATA_HOME', os.getenv('HOME'))}/.flwr", + ) + ) From 567e7e5de5d787eb260ee769a6741b77c825d5c0 Mon Sep 17 00:00:00 2001 From: Chong Shen Ng Date: Thu, 13 Jun 2024 08:04:12 +0100 Subject: [PATCH 21/74] Move functions --- src/py/flwr/cli/log.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/py/flwr/cli/log.py b/src/py/flwr/cli/log.py index b5d53967f109..89bb8f9013ae 100644 --- a/src/py/flwr/cli/log.py +++ b/src/py/flwr/cli/log.py @@ -28,6 +28,16 @@ from flwr.common.logger import log as logger +# pylint: disable=unused-argument +def stream_logs(run_id: int, channel: grpc.Channel, period: int) -> None: + """Stream logs from the beginning of a run with connection refresh.""" + + +# pylint: disable=unused-argument +def print_logs(run_id: int, channel: grpc.Channel, timeout: int) -> None: + """Print logs from the beginning of a run.""" + + def log( run_id: Annotated[ int, @@ -55,14 +65,6 @@ def on_channel_state_change(channel_connectivity: str) -> None: """Log channel connectivity.""" logger(DEBUG, channel_connectivity) - # pylint: disable=unused-argument - def stream_logs(run_id: int, channel: grpc.Channel, period: int) -> None: - """Stream logs from the beginning of a run with connection refresh.""" - - # pylint: disable=unused-argument - def print_logs(run_id: int, channel: grpc.Channel, timeout: int) -> None: - """Print logs from the beginning of a run.""" - if superexec_address is None: global_config = config_utils.load( config_utils.get_flower_home() / "config.toml" From 034e298fc0abfc489c807a84f80e640df5c53edd Mon Sep 17 00:00:00 2001 From: Chong Shen Ng Date: Thu, 13 Jun 2024 09:25:42 +0100 Subject: [PATCH 22/74] Handle run_id not found --- src/py/flwr/cli/log.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/py/flwr/cli/log.py b/src/py/flwr/cli/log.py index 89bb8f9013ae..faa409efed1a 100644 --- a/src/py/flwr/cli/log.py +++ b/src/py/flwr/cli/log.py @@ -16,7 +16,7 @@ import sys import time -from logging import DEBUG, INFO +from logging import DEBUG, ERROR, INFO from typing import Optional import grpc @@ -93,12 +93,18 @@ def on_channel_state_change(channel_connectivity: str) -> None: if follow: try: while True: - logger(INFO, "Streaming logs") + logger(INFO, "Starting logstream") stream_logs(run_id, channel, period) time.sleep(2) logger(INFO, "Reconnecting to logstream") except KeyboardInterrupt: logger(INFO, "Exiting logstream") + except grpc.RpcError as e: + # pylint: disable=E1101 + if e.code() == grpc.StatusCode.NOT_FOUND: + logger(ERROR, "`run_id` is invalid, exiting") + finally: channel.close() else: + logger(INFO, "Printing logstream") print_logs(run_id, channel, timeout=1) From 29d0dd26e607b7097776cd378478da2f9ce1eabd Mon Sep 17 00:00:00 2001 From: Chong Shen Ng Date: Fri, 14 Jun 2024 13:35:06 +0100 Subject: [PATCH 23/74] Update get_flwr_dir --- src/py/flwr/cli/log.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/py/flwr/cli/log.py b/src/py/flwr/cli/log.py index faa409efed1a..404e2b0ebb1e 100644 --- a/src/py/flwr/cli/log.py +++ b/src/py/flwr/cli/log.py @@ -26,6 +26,7 @@ from flwr.cli import config_utils from flwr.common.grpc import GRPC_MAX_MESSAGE_LENGTH, create_channel from flwr.common.logger import log as logger +from flwr.common.config import get_flwr_dir # pylint: disable=unused-argument @@ -67,7 +68,7 @@ def on_channel_state_change(channel_connectivity: str) -> None: if superexec_address is None: global_config = config_utils.load( - config_utils.get_flower_home() / "config.toml" + get_flwr_dir / "config.toml" ) if global_config: superexec_address = global_config["federation"]["default"] From 8cdffba6636116de924e59c5ec240d87055d696c Mon Sep 17 00:00:00 2001 From: Chong Shen Ng Date: Fri, 14 Jun 2024 13:49:10 +0100 Subject: [PATCH 24/74] Lint --- src/py/flwr/cli/log.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/py/flwr/cli/log.py b/src/py/flwr/cli/log.py index 404e2b0ebb1e..636eff59531d 100644 --- a/src/py/flwr/cli/log.py +++ b/src/py/flwr/cli/log.py @@ -24,9 +24,9 @@ from typing_extensions import Annotated from flwr.cli import config_utils +from flwr.common.config import get_flwr_dir from flwr.common.grpc import GRPC_MAX_MESSAGE_LENGTH, create_channel from flwr.common.logger import log as logger -from flwr.common.config import get_flwr_dir # pylint: disable=unused-argument @@ -67,9 +67,7 @@ def on_channel_state_change(channel_connectivity: str) -> None: logger(DEBUG, channel_connectivity) if superexec_address is None: - global_config = config_utils.load( - get_flwr_dir / "config.toml" - ) + global_config = config_utils.load(get_flwr_dir / "config.toml") if global_config: superexec_address = global_config["federation"]["default"] else: From fe630967da8ae23cf0714338481ceb9b2b8a2a44 Mon Sep 17 00:00:00 2001 From: Chong Shen Ng Date: Fri, 14 Jun 2024 13:55:40 +0100 Subject: [PATCH 25/74] Fix bug --- src/py/flwr/cli/log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/py/flwr/cli/log.py b/src/py/flwr/cli/log.py index 636eff59531d..da1135f6bbc6 100644 --- a/src/py/flwr/cli/log.py +++ b/src/py/flwr/cli/log.py @@ -67,7 +67,7 @@ def on_channel_state_change(channel_connectivity: str) -> None: logger(DEBUG, channel_connectivity) if superexec_address is None: - global_config = config_utils.load(get_flwr_dir / "config.toml") + global_config = config_utils.load(get_flwr_dir() / "config.toml") if global_config: superexec_address = global_config["federation"]["default"] else: From d0a23c72d4751a9067388314a38de35b15e765d8 Mon Sep 17 00:00:00 2001 From: Chong Shen Ng Date: Fri, 14 Jun 2024 14:08:33 +0100 Subject: [PATCH 26/74] Add autocancel --- src/py/flwr/cli/log.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/py/flwr/cli/log.py b/src/py/flwr/cli/log.py index da1135f6bbc6..be83794ffddb 100644 --- a/src/py/flwr/cli/log.py +++ b/src/py/flwr/cli/log.py @@ -102,6 +102,8 @@ def on_channel_state_change(channel_connectivity: str) -> None: # pylint: disable=E1101 if e.code() == grpc.StatusCode.NOT_FOUND: logger(ERROR, "`run_id` is invalid, exiting") + if e.code() == grpc.StatusCode.CANCELLED: + pass finally: channel.close() else: From 4448a23462270367367984f23cbb81e2f6a8461e Mon Sep 17 00:00:00 2001 From: Chong Shen Ng Date: Fri, 14 Jun 2024 22:19:59 +0100 Subject: [PATCH 27/74] Standardize log info --- src/py/flwr/cli/log.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/py/flwr/cli/log.py b/src/py/flwr/cli/log.py index be83794ffddb..0d666de9765c 100644 --- a/src/py/flwr/cli/log.py +++ b/src/py/flwr/cli/log.py @@ -92,7 +92,7 @@ def on_channel_state_change(channel_connectivity: str) -> None: if follow: try: while True: - logger(INFO, "Starting logstream") + logger(INFO, "Starting logstream for run_id `%s`", run_id) stream_logs(run_id, channel, period) time.sleep(2) logger(INFO, "Reconnecting to logstream") @@ -101,11 +101,11 @@ def on_channel_state_change(channel_connectivity: str) -> None: except grpc.RpcError as e: # pylint: disable=E1101 if e.code() == grpc.StatusCode.NOT_FOUND: - logger(ERROR, "`run_id` is invalid, exiting") + logger(ERROR, "Invalid run_id `%s`, exiting", run_id) if e.code() == grpc.StatusCode.CANCELLED: pass finally: channel.close() else: - logger(INFO, "Printing logstream") - print_logs(run_id, channel, timeout=1) + logger(INFO, "Printing logstream for run_id `%s`", run_id) + print_logs(run_id, channel, timeout=5) From 07a3ebfbbf5d4fc36ab4d88ff299be9721967901 Mon Sep 17 00:00:00 2001 From: Javier Date: Fri, 12 Jul 2024 17:29:38 +0200 Subject: [PATCH 28/74] feat(framework) Capture `node_id`/`node_config` in `Context` via `NodeState` (#3780) Co-authored-by: Daniel J. Beutel --- src/py/flwr/client/app.py | 40 ++++++++++++++++--- .../client/grpc_adapter_client/connection.py | 2 +- src/py/flwr/client/grpc_client/connection.py | 2 +- .../client/grpc_rere_client/connection.py | 5 ++- .../message_handler/message_handler_test.py | 4 +- .../secure_aggregation/secaggplus_mod_test.py | 7 +++- src/py/flwr/client/mod/utils_test.py | 4 +- src/py/flwr/client/node_state.py | 11 +++-- src/py/flwr/client/node_state_tests.py | 2 +- src/py/flwr/client/rest_client/connection.py | 7 ++-- src/py/flwr/common/context.py | 15 ++++++- src/py/flwr/server/compat/legacy_context.py | 2 +- src/py/flwr/server/run_serverapp.py | 4 +- src/py/flwr/server/server_app_test.py | 2 +- .../fleet/vce/backend/raybackend_test.py | 2 +- .../server/superlink/fleet/vce/vce_api.py | 4 +- .../ray_transport/ray_client_proxy.py | 4 +- .../ray_transport/ray_client_proxy_test.py | 8 +++- 18 files changed, 95 insertions(+), 30 deletions(-) diff --git a/src/py/flwr/client/app.py b/src/py/flwr/client/app.py index bfe5147f78e1..fa17ba9a8481 100644 --- a/src/py/flwr/client/app.py +++ b/src/py/flwr/client/app.py @@ -319,7 +319,13 @@ def _on_backoff(retry_state: RetryState) -> None: on_backoff=_on_backoff, ) - node_state = NodeState(partition_id=partition_id) + # Empty dict (for now) + # This will be removed once users can pass node_config via flower-supernode + node_config: Dict[str, str] = {} + + # NodeState gets initialized when the first connection is established + node_state: Optional[NodeState] = None + runs: Dict[int, Run] = {} while not app_state_tracker.interrupt: @@ -334,9 +340,33 @@ def _on_backoff(retry_state: RetryState) -> None: ) as conn: receive, send, create_node, delete_node, get_run = conn - # Register node - if create_node is not None: - create_node() # pylint: disable=not-callable + # Register node when connecting the first time + if node_state is None: + if create_node is None: + if transport not in ["grpc-bidi", None]: + raise NotImplementedError( + "All transports except `grpc-bidi` require " + "an implementation for `create_node()`.'" + ) + # gRPC-bidi doesn't have the concept of node_id, + # so we set it to -1 + node_state = NodeState( + node_id=-1, + node_config={}, + partition_id=partition_id, + ) + else: + # Call create_node fn to register node + node_id: Optional[int] = ( # pylint: disable=assignment-from-none + create_node() + ) # pylint: disable=not-callable + if node_id is None: + raise ValueError("Node registration failed") + node_state = NodeState( + node_id=node_id, + node_config=node_config, + partition_id=partition_id, + ) app_state_tracker.register_signal_handler() while not app_state_tracker.interrupt: @@ -580,7 +610,7 @@ def _init_connection(transport: Optional[str], server_address: str) -> Tuple[ Tuple[ Callable[[], Optional[Message]], Callable[[Message], None], - Optional[Callable[[], None]], + Optional[Callable[[], Optional[int]]], Optional[Callable[[], None]], Optional[Callable[[int], Run]], ] diff --git a/src/py/flwr/client/grpc_adapter_client/connection.py b/src/py/flwr/client/grpc_adapter_client/connection.py index 971b630e470b..80a5cf0b4656 100644 --- a/src/py/flwr/client/grpc_adapter_client/connection.py +++ b/src/py/flwr/client/grpc_adapter_client/connection.py @@ -44,7 +44,7 @@ def grpc_adapter( # pylint: disable=R0913 Tuple[ Callable[[], Optional[Message]], Callable[[Message], None], - Optional[Callable[[], None]], + Optional[Callable[[], Optional[int]]], Optional[Callable[[], None]], Optional[Callable[[int], Run]], ] diff --git a/src/py/flwr/client/grpc_client/connection.py b/src/py/flwr/client/grpc_client/connection.py index 3e9f261c1ecf..a6417106d51b 100644 --- a/src/py/flwr/client/grpc_client/connection.py +++ b/src/py/flwr/client/grpc_client/connection.py @@ -72,7 +72,7 @@ def grpc_connection( # pylint: disable=R0913, R0915 Tuple[ Callable[[], Optional[Message]], Callable[[Message], None], - Optional[Callable[[], None]], + Optional[Callable[[], Optional[int]]], Optional[Callable[[], None]], Optional[Callable[[int], Run]], ] diff --git a/src/py/flwr/client/grpc_rere_client/connection.py b/src/py/flwr/client/grpc_rere_client/connection.py index 8062ce28fcc7..e573df6854bc 100644 --- a/src/py/flwr/client/grpc_rere_client/connection.py +++ b/src/py/flwr/client/grpc_rere_client/connection.py @@ -79,7 +79,7 @@ def grpc_request_response( # pylint: disable=R0913, R0914, R0915 Tuple[ Callable[[], Optional[Message]], Callable[[Message], None], - Optional[Callable[[], None]], + Optional[Callable[[], Optional[int]]], Optional[Callable[[], None]], Optional[Callable[[int], Run]], ] @@ -176,7 +176,7 @@ def ping() -> None: if not ping_stop_event.is_set(): ping_stop_event.wait(next_interval) - def create_node() -> None: + def create_node() -> Optional[int]: """Set create_node.""" # Call FleetAPI create_node_request = CreateNodeRequest(ping_interval=PING_DEFAULT_INTERVAL) @@ -189,6 +189,7 @@ def create_node() -> None: nonlocal node, ping_thread node = cast(Node, create_node_response.node) ping_thread = start_ping_loop(ping, ping_stop_event) + return node.node_id def delete_node() -> None: """Set delete_node.""" diff --git a/src/py/flwr/client/message_handler/message_handler_test.py b/src/py/flwr/client/message_handler/message_handler_test.py index 9ce4c9620c43..96de7ce0c2cb 100644 --- a/src/py/flwr/client/message_handler/message_handler_test.py +++ b/src/py/flwr/client/message_handler/message_handler_test.py @@ -145,7 +145,7 @@ def test_client_without_get_properties() -> None: actual_msg = handle_legacy_message_from_msgtype( client_fn=_get_client_fn(client), message=message, - context=Context(state=RecordSet(), run_config={}), + context=Context(node_id=1123, node_config={}, state=RecordSet(), run_config={}), ) # Assert @@ -209,7 +209,7 @@ def test_client_with_get_properties() -> None: actual_msg = handle_legacy_message_from_msgtype( client_fn=_get_client_fn(client), message=message, - context=Context(state=RecordSet(), run_config={}), + context=Context(node_id=1123, node_config={}, state=RecordSet(), run_config={}), ) # Assert diff --git a/src/py/flwr/client/mod/secure_aggregation/secaggplus_mod_test.py b/src/py/flwr/client/mod/secure_aggregation/secaggplus_mod_test.py index 5e4c4411e1f7..2832576fb4fc 100644 --- a/src/py/flwr/client/mod/secure_aggregation/secaggplus_mod_test.py +++ b/src/py/flwr/client/mod/secure_aggregation/secaggplus_mod_test.py @@ -73,7 +73,12 @@ def func(configs: Dict[str, ConfigsRecordValues]) -> ConfigsRecord: def _make_ctxt() -> Context: cfg = ConfigsRecord(SecAggPlusState().to_dict()) - return Context(RecordSet(configs_records={RECORD_KEY_STATE: cfg}), run_config={}) + return Context( + node_id=123, + node_config={}, + state=RecordSet(configs_records={RECORD_KEY_STATE: cfg}), + run_config={}, + ) def _make_set_state_fn( diff --git a/src/py/flwr/client/mod/utils_test.py b/src/py/flwr/client/mod/utils_test.py index 7a1dd8988399..a5bbd0a0bb4d 100644 --- a/src/py/flwr/client/mod/utils_test.py +++ b/src/py/flwr/client/mod/utils_test.py @@ -104,7 +104,7 @@ def test_multiple_mods(self) -> None: state = RecordSet() state.metrics_records[METRIC] = MetricsRecord({COUNTER: 0.0}) - context = Context(state=state, run_config={}) + context = Context(node_id=0, node_config={}, state=state, run_config={}) message = _get_dummy_flower_message() # Execute @@ -129,7 +129,7 @@ def test_filter(self) -> None: # Prepare footprint: List[str] = [] mock_app = make_mock_app("app", footprint) - context = Context(state=RecordSet(), run_config={}) + context = Context(node_id=0, node_config={}, state=RecordSet(), run_config={}) message = _get_dummy_flower_message() def filter_mod( diff --git a/src/py/flwr/client/node_state.py b/src/py/flwr/client/node_state.py index 2b090eba9720..d0a349b0cae0 100644 --- a/src/py/flwr/client/node_state.py +++ b/src/py/flwr/client/node_state.py @@ -17,7 +17,7 @@ from dataclasses import dataclass from pathlib import Path -from typing import Any, Dict, Optional +from typing import Dict, Optional from flwr.common import Context, RecordSet from flwr.common.config import get_fused_config @@ -35,8 +35,11 @@ class RunInfo: class NodeState: """State of a node where client nodes execute runs.""" - def __init__(self, partition_id: Optional[int]) -> None: - self._meta: Dict[str, Any] = {} # holds metadata about the node + def __init__( + self, node_id: int, node_config: Dict[str, str], partition_id: Optional[int] + ) -> None: + self.node_id = node_id + self.node_config = node_config self.run_infos: Dict[int, RunInfo] = {} self._partition_id = partition_id @@ -52,6 +55,8 @@ def register_context( self.run_infos[run_id] = RunInfo( initial_run_config=initial_run_config, context=Context( + node_id=self.node_id, + node_config=self.node_config, state=RecordSet(), run_config=initial_run_config.copy(), partition_id=self._partition_id, diff --git a/src/py/flwr/client/node_state_tests.py b/src/py/flwr/client/node_state_tests.py index effd64a3ae7a..8d7971fa5280 100644 --- a/src/py/flwr/client/node_state_tests.py +++ b/src/py/flwr/client/node_state_tests.py @@ -41,7 +41,7 @@ def test_multirun_in_node_state() -> None: expected_values = {0: "1", 1: "1" * 3, 2: "1" * 2, 3: "1", 5: "1"} # NodeState - node_state = NodeState(partition_id=None) + node_state = NodeState(node_id=0, node_config={}, partition_id=None) for task in tasks: run_id = task.run_id diff --git a/src/py/flwr/client/rest_client/connection.py b/src/py/flwr/client/rest_client/connection.py index 0efa5731ae51..3e81969d898c 100644 --- a/src/py/flwr/client/rest_client/connection.py +++ b/src/py/flwr/client/rest_client/connection.py @@ -90,7 +90,7 @@ def http_request_response( # pylint: disable=,R0913, R0914, R0915 Tuple[ Callable[[], Optional[Message]], Callable[[Message], None], - Optional[Callable[[], None]], + Optional[Callable[[], Optional[int]]], Optional[Callable[[], None]], Optional[Callable[[int], Run]], ] @@ -237,19 +237,20 @@ def ping() -> None: if not ping_stop_event.is_set(): ping_stop_event.wait(next_interval) - def create_node() -> None: + def create_node() -> Optional[int]: """Set create_node.""" req = CreateNodeRequest(ping_interval=PING_DEFAULT_INTERVAL) # Send the request res = _request(req, CreateNodeResponse, PATH_CREATE_NODE) if res is None: - return + return None # Remember the node and the ping-loop thread nonlocal node, ping_thread node = res.node ping_thread = start_ping_loop(ping, ping_stop_event) + return node.node_id def delete_node() -> None: """Set delete_node.""" diff --git a/src/py/flwr/common/context.py b/src/py/flwr/common/context.py index 8120723ce9e9..e65300278c84 100644 --- a/src/py/flwr/common/context.py +++ b/src/py/flwr/common/context.py @@ -27,6 +27,11 @@ class Context: Parameters ---------- + node_id : int + The ID that identifies the node. + node_config : Dict[str, str] + A config (key/value mapping) unique to the node and independent of the + `run_config`. This config persists across all runs this node participates in. state : RecordSet Holds records added by the entity in a given run and that will stay local. This means that the data it holds will never leave the system it's running from. @@ -44,16 +49,22 @@ class Context: simulation or proto typing setups. """ + node_id: int + node_config: Dict[str, str] state: RecordSet - partition_id: Optional[int] run_config: Dict[str, str] + partition_id: Optional[int] - def __init__( + def __init__( # pylint: disable=too-many-arguments self, + node_id: int, + node_config: Dict[str, str], state: RecordSet, run_config: Dict[str, str], partition_id: Optional[int] = None, ) -> None: + self.node_id = node_id + self.node_config = node_config self.state = state self.run_config = run_config self.partition_id = partition_id diff --git a/src/py/flwr/server/compat/legacy_context.py b/src/py/flwr/server/compat/legacy_context.py index 9e120c824103..ee09d79012dc 100644 --- a/src/py/flwr/server/compat/legacy_context.py +++ b/src/py/flwr/server/compat/legacy_context.py @@ -52,4 +52,4 @@ def __init__( self.strategy = strategy self.client_manager = client_manager self.history = History() - super().__init__(state, run_config={}) + super().__init__(node_id=0, node_config={}, state=state, run_config={}) diff --git a/src/py/flwr/server/run_serverapp.py b/src/py/flwr/server/run_serverapp.py index b4697e99913f..4cc25feb7e0e 100644 --- a/src/py/flwr/server/run_serverapp.py +++ b/src/py/flwr/server/run_serverapp.py @@ -78,7 +78,9 @@ def _load() -> ServerApp: server_app = _load() # Initialize Context - context = Context(state=RecordSet(), run_config=server_app_run_config) + context = Context( + node_id=0, node_config={}, state=RecordSet(), run_config=server_app_run_config + ) # Call ServerApp server_app(driver=driver, context=context) diff --git a/src/py/flwr/server/server_app_test.py b/src/py/flwr/server/server_app_test.py index 7de8774d4c81..b0672b3202ed 100644 --- a/src/py/flwr/server/server_app_test.py +++ b/src/py/flwr/server/server_app_test.py @@ -29,7 +29,7 @@ def test_server_app_custom_mode() -> None: # Prepare app = ServerApp() driver = MagicMock() - context = Context(state=RecordSet(), run_config={}) + context = Context(node_id=0, node_config={}, state=RecordSet(), run_config={}) called = {"called": False} diff --git a/src/py/flwr/server/superlink/fleet/vce/backend/raybackend_test.py b/src/py/flwr/server/superlink/fleet/vce/backend/raybackend_test.py index 287983003f8c..da4390194d05 100644 --- a/src/py/flwr/server/superlink/fleet/vce/backend/raybackend_test.py +++ b/src/py/flwr/server/superlink/fleet/vce/backend/raybackend_test.py @@ -120,7 +120,7 @@ def _create_message_and_context() -> Tuple[Message, Context, float]: ) # Construct emtpy Context - context = Context(state=RecordSet(), run_config={}) + context = Context(node_id=0, node_config={}, state=RecordSet(), run_config={}) # Expected output expected_output = pi * mult_factor diff --git a/src/py/flwr/server/superlink/fleet/vce/vce_api.py b/src/py/flwr/server/superlink/fleet/vce/vce_api.py index 3c0b36e1ca3c..134fd34ed8f0 100644 --- a/src/py/flwr/server/superlink/fleet/vce/vce_api.py +++ b/src/py/flwr/server/superlink/fleet/vce/vce_api.py @@ -284,7 +284,9 @@ def start_vce( # Construct mapping of NodeStates node_states: Dict[int, NodeState] = {} for node_id, partition_id in nodes_mapping.items(): - node_states[node_id] = NodeState(partition_id=partition_id) + node_states[node_id] = NodeState( + node_id=node_id, node_config={}, partition_id=partition_id + ) # Load backend config log(DEBUG, "Supported backends: %s", list(supported_backends.keys())) diff --git a/src/py/flwr/simulation/ray_transport/ray_client_proxy.py b/src/py/flwr/simulation/ray_transport/ray_client_proxy.py index 31bc22c84bd5..f2684016048e 100644 --- a/src/py/flwr/simulation/ray_transport/ray_client_proxy.py +++ b/src/py/flwr/simulation/ray_transport/ray_client_proxy.py @@ -59,7 +59,9 @@ def _load_app() -> ClientApp: self.app_fn = _load_app self.actor_pool = actor_pool - self.proxy_state = NodeState(partition_id=self.partition_id) + self.proxy_state = NodeState( + node_id=node_id, node_config={}, partition_id=self.partition_id + ) def _submit_job(self, message: Message, timeout: Optional[float]) -> Message: """Sumbit a message to the ActorPool.""" diff --git a/src/py/flwr/simulation/ray_transport/ray_client_proxy_test.py b/src/py/flwr/simulation/ray_transport/ray_client_proxy_test.py index 83f6cfe05313..8831e5f475ea 100644 --- a/src/py/flwr/simulation/ray_transport/ray_client_proxy_test.py +++ b/src/py/flwr/simulation/ray_transport/ray_client_proxy_test.py @@ -218,7 +218,13 @@ def _load_app() -> ClientApp: _load_app, message, str(node_id), - Context(state=RecordSet(), run_config={}, partition_id=node_id), + Context( + node_id=0, + node_config={}, + state=RecordSet(), + run_config={}, + partition_id=node_id, + ), ), ) From 596f859625ca7baf68198944b53c5552b6b3be5c Mon Sep 17 00:00:00 2001 From: Charles Beauville Date: Fri, 12 Jul 2024 18:26:06 +0200 Subject: [PATCH 29/74] feat(framework) Add `node-config` arg to SuperNode (#3782) Co-authored-by: jafermarq Co-authored-by: Daniel J. Beutel --- src/py/flwr/client/app.py | 16 ++++++---------- src/py/flwr/client/supernode/app.py | 20 +++++++++++++------- src/py/flwr/common/config.py | 6 +++--- 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/src/py/flwr/client/app.py b/src/py/flwr/client/app.py index fa17ba9a8481..ffcc95489d62 100644 --- a/src/py/flwr/client/app.py +++ b/src/py/flwr/client/app.py @@ -160,6 +160,7 @@ class `flwr.client.Client` (default: None) event(EventType.START_CLIENT_ENTER) _start_client_internal( server_address=server_address, + node_config={}, load_client_app_fn=None, client_fn=client_fn, client=client, @@ -181,6 +182,7 @@ class `flwr.client.Client` (default: None) def _start_client_internal( *, server_address: str, + node_config: Dict[str, str], load_client_app_fn: Optional[Callable[[str, str], ClientApp]] = None, client_fn: Optional[ClientFnExt] = None, client: Optional[Client] = None, @@ -193,7 +195,6 @@ def _start_client_internal( ] = None, max_retries: Optional[int] = None, max_wait_time: Optional[float] = None, - partition_id: Optional[int] = None, flwr_dir: Optional[Path] = None, ) -> None: """Start a Flower client node which connects to a Flower server. @@ -204,6 +205,8 @@ def _start_client_internal( The IPv4 or IPv6 address of the server. If the Flower server runs on the same machine on port 8080, then `server_address` would be `"[::]:8080"`. + node_config: Dict[str, str] + The configuration of the node. load_client_app_fn : Optional[Callable[[], ClientApp]] (default: None) A function that can be used to load a `ClientApp` instance. client_fn : Optional[ClientFnExt] @@ -238,9 +241,6 @@ class `flwr.client.Client` (default: None) The maximum duration before the client stops trying to connect to the server in case of connection error. If set to None, there is no limit to the total time. - partition_id: Optional[int] (default: None) - The data partition index associated with this node. Better suited for - prototyping purposes. flwr_dir: Optional[Path] (default: None) The fully resolved path containing installed Flower Apps. """ @@ -319,10 +319,6 @@ def _on_backoff(retry_state: RetryState) -> None: on_backoff=_on_backoff, ) - # Empty dict (for now) - # This will be removed once users can pass node_config via flower-supernode - node_config: Dict[str, str] = {} - # NodeState gets initialized when the first connection is established node_state: Optional[NodeState] = None @@ -353,7 +349,7 @@ def _on_backoff(retry_state: RetryState) -> None: node_state = NodeState( node_id=-1, node_config={}, - partition_id=partition_id, + partition_id=None, ) else: # Call create_node fn to register node @@ -365,7 +361,7 @@ def _on_backoff(retry_state: RetryState) -> None: node_state = NodeState( node_id=node_id, node_config=node_config, - partition_id=partition_id, + partition_id=None, ) app_state_tracker.register_signal_handler() diff --git a/src/py/flwr/client/supernode/app.py b/src/py/flwr/client/supernode/app.py index 355a2a13a0e5..d61b986bc7af 100644 --- a/src/py/flwr/client/supernode/app.py +++ b/src/py/flwr/client/supernode/app.py @@ -29,7 +29,12 @@ from flwr.client.client_app import ClientApp, LoadClientAppError from flwr.common import EventType, event -from flwr.common.config import get_flwr_dir, get_project_config, get_project_dir +from flwr.common.config import ( + get_flwr_dir, + get_project_config, + get_project_dir, + parse_config_args, +) from flwr.common.constant import ( TRANSPORT_TYPE_GRPC_ADAPTER, TRANSPORT_TYPE_GRPC_RERE, @@ -67,7 +72,7 @@ def run_supernode() -> None: authentication_keys=authentication_keys, max_retries=args.max_retries, max_wait_time=args.max_wait_time, - partition_id=args.partition_id, + node_config=parse_config_args(args.node_config), flwr_dir=get_flwr_dir(args.flwr_dir), ) @@ -93,6 +98,7 @@ def run_client_app() -> None: _start_client_internal( server_address=args.superlink, + node_config=parse_config_args(args.node_config), load_client_app_fn=load_fn, transport=args.transport, root_certificates=root_certificates, @@ -389,11 +395,11 @@ def _parse_args_common(parser: argparse.ArgumentParser) -> None: help="The SuperNode's public key (as a path str) to enable authentication.", ) parser.add_argument( - "--partition-id", - type=int, - help="The data partition index associated with this SuperNode. Better suited " - "for prototyping purposes where a SuperNode might only load a fraction of an " - "artificially partitioned dataset (e.g. using `flwr-datasets`)", + "--node-config", + type=str, + help="A comma separated list of key/value pairs (separated by `=`) to " + "configure the SuperNode. " + "E.g. --node-config 'key1=\"value1\",partition-id=0,num-partitions=100'", ) diff --git a/src/py/flwr/common/config.py b/src/py/flwr/common/config.py index 9770bdb4af2b..54d74353e4ed 100644 --- a/src/py/flwr/common/config.py +++ b/src/py/flwr/common/config.py @@ -121,16 +121,16 @@ def flatten_dict(raw_dict: Dict[str, Any], parent_key: str = "") -> Dict[str, st def parse_config_args( - config_overrides: Optional[str], + config: Optional[str], separator: str = ",", ) -> Dict[str, str]: """Parse separator separated list of key-value pairs separated by '='.""" overrides: Dict[str, str] = {} - if config_overrides is None: + if config is None: return overrides - overrides_list = config_overrides.split(separator) + overrides_list = config.split(separator) if ( len(overrides_list) == 1 and "=" not in overrides_list From 889eadfecf6a7f86ceb676e43d5fe82b90234f55 Mon Sep 17 00:00:00 2001 From: Javier Date: Sat, 13 Jul 2024 08:39:50 +0200 Subject: [PATCH 30/74] feat(framework) Introduce new `client_fn` signature passing the `Context` (#3779) Co-authored-by: Daniel J. Beutel --- src/py/flwr/client/app.py | 7 ++-- src/py/flwr/client/client_app.py | 34 +++++++++++++++---- .../client/message_handler/message_handler.py | 2 +- .../message_handler/message_handler_test.py | 6 ++-- src/py/flwr/client/typing.py | 4 +-- .../fleet/vce/backend/raybackend_test.py | 4 +-- .../server/superlink/fleet/vce/vce_api.py | 4 ++- .../ray_transport/ray_client_proxy.py | 17 ++++++---- .../ray_transport/ray_client_proxy_test.py | 32 +++++++++-------- 9 files changed, 66 insertions(+), 44 deletions(-) diff --git a/src/py/flwr/client/app.py b/src/py/flwr/client/app.py index ffcc95489d62..380185ed26e7 100644 --- a/src/py/flwr/client/app.py +++ b/src/py/flwr/client/app.py @@ -28,7 +28,7 @@ from flwr.client.client import Client from flwr.client.client_app import ClientApp, LoadClientAppError from flwr.client.typing import ClientFnExt -from flwr.common import GRPC_MAX_MESSAGE_LENGTH, EventType, Message, event +from flwr.common import GRPC_MAX_MESSAGE_LENGTH, Context, EventType, Message, event from flwr.common.address import parse_address from flwr.common.constant import ( MISSING_EXTRA_REST, @@ -138,7 +138,7 @@ class `flwr.client.Client` (default: None) Starting an SSL-enabled gRPC client using system certificates: - >>> def client_fn(node_id: int, partition_id: Optional[int]): + >>> def client_fn(context: Context): >>> return FlowerClient() >>> >>> start_client( @@ -253,8 +253,7 @@ class `flwr.client.Client` (default: None) if client_fn is None: # Wrap `Client` instance in `client_fn` def single_client_factory( - node_id: int, # pylint: disable=unused-argument - partition_id: Optional[int], # pylint: disable=unused-argument + context: Context, # pylint: disable=unused-argument ) -> Client: if client is None: # Added this to keep mypy happy raise ValueError( diff --git a/src/py/flwr/client/client_app.py b/src/py/flwr/client/client_app.py index 663d83a8b19e..9566302d0721 100644 --- a/src/py/flwr/client/client_app.py +++ b/src/py/flwr/client/client_app.py @@ -30,21 +30,41 @@ from .typing import ClientAppCallable +def _alert_erroneous_client_fn() -> None: + raise ValueError( + "A `ClientApp` cannot make use of a `client_fn` that does " + "not have a signature in the form: `def client_fn(context: " + "Context)`. You can import the `Context` like this: " + "`from flwr.common import Context`" + ) + + def _inspect_maybe_adapt_client_fn_signature(client_fn: ClientFnExt) -> ClientFnExt: client_fn_args = inspect.signature(client_fn).parameters + first_arg = list(client_fn_args.keys())[0] + + if len(client_fn_args) != 1: + _alert_erroneous_client_fn() + + first_arg_type = client_fn_args[first_arg].annotation - if not all(key in client_fn_args for key in ["node_id", "partition_id"]): + if first_arg_type is str or first_arg == "cid": + # Warn previous signature for `client_fn` seems to be used warn_deprecated_feature( - "`client_fn` now expects a signature `def client_fn(node_id: int, " - "partition_id: Optional[int])`.\nYou provided `client_fn` with signature: " - f"{dict(client_fn_args.items())}" + "`client_fn` now expects a signature `def client_fn(context: Context)`." + "The provided `client_fn` has signature: " + f"{dict(client_fn_args.items())}. You can import the `Context` like this:" + " `from flwr.common import Context`" ) # Wrap depcreated client_fn inside a function with the expected signature def adaptor_fn( - node_id: int, partition_id: Optional[int] # pylint: disable=unused-argument - ) -> Client: - return client_fn(str(partition_id)) # type: ignore + context: Context, + ) -> Client: # pylint: disable=unused-argument + # if patition-id is defined, pass it. Else pass node_id that should + # always be defined during Context init. + cid = context.node_config.get("partition-id", context.node_id) + return client_fn(str(cid)) # type: ignore return adaptor_fn diff --git a/src/py/flwr/client/message_handler/message_handler.py b/src/py/flwr/client/message_handler/message_handler.py index e9a853a92101..1ab84eb01468 100644 --- a/src/py/flwr/client/message_handler/message_handler.py +++ b/src/py/flwr/client/message_handler/message_handler.py @@ -92,7 +92,7 @@ def handle_legacy_message_from_msgtype( client_fn: ClientFnExt, message: Message, context: Context ) -> Message: """Handle legacy message in the inner most mod.""" - client = client_fn(message.metadata.dst_node_id, context.partition_id) + client = client_fn(context) # Check if NumPyClient is returend if isinstance(client, NumPyClient): diff --git a/src/py/flwr/client/message_handler/message_handler_test.py b/src/py/flwr/client/message_handler/message_handler_test.py index 96de7ce0c2cb..557d61ffb32a 100644 --- a/src/py/flwr/client/message_handler/message_handler_test.py +++ b/src/py/flwr/client/message_handler/message_handler_test.py @@ -19,7 +19,7 @@ import unittest import uuid from copy import copy -from typing import List, Optional +from typing import List from flwr.client import Client from flwr.client.typing import ClientFnExt @@ -114,9 +114,7 @@ def evaluate(self, ins: EvaluateIns) -> EvaluateRes: def _get_client_fn(client: Client) -> ClientFnExt: - def client_fn( - node_id: int, partition_id: Optional[int] # pylint: disable=unused-argument - ) -> Client: + def client_fn(contex: Context) -> Client: # pylint: disable=unused-argument return client return client_fn diff --git a/src/py/flwr/client/typing.py b/src/py/flwr/client/typing.py index bf66a9082c77..9faed4bc7283 100644 --- a/src/py/flwr/client/typing.py +++ b/src/py/flwr/client/typing.py @@ -15,7 +15,7 @@ """Custom types for Flower clients.""" -from typing import Callable, Optional +from typing import Callable from flwr.common import Context, Message @@ -23,7 +23,7 @@ # Compatibility ClientFn = Callable[[str], Client] -ClientFnExt = Callable[[int, Optional[int]], Client] +ClientFnExt = Callable[[Context], Client] ClientAppCallable = Callable[[Message, Context], Message] Mod = Callable[[Message, Context, ClientAppCallable], Message] diff --git a/src/py/flwr/server/superlink/fleet/vce/backend/raybackend_test.py b/src/py/flwr/server/superlink/fleet/vce/backend/raybackend_test.py index da4390194d05..3abdac7a232b 100644 --- a/src/py/flwr/server/superlink/fleet/vce/backend/raybackend_test.py +++ b/src/py/flwr/server/superlink/fleet/vce/backend/raybackend_test.py @@ -53,9 +53,7 @@ def get_properties(self, config: Config) -> Dict[str, Scalar]: return {"result": result} -def get_dummy_client( - node_id: int, partition_id: Optional[int] # pylint: disable=unused-argument -) -> Client: +def get_dummy_client(context: Context) -> Client: # pylint: disable=unused-argument """Return a DummyClient converted to Client type.""" return DummyClient().to_client() diff --git a/src/py/flwr/server/superlink/fleet/vce/vce_api.py b/src/py/flwr/server/superlink/fleet/vce/vce_api.py index 134fd34ed8f0..422148489b4a 100644 --- a/src/py/flwr/server/superlink/fleet/vce/vce_api.py +++ b/src/py/flwr/server/superlink/fleet/vce/vce_api.py @@ -285,7 +285,9 @@ def start_vce( node_states: Dict[int, NodeState] = {} for node_id, partition_id in nodes_mapping.items(): node_states[node_id] = NodeState( - node_id=node_id, node_config={}, partition_id=partition_id + node_id=node_id, + node_config={"partition-id": str(partition_id)}, + partition_id=None, ) # Load backend config diff --git a/src/py/flwr/simulation/ray_transport/ray_client_proxy.py b/src/py/flwr/simulation/ray_transport/ray_client_proxy.py index f2684016048e..b62e04aeed79 100644 --- a/src/py/flwr/simulation/ray_transport/ray_client_proxy.py +++ b/src/py/flwr/simulation/ray_transport/ray_client_proxy.py @@ -60,7 +60,9 @@ def _load_app() -> ClientApp: self.app_fn = _load_app self.actor_pool = actor_pool self.proxy_state = NodeState( - node_id=node_id, node_config={}, partition_id=self.partition_id + node_id=node_id, + node_config={"partition-id": str(partition_id)}, + partition_id=None, ) def _submit_job(self, message: Message, timeout: Optional[float]) -> Message: @@ -70,18 +72,19 @@ def _submit_job(self, message: Message, timeout: Optional[float]) -> Message: # Register state self.proxy_state.register_context(run_id=run_id) - # Retrieve state - state = self.proxy_state.retrieve_context(run_id=run_id) + # Retrieve context + context = self.proxy_state.retrieve_context(run_id=run_id) + partition_id_str = context.node_config["partition-id"] try: self.actor_pool.submit_client_job( - lambda a, a_fn, mssg, partition_id, state: a.run.remote( - a_fn, mssg, partition_id, state + lambda a, a_fn, mssg, partition_id, context: a.run.remote( + a_fn, mssg, partition_id, context ), - (self.app_fn, message, str(self.partition_id), state), + (self.app_fn, message, partition_id_str, context), ) out_mssg, updated_context = self.actor_pool.get_client_result( - str(self.partition_id), timeout + partition_id_str, timeout ) # Update state diff --git a/src/py/flwr/simulation/ray_transport/ray_client_proxy_test.py b/src/py/flwr/simulation/ray_transport/ray_client_proxy_test.py index 8831e5f475ea..1d44ea1d8d2b 100644 --- a/src/py/flwr/simulation/ray_transport/ray_client_proxy_test.py +++ b/src/py/flwr/simulation/ray_transport/ray_client_proxy_test.py @@ -17,7 +17,7 @@ from math import pi from random import shuffle -from typing import Dict, List, Optional, Tuple, Type +from typing import Dict, List, Tuple, Type import ray @@ -39,7 +39,10 @@ recordset_to_getpropertiesres, ) from flwr.common.recordset_compat_test import _get_valid_getpropertiesins -from flwr.simulation.app import _create_node_id_to_partition_mapping +from flwr.simulation.app import ( + NodeToPartitionMapping, + _create_node_id_to_partition_mapping, +) from flwr.simulation.ray_transport.ray_actor import ( ClientAppActor, VirtualClientEngineActor, @@ -65,16 +68,16 @@ def get_properties(self, config: Config) -> Dict[str, Scalar]: return {"result": result} -def get_dummy_client( - node_id: int, partition_id: Optional[int] # pylint: disable=unused-argument -) -> Client: +def get_dummy_client(context: Context) -> Client: """Return a DummyClient converted to Client type.""" - return DummyClient(node_id).to_client() + return DummyClient(context.node_id).to_client() def prep( actor_type: Type[VirtualClientEngineActor] = ClientAppActor, -) -> Tuple[List[RayActorClientProxy], VirtualClientEngineActorPool]: # pragma: no cover +) -> Tuple[ + List[RayActorClientProxy], VirtualClientEngineActorPool, NodeToPartitionMapping +]: # pragma: no cover """Prepare ClientProxies and pool for tests.""" client_resources = {"num_cpus": 1, "num_gpus": 0.0} @@ -101,7 +104,7 @@ def create_actor_fn() -> Type[VirtualClientEngineActor]: for node_id, partition_id in mapping.items() ] - return proxies, pool + return proxies, pool, mapping def test_cid_consistency_one_at_a_time() -> None: @@ -109,7 +112,7 @@ def test_cid_consistency_one_at_a_time() -> None: Submit one job and waits for completion. Then submits the next and so on """ - proxies, _ = prep() + proxies, _, _ = prep() getproperties_ins = _get_valid_getpropertiesins() recordset = getpropertiesins_to_recordset(getproperties_ins) @@ -139,7 +142,7 @@ def test_cid_consistency_all_submit_first_run_consistency() -> None: All jobs are submitted at the same time. Then fetched one at a time. This also tests NodeState (at each Proxy) and RunState basic functionality. """ - proxies, _ = prep() + proxies, _, _ = prep() run_id = 0 getproperties_ins = _get_valid_getpropertiesins() @@ -186,9 +189,8 @@ def test_cid_consistency_all_submit_first_run_consistency() -> None: def test_cid_consistency_without_proxies() -> None: """Test cid consistency of jobs submitted/retrieved to/from pool w/o ClientProxy.""" - proxies, pool = prep() - num_clients = len(proxies) - node_ids = list(range(num_clients)) + _, pool, mapping = prep() + node_ids = list(mapping.keys()) getproperties_ins = _get_valid_getpropertiesins() recordset = getpropertiesins_to_recordset(getproperties_ins) @@ -219,11 +221,11 @@ def _load_app() -> ClientApp: message, str(node_id), Context( - node_id=0, + node_id=node_id, node_config={}, state=RecordSet(), run_config={}, - partition_id=node_id, + partition_id=mapping[node_id], ), ), ) From 76244be78d9b49f01bbbcb5027c22c3d3044826f Mon Sep 17 00:00:00 2001 From: Javier Date: Sat, 13 Jul 2024 10:31:07 +0200 Subject: [PATCH 31/74] refactor(framework) Remove `partition_id` from `Context` (#3792) --- src/py/flwr/client/app.py | 2 - src/py/flwr/client/node_state.py | 6 +-- src/py/flwr/client/node_state_tests.py | 2 +- src/py/flwr/common/constant.py | 3 ++ src/py/flwr/common/context.py | 9 +---- .../superlink/fleet/vce/backend/raybackend.py | 3 +- .../fleet/vce/backend/raybackend_test.py | 12 ++++-- .../server/superlink/fleet/vce/vce_api.py | 17 +++++++-- src/py/flwr/simulation/app.py | 1 + .../ray_transport/ray_client_proxy.py | 18 ++++++--- .../ray_transport/ray_client_proxy_test.py | 38 +++++++++++-------- 11 files changed, 68 insertions(+), 43 deletions(-) diff --git a/src/py/flwr/client/app.py b/src/py/flwr/client/app.py index 380185ed26e7..700ac85f341f 100644 --- a/src/py/flwr/client/app.py +++ b/src/py/flwr/client/app.py @@ -348,7 +348,6 @@ def _on_backoff(retry_state: RetryState) -> None: node_state = NodeState( node_id=-1, node_config={}, - partition_id=None, ) else: # Call create_node fn to register node @@ -360,7 +359,6 @@ def _on_backoff(retry_state: RetryState) -> None: node_state = NodeState( node_id=node_id, node_config=node_config, - partition_id=None, ) app_state_tracker.register_signal_handler() diff --git a/src/py/flwr/client/node_state.py b/src/py/flwr/client/node_state.py index d0a349b0cae0..393ca4564a35 100644 --- a/src/py/flwr/client/node_state.py +++ b/src/py/flwr/client/node_state.py @@ -36,12 +36,13 @@ class NodeState: """State of a node where client nodes execute runs.""" def __init__( - self, node_id: int, node_config: Dict[str, str], partition_id: Optional[int] + self, + node_id: int, + node_config: Dict[str, str], ) -> None: self.node_id = node_id self.node_config = node_config self.run_infos: Dict[int, RunInfo] = {} - self._partition_id = partition_id def register_context( self, @@ -59,7 +60,6 @@ def register_context( node_config=self.node_config, state=RecordSet(), run_config=initial_run_config.copy(), - partition_id=self._partition_id, ), ) diff --git a/src/py/flwr/client/node_state_tests.py b/src/py/flwr/client/node_state_tests.py index 8d7971fa5280..26ac4fea6855 100644 --- a/src/py/flwr/client/node_state_tests.py +++ b/src/py/flwr/client/node_state_tests.py @@ -41,7 +41,7 @@ def test_multirun_in_node_state() -> None: expected_values = {0: "1", 1: "1" * 3, 2: "1" * 2, 3: "1", 5: "1"} # NodeState - node_state = NodeState(node_id=0, node_config={}, partition_id=None) + node_state = NodeState(node_id=0, node_config={}) for task in tasks: run_id = task.run_id diff --git a/src/py/flwr/common/constant.py b/src/py/flwr/common/constant.py index f14959589458..72256a62add7 100644 --- a/src/py/flwr/common/constant.py +++ b/src/py/flwr/common/constant.py @@ -57,6 +57,9 @@ FAB_CONFIG_FILE = "pyproject.toml" FLWR_HOME = "FLWR_HOME" +# Constants entries in Node config for Simulation +PARTITION_ID_KEY = "partition-id" +NUM_PARTITIONS_KEY = "num-partitions" GRPC_ADAPTER_METADATA_FLOWER_VERSION_KEY = "flower-version" GRPC_ADAPTER_METADATA_SHOULD_EXIT_KEY = "should-exit" diff --git a/src/py/flwr/common/context.py b/src/py/flwr/common/context.py index e65300278c84..4da52ba44481 100644 --- a/src/py/flwr/common/context.py +++ b/src/py/flwr/common/context.py @@ -16,7 +16,7 @@ from dataclasses import dataclass -from typing import Dict, Optional +from typing import Dict from .record import RecordSet @@ -43,17 +43,12 @@ class Context: A config (key/value mapping) held by the entity in a given run and that will stay local. It can be used at any point during the lifecycle of this entity (e.g. across multiple rounds) - partition_id : Optional[int] (default: None) - An index that specifies the data partition that the ClientApp using this Context - object should make use of. Setting this attribute is better suited for - simulation or proto typing setups. """ node_id: int node_config: Dict[str, str] state: RecordSet run_config: Dict[str, str] - partition_id: Optional[int] def __init__( # pylint: disable=too-many-arguments self, @@ -61,10 +56,8 @@ def __init__( # pylint: disable=too-many-arguments node_config: Dict[str, str], state: RecordSet, run_config: Dict[str, str], - partition_id: Optional[int] = None, ) -> None: self.node_id = node_id self.node_config = node_config self.state = state self.run_config = run_config - self.partition_id = partition_id diff --git a/src/py/flwr/server/superlink/fleet/vce/backend/raybackend.py b/src/py/flwr/server/superlink/fleet/vce/backend/raybackend.py index 0d2f4d193f0b..0ab29a234f88 100644 --- a/src/py/flwr/server/superlink/fleet/vce/backend/raybackend.py +++ b/src/py/flwr/server/superlink/fleet/vce/backend/raybackend.py @@ -21,6 +21,7 @@ import ray from flwr.client.client_app import ClientApp +from flwr.common.constant import PARTITION_ID_KEY from flwr.common.context import Context from flwr.common.logger import log from flwr.common.message import Message @@ -168,7 +169,7 @@ def process_message( Return output message and updated context. """ - partition_id = context.partition_id + partition_id = context.node_config[PARTITION_ID_KEY] try: # Submit a task to the pool diff --git a/src/py/flwr/server/superlink/fleet/vce/backend/raybackend_test.py b/src/py/flwr/server/superlink/fleet/vce/backend/raybackend_test.py index 3abdac7a232b..a38cff96ceef 100644 --- a/src/py/flwr/server/superlink/fleet/vce/backend/raybackend_test.py +++ b/src/py/flwr/server/superlink/fleet/vce/backend/raybackend_test.py @@ -23,6 +23,7 @@ from flwr.client import Client, NumPyClient from flwr.client.client_app import ClientApp, LoadClientAppError +from flwr.client.node_state import NodeState from flwr.common import ( DEFAULT_TTL, Config, @@ -32,9 +33,9 @@ Message, MessageTypeLegacy, Metadata, - RecordSet, Scalar, ) +from flwr.common.constant import PARTITION_ID_KEY from flwr.common.object_ref import load_app from flwr.common.recordset_compat import getpropertiesins_to_recordset from flwr.server.superlink.fleet.vce.backend.backend import BackendConfig @@ -101,12 +102,13 @@ def _create_message_and_context() -> Tuple[Message, Context, float]: # Construct a Message mult_factor = 2024 + run_id = 0 getproperties_ins = GetPropertiesIns(config={"factor": mult_factor}) recordset = getpropertiesins_to_recordset(getproperties_ins) message = Message( content=recordset, metadata=Metadata( - run_id=0, + run_id=run_id, message_id="", group_id="", src_node_id=0, @@ -117,8 +119,10 @@ def _create_message_and_context() -> Tuple[Message, Context, float]: ), ) - # Construct emtpy Context - context = Context(node_id=0, node_config={}, state=RecordSet(), run_config={}) + # Construct NodeState and retrieve context + node_state = NodeState(node_id=run_id, node_config={PARTITION_ID_KEY: str(0)}) + node_state.register_context(run_id=run_id) + context = node_state.retrieve_context(run_id=run_id) # Expected output expected_output = pi * mult_factor diff --git a/src/py/flwr/server/superlink/fleet/vce/vce_api.py b/src/py/flwr/server/superlink/fleet/vce/vce_api.py index 422148489b4a..cd30c40167c5 100644 --- a/src/py/flwr/server/superlink/fleet/vce/vce_api.py +++ b/src/py/flwr/server/superlink/fleet/vce/vce_api.py @@ -29,7 +29,12 @@ from flwr.client.client_app import ClientApp, ClientAppException, LoadClientAppError from flwr.client.node_state import NodeState -from flwr.common.constant import PING_MAX_INTERVAL, ErrorCode +from flwr.common.constant import ( + NUM_PARTITIONS_KEY, + PARTITION_ID_KEY, + PING_MAX_INTERVAL, + ErrorCode, +) from flwr.common.logger import log from flwr.common.message import Error from flwr.common.object_ref import load_app @@ -73,7 +78,7 @@ def worker( task_ins: TaskIns = taskins_queue.get(timeout=1.0) node_id = task_ins.task.consumer.node_id - # Register and retrieve runstate + # Register and retrieve context node_states[node_id].register_context(run_id=task_ins.run_id) context = node_states[node_id].retrieve_context(run_id=task_ins.run_id) @@ -283,11 +288,15 @@ def start_vce( # Construct mapping of NodeStates node_states: Dict[int, NodeState] = {} + # Number of unique partitions + num_partitions = len(set(nodes_mapping.values())) for node_id, partition_id in nodes_mapping.items(): node_states[node_id] = NodeState( node_id=node_id, - node_config={"partition-id": str(partition_id)}, - partition_id=None, + node_config={ + PARTITION_ID_KEY: str(partition_id), + NUM_PARTITIONS_KEY: str(num_partitions), + }, ) # Load backend config diff --git a/src/py/flwr/simulation/app.py b/src/py/flwr/simulation/app.py index 446b0bdeba38..fc52267f9a04 100644 --- a/src/py/flwr/simulation/app.py +++ b/src/py/flwr/simulation/app.py @@ -327,6 +327,7 @@ def update_resources(f_stop: threading.Event) -> None: client_fn=client_fn, node_id=node_id, partition_id=partition_id, + num_partitions=num_clients, actor_pool=pool, ) initialized_server.client_manager().register(client=client_proxy) diff --git a/src/py/flwr/simulation/ray_transport/ray_client_proxy.py b/src/py/flwr/simulation/ray_transport/ray_client_proxy.py index b62e04aeed79..895272c2fd79 100644 --- a/src/py/flwr/simulation/ray_transport/ray_client_proxy.py +++ b/src/py/flwr/simulation/ray_transport/ray_client_proxy.py @@ -24,7 +24,12 @@ from flwr.client.client_app import ClientApp from flwr.client.node_state import NodeState from flwr.common import DEFAULT_TTL, Message, Metadata, RecordSet -from flwr.common.constant import MessageType, MessageTypeLegacy +from flwr.common.constant import ( + NUM_PARTITIONS_KEY, + PARTITION_ID_KEY, + MessageType, + MessageTypeLegacy, +) from flwr.common.logger import log from flwr.common.recordset_compat import ( evaluateins_to_recordset, @@ -43,11 +48,12 @@ class RayActorClientProxy(ClientProxy): """Flower client proxy which delegates work using Ray.""" - def __init__( + def __init__( # pylint: disable=too-many-arguments self, client_fn: ClientFnExt, node_id: int, partition_id: int, + num_partitions: int, actor_pool: VirtualClientEngineActorPool, ): super().__init__(cid=str(node_id)) @@ -61,8 +67,10 @@ def _load_app() -> ClientApp: self.actor_pool = actor_pool self.proxy_state = NodeState( node_id=node_id, - node_config={"partition-id": str(partition_id)}, - partition_id=None, + node_config={ + PARTITION_ID_KEY: str(partition_id), + NUM_PARTITIONS_KEY: str(num_partitions), + }, ) def _submit_job(self, message: Message, timeout: Optional[float]) -> Message: @@ -74,7 +82,7 @@ def _submit_job(self, message: Message, timeout: Optional[float]) -> Message: # Retrieve context context = self.proxy_state.retrieve_context(run_id=run_id) - partition_id_str = context.node_config["partition-id"] + partition_id_str = context.node_config[PARTITION_ID_KEY] try: self.actor_pool.submit_client_job( diff --git a/src/py/flwr/simulation/ray_transport/ray_client_proxy_test.py b/src/py/flwr/simulation/ray_transport/ray_client_proxy_test.py index 1d44ea1d8d2b..62e0cfd61c99 100644 --- a/src/py/flwr/simulation/ray_transport/ray_client_proxy_test.py +++ b/src/py/flwr/simulation/ray_transport/ray_client_proxy_test.py @@ -23,6 +23,7 @@ from flwr.client import Client, NumPyClient from flwr.client.client_app import ClientApp +from flwr.client.node_state import NodeState from flwr.common import ( DEFAULT_TTL, Config, @@ -31,9 +32,9 @@ Message, MessageTypeLegacy, Metadata, - RecordSet, Scalar, ) +from flwr.common.constant import NUM_PARTITIONS_KEY, PARTITION_ID_KEY from flwr.common.recordset_compat import ( getpropertiesins_to_recordset, recordset_to_getpropertiesres, @@ -99,6 +100,7 @@ def create_actor_fn() -> Type[VirtualClientEngineActor]: client_fn=get_dummy_client, node_id=node_id, partition_id=partition_id, + num_partitions=num_proxies, actor_pool=pool, ) for node_id, partition_id in mapping.items() @@ -192,6 +194,17 @@ def test_cid_consistency_without_proxies() -> None: _, pool, mapping = prep() node_ids = list(mapping.keys()) + # register node states + node_states: Dict[int, NodeState] = {} + for node_id, partition_id in mapping.items(): + node_states[node_id] = NodeState( + node_id=node_id, + node_config={ + PARTITION_ID_KEY: str(partition_id), + NUM_PARTITIONS_KEY: str(len(node_ids)), + }, + ) + getproperties_ins = _get_valid_getpropertiesins() recordset = getpropertiesins_to_recordset(getproperties_ins) @@ -200,11 +213,12 @@ def _load_app() -> ClientApp: # submit all jobs (collect later) shuffle(node_ids) + run_id = 0 for node_id in node_ids: message = Message( content=recordset, metadata=Metadata( - run_id=0, + run_id=run_id, message_id="", group_id=str(0), src_node_id=0, @@ -214,26 +228,20 @@ def _load_app() -> ClientApp: message_type=MessageTypeLegacy.GET_PROPERTIES, ), ) + # register and retrieve context + node_states[node_id].register_context(run_id=run_id) + context = node_states[node_id].retrieve_context(run_id=run_id) + partition_id_str = context.node_config[PARTITION_ID_KEY] pool.submit_client_job( lambda a, c_fn, j_fn, nid_, state: a.run.remote(c_fn, j_fn, nid_, state), - ( - _load_app, - message, - str(node_id), - Context( - node_id=node_id, - node_config={}, - state=RecordSet(), - run_config={}, - partition_id=mapping[node_id], - ), - ), + (_load_app, message, partition_id_str, context), ) # fetch results one at a time shuffle(node_ids) for node_id in node_ids: - message_out, _ = pool.get_client_result(str(node_id), timeout=None) + partition_id_str = str(mapping[node_id]) + message_out, _ = pool.get_client_result(partition_id_str, timeout=None) res = recordset_to_getpropertiesres(message_out.content) assert node_id * pi == res.properties["result"] From ca48ae1bc99ca7b55b99c262e1ad3747d596ade5 Mon Sep 17 00:00:00 2001 From: Daniel Nata Nugraha Date: Sat, 13 Jul 2024 16:29:36 +0200 Subject: [PATCH 32/74] ci(*:skip) Update client_fn args in e2e tests (#3775) Co-authored-by: Daniel J. Beutel --- e2e/bare-client-auth/client.py | 9 +++++---- e2e/bare-https/client.py | 11 ++++++----- e2e/bare/client.py | 14 ++++++-------- e2e/docker/client.py | 3 ++- e2e/framework-fastai/client.py | 11 ++++++----- e2e/framework-jax/client.py | 13 ++++++------- e2e/framework-opacus/client.py | 11 ++++++----- e2e/framework-pandas/client.py | 11 ++++++----- e2e/framework-pytorch-lightning/client.py | 11 ++++++----- e2e/framework-pytorch/client.py | 12 ++++++------ e2e/framework-scikit-learn/client.py | 13 ++++++------- e2e/framework-tensorflow/client.py | 13 ++++++------- e2e/strategies/client.py | 13 ++++++------- e2e/strategies/test.py | 12 ++++++------ 14 files changed, 79 insertions(+), 78 deletions(-) diff --git a/e2e/bare-client-auth/client.py b/e2e/bare-client-auth/client.py index e82f17088bd9..c7b0d59b8ea5 100644 --- a/e2e/bare-client-auth/client.py +++ b/e2e/bare-client-auth/client.py @@ -1,13 +1,14 @@ import numpy as np -import flwr as fl +from flwr.client import ClientApp, NumPyClient +from flwr.common import Context model_params = np.array([1]) objective = 5 # Define Flower client -class FlowerClient(fl.client.NumPyClient): +class FlowerClient(NumPyClient): def get_parameters(self, config): return model_params @@ -23,10 +24,10 @@ def evaluate(self, parameters, config): return loss, 1, {"accuracy": accuracy} -def client_fn(cid): +def client_fn(context: Context): return FlowerClient().to_client() -app = fl.client.ClientApp( +app = ClientApp( client_fn=client_fn, ) diff --git a/e2e/bare-https/client.py b/e2e/bare-https/client.py index 8f5c1412fd01..4a682af3aec3 100644 --- a/e2e/bare-https/client.py +++ b/e2e/bare-https/client.py @@ -2,14 +2,15 @@ import numpy as np -import flwr as fl +from flwr.client import ClientApp, NumPyClient, start_client +from flwr.common import Context model_params = np.array([1]) objective = 5 # Define Flower client -class FlowerClient(fl.client.NumPyClient): +class FlowerClient(NumPyClient): def get_parameters(self, config): return model_params @@ -25,17 +26,17 @@ def evaluate(self, parameters, config): return loss, 1, {"accuracy": accuracy} -def client_fn(cid): +def client_fn(context: Context): return FlowerClient().to_client() -app = fl.client.ClientApp( +app = ClientApp( client_fn=client_fn, ) if __name__ == "__main__": # Start Flower client - fl.client.start_client( + start_client( server_address="127.0.0.1:8080", client=FlowerClient().to_client(), root_certificates=Path("certificates/ca.crt").read_bytes(), diff --git a/e2e/bare/client.py b/e2e/bare/client.py index 402d775ac3a9..943e60d5db9f 100644 --- a/e2e/bare/client.py +++ b/e2e/bare/client.py @@ -2,8 +2,8 @@ import numpy as np -import flwr as fl -from flwr.common import ConfigsRecord +from flwr.client import ClientApp, NumPyClient, start_client +from flwr.common import ConfigsRecord, Context SUBSET_SIZE = 1000 STATE_VAR = "timestamp" @@ -14,7 +14,7 @@ # Define Flower client -class FlowerClient(fl.client.NumPyClient): +class FlowerClient(NumPyClient): def get_parameters(self, config): return model_params @@ -51,16 +51,14 @@ def evaluate(self, parameters, config): ) -def client_fn(cid): +def client_fn(context: Context): return FlowerClient().to_client() -app = fl.client.ClientApp( +app = ClientApp( client_fn=client_fn, ) if __name__ == "__main__": # Start Flower client - fl.client.start_client( - server_address="127.0.0.1:8080", client=FlowerClient().to_client() - ) + start_client(server_address="127.0.0.1:8080", client=FlowerClient().to_client()) diff --git a/e2e/docker/client.py b/e2e/docker/client.py index 8451b810416b..44313c7c3af6 100644 --- a/e2e/docker/client.py +++ b/e2e/docker/client.py @@ -9,6 +9,7 @@ from torchvision.transforms import Compose, Normalize, ToTensor from flwr.client import ClientApp, NumPyClient +from flwr.common import Context # ############################################################################# # 1. Regular PyTorch pipeline: nn.Module, train, test, and DataLoader @@ -122,7 +123,7 @@ def evaluate(self, parameters, config): return loss, len(testloader.dataset), {"accuracy": accuracy} -def client_fn(cid: str): +def client_fn(context: Context): """Create and return an instance of Flower `Client`.""" return FlowerClient().to_client() diff --git a/e2e/framework-fastai/client.py b/e2e/framework-fastai/client.py index 1d98a1134941..161b27b5a548 100644 --- a/e2e/framework-fastai/client.py +++ b/e2e/framework-fastai/client.py @@ -5,7 +5,8 @@ import torch from fastai.vision.all import * -import flwr as fl +from flwr.client import ClientApp, NumPyClient, start_client +from flwr.common import Context warnings.filterwarnings("ignore", category=UserWarning) @@ -29,7 +30,7 @@ # Define Flower client -class FlowerClient(fl.client.NumPyClient): +class FlowerClient(NumPyClient): def get_parameters(self, config): return [val.cpu().numpy() for _, val in learn.model.state_dict().items()] @@ -49,18 +50,18 @@ def evaluate(self, parameters, config): return loss, len(dls.valid), {"accuracy": 1 - error_rate} -def client_fn(cid): +def client_fn(context: Context): return FlowerClient().to_client() -app = fl.client.ClientApp( +app = ClientApp( client_fn=client_fn, ) if __name__ == "__main__": # Start Flower client - fl.client.start_client( + start_client( server_address="127.0.0.1:8080", client=FlowerClient().to_client(), ) diff --git a/e2e/framework-jax/client.py b/e2e/framework-jax/client.py index 347a005d923a..c9ff67b3e38e 100644 --- a/e2e/framework-jax/client.py +++ b/e2e/framework-jax/client.py @@ -6,7 +6,8 @@ import jax_training import numpy as np -import flwr as fl +from flwr.client import ClientApp, NumPyClient, start_client +from flwr.common import Context # Load data and determine model shape train_x, train_y, test_x, test_y = jax_training.load_data() @@ -14,7 +15,7 @@ model_shape = train_x.shape[1:] -class FlowerClient(fl.client.NumPyClient): +class FlowerClient(NumPyClient): def __init__(self): self.params = jax_training.load_model(model_shape) @@ -48,16 +49,14 @@ def evaluate( return float(loss), num_examples, {"loss": float(loss)} -def client_fn(cid): +def client_fn(context: Context): return FlowerClient().to_client() -app = fl.client.ClientApp( +app = ClientApp( client_fn=client_fn, ) if __name__ == "__main__": # Start Flower client - fl.client.start_client( - server_address="127.0.0.1:8080", client=FlowerClient().to_client() - ) + start_client(server_address="127.0.0.1:8080", client=FlowerClient().to_client()) diff --git a/e2e/framework-opacus/client.py b/e2e/framework-opacus/client.py index c9ebe319063a..167fa4584e37 100644 --- a/e2e/framework-opacus/client.py +++ b/e2e/framework-opacus/client.py @@ -9,7 +9,8 @@ from torch.utils.data import DataLoader from torchvision.datasets import CIFAR10 -import flwr as fl +from flwr.client import ClientApp, NumPyClient, start_client +from flwr.common import Context # Define parameters. PARAMS = { @@ -95,7 +96,7 @@ def load_data(): # Define Flower client. -class FlowerClient(fl.client.NumPyClient): +class FlowerClient(NumPyClient): def __init__(self, model) -> None: super().__init__() # Create a privacy engine which will add DP and keep track of the privacy budget. @@ -139,16 +140,16 @@ def evaluate(self, parameters, config): return float(loss), len(testloader), {"accuracy": float(accuracy)} -def client_fn(cid): +def client_fn(context: Context): model = Net() return FlowerClient(model).to_client() -app = fl.client.ClientApp( +app = ClientApp( client_fn=client_fn, ) if __name__ == "__main__": - fl.client.start_client( + start_client( server_address="127.0.0.1:8080", client=FlowerClient(model).to_client() ) diff --git a/e2e/framework-pandas/client.py b/e2e/framework-pandas/client.py index 19e15f5a3b11..0c3300e1dd3f 100644 --- a/e2e/framework-pandas/client.py +++ b/e2e/framework-pandas/client.py @@ -3,7 +3,8 @@ import numpy as np import pandas as pd -import flwr as fl +from flwr.client import ClientApp, NumPyClient, start_client +from flwr.common import Context df = pd.read_csv("./data/client.csv") @@ -16,7 +17,7 @@ def compute_hist(df: pd.DataFrame, col_name: str) -> np.ndarray: # Define Flower client -class FlowerClient(fl.client.NumPyClient): +class FlowerClient(NumPyClient): def fit( self, parameters: List[np.ndarray], config: Dict[str, str] ) -> Tuple[List[np.ndarray], int, Dict]: @@ -32,17 +33,17 @@ def fit( ) -def client_fn(cid): +def client_fn(context: Context): return FlowerClient().to_client() -app = fl.client.ClientApp( +app = ClientApp( client_fn=client_fn, ) if __name__ == "__main__": # Start Flower client - fl.client.start_client( + start_client( server_address="127.0.0.1:8080", client=FlowerClient().to_client(), ) diff --git a/e2e/framework-pytorch-lightning/client.py b/e2e/framework-pytorch-lightning/client.py index fdd55b3dc344..bf291a1ca2c5 100644 --- a/e2e/framework-pytorch-lightning/client.py +++ b/e2e/framework-pytorch-lightning/client.py @@ -4,10 +4,11 @@ import pytorch_lightning as pl import torch -import flwr as fl +from flwr.client import ClientApp, NumPyClient, start_client +from flwr.common import Context -class FlowerClient(fl.client.NumPyClient): +class FlowerClient(NumPyClient): def __init__(self, model, train_loader, val_loader, test_loader): self.model = model self.train_loader = train_loader @@ -51,7 +52,7 @@ def _set_parameters(model, parameters): model.load_state_dict(state_dict, strict=True) -def client_fn(cid): +def client_fn(context: Context): model = mnist.LitAutoEncoder() train_loader, val_loader, test_loader = mnist.load_data() @@ -59,7 +60,7 @@ def client_fn(cid): return FlowerClient(model, train_loader, val_loader, test_loader).to_client() -app = fl.client.ClientApp( +app = ClientApp( client_fn=client_fn, ) @@ -71,7 +72,7 @@ def main() -> None: # Flower client client = FlowerClient(model, train_loader, val_loader, test_loader).to_client() - fl.client.start_client(server_address="127.0.0.1:8080", client=client) + start_client(server_address="127.0.0.1:8080", client=client) if __name__ == "__main__": diff --git a/e2e/framework-pytorch/client.py b/e2e/framework-pytorch/client.py index dbfbfed1ffa7..ab4bc7b5c5b9 100644 --- a/e2e/framework-pytorch/client.py +++ b/e2e/framework-pytorch/client.py @@ -10,8 +10,8 @@ from torchvision.transforms import Compose, Normalize, ToTensor from tqdm import tqdm -import flwr as fl -from flwr.common import ConfigsRecord +from flwr.client import ClientApp, NumPyClient, start_client +from flwr.common import ConfigsRecord, Context # ############################################################################# # 1. Regular PyTorch pipeline: nn.Module, train, test, and DataLoader @@ -89,7 +89,7 @@ def load_data(): # Define Flower client -class FlowerClient(fl.client.NumPyClient): +class FlowerClient(NumPyClient): def get_parameters(self, config): return [val.cpu().numpy() for _, val in net.state_dict().items()] @@ -136,18 +136,18 @@ def set_parameters(model, parameters): return -def client_fn(cid): +def client_fn(context: Context): return FlowerClient().to_client() -app = fl.client.ClientApp( +app = ClientApp( client_fn=client_fn, ) if __name__ == "__main__": # Start Flower client - fl.client.start_client( + start_client( server_address="127.0.0.1:8080", client=FlowerClient().to_client(), ) diff --git a/e2e/framework-scikit-learn/client.py b/e2e/framework-scikit-learn/client.py index b0691e75a79d..24c6617c1289 100644 --- a/e2e/framework-scikit-learn/client.py +++ b/e2e/framework-scikit-learn/client.py @@ -5,7 +5,8 @@ from sklearn.linear_model import LogisticRegression from sklearn.metrics import log_loss -import flwr as fl +from flwr.client import ClientApp, NumPyClient, start_client +from flwr.common import Context # Load MNIST dataset from https://www.openml.org/d/554 (X_train, y_train), (X_test, y_test) = utils.load_mnist() @@ -26,7 +27,7 @@ # Define Flower client -class FlowerClient(fl.client.NumPyClient): +class FlowerClient(NumPyClient): def get_parameters(self, config): # type: ignore return utils.get_model_parameters(model) @@ -45,16 +46,14 @@ def evaluate(self, parameters, config): # type: ignore return loss, len(X_test), {"accuracy": accuracy} -def client_fn(cid): +def client_fn(context: Context): return FlowerClient().to_client() -app = fl.client.ClientApp( +app = ClientApp( client_fn=client_fn, ) if __name__ == "__main__": # Start Flower client - fl.client.start_client( - server_address="0.0.0.0:8080", client=FlowerClient().to_client() - ) + start_client(server_address="0.0.0.0:8080", client=FlowerClient().to_client()) diff --git a/e2e/framework-tensorflow/client.py b/e2e/framework-tensorflow/client.py index 779be0c3746d..351f495a3acb 100644 --- a/e2e/framework-tensorflow/client.py +++ b/e2e/framework-tensorflow/client.py @@ -2,7 +2,8 @@ import tensorflow as tf -import flwr as fl +from flwr.client import ClientApp, NumPyClient, start_client +from flwr.common import Context SUBSET_SIZE = 1000 @@ -18,7 +19,7 @@ # Define Flower client -class FlowerClient(fl.client.NumPyClient): +class FlowerClient(NumPyClient): def get_parameters(self, config): return model.get_weights() @@ -33,16 +34,14 @@ def evaluate(self, parameters, config): return loss, len(x_test), {"accuracy": accuracy} -def client_fn(cid): +def client_fn(context: Context): return FlowerClient().to_client() -app = fl.client.ClientApp( +app = ClientApp( client_fn=client_fn, ) if __name__ == "__main__": # Start Flower client - fl.client.start_client( - server_address="127.0.0.1:8080", client=FlowerClient().to_client() - ) + start_client(server_address="127.0.0.1:8080", client=FlowerClient().to_client()) diff --git a/e2e/strategies/client.py b/e2e/strategies/client.py index 505340e013a5..0403416cc3b7 100644 --- a/e2e/strategies/client.py +++ b/e2e/strategies/client.py @@ -2,7 +2,8 @@ import tensorflow as tf -import flwr as fl +from flwr.client import ClientApp, NumPyClient, start_client +from flwr.common import Context SUBSET_SIZE = 1000 @@ -33,7 +34,7 @@ def get_model(): # Define Flower client -class FlowerClient(fl.client.NumPyClient): +class FlowerClient(NumPyClient): def get_parameters(self, config): return model.get_weights() @@ -48,17 +49,15 @@ def evaluate(self, parameters, config): return loss, len(x_test), {"accuracy": accuracy} -def client_fn(cid): +def client_fn(context: Context): return FlowerClient().to_client() -app = fl.client.ClientApp( +app = ClientApp( client_fn=client_fn, ) if __name__ == "__main__": # Start Flower client - fl.client.start_client( - server_address="127.0.0.1:8080", client=FlowerClient().to_client() - ) + start_client(server_address="127.0.0.1:8080", client=FlowerClient().to_client()) diff --git a/e2e/strategies/test.py b/e2e/strategies/test.py index abf9cdb5a5c7..c567f33b236b 100644 --- a/e2e/strategies/test.py +++ b/e2e/strategies/test.py @@ -3,8 +3,8 @@ import tensorflow as tf from client import SUBSET_SIZE, FlowerClient, get_model -import flwr as fl -from flwr.common import ndarrays_to_parameters +from flwr.common import Context, ndarrays_to_parameters +from flwr.server import ServerConfig from flwr.server.strategy import ( FaultTolerantFedAvg, FedAdagrad, @@ -15,6 +15,7 @@ FedYogi, QFedAvg, ) +from flwr.simulation import start_simulation STRATEGY_LIST = [ FedMedian, @@ -42,8 +43,7 @@ def get_strat(name): init_model = get_model() -def client_fn(cid): - _ = cid +def client_fn(context: Context): return FlowerClient() @@ -71,10 +71,10 @@ def evaluate(server_round, parameters, config): if start_idx >= OPT_IDX: strat_args["tau"] = 0.01 -hist = fl.simulation.start_simulation( +hist = start_simulation( client_fn=client_fn, num_clients=2, - config=fl.server.ServerConfig(num_rounds=3), + config=ServerConfig(num_rounds=3), strategy=strategy(**strat_args), ) From 57907597ab024c89cfb8537f238368c7961b048f Mon Sep 17 00:00:00 2001 From: "Daniel J. Beutel" Date: Sat, 13 Jul 2024 17:23:36 +0200 Subject: [PATCH 33/74] docs(framework) Update client_fn docstrings to new signature (#3793) --- src/py/flwr/client/app.py | 2 +- src/py/flwr/client/client_app.py | 2 +- src/py/flwr/simulation/app.py | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/py/flwr/client/app.py b/src/py/flwr/client/app.py index 700ac85f341f..348ef8910dd3 100644 --- a/src/py/flwr/client/app.py +++ b/src/py/flwr/client/app.py @@ -139,7 +139,7 @@ class `flwr.client.Client` (default: None) Starting an SSL-enabled gRPC client using system certificates: >>> def client_fn(context: Context): - >>> return FlowerClient() + >>> return FlowerClient().to_client() >>> >>> start_client( >>> server_address=localhost:8080, diff --git a/src/py/flwr/client/client_app.py b/src/py/flwr/client/client_app.py index 9566302d0721..2a913b3a248d 100644 --- a/src/py/flwr/client/client_app.py +++ b/src/py/flwr/client/client_app.py @@ -91,7 +91,7 @@ class ClientApp: >>> class FlowerClient(NumPyClient): >>> # ... >>> - >>> def client_fn(node_id: int, partition_id: Optional[int]): + >>> def client_fn(context: Context): >>> return FlowerClient().to_client() >>> >>> app = ClientApp(client_fn) diff --git a/src/py/flwr/simulation/app.py b/src/py/flwr/simulation/app.py index fc52267f9a04..973a9a89e652 100644 --- a/src/py/flwr/simulation/app.py +++ b/src/py/flwr/simulation/app.py @@ -111,9 +111,9 @@ def start_simulation( Parameters ---------- client_fn : ClientFnExt - A function creating Client instances. The function must have the signature - `client_fn(node_id: int, partition_id: Optional[int]). It should return - a single client instance of type Client. Note that the created client + A function creating `Client` instances. The function must have the signature + `client_fn(context: Context). It should return + a single client instance of type `Client`. Note that the created client instances are ephemeral and will often be destroyed after a single method invocation. Since client instances are not long-lived, they should not attempt to carry state over method invocations. Any state required by the instance From e8bf5f84d6a72c842367a655848e55527aac32fe Mon Sep 17 00:00:00 2001 From: Javier Date: Sat, 13 Jul 2024 22:31:25 +0200 Subject: [PATCH 34/74] refactor(framework) Update `flwr new` templates with new `client_fn` signature (#3795) Co-authored-by: Daniel J. Beutel --- src/py/flwr/cli/new/templates/app/code/client.hf.py.tpl | 8 ++++++-- src/py/flwr/cli/new/templates/app/code/client.jax.py.tpl | 3 ++- src/py/flwr/cli/new/templates/app/code/client.mlx.py.tpl | 7 +++++-- .../flwr/cli/new/templates/app/code/client.numpy.py.tpl | 3 ++- .../flwr/cli/new/templates/app/code/client.pytorch.py.tpl | 7 +++++-- .../flwr/cli/new/templates/app/code/client.sklearn.py.tpl | 6 ++++-- .../cli/new/templates/app/code/client.tensorflow.py.tpl | 7 +++++-- src/py/flwr/cli/new/templates/app/code/task.hf.py.tpl | 4 ++-- src/py/flwr/cli/new/templates/app/code/task.mlx.py.tpl | 4 ++-- .../flwr/cli/new/templates/app/code/task.pytorch.py.tpl | 2 +- 10 files changed, 34 insertions(+), 17 deletions(-) diff --git a/src/py/flwr/cli/new/templates/app/code/client.hf.py.tpl b/src/py/flwr/cli/new/templates/app/code/client.hf.py.tpl index 314da2120c53..56bac8543c50 100644 --- a/src/py/flwr/cli/new/templates/app/code/client.hf.py.tpl +++ b/src/py/flwr/cli/new/templates/app/code/client.hf.py.tpl @@ -1,6 +1,7 @@ """$project_name: A Flower / HuggingFace Transformers app.""" from flwr.client import ClientApp, NumPyClient +from flwr.common import Context from transformers import AutoModelForSequenceClassification from $import_name.task import ( @@ -38,12 +39,15 @@ class FlowerClient(NumPyClient): return float(loss), len(self.testloader), {"accuracy": accuracy} -def client_fn(cid): +def client_fn(context: Context): # Load model and data net = AutoModelForSequenceClassification.from_pretrained( CHECKPOINT, num_labels=2 ).to(DEVICE) - trainloader, valloader = load_data(int(cid), 2) + + partition_id = int(context.node_config['partition-id']) + num_partitions = int(context.node_config['num-partitions]) + trainloader, valloader = load_data(partition_id, num_partitions) # Return Client instance return FlowerClient(net, trainloader, valloader).to_client() diff --git a/src/py/flwr/cli/new/templates/app/code/client.jax.py.tpl b/src/py/flwr/cli/new/templates/app/code/client.jax.py.tpl index 3c6d2f03637a..48b667665f3f 100644 --- a/src/py/flwr/cli/new/templates/app/code/client.jax.py.tpl +++ b/src/py/flwr/cli/new/templates/app/code/client.jax.py.tpl @@ -2,6 +2,7 @@ import jax from flwr.client import NumPyClient, ClientApp +from flwr.common import Context from $import_name.task import ( evaluation, @@ -44,7 +45,7 @@ class FlowerClient(NumPyClient): ) return float(loss), num_examples, {"loss": float(loss)} -def client_fn(cid): +def client_fn(context: Context): # Return Client instance return FlowerClient().to_client() diff --git a/src/py/flwr/cli/new/templates/app/code/client.mlx.py.tpl b/src/py/flwr/cli/new/templates/app/code/client.mlx.py.tpl index 1722561370a8..37207c940d83 100644 --- a/src/py/flwr/cli/new/templates/app/code/client.mlx.py.tpl +++ b/src/py/flwr/cli/new/templates/app/code/client.mlx.py.tpl @@ -4,6 +4,7 @@ import mlx.core as mx import mlx.nn as nn import mlx.optimizers as optim from flwr.client import NumPyClient, ClientApp +from flwr.common import Context from $import_name.task import ( batch_iterate, @@ -57,8 +58,10 @@ class FlowerClient(NumPyClient): return loss.item(), len(self.test_images), {"accuracy": accuracy.item()} -def client_fn(cid): - data = load_data(int(cid), 2) +def client_fn(context: Context): + partition_id = int(context.node_config["partition-id"]) + num_partitions = int(context.node_config["num-partitions"]) + data = load_data(partition_id, num_partitions) # Return Client instance return FlowerClient(data).to_client() diff --git a/src/py/flwr/cli/new/templates/app/code/client.numpy.py.tpl b/src/py/flwr/cli/new/templates/app/code/client.numpy.py.tpl index 232c305fc2a9..1dd83e108bb5 100644 --- a/src/py/flwr/cli/new/templates/app/code/client.numpy.py.tpl +++ b/src/py/flwr/cli/new/templates/app/code/client.numpy.py.tpl @@ -1,6 +1,7 @@ """$project_name: A Flower / NumPy app.""" from flwr.client import NumPyClient, ClientApp +from flwr.common import Context import numpy as np @@ -15,7 +16,7 @@ class FlowerClient(NumPyClient): return float(0.0), 1, {"accuracy": float(1.0)} -def client_fn(cid: str): +def client_fn(context: Context): return FlowerClient().to_client() diff --git a/src/py/flwr/cli/new/templates/app/code/client.pytorch.py.tpl b/src/py/flwr/cli/new/templates/app/code/client.pytorch.py.tpl index c68974efaadf..addc71023a09 100644 --- a/src/py/flwr/cli/new/templates/app/code/client.pytorch.py.tpl +++ b/src/py/flwr/cli/new/templates/app/code/client.pytorch.py.tpl @@ -1,6 +1,7 @@ """$project_name: A Flower / PyTorch app.""" from flwr.client import NumPyClient, ClientApp +from flwr.common import Context from $import_name.task import ( Net, @@ -31,10 +32,12 @@ class FlowerClient(NumPyClient): return loss, len(self.valloader.dataset), {"accuracy": accuracy} -def client_fn(cid): +def client_fn(context: Context): # Load model and data net = Net().to(DEVICE) - trainloader, valloader = load_data(int(cid), 2) + partition_id = int(context.node_config["partition-id"]) + num_partitions = int(context.node_config["num-partitions"]) + trainloader, valloader = load_data(partition_id, num_partitions) # Return Client instance return FlowerClient(net, trainloader, valloader).to_client() diff --git a/src/py/flwr/cli/new/templates/app/code/client.sklearn.py.tpl b/src/py/flwr/cli/new/templates/app/code/client.sklearn.py.tpl index 9181389cad1c..a1eefa034e7b 100644 --- a/src/py/flwr/cli/new/templates/app/code/client.sklearn.py.tpl +++ b/src/py/flwr/cli/new/templates/app/code/client.sklearn.py.tpl @@ -4,6 +4,7 @@ import warnings import numpy as np from flwr.client import NumPyClient, ClientApp +from flwr.common import Context from flwr_datasets import FederatedDataset from sklearn.linear_model import LogisticRegression from sklearn.metrics import log_loss @@ -68,8 +69,9 @@ class FlowerClient(NumPyClient): fds = FederatedDataset(dataset="mnist", partitioners={"train": 2}) -def client_fn(cid: str): - dataset = fds.load_partition(int(cid), "train").with_format("numpy") +def client_fn(context: Context): + partition_id = int(context.node_config["partition-id"]) + dataset = fds.load_partition(partition_id, "train").with_format("numpy") X, y = dataset["image"].reshape((len(dataset), -1)), dataset["label"] diff --git a/src/py/flwr/cli/new/templates/app/code/client.tensorflow.py.tpl b/src/py/flwr/cli/new/templates/app/code/client.tensorflow.py.tpl index dc55d4ca6569..0fe1c405a110 100644 --- a/src/py/flwr/cli/new/templates/app/code/client.tensorflow.py.tpl +++ b/src/py/flwr/cli/new/templates/app/code/client.tensorflow.py.tpl @@ -1,6 +1,7 @@ """$project_name: A Flower / TensorFlow app.""" from flwr.client import NumPyClient, ClientApp +from flwr.common import Context from $import_name.task import load_data, load_model @@ -28,10 +29,12 @@ class FlowerClient(NumPyClient): return loss, len(self.x_test), {"accuracy": accuracy} -def client_fn(cid): +def client_fn(context: Context): # Load model and data net = load_model() - x_train, y_train, x_test, y_test = load_data(int(cid), 2) + + partition_id = int(context.node_config["partition-id"]) + x_train, y_train, x_test, y_test = load_data(partition_id, 2) # Return Client instance return FlowerClient(net, x_train, y_train, x_test, y_test).to_client() diff --git a/src/py/flwr/cli/new/templates/app/code/task.hf.py.tpl b/src/py/flwr/cli/new/templates/app/code/task.hf.py.tpl index 8e89add66835..eb43acfce976 100644 --- a/src/py/flwr/cli/new/templates/app/code/task.hf.py.tpl +++ b/src/py/flwr/cli/new/templates/app/code/task.hf.py.tpl @@ -16,9 +16,9 @@ DEVICE = torch.device("cpu") CHECKPOINT = "distilbert-base-uncased" # transformer model checkpoint -def load_data(partition_id, num_clients): +def load_data(partition_id: int, num_partitions: int): """Load IMDB data (training and eval)""" - fds = FederatedDataset(dataset="imdb", partitioners={"train": num_clients}) + fds = FederatedDataset(dataset="imdb", partitioners={"train": num_partitions}) partition = fds.load_partition(partition_id) # Divide data: 80% train, 20% test partition_train_test = partition.train_test_split(test_size=0.2, seed=42) diff --git a/src/py/flwr/cli/new/templates/app/code/task.mlx.py.tpl b/src/py/flwr/cli/new/templates/app/code/task.mlx.py.tpl index bcd4dde93310..88053b0cd590 100644 --- a/src/py/flwr/cli/new/templates/app/code/task.mlx.py.tpl +++ b/src/py/flwr/cli/new/templates/app/code/task.mlx.py.tpl @@ -43,8 +43,8 @@ def batch_iterate(batch_size, X, y): yield X[ids], y[ids] -def load_data(partition_id, num_clients): - fds = FederatedDataset(dataset="mnist", partitioners={"train": num_clients}) +def load_data(partition_id: int, num_partitions: int): + fds = FederatedDataset(dataset="mnist", partitioners={"train": num_partitions}) partition = fds.load_partition(partition_id) partition_splits = partition.train_test_split(test_size=0.2, seed=42) diff --git a/src/py/flwr/cli/new/templates/app/code/task.pytorch.py.tpl b/src/py/flwr/cli/new/templates/app/code/task.pytorch.py.tpl index b30c65a285b5..d5971ffb6ce5 100644 --- a/src/py/flwr/cli/new/templates/app/code/task.pytorch.py.tpl +++ b/src/py/flwr/cli/new/templates/app/code/task.pytorch.py.tpl @@ -34,7 +34,7 @@ class Net(nn.Module): return self.fc3(x) -def load_data(partition_id, num_partitions): +def load_data(partition_id: int, num_partitions: int): """Load partition CIFAR10 data.""" fds = FederatedDataset(dataset="cifar10", partitioners={"train": num_partitions}) partition = fds.load_partition(partition_id) From 942adfd72c42ec4f2883929d83f42717d3f17b48 Mon Sep 17 00:00:00 2001 From: Charles Beauville Date: Sun, 14 Jul 2024 13:49:35 +0200 Subject: [PATCH 35/74] feat(framework) Add SuperExec `--executor-config` (#3720) Co-authored-by: Daniel J. Beutel --- src/py/flwr/superexec/app.py | 16 ++++-- src/py/flwr/superexec/deployment.py | 86 ++++++++++++++++++++++++----- src/py/flwr/superexec/exec_grpc.py | 7 ++- src/py/flwr/superexec/executor.py | 19 ++++++- 4 files changed, 106 insertions(+), 22 deletions(-) diff --git a/src/py/flwr/superexec/app.py b/src/py/flwr/superexec/app.py index 372ccb443a76..b51c3e6821dc 100644 --- a/src/py/flwr/superexec/app.py +++ b/src/py/flwr/superexec/app.py @@ -24,6 +24,7 @@ from flwr.common import EventType, event, log from flwr.common.address import parse_address +from flwr.common.config import parse_config_args from flwr.common.constant import SUPEREXEC_DEFAULT_ADDRESS from flwr.common.exit_handlers import register_exit_handlers from flwr.common.object_ref import load_app, validate @@ -55,6 +56,7 @@ def run_superexec() -> None: address=address, executor=_load_executor(args), certificates=certificates, + config=parse_config_args(args.executor_config), ) grpc_servers = [superexec_server] @@ -74,21 +76,25 @@ def _parse_args_run_superexec() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( description="Start a Flower SuperExec", ) - parser.add_argument( - "executor", - help="For example: `deployment:exec` or `project.package.module:wrapper.exec`.", - default="flwr.superexec.deployment:executor", - ) parser.add_argument( "--address", help="SuperExec (gRPC) server address (IPv4, IPv6, or a domain name)", default=SUPEREXEC_DEFAULT_ADDRESS, ) + parser.add_argument( + "--executor", + help="For example: `deployment:exec` or `project.package.module:wrapper.exec`.", + default="flwr.superexec.deployment:executor", + ) parser.add_argument( "--executor-dir", help="The directory for the executor.", default=".", ) + parser.add_argument( + "--executor-config", + help="Key-value pairs for the executor config, separated by commas.", + ) parser.add_argument( "--insecure", action="store_true", diff --git a/src/py/flwr/superexec/deployment.py b/src/py/flwr/superexec/deployment.py index d117f280b38d..f9a272e6b0bf 100644 --- a/src/py/flwr/superexec/deployment.py +++ b/src/py/flwr/superexec/deployment.py @@ -17,6 +17,7 @@ import subprocess import sys from logging import ERROR, INFO +from pathlib import Path from typing import Dict, Optional from typing_extensions import override @@ -33,25 +34,73 @@ class DeploymentEngine(Executor): - """Deployment engine executor.""" + """Deployment engine executor. + + Parameters + ---------- + superlink: str (default: "0.0.0.0:9091") + Address of the SuperLink to connect to. + root_certificates: Optional[str] (default: None) + Specifies the path to the PEM-encoded root certificate file for + establishing secure HTTPS connections. + flwr_dir: Optional[str] (default: None) + The path containing installed Flower Apps. + """ def __init__( self, - address: str = DEFAULT_SERVER_ADDRESS_DRIVER, - root_certificates: Optional[bytes] = None, + superlink: str = DEFAULT_SERVER_ADDRESS_DRIVER, + root_certificates: Optional[str] = None, + flwr_dir: Optional[str] = None, ) -> None: - self.address = address - self.root_certificates = root_certificates + self.superlink = superlink + if root_certificates is None: + self.root_certificates = None + self.root_certificates_bytes = None + else: + self.root_certificates = root_certificates + self.root_certificates_bytes = Path(root_certificates).read_bytes() + self.flwr_dir = flwr_dir self.stub: Optional[DriverStub] = None + @override + def set_config( + self, + config: Dict[str, str], + ) -> None: + """Set executor config arguments. + + Parameters + ---------- + config : Dict[str, str] + A dictionary for configuration values. + Supported configuration key/value pairs: + - "superlink": str + The address of the SuperLink Driver API. + - "root-certificates": str + The path to the root certificates. + - "flwr-dir": str + The path to the Flower directory. + """ + if not config: + return + if superlink_address := config.get("superlink"): + self.superlink = superlink_address + if root_certificates := config.get("root-certificates"): + self.root_certificates = root_certificates + self.root_certificates_bytes = Path(root_certificates).read_bytes() + if flwr_dir := config.get("flwr-dir"): + self.flwr_dir = flwr_dir + def _connect(self) -> None: - if self.stub is None: - channel = create_channel( - server_address=self.address, - insecure=(self.root_certificates is None), - root_certificates=self.root_certificates, - ) - self.stub = DriverStub(channel) + if self.stub is not None: + return + channel = create_channel( + server_address=self.superlink, + insecure=(self.root_certificates_bytes is None), + root_certificates=self.root_certificates_bytes, + ) + self.stub = DriverStub(channel) def _create_run( self, @@ -74,7 +123,9 @@ def _create_run( @override def start_run( - self, fab_file: bytes, override_config: Dict[str, str] + self, + fab_file: bytes, + override_config: Dict[str, str], ) -> Optional[RunTracker]: """Start run using the Flower Deployment Engine.""" try: @@ -99,7 +150,14 @@ def start_run( "flower-server-app", "--run-id", str(run_id), - "--insecure", + f"--flwr-dir {self.flwr_dir}" if self.flwr_dir else "", + "--superlink", + self.superlink, + ( + "--insecure" + if self.root_certificates is None + else f"--root-certificates {self.root_certificates}" + ), ], stdout=subprocess.PIPE, stderr=subprocess.PIPE, diff --git a/src/py/flwr/superexec/exec_grpc.py b/src/py/flwr/superexec/exec_grpc.py index 127d5615dd84..d90cec3e47cd 100644 --- a/src/py/flwr/superexec/exec_grpc.py +++ b/src/py/flwr/superexec/exec_grpc.py @@ -15,7 +15,7 @@ """SuperExec gRPC API.""" from logging import INFO -from typing import Optional, Tuple +from typing import Dict, Optional, Tuple import grpc @@ -32,8 +32,11 @@ def run_superexec_api_grpc( address: str, executor: Executor, certificates: Optional[Tuple[bytes, bytes, bytes]], + config: Dict[str, str], ) -> grpc.Server: """Run SuperExec API (gRPC, request-response).""" + executor.set_config(config) + exec_servicer: grpc.Server = ExecServicer( executor=executor, ) @@ -45,7 +48,7 @@ def run_superexec_api_grpc( certificates=certificates, ) - log(INFO, "Flower ECE: Starting SuperExec API (gRPC-rere) on %s", address) + log(INFO, "Starting Flower SuperExec gRPC server on %s", address) superexec_grpc_server.start() return superexec_grpc_server diff --git a/src/py/flwr/superexec/executor.py b/src/py/flwr/superexec/executor.py index 85b6e5c3e095..62d64f366cec 100644 --- a/src/py/flwr/superexec/executor.py +++ b/src/py/flwr/superexec/executor.py @@ -31,9 +31,24 @@ class RunTracker: class Executor(ABC): """Execute and monitor a Flower run.""" + @abstractmethod + def set_config( + self, + config: Dict[str, str], + ) -> None: + """Register provided config as class attributes. + + Parameters + ---------- + config : Optional[Dict[str, str]] + A dictionary for configuration values. + """ + @abstractmethod def start_run( - self, fab_file: bytes, override_config: Dict[str, str] + self, + fab_file: bytes, + override_config: Dict[str, str], ) -> Optional[RunTracker]: """Start a run using the given Flower FAB ID and version. @@ -44,6 +59,8 @@ def start_run( ---------- fab_file : bytes The Flower App Bundle file bytes. + override_config: Dict[str, str] + The config overrides dict sent by the user (using `flwr run`). Returns ------- From 3b5963f1b5d44ac8b4fe87c540e03d7e7de78ec4 Mon Sep 17 00:00:00 2001 From: Javier Date: Sun, 14 Jul 2024 14:18:56 +0200 Subject: [PATCH 36/74] refactor(framework) Update `flwr new` templates with new `server_fn` signature (#3796) Co-authored-by: Daniel J. Beutel --- .../new/templates/app/code/server.hf.py.tpl | 24 +++++++++-------- .../new/templates/app/code/server.jax.py.tpl | 20 ++++++++------ .../new/templates/app/code/server.mlx.py.tpl | 15 ++++++----- .../templates/app/code/server.numpy.py.tpl | 20 ++++++++------ .../templates/app/code/server.pytorch.py.tpl | 27 +++++++++---------- .../templates/app/code/server.sklearn.py.tpl | 23 +++++++++------- .../app/code/server.tensorflow.py.tpl | 26 +++++++++--------- 7 files changed, 84 insertions(+), 71 deletions(-) diff --git a/src/py/flwr/cli/new/templates/app/code/server.hf.py.tpl b/src/py/flwr/cli/new/templates/app/code/server.hf.py.tpl index d7d86931335b..039ea8619532 100644 --- a/src/py/flwr/cli/new/templates/app/code/server.hf.py.tpl +++ b/src/py/flwr/cli/new/templates/app/code/server.hf.py.tpl @@ -1,17 +1,19 @@ """$project_name: A Flower / HuggingFace Transformers app.""" +from flwr.common import Context from flwr.server.strategy import FedAvg -from flwr.server import ServerApp, ServerConfig +from flwr.server import ServerApp, ServerAppComponents, ServerConfig -# Define strategy -strategy = FedAvg( - fraction_fit=1.0, - fraction_evaluate=1.0, -) +def server_fn(context: Context): + # Define strategy + strategy = FedAvg( + fraction_fit=1.0, + fraction_evaluate=1.0, + ) + config = ServerConfig(num_rounds=3) -# Start server -app = ServerApp( - config=ServerConfig(num_rounds=3), - strategy=strategy, -) + return ServerAppComponents(strategy=strategy, config=config) + +# Create ServerApp +app = ServerApp(server_fn=server_fn) diff --git a/src/py/flwr/cli/new/templates/app/code/server.jax.py.tpl b/src/py/flwr/cli/new/templates/app/code/server.jax.py.tpl index 53cff7b905f4..122b884ab8bb 100644 --- a/src/py/flwr/cli/new/templates/app/code/server.jax.py.tpl +++ b/src/py/flwr/cli/new/templates/app/code/server.jax.py.tpl @@ -1,12 +1,16 @@ """$project_name: A Flower / JAX app.""" -import flwr as fl +from flwr.common import Context +from flwr.server.strategy import FedAvg +from flwr.server import ServerApp, ServerAppComponents, ServerConfig -# Configure the strategy -strategy = fl.server.strategy.FedAvg() -# Flower ServerApp -app = fl.server.ServerApp( - config=fl.server.ServerConfig(num_rounds=3), - strategy=strategy, -) +def server_fn(context: Context): + # Define strategy + strategy = FedAvg() + config = ServerConfig(num_rounds=3) + + return ServerAppComponents(strategy=strategy, config=config) + +# Create ServerApp +app = ServerApp(server_fn=server_fn) diff --git a/src/py/flwr/cli/new/templates/app/code/server.mlx.py.tpl b/src/py/flwr/cli/new/templates/app/code/server.mlx.py.tpl index b475e0e7dc36..403c68ac3405 100644 --- a/src/py/flwr/cli/new/templates/app/code/server.mlx.py.tpl +++ b/src/py/flwr/cli/new/templates/app/code/server.mlx.py.tpl @@ -1,15 +1,16 @@ """$project_name: A Flower / MLX app.""" -from flwr.server import ServerApp, ServerConfig +from flwr.common import Context +from flwr.server import ServerApp, ServerAppComponents, ServerConfig from flwr.server.strategy import FedAvg -# Define strategy -strategy = FedAvg() +def server_fn(context: Context): + # Define strategy + strategy = FedAvg() + config = ServerConfig(num_rounds=3) + return ServerAppComponents(strategy=strategy, config=config) # Create ServerApp -app = ServerApp( - config=ServerConfig(num_rounds=3), - strategy=strategy, -) +app = ServerApp(server_fn=server_fn) diff --git a/src/py/flwr/cli/new/templates/app/code/server.numpy.py.tpl b/src/py/flwr/cli/new/templates/app/code/server.numpy.py.tpl index 03f95ae35cfd..1ed2d36339db 100644 --- a/src/py/flwr/cli/new/templates/app/code/server.numpy.py.tpl +++ b/src/py/flwr/cli/new/templates/app/code/server.numpy.py.tpl @@ -1,12 +1,16 @@ """$project_name: A Flower / NumPy app.""" -import flwr as fl +from flwr.common import Context +from flwr.server import ServerApp, ServerAppComponents, ServerConfig +from flwr.server.strategy import FedAvg -# Configure the strategy -strategy = fl.server.strategy.FedAvg() -# Flower ServerApp -app = fl.server.ServerApp( - config=fl.server.ServerConfig(num_rounds=1), - strategy=strategy, -) +def server_fn(context: Context): + # Define strategy + strategy = FedAvg() + config = ServerConfig(num_rounds=3) + + return ServerAppComponents(strategy=strategy, config=config) + +# Create ServerApp +app = ServerApp(server_fn=server_fn) diff --git a/src/py/flwr/cli/new/templates/app/code/server.pytorch.py.tpl b/src/py/flwr/cli/new/templates/app/code/server.pytorch.py.tpl index dc635f79a664..3638b9eba7b0 100644 --- a/src/py/flwr/cli/new/templates/app/code/server.pytorch.py.tpl +++ b/src/py/flwr/cli/new/templates/app/code/server.pytorch.py.tpl @@ -1,7 +1,7 @@ """$project_name: A Flower / PyTorch app.""" -from flwr.common import ndarrays_to_parameters -from flwr.server import ServerApp, ServerConfig +from flwr.common import Context, ndarrays_to_parameters +from flwr.server import ServerApp, ServerAppComponents, ServerConfig from flwr.server.strategy import FedAvg from $import_name.task import Net, get_weights @@ -11,18 +11,17 @@ from $import_name.task import Net, get_weights ndarrays = get_weights(Net()) parameters = ndarrays_to_parameters(ndarrays) +def server_fn(context: Context): + # Define strategy + strategy = FedAvg( + fraction_fit=1.0, + fraction_evaluate=1.0, + min_available_clients=2, + initial_parameters=parameters, + ) + config = ServerConfig(num_rounds=3) -# Define strategy -strategy = FedAvg( - fraction_fit=1.0, - fraction_evaluate=1.0, - min_available_clients=2, - initial_parameters=parameters, -) - + return ServerAppComponents(strategy=strategy, config=config) # Create ServerApp -app = ServerApp( - config=ServerConfig(num_rounds=3), - strategy=strategy, -) +app = ServerApp(server_fn=server_fn) diff --git a/src/py/flwr/cli/new/templates/app/code/server.sklearn.py.tpl b/src/py/flwr/cli/new/templates/app/code/server.sklearn.py.tpl index 266a53ac5794..2e463e8da09e 100644 --- a/src/py/flwr/cli/new/templates/app/code/server.sklearn.py.tpl +++ b/src/py/flwr/cli/new/templates/app/code/server.sklearn.py.tpl @@ -1,17 +1,20 @@ """$project_name: A Flower / Scikit-Learn app.""" -from flwr.server import ServerApp, ServerConfig +from flwr.common import Context +from flwr.server import ServerApp, ServerAppComponents, ServerConfig from flwr.server.strategy import FedAvg -strategy = FedAvg( - fraction_fit=1.0, - fraction_evaluate=1.0, - min_available_clients=2, -) +def server_fn(context: Context): + # Define strategy + strategy = FedAvg( + fraction_fit=1.0, + fraction_evaluate=1.0, + min_available_clients=2, + ) + config = ServerConfig(num_rounds=3) + + return ServerAppComponents(strategy=strategy, config=config) # Create ServerApp -app = ServerApp( - config=ServerConfig(num_rounds=3), - strategy=strategy, -) +app = ServerApp(server_fn=server_fn) diff --git a/src/py/flwr/cli/new/templates/app/code/server.tensorflow.py.tpl b/src/py/flwr/cli/new/templates/app/code/server.tensorflow.py.tpl index 8d092164a468..eee727ba9025 100644 --- a/src/py/flwr/cli/new/templates/app/code/server.tensorflow.py.tpl +++ b/src/py/flwr/cli/new/templates/app/code/server.tensorflow.py.tpl @@ -1,7 +1,7 @@ """$project_name: A Flower / TensorFlow app.""" -from flwr.common import ndarrays_to_parameters -from flwr.server import ServerApp, ServerConfig +from flwr.common import Context, ndarrays_to_parameters +from flwr.server import ServerApp, ServerAppComponents, ServerConfig from flwr.server.strategy import FedAvg from $import_name.task import load_model @@ -11,17 +11,17 @@ config = ServerConfig(num_rounds=3) parameters = ndarrays_to_parameters(load_model().get_weights()) -# Define strategy -strategy = FedAvg( - fraction_fit=1.0, - fraction_evaluate=1.0, - min_available_clients=2, - initial_parameters=parameters, -) +def server_fn(context: Context): + # Define strategy + strategy = strategy = FedAvg( + fraction_fit=1.0, + fraction_evaluate=1.0, + min_available_clients=2, + initial_parameters=parameters, + ) + config = ServerConfig(num_rounds=3) + return ServerAppComponents(strategy=strategy, config=config) # Create ServerApp -app = ServerApp( - config=config, - strategy=strategy, -) +app = ServerApp(server_fn=server_fn) From 6aff69d7b24825f4b413d30ce76b647d0898f202 Mon Sep 17 00:00:00 2001 From: Charles Beauville Date: Sun, 14 Jul 2024 14:49:30 +0200 Subject: [PATCH 37/74] fix(framework:skip) Remove dependency installation from SuperExec (#3781) Co-authored-by: Daniel J. Beutel --- src/py/flwr/superexec/deployment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/py/flwr/superexec/deployment.py b/src/py/flwr/superexec/deployment.py index f9a272e6b0bf..3a3bc3bf2b1e 100644 --- a/src/py/flwr/superexec/deployment.py +++ b/src/py/flwr/superexec/deployment.py @@ -135,7 +135,7 @@ def start_run( # Install FAB Python package subprocess.check_call( - [sys.executable, "-m", "pip", "install", str(fab_path)], + [sys.executable, "-m", "pip", "install", "--no-deps", str(fab_path)], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) From ba20db4b72141e7a3ccdfe49133d84f00ea3180b Mon Sep 17 00:00:00 2001 From: "Daniel J. Beutel" Date: Sun, 14 Jul 2024 15:46:41 +0200 Subject: [PATCH 38/74] refactor(framework) Rename `--config` to `--run-config` (#3798) --- src/py/flwr/cli/run/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/py/flwr/cli/run/run.py b/src/py/flwr/cli/run/run.py index 4ee2368f5794..b23ba3f7d0cf 100644 --- a/src/py/flwr/cli/run/run.py +++ b/src/py/flwr/cli/run/run.py @@ -62,7 +62,7 @@ def run( config_overrides: Annotated[ Optional[str], typer.Option( - "--config", + "--run-config", "-c", help="Override configuration key-value pairs", ), From b0de25d6bfd1e8438b2ac876ad16d6852aa61570 Mon Sep 17 00:00:00 2001 From: "Daniel J. Beutel" Date: Sun, 14 Jul 2024 22:27:28 +0200 Subject: [PATCH 39/74] docs(framework) Document public/private API approach (#3562) --- ...or-explanation-public-and-private-apis.rst | 118 ++++++++++++++++++ doc/source/index.rst | 1 + 2 files changed, 119 insertions(+) create mode 100644 doc/source/contributor-explanation-public-and-private-apis.rst diff --git a/doc/source/contributor-explanation-public-and-private-apis.rst b/doc/source/contributor-explanation-public-and-private-apis.rst new file mode 100644 index 000000000000..1dfdf88f97d3 --- /dev/null +++ b/doc/source/contributor-explanation-public-and-private-apis.rst @@ -0,0 +1,118 @@ +Public and private APIs +======================= + +In Python, everything is public. +To enable developers to understand which components can be relied upon, Flower declares a public API. +Components that are part of the public API can be relied upon. +Changes to the public API are announced in the release notes and are subject to deprecation policies. + +Everything that is not part of the public API is part of the private API. +Even though Python allows accessing them, user code should never use those components. +Private APIs can change at any time, even in patch releases. + +How can you determine whether a component is part of the public API or not? Easy: + +- `Use the Flower API reference documentation `_ +- `Use the Flower CLI reference documentation `_ + +Everything listed in the reference documentation is part of the public API. +This document explains how Flower maintainers define the public API and how you can determine whether a component is part of the public API or not by reading the Flower source code. + +Flower public API +----------------- + +Flower has a well-defined public API. Let's look at this in more detail. + +.. important:: + + Every component that is reachable by recursively following ``__init__.__all__`` starting from the root package (``flwr``) is part of the public API. + +If you want to determine whether a component (class/function/generator/...) is part of the public API or not, you need to start at the root of the ``flwr`` package. +Let's use ``tree -L 1 -d src/py/flwr`` to look at the Python sub-packages contained ``flwr``: + +.. code-block:: bash + + flwr + ├── cli + ├── client + ├── common + ├── proto + ├── server + └── simulation + +Contrast this with the definition of ``__all__`` in the root ``src/py/flwr/__init__.py``: + +.. code-block:: python + + # From `flwr/__init__.py` + __all__ = [ + "client", + "common", + "server", + "simulation", + ] + +You can see that ``flwr`` has six subpackages (``cli``, ``client``, ``common``, ``proto``, ``server``, ``simulation``), but only four of them are "exported" via ``__all__`` (``client``, ``common``, ``server``, ``simulation``). + +What does this mean? It means that ``client``, ``common``, ``server`` and ``simulation`` are part of the public API, but ``cli`` and ``proto`` are not. +The ``flwr`` subpackages ``cli`` and ``proto`` are private APIs. +A private API can change completely from one release to the next (even in patch releases). +It can change in a breaking way, it can be renamed (for example, ``flwr.cli`` could be renamed to ``flwr.command``) and it can even be removed completely. + +Therefore, as a Flower user: + +- ``from flwr import client`` ✅ Ok, you're importing a public API. +- ``from flwr import proto`` ❌ Not recommended, you're importing a private API. + +What about components that are nested deeper in the hierarchy? Let's look at Flower strategies to see another typical pattern. +Flower strategies like ``FedAvg`` are often imported using ``from flwr.server.strategy import FedAvg``. +Let's look at ``src/py/flwr/server/strategy/__init__.py``: + +.. code-block:: python + + from .fedavg import FedAvg as FedAvg + # ... more imports + + __all__ = [ + "FedAvg", + # ... more exports + ] + +What's notable here is that all strategies are implemented in dedicated modules (e.g., ``fedavg.py``). +In ``__init__.py``, we *import* the components we want to make part of the public API and then *export* them via ``__all__``. +Note that we export the component itself (for example, the ``FedAvg`` class), but not the module it is defined in (for example, ``fedavg.py``). +This allows us to move the definition of ``FedAvg`` into a different module (or even a module in a subpackage) without breaking the public API (as long as we update the import path in ``__init__.py``). + +Therefore: + +- ``from flwr.server.strategy import FedAvg`` ✅ Ok, you're importing a class that is part of the public API. +- ``from flwr.server.strategy import fedavg`` ❌ Not recommended, you're importing a private module. + +This approach is also implemented in the tooling that automatically builds API reference docs. + +Flower public API of private packages +------------------------------------- + +We also use this to define the public API of private subpackages. +Public, in this context, means the API that other ``flwr`` subpackages should use. +For example, ``flwr.server.driver`` is a private subpackage (it's not exported via ``src/py/flwr/server/__init__.py``'s ``__all__``). + +Still, the private sub-package ``flwr.server.driver`` defines a "public" API using ``__all__`` in ``src/py/flwr/server/driver/__init__.py``: + +.. code-block:: python + + from .driver import Driver + from .grpc_driver import GrpcDriver + from .inmemory_driver import InMemoryDriver + + __all__ = [ + "Driver", + "GrpcDriver", + "InMemoryDriver", + ] + +The interesting part is that both ``GrpcDriver`` and ``InMemoryDriver`` are never used by Flower framework users, only by other parts of the Flower framework codebase. +Those other parts of the codebase import, for example, ``InMemoryDriver`` using ``from flwr.server.driver import InMemoryDriver`` (i.e., the ``InMemoryDriver`` exported via ``__all__``), not ``from flwr.server.driver.in_memory_driver import InMemoryDriver`` (``in_memory_driver.py`` is the module containing the actual ``InMemoryDriver`` class definition). + +This is because ``flwr.server.driver`` defines a public interface for other ``flwr`` subpackages. +This allows codeowners of ``flwr.server.driver`` to refactor the package without breaking other ``flwr``-internal users. diff --git a/doc/source/index.rst b/doc/source/index.rst index f62c5ebf4786..a0115620fce9 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -174,6 +174,7 @@ The Flower community welcomes contributions. The following docs are intended to :caption: Contributor explanations contributor-explanation-architecture + contributor-explanation-public-and-private-apis .. toctree:: :maxdepth: 1 From 0e32d8370d31f17d6a8199531e9d64fb80a7e570 Mon Sep 17 00:00:00 2001 From: Charles Beauville Date: Sun, 14 Jul 2024 22:43:38 +0200 Subject: [PATCH 40/74] fix(framework:skip) Use correct arguments (#3799) --- src/py/flwr/superexec/deployment.py | 34 +++++++++++++++++------------ 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/src/py/flwr/superexec/deployment.py b/src/py/flwr/superexec/deployment.py index 3a3bc3bf2b1e..bbe7882692f0 100644 --- a/src/py/flwr/superexec/deployment.py +++ b/src/py/flwr/superexec/deployment.py @@ -144,21 +144,27 @@ def start_run( run_id: int = self._create_run(fab_id, fab_version, override_config) log(INFO, "Created run %s", str(run_id)) - # Start ServerApp + command = [ + "flower-server-app", + "--run-id", + str(run_id), + "--superlink", + str(self.superlink), + ] + + if self.flwr_dir: + command.append("--flwr-dir") + command.append(self.flwr_dir) + + if self.root_certificates is None: + command.append("--insecure") + else: + command.append("--root-certificates") + command.append(self.root_certificates) + + # Execute the command proc = subprocess.Popen( # pylint: disable=consider-using-with - [ - "flower-server-app", - "--run-id", - str(run_id), - f"--flwr-dir {self.flwr_dir}" if self.flwr_dir else "", - "--superlink", - self.superlink, - ( - "--insecure" - if self.root_certificates is None - else f"--root-certificates {self.root_certificates}" - ), - ], + command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, From 27ee0db4b3e9332faab27675ffb83ac896415c44 Mon Sep 17 00:00:00 2001 From: Javier Date: Mon, 15 Jul 2024 10:12:54 +0200 Subject: [PATCH 41/74] feat(framework) Add simulation engine `SuperExec` plugin (#3589) Co-authored-by: Charles Beauville Co-authored-by: Daniel J. Beutel --- src/py/flwr/superexec/simulation.py | 157 ++++++++++++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 src/py/flwr/superexec/simulation.py diff --git a/src/py/flwr/superexec/simulation.py b/src/py/flwr/superexec/simulation.py new file mode 100644 index 000000000000..9a8e19365ab9 --- /dev/null +++ b/src/py/flwr/superexec/simulation.py @@ -0,0 +1,157 @@ +# Copyright 2024 Flower Labs GmbH. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Simulation engine executor.""" + + +import subprocess +import sys +from logging import ERROR, INFO, WARN +from typing import Dict, Optional + +from typing_extensions import override + +from flwr.cli.config_utils import load_and_validate +from flwr.cli.install import install_from_fab +from flwr.common.constant import RUN_ID_NUM_BYTES +from flwr.common.logger import log +from flwr.server.superlink.state.utils import generate_rand_int_from_bytes + +from .executor import Executor, RunTracker + + +class SimulationEngine(Executor): + """Simulation engine executor. + + Parameters + ---------- + num_supernodes: Opitonal[str] (default: None) + Total number of nodes to involve in the simulation. + """ + + def __init__( + self, + num_supernodes: Optional[str] = None, + ) -> None: + self.num_supernodes = num_supernodes + + @override + def set_config( + self, + config: Dict[str, str], + ) -> None: + """Set executor config arguments. + + Parameters + ---------- + config : Dict[str, str] + A dictionary for configuration values. + Supported configuration key/value pairs: + - "num-supernodes": str + Number of nodes to register for the simulation. + """ + if not config: + return + if num_supernodes := config.get("num-supernodes"): + self.num_supernodes = num_supernodes + + # Validate config + if self.num_supernodes is None: + log( + ERROR, + "To start a run with the simulation plugin, please specify " + "the number of SuperNodes. This can be done by using the " + "`--executor-config` argument when launching the SuperExec.", + ) + raise ValueError("`num-supernodes` must not be `None`") + + @override + def start_run( + self, fab_file: bytes, override_config: Dict[str, str] + ) -> Optional[RunTracker]: + """Start run using the Flower Simulation Engine.""" + try: + if override_config: + raise ValueError( + "Overriding the run config is not yet supported with the " + "simulation executor.", + ) + + # Install FAB to flwr dir + fab_path = install_from_fab(fab_file, None, True) + + # Install FAB Python package + subprocess.check_call( + [sys.executable, "-m", "pip", "install", "--no-deps", str(fab_path)], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + # Load and validate config + config, errors, warnings = load_and_validate(fab_path / "pyproject.toml") + if errors: + raise ValueError(errors) + + if warnings: + log(WARN, warnings) + + if config is None: + raise ValueError( + "Config extracted from FAB's pyproject.toml is not valid" + ) + + # Get ClientApp and SeverApp components + flower_components = config["flower"]["components"] + clientapp = flower_components["clientapp"] + serverapp = flower_components["serverapp"] + + # In Simulation there is no SuperLink, still we create a run_id + run_id = generate_rand_int_from_bytes(RUN_ID_NUM_BYTES) + log(INFO, "Created run %s", str(run_id)) + + # Prepare commnand + command = [ + "flower-simulation", + "--client-app", + f"{clientapp}", + "--server-app", + f"{serverapp}", + "--num-supernodes", + f"{self.num_supernodes}", + "--run-id", + str(run_id), + ] + + # Start Simulation + proc = subprocess.Popen( # pylint: disable=consider-using-with + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + log(INFO, "Started run %s", str(run_id)) + + return RunTracker( + run_id=run_id, + proc=proc, + ) + + # pylint: disable-next=broad-except + except Exception as e: + log(ERROR, "Could not start run: %s", str(e)) + return None + + +executor = SimulationEngine() From 848734d1ec93352837f3746476b16a41a98609c8 Mon Sep 17 00:00:00 2001 From: Javier Date: Mon, 15 Jul 2024 16:06:16 +0200 Subject: [PATCH 42/74] refactor(framework) Replace `run_id` with `Run` in simulation (#3802) --- src/py/flwr/simulation/run_simulation.py | 48 ++++++++++++------------ 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/src/py/flwr/simulation/run_simulation.py b/src/py/flwr/simulation/run_simulation.py index de101fe3e09f..7060a972dd9a 100644 --- a/src/py/flwr/simulation/run_simulation.py +++ b/src/py/flwr/simulation/run_simulation.py @@ -26,14 +26,16 @@ from flwr.client import ClientApp from flwr.common import EventType, event, log +from flwr.common.constant import RUN_ID_NUM_BYTES from flwr.common.logger import set_logger_propagation, update_console_handler from flwr.common.typing import Run from flwr.server.driver import Driver, InMemoryDriver -from flwr.server.run_serverapp import run +from flwr.server.run_serverapp import run as run_server_app from flwr.server.server_app import ServerApp from flwr.server.superlink.fleet import vce from flwr.server.superlink.fleet.vce.backend.backend import BackendConfig from flwr.server.superlink.state import StateFactory +from flwr.server.superlink.state.utils import generate_rand_int_from_bytes from flwr.simulation.ray_transport.utils import ( enable_tf_gpu_growth as enable_gpu_growth, ) @@ -54,7 +56,11 @@ def run_simulation_from_cli() -> None: backend_name=args.backend, backend_config=backend_config_dict, app_dir=args.app_dir, - run_id=args.run_id, + run=( + Run(run_id=args.run_id, fab_id="", fab_version="", override_config={}) + if args.run_id + else None + ), enable_tf_gpu_growth=args.enable_tf_gpu_growth, verbose_logging=args.verbose, ) @@ -156,7 +162,7 @@ def server_th_with_start_checks( enable_gpu_growth() # Run ServerApp - run( + run_server_app( driver=_driver, server_app_dir=_server_app_dir, server_app_run_config=_server_app_run_config, @@ -193,16 +199,6 @@ def server_th_with_start_checks( return serverapp_th -def _override_run_id(state: StateFactory, run_id_to_replace: int, run_id: int) -> None: - """Override the run_id of an existing Run.""" - log(DEBUG, "Pre-registering run with id %s", run_id) - # Remove run - run_info: Run = state.state().run_ids.pop(run_id_to_replace) # type: ignore - # Update with new run_id and insert back in state - run_info.run_id = run_id - state.state().run_ids[run_id] = run_info # type: ignore - - # pylint: disable=too-many-locals def _main_loop( num_supernodes: int, @@ -210,7 +206,7 @@ def _main_loop( backend_config_stream: str, app_dir: str, enable_tf_gpu_growth: bool, - run_id: Optional[int] = None, + run: Run, client_app: Optional[ClientApp] = None, client_app_attr: Optional[str] = None, server_app: Optional[ServerApp] = None, @@ -225,16 +221,13 @@ def _main_loop( server_app_thread_has_exception = threading.Event() serverapp_th = None try: - # Create run (with empty fab_id and fab_version) - run_id_ = state_factory.state().create_run("", "", {}) + # Register run + log(DEBUG, "Pre-registering run with id %s", run.run_id) + state_factory.state().run_ids[run.run_id] = run # type: ignore server_app_run_config: Dict[str, str] = {} - if run_id: - _override_run_id(state_factory, run_id_to_replace=run_id_, run_id=run_id) - run_id_ = run_id - # Initialize Driver - driver = InMemoryDriver(run_id=run_id_, state_factory=state_factory) + driver = InMemoryDriver(run_id=run.run_id, state_factory=state_factory) # Get and run ServerApp thread serverapp_th = run_serverapp_th( @@ -289,7 +282,7 @@ def _run_simulation( client_app_attr: Optional[str] = None, server_app_attr: Optional[str] = None, app_dir: str = "", - run_id: Optional[int] = None, + run: Optional[Run] = None, enable_tf_gpu_growth: bool = False, verbose_logging: bool = False, ) -> None: @@ -332,8 +325,8 @@ def _run_simulation( Add specified directory to the PYTHONPATH and load `ClientApp` from there. (Default: current working directory.) - run_id : Optional[int] - An integer specifying the ID of the run started when running this function. + run : Optional[Run] + An object carrying details about the run. enable_tf_gpu_growth : bool (default: False) A boolean to indicate whether to enable GPU growth on the main thread. This is @@ -371,13 +364,18 @@ def _run_simulation( # Convert config to original JSON-stream format backend_config_stream = json.dumps(backend_config) + # If no `Run` object is set, create one + if run is None: + run_id = generate_rand_int_from_bytes(RUN_ID_NUM_BYTES) + run = Run(run_id=run_id, fab_id="", fab_version="", override_config={}) + args = ( num_supernodes, backend_name, backend_config_stream, app_dir, enable_tf_gpu_growth, - run_id, + run, client_app, client_app_attr, server_app, From ce60e52a70fca0af6e4d0a424c6eb9cb638c40e4 Mon Sep 17 00:00:00 2001 From: Javier Date: Mon, 15 Jul 2024 16:29:08 +0200 Subject: [PATCH 43/74] refactor(framework) Register `Context` early in Simulation Engine (#3804) --- .../server/superlink/fleet/vce/vce_api.py | 42 ++++++++++++------- .../superlink/fleet/vce/vce_api_test.py | 3 ++ src/py/flwr/simulation/run_simulation.py | 1 + 3 files changed, 31 insertions(+), 15 deletions(-) diff --git a/src/py/flwr/server/superlink/fleet/vce/vce_api.py b/src/py/flwr/server/superlink/fleet/vce/vce_api.py index cd30c40167c5..66bca5a391c7 100644 --- a/src/py/flwr/server/superlink/fleet/vce/vce_api.py +++ b/src/py/flwr/server/superlink/fleet/vce/vce_api.py @@ -39,6 +39,7 @@ from flwr.common.message import Error from flwr.common.object_ref import load_app from flwr.common.serde import message_from_taskins, message_to_taskres +from flwr.common.typing import Run from flwr.proto.task_pb2 import TaskIns, TaskRes # pylint: disable=E0611 from flwr.server.superlink.state import State, StateFactory @@ -60,6 +61,27 @@ def _register_nodes( return nodes_mapping +def _register_node_states( + nodes_mapping: NodeToPartitionMapping, run: Run +) -> Dict[int, NodeState]: + """Create NodeState objects and pre-register the context for the run.""" + node_states: Dict[int, NodeState] = {} + num_partitions = len(set(nodes_mapping.values())) + for node_id, partition_id in nodes_mapping.items(): + node_states[node_id] = NodeState( + node_id=node_id, + node_config={ + PARTITION_ID_KEY: str(partition_id), + NUM_PARTITIONS_KEY: str(num_partitions), + }, + ) + + # Pre-register Context objects + node_states[node_id].register_context(run_id=run.run_id, run=run) + + return node_states + + # pylint: disable=too-many-arguments,too-many-locals def worker( app_fn: Callable[[], ClientApp], @@ -78,8 +100,7 @@ def worker( task_ins: TaskIns = taskins_queue.get(timeout=1.0) node_id = task_ins.task.consumer.node_id - # Register and retrieve context - node_states[node_id].register_context(run_id=task_ins.run_id) + # Retrieve context context = node_states[node_id].retrieve_context(run_id=task_ins.run_id) # Convert TaskIns to Message @@ -151,7 +172,7 @@ def put_taskres_into_state( pass -def run( +def run_api( app_fn: Callable[[], ClientApp], backend_fn: Callable[[], Backend], nodes_mapping: NodeToPartitionMapping, @@ -237,6 +258,7 @@ def start_vce( backend_config_json_stream: str, app_dir: str, f_stop: threading.Event, + run: Run, client_app: Optional[ClientApp] = None, client_app_attr: Optional[str] = None, num_supernodes: Optional[int] = None, @@ -287,17 +309,7 @@ def start_vce( ) # Construct mapping of NodeStates - node_states: Dict[int, NodeState] = {} - # Number of unique partitions - num_partitions = len(set(nodes_mapping.values())) - for node_id, partition_id in nodes_mapping.items(): - node_states[node_id] = NodeState( - node_id=node_id, - node_config={ - PARTITION_ID_KEY: str(partition_id), - NUM_PARTITIONS_KEY: str(num_partitions), - }, - ) + node_states = _register_node_states(nodes_mapping=nodes_mapping, run=run) # Load backend config log(DEBUG, "Supported backends: %s", list(supported_backends.keys())) @@ -348,7 +360,7 @@ def _load() -> ClientApp: _ = app_fn() # Run main simulation loop - run( + run_api( app_fn, backend_fn, nodes_mapping, diff --git a/src/py/flwr/server/superlink/fleet/vce/vce_api_test.py b/src/py/flwr/server/superlink/fleet/vce/vce_api_test.py index 7d37f03f6ade..4dfc08560523 100644 --- a/src/py/flwr/server/superlink/fleet/vce/vce_api_test.py +++ b/src/py/flwr/server/superlink/fleet/vce/vce_api_test.py @@ -165,6 +165,8 @@ def start_and_shutdown( if not app_dir: app_dir = _autoresolve_app_dir() + run = Run(run_id=1234, fab_id="", fab_version="", override_config={}) + start_vce( num_supernodes=num_supernodes, client_app_attr=client_app_attr, @@ -173,6 +175,7 @@ def start_and_shutdown( state_factory=state_factory, app_dir=app_dir, f_stop=f_stop, + run=run, existing_nodes_mapping=nodes_mapping, ) diff --git a/src/py/flwr/simulation/run_simulation.py b/src/py/flwr/simulation/run_simulation.py index 7060a972dd9a..60e6e16eed27 100644 --- a/src/py/flwr/simulation/run_simulation.py +++ b/src/py/flwr/simulation/run_simulation.py @@ -252,6 +252,7 @@ def _main_loop( app_dir=app_dir, state_factory=state_factory, f_stop=f_stop, + run=run, ) except Exception as ex: From dd37449dc67435664fe2120a009b2ad389fa7691 Mon Sep 17 00:00:00 2001 From: Charles Beauville Date: Mon, 15 Jul 2024 18:39:36 +0200 Subject: [PATCH 44/74] feat(framework) Use federations config in `flwr run` (#3800) Co-authored-by: Daniel J. Beutel --- .../app/pyproject.flowertune.toml.tpl | 14 +-- .../new/templates/app/pyproject.hf.toml.tpl | 11 +- .../new/templates/app/pyproject.jax.toml.tpl | 9 +- .../new/templates/app/pyproject.mlx.toml.tpl | 11 +- .../templates/app/pyproject.numpy.toml.tpl | 11 +- .../templates/app/pyproject.pytorch.toml.tpl | 11 +- .../templates/app/pyproject.sklearn.toml.tpl | 11 +- .../app/pyproject.tensorflow.toml.tpl | 11 +- src/py/flwr/cli/run/run.py | 111 +++++++++--------- 9 files changed, 87 insertions(+), 113 deletions(-) diff --git a/src/py/flwr/cli/new/templates/app/pyproject.flowertune.toml.tpl b/src/py/flwr/cli/new/templates/app/pyproject.flowertune.toml.tpl index 2ed6bd36fd89..109cbf66a35b 100644 --- a/src/py/flwr/cli/new/templates/app/pyproject.flowertune.toml.tpl +++ b/src/py/flwr/cli/new/templates/app/pyproject.flowertune.toml.tpl @@ -6,9 +6,6 @@ build-backend = "hatchling.build" name = "$package_name" version = "1.0.0" description = "" -authors = [ - { name = "The Flower Authors", email = "hello@flower.ai" }, -] license = { text = "Apache License (2.0)" } dependencies = [ "flwr[simulation]>=1.9.0,<2.0", @@ -32,11 +29,8 @@ publisher = "$username" serverapp = "$import_name.app:server" clientapp = "$import_name.app:client" -[flower.engine] -name = "simulation" - -[flower.engine.simulation.supernode] -num = $num_clients +[flower.federations] +default = "localhost" -[flower.engine.simulation] -backend_config = { client_resources = { num_cpus = 8, num_gpus = 1.0 } } +[flower.federations.localhost] +options.num-supernodes = 10 diff --git a/src/py/flwr/cli/new/templates/app/pyproject.hf.toml.tpl b/src/py/flwr/cli/new/templates/app/pyproject.hf.toml.tpl index 71004f3421cd..6c7e50393098 100644 --- a/src/py/flwr/cli/new/templates/app/pyproject.hf.toml.tpl +++ b/src/py/flwr/cli/new/templates/app/pyproject.hf.toml.tpl @@ -6,9 +6,6 @@ build-backend = "hatchling.build" name = "$package_name" version = "1.0.0" description = "" -authors = [ - { name = "The Flower Authors", email = "hello@flower.ai" }, -] license = { text = "Apache License (2.0)" } dependencies = [ "flwr[simulation]>=1.9.0,<2.0", @@ -30,8 +27,8 @@ publisher = "$username" serverapp = "$import_name.server:app" clientapp = "$import_name.client:app" -[flower.engine] -name = "simulation" +[flower.federations] +default = "localhost" -[flower.engine.simulation.supernode] -num = 2 +[flower.federations.localhost] +options.num-supernodes = 10 diff --git a/src/py/flwr/cli/new/templates/app/pyproject.jax.toml.tpl b/src/py/flwr/cli/new/templates/app/pyproject.jax.toml.tpl index c5463e08b92c..f5c66cc729b8 100644 --- a/src/py/flwr/cli/new/templates/app/pyproject.jax.toml.tpl +++ b/src/py/flwr/cli/new/templates/app/pyproject.jax.toml.tpl @@ -6,9 +6,6 @@ build-backend = "hatchling.build" name = "$package_name" version = "1.0.0" description = "" -authors = [ - { name = "The Flower Authors", email = "hello@flower.ai" }, -] license = {text = "Apache License (2.0)"} dependencies = [ "flwr[simulation]>=1.9.0,<2.0", @@ -26,3 +23,9 @@ publisher = "$username" [flower.components] serverapp = "$import_name.server:app" clientapp = "$import_name.client:app" + +[flower.federations] +default = "localhost" + +[flower.federations.localhost] +options.num-supernodes = 10 diff --git a/src/py/flwr/cli/new/templates/app/pyproject.mlx.toml.tpl b/src/py/flwr/cli/new/templates/app/pyproject.mlx.toml.tpl index a850135a1fc5..eaeec144adb2 100644 --- a/src/py/flwr/cli/new/templates/app/pyproject.mlx.toml.tpl +++ b/src/py/flwr/cli/new/templates/app/pyproject.mlx.toml.tpl @@ -6,9 +6,6 @@ build-backend = "hatchling.build" name = "$package_name" version = "1.0.0" description = "" -authors = [ - { name = "The Flower Authors", email = "hello@flower.ai" }, -] license = { text = "Apache License (2.0)" } dependencies = [ "flwr[simulation]>=1.9.0,<2.0", @@ -27,8 +24,8 @@ publisher = "$username" serverapp = "$import_name.server:app" clientapp = "$import_name.client:app" -[flower.engine] -name = "simulation" +[flower.federations] +default = "localhost" -[flower.engine.simulation.supernode] -num = 2 +[flower.federations.localhost] +options.num-supernodes = 10 diff --git a/src/py/flwr/cli/new/templates/app/pyproject.numpy.toml.tpl b/src/py/flwr/cli/new/templates/app/pyproject.numpy.toml.tpl index d49015eb567f..6f386990ba6e 100644 --- a/src/py/flwr/cli/new/templates/app/pyproject.numpy.toml.tpl +++ b/src/py/flwr/cli/new/templates/app/pyproject.numpy.toml.tpl @@ -6,9 +6,6 @@ build-backend = "hatchling.build" name = "$package_name" version = "1.0.0" description = "" -authors = [ - { name = "The Flower Authors", email = "hello@flower.ai" }, -] license = { text = "Apache License (2.0)" } dependencies = [ "flwr[simulation]>=1.9.0,<2.0", @@ -25,8 +22,8 @@ publisher = "$username" serverapp = "$import_name.server:app" clientapp = "$import_name.client:app" -[flower.engine] -name = "simulation" +[flower.federations] +default = "localhost" -[flower.engine.simulation.supernode] -num = 2 +[flower.federations.localhost] +options.num-supernodes = 10 diff --git a/src/py/flwr/cli/new/templates/app/pyproject.pytorch.toml.tpl b/src/py/flwr/cli/new/templates/app/pyproject.pytorch.toml.tpl index b56c0041b96c..4313079fa74a 100644 --- a/src/py/flwr/cli/new/templates/app/pyproject.pytorch.toml.tpl +++ b/src/py/flwr/cli/new/templates/app/pyproject.pytorch.toml.tpl @@ -6,9 +6,6 @@ build-backend = "hatchling.build" name = "$package_name" version = "1.0.0" description = "" -authors = [ - { name = "The Flower Authors", email = "hello@flower.ai" }, -] license = { text = "Apache License (2.0)" } dependencies = [ "flwr[simulation]>=1.9.0,<2.0", @@ -27,8 +24,8 @@ publisher = "$username" serverapp = "$import_name.server:app" clientapp = "$import_name.client:app" -[flower.engine] -name = "simulation" +[flower.federations] +default = "localhost" -[flower.engine.simulation.supernode] -num = 2 +[flower.federations.localhost] +options.num-supernodes = 10 diff --git a/src/py/flwr/cli/new/templates/app/pyproject.sklearn.toml.tpl b/src/py/flwr/cli/new/templates/app/pyproject.sklearn.toml.tpl index 6f914ae659b1..8ab7c10d0107 100644 --- a/src/py/flwr/cli/new/templates/app/pyproject.sklearn.toml.tpl +++ b/src/py/flwr/cli/new/templates/app/pyproject.sklearn.toml.tpl @@ -6,9 +6,6 @@ build-backend = "hatchling.build" name = "$package_name" version = "1.0.0" description = "" -authors = [ - { name = "The Flower Authors", email = "hello@flower.ai" }, -] license = { text = "Apache License (2.0)" } dependencies = [ "flwr[simulation]>=1.9.0,<2.0", @@ -26,8 +23,8 @@ publisher = "$username" serverapp = "$import_name.server:app" clientapp = "$import_name.client:app" -[flower.engine] -name = "simulation" +[flower.federations] +default = "localhost" -[flower.engine.simulation.supernode] -num = 2 +[flower.federations.localhost] +options.num-supernodes = 10 diff --git a/src/py/flwr/cli/new/templates/app/pyproject.tensorflow.toml.tpl b/src/py/flwr/cli/new/templates/app/pyproject.tensorflow.toml.tpl index 4ecd16143dcc..a64dfbe6bf77 100644 --- a/src/py/flwr/cli/new/templates/app/pyproject.tensorflow.toml.tpl +++ b/src/py/flwr/cli/new/templates/app/pyproject.tensorflow.toml.tpl @@ -6,9 +6,6 @@ build-backend = "hatchling.build" name = "$package_name" version = "1.0.0" description = "" -authors = [ - { name = "The Flower Authors", email = "hello@flower.ai" }, -] license = { text = "Apache License (2.0)" } dependencies = [ "flwr[simulation]>=1.9.0,<2.0", @@ -26,8 +23,8 @@ publisher = "$username" serverapp = "$import_name.server:app" clientapp = "$import_name.client:app" -[flower.engine] -name = "simulation" +[flower.federations] +default = "localhost" -[flower.engine.simulation.supernode] -num = 2 +[flower.federations.localhost] +options.num-supernodes = 10 diff --git a/src/py/flwr/cli/run/run.py b/src/py/flwr/cli/run/run.py index b23ba3f7d0cf..76d1f47e4fa9 100644 --- a/src/py/flwr/cli/run/run.py +++ b/src/py/flwr/cli/run/run.py @@ -15,18 +15,16 @@ """Flower command line interface `run` command.""" import sys -from enum import Enum from logging import DEBUG from pathlib import Path -from typing import Dict, Optional +from typing import Any, Dict, Optional import typer from typing_extensions import Annotated -from flwr.cli import config_utils from flwr.cli.build import build +from flwr.cli.config_utils import load_and_validate from flwr.common.config import parse_config_args -from flwr.common.constant import SUPEREXEC_DEFAULT_ADDRESS from flwr.common.grpc import GRPC_MAX_MESSAGE_LENGTH, create_channel from flwr.common.logger import log from flwr.proto.exec_pb2 import StartRunRequest # pylint: disable=E0611 @@ -34,31 +32,12 @@ from flwr.simulation.run_simulation import _run_simulation -class Engine(str, Enum): - """Enum defining the engine to run on.""" - - SIMULATION = "simulation" - - # pylint: disable-next=too-many-locals def run( - engine: Annotated[ - Optional[Engine], - typer.Option( - case_sensitive=False, - help="The engine to run FL with (currently only simulation is supported).", - ), - ] = None, - use_superexec: Annotated[ - bool, - typer.Option( - case_sensitive=False, help="Use this flag to use the new SuperExec API" - ), - ] = False, directory: Annotated[ - Optional[Path], - typer.Option(help="Path of the Flower project to run"), - ] = None, + Path, + typer.Argument(help="Path of the Flower project to run"), + ] = Path("."), config_overrides: Annotated[ Optional[str], typer.Option( @@ -72,7 +51,7 @@ def run( typer.secho("Loading project configuration... ", fg=typer.colors.BLUE) pyproject_path = directory / "pyproject.toml" if directory else None - config, errors, warnings = config_utils.load_and_validate(path=pyproject_path) + config, errors, warnings = load_and_validate(path=pyproject_path) if config is None: typer.secho( @@ -94,48 +73,37 @@ def run( typer.secho("Success", fg=typer.colors.GREEN) - if use_superexec: - _start_superexec_run( - parse_config_args(config_overrides, separator=","), directory - ) - return - - server_app_ref = config["flower"]["components"]["serverapp"] - client_app_ref = config["flower"]["components"]["clientapp"] - - if engine is None: - engine = config["flower"]["engine"]["name"] - - if engine == Engine.SIMULATION: - num_supernodes = config["flower"]["engine"]["simulation"]["supernode"]["num"] - backend_config = config["flower"]["engine"]["simulation"].get( - "backend_config", None - ) - - typer.secho("Starting run... ", fg=typer.colors.BLUE) - _run_simulation( - server_app_attr=server_app_ref, - client_app_attr=client_app_ref, - num_supernodes=num_supernodes, - backend_config=backend_config, - ) - else: + try: + federation_name = config["flower"]["federations"]["default"] + federation = config["flower"]["federations"][federation_name] + except KeyError as err: typer.secho( - f"Engine '{engine}' is not yet supported in `flwr run`", + "❌ The project's `pyproject.toml` needs to declare " + "a default federation (with a SuperExec address or an " + "`options.num-supernodes` value).", fg=typer.colors.RED, bold=True, ) + raise typer.Exit(code=1) from err + if "address" in federation: + _run_with_superexec(federation, directory, config_overrides) + else: + _run_without_superexec(config, federation, federation_name) -def _start_superexec_run( - override_config: Dict[str, str], directory: Optional[Path] + +def _run_with_superexec( + federation: Dict[str, str], + directory: Optional[Path], + config_overrides: Optional[str], ) -> None: + def on_channel_state_change(channel_connectivity: str) -> None: """Log channel connectivity.""" log(DEBUG, channel_connectivity) channel = create_channel( - server_address=SUPEREXEC_DEFAULT_ADDRESS, + server_address=federation["address"], insecure=True, root_certificates=None, max_message_length=GRPC_MAX_MESSAGE_LENGTH, @@ -148,7 +116,34 @@ def on_channel_state_change(channel_connectivity: str) -> None: req = StartRunRequest( fab_file=Path(fab_path).read_bytes(), - override_config=override_config, + override_config=parse_config_args(config_overrides, separator=","), ) res = stub.StartRun(req) typer.secho(f"🎊 Successfully started run {res.run_id}", fg=typer.colors.GREEN) + + +def _run_without_superexec( + config: Dict[str, Any], federation: Dict[str, Any], federation_name: str +) -> None: + server_app_ref = config["flower"]["components"]["serverapp"] + client_app_ref = config["flower"]["components"]["clientapp"] + + try: + num_supernodes = federation["options"]["num-supernodes"] + except KeyError as err: + typer.secho( + "❌ The project's `pyproject.toml` needs to declare the number of" + " SuperNodes in the simulation. To simulate 10 SuperNodes," + " use the following notation:\n\n" + f"[flower.federations.{federation_name}]\n" + "options.num-supernodes = 10\n", + fg=typer.colors.RED, + bold=True, + ) + raise typer.Exit(code=1) from err + + _run_simulation( + server_app_attr=server_app_ref, + client_app_attr=client_app_ref, + num_supernodes=num_supernodes, + ) From 0f7c0f7f2b65254f015554fb90de3626cc7aaae6 Mon Sep 17 00:00:00 2001 From: Javier Date: Mon, 15 Jul 2024 20:19:16 +0200 Subject: [PATCH 45/74] refactor(framework) Refactor `ClientApp` loading to use explicit arguments (#3805) --- src/py/flwr/client/supernode/app.py | 37 +++++++++++++++++++---------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/src/py/flwr/client/supernode/app.py b/src/py/flwr/client/supernode/app.py index d61b986bc7af..2f2fa58b428c 100644 --- a/src/py/flwr/client/supernode/app.py +++ b/src/py/flwr/client/supernode/app.py @@ -60,7 +60,12 @@ def run_supernode() -> None: _warn_deprecated_server_arg(args) root_certificates = _get_certificates(args) - load_fn = _get_load_client_app_fn(args, multi_app=True) + load_fn = _get_load_client_app_fn( + default_app_ref=getattr(args, "client-app"), + dir_arg=args.dir, + flwr_dir_arg=args.flwr_dir, + multi_app=True, + ) authentication_keys = _try_setup_client_authentication(args) _start_client_internal( @@ -93,7 +98,11 @@ def run_client_app() -> None: _warn_deprecated_server_arg(args) root_certificates = _get_certificates(args) - load_fn = _get_load_client_app_fn(args, multi_app=False) + load_fn = _get_load_client_app_fn( + default_app_ref=getattr(args, "client-app"), + dir_arg=args.dir, + multi_app=False, + ) authentication_keys = _try_setup_client_authentication(args) _start_client_internal( @@ -166,7 +175,10 @@ def _get_certificates(args: argparse.Namespace) -> Optional[bytes]: def _get_load_client_app_fn( - args: argparse.Namespace, multi_app: bool + default_app_ref: str, + dir_arg: str, + multi_app: bool, + flwr_dir_arg: Optional[str] = None, ) -> Callable[[str, str], ClientApp]: """Get the load_client_app_fn function. @@ -178,25 +190,24 @@ def _get_load_client_app_fn( loads a default ClientApp. """ # Find the Flower directory containing Flower Apps (only for multi-app) - flwr_dir = Path("") - if "flwr_dir" in args: - if args.flwr_dir is None: + if not multi_app: + flwr_dir = Path("") + else: + if flwr_dir_arg is None: flwr_dir = get_flwr_dir() else: - flwr_dir = Path(args.flwr_dir).absolute() + flwr_dir = Path(flwr_dir_arg).absolute() inserted_path = None - default_app_ref: str = getattr(args, "client-app") - if not multi_app: log( DEBUG, "Flower SuperNode will load and validate ClientApp `%s`", - getattr(args, "client-app"), + default_app_ref, ) # Insert sys.path - dir_path = Path(args.dir).absolute() + dir_path = Path(dir_arg).absolute() sys.path.insert(0, str(dir_path)) inserted_path = str(dir_path) @@ -208,7 +219,7 @@ def _load(fab_id: str, fab_version: str) -> ClientApp: # If multi-app feature is disabled if not multi_app: # Get sys path to be inserted - dir_path = Path(args.dir).absolute() + dir_path = Path(dir_arg).absolute() # Set app reference client_app_ref = default_app_ref @@ -221,7 +232,7 @@ def _load(fab_id: str, fab_version: str) -> ClientApp: log(WARN, "FAB ID is not provided; the default ClientApp will be loaded.") # Get sys path to be inserted - dir_path = Path(args.dir).absolute() + dir_path = Path(dir_arg).absolute() # Set app reference client_app_ref = default_app_ref From 17049ca900adb9f862583188aa5c28dde5ff1e94 Mon Sep 17 00:00:00 2001 From: Charles Beauville Date: Mon, 15 Jul 2024 21:22:17 +0200 Subject: [PATCH 46/74] feat(framework) Add federation argument to `flwr run` (#3807) --- src/py/flwr/cli/run/run.py | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/src/py/flwr/cli/run/run.py b/src/py/flwr/cli/run/run.py index 76d1f47e4fa9..1ae4017492b0 100644 --- a/src/py/flwr/cli/run/run.py +++ b/src/py/flwr/cli/run/run.py @@ -38,6 +38,10 @@ def run( Path, typer.Argument(help="Path of the Flower project to run"), ] = Path("."), + federation_name: Annotated[ + Optional[str], + typer.Argument(help="Name of the federation to run the app on"), + ] = None, config_overrides: Annotated[ Optional[str], typer.Option( @@ -73,18 +77,30 @@ def run( typer.secho("Success", fg=typer.colors.GREEN) - try: - federation_name = config["flower"]["federations"]["default"] - federation = config["flower"]["federations"][federation_name] - except KeyError as err: + federation_name = federation_name or config["flower"]["federations"].get("default") + + if federation_name is None: typer.secho( - "❌ The project's `pyproject.toml` needs to declare " - "a default federation (with a SuperExec address or an " + "❌ No federation name was provided and the project's `pyproject.toml` " + "doesn't declare a default federation (with a SuperExec address or an " "`options.num-supernodes` value).", fg=typer.colors.RED, bold=True, ) - raise typer.Exit(code=1) from err + raise typer.Exit(code=1) + + # Validate the federation exists in the configuration + federation = config["flower"]["federations"].get(federation_name) + if federation is None: + available_feds = list(config["flower"]["federations"]) + typer.secho( + f"❌ There is no `{federation_name}` federation declared in the " + "`pyproject.toml`.\n The following federations were found:\n\n" + "\n".join(available_feds) + "\n\n", + fg=typer.colors.RED, + bold=True, + ) + raise typer.Exit(code=1) if "address" in federation: _run_with_superexec(federation, directory, config_overrides) From 99885922dbc2328e06133f1b69990886bc3c2510 Mon Sep 17 00:00:00 2001 From: Javier Date: Mon, 15 Jul 2024 21:36:18 +0200 Subject: [PATCH 47/74] refactor(framework) Improve app loading in simulation engine (#3806) --- examples/simulation-pytorch/sim.py | 37 +++++++++++-------- .../server/superlink/fleet/vce/vce_api.py | 31 +++++++++------- src/py/flwr/simulation/run_simulation.py | 18 +++++++++ 3 files changed, 58 insertions(+), 28 deletions(-) diff --git a/examples/simulation-pytorch/sim.py b/examples/simulation-pytorch/sim.py index db68e75653fc..dcc0f39a79ef 100644 --- a/examples/simulation-pytorch/sim.py +++ b/examples/simulation-pytorch/sim.py @@ -87,11 +87,13 @@ def get_client_fn(dataset: FederatedDataset): the strategy to participate. """ - def client_fn(cid: str) -> fl.client.Client: + def client_fn(context) -> fl.client.Client: """Construct a FlowerClient with its own dataset partition.""" # Let's get the partition corresponding to the i-th client - client_dataset = dataset.load_partition(int(cid), "train") + client_dataset = dataset.load_partition( + int(context.node_config["partition-id"]), "train" + ) # Now let's split it into train (90%) and validation (10%) client_dataset_splits = client_dataset.train_test_split(test_size=0.1, seed=42) @@ -171,15 +173,23 @@ def evaluate( mnist_fds = FederatedDataset(dataset="mnist", partitioners={"train": NUM_CLIENTS}) centralized_testset = mnist_fds.load_split("test") -# Configure the strategy -strategy = fl.server.strategy.FedAvg( - fraction_fit=0.1, # Sample 10% of available clients for training - fraction_evaluate=0.05, # Sample 5% of available clients for evaluation - min_available_clients=10, - on_fit_config_fn=fit_config, - evaluate_metrics_aggregation_fn=weighted_average, # Aggregate federated metrics - evaluate_fn=get_evaluate_fn(centralized_testset), # Global evaluation function -) +from flwr.server import ServerAppComponents + + +def server_fn(context): + # Configure the strategy + strategy = fl.server.strategy.FedAvg( + fraction_fit=0.1, # Sample 10% of available clients for training + fraction_evaluate=0.05, # Sample 5% of available clients for evaluation + min_available_clients=10, + on_fit_config_fn=fit_config, + evaluate_metrics_aggregation_fn=weighted_average, # Aggregate federated metrics + evaluate_fn=get_evaluate_fn(centralized_testset), # Global evaluation function + ) + return ServerAppComponents( + strategy=strategy, config=fl.server.ServerConfig(num_rounds=NUM_ROUNDS) + ) + # ClientApp for Flower-Next client = fl.client.ClientApp( @@ -187,10 +197,7 @@ def evaluate( ) # ServerApp for Flower-Next -server = fl.server.ServerApp( - config=fl.server.ServerConfig(num_rounds=NUM_ROUNDS), - strategy=strategy, -) +server = fl.server.ServerApp(server_fn=server_fn) def main(): diff --git a/src/py/flwr/server/superlink/fleet/vce/vce_api.py b/src/py/flwr/server/superlink/fleet/vce/vce_api.py index 66bca5a391c7..b652207961a1 100644 --- a/src/py/flwr/server/superlink/fleet/vce/vce_api.py +++ b/src/py/flwr/server/superlink/fleet/vce/vce_api.py @@ -16,7 +16,6 @@ import json -import sys import threading import time import traceback @@ -29,6 +28,7 @@ from flwr.client.client_app import ClientApp, ClientAppException, LoadClientAppError from flwr.client.node_state import NodeState +from flwr.client.supernode.app import _get_load_client_app_fn from flwr.common.constant import ( NUM_PARTITIONS_KEY, PARTITION_ID_KEY, @@ -37,7 +37,6 @@ ) from flwr.common.logger import log from flwr.common.message import Error -from flwr.common.object_ref import load_app from flwr.common.serde import message_from_taskins, message_to_taskres from flwr.common.typing import Run from flwr.proto.task_pb2 import TaskIns, TaskRes # pylint: disable=E0611 @@ -259,6 +258,7 @@ def start_vce( app_dir: str, f_stop: threading.Event, run: Run, + flwr_dir: Optional[str] = None, client_app: Optional[ClientApp] = None, client_app_attr: Optional[str] = None, num_supernodes: Optional[int] = None, @@ -338,16 +338,12 @@ def backend_fn() -> Backend: def _load() -> ClientApp: if client_app_attr: - - if app_dir is not None: - sys.path.insert(0, app_dir) - - app: ClientApp = load_app(client_app_attr, LoadClientAppError, app_dir) - - if not isinstance(app, ClientApp): - raise LoadClientAppError( - f"Attribute {client_app_attr} is not of type {ClientApp}", - ) from None + app = _get_load_client_app_fn( + default_app_ref=client_app_attr, + dir_arg=app_dir, + flwr_dir_arg=flwr_dir, + multi_app=True, + )(run.fab_id, run.fab_version) if client_app: app = client_app @@ -357,7 +353,16 @@ def _load() -> ClientApp: try: # Test if ClientApp can be loaded - _ = app_fn() + client_app = app_fn() + + # Cache `ClientApp` + if client_app_attr: + # Now wrap the loaded ClientApp in a dummy function + # this prevent unnecesary low-level loading of ClientApp + def _load_client_app() -> ClientApp: + return client_app + + app_fn = _load_client_app # Run main simulation loop run_api( diff --git a/src/py/flwr/simulation/run_simulation.py b/src/py/flwr/simulation/run_simulation.py index 60e6e16eed27..8c70bf8374d0 100644 --- a/src/py/flwr/simulation/run_simulation.py +++ b/src/py/flwr/simulation/run_simulation.py @@ -207,6 +207,7 @@ def _main_loop( app_dir: str, enable_tf_gpu_growth: bool, run: Run, + flwr_dir: Optional[str] = None, client_app: Optional[ClientApp] = None, client_app_attr: Optional[str] = None, server_app: Optional[ServerApp] = None, @@ -253,6 +254,7 @@ def _main_loop( state_factory=state_factory, f_stop=f_stop, run=run, + flwr_dir=flwr_dir, ) except Exception as ex: @@ -283,6 +285,7 @@ def _run_simulation( client_app_attr: Optional[str] = None, server_app_attr: Optional[str] = None, app_dir: str = "", + flwr_dir: Optional[str] = None, run: Optional[Run] = None, enable_tf_gpu_growth: bool = False, verbose_logging: bool = False, @@ -326,6 +329,9 @@ def _run_simulation( Add specified directory to the PYTHONPATH and load `ClientApp` from there. (Default: current working directory.) + flwr_dir : Optional[str] + The path containing installed Flower Apps. + run : Optional[Run] An object carrying details about the run. @@ -377,6 +383,7 @@ def _run_simulation( app_dir, enable_tf_gpu_growth, run, + flwr_dir, client_app, client_app_attr, server_app, @@ -464,6 +471,17 @@ def _parse_args_run_simulation() -> argparse.ArgumentParser: "ClientApp and ServerApp from there." " Default: current working directory.", ) + parser.add_argument( + "--flwr-dir", + default=None, + help="""The path containing installed Flower Apps. + By default, this value is equal to: + + - `$FLWR_HOME/` if `$FLWR_HOME` is defined + - `$XDG_DATA_HOME/.flwr/` if `$XDG_DATA_HOME` is defined + - `$HOME/.flwr/` in all other cases + """, + ) parser.add_argument( "--run-id", type=int, From 1e14dc66d32052b13fc3a025b235033f3673e526 Mon Sep 17 00:00:00 2001 From: Charles Beauville Date: Mon, 15 Jul 2024 22:26:37 +0200 Subject: [PATCH 48/74] refactor(framework) Switch to `tool.flwr` instead of `flower` in `pyproject.toml` (#3809) --- src/py/flwr/cli/build.py | 2 +- src/py/flwr/cli/config_utils.py | 30 +++--- src/py/flwr/cli/config_utils_test.py | 96 ++++++++----------- src/py/flwr/cli/install.py | 2 +- .../app/pyproject.flowertune.toml.tpl | 8 +- .../new/templates/app/pyproject.hf.toml.tpl | 8 +- .../new/templates/app/pyproject.jax.toml.tpl | 8 +- .../new/templates/app/pyproject.mlx.toml.tpl | 8 +- .../templates/app/pyproject.numpy.toml.tpl | 8 +- .../templates/app/pyproject.pytorch.toml.tpl | 8 +- .../templates/app/pyproject.sklearn.toml.tpl | 8 +- .../app/pyproject.tensorflow.toml.tpl | 8 +- src/py/flwr/cli/run/run.py | 14 +-- src/py/flwr/client/supernode/app.py | 2 +- src/py/flwr/common/config.py | 2 +- src/py/flwr/common/config_test.py | 38 ++++---- src/py/flwr/server/run_serverapp.py | 2 +- src/py/flwr/superexec/simulation.py | 2 +- 18 files changed, 120 insertions(+), 134 deletions(-) diff --git a/src/py/flwr/cli/build.py b/src/py/flwr/cli/build.py index f63d0acd5d73..599ce613698c 100644 --- a/src/py/flwr/cli/build.py +++ b/src/py/flwr/cli/build.py @@ -85,7 +85,7 @@ def build( # Set the name of the zip file fab_filename = ( - f"{conf['flower']['publisher']}" + f"{conf['tool']['flwr']['publisher']}" f".{directory.name}" f".{conf['project']['version'].replace('.', '-')}.fab" ) diff --git a/src/py/flwr/cli/config_utils.py b/src/py/flwr/cli/config_utils.py index 33bf12e34b04..9147ebba4995 100644 --- a/src/py/flwr/cli/config_utils.py +++ b/src/py/flwr/cli/config_utils.py @@ -60,7 +60,7 @@ def get_fab_metadata(fab_file: Union[Path, bytes]) -> Tuple[str, str]: return ( conf["project"]["version"], - f"{conf['flower']['publisher']}/{conf['project']['name']}", + f"{conf['tool']['flwr']['publisher']}/{conf['project']['name']}", ) @@ -136,20 +136,20 @@ def validate_fields(config: Dict[str, Any]) -> Tuple[bool, List[str], List[str]] if "authors" not in config["project"]: warnings.append('Recommended property "authors" missing in [project]') - if "flower" not in config: - errors.append("Missing [flower] section") + if "tool" not in config or "flwr" not in config["tool"]: + errors.append("Missing [tool.flwr] section") else: - if "publisher" not in config["flower"]: - errors.append('Property "publisher" missing in [flower]') - if "config" in config["flower"]: - _validate_run_config(config["flower"]["config"], errors) - if "components" not in config["flower"]: - errors.append("Missing [flower.components] section") + if "publisher" not in config["tool"]["flwr"]: + errors.append('Property "publisher" missing in [tool.flwr]') + if "config" in config["tool"]["flwr"]: + _validate_run_config(config["tool"]["flwr"]["config"], errors) + if "components" not in config["tool"]["flwr"]: + errors.append("Missing [tool.flwr.components] section") else: - if "serverapp" not in config["flower"]["components"]: - errors.append('Property "serverapp" missing in [flower.components]') - if "clientapp" not in config["flower"]["components"]: - errors.append('Property "clientapp" missing in [flower.components]') + if "serverapp" not in config["tool"]["flwr"]["components"]: + errors.append('Property "serverapp" missing in [tool.flwr.components]') + if "clientapp" not in config["tool"]["flwr"]["components"]: + errors.append('Property "clientapp" missing in [tool.flwr.components]') return len(errors) == 0, errors, warnings @@ -165,14 +165,14 @@ def validate( # Validate serverapp is_valid, reason = object_ref.validate( - config["flower"]["components"]["serverapp"], check_module + config["tool"]["flwr"]["components"]["serverapp"], check_module ) if not is_valid and isinstance(reason, str): return False, [reason], [] # Validate clientapp is_valid, reason = object_ref.validate( - config["flower"]["components"]["clientapp"], check_module + config["tool"]["flwr"]["components"]["clientapp"], check_module ) if not is_valid and isinstance(reason, str): diff --git a/src/py/flwr/cli/config_utils_test.py b/src/py/flwr/cli/config_utils_test.py index b24425cd08f4..35d9900703b6 100644 --- a/src/py/flwr/cli/config_utils_test.py +++ b/src/py/flwr/cli/config_utils_test.py @@ -34,27 +34,18 @@ def test_load_pyproject_toml_load_from_cwd(tmp_path: Path) -> None: name = "fedgpt" version = "1.0.0" description = "" - authors = [ - { name = "The Flower Authors", email = "hello@flower.ai" }, - ] license = {text = "Apache License (2.0)"} dependencies = [ "flwr[simulation]>=1.9.0,<2.0", "numpy>=1.21.0", ] - [flower] + [tool.flwr] publisher = "flwrlabs" - [flower.components] + [tool.flwr.components] serverapp = "fedgpt.server:app" clientapp = "fedgpt.client:app" - - [flower.engine] - name = "simulation" # optional - - [flower.engine.simulation.supernode] - count = 10 # optional """ expected_config = { "build-system": {"build-backend": "hatchling.build", "requires": ["hatchling"]}, @@ -62,19 +53,16 @@ def test_load_pyproject_toml_load_from_cwd(tmp_path: Path) -> None: "name": "fedgpt", "version": "1.0.0", "description": "", - "authors": [{"email": "hello@flower.ai", "name": "The Flower Authors"}], "license": {"text": "Apache License (2.0)"}, "dependencies": ["flwr[simulation]>=1.9.0,<2.0", "numpy>=1.21.0"], }, - "flower": { - "publisher": "flwrlabs", - "components": { - "serverapp": "fedgpt.server:app", - "clientapp": "fedgpt.client:app", - }, - "engine": { - "name": "simulation", - "simulation": {"supernode": {"count": 10}}, + "tool": { + "flwr": { + "publisher": "flwrlabs", + "components": { + "serverapp": "fedgpt.server:app", + "clientapp": "fedgpt.client:app", + }, }, }, } @@ -109,27 +97,18 @@ def test_load_pyproject_toml_from_path(tmp_path: Path) -> None: name = "fedgpt" version = "1.0.0" description = "" - authors = [ - { name = "The Flower Authors", email = "hello@flower.ai" }, - ] license = {text = "Apache License (2.0)"} dependencies = [ "flwr[simulation]>=1.9.0,<2.0", "numpy>=1.21.0", ] - [flower] + [tool.flwr] publisher = "flwrlabs" - [flower.components] + [tool.flwr.components] serverapp = "fedgpt.server:app" clientapp = "fedgpt.client:app" - - [flower.engine] - name = "simulation" # optional - - [flower.engine.simulation.supernode] - count = 10 # optional """ expected_config = { "build-system": {"build-backend": "hatchling.build", "requires": ["hatchling"]}, @@ -137,19 +116,16 @@ def test_load_pyproject_toml_from_path(tmp_path: Path) -> None: "name": "fedgpt", "version": "1.0.0", "description": "", - "authors": [{"email": "hello@flower.ai", "name": "The Flower Authors"}], "license": {"text": "Apache License (2.0)"}, "dependencies": ["flwr[simulation]>=1.9.0,<2.0", "numpy>=1.21.0"], }, - "flower": { - "publisher": "flwrlabs", - "components": { - "serverapp": "fedgpt.server:app", - "clientapp": "fedgpt.client:app", - }, - "engine": { - "name": "simulation", - "simulation": {"supernode": {"count": 10}}, + "tool": { + "flwr": { + "publisher": "flwrlabs", + "components": { + "serverapp": "fedgpt.server:app", + "clientapp": "fedgpt.client:app", + }, }, }, } @@ -219,7 +195,7 @@ def test_validate_pyproject_toml_fields_no_flower_components() -> None: "license": "", "authors": [], }, - "flower": {}, + "tool": {"flwr": {}}, } # Execute @@ -242,7 +218,7 @@ def test_validate_pyproject_toml_fields_no_server_and_client_app() -> None: "license": "", "authors": [], }, - "flower": {"components": {}}, + "tool": {"flwr": {"components": {}}}, } # Execute @@ -265,9 +241,11 @@ def test_validate_pyproject_toml_fields() -> None: "license": "", "authors": [], }, - "flower": { - "publisher": "flwrlabs", - "components": {"serverapp": "", "clientapp": ""}, + "tool": { + "flwr": { + "publisher": "flwrlabs", + "components": {"serverapp": "", "clientapp": ""}, + }, }, } @@ -291,11 +269,13 @@ def test_validate_pyproject_toml() -> None: "license": "", "authors": [], }, - "flower": { - "publisher": "flwrlabs", - "components": { - "serverapp": "flwr.cli.run:run", - "clientapp": "flwr.cli.run:run", + "tool": { + "flwr": { + "publisher": "flwrlabs", + "components": { + "serverapp": "flwr.cli.run:run", + "clientapp": "flwr.cli.run:run", + }, }, }, } @@ -320,11 +300,13 @@ def test_validate_pyproject_toml_fail() -> None: "license": "", "authors": [], }, - "flower": { - "publisher": "flwrlabs", - "components": { - "serverapp": "flwr.cli.run:run", - "clientapp": "flwr.cli.run:runa", + "tool": { + "flwr": { + "publisher": "flwrlabs", + "components": { + "serverapp": "flwr.cli.run:run", + "clientapp": "flwr.cli.run:runa", + }, }, }, } diff --git a/src/py/flwr/cli/install.py b/src/py/flwr/cli/install.py index de9227bee450..7444f10c1eb7 100644 --- a/src/py/flwr/cli/install.py +++ b/src/py/flwr/cli/install.py @@ -149,7 +149,7 @@ def validate_and_install( ) raise typer.Exit(code=1) - publisher = config["flower"]["publisher"] + publisher = config["tool"]["flwr"]["publisher"] project_name = config["project"]["name"] version = config["project"]["version"] diff --git a/src/py/flwr/cli/new/templates/app/pyproject.flowertune.toml.tpl b/src/py/flwr/cli/new/templates/app/pyproject.flowertune.toml.tpl index 109cbf66a35b..17630dd9d0dc 100644 --- a/src/py/flwr/cli/new/templates/app/pyproject.flowertune.toml.tpl +++ b/src/py/flwr/cli/new/templates/app/pyproject.flowertune.toml.tpl @@ -22,15 +22,15 @@ dependencies = [ [tool.hatch.build.targets.wheel] packages = ["."] -[flower] +[tool.flwr] publisher = "$username" -[flower.components] +[tool.flwr.components] serverapp = "$import_name.app:server" clientapp = "$import_name.app:client" -[flower.federations] +[tool.flwr.federations] default = "localhost" -[flower.federations.localhost] +[tool.flwr.federations.localhost] options.num-supernodes = 10 diff --git a/src/py/flwr/cli/new/templates/app/pyproject.hf.toml.tpl b/src/py/flwr/cli/new/templates/app/pyproject.hf.toml.tpl index 6c7e50393098..6f46d6de5bf5 100644 --- a/src/py/flwr/cli/new/templates/app/pyproject.hf.toml.tpl +++ b/src/py/flwr/cli/new/templates/app/pyproject.hf.toml.tpl @@ -20,15 +20,15 @@ dependencies = [ [tool.hatch.build.targets.wheel] packages = ["."] -[flower] +[tool.flwr] publisher = "$username" -[flower.components] +[tool.flwr.components] serverapp = "$import_name.server:app" clientapp = "$import_name.client:app" -[flower.federations] +[tool.flwr.federations] default = "localhost" -[flower.federations.localhost] +[tool.flwr.federations.localhost] options.num-supernodes = 10 diff --git a/src/py/flwr/cli/new/templates/app/pyproject.jax.toml.tpl b/src/py/flwr/cli/new/templates/app/pyproject.jax.toml.tpl index f5c66cc729b8..045a1f4e57eb 100644 --- a/src/py/flwr/cli/new/templates/app/pyproject.jax.toml.tpl +++ b/src/py/flwr/cli/new/templates/app/pyproject.jax.toml.tpl @@ -17,15 +17,15 @@ dependencies = [ [tool.hatch.build.targets.wheel] packages = ["."] -[flower] +[tool.flwr] publisher = "$username" -[flower.components] +[tool.flwr.components] serverapp = "$import_name.server:app" clientapp = "$import_name.client:app" -[flower.federations] +[tool.flwr.federations] default = "localhost" -[flower.federations.localhost] +[tool.flwr.federations.localhost] options.num-supernodes = 10 diff --git a/src/py/flwr/cli/new/templates/app/pyproject.mlx.toml.tpl b/src/py/flwr/cli/new/templates/app/pyproject.mlx.toml.tpl index eaeec144adb2..5ea2c420d6f8 100644 --- a/src/py/flwr/cli/new/templates/app/pyproject.mlx.toml.tpl +++ b/src/py/flwr/cli/new/templates/app/pyproject.mlx.toml.tpl @@ -17,15 +17,15 @@ dependencies = [ [tool.hatch.build.targets.wheel] packages = ["."] -[flower] +[tool.flwr] publisher = "$username" -[flower.components] +[tool.flwr.components] serverapp = "$import_name.server:app" clientapp = "$import_name.client:app" -[flower.federations] +[tool.flwr.federations] default = "localhost" -[flower.federations.localhost] +[tool.flwr.federations.localhost] options.num-supernodes = 10 diff --git a/src/py/flwr/cli/new/templates/app/pyproject.numpy.toml.tpl b/src/py/flwr/cli/new/templates/app/pyproject.numpy.toml.tpl index 6f386990ba6e..d166616bb616 100644 --- a/src/py/flwr/cli/new/templates/app/pyproject.numpy.toml.tpl +++ b/src/py/flwr/cli/new/templates/app/pyproject.numpy.toml.tpl @@ -15,15 +15,15 @@ dependencies = [ [tool.hatch.build.targets.wheel] packages = ["."] -[flower] +[tool.flwr] publisher = "$username" -[flower.components] +[tool.flwr.components] serverapp = "$import_name.server:app" clientapp = "$import_name.client:app" -[flower.federations] +[tool.flwr.federations] default = "localhost" -[flower.federations.localhost] +[tool.flwr.federations.localhost] options.num-supernodes = 10 diff --git a/src/py/flwr/cli/new/templates/app/pyproject.pytorch.toml.tpl b/src/py/flwr/cli/new/templates/app/pyproject.pytorch.toml.tpl index 4313079fa74a..c0323126516d 100644 --- a/src/py/flwr/cli/new/templates/app/pyproject.pytorch.toml.tpl +++ b/src/py/flwr/cli/new/templates/app/pyproject.pytorch.toml.tpl @@ -17,15 +17,15 @@ dependencies = [ [tool.hatch.build.targets.wheel] packages = ["."] -[flower] +[tool.flwr] publisher = "$username" -[flower.components] +[tool.flwr.components] serverapp = "$import_name.server:app" clientapp = "$import_name.client:app" -[flower.federations] +[tool.flwr.federations] default = "localhost" -[flower.federations.localhost] +[tool.flwr.federations.localhost] options.num-supernodes = 10 diff --git a/src/py/flwr/cli/new/templates/app/pyproject.sklearn.toml.tpl b/src/py/flwr/cli/new/templates/app/pyproject.sklearn.toml.tpl index 8ab7c10d0107..0e63375aab00 100644 --- a/src/py/flwr/cli/new/templates/app/pyproject.sklearn.toml.tpl +++ b/src/py/flwr/cli/new/templates/app/pyproject.sklearn.toml.tpl @@ -16,15 +16,15 @@ dependencies = [ [tool.hatch.build.targets.wheel] packages = ["."] -[flower] +[tool.flwr] publisher = "$username" -[flower.components] +[tool.flwr.components] serverapp = "$import_name.server:app" clientapp = "$import_name.client:app" -[flower.federations] +[tool.flwr.federations] default = "localhost" -[flower.federations.localhost] +[tool.flwr.federations.localhost] options.num-supernodes = 10 diff --git a/src/py/flwr/cli/new/templates/app/pyproject.tensorflow.toml.tpl b/src/py/flwr/cli/new/templates/app/pyproject.tensorflow.toml.tpl index a64dfbe6bf77..aeca4a17805f 100644 --- a/src/py/flwr/cli/new/templates/app/pyproject.tensorflow.toml.tpl +++ b/src/py/flwr/cli/new/templates/app/pyproject.tensorflow.toml.tpl @@ -16,15 +16,15 @@ dependencies = [ [tool.hatch.build.targets.wheel] packages = ["."] -[flower] +[tool.flwr] publisher = "$username" -[flower.components] +[tool.flwr.components] serverapp = "$import_name.server:app" clientapp = "$import_name.client:app" -[flower.federations] +[tool.flwr.federations] default = "localhost" -[flower.federations.localhost] +[tool.flwr.federations.localhost] options.num-supernodes = 10 diff --git a/src/py/flwr/cli/run/run.py b/src/py/flwr/cli/run/run.py index 1ae4017492b0..c39ae0decd4b 100644 --- a/src/py/flwr/cli/run/run.py +++ b/src/py/flwr/cli/run/run.py @@ -77,7 +77,9 @@ def run( typer.secho("Success", fg=typer.colors.GREEN) - federation_name = federation_name or config["flower"]["federations"].get("default") + federation_name = federation_name or config["tool"]["flwr"]["federations"].get( + "default" + ) if federation_name is None: typer.secho( @@ -90,9 +92,9 @@ def run( raise typer.Exit(code=1) # Validate the federation exists in the configuration - federation = config["flower"]["federations"].get(federation_name) + federation = config["tool"]["flwr"]["federations"].get(federation_name) if federation is None: - available_feds = list(config["flower"]["federations"]) + available_feds = list(config["tool"]["flwr"]["federations"]) typer.secho( f"❌ There is no `{federation_name}` federation declared in the " "`pyproject.toml`.\n The following federations were found:\n\n" @@ -141,8 +143,8 @@ def on_channel_state_change(channel_connectivity: str) -> None: def _run_without_superexec( config: Dict[str, Any], federation: Dict[str, Any], federation_name: str ) -> None: - server_app_ref = config["flower"]["components"]["serverapp"] - client_app_ref = config["flower"]["components"]["clientapp"] + server_app_ref = config["tool"]["flwr"]["components"]["serverapp"] + client_app_ref = config["tool"]["flwr"]["components"]["clientapp"] try: num_supernodes = federation["options"]["num-supernodes"] @@ -151,7 +153,7 @@ def _run_without_superexec( "❌ The project's `pyproject.toml` needs to declare the number of" " SuperNodes in the simulation. To simulate 10 SuperNodes," " use the following notation:\n\n" - f"[flower.federations.{federation_name}]\n" + f"[tool.flwr.federations.{federation_name}]\n" "options.num-supernodes = 10\n", fg=typer.colors.RED, bold=True, diff --git a/src/py/flwr/client/supernode/app.py b/src/py/flwr/client/supernode/app.py index 2f2fa58b428c..027c3376b7f3 100644 --- a/src/py/flwr/client/supernode/app.py +++ b/src/py/flwr/client/supernode/app.py @@ -248,7 +248,7 @@ def _load(fab_id: str, fab_version: str) -> ClientApp: dir_path = Path(project_dir).absolute() # Set app reference - client_app_ref = config["flower"]["components"]["clientapp"] + client_app_ref = config["tool"]["flwr"]["components"]["clientapp"] # Set sys.path nonlocal inserted_path diff --git a/src/py/flwr/common/config.py b/src/py/flwr/common/config.py index 54d74353e4ed..e2b06ff86110 100644 --- a/src/py/flwr/common/config.py +++ b/src/py/flwr/common/config.py @@ -97,7 +97,7 @@ def get_fused_config(run: Run, flwr_dir: Optional[Path]) -> Dict[str, str]: project_dir = get_project_dir(run.fab_id, run.fab_version, flwr_dir) - default_config = get_project_config(project_dir)["flower"].get("config", {}) + default_config = get_project_config(project_dir)["tool"]["flwr"].get("config", {}) flat_default_config = flatten_dict(default_config) return _fuse_dicts(flat_default_config, run.override_config) diff --git a/src/py/flwr/common/config_test.py b/src/py/flwr/common/config_test.py index fe429bab9cb5..899240c1e76a 100644 --- a/src/py/flwr/common/config_test.py +++ b/src/py/flwr/common/config_test.py @@ -93,20 +93,20 @@ def test_get_fused_config_valid(tmp_path: Path) -> None: "numpy>=1.21.0", ] - [flower] + [tool.flwr] publisher = "flwrlabs" - [flower.components] + [tool.flwr.components] serverapp = "fedgpt.server:app" clientapp = "fedgpt.client:app" - [flower.config] + [tool.flwr.config] num_server_rounds = "10" momentum = "0.1" lr = "0.01" serverapp.test = "key" - [flower.config.clientapp] + [tool.flwr.config.clientapp] test = "key" """ overrides = { @@ -131,7 +131,7 @@ def test_get_fused_config_valid(tmp_path: Path) -> None: f.write(textwrap.dedent(pyproject_toml_content)) # Execute - default_config = get_project_config(tmp_path)["flower"].get("config", {}) + default_config = get_project_config(tmp_path)["tool"]["flwr"].get("config", {}) config = _fuse_dicts(flatten_dict(default_config), overrides) @@ -158,14 +158,14 @@ def test_get_project_config_file_valid(tmp_path: Path) -> None: "numpy>=1.21.0", ] - [flower] + [tool.flwr] publisher = "flwrlabs" - [flower.components] + [tool.flwr.components] serverapp = "fedgpt.server:app" clientapp = "fedgpt.client:app" - [flower.config] + [tool.flwr.config] num_server_rounds = "10" momentum = "0.1" lr = "0.01" @@ -179,16 +179,18 @@ def test_get_project_config_file_valid(tmp_path: Path) -> None: "license": {"text": "Apache License (2.0)"}, "dependencies": ["flwr[simulation]>=1.9.0,<2.0", "numpy>=1.21.0"], }, - "flower": { - "publisher": "flwrlabs", - "components": { - "serverapp": "fedgpt.server:app", - "clientapp": "fedgpt.client:app", - }, - "config": { - "num_server_rounds": "10", - "momentum": "0.1", - "lr": "0.01", + "tool": { + "flwr": { + "publisher": "flwrlabs", + "components": { + "serverapp": "fedgpt.server:app", + "clientapp": "fedgpt.client:app", + }, + "config": { + "num_server_rounds": "10", + "momentum": "0.1", + "lr": "0.01", + }, }, }, } diff --git a/src/py/flwr/server/run_serverapp.py b/src/py/flwr/server/run_serverapp.py index 4cc25feb7e0e..efaba24f05f9 100644 --- a/src/py/flwr/server/run_serverapp.py +++ b/src/py/flwr/server/run_serverapp.py @@ -186,7 +186,7 @@ def run_server_app() -> None: # pylint: disable=too-many-branches run_ = driver.run server_app_dir = str(get_project_dir(run_.fab_id, run_.fab_version, flwr_dir)) config = get_project_config(server_app_dir) - server_app_attr = config["flower"]["components"]["serverapp"] + server_app_attr = config["tool"]["flwr"]["components"]["serverapp"] server_app_run_config = get_fused_config(run_, flwr_dir) else: # User provided `server-app`, but not `--run-id` diff --git a/src/py/flwr/superexec/simulation.py b/src/py/flwr/superexec/simulation.py index 9a8e19365ab9..fa7a8ad9b0d3 100644 --- a/src/py/flwr/superexec/simulation.py +++ b/src/py/flwr/superexec/simulation.py @@ -112,7 +112,7 @@ def start_run( ) # Get ClientApp and SeverApp components - flower_components = config["flower"]["components"] + flower_components = config["tool"]["flwr"]["components"] clientapp = flower_components["clientapp"] serverapp = flower_components["serverapp"] From 8f5b4e5b8292467e028d19bafd7ef0e5640cb729 Mon Sep 17 00:00:00 2001 From: Charles Beauville Date: Mon, 15 Jul 2024 22:39:08 +0200 Subject: [PATCH 49/74] feat(framework) Add secure channel support for SuperExec (#3808) Co-authored-by: Daniel J. Beutel --- src/py/flwr/cli/run/run.py | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/src/py/flwr/cli/run/run.py b/src/py/flwr/cli/run/run.py index c39ae0decd4b..512da83d13fe 100644 --- a/src/py/flwr/cli/run/run.py +++ b/src/py/flwr/cli/run/run.py @@ -120,10 +120,38 @@ def on_channel_state_change(channel_connectivity: str) -> None: """Log channel connectivity.""" log(DEBUG, channel_connectivity) + insecure_str = federation.get("insecure") + if root_certificates := federation.get("root-certificates"): + root_certificates_bytes = Path(root_certificates).read_bytes() + if insecure := bool(insecure_str): + typer.secho( + "❌ `root_certificates` were provided but the `insecure` parameter" + "is set to `True`.", + fg=typer.colors.RED, + bold=True, + ) + raise typer.Exit(code=1) + else: + root_certificates_bytes = None + if insecure_str is None: + typer.secho( + "❌ To disable TLS, set `insecure = true` in `pyproject.toml`.", + fg=typer.colors.RED, + bold=True, + ) + raise typer.Exit(code=1) + if not (insecure := bool(insecure_str)): + typer.secho( + "❌ No certificate were given yet `insecure` is set to `False`.", + fg=typer.colors.RED, + bold=True, + ) + raise typer.Exit(code=1) + channel = create_channel( server_address=federation["address"], - insecure=True, - root_certificates=None, + insecure=insecure, + root_certificates=root_certificates_bytes, max_message_length=GRPC_MAX_MESSAGE_LENGTH, interceptors=None, ) From 9f78948fb0c44c55691adf2e3a452b9507569966 Mon Sep 17 00:00:00 2001 From: Chong Shen Ng Date: Tue, 16 Jul 2024 09:29:01 +0100 Subject: [PATCH 50/74] Set refresh period as constant --- src/py/flwr/cli/log.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/py/flwr/cli/log.py b/src/py/flwr/cli/log.py index 0d666de9765c..96fd328b081a 100644 --- a/src/py/flwr/cli/log.py +++ b/src/py/flwr/cli/log.py @@ -28,6 +28,8 @@ from flwr.common.grpc import GRPC_MAX_MESSAGE_LENGTH, create_channel from flwr.common.logger import log as logger +CONN_REFRESH_PERIOD = 60 # Connection refresh period for log streaming (seconds) + # pylint: disable=unused-argument def stream_logs(run_id: int, channel: grpc.Channel, period: int) -> None: @@ -48,13 +50,6 @@ def log( Optional[str], typer.Option(case_sensitive=False, help="The address of the SuperExec server"), ] = None, - period: Annotated[ - int, - typer.Option( - case_sensitive=False, - help="Use this to set connection refresh time period (in seconds)", - ), - ] = 60, follow: Annotated[ bool, typer.Option(case_sensitive=False, help="Use this flag to follow logstream"), @@ -93,7 +88,7 @@ def on_channel_state_change(channel_connectivity: str) -> None: try: while True: logger(INFO, "Starting logstream for run_id `%s`", run_id) - stream_logs(run_id, channel, period) + stream_logs(run_id, channel, CONN_REFRESH_PERIOD) time.sleep(2) logger(INFO, "Reconnecting to logstream") except KeyboardInterrupt: From 603cee698c4a9bc49efb46def2a80e66848d000d Mon Sep 17 00:00:00 2001 From: Chong Shen Ng Date: Thu, 18 Jul 2024 22:11:25 +0100 Subject: [PATCH 51/74] Use directory and federation_name --- src/py/flwr/cli/log.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/py/flwr/cli/log.py b/src/py/flwr/cli/log.py index 96fd328b081a..c350ff1408d5 100644 --- a/src/py/flwr/cli/log.py +++ b/src/py/flwr/cli/log.py @@ -17,6 +17,7 @@ import sys import time from logging import DEBUG, ERROR, INFO +from pathlib import Path from typing import Optional import grpc @@ -44,11 +45,15 @@ def print_logs(run_id: int, channel: grpc.Channel, timeout: int) -> None: def log( run_id: Annotated[ int, - typer.Option(case_sensitive=False, help="The Flower run ID to query"), - ], - superexec_address: Annotated[ + typer.Argument(help="The Flower run ID to query"), + ] = None, + directory: Annotated[ + Path, + typer.Argument(help="Path of the Flower project to run"), + ] = Path("."), + federation_name: Annotated[ Optional[str], - typer.Option(case_sensitive=False, help="The address of the SuperExec server"), + typer.Argument(help="Name of the federation to run the app on"), ] = None, follow: Annotated[ bool, From fbdb35c52e42614e6963b4d15d571be012cd27e6 Mon Sep 17 00:00:00 2001 From: Chong Shen Ng Date: Thu, 18 Jul 2024 22:17:04 +0100 Subject: [PATCH 52/74] Add pyproject.toml validation --- src/py/flwr/cli/log.py | 67 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 66 insertions(+), 1 deletion(-) diff --git a/src/py/flwr/cli/log.py b/src/py/flwr/cli/log.py index c350ff1408d5..72f67739b195 100644 --- a/src/py/flwr/cli/log.py +++ b/src/py/flwr/cli/log.py @@ -18,13 +18,14 @@ import time from logging import DEBUG, ERROR, INFO from pathlib import Path -from typing import Optional +from typing import Optional, Dict import grpc import typer from typing_extensions import Annotated from flwr.cli import config_utils +from flwr.cli.config_utils import load_and_validate from flwr.common.config import get_flwr_dir from flwr.common.grpc import GRPC_MAX_MESSAGE_LENGTH, create_channel from flwr.common.logger import log as logger @@ -61,6 +62,70 @@ def log( ] = True, ) -> None: """Get logs from Flower run.""" + typer.secho("Loading project configuration... ", fg=typer.colors.BLUE) + + pyproject_path = directory / "pyproject.toml" if directory else None + config, errors, warnings = load_and_validate(path=pyproject_path) + + if config is None: + typer.secho( + "Project configuration could not be loaded.\n" + "pyproject.toml is invalid:\n" + + "\n".join([f"- {line}" for line in errors]), + fg=typer.colors.RED, + bold=True, + ) + sys.exit() + + if warnings: + typer.secho( + "Project configuration is missing the following " + "recommended properties:\n" + "\n".join([f"- {line}" for line in warnings]), + fg=typer.colors.RED, + bold=True, + ) + + typer.secho("Success", fg=typer.colors.GREEN) + + federation_name = federation_name or config["tool"]["flwr"]["federations"].get( + "default" + ) + + if federation_name is None: + typer.secho( + "❌ No federation name was provided and the project's `pyproject.toml` " + "doesn't declare a default federation (with a SuperExec address or an " + "`options.num-supernodes` value).", + fg=typer.colors.RED, + bold=True, + ) + raise typer.Exit(code=1) + + # Validate the federation exists in the configuration + federation = config["tool"]["flwr"]["federations"].get(federation_name) + if federation is None: + available_feds = { + fed for fed in config["tool"]["flwr"]["federations"] if fed != "default" + } + typer.secho( + f"❌ There is no `{federation_name}` federation declared in the " + "`pyproject.toml`.\n The following federations were found:\n\n" + + "\n".join(available_feds), + fg=typer.colors.RED, + bold=True, + ) + raise typer.Exit(code=1) + + if "address" in federation: + _log_with_superexec(federation, directory) + else: + pass + + +def _log_with_superexec( + federation: Dict[str, str], + directory: Optional[Path], +) -> None: def on_channel_state_change(channel_connectivity: str) -> None: """Log channel connectivity.""" From 51d1700ed30c07d86ad58843824b6fc1fd7f525c Mon Sep 17 00:00:00 2001 From: Chong Shen Ng Date: Thu, 18 Jul 2024 22:37:24 +0100 Subject: [PATCH 53/74] Add parsing server_address from pyproject.toml --- src/py/flwr/cli/log.py | 52 +++++++++++++++++++++++++++++------------- 1 file changed, 36 insertions(+), 16 deletions(-) diff --git a/src/py/flwr/cli/log.py b/src/py/flwr/cli/log.py index 72f67739b195..e5f85967c1f2 100644 --- a/src/py/flwr/cli/log.py +++ b/src/py/flwr/cli/log.py @@ -58,10 +58,14 @@ def log( ] = None, follow: Annotated[ bool, - typer.Option(case_sensitive=False, help="Use this flag to follow logstream"), + typer.Option( + "--follow/--no-follow", + "-f/-F", + help="Use this flag to follow logstream", + ), ] = True, ) -> None: - """Get logs from Flower run.""" + """Get logs from a Flower project run.""" typer.secho("Loading project configuration... ", fg=typer.colors.BLUE) pyproject_path = directory / "pyproject.toml" if directory else None @@ -117,38 +121,54 @@ def log( raise typer.Exit(code=1) if "address" in federation: - _log_with_superexec(federation, directory) + _log_with_superexec(federation, run_id, follow) else: pass +# pylint: disable-next=too-many-branches def _log_with_superexec( federation: Dict[str, str], - directory: Optional[Path], + run_id: int, + follow: bool, ) -> None: def on_channel_state_change(channel_connectivity: str) -> None: """Log channel connectivity.""" logger(DEBUG, channel_connectivity) - if superexec_address is None: - global_config = config_utils.load(get_flwr_dir() / "config.toml") - if global_config: - superexec_address = global_config["federation"]["default"] - else: + insecure_str = federation.get("insecure") + if root_certificates := federation.get("root-certificates"): + root_certificates_bytes = Path(root_certificates).read_bytes() + if insecure := bool(insecure_str): typer.secho( - "No SuperExec address was provided and no global config was found.", + "❌ `root_certificates` were provided but the `insecure` parameter" + "is set to `True`.", fg=typer.colors.RED, bold=True, ) - sys.exit() - - assert superexec_address is not None + raise typer.Exit(code=1) + else: + root_certificates_bytes = None + if insecure_str is None: + typer.secho( + "❌ To disable TLS, set `insecure = true` in `pyproject.toml`.", + fg=typer.colors.RED, + bold=True, + ) + raise typer.Exit(code=1) + if not (insecure := bool(insecure_str)): + typer.secho( + "❌ No certificate were given yet `insecure` is set to `False`.", + fg=typer.colors.RED, + bold=True, + ) + raise typer.Exit(code=1) channel = create_channel( - server_address=superexec_address, - insecure=True, - root_certificates=None, + server_address=federation["address"], + insecure=insecure, + root_certificates=root_certificates_bytes, max_message_length=GRPC_MAX_MESSAGE_LENGTH, interceptors=None, ) From 471dec8c378442ce085737241c903d38dc4c6e83 Mon Sep 17 00:00:00 2001 From: Chong Shen Ng Date: Thu, 18 Jul 2024 22:37:41 +0100 Subject: [PATCH 54/74] Remove unused imports --- src/py/flwr/cli/log.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/py/flwr/cli/log.py b/src/py/flwr/cli/log.py index e5f85967c1f2..2ee90d023eab 100644 --- a/src/py/flwr/cli/log.py +++ b/src/py/flwr/cli/log.py @@ -24,9 +24,7 @@ import typer from typing_extensions import Annotated -from flwr.cli import config_utils from flwr.cli.config_utils import load_and_validate -from flwr.common.config import get_flwr_dir from flwr.common.grpc import GRPC_MAX_MESSAGE_LENGTH, create_channel from flwr.common.logger import log as logger From 3bd943d1b1597d8267a61c7b991185e0bbe01007 Mon Sep 17 00:00:00 2001 From: Chong Shen Ng Date: Fri, 19 Jul 2024 16:00:23 +0100 Subject: [PATCH 55/74] Run isort --- src/py/flwr/cli/log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/py/flwr/cli/log.py b/src/py/flwr/cli/log.py index 2ee90d023eab..11f0bd3fa0e7 100644 --- a/src/py/flwr/cli/log.py +++ b/src/py/flwr/cli/log.py @@ -18,7 +18,7 @@ import time from logging import DEBUG, ERROR, INFO from pathlib import Path -from typing import Optional, Dict +from typing import Dict, Optional import grpc import typer From 8abe6c33e7924fb60ef13a1ba6825f1b781e7211 Mon Sep 17 00:00:00 2001 From: Chong Shen Ng Date: Fri, 19 Jul 2024 16:13:46 +0100 Subject: [PATCH 56/74] Remove default value for run_id --- src/py/flwr/cli/log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/py/flwr/cli/log.py b/src/py/flwr/cli/log.py index 11f0bd3fa0e7..b59a3714d568 100644 --- a/src/py/flwr/cli/log.py +++ b/src/py/flwr/cli/log.py @@ -45,7 +45,7 @@ def log( run_id: Annotated[ int, typer.Argument(help="The Flower run ID to query"), - ] = None, + ], directory: Annotated[ Path, typer.Argument(help="Path of the Flower project to run"), From 463f7525bad72eb79946a00a9a809c9ce9fff5a3 Mon Sep 17 00:00:00 2001 From: Chong Shen Ng Date: Tue, 23 Jul 2024 16:14:09 +0100 Subject: [PATCH 57/74] Set log level to debug --- src/py/flwr/cli/log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/py/flwr/cli/log.py b/src/py/flwr/cli/log.py index b59a3714d568..10ffb218861b 100644 --- a/src/py/flwr/cli/log.py +++ b/src/py/flwr/cli/log.py @@ -178,7 +178,7 @@ def on_channel_state_change(channel_connectivity: str) -> None: logger(INFO, "Starting logstream for run_id `%s`", run_id) stream_logs(run_id, channel, CONN_REFRESH_PERIOD) time.sleep(2) - logger(INFO, "Reconnecting to logstream") + logger(DEBUG, "Reconnecting to logstream") except KeyboardInterrupt: logger(INFO, "Exiting logstream") except grpc.RpcError as e: From 3b04c29fbedd34eb2383194291d898eb2f6bb52f Mon Sep 17 00:00:00 2001 From: Charles Beauville Date: Tue, 23 Jul 2024 17:39:01 +0200 Subject: [PATCH 58/74] fix(framework:skip) Use full name for HuggingFace template (#3883) --- .github/workflows/e2e.yml | 2 +- src/py/flwr/cli/new/new.py | 2 +- .../app/code/{client.hf.py.tpl => client.huggingface.py.tpl} | 0 .../app/code/{server.hf.py.tpl => server.huggingface.py.tpl} | 0 .../app/code/{task.hf.py.tpl => task.huggingface.py.tpl} | 0 .../{pyproject.hf.toml.tpl => pyproject.huggingface.toml.tpl} | 0 6 files changed, 2 insertions(+), 2 deletions(-) rename src/py/flwr/cli/new/templates/app/code/{client.hf.py.tpl => client.huggingface.py.tpl} (100%) rename src/py/flwr/cli/new/templates/app/code/{server.hf.py.tpl => server.huggingface.py.tpl} (100%) rename src/py/flwr/cli/new/templates/app/code/{task.hf.py.tpl => task.huggingface.py.tpl} (100%) rename src/py/flwr/cli/new/templates/app/{pyproject.hf.toml.tpl => pyproject.huggingface.toml.tpl} (100%) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 241ec8057c7c..f5ed1d99012a 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -213,7 +213,7 @@ jobs: needs: wheel strategy: matrix: - framework: ["numpy", "pytorch", "tensorflow", "hf", "jax", "sklearn"] + framework: ["numpy", "pytorch", "tensorflow", "huggingface", "jax", "sklearn"] name: Template / ${{ matrix.framework }} diff --git a/src/py/flwr/cli/new/new.py b/src/py/flwr/cli/new/new.py index 306f20efccfa..237b8847e193 100644 --- a/src/py/flwr/cli/new/new.py +++ b/src/py/flwr/cli/new/new.py @@ -38,7 +38,7 @@ class MlFramework(str, Enum): PYTORCH = "PyTorch" TENSORFLOW = "TensorFlow" JAX = "JAX" - HUGGINGFACE = "HF" + HUGGINGFACE = "HuggingFace" MLX = "MLX" SKLEARN = "sklearn" FLOWERTUNE = "FlowerTune" diff --git a/src/py/flwr/cli/new/templates/app/code/client.hf.py.tpl b/src/py/flwr/cli/new/templates/app/code/client.huggingface.py.tpl similarity index 100% rename from src/py/flwr/cli/new/templates/app/code/client.hf.py.tpl rename to src/py/flwr/cli/new/templates/app/code/client.huggingface.py.tpl diff --git a/src/py/flwr/cli/new/templates/app/code/server.hf.py.tpl b/src/py/flwr/cli/new/templates/app/code/server.huggingface.py.tpl similarity index 100% rename from src/py/flwr/cli/new/templates/app/code/server.hf.py.tpl rename to src/py/flwr/cli/new/templates/app/code/server.huggingface.py.tpl diff --git a/src/py/flwr/cli/new/templates/app/code/task.hf.py.tpl b/src/py/flwr/cli/new/templates/app/code/task.huggingface.py.tpl similarity index 100% rename from src/py/flwr/cli/new/templates/app/code/task.hf.py.tpl rename to src/py/flwr/cli/new/templates/app/code/task.huggingface.py.tpl diff --git a/src/py/flwr/cli/new/templates/app/pyproject.hf.toml.tpl b/src/py/flwr/cli/new/templates/app/pyproject.huggingface.toml.tpl similarity index 100% rename from src/py/flwr/cli/new/templates/app/pyproject.hf.toml.tpl rename to src/py/flwr/cli/new/templates/app/pyproject.huggingface.toml.tpl From ae57fdbc7740ea5cc2e94f2ec8d4e41bc2d0bdd2 Mon Sep 17 00:00:00 2001 From: Chong Shen Ng Date: Tue, 23 Jul 2024 17:29:40 +0100 Subject: [PATCH 59/74] fix(framework:skip) Fix `SimulationEngine` log (#3888) --- src/py/flwr/superexec/simulation.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/py/flwr/superexec/simulation.py b/src/py/flwr/superexec/simulation.py index 5f0b5bf814fa..3c182fd8b002 100644 --- a/src/py/flwr/superexec/simulation.py +++ b/src/py/flwr/superexec/simulation.py @@ -152,9 +152,8 @@ def start_run( command.extend(["--run-config", f"{override_config_str}"]) # Start Simulation - proc = subprocess.run( # pylint: disable=consider-using-with + proc = subprocess.Popen( # pylint: disable=consider-using-with command, - check=True, text=True, ) @@ -162,7 +161,7 @@ def start_run( return RunTracker( run_id=run_id, - proc=proc, # type:ignore + proc=proc, ) # pylint: disable-next=broad-except From 3c37de258ef4bc97cd0846031ba67b381cb59e82 Mon Sep 17 00:00:00 2001 From: Charles Beauville Date: Tue, 23 Jul 2024 18:37:17 +0200 Subject: [PATCH 60/74] refactor(framework:skip) Improve SuperExec docs (#3889) --- src/py/flwr/superexec/app.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/py/flwr/superexec/app.py b/src/py/flwr/superexec/app.py index b852577d943f..2ad5f12d227f 100644 --- a/src/py/flwr/superexec/app.py +++ b/src/py/flwr/superexec/app.py @@ -93,7 +93,9 @@ def _parse_args_run_superexec() -> argparse.ArgumentParser: ) parser.add_argument( "--executor-config", - help="Key-value pairs for the executor config, separated by commas.", + help="Key-value pairs for the executor config, separated by commas. " + 'For example:\n\n`--executor-config superlink="superlink:9091",' + 'root-certificates="certificates/superlink-ca.crt"`', ) parser.add_argument( "--insecure", From 06417b9af7f1e46b9de5f8efcb58afa0ffe789b0 Mon Sep 17 00:00:00 2001 From: Javier Date: Tue, 23 Jul 2024 18:07:17 +0100 Subject: [PATCH 61/74] fix(framework:skip) Unify default `client_resources` and deprecate setting GPU growth as input argument (#3890) --- src/py/flwr/simulation/run_simulation.py | 36 +++++++++++++++++++++--- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/src/py/flwr/simulation/run_simulation.py b/src/py/flwr/simulation/run_simulation.py index 7cebb90451d6..51799074ef6f 100644 --- a/src/py/flwr/simulation/run_simulation.py +++ b/src/py/flwr/simulation/run_simulation.py @@ -32,7 +32,11 @@ from flwr.common import EventType, event, log from flwr.common.config import get_fused_config_from_dir, parse_config_args from flwr.common.constant import RUN_ID_NUM_BYTES -from flwr.common.logger import set_logger_propagation, update_console_handler +from flwr.common.logger import ( + set_logger_propagation, + update_console_handler, + warn_deprecated_feature_with_example, +) from flwr.common.typing import Run, UserConfig from flwr.server.driver import Driver, InMemoryDriver from flwr.server.run_serverapp import run as run_server_app @@ -93,6 +97,14 @@ def run_simulation_from_cli() -> None: """Run Simulation Engine from the CLI.""" args = _parse_args_run_simulation().parse_args() + if args.enable_tf_gpu_growth: + warn_deprecated_feature_with_example( + "Passing `--enable-tf-gpu-growth` is deprecated.", + example_message="Instead, set the `TF_FORCE_GPU_ALLOW_GROWTH` environmnet " + "variable to true.", + code_example='TF_FORCE_GPU_ALLOW_GROWTH="true" flower-simulation <...>', + ) + # We are supporting two modes for the CLI entrypoint: # 1) Running an app dir containing a `pyproject.toml` # 2) Running any ClientApp and SeverApp w/o pyproject.toml being present @@ -223,6 +235,15 @@ def run_simulation( When disabled, only INFO, WARNING and ERROR log messages will be shown. If enabled, DEBUG-level logs will be displayed. """ + if enable_tf_gpu_growth: + warn_deprecated_feature_with_example( + "Passing `enable_tf_gpu_growth=True` is deprecated.", + example_message="Instead, set the `TF_FORCE_GPU_ALLOW_GROWTH` environmnet " + "variable to true.", + code_example='import os;os.environ["TF_FORCE_GPU_ALLOW_GROWTH"]="true"' + "\n\tflwr.simulation.run_simulationt(...)", + ) + _run_simulation( num_supernodes=num_supernodes, client_app=client_app, @@ -264,7 +285,7 @@ def server_th_with_start_checks( """ try: if tf_gpu_growth: - log(INFO, "Enabling GPU growth for Tensorflow on the main thread.") + log(INFO, "Enabling GPU growth for Tensorflow on the server thread.") enable_gpu_growth() # Run ServerApp @@ -475,6 +496,14 @@ def _run_simulation( if "init_args" not in backend_config: backend_config["init_args"] = {} + # Set default client_resources if not passed + if "client_resources" not in backend_config: + backend_config["client_resources"] = {"num_cpus": 2, "num_gpus": 0} + + # Initialization of backend config to enable GPU growth globally when set + if "actor" not in backend_config: + backend_config["actor"] = {"tensorflow": 0} + # Set logging level logger = logging.getLogger("flwr") if verbose_logging: @@ -580,8 +609,7 @@ def _parse_args_run_simulation() -> argparse.ArgumentParser: parser.add_argument( "--backend-config", type=str, - default='{"client_resources": {"num_cpus":2, "num_gpus":0.0},' - '"actor": {"tensorflow": 0}}', + default="{}", help='A JSON formatted stream, e.g \'{"":, "":}\' to ' "configure a backend. Values supported in are those included by " "`flwr.common.typing.ConfigsRecordValues`. ", From 7a89b4b3319a96b3b068863daf89946b94b97413 Mon Sep 17 00:00:00 2001 From: Adam Narozniak <51029327+adam-narozniak@users.noreply.github.com> Date: Wed, 24 Jul 2024 00:18:53 +0200 Subject: [PATCH 62/74] docs(datasets) Rewrite Flower Datasets quickstart tutorial as a notebook (#3854) Co-authored-by: jafermarq --- .../tutorial-quickstart/choose-hf-dataset.png | Bin 0 -> 808670 bytes .../tutorial-quickstart/copy-dataset-name.png | Bin 0 -> 392600 bytes datasets/doc/source/tutorial-quickstart.ipynb | 550 ++++++++++++++++++ datasets/doc/source/tutorial-quickstart.rst | 99 ---- 4 files changed, 550 insertions(+), 99 deletions(-) create mode 100644 datasets/doc/source/_static/tutorial-quickstart/choose-hf-dataset.png create mode 100644 datasets/doc/source/_static/tutorial-quickstart/copy-dataset-name.png create mode 100644 datasets/doc/source/tutorial-quickstart.ipynb delete mode 100644 datasets/doc/source/tutorial-quickstart.rst diff --git a/datasets/doc/source/_static/tutorial-quickstart/choose-hf-dataset.png b/datasets/doc/source/_static/tutorial-quickstart/choose-hf-dataset.png new file mode 100644 index 0000000000000000000000000000000000000000..ffce2008e178883307b951efe0f77b3ba2730b6c GIT binary patch literal 808670 zcmbrl1z4L+)<29DC@of6C~hsZMS{C)TePLPI~12-K}&F_1&Tu{R$Ph`+$FdZT#5w` zo-f_qcXyxX-FN@j^?b>lE0dWy_sp4d&Tq~+b09!gkCORK(-mZnp6cCfIvHAh2x9hRVpqovkInxPvXD`AC>^&Jeu zildNti_7MJ%KnP}KGECAfGiVvvL||tTCKsQ;V;}$?#b@4hT~tGNJ?sxJ#gDl!rN{r z^LBtZ9NZu>xsie{2ZPP-XsW_}#9_|wfN00yd#hiK{Jh>fWSLAayWM0N=#Mit&@y59n*8B=dH}^G$*EwEl1k|Z~r6Imw!PH5MC6X-K`Qo|4qep;RdHlLd z*00=Im=%mMBt+O1=$(@73!#qpGBfCLO{5=c;tDCLwnhhPk*&(!+_Y*14UrzLKMev@ zvr7+}yt+7{su(Prn8tjgETWc;6pDaqY@7a&)piOaQlsMoJbor4QH)n1?W=72wcWz@ z9Vv@RuN-SwT(<*((?i3jdw}U4Jnl|OPQ2@}rkm_b6WvOzJ=}X@TCq>a7@&kx{Tscj zAuesDhe3lE^whq-?=MF5JZ}6YE*C0jARJ;E*(Z`8mT4JjBh+N--{FB4zGPCeZYa^a z^#LN)BfCiOmOp*jT_gYYsX)Lp&Ep*(GtXNZLI@QYlN1-tl`epnS`m9zVg#nSgacm} zmGsB#q$4>DX#PY_5%*%3xZQ^J1%{tgC(dQZ+KqX$w<_*sSx52BM?x;M=AcSq!R42o zo^%h_+d*NrUmRz2F+4EH+r*n(j5gXmT#OUWks#mD+hGoRl2W5*(loz5N+)(on4ocj zpUPCXojK7iJRl6w6xT|3#6IBUiGiPk^T7n|jP3YRWGm;-Br7iRU-Sg97tnt1mA#6o zZ)MBQV>yTcNdpTacbBhPY1{*Rzt#ax_wck68SJe!w#BWHcG%(~_(9aOagp)zvA4Zb z;xQk7sspJt;x?~(ZhB3XXovZ2>b5MA#k7^sK)V@e;tazcksE<<+~-DPsgP&JOO~io$m1U zoPsV+-H(>$u>Q6)$eAF?0QZnB8{0jFh6BT*m14N(ejPes83#l1MJSdN9l0WS)Upl8 zj;+?F$U*qUpR1gz7@e}!q8xt@1Jov|N*EoWw#-_GRq9W<%&sT8xO{IL2i|IlCHWfn zNjAM#D5pul481Zg=8+^uoZ~a>H*#aKiZY?EW0QNmzdzItvz4e}nCwOWPAnLtCm|F& z-z)u{%rnYciu1$0Nh$ny3a&i;63JFiDMFMbxJO@F5jUXehnGqhjfh%Nd*L;MVG@la zWZP)i9OjBR+--ZD*xl_66^~glNldFM-WO99wSTN&x=1TC*4LsvCuUydNsFHEctZ85 zXHCx=PmI#=lS8Zf3ebba7vC=wv9^T!;S+jW^zs`cl6c%?a3T1Sl%O%r3hv6o1%*Yh zy9~p-5Gz^N5D;hbSEV#bJ*kD)3-Tgw_UOcz2x4D96OE_tOMU!iH5Tvn^Y3ThMZa5& zslVs{6&PO`UlITLeeC=2c-2C!LOtu+vHQ72Z=qkrK2P?MnyENHOP0@2HjslU+cOw4 zxqlW{<9VU+*x@bJ^Hi>w%INnBg3|T!o;k(EwmD#~;ywFMpDQ7utx z{&Dp_s@NjC`$KAh#&?ZUx7YiXI+;S+i|^BOhu>wE7-Yi=V1;5?lH*TvhzsNLD`ZS- zyYEHOD8=XHT4XV)xCCtS4UMP7SU(+#?EdCtePTUtZE3ZrBg4kRR-;j)kvwHk;#T5o zr({RAkF{?-_hhc;XQL|zwE}gGa_KNvdY}+LP%Mgrs*S3Rw!%YZKeUJuwi`WhHomc& zxN|VxoPGT!rj)E8DW_Jd%m5A%D|INMEfH7u`+P@!3yITVY?gDDPmC53IF3gDXk%A*#TGE1?Pl+W(HP}Mor9YrB6nY2J!+>~v%+$YY;Jl++;zkW*ZqZCfxCy`oVRQL z_LCfF)ZOOx&ez?##iN6#L!8;B;XOQ2p8ICIoxmTC4F`%q0_Tjih2@+6uHhhfW#2|3 z%MdVaS8AGcy3GE--lSHi*1+8b!Gm~^V3ua@L}Eqp~ir@VS1P-$G?wu@Ey81_%J;YKaomO%%qs=X#DN2b0MsU zxln0UQRs`@YX!ym^z$H+zQWaq->b%1#;@yH4|EP5WY9YBmLR*tS)e-@T; z`}XMF^ke8to4nJAj!xm@xSRWz`1f=UvMmeqqi$$}1>9omE1x|WPpcM-6C!2YRncZ~5ArOCeXc}OMG?$(VrhIbu6;UQt!M6@bgmM+)!}p0H5C+ z!;G_y&EZtTtAQWKcAa)X^BMLkGrkA>-42t&r|w_&FB;V9s%$g|OQ${}c6JuA8dQ(N|rGy*S$X5C+%Osplo zVRKhuPKUrK7Uz7K_69GjBrR1kO*3r7;68UV_n;4=$jHfd0rmPnY+USwj~jR4@^g!G zyX=jI_I0aHxrL^-Jz+%NJ{V(`o7?0SRnA5 zs6i`WSuhr8k2u_y-s`BoY+WTvoW)S5MdT)F$8s4TOd0(~{Fdecs-j?ww2kz5{k4bKZ7Wr04rEBiM3QDC1_Y?>!E!(a48r z25>C@l!s{e8PFGN8(ZSqlIxZlWzj@fp+h2Wi+(6=#>8~zej)Zk3JY^Xd%{U zSpVpwgZlpcibZ`;)c^R#j1NY`LH&A!`hc@A{-^i-!z|4IDPw*^wV_F>Nxyl6`c^Y_ zHaE9-v2t*IoIj?Es<`j?TH6H;jr95N5Bi(;&rtCY#)9<+EmtiiMIloMJ9c9;2NQF4 z4?D--_dyf!5JDC0%w3J?JnU@kU4%SD8UEHo2vz=F4Pc=ATNhUwQ3fp~RXS-0XLC9} zb`Ew91~EK3Iyw<&GYg^jGIIYQNBt6IuyS>E6aoO;-QC&Ux!E0@EdiW@f`R}JE&vx7 z8>$DJi>JM-u?L&I3*+B!@;~>HF?TU_wsv&2cCe@WeP3e}2cWAc1HQ*WZT|`F%4Xbr*AIX$Lzyb9+~@ zf33L4-$wdF7@i2=pPDO%*V2yRiiReE_C`kX zg9rLvI!=?mdUEZw-LGaG+&8Z=?xQ~vdeO`%OZxzkf|DplCf#Bs|(iu9G%zn}QHGvZa*rOE2kY8e&UiL2uTt)&zY!UOXYCAsG zv*lgvZK3afN!R^RGRp`3#c5+T1fUdV5luhj@n13(B9zQr3ID2<@)O1Smr-Wz#{R{X zLmBzOUz|4a|8FDz5%l*p&$7AC!_VKS-{{uZuy|Gq$!_E2<7>#v+bm@RNB;=|mtV24 z=&7qG6cv`b`CL!b`QBdDg@@zC5&w5C0@xWA3+CnXikUgXkFu{t#M|*%8Ts*> z9ZKj5gP`kxV-@6`17uMkB}39z*cZT0|z(zBY(ygzAYE=lk6^@wS;kfr(}7-+)= z7kEM35iHPTHSqs{-K+(ap|kM6+hG1Jt_(D4A4)#^`>V|m9~(VCWJuR)lZRM6+p{UZ zKMj~JM@PEn6zeg&={_~tD1_{hNQ+|(?k6$K*T*+!{e$TLM0w4Qvd{BHn=)=PbmDCp zj9ork0%I4YCQFm%^@(t!T{~^HKP~Jqvyqgja6B9LPfwB~VrBQ!UgN|Mf9mwVwCTlz zfStl>=*v3;!q+iD{xNc6+I2J^9X5H#Jk2 zf;F{iS}qTx((gKe|Hc!FMOhLQZdjSkN&Xg}GYOsR!TDS|y(H3?ZB>+Hn=@+bjoE;; z;O2Z{qQUN;Wa5kx-b4;8-6nNIZV}=OQqXVlLqhO!ZH#!H{4c)#wVMPy!ksAd9Gi6T z80NLrS*56L-lMRYuHRhL)pwEB?=AV(P*$|f-KmaGwf_TK#c{GDzb++?TW@G)_BmC_ z7W^AggD8@s5R968b7<=(+x%j5dZQlF*GInvGAWs5DJ7D3B$97FV4eTfL08ACi2#@~9qFzUJ8RGGR|i{LXWg28+~e+rlYir!(D!ELqOgBICG|$aEP=%(weLheH;KxxRp1bLUbq7FfZHaaf>?^Q-I7I z)(g2J{Bid>u-p!cT~cdoU3{)4>Lf9*upey#0U4F_=cQy&c*lk7&L2!+gbhovpr_3) z!}>U8f8Kqfez9?P@i04(I@{I!4;}xP+h&JC_XD?|Du!EbA}1^@74>$XDoYC;F$qAa zlpdXpda~-7m75Fd$-q>9!fb<%oy*eE9v=G{#<>CDn2)GYF}%3Q`T3~VZ!_C=!ypxN zACZ7p{>&b<>0p9(&b&A?D z=cv@AhAaFCz2@m8-l@_s=e~>*DvoY^p({f#bE`%6Cx`TNMWCGqw|k&jkE)~}w_QaP z!n!69TUMP%C~nZPrtD_+c6>r$BlMX1z{g_KSl*9EWk7sU5$QGTuu{)IutgMbOw7?{ zkd{*tkv}+Zh-vIvJJmoTZe)oJ~ewlQp1Zo{uH6?(HpgklG=%p!e?)S zmFC-NuA?Y_TB<1h34#Cc(`f0rYRigXt@**$d+ALAk=1njS>-aK{qV*9AoAs^;j=j zXl^?r=X+@;Z?$L_<2%i3>mkNE@ApK^7GBQXGQKA3ozAe#Xl7qR22VaqcKcTJG!4JE z252@{SDI-96C4Sd)~BkB3oT>h`B{eYs^sr?VnF))`y(k9d|$_pN7JZWqu}%3!Q@M( zf7r!pVc11VN=h1*_l9LD)t{LEcmC#>N5-+d!9$BYTOD>H@H63U5n9N(%N_WY-tAO^ zy{2h`O>y~1E!$S|gi1M;_vz)!;7yDwS7T$S-%-O`T7%lkdX&{sY}N8zb3hE;8x0yn zl@AU(`ESx_&$k^L*ep}k_oCT|xcEuolZq)*FIPq1YV&#}6^V@$z09Yf#ZxLUXu9kB zJ)#X8qw9j8l^uuC;=JrbR`6tHKkh#de2c%^?xJFFmw4?Kv^|09ADtH#Gnb` z>34`1GTl6vhfaP(vtM-coSMOWP9G79I8InNF1L0$Lz|wk1Q#t;GEjXsRSTX;aVtJ+ zhm>^|c`bNr-}Cl>Ny}JHZH>Gsb?(I0-K)0litQ@h%<7Q7!-qV4bOfaM_}~4t|GX8I z6OzoFAJ*#MHUw#75e=j8S=v~CKiIsq)XVHEe}%rr>X&;F;qjG)N@9u3<3^a>jlp(B z?X2dn!!KU2m>(2sXrLSJc@iEO$hMT|yTKAV41m0!ErZwSS35|u6o0F-7e&BejJyiV z*M7!7x?V)M1;eG3fAh2cT?_ub>`XHAb5FU}<%muDK5nYbYV$s&M%~p9;CpB!!4xZ0y)h|@*@#m+5C%*)7b;%{a1gTV4-oG$u(>p^qZTQe+ zX~Ysz=FF=WjThm85jkLH&9g6xML|iq=!00?^1acAhKhOA7^!-1t*Z3JlyNxVBbJ>7^;$yaBY;n^*E zcd=$akjQ52Gscn`hEN=_97wd3MaCI))(Ww{XX+=VG~c4~N>NYBk-*(xb+sS*PA+D9 zkm78n`<^C2I;s9g+-Lo@;M{D*$+c)#b<-Humw^2VAFNGiCVW^JE>i1e7r~Vn--Nt{ zyH6W{><~}h%9H*Vo&AqZ0X|iy+h79GQ`PA*Lvc;@Up^E2vrEUk z#F-oByZXq{Qsdo$QmBDGc+Dt=|T_~ZL6O&I}^pdL40^g(s4!Q{iEnI28Nv#LE;ehv|+H^ho7RqJL-wiouEOrlYa6GVHhEKC)orF4nKOOa&7dNDzI!kDhmpqeG@OizXNJ&cs*R@k^FNE&AKNec-$9N? zYb{BuLlQoAbPr|AOioVPZ^ufSvn|?-H<-@X%*9&L8VXX0co=D)z>5dnuCA>@S~AP& z192|H?X-V*rmO31u>3>Un}hv%ub#Jt~;jhr7Y;$?+HSRcNPcRg?gt zpAP`%3V`OvQp2b4rKSDaC0}zkBF6P3c72ND;#2%zpP5zIZSFuW7B{6@0$IxeM$kTu zi^88T+a`QN&MioV_=$UEjn?wm>iNv4VxEfeaMMD|9ZKz-gogeO$B*TWuFt8XAqA3w z*kDM(>bb_pzrhyD$Nrn2yq0Wep5EQc2>bO>Zpig)UZY_V)CoGOwH1Lz*9`I>)!5B1 z#6MS9Knd~@7=%OXU7mA`fxcAmF6UD@VZf3k4)uC>H1+Ne;-h`egb+_Otm3SE>9`B4iPX$zJG@0c6~QAbM<0XD^mJIU(sJIf6`{kJi1|V8dIBw->n+#P5vt7 zmo*kCo6h>k$H6qg@$41}q7MY~KW&m4FE>*=jPh!mHrUwjHebo#^-E)UaRpFE+-*lp zr@b_pY_0h1QH5+Ybbr)*H~T6lq6_%!H9HBrJFLmmzSSRH632Crv!`w-mA^4K7@Oa$ zmx$kXrhFt=^2Op-28ex6+;>4hGp@+)#mXXpsw}VT-2?N5pX)Vr+cqz~tJd_q;>jS< z4=&CrXcXq|H&u@G$ zH9ySXeFgC5(P}>KXR%XWkb6)&yR13#`g0BR>Ws~xdhC3bY>L{cllb6Qex|+459SOH zlT=_cP`30sH})h<`T<$IfptTM#Bopl!t=U)IKmvu%VR#Zxo3dp!~*w*1IzoSlHcXF6Qyj2(*&ct&229ZR}PA+hF_W0vcKu@>}$wCqZxPij6nwi z8zZjRR@_7Qp4?qff4lu|xCANnOKwJ#>HO?p^h4g-wT^^ISw71kI0U~W2`FVa>|L|F z#y+N|te{O~x-d;IfwL0RLSGq3qc#*;k&vdt_6N}%hVryNt{3~uf;)TXtKQz#CGNxp zC^vcWfLEpA3hV(L{?;9Ffj!*N4-EPW(59{A;*iC1Sq{JnUcwccHU|OFPwGO90M^w* zi7sH^j8>+uEfeG)2Igt=&vM5)c|u=?m5183^6rVj)i zabr$B)MQPdxFzK660)A9tf`VpyAgFdH085r3Vr}=dpTZdKEN|m%xaeK#(}QfYWcDB@t zL**#(y4G<;3Msk(je07`RrHYcJ2hi50s|sP`?j_}WGuIMp61%u!f@EUjQFq%hIl1k zd<5p+nzdjeg;x;MFR1}|>fydkx2geqHmjoZN!B2Y)`P=#`Fb;I=EEol?+$(6dOwew z0;4Jf1ks$0VKI;r-U@lUyw&pYmjuU(0lTCDl@J-a;bHHVtj`)$tX^T1&7iV<5qU^r zM^RucpnED6F?Ll>KHjud-_cO5_xnUk;s`6TMC%ttDur~&ubELhmY%My8JBadD!0IS zh-paCk7DP~1gs8P6&WsxufqQ2s-G%i2sa}zgs>^S*X%&St%UTn^o>^4`>Y)~7#WWT zqlI<~I9@Fzx+symdLGnoS@4lH7+T;A?ImFGIn2(joP5JHO(Mh4xQzbg{NRF=RVxd5>;9kiAxJ*ZUPX_Tn`>{9@Tp6EzxG=(&`EtlJEI*f$$YNe zz|YcH9|Lb|S)${WlCrdJUZ3ek*hc)jpu6Lw7Bd2uZgR`I-O|R=@yzr=bl!26`*aIM zi)%d3m+sT;&5LYxl(K-m`>s?=omYYopB}I~E+Ov_(pHUnzZy($0|~{pcbXCNoTlA% zIoG14Y6ciMTi6!$suJmx#p7|6Y^afSA=~-q0CNSAuAT8ltgWQcPE*IIswsNh~DWJ^LlJ&hx9WogKmy`xMe%ENia=_ zp7zNYIWU*o%%+#tmzuXSZ0XFp7wp12MojQ|J5bN=xboq%sNKy&d>ZHLoJ=o!6buU1`Q}bNNWc3!6 z0NHTi()SF`!%AtcQ#o-AtX-EwYlEg*(Lu5kL)QL*+Dy;=h;h0*8|bT$TJSSlKza>A z1mnZ0^ zz22+Zr`b{QyHabk{$jMm=*;Tq|?D4q6rmr)MWKqTU_;c}af^$;H z$m{%cH=T=)oDG;6O1prY!1n8p$7RiH_|#l=N{@W`@rbW~&T{sIC$MM+U-P_#DKNF5 zv}(aPgnHAyZY6k~i7i5cPPM=T&}}XycD`>fO?NWCbjK&Kaend?y2093(773NPTB-I3?%Ni2?mBCdJ7>COApVy;ppC&*ADJWKJIs)C#Gl= z{Nbg3SNVR1H!G5A_L|Qp!s<&?QEM^4?ti}&rq@6&u;n9-5J9e z;X->>)x~g7Wi-s8|I&9Bv@~Bg!K-<3SdY7Nexog=tE^mWx40u0LqpwkmGYV}o5s(A z4_e(Had|j)xl>dYJd|t}rwXt&@`N@Fau!`-0ldHb%pG*Y+KS(rXvp2&5?c4GHjcU8 z+-ykF@5!X-}fi+;>%)-_hS#PH`-r61|nB671u( z*P1dncGZn3sgHitD!SQ4X1FkFTit@Y$BRV-UwK0)-K1Gu5fcR4SSunrGOXwc9dD_0 zj?%qYh)7nK#wK))bfo6D_(DioHD99BJj|wl7Y>>Xne4IxGd&xmo4(ABw(`oM`{BGVd9*t42iMrU7=scxa7#!cp#1Pfs}mpng({jbBVe|XU`Q7%Dc@x zA`Qdg!jf|E=%AE(Dj`#b`v!8m?+{`cy0tYbPrF~vpZ$SIXSV467Y0~_%1qNae%jfA z2QYt-6WU6@_z&WX-}u6t$1Aw#9XtE%Z~V zcTo??^-tP#NTl#lHhOgBi=7oNzlq|2E+pZuXL=vML0m5(#lEcPCK)(;h~GkyH!28J zFjX0$kl*n{3XKrtQQn0MjL@G_w30_FVEo1c9)i z`ZpR$1HxX}X7(mVL2pVcC|K#`x6q)si)A&HI7FnTZ%|BbDF5joWhAl~FDdclXFKED6#< zC@un?rCx6?nwppZo91nhmql#NX19YLbiO!=w)CUiyaaeEC*v=2s8#v{SiBEK^aN(xUSQQNfPyZo!@2Q~}E(m&e8Xd5jwzYYu4aec<=X(f87 z@$d(Ml-RtM&oa7XtEGvw?h<9b_ntOmSVE=;lsV1u6I<{vdw_r2J=?LTE4*CKNvm_$ zaDrz}PPU+g2~;LZDw$%7#5DB#I8l^$yk(wBAE~^2t*mG00sT?e3de%$vC^MXKMGrj zL=7NrPN0pGMOB7*U(ly}DvsI)Dh5qxItTB1#P5jZvZ$av{lL+g;2%9fZhXo$$(6 zZTt05<76KmRN7~x3U}`?T^3ZezmL(?d^z`P;h$L^FWTSP@C&}+;{;0jWz<%@5P;)q zuZ#JP0dmm=z&2Lb`SnZWJBcn6Mm;WF>w|W+CEo};#5agkWTM;oJ|3lzs}-(pcg41V z;8vPTyK%|qCn_dS?Seyfmae91gEyS- z_iZN=mSQK#%NxV(nQQ3y^oH(~K4z7xxWyix3A!y#E*kpHmEX9)y^uW4L1Fen^C$dy;`yA(lpf9Q4XzqTpTO^hQ5RnX3oyR)g{O!)>_u zZgJIHA@tLSUFR`mURf(0AvmMH*}oD>cke3-ZnyGbh+(7ESRNU5Kzy$2-Z9Eo*A9qP zG{fYr?Ch=fS&EO1t!+@z{7@9Iw2;duOZ&YHS)kN>M;4lI+160%)>N_f8a6(~`c#=A zZ71mAN9hGkRQ{$wA;)kI6&G-Sf_Z!P^i*8-L!_m&Uvir}6x&_uucZmprc_k1S=TBl zkRED>70J==Sx(QJd}a8iFlP&>6#I4x_}OL`qdv>$q8(W9wqa_b?mE3M>WAk z-89MC2X@@FH_JHlJQ&gB+)N9-NA+0}=HJc6?)I1NhI~-DBc)P;9~u<2g{ovW{EMz? zM5bNWw=0Bxjt;&jscY)@hI%%x+X#J^eQ(xiYiDCz)_O+vjt2nlk>5h`6WB5M`u#)M zhEV=Ayx#A2+D~cz@l>3=VDh?%DmfqB2eZ!I>6W{*L8}(MS>Im@hR&IHr}@-?4)2A#f*UyU&LJei$RML)RO>b_#`~G2=iP?kU6;?cG%a2Z;ZgMqn@s

krJ^quga%Cq!fB)2|Ltj^v$tBotB!) z$2ekQR~r^a5h|>`*z&im%9h3sI(ZENte#q*=+@df9Pup+zv{jO)vECPXm29Mq}y@JvuBx#|I#vRRQ$RN7T9Ir0t0ku zx$9k<^d}`Ix~v;)WbBXUcXLydV@+CHf8O=59Py}a=DUU{iQHY2>t!$9`i%XD+f z5QLEP(7bR>%|A~VP9I|E`uLbdA(^O!TvZQcG@{>+j@CJjK5~a?GuNX+Z+7`aZO@Ur zU`=u%3|fz6bob-FUj|1*YdFH=_VHAqsR`C50RhFRLP3qtL%OejiPbt~P;IP;IEv)U14DNF+BWGeU;6S|wS>keT&%P@gOoW3Nrxk*%IUh@&Po2x#2NSZ3| zpseE8DufI{jMGDI?MvV)*=#AXftZL1wp&(YWh4ZgVZ@y7`o7=+6)F&HTe)v8)hWLO z6kBWV6aMT@i(os;z$6f$TVgMqcNCKL`Q#ebNMK|>E@i~Nbxv2P5eURtM)$6F+3mQcIF)}9ZdcW4NK)l91rqfgB)_=)-n0rd8 z^B2#O)bdbL)9?{93t_+&KGfX&LzO(}Cura;yxH;t(bV&nAZPY#I2G0BqaK$Y&{xYq zA=Z!CiS>+j60Hxv>JV|kEAw8$06<&XoD`;d+t>+7{O5bPwQ_>A?- z=QpW?15K-_M87Mp@KJ+iDE;!9&e{fG8b>(Cl31>0x_9v0y=jelC6~hG#t#Wc3QFNr zTZST4-*UgL(8F)rr&gQS#dS>}gZ!{EyVzvY6$WRz4|T+d?;B_}4bo$W+hvg@q`P$6 zIERlIoYQk{ieB0hl{{Ob_rMUjt|kfOU>zRcbhsJcmkjOc*-16(Nu_bJ_-km;eBmb?um;Sm^A# z>Ee3W^?Wv7N=-%wnjgUyIF`OVazImG|A~lt+U7~m?^ssmuB_8)76k91%n^(nH_*^Es^4h)CcOtIe(!<)L zb;)nQXUoj{U_p0*EL3N{YB8fDM@580#4wNdhhkcJrts~$X%$5f90IR!1n|h5;ZoR@ zUY`vmcQ~ma$5VC0KfM)HX5tpF+78Nvq&;YFep;4ak>cVr^gf<_>r}87U zd+N#6V4O$wPB;A5mN+Wkv@{K_uW3@Y47`7BI4+Rm(5@>BoQHB`-oOcK2k(DUp2~FJ zv{}56$>)VXDizk-7hxlW1y&KJ9uQ;-4r!Px2^t;2?SEPl_g{9<-i<^&52=|S?uh=f!D4v% zHWfYDwHvw4EZ*k{)5cNK&MMh2GQ9m^+G;;GEQ(m*c#+UcT$!Qb@4$sktXp6OHm^S7 zBoeU3{C)O^a}>8vtMZGLVQE8e z15FPho0rk}oDUIasAM%w8No@H7-6EMe1_ye4!_C~L=%qeH(!?(fR!G0CTUoFlkDnl z^lk^=Kl(v14Z`yQyob}B3Y)NU`!*i#n0JF{9PBJ`Tkrq;m@RB43SH8L9Ve>FS8h$# z>_@3(C`4ZI-U?k`^Bu%7>R`ui2`}jz6t{Tvg)d9A&%Hazg%8@{UU(WNYeK|POdPb~ z&?-LB8L#!(|2ZA(Zm1*r34|ffv`_x&Gu(SixNz9o0C_bXawRnun0ruv^WjpA1Kse3 zwQ3rbR3j^W`Asat1`-InJ#e?2ym!AZB1v3fz;}X@xH(t5Vw$LI;%_0b1}ak`TNktH zy|&+^sVs8T>F9T}O4ed^(j26gL+HJXd32zCvXDQ~wh}jibr}7Y;_`{}5))#)61Q4D z%XmD#_!%!_e#i!sX2RK!4DuaF&X5Yj0yaCLQn+?J$QA2&x;1moija>Q-pHn;er@nRS=NfUMU)Np+|nvuqrCML)fnw_gaNUnmmvc&#BYKLIw^ zvhE4*`s7qInWJl;7CgC8gxhn8nDVqg$`zG}v3mG0zJQ*Py zo*LkWg~_Gc)EBvcbwVJh=Nsp4Fs1!#CAoWva=)Y~zV`VOrqy9M4zSfeeg>-wRLfqK zemf-=FbNYzpCX2X_GxpQUYeBQgbExsd-6Q7aqx+sKKXRWP@YwS? z3rIxq76wNk8fozi_Dnd_o_FR{1hx0hgLchHJ1 zYjl-&pXKBZ@E}+mwkf^7S3apd9+k>>?b6%<>el@XqLnbav&30rv|_^F zbMiWounnd#VBiE64MxD5pKd<^cq>wTDoed{&Y$0x7y3>PSH%({6g~eyc|`L|5GOjY zAa?ij0L>AGv=&nr=zIP{D#pt@<;|`N0nPsS$M85(xHVPaeiw!L#pkV9CKw~&pij_U zbeqQeyLq{E#bthO56KGLA!QAf@}8CR?`pm`olmw&<{~ZxCr9iqA%~g|84N_nDw(r$r=d7xsi9{0>K%=R*-Q!$A4h1D=j$-5UM8ODknh5H82Q zl_%l(R7FK)C?6xDJl450a_z(d0u%4h&}e$Xv=6Nv0^(A>`OE_gj2ZIlrpHjEfH#XT z93%KvMaSZ%cYf|JIv`zjiMF02e!3I>I(&9DJrB1&QJjZkArxOui+C!s{mYXT!BjT{<-FBI>z6$d7dJuVM+T+U8(J#QFn7^p9)g+czYk@Ki;v577->^{ zJ0U-N4i*F_DTE_t2v@LH>4m%wvEv7}9Yr?2s7d9kbKzq||IQc0xcAu|aV^!$*k-Yk zUa`ZXL&2tW6R)P059g*KeCkc|2JQ<^A;OzqO+Ojs&NL-JK)B1Iao1g}p2$_Z7!c)X zH}ACGTKuZ$dCoquF;#ox42|b{+#%KTc_!@|aV(Xm(0kwN)}n>kWr*LnHGB>>U5nz< zw6f_8puIZ`QJa^VC2I>7y(I;L2edTIBpqC}zkjn1<`0i~BG3z!x3q;IYdqQ{?Yy>USBVURRk_^gs- zT9Ign@+^?#U-l}1NfjvO4|%|797;{{UUMt;`7X03P;pw3)WiADkKq-BErgb7PI5#A zot|HgMP}}D#R6;m95NHfuMJd;;`WMT=0^wK!C!7R*LfVR9#S3cby`3v)jCBjR7B4{ z5P_JxK(@^L$U8U^8{T%By}wT?jBKrtXpdx2gz-B;}@i(HhnLLh2Qr!Ww_IhoZJYOcDT7Pj=#V<{ZJOiv{B z#_dQLTqgnve&{BF3)a>d>w)JAw7f5+H_<#^2e9e6H>G<$2O93ATQBK89PB+S>1ysl zJXiJ_+X>PLM*R&`j|n4PGRsM~h$G{sC|HRg#Jbep$$u2axwe(!D~KI)=+l=ajZMI? zkSjLmt{g>n97QEK#<>j#lPKfv5*4!hWh@bS;-kaxP-~V=>3G4`M^mLp zLDVQP3eS4y65zkNIi+#5UaUH?<4|2%I#csyVe}w~YV!0cFv0qm?b5)24yS(jq6_wQ&TzMYd7a~F* zyr$aVUCe=({j~9@n=)`N*~d!Y+oM#A0wl9aMw(X({gpQyv_#pS_kF(z$cBaMuFK8{ zX4e=N8Sm%&)q2~kTCianMZ41CanwEdORX8#$?HdLUbTo8Vp{m%h*3*dEc#+p)T6(X zV<&=GA9_Xf28{e(v1 zv|`xhD}Z^GR7&Z+K~+x^S;>2mMkiYtp{Fzmf*A+BL+1c@0105FFjkr4Iy767q%5GU zRB!j55L*;AM=J7^!Tn}5L^qCW97OXqNnJzgV3jTE-6IywGJFiVN8@jLfIhpqTR18n z$q#QHA|7U!4H|Kdh0tY`BL(CON}iMzHXV_e2A)b;(5FseVyOokf=)o!mHQu>o zb~QZT7(b?XjRe@A_O8@deyqB?3(4#a9T+U4%eZm7AS}!sFu?!DZBw+cG{&!M7h>O~L?4 zJ?ZH&3wd993BK(Cxuphj5XL2wBNS{s3OgFE=6cLu0Pj2jGm-bIJhoE%tCpL;LymV+ z=pSY#!ECCM;@$^jL9lIT&ct z?7nbZ=(qtcuEM!t1Jw1KN1QzW{Hwr7o3yV(@}5M+~{D zP=C+P>B@OXH@{DbfAO)9m3_{3QMCrim&OgihQ6TAPSSVNZqnVxvQTYGh#7t^ zlFIqq3^olp|LN$P#@NFPO+$ft`a2{F*3$&pp}cZCVW7N{c9>+Oq39L7}y$x&4~P7&{n84U4>qn>d=(#wkRsebi-GkXhz6 z9bQb36*E3!66$+iRlHXZs59gg60c7permti!21O%^X4Xcl)0UooDx9=x0t&a@;E2z zJ8tYe>EI4Z_u45m?)M92cw_R=$w!EqzP~;NE^Y<@NTI4UOB50~XcDOR4qXoxjTTdpR{q(mvpEZ#6yvjER#3O=i9 zu2N$Be?*;!Bh`QW@4rb>WMpNpjEa!WaTK9KQrW9)S=onURJMbY?0G^)+3Q#b*_)7g zI5@{%$1%=19OItu_}xF@9LM|fdcU5}$K#{-mFwyglQZOZ@I&Dg;HdjAb_$pb#_amk zol%+Tuf^_--t12PNddPe4hHJz7Q)#V-<)}bEJZH_xBXN_V1uwUKKj~&Nm!AR>r34U zSIqzVpPoZXTB5`lFijz4b!d!|KX8ryt8h z{{#2LvWIgO>So0JO1aQ^&@A*UC$kK@<)rOys}(;tZ+wJ;p3g;2EmU-H`viDII<(nT z6yG}qU3dX^Y~F`x_kCAy?&4M99w;7C?k&&7lro0uW-G~-q6H-`^xq35n=|iLNew6r zJAMa|FCpx5QcaMXZsQC$eIt-g@mdAZ5&RqiyVoUwKo+?1Z=;vOIFFcgsOqN)eCIGB z*~pSz4oC5$J%Ta&&brPjiqw-oO;u_83KF1BB=QE())u&ptcNxe&n80ujE*yYBm8tQMU?P&<%;c# zcH;>hKWc*d-iJBN>~t1g*9$l2rHxM_`K)JU34I_9?s{=A|CH2xByueFkWXh>Q1NMO6typ1F?+VUH}g)KdilWID~3TIhj1fU()8 zm85N4$G6q=$)eM*%~6D*LY)kTE?C&@p1qKb8-CGk<>7O`Cd%scT}Vu6V6#2621H>!fgY2ALG!|j17{hXuvUFaxIk|RH_##mPcA}K3WPuvom5p z&m;mV$vs{Lk&FE$M}b!y?KdVf>VqANp{P5Qm ziq?-tu8s-htV!-Qo3BqzP6L6xuh(q~qX;9aC3Oio;LVNM$Os3osrR$FS1*8sHi}0tD~swsjTkjXWWxWt-0C?{#(K zc;|w$%KzsA&^akfOdEN-=Nx8KTV)k;e!P5wZs^W3+sTa#Vx&gOlUp*pyicKAZ2F6> z){R{OCnKu5h+e)Cr{Iqt8)IJ{U87;OMO^HUV4$S`pf97)I}4V5QjDOFEjWO6&tdyv z*tABhZy>7shirZ)&+z9 zxWTpekNx>ZU&Ef@7dT`08^MWVmabUE5t=I-zdwt+Ual4sm`Ro^>?`zRH0Snr&K>(J z*Gr2Q2cBArO<36lrTqO&C>i3+%i@%?D&y#1JB+^U}w&`YPnC%6vi`8t`r-8yKj7*Vkc2WS`CXOEivy{AiHuV*AL=>(k9Ra1T;|?1z&77-?7}!( zrO7Rz4VFV_-LCRbt6%Rl4N6*^_4Qyl|2}E?Of0f~&)Is#haokMo z*=V&bW}vb7s-vx7D6Ppl1C1%O$tacSM!v`{8cU+L0E#*N?q7W_c_F%LylVUU!z>aB zO*M9r`Jz9z(wpv8_}Ov3!Q;uNPB?w=eyGR6P#{-uc&hdMv*hP14jlUOl-#Zs*P=$*2KUK9I#7g1ez1Cw+Cln3Oxk9j2ZL`0f0`}_%W1y-a36KQ3)sQROZlad6* zfeb<-ZM&8E?x8w4F;o%*ouGKebDVi^kEhs5_kn5kXs}H>@z7`y?OtT<>3Kiy$+NhM(V+Ua#cfHeWP-8lh-1rv% zJy25_!X7G#d(8$BpV1U~D~8CwDm9Wi!l^90iTlM2AsB0|TzkrMB8EqI;jD2vyIQ2B zei0ZKE6$j8T1s5E{2mH6G(LCDNy5clvSfD=StR$kN^pH_CT-#2Oaelw+I^hnX3n~` zdTm&5IeFJ6L++#W-H!A9;d*E6_U>;Ft;Xc5dO=nH^#h*F`2v*E$V1Jfz4V(osYJAj z(tMe8`pbYE*pIYOyF=6G_TIv5&TIZ(&KY9wBFXELRf`ESFl=3eTA#qQC*zAe+21=e ze!7eGnQLa`poldxM~N z2JVou#?*K1IZrW3i3IzKi7$KEcxY?EB46h`c>pOqx#3-)z`!qODZseXeGb5qTlUi7 zVp=Ul(u)vkiXl+_AwVe3K_Hka5m`4ceM2Wh=H460FrTW~$5*EK$03XEDe)IWGtE4q z`qrI^j`O3jL`4&k-1t%Xr#nlU1aqA#pB#7Ett8q%Q7vBi7Lr5Uxs~ z)rTC&<&FJThfNpdiS6K%>xBN%pqkDQzlFnBS-U{x#rZ#5E|k=3!*r1!`rT9R-i7yu z91+slROZb0dwP|bJnjp$nyP;N=F3JINR8h|j~;$9v?uS#b6Yy&8P}b+=Q80bEkbq& z^K;0y8aP`;EAq{^ttu_}kiitQ1-!GwH0)}d$-+wT$zyef_4G9Q&pbK!3j)b!FQ0dD z6X-kqkZ{yQC#UnBm{iCAu4)l;tDmZ&;l1R`RC5?}vdNqRH&5zE<*eFTpMIHcu|xHX z5x-VxXEwB5QFw8`vN5Z+eZF-O8&**D@It%Uh=F&YDgM(93+lKz7JLTAp;^A=!bRRq2Z>p(y8(UY zHxeYu)H68{rX|kgyOexSH%!B{!_ic9*jiJc?^X1*d*l$hUcKn01DEG3&HZGZ%5gAE zI&(ApJ$2d(DL`zuT{VEA4Ci1tP;7%I?@R6u3OIAna54Vq?$Fd$7>^W_#VE#y`<7a@ zz$YIpBxtSpngtFeNJ}Rx1hbL1vQD!B@-q(}9fs)$J$1RqYDIt8mzFYmfidN@ffM%; z;d3+3_Ccuxbc1h@bFW3NKXXQWWj&s&l3r~Jf4@C^jk~C`Cb!~)gW>Y6D(TV|442GL zpCbiJ9JOvXzBxh6?B7viX=Djjw%00R5~&4hK`f28=9uO@0BYKq%XJdd%EJhmZ*s#m zpI1NkY9jCjCO@YE;u8NKfuCun-a%PlIUVJjwGitrR0~Ix&FGj1kGhEyu#2R0hsQ#i zMfyN213c7!+vWsfb!I)H1myW$Di{;7rQrEAFi^>BoRZfU5%`8GJVPk|<4i!Ihlpvl z&ApeYSt7JQ*!gpF^qwA9x`IjG#~D6uRcv(pXiUW&ejcZjP8uOK!-Fk1ZTPJowJiKV zPYYLM=*tLm7xzoGdM3kuz(S?oYui3PM`@*)&EKx0*3fSNn3Dkgfn@uG!-6bc3QOs( zMv#mS9EtSRg@{yoOr)Uw25PRmE`e^IlB2Y*tE&kX4Dq*xGum<9^Xt+m0#1dqs6f0A zU{DRb_bhHgczu&04*BhyE3T?93EcGOnId-(&kKL5#P!0}auG?p0cMAgFx7Q3+PBkB z>z)Am{OG~|Jt4}Y01-IOa3TZa=TZVwlm^~|ccUYmCiD%na;HqMAY6P;uv^y{u66)<<)h1ZQr@>YZB?uKH|BjF4wyP>7KyK`lQ6=b{@{kl!N)T z+(`JX8I6*JrA}ys42>BhG_6Vo0GI+fZhIQMxa0cxnNe(l1@0LoQhu5~-c_PEDNbBZD7!BH@OSvbe9Sk&4Nq=qR*6CjHoPDpwo!33nSR?px{$s6{KR>*SV3^5 z-{^|71hgfHi!r&7=p|MWtR#n;!dRH)@PQR2oh=ID>T3$P{zMe&*;z+ILR=nrO(Qf#ly!~p; z^Xq%CCiby+S!~v+BbC6`18cR1nJL=}S8v{_TJVF4=qlSMVgbeTg7F#}7)4g(nR1HPEyi#_gWo-56)L-t7Tr_uiciX5Yg#30kZ zEi&eu?XyqJ=T6Xnz=$VdTw`(uO;N~%)G_nb***1z)#@80=x|+svgfduec`SDPiy_@ zr*yo`mAc z!B-Grc8~nYfmhxH0;s<9@V`Ce=A^e*JzJ(%{}#y`tPmEJ7s^!iuGMQa)xmvU&TYr> z(xOH%aPjV-s{Px5X1h7^qV}7SpV{k~uSarBsaSG@niiGLpfHB}<`VhTaaup$Vl6?7 z$0N-6>4p+{rBu2edzNU|qs^DyQLB9r=_j<;4-G!b=eMoZV2&Q_X-s2W2>Upq!@Ox|)Ao08MJPWU$xGkhaY@r_R{;ces*fNu%o-nBojM4Ip{vdEMKMWr-cGFbFb z$j|NI-Vy$&(>JEKCRQ^;l!=)3lYYehOEr*bj$%%9?F9uwHT}94)p{l)y6TVIX*`g= z01#^UQnwlU80bN*W)LP{)mv?AA-L!s(i))W(8k8tmjPmdem<%XTDr zXrhJi_gmpud+Aq7bz_1!?|yhzp8v0CokhN)GUFu1_(Qc2WS1`lE^Zp(UouyT5wY{T70|V}E@^ zY@JUubZ}9bn%t){XdY-UW2+39=c)K?p~GWz+Q=~uVqEtk&82G$ABXx5+$#!W6n^Vs zJw0W}3M~DmIYW70u|L5|#y0Pp#$$&#A6)BOvigOoLV^IG_v1HwDICrvA%5$YmeZ}L zDRTFUJA)qP;J?LLr2wb@QtG)tA5?jq{&;`_nE_zhzuAhM+;dVra;JZ`An-lAta~X8 zO=q*7RqHX-HWw3+4!0A;4|1NJyNTP;>_ajI7LSPJ1dcEYOn141=GEVN9_^%z2@5ar zyYCr2@B{b$I?NWCY_0y3Kwz?zB<|!1mDd1bTo+Irp5Zp}+$!Im&z9v?Pb!<1zj*oc z`mST}bE;4z?y-X7yUW;55_sVTBzQ)WQ@N?ER}-1yo*|~)P8cp@$OvW= z5{p&l_UX8+&9)4W-MIco1S_?xk$_f5(DAV@uDm^t}qdc}~irmBj9Tf7w6Ne=`7FxtpNDH0xo6hKW-lXN_~|t zsbw7X{?Sj5)N*Ikt;!F|EL)VEnRXwa&{i-DFF}y*)>4b5CN&3k30`K^V(uX%q*q~_ z{2z9!;$`;z&)0cAg3!{VB4Nv`(D=Xqkej_>_h7Hw+2bv5T0^CAdYx)K9o2*PPGm)r zJ`;fqtBBG@l>EA18Z?i`mO9PjbV+U*-)WcsOd473x;5{@NA7EG%<}^_>)XzE#q^ZI zxft}s^#Hi1)aj88%oxj6==wK0n+t~3jP6ms+*myWV z(7A;Lp7DQLopq{ES01UTAEoD!sT{u7<@y5GV?EwOO18<7lT%ALtbf=3h0neNA`l_Z zk4P72eNey+xM+#$_A+!++Gk(l1sm46o)mhtsg(zp)}AY$0+J4B3zPHf%eQUxN0gQ+ zm89LD677#{_?L24RP4v9URpU+XDgaaM`3LSMRL}D5xy3ZFCSsAG??GzhWK0h=MdGu z62m0|`5jc<&4Y&tViX=h{-;Nq=^foc8Eb!pYow$?r{ABx&~=RPV&0y%tSv66h_kTH zH4uRO)mZR)7)ZnAJM*3$7}l2s@b|nU8YYh!%U2Hh^Fpiqec=*!R`fPyzn>^NXlo9U z_zy3m9M(898}?(z-jfa^f@``3fFm_3xSXo5#F1y-fu>^)&r&{BI86&1$riVj9PxW+ zJ}_-HNF;cGwF(v!1C|X#RE@xlyg?-LJWaQI-360*-Gg`Z=G9jW`z+wt(nm6~N=7ED zqF|RSGrh`h*8+naS_Wl@(y<)V61b4rqVc8fK-meZ>p1fXc)&EGMV6Km-AZ1Vdqk>^ zEwQ?|9|u1*UP!mJIYussZ=>dl1f<;?+qMUQa+yMRWO8|We_3Vi{w%C^ZyN#{WE=C3 zaNV`DpR09xXryl1%Gh|%B-w}Mp+QCC8i$q;zZ}N|HRsnsYrl4njDI_)pL?G}4iaHq z{ncy#0t)+ZoxZu5TQsZouLtfeePL&`}&S`3RzG!I;}nS zxF+P&!yZ1LSei)>@^UK+duW4@%$*2qz3fo=12)5Yh z9txZ9@VXoE-?7G`6Y-#=WZ~L+7Mj{3P6#%rmXlxm2gzSUaq$43!S=P_+yA$tJT7AV zou5^g$v?nH$)-ut=I0j1eR_d+u2@#2Gsu`vJ(uFB(XO%$rX3OQvX)>B|Dn~@``s$c z&|5Oyn6lzk*k#ba1RP`wvFs?HCwGZN&0A)rEOnL9B`+x8&!S_EFyk?<572A)ae`PKzfa zepxq89CBN@P2PITSZ_sN_{EaA*Y{N zj&-Y>U}ru(a`)v>mlHY&V;klzHibV1$1Bn0K*2G|p0|{ncaEG6fYpBVsKAh;eBSV= zFMsbXIfVWwcE-{yz7a-eD%riZ90g|wx#B0!N~q#}b(N>=U$%LW?n^8oM-6079DNLC zk9|nPMQ#0Svtw)0Z`$Y>gM{cB&)(hHN((wYx<%KqbFK|jy;?Hf1y0c`e6>47S@Ajf zDz_ap&TIRIp<(iq$&FT^xvMs{T9|h->Z;`%a{r4-vy1Q#P!b77j{8R$V-H%(a>AeW zJ_?clD|?-v*OK(Bu(ntZR~h+C624MZDyHirPVr;wWqECn-R8i_j|Y?1(DN#2kXRT? zrT)7Uyn`*R+hXG07gt^mi=pJUvO#&fBR-zffQ;nC=MFHlhf+d-R3OCde*^lb_V+mKIBSQaR1cyWn=RXNJl zl=r1KWbF(*J6&lKsM+!9Pvjv`XDUfT`5xl$vq6IQCF{K$2C;ZKEJq_VWKgYOcQ@4{ z=_hT=hl26lAgPMA$RwE(ay5G2>R&qJsrct`g3$KUCAP}3xUpqvBB6_^t=&?+4% zIpkt<4Pe$}E?#HQKP#H42i(33iu^>6&DBnA?l{<}y5=;J{X6<9jftT;dDQY~P4&Z0ux8O6P#y)n%;8~|ssFsa-Us!## zIcwz87^Kzy*?YBy8OXEp!11YGyC?57lo^hDMkJr||7%yNuTOCM&HwL$IF+6Q7er2N zWc?o-uZ;QAB0QS&a8CNGcg_-SL=5lP*+eGyWwa55Rf!PAs8nklgOk#STJ>JE${OQ? z?$9tAf8kALwT&*`QCWT4^0tf9R{+YOS}QY=F zDl$E7;-Kqde+P~M%!jnBrtl!eh2JJd$Hf!SZ4TzrzX5uIud`1qyGYaZt;c0ct4Weg zR;S~skB-F=uz#(yYsdcGtCN8QhVQ{#%4xafPORX{xa~Q(N{+w`B!9G<{1OXD_`P1F z)xBkHpV98x=HOPY>B<`IIEPnMluOCki>Lpj0mJ;bA8wYJNSApUBR+JZ5B=v~?M&w9 zO)0>j7k?W3e|lj8RUI)ed0+TF{*W0JlsKDoj@~=Dxk&Z7=H6CVrkv|d3Sg_^NzaqB zf7}$w(l30s*2oyR7S)l`U(eu-YFRM&3%bH<2aWyrFZ~*#gUL9_-ATS`#+mg&n9h5W ziK+=$6X7uOfECMM*~x%fzK1QIC|U4B&Gia(gN5g7bc`>i?#b>gT+X3? zG|7KGKJeIvZ|ft6Q$QXF`Y1K=IY+2x8Ky~kOT=j%7Sh`M4}T`NOkP!v1ItT$PCVSj zEHcgyV(O8BXH{<7X*uK)_xrp5cZ=WY~pp&bO?cxSFd5}uTV#$}PAkY1e5q@$tET|F_ ztzkf30{zaAmub~WImpguZFXzBnpk))!j;?~u|6KpsG3@0STXox^H(~) z+}|i!z4w3Yxf@&*G9$hzENI0BuHIlWsK#CWWXI;q4Zvp>>;6~y)QX<$uBwE^kPv(6 z_gh2MQV#aqk#O^kr0PG0ksdrx12$ap^>gGNbd+Q;MBBipfwQx#uP?%x8pw{7&_BcY z-=Wqn9$=xX1x^HQ=jnJtzdUT4#54-Br&S8Ua4x0^p=w;abv8j-X^_6ZvHmoix*`?q zIV7bIjU(dXR+MIyPi43u^`I1671-nXifVh^J>Qx@hue!@RrShYg=<0UGd55k0B7SI zY2pAHVzyxUk!6(icRg97n&Suv}h)n0jSaUgp2tM7dNJK_usz1VS3r3uyB*AZs=JRLG%p7+3XUc(7#1T*9k)24I20Uv$-70zik$+Kz_S2KluuACB3JW$hTfPzQ zJ{%|ha;E)!|IKzb8qc(@qX?{(hygM4-fU$2RfeHfP<fZSNAV=EM|a%6KN>sJMA~as&%d5e@3=$!NA{XG7p{&iY!3|okRH9By1hRtqlBlo@j2WKz+Zj@qSsX>2mmk2RUTV z-b~`x22dedLXNDHoR`M&bf_|2rpvd2V4kCkO6x$s!^J&MWdi8#VDe>#PZ`An&F7hs z2pmS!fy2^YjyL#h)7+%iMDNsi!@IBcx$AhP0G(c9#`oXJWVAHmK?dP(jY)DOMa{-> ze@`q=f?}jUs>bMH=H=f!THgDHi6V9JmeB4GgGQyaU+QWDw|wj0mXE`$+>VN;+9?UD zXHUy*?W{&mvI4D3dR6w?PfsK0xFZ?rtAt?3AhT8n*WJ^pb(c1rjr6D?A_N>~JSl6C zzEW4G{OAy;qZrS$&Nd3=P@LxmJ>FhQFfxQssVfHzZ$6b#f0_j?D1a1Yd+heZ{;1lU z)lbg3Ofk2Y-oRGNF;_mmbEsf`JP7Wp$0;+>N1Z%vyv9|ya`&XX(QbeQg!y=bTM>1- z|4+F1U54VX2Z`X$UTkJcw+W8B1N^Hy(^l7-mh%hXym(Y7Z)f-PZH)1&)H=^i@v`O3OGTkXVYlA*e{4NJRS3=J^foKnHIh$Zai@K`M3apb zcB;-U9`duxTW0*32%E_)m;$PL0WvI>R^ww5P=ucxno(WWJlqq5PW+u--&^9h5F!277Z+ZVKCh-`-By#L!wve|keQqJ!uN90*M>7wO&)p|ws<*9@B*mQ zqta^Wfy8&Vr-Os5_7`j-kS3x|U+#=J`X%B+jE%oj!c$(-Pw)Xlbm!J}%`mF@( zvk{>{pDN8~!y%;lkii|zkfPFcCgNm8FpEHs$#WbRz{S{^cZ5-kYk7lBwzI_TA*A!3 zmG_uNdb`ArkWNn2*V$dNFs@wPdt84)Wm^AB_wFM5bY;WsTG%O>_4)A3n|wE*I`1L3 zxUnCcokN2$WJZGeSgp0pc9SwVWTI@ZllSuV)M63vQhXX^g%}k9l5P?DY|4BZj|6u) zot-sD`jo~_RpJ*vP(eo(jQU?Jv6W(_W{o-@h5Th{h-BQ4<55wHDkUB7aw2rRDwvc~ zuUx5`_CSgNQ+!y2piD*YTRoT3DhCGKkcsQ|+sy%{M-%3RBcGq-*Na$gXK&*LT>A7gIu`(h$yYLDQ1 ztEO#|3YH=el)+S}mOnM62HV(w;8u zV6RzONG?}5-rz$EGC!OWK$|fG4wjRls+8p8Mevr>Y;RYH@ScsC!=1dJlg`u4ahdQ2 z&P(rRv$bwxCwOqDu)Mn-(Ht%)8S6T*a`h49UpA-i*!s;R-oY1JK6)V5*DM^o=rYfA z^AT!2p_i>eNPAFFpt~8y{-unJd8GaPN>09<6G3~PZ}%Qq=aG0^kWuGb zzv|qU^Vp;}@w}?SVUO!r!rsWKKlNE(pm#v@%m#D>T|uN!{+3TsdRS zdqH<=^jg`UmVvaTF^bO)^G<$u9(nk8l~ky#_OtH065hAhR4fdl z1~H4+U{S%CxY71Ce^&R#<2?oxa%wioLAEs>gO?OS_;cMn+XM3shyAS!a($8hZME7p zTt~ZQ4RF2fq_b@Aba5WN*VtAdEe^EQs_*r8`X7(*_cO{u9+|PY7%G(a5w$cOurE7g z>N)pt;nF$I_wn_d*=Cf1`8rsv)t@ZsdM#+m_eEeQk3BaLO|`AyeR~%SJN;WU7nmQ)_hlp}g+1@ayW*l+ki%)Rfbl7c~QXBg47; zMrgITw{$@>`RR>h{`Rrop7qh+lytNPt-Jq;nA>%jvY%V#5AXdh*Zls^BlDfKz<*N# z(%#DKN#vO_c?Ti4i*XUT>$EfbO-_e)*+9$}t*FDJeD~;bEE+bIu!w4`x+H^uOmig6 zBjZ*b8$FjO23rfgPlSg-Jk&aj)%^Rc0E-nJA(qo=iqtu&lUU%+WRb+n6Su`S@Ma2#-K@oY^#eG#nzYn=pNt&3ZXD@@XVW$t(G?N1Aal z_ky%t*-1x}!3==Wama`=UZWg_o9of(%!h5qsG4($7Ws|%~Ys`C7bxrVkiW~)j zTtuhe+I5;Jye3<+D)n*cx1`c%#^eg{!glpMO=3jkT@zfv=Ll&3=0uT$Z1MMRtNGs| z;}Q0wc=`Z(MtpK&bk%n0JzgG}w*?+htu~f`K#nVpd{LasE>;`@Xnjq1PLBm#$YtXdhFJdp z0>+GH(30giDeZLFBmXa8-?48)Cwy<4#~?#=IgiH3vr1Q)MO(kP1wN*NVejUwUX~MbnEY}ET+xA?ZU)9t8{`s@9e@6UbrDr@>^HQaqd!r(H zwzw}JP1b~Vs`SZrGN`-G#(e$IY_^OYc6vkSO;&YdLZnFzY^mjax5MK(mFKEyC$!eZ zseA3O=s!SrsCCS=t~Jb(`NjGuR>p({-}Q61gRb)ha?iPdYw+xR5p@ojqN!nRNYOM8 zs|#tySX$wuFXrBYP$eCT@Lr$4eeX@y)fMA2e2N{6u~R4OTa=dw-xUQw1wlnpB1*0w zypxvTHXe4~kpyuR*)bRxv61e;NIDnyn*VW?Kp5Rj+vnoD0AF;krmz2m z`st5IJNn|CM5rWw&9ebS-bQegTS5K;o{Z2hY$ORwd@)X6jv4ueBi$BMYD8 z%$?3~f%&}M$TP<8fTKU^59zKwM>6k*M42eC=6)Pi+AjTLuaWbVZW&7VO6i5uVpk1y zZv%eLS-QKCti5@3n6YU|VQ#Z0-+Ik)(eAgG7%u@>O+yZc%PH7wQT!m>0fI6->_TL7*Q09_xfB52cI)hfu&aS8$USIqY!S zR=70Z*0u|seuJu;N{J_?tA_a$D5&)Ln0LIap%-8z|87GO zAsE2Sh&z=1T+ANiRuG!#5EdV-CWc`d*mKZM&3d#HOx^&#uUi09n>JMVuQ9;R2qBk# zeUWl1;E>c)fcFZ!PD+cs1Qvgul(`<*&{^~eR58B{QZ_FZ4UL}yPnDS&9e@*BM`nP; zcKRlIvY_V4N9`a;(~Tgrv#D3~>OY2+)BIG=!xaXJ6=dXGhqV#VYJYp@n)GUn>EBsV zYPR@y^!(RREVvkFtgW(KLg#)OCk@O6sRcY>^uxr-+jr?dmCtvJQ)fbXou%|9^hLc( z-2rzCGGi^6Xb(t3?(#m)!SzNJ_|S@=jW>ti2ZDTN?NT>$Y9N2E78aPRx7Ife5Z$WX z#Zg*+nTj9e)kwNtBgk*=-xPaqg?w}GoQTw_d{l0rR-Dqw>pN6fA0=)J6T}o z8k&whxSGR(_*79i$rvRRMdI0Qwd2IPQ$_5b_hV%jUproFDEhhWX_YTkj2rx+Ve_;l z>4}in`iKvxU`1KH726NPboW+yN|qIu-hgMMbgX*mJ}kQ~AL5+tg`eFa~!n zllo1mk*Wb3moYt~hgA|IZ<;?z@I-S#J>#t#i3Bt3OZO$jnZ~fsv}Gbq*bJM7F7F93 zWaKgrhW7=q1gl&&*4L|VYGA^2H3DIJ4|)sepAOAf3)QXzNSnjgfeP=^-Sd7}_xV1t z^=9n;RM%VDb2dlH|5SBO+)@B~#&bz9!;a#Es=7~N1ZG5E#fjg%a>7G%c~lgvyVfwI zF#W^e!gyHxW+(Z(@+D17@aA{_@*m7Pyd!w?H;V9?es%fFlhW!w-vyr{dCxtJgAKE@ zXE_F%OdsNXFma?$%hf#wdv+~;l{IALDX$c)!7cB5g|pBEvGJ%Q@4H-#g%vRsHi6jo zQ~e5!Yqg0j_dt`jbB=0i&lPZwQZuZMvS8d7vELU(?c=g$Qx^18e^K7$yK>{%V~X?J zQS_E7ia{IesH3?@z^_x5+@Sh|ie|j`#bYy)I(Ks&*2W3}ts4TH;NJH%{Okq0;(%sn z59&piqB3Rmzv{brcuDf?vddH1K+whHKRG(6Ui8tXscy7yf;K?*b0);%{12YYpwAmJ9z|vsx?~%qo@47kGL+0jY zH9VXH-^rgw6CxJ(Bji^?cI7I&OxgTn6cd2kfbgfuX>-=7QDX+-e)vVl?`JG znZ8pMLP56;TyPl=E__^Z=OnUJqrrKag+)0y@YQ_fZemOanrMVX zUwEBCzCg>xf#10_&Txl`usEDF+!+FEUL7@<>wa^9_6tIwqQ&=?wLIQUvtEnD@sv%Q zvtH+A#--0L-RKlEDmdqA*;U|*=}@-?Pwf%G9; zi?T&+%UBsBi_5OH4)x8SWs$u?jyP$kc=`H1&UjugXzR_1dMu9cx>Sx^wIUxSI#=0! zUbY`Op|T3%<|OovWGhbxl8X#OSkA;r_$bT{R98f+iD@O9EqhKnXazBFq4WshnEGD2Ahc~KH< zJGW?d^S(+L4noSfM?Vi-<~@^`>tXG!KIOsriru6ZjP$^vDkrA4z*3Rt@KehnfnU1L zC26-D(uH{J=7O)xp{6gCQohIPS&3J7L()ZOAbPEp=PP;_cdbSWaMr?>e7krx=W@DK zN5ltV?H|=e>KNz-NYJ}p9A8g%fSMtRHmbm6`4YX;DETUM+&eh&>2q_=UHO6;qY04o zLmusAcrZnf>PvY;HPEnM5cnZ5`b*gyej}5260ud$V*3jO#e3MSO)j3e4kx;kewn}3 zK$MA0=)y#0azPUK?ill`FILQE$!36*1G~jj-C^z4latl*8@v)XXRO1c7T}_8r+Y?- z2LG+;o-N}ASDW(1M8lskU%23cT0IiB==HN-^({loGqH^cZ%E;9O6LvT+BcFVNNd6| zB_9wxnKyr19uK6-wC6cPlJS<=_dRB)$7uu;|6w^Z?xC`L27H|NrZ!n4_XR- z#KNK2Bg-_c)~NEcuE}f%GghYmkjxD zHVj(#LlmE$bL_gmTuk%`G(ps-!d%J6cLjJLQFF6r-dHGcshYH@mHg@doU89tDBYe_ zNMY(`%Ao1y-A!C77j)6nUlxxK*z`*K%;9F??w3KVFiMz%^QEpaSj=T!CssZy|349) z-O0M{4hnR^vDqNcm-ktu%I~2=> zIq$*0xEsqZ-!f-b`iT`;VtM=vVQpM$KOTG;1y8{IGaj#5{Cu~$mg%QT;Ht5b&53_i zKHl1A{<6rqDsa3*;5@0fmUq;A529ADpjVY=%?!C9D~{3t)9|Zn5}ThH=_~e z(Pq0rkp?3^EH+)EIMEj|B#zzQU#b2*gjc0q4ano$y%IUYe2 zBxX?vC*3xWK817?ERtFl2TP1oNJ!uwf0qxp{ADwosKT(=R%LAfHd`URpT)N0iPv%4 z2(+cC4>lt8Oby!&-knw-th8M7(L-n&udQ1L!m8Cw6(fm?+@$DK?85bl>=oMRPAU(*Wj-sfI`_*--RMK@28lKbx<6~wDZ|cyum5P!R{|_cjA9}?<7!W{-P=e z3R}_W1UybjCdL?ZH468+3Y1NPUW};@@Cw9gy_7~3kbrZ5BdyC?|0UJ);kDNKmfA+_ zwAEbJiQnRM7U>-3?hCv`t?l<`W;(+zF~hQ>{y{m~Lr!LJ^`8SC=6D6cl0(0#hNFL6a_dRBL5uVXOFbc4 zPrO^GV_7zuF8)BP@?X@RLqY@5P%JUO#2KRZy1R&WQ2Zha!X|>F;Nxj#47mfZFD6PX z)<)@dwXVrlILT*jZ-Z>01=co0#=s_cz+!1UhTa(1&fv9iYWqI06XiXjn_r$oZa1&% z$;DD^LY97WhIctK>zW&ZeBc&$_=2I&Rb}) zi+{62%XYc0?g{Grjy}clavnv=^ZDbot2o zKx#AKAtY@8@8ODAwK~(hNb0u&;V1C56jrlZfVE@Drj*V4#J4d$(0tj6$GBbQ!dXzX zI$#{+ux@>Su1SbxG#=9Ct6bh9VR#BjmXyd^?8o$uLcfhxV^$7fF%)fCo^g3dGhVL= z^XXB6b-)_&Pv^QctWPxti6+V4j2?7dSO*@&V1F7v=ezj>Su&j5I2l;l<+q-ScG$S8 zTl$zIymmKpunr@@Q?lIE7zM*3_kHjCd;a6coHOUS_g?E*$2!*9`waKQC~!ruR_XhU;CELY zJB+Ny4yBBD=c$koLC>YLyqXW8EaaNGLsaZ$S$@sI)$yuoim=_2HGg=2NOc9}%zV4J z@P03>B>B+Hz3_%AXTWY*rRrF&q0`O7Wip=BnX{`S=2a}h5YyiHj&&^O*$cVMg6sZr z9U%q*&78~cKazb}Pwi@*Ff0&+Z4#dnAT9l^qrh zkLU5RAS;b}lLM{TSa7SR`GZ!299cf>Qt^qKuUB1J$@#eY3X#~SLNAMCQaPRs4w>D( zIz!np(}xV*G+X!b6%sPLA%`DW8k9V;&d;xNy4iFAu}2~7GVdlx(cEkrjo>>04QaWK+YB0Iu9qq0jgt$P=|9ZS-Otc%^GN%+=rcGy|F9Y>IQhYH zDwz0b1*T=U)T z8BVh;lWy?{7h%;={~} zBV*eB=t5+5C@ygHvAc)2N^RiW_p<XObpqx5FlRK?rG+}( zuA?{mQd3yb>G#EJzg{%o&C7wjXkNo^~xy+rs+k-akA~Olrps$m)UWt z&MC6smjS!h?jDHo(E~)ukzytKiV0=K{QZ*6iV#>Nh8$!zxvw7rO|(7DedHOyBQiZ5 z#3KT8JHHoIR-NHAwy48%a~JW*>>%|boANXnU6cRKY0|p`B*F(X(IQLW%%2guxVv7e z)i?ZwA=9Myw@BGs)5=-hdm}Ut&R=9p;;G1g)yhAAUN~VI^ImnB&Cb%$!HZYmof~i$ z_q3cS(}ZKk`pX`OX1+jE>hWilEH&_=XwrPEt{MIyxu&N0lJzE)Y8;Xfc@q!zRPH19 zz|{eC$YgcFHOQg8Y^NG}F6~Y5>Q?ypJ; zQFT4Nb*9npy=q0i{#=a;jBvF=N#qmjq~_ek9U74R>zzl}g)An)Ajs}ar1xh2-5RRE zU6QGpn-|ovy{HggR4q&y&FTuXRE5M(G@0LQmL2a*bX7TM(!a5cLt~#;Q^3b_KPf9V%niq(l7@o7w&90G+%d(%k#aeuu9^g`R}Io(v|)Y((an zRj*#Yj7%e;fdo58t|=H@Xsh)S{^YvrQ2Sr{iDO2uH22 z?%%tB=_V(c%%7dV^m)Y~QMWLzmER_W%C;=ybtR&#jisQ3zJocCW0zm-;8c)YUXySc zSVE@Nd*>%g`kr8!UkDWu#yDD__ULn+B*4nWYW!kCvbx>iMCP(RYBBTRCLKzwxe)4! z@jg)`?*sXv@8-7#PVLq`i!jacZYu43_UJo$vlgo4X_O@z;ig71=sGOy2`e%+5ZI&{# zxO9(Zj|UEBlHCk8k0Hm0PWOxoy0a<#SzJF|@(sD1IQV)TGsCiwc_+}3nVh6qp=Wo0 zpjL=#w}9tRt+T{J#?H)yqLTho`#ay9#XIYEL@Q^xRs;1RHb!wAs|R7QVGCre>%sd2 zi^{j=(H+6#8%xy=^Wn*|;Rl;DQR3rJtwYjU2;a_ZG|am?P&YyAt<|XIMlOB%t^G#P zxuc9=gBKJPdx&gai?@Cz;_Es@D|6Eyw5oZ`qOF7bcSjZZ3^a)InNtESBGmCKOam>R z6$GJw$WCD4ft+u5RQ_zwvU$8@L{L!p_OSBcO>7>MMRQo)n=^OMCnpDM5X&IL>aKfD zIIXUR1ndT9HitkKFuUW;QmXWE99Iell@=Snsb{DK`SYJO=gPM~YueOOA?mQ>2s|8% zc<&B~taH{-9rjXDEISXi>i}ZUcD;wx3;2t+4PrKpuF_-j@K|aiI==a+qP9l>_S|3( zo}gevbj?a{Deyye&j!`g+i5Zf27!gThhNlqLR>~?;s|PYkDnwR z#<*-4*ci;!RPXEL)j7b!l*>tvv5M>8xH2TUIbVhwjZbJ*Z-&6FnQRM)5y7Jyd9dkd zp%_BpWJ;NMumOfkH?fUm)tq(Jl{UX+Hx`B{Nq*(oXj9g`Wa8Z`ZkAWtEV|sk8!WJM zeJa)&5!alhvRfe6J!kl_WP0GpUvDY4N zBrtQSbA6^cUe0xzP=@*G9#bv5`*W^QeY+Yp$VTK)60ep1m*Vo`CnKPWvO5o%UWcq- z><4cHaC3^Red~CGe^1k6(V}--%(h>en;i%HB-i%JzixT(W`?g2yl0V<&8?TG>u|v< zKOsV5+23>CT`tD#`iaW<*Xwj4v@TSHX!>5hm>FtMS3K2T7?p|6dVm^d*{HRYP6WLB zXsJ66rJ{FsA9Ofk7T50X9_g6hTDl!3Vu68AQSpZF@a(aj4a~t^pR?(si_mz2Q}^+p z7Ba}A)~P}6@V3oPy-la4`|{I@HCB@2O7KQ9h*sHHX)&8{rq`pBv zL_KDShRsPL*bN?XWVfkOCiYP0t-=iHF|F#TR5QU$S|sB6(LwY05V>0i)LM9Ct@aL% zNc#J%<_?+ueh2_uKjOQ>K?=VNa z`8-Hnf^A8qY~sc?V)S9jn4x-kG*ucIOIo(Y#lZfPK)T|7KwQnkx}YxfE;rP?5E^W# zsk{9~Z4NfE>ozfP@TKtRiw;5JlifSQTX#Cy4nD9+e;l}^&J}J4TTveookXQ4gcYdt zM3++8Up!Qoal!tP83~x|$}ykmA%Qb+Fzk8W_Xx(W7fgUZF<#&4v>Nlo3Vry7l7EfAI(tca$I-zTp zE29;#TeX|F;wKI|ClX@qs)HS&@#CCt#$ud2&+c$}l@Q1Zo!{QdAta0MKD~;Jg{2Db z1^KVb32nP5Sxpek4ayx5cx3JE9dGK+PS>ahu~E8_;rWBO@=2xDnHx4pM%tM2ks2~z-OR{clsY*UIM_u;nbJDGt&iSc{Bg!^h&6f9-IWPhwG~!n z(K@A7P3r5?K)gq}*XE|(Nub3Hxml^&KZ5rH^*$4syw3mx$v2gRx0O28JnS@rth!j- zV-Ggij;%L0g^xFFC@c11xsTuO|F*1f5=VBJyk^w24cEur1)Zc|%N4p8Qv9U8hL|4^ ztw0`q{ZX0+g5da}`_dTM_}X(O+bCJ7lV7-g#p~V~utV>y)5yqpl-dyTv&G@(9M+_b zh%8Q&`Vzm$DYV}kBKyf)nv462SJD}+&1x!>;4y&;@3D%IgLc@`>T!v{wA{hg@#h0M z;r0FJ;&4iET9$IPTBb6|>bu<%HhqN-(qqStErS9XLkk6al8>GW)ufm%Q#MmN#*NR{ zA1=xF8N0y8+0wB_N;==6Ph@N%MzNucuEp=DlB+swS>0h%us*deE^!ONUUG@!;gK%7 zu37O#N@Sd7T*JUpsYB=S_QK6dpF46H0N2u05cvD{1KJLAR9}R5ZG?FU?uTz`bnYkHJ&iU_^u*M48THZXPapkm;Yt`QwZ}&=a5{I#HnGCwy!{OvGbpaB6&9=D z+8%7&<9LEIBmue6h*PnjTgxISyW6a^B)^%udz=M3dyL46cc`5|orYd*Oukwsc2ma~ z>88#P$ln^#%KCPq!+5OPRuwXz%{*A_2Jx&mCCQh5)p4@YOQoIn+@eD`!TBR&R$HoO zfWF^rnqNiCckU6?I7JBfy86ipKVE5`*EyKJ)0weVSVK{>XIfI@IM6&ir8P==&)vr2 z-joix$3bkNRLH^MQAJhdD}oo|aCe~Tn+fe!baN#iE*Wj6*%`oh$SkUF3X30t7bTc) zHqr@M$(`v+5(K*yK&t0<&jYoqag#lUXVmhJ4OE7QtK0c=NnDoTvdLb&i5!J_b}wVB znK3SY)noK!bO-|TLtkMSR;^YKR~F3j$EgmjBgbgtdHDf2MRrI};^}l5ny>Y)pDsed zidW?M!S|p7IoKsg^>hW9Pi4-UTD=Mspw&| zi`iQ!CDOqqwcyYg~st}IWqgAifrmz|c_#kfQ{ z_sYsz(`FSpH!+)qFWnBymp|fDRIz9iHOD(DuXN1|@dq@MDwI2H-R!oXtHEKs3=mWW zgLzVuSOOXLJ7R{`J;qssDQL1xjx6z6*?#AdKREfxpL(}&?w+4GT-{&Ki9}o*`NrgH zV4N7KASLV=9pQCFK@o^p-9LF{`i!;fe zfE;-Ij_;VxLJrdjj*t>yGfKw)9dn%TVj zHfZ1N(`0Y1LLRB+gCcPx{b*pP%I=ldk|OAC=gK$Rchhz0qvOde%DGkC2$E`KX&jNH zl2isGSwE4sB7XGc4>gE5k9FtcB$w{l0c42NR!}ZkPtqUf=y!NrGC*JF9phNtYkfRl zAeSCu zes<117cTZDkVZnzjM~X*?Agrg+Z@_u#e??@y^ONTc)`|`5n5PCzNGdI)W612^(2ve zPmB5K3{-Z=-5gBu)Er`UAlAD?=^y2g~e zNO_x$#{NB@TTB<*SpRhP7L zyyJ7-KHHf}tu{{^%_cZ1u(4LtnNgx^#yYkOe>|TQ!Uf(6xzy*Q+lEX#w>CU~?0B?4 z?y(mD+wd{6K0mQVZKD)UKTRg2xI6f?8@z}lDY?D*DfHNxX>ue}A}$aGT6X8aD@!*! zYj%ZuwCwteVU?z`_9ScfC<{K61(!@+{1N5UU!MBtV#OKKb#K*UQp;xQgRD)1-2f+@ zEV-`PsgPGc0L9OLT7nHuuJGHNlYZl)D5)tkwyDHStyX(8VSs?DeT36!e2==b%V<2A zGUS%JQ{iH7g_^J_Qgz%X0Mj*89yQC%5?9Q>xUEa=&gz&~PsPDn!djab`f9#^{4Jrq zUgU=H1k*K|Rr|?G#jaPvT7KE+0=CWj)lacue`je$h8UDi3oEeE<|7&Byp@r-_W4y^ z6MMftYNx{ET;VbDsy%qcM4&@v^P>AwVyAESp}P&B35|56@zevO!{japJeJbJSC++q4~U$byje*HeL|ikmv}M%11Uh2JJhi6^z!Q*nUSL5B8D&P zCcCYr!n8E(Mw;i%2lm*>{uW$4&b<&{OgbK|ipNX#u&~*i21iJ_EiCkRs)Qh=I@jr{ zQ!4h~ld^nj=B9jvHo=`lYy%c%hs7%~kTkw(1;RG((SUN<=HTD}#IUc+a^ddsBN?q9 zJyz`^TELFJ#yZ!ry*(@U?zjhc_YN$+zycid;q2Qv3jaRaKls_YJ%Y0rmJ126E*UX=PX1f8dWL-5;Zl4N>IYcp&hc!$c4RYO zOLUqXUC7jpwMSw-tdL{-;Vx)6c_AXmcYL}^)ue^+nRjm0O=XT2Zu5;{fA8VsMCc@I z6cHTD0su7MT__jIuXaFyb)MQJjl=FrYB#Vs9ZKeXekmZIO92}w{r~!^pt||=Zj-aq$S>m(wzaHs_r~CHjWWZHD zv5vG*NcKM>{)fpR;rH8+-Fs19RL7!wuU#yr-=u@?0Xk#@$3WJk3Uspj}9sy zb);2^ zFCh@(Y@2q}sF^{m=fd!Cmjwp+jWC9JK!NZFzf8meG?n(fa2E^e0RB$+1t& zt1k15dR-aAMljQ#Kl@=^fAuGJoF}F$_Wsr_r+@swgG6X}2=061R)23Le=z><^F1j- zSme1XlI%|RGfMr<pen*_Y8rNT-4Y@JOWEY^Tv;X*k^_0Ncj1W0B ze{XI1>5=~8UjG`2*J&AG5U1Vt6RZF61DgeawTZvBTK-?%;4fF)G6hGy&LKsvHaz+l zC+GJEmN{SKX`wVr@iP#LFyepF@{O%+%qAEDPxQWMKc2CThU^Y$6}}L9s9`MH&|=+huD)3vZdg8#?c97 zFoYKRd=kyn-E6!S0p%=DS}<0L!|FKQquZV(4=K2>XAnmyaU5@V*k^7v*0|LA-p2mj zGu{`6Zj1(Yv@Knb2>;$=9l>S$fp+*jEVb9pj=sW6H-AH$b0ImlvHYfNAsN)@+yQ@N zF^}cp4H?C3r|DcZ=h>`*K?3U!7WX&th!@fcl{finJ|Ed-jz|_B*k3&x_eo$~C<1)J ziDRqJ8Dv^3GeBgP+BL-f1<$Ulzh1PC4)J-ONH*C)Cfv7mR%cp$JcK<0#qFF@oe{I~nQX#|TVYLMU}7`(Yo{{GCSp&9ov9(YbB0 z0&;1NH%`-uan`L3Hm;q7$*Cp|;~HbO~8U+KEst0|_7--tpmcUwIPI>l2x6f#kg`FxXvS#iEB&iOXG3`a}!ole4uXSW>= z2bfA|eB$UJ))@`K&~O|5V_IU~Irvpb1p7_?q1>#ol{QJI2HaR>Iu809OoHnlBgUIB zgMIR7)q0rLBPr2s7@+ehadhK;?Dl-Aeb54NL+9VTR^{Na6MbBAu7tQj8_%vM>rNC2 z!=o~yH%&6s+$Aa-*MUVva{8~b`4T-Kbw}(E5F;kjer3FYsPS6#h82d+1dLRqA(7G{ z`(nc?Zcju+hto5?O0Ioj73eD=ZUgi^1I z6~X)q@3b1q4d4awq!C?b=XZZ;azwnTeyK3?K;W^k`>N!7nn|$4q=v6KbDXB#=nbOb z;FdP@aO?Y>gsXQhxnM$5bE96}Oa+sjzH$4R8b3mtYc9%92E)1v+#)Lz=45IoXWTl( z-mcBn^wyJL`d?cP{tI6yqW1~{bpV=MHTRq!V-3%gzlhn9NpKq|uytWhaSB3XRR{r-9gA>l?zY8uTcsma>{7vXyE!lW zV+Bt`&(uoLM6*wUQDw9>!b@TPa*UW3%rdH*jigG>YX#s7cP;n$#}xT*IoS<1uRT0Oc`-jZ+s`mvCLzlzxGEd)>q7(5)+5 zxUbe?H3~80|HN^jaQ~C1cZmcAkw|c`lr*@LQst@W^orh(ZhN47JVSxSS?XQ=*+)vI7 zWQ0yRqWmH^mNK@_w3AS`c;myf3kR~v*6j_Mm*x*$woBttQ9v>!f6Dm7l9XIEK~ORC zq)$027yox$mswJ82GcdKqZwmul#bE`cO*&;cT5b^lIVRTrUkW}J%vN)X5!+o`!u#% z#Y23ruXI4}&X+4tfRGd#w`v-rp`3avDUnx%`+ataOFd?urSV#)_MbNckpKD3c)Q@CK!qrV@SG z{{r~Fb5&goZk@WaT1%E>7M?md&R?>C4^NeQTJpD57&sRSVg9RBGMq@3`5D`@&Hm7K z&sbXb#5Op?p9nxZ@nI9BJFh8?0M~lWMe#zK)d`GORqKFN=VF8!wwQEFy7^dCSxxD~ zO?Ybipyhg2r!u!nS^oQatmy3^5H#5FGBd%#Fd=do-YHrN z7E7&})igrwHVGQkb!RPhW_I&(rm8UmbMxu0Lpgn1`2hctp|>6IsA1P+2ZW7Djem(( zkdJQ1LeWbpiE>(sfiNw+#(8+TfGDAuL=7yYG`xn{m|5);S*bxLQ+?|eDl7n`)0WNN zDqU2FrMxqVRC0!u*`CmgG=ZzFN;ky9S)?rV(e$j%7~mH^08>)wyAG)(QhcP1m|E0T zoAe3!l@S61-$RGo6NlUmW-_0D+tp!g|B0us;EU2}lJa6T>`k3!DSaJr*!!S)YNWp1U|#rAJJMuHIZ|@qYDJ(vyuff z=p~|$mXICR0{iHgX4l2&zIkt@j&b#+j4NTi>ju)Bg)z--j$Cu|H<9ZuYHwN4z>-$c z6J|(!H4o`pU;%d|qVIe~Hl`SS?ROC`QQ?6mlJrJ(U;XMEhB5)OYoR0HUwm9#-I}>B zx23wG9<02%8HMS3PVLuV&#c-r1PPJW8I)c*T~P!_#*y6kJwj~ zTbX3!`E=D&R~_nQSg#4^bNkOje*sgQdsz^dpO`S0uMiNFe;egxEVz3$pG2 zzMnO~zw&%NkMo0$XcI>t-tC)JoR{rZ62ffrE|rz5ij_>ADL?h!!T^|xf&_h^Y{DH& z>#e}z->$z0h@Zvn*0b^m7J181T=zU5@1`LX(@&1p5=MNCT#~BG$kLXIy$E<6iUr4- z12FJ=#p?Gpvr)T2Q8|1s7ENa7iwRFwuu$<$0es=;Os&l+|8tnH0CzFSa7uc*xYpMl z*AF~?FBEwEqiv7B8vdUm0F35_2O1TRsvotstdQ!PnM(!+$>vU(ybw7GVpe6QS9OZx z=xl<^%4aNRoTIYFBJf#Hvxd|kZm-X=@02dea!H9@dqq{zH93r%!AdcD=iliI^l z*!&=AMm4J=i4wI$c3n|kCdwz+ydoW%=*B?!`WOU{uiPwqK#2wiki>FY;f&f=DRfxO~AQWzv4mV$&+gm*p+Fb9~TS zdadAr8|UlEd7aZ&i+n)9S3X}S5>EV-!OlFN?M+A048VvZl`bZZxJfEKcnI~_5S+;hIe{Ib_bi1(l|i_Z^u|nh<6=om{3}=H zZwGcHWBU+fo6y<8DYb@kSZmtfU{#Hom8CwJ4Y zi_Ls>a;F-Q?3*fQwueHKBKNT->}NiF1m@+Qk!4jseM%Z2fM%==PWe`;DV)=DsG;|% zuB7u;h^`3FO~F29DDX+u+;^8aaFL|SIu`PbAP2d{HTTL-?(W7Ba4nPpysM-xon(&R zB(=9D1y$7)re86D&^j%QD7ML&=Zlu=svXpLuE~;^q7|>l2g7BYq+FxsdEbe7l8rAl zuRc&}aKa-|)0DfPX@a$94dQ;h{6fL8v*G(k@|%rn1-64$&epO^6Y1z#0)mavinL;M zfSZnJGw>R)6o93S>2rC|qh65BW85sQ<{b`)ntXNQ1bMj0KpMN?^Sa%vtquz~xD>}% z%<1;4EWMYsno;A8=ykJ$ihVTCP{x!l;p)MV-cq^NFi=}ihfVb>BLpVCf$16;?CMLd zHri#F7~8v)ZIQ0spM}M2$&(|%9&-pyIxUVF7*3I{^}MXlAB6mhNO)8k;L=rxX$d^+ z1y0jareCfkXL9Og4tyypgEpJIVF))8hA2tIBm$2hi_mJueC*RQxEnj?xP^&3c>ksB zH~+P*>2c(c6up)<(;NfSy%`d@{-mwj05Gb8N1CW4&|ZO7GW=T%aQ{;bc*1ar5JW;L zMm9ai44(-VC3C02yI0C`fjaTzNm7~W?z5-_mXc8w17LO?L2+I5SqD@|rq zCnI{3VeRMI5Oo;1sEjsR zBFJwir1az_9{0{}rfmXN$JUxeyLVG;7TUokWB6P-@VC_K(x@1(ijDbXc39gI3rkgx z%mud}23AP{+m$DiK_M%p}ZK;l38gG!%Uz`kiEZ z)dYr1>|+27@M*O1zY0Nxdi=KU0@+W;E%igqWhvbB4IIhcS!F6<4TM*5uip|}Zj#&- z82LPvk<66~JbvRU@c5Q5$NwJsKn~O)rFVobWS)mFUYX#Ur`*iZ)Fd>vA#LTF%+IKm zt$N)nqZ(7kc{u0XWYadYTonU_7M2#J4{~v)-EhfdAeAU8G7CiBcx8TRj+U%jdk6R6 zfbFDe>V%XCH0R643JRiRnlA|~r=i&=Jbx_q$}OZK?cHPGEJB9&1l&Dm=9PkzREl^3 z7Dy%lu*j67L&z{zY_UJ&UscWQ90xO8t7zF`F=f)7)K5%~ZjD*_2Df0Z-pf#f?}0SI zMCxzt08_qGr!sBa^KXHp_~PplRH0KHDF6G+_r9J2`X=cb3x6N6HF=puEdKV(##|w; zlu0v1Y|PNKnLg4vYHvQvt@{%Xv4JgTU-R^$Wg1reGOD4g94puiFyb&bLG7`pZmhL- zIvTqSKdT_nq+3~R`6XVUOd5y)nRL7r`jo!LrBq0vlEa+x8TCP09E01PZBb*S+SU^PH3U%KiR$NEOvZw5uI@*g`tk}L$% z_^}O-P;%{-TN?{O{Nse(RbV8~Gsm2myxu=g!zp0{55xHn<-lTgD#ALOWXB~2Gff=^ zTQcH0g&tLs>L7t3yHU|DhXPZiKNS!J`7prfA~HpU&gS??*TVcAjF-1A=*mJpD|}SF zPh>$wn`_r!z}ND%@!VUP`N$`-*j48p=>B92UxR~Z+Nbj<6`&s1qvMg*De-h{!+qNz z-7NMUtOBY`30yGLZOM=buHeHfbO+BPyx)N{JVm*Zp{C2aMwJQG8~YMgeJS6ws6s#o z|C7wX_5J9nEV_gHP|`$9)5dvK{EIuc-n?WMT;`9!EmIJsF8i#O(UVdOfWd>acd0BP z1j0LPjqrb9~PAxy@xwA$;5Z+$FH;mqUc;Tk2OOBwjXg z0b#~)X~UTTT$C*xeKkvvY5Oc>xyd&WnAd@%+>G}}v>1Q@@7d!oSb~VyXL72uIj%T1s6ueIgm4^PdF$dy4;Qrdj z+ihm8DM0Ghf)~wf%_a=iN?^)5Ilo}bkbWr(Qkb$TMaw1)E%1alGj`vm6a$5!CpfkM zgASSn{p!SjTLqA-zbNybxyB#Uv$Z%QUFrt6ejLhy^%YsK54G#^ymgM>w5v9;$H;-=zm-Qz#}ROyO;=)Ih6mx zom1BzM|j^AYYw^DS*uUUG}9lE+zc3%cI$hT zsauo8uFcIH-6--GF;@d6q)(p&Eb>=ySoky_SL8OSldOqq{Yv`)2@Md$g;tR4UM9lY zMcnktQgirw&smV913BpB579=lub4j;B^!z^TL1*eCwEld@09yP8SE%bi;VfLZKYG< zFcYyo+tumK^r2drwjuh$0bsVPZOnY=sMRsr`AwP{mCv<317DA1Bt)&TP!3V`SteV> z)b^Cg{4G==fC-PWo%n!OkV8WzD8v`ZE$=6YABF)3m-73x5G1D znnJouMWAOxYZ4_jmKsrlP^OeCSwA}DsQ?q~zutdzQhEhDDS_6B#2ffhGAG6>D7QyM zkJrZ!0y3j-@Cn&U-cnJ{2B?8N?xX z8Zoaz+kTSgOT?gJd>%gTrsTL7lB&IBLkU4&^F36d@I=xkm#ifJs%z@h+b%pIHN@gX)zlLB z_Zyqmcn`GVtULYaZWGZ;%o;RYiIH9pLd{{m1`a6alJ0Omy7=rt7FW zMX)X3o_bEO(aRvW%DLESgpi$bxqSY8bSWpi>a9|xt93z(50dPa_{)DQ7r@#bXpU|l zgcjUkU}loD0O2afHs*1toWsUmU>~Y9SXNDGqM7DVv+1F%SzU5i(?h8upc9nY7JuK{b|o-r9Z@d277 zn9IbAF^g-fLaepvbGp6M`fUs)_qE1yr*B*hg5grAE(jHR?k@E{`NZo4; z7~7<#Z93B|ulWQ!#g|L|YM?AwKa`G-dj&}QY;EEgHj@iskOq7p-zwZYby`NXG~-jn zJ4iOWOE8eYk!pN}9j9v9=PK`?eiUb`Sz@P!N;0XWofYmI@ zTMymY=hRk?#c!Sg#7d)ct0h7QOk+SKDQ(20!+wM&C3h$9>I5*7-(;#bRnecG6(@YG z3m(S!r6C$sqsp(*Au_f}14AKN&v~KsL8Za>)rt$;jGk#J`OJ=FPHz{>eG*o+26hOA z%iGS{oB^X_knwW8tcV@!4wkoP7)}F8$V9Wj&c`^(4bowE@1KKCX;zUOyn0{_#3jSL zu2tDT*#eL*KyWxEo zZ>~$G0c9g!4cqt0EWi7+r7P?w-2iU#p+0}p4WRmk%ylFPk2IjS|K9hr003<6MJ6?2 z2A;kFsDYUwH>yNjkO4jQm_WSDjRD>(?2b0pxOKk-r5bDIe}3FSzoaYgR3n-L&}j8j zgcPOF`Bo&b50o){{u12dU1El+6KaDxp?ATraAOg4oAcj#1Rg%P8Nz(hBQO`>Y#_%y z$FdHDzE79Y%pFUpYK>&h&a_551Zkwe8nE& zxCt9u=RmRVVj?P?ambi_`K}x+tG?u_p{M~+8$dNs8%H)%F6*m9Ti*uqXh_@EMf8c} zK91T9Fc?~*%X2sqN!~Z(%pN!s;++aNkdlZ#hcc{JwCjf%q1P5a5a=FPqEos4$_N4b z2Zaw7{taSlEvsA6c8|qM(jZ~>Km{zS8%y74(c6)8!6zqXmbDMHzBDdLrvt*RYNqD+ zTOh;igjkQal*9bl@^gG#fb#fcVWvQ}qGNqFa5wbK7?SX$kmA9rKjX8VGV8SSB zrk9jnkNPG*NbFF)4Bfsx(@Br%|1y}$AkO2XfNE1NgK!9Fx2Ssc6ea(R+>51``BviG z2Nv*iJb07wNE&2SY&q4QQ<3=YYBCjb4+*_SQRU^F94F8WLa1T=zQbDWpB4c}cMZ{@ z?v;8^<+G#OC^pV&?ywaoVOlnzzgG|)?eOM)%VyCFVihO^rsAU=e2Bwm#v7OHhA(EF z3hscu+<=dNoR=u^h>s{y&?y8DJs$bM8so(6moCMKp>n$j9Po7}69?arL3P2OEptniL^+BHA>;hGG=g3G+e|%Z->%m?ZRmXWMD>SKIErPe0IModEQ*3k zT^evrHu1_NyW(;yvUJcNB|k_sSx{f%ic1Zkt=@S`&}&qI?4H8-9W|r2NXovVWtlCmoBP%JzZi@j+bjrI~xo{8$w)GlmraqVb2d1Ufw>+ zYX%fVoLxJh_;*k}`(tOoN$v+)oc^aX;QwEp0TCf*V0FFJZhjOXeNPKGg%y`)ER|zp z>ZV}0Lk>FRb_=a_V$s|U+w8)yS~%h5TcA2kTh;@=eidcXpeH0gX>vBeiRg8&J17{& z^|md&kfZlM2kN;17#Qf}frc&5AblP4;36TWajug4coe1?aHdiK45dI9W+0>nslY($ z>|4Jrlgey%ieOZY{aqY9jG+c#4lxs(G=2T*7OGSjXEk06+D9(M+0W-=3||hW?SO!X z>ABdBF7zVz*QgOFkvF%wN}uX>n=Z~Odg_$X^HlkixTxlXHSUc^16J646BPx^bncds zWUzBU4OWUn-{VZjXFmifUFtiilq;XiX&Al+&Pkpi(di$0C9|Q-o&`ZuGHZ!yXh@t$ z{8N_*O2%-=I2EiYe()r69GmIjX1Em^c^G8`9Lfa>?t^6I?lH6sfPC~b2`YZ(-SJ6S zBU1!rLUuStzpHHFJC8)r*3z~}_ff*n1*vb3EgLqFy$_nZa zn@oa1cqG_*fO!tJnMs}$471QWD1*(BihXGw2k4dl@UX|RKs7GY$nT6%7_3d%aQ0CI z2sEC<5G~+(?bR;${SyP|r%1cL`w3a{dI`+}kUE8N%<19q**6;ukf{2JcsQ$CnP zDl?vGBi?RGGHcVE20~vB7y9TL=wa)QyWFK&R6hJqTLdNRFPU0_ED02afwl_N2GLuO zD=#E~EZ|$ykRv<5{nN}He# zL{~a%7CTq)Ps@PitL+$>hqm>@De^3lw`R%SFX?J!;Z8osG)wa?kHviYNO$}-AVBy9 z)^{k-5x!g$>Lk^XNzef~H1gQ2OLNeg!0zzJ$J`7?Y1KO5p=+2*X3w0s`VhyBAXIM0 zKnQ9cs&Awdz#1dc71y})dL(>VeL!^q)gzM6oXxDZ7QfYD%wSmgeWs!oed@8=wqBfc z{$<@-dnug@V(eqk50n+nUa%(wMVH|b*|BFdDJDe853#K4PnGcy@7MEU!vFn#h#gW? zBjhrhR&I9`+0>T6oKgz&{Y4;c=MeD+<6SYr@3FwtisLW)Vv0+#0}ODvH)k_eG4MfA zaV<~A3@8Wr$!U;qb_txFOje6!KFRrM9-?A=2|v9&^}vcZ$BiQNkwK+w?&M%`GIAi= z&=F~U>k&KUJAsagoio|YPj8&C42pkyPOaNMx>~J}lRluYx!*}gC;)>$bcAY0T+PJ8a&L~_N0x7v_-2Bjce73{92Z_Mr3HaQ;+*LiM`-N_>`+7Pk zP29@Bj<;`a)*ypF=_dCMJjy6w2Kh;0>h`;9ILvb_Ykg2z-%~sVMoz;r@JaT`jfLKLRXJoW2hFiQV|?EfiZbqH&;3W$aIPK$P2y^uFVc zC+D)0S1@@ll&MEW#j=wCpBdWLs_%@zH&nL=nDU)EEi1kV1(cD^7U4Vy+J&=ztwa`6 zi)t4x2%RZ+Yq!ldx~{XGgNA7o&QI@2j_MSqI^XfFH&Ad{j2P`V&Ii3wiu0fMU$-ss zK|rUN?p!td>4Ub2#cO9nP5P|hpwmg(JQXT&8)ecwsk7XI^{vF4^mo$^LBWtKEpq5Q z=qbGl>bU?I6lGID0+6B@a~~IRlNPr}HS3_71#b`_p$Py6W1tIzx+j{D7BJvKIp|#)rpb!o`9oDg`6|h9>7RAIG4LLaZ0(!Wn$b=5A#nuh9dW3 zP9-554dfn1=Dk48AqQMtel=0CWp=Tm@=y7ANM0$$g4P`(SzR zm}W&Itie_RiqZ>>sN5htj_|6ew4cSY#+uaQ)+v<8$6tHvPqu)PF;*ne$TNZw{fspP zst5zx8Fb!jg7nZV(?9MLNYArc!seUeNe8XMkNR-l>?P=!9`i%zW+MmW0>ZJrXHm|; zD1Xch>Q*pVb<4AyX1zjCr+nCMw(PNSM|XSVn6M_Avg$hX9(q@oBkNawP*Vq)z3zy| z-w`h`kE5oGzte1C0=C_Q4d@#Zol~Ys!{n7;nmS>yg+GLys4J-6V;bv;Xg|OvX4n5O zW)nres5#B8*oxfcB(2?W>74VcE%Ln-_v&{Pu6v7tuYEWOTS!ZqAb0wtEBn!vhh-$O z`48I>)tOLQefbo>bUg89%>Rz2#IsC~6camA>hiux(U;qax3QLE;F6XAZ-5h_P?wNV zaM@Wov-uq2C?aW(Mf;@GK`(cIgXm@NqxQab->0*mQ`6g7|CW8c53i#pa4V>AoVVbN zUUBn~yx|`8<2GfjJq&yEX|409-i)eiH<*R|r|r!l zH0O8G6J39G!QB-y|A)>6 zovXm1SljNJd~d%6qyQ%0Hy3(R&Dp`tyrafKV{jY9L5VrvGljNmRZmcL$}_X};9s&b zoj{dJ>UmXS7*+M;cY$)`-sHwdj?S)mkNgVT?o~52{m0b5Pezd{y14lPQ)h8=O(vi1 z*6_RFMZG^b8|+(M;}rtc+rWf3NUx=-{FT0J@l+Lg|7BN$Qpn(cAA2EpWTzr+JpaIp zUAywDaQpvE!r)!NU>!q39?#nxS68^gM@7Y_4u0tLyZ|6cmb*D=?a)L8GT(aVXU(JT z^95An7?m?5qZ^89WMz#>C$hds;1j7Owlw)N`lgVzj_oBZ8rxv6Mn44|;evIXyje4BwtGmxyOLApCZINf8Oss>18413LM?VK!~J?gtS- z_$Y}?D*v(NK>?Tssv7#=*z%y~1K{V0{n1u*e4xYeXRL5b~;ZZtu zm;8tQ=X&utz?h0urzQm=#hDLrre zCsDN7h4QVBlf5;iR8-&kCPT1Nyo0>fn3NuWO&WH-OCxZZ(~q3Ll}ec8bOB*ay?z(f zOLMz5KcT05@+HY5HB=qN;43k@@^{2Y**Nlln>fOK|4P8_(JVMLi1OgmA8hIH_T*X) zIF_RpCeAWF!baBt&ae0#rz}NKjfEkRdy_#w*HMc8U#O!rUv@p=Ot`GE|J`*Id?K%{ z!O_j+{Re8}x8&)@TK`3*z2>aMvljqYE3$R=lB zQkm?(F3}4DefDS#?&6VQK!b;5 zW#_gcvceGslCL?30Y7xMlg}0_mE(wb%*X+<2wZ@9vN9F3Q-opFpxvdrr}P#mx}Ra;ZHFEBCXRjt2V^Wremx|U>%!pP zX#5HOTo$(lhqQe69xo=}y54w52H(!J0c{4T73B|31rBT!qWdPaTv5MC!1{k)^M%(o zVY%%pG7c~K$SCk^f~p3e^)g4iy3^cRD;<|#@Wd+;UA|bIQgZD);U5n|QYY|gCyj59 z?&3mGZ>QPOtWC*|=q^|Knh+b4)b&E=J^~a!8o%A_9Qr$nJgMcYoNVcg99n9CfGwSQ z92{{t+d-wdU>6tW3T>n~R*A>3&1ANo)mxv4_BB=hb;ded z0@BMCVhr~m*&bYk9x%bcG43*ZP95$C(f3?tw782Ck0uyWN%C0dRJ=5TZ`Q~1d1Axs z7TR=T{b7|zX;E;vDvE0(#|EzZ8x?VBr3C+ z8$Pk&v<}`Uvce8c_eniR*GJ2#+FY8;>I9iRxN5G&zRz6$K)OoAT2DFh;TO%TLk`@d z%$eF0ucqEgT*`zHzjEVw$ZM-2YK_5}LkumDg?kiF4WA#Np;3Lx`rA#>x96a&VVncK zSe+&1!`ZNo#4MTp-NiKaBmZx6^_io{a)*~=K7-@QYPOUb)L>8{6B*$0X`EI=?E7SX zoi65gzjaV~l*R6LP0Z@le~4@DF>OeWFL3*uJtTDb57&NOhsPZ99_Un5d6oA`aGE6T zT$|Os=fbiU{52Werf-{wT`*}ELhsfez9)f&n5?{X~5G+9O z;0=V}5G;7m#sk5-4tY zonIo&vI%{@j`PM8_rIna9f2B)QCx?UK`Gc{5y_@}COjQ-X!q)`srw(3vt6-Y=nln0 z`VUJg{(G|JajFx-LgpSw``7B48k8tlx}~8m)?mPWJZxAPZ(`U4u_D`1JoW3a#BR7$ zL4lBV+iY8%0(|$RcC?iZiu6p{Hg*T0E8deW)3} zBt}@#-|x;1I5e9tSht2CSm3DOo*APm{e^7 z!#82)_bXd}-O?#8&gBo6xW5Qb?sQq8|DabVTb z5st=RF2TO{Rywh^a$VJKk2~}b!L!x>zYF{op(zu>V(M<0myK3(BxCSZ8h3|`rA3?J z_f&ZwgO8e=D?rH7Jdei_?F%`xtXI3f||(${_gx>>`62+d?9lK z!d`$a&Nd!-{RAE>R#HK}N-5(h7xK~B&mxcl7qWwDIJch*OVKiPE*>eSsAB^^9ndFv zze_K<8FkVZr?<147~#1&bHE8vg`WYlHtqGeb_d?Hz5#`G?z%_)ZreALg*M1SC8~tb z)ARQYcUl33<(gH_*CTq!2-5|0(NGt#5ur-!?U+wb3)3JpJ~w^p4T0)!2gPe6&l`|) z>RF#JEsE{rBn_6dT2P@ZOE1Q?W%y3;ZWIz6;Rc9Ga%^gwfujqqV*LT9>FuFel8tAs%T20?mN76hJ zuvr-z0sYz?mhl)utEq3o+5U;dcw`8uH2k3SqcEMuWoth{yx5C(O0O-b{*LBO0377WHPPC9yF0CHG*hOX7(fx%8?;tYYE_oZ?wpa# zXeZ6iMTdw1GpRz9p#3Uev)(#S;$lBfb%Zp}`5+@k;%ke4ZgV&8CEm$>SFS3zRZweO zxEs#oWCq+m=H(_nzIPeILPWzC$6(C5^jRz}S$kR^!b_8WAcT>E>j+rwCUUrc4&%$Q z7?qom^uK7}EdpaCK)X4>NHS|0tZ=+?(U6w`W~eyR#|`nHs9SR*Nq6X_ki@fr?~NXG zGd!FKtV&4S8>Z$#gFWxmXGUFrg)!IP)F`2K`9RQg>K`*i8HUV1XOkB?+uy2Fj#(KK zv_r55qebc)cL7nCLp32hskH(o+`o%{r+*y78%Ul%gr&ho6a~|Iq&1Q$369w7fk;ieq-1ddMz#J8lp&(Ealri+u|^PCkgiKfZ*6)#?DjR(L#W6v<~=yqe0P+-(x zRTqAR#7a$(yQz((Vxy&~um=RraPU6f zwY))}PL)uS(z?p;@A~}MIcnFI#lST1McW!{KT#uE@s_ABo5hC=BM1tPqU>q&PA5t_ zVYsX60)YcG=I}A_ENL)?p)MoFEzoMVe~*$WPGO!DZVOx;T~fT0L`bW;dr7F0D*lBQ z76a`Ye3((2?;(LovSCum$O`!z!RJ|P5&E9K-D(UsburwG)iR-G{)A`M^ zy-#1R7~&~q#?~dY|_A98P5}run`6tXhb)SFZ>0*F_NZe^qEBK4p`B%53Yxu(EL~dZW`l ziU-5mFcrMpEQFbWlcfjeDeFTKEG-Akhm ziboJ4ZHZRE7x=ns(=U_wg-Dcy=J@ZDwk46b42@U5?&J*zk@28dWorB+SNn2AaM8)e z2q(MA5NE5~ta7EsoohL=0`I*{s=&lx63>ErySYxQ+u5W=ytTatB%b-#aj=7R|4`!( z?==`CPtAyXc9X;>&9S7=!<(K`i z>J<}K_)ZjBqU_(rP%}!`w_+hSND=T79@pYCHsVT%WhX{gX;xqJSbx#=Xi5LhD1t^L zWS6`h!?*7I287GFfJ@rH#YQya7T;l@@-d_cF4pin$Nb7JfbA%ctn{T?i$IwU zv6cUtbXia&+mTl&h%4sqt%hP%I;~MHzeF-Or}r6w3O9ii$+pd~3{*0lOohTxwF*L2 z`P21Njf628vv^-N`GW`!CYD7TtjbMwSs+DgU^Y7psGAcokPfLVKo}!wMyg|?RkvAx zu9=;Rk7~W2!=U@j0vEZdd}SH!3HLou;TDN9j@e{1iiWIb>dC*F>La%tX6XtxTkqch zgRS9jv&k8BM$z4V@<6zkl}Xy2 ziw#vq4 zuM)Fk4 z?(om*1ixBVmq%p0EB-|hUE9{OR7KUIfgfBKn_D|y%0MySR|S3eiyvh?mFdh7SKDunh?$}& z2mN$;&!ZcJp2;-7#KydX2bF@#7_3llYoSkjmAp1T2u&{to{0yg5iMXED8O=^(*etrx|63C0F(?u$W~fjYD1yoZ+Q zg)u3_xPC%k@G%{EQauYLKodfM=)n_R7>j!M?_X)oQxmCXc70F@CZhOJm(TU6LFOhq zp};JOGYtF;L_YQ^>F^{dIye~Jpo`2(zC_VNq1ayn->%tcz7-X8^mAq5iQiqITF>CV z&EAFiY{paU8~Z)JU~k^cAmYR~k>WDsY@wwzEQf(O44-aS4E4V(emy^Pf{?zl%+xy% z8~#HUWDM5d*^T96c31~KHO+DJVci9~sUJKRuvzgW^h7ZbXA) z|7zl0&>TYt^sAGBbL+~&)h^}{@;)1PgYB4GC^aWs=ey!#c}{7@MD!;Q?dHy8h);Ub z;^v=dDT;)Lih#P=qN^mRjclzEt4&Sys~}y#KxOBQt|2*ere#i*xSv)Krl3q$JNwxV ztN1@EXNGt*5GKs(vc<9U49n}u*rH>+MX|NwoS1GK7k>G#SZ^0kl6(E!o2Pd-(~d*R zsKO26C3L>!y%xh7*I}GM6m75Sd7@&`cZIqsn6_InPY@Gx2BWgpeMxeO7rg1nO)LK9 zYHRQ0Td~Zk05^uAGVd)5cCjL5BQE{viUTSAAyeU(_=}{LKwKYWOrlqj{Vow9n_F_n zg#M~}PB_c=zKzmWVdrtNF-y+8>CDG%M zftX5Yf#4OTx|8?@3*-Cny}cK#@Q-xWa!AH^^Q2X?G*e@84E$IT4d;*yo~?W|B{Hej zj1`Y)mzTC2^(8hb>n*=qD2{OOQoE zoqFyrL2ZS;QIo_VxkkuzBCxU)A)+x}4FA+~0P|3LznPu#VRznHR2FI^u6FmlZCtBD z?_0|0&Q#ZFOgBHDsoYD_hOgX(D=M{a^ZyxLv)I{uEGYPJigHi4mXy6fV_gg}yJb8v z-CQQae6n9UoCsCMOnZx+ELz1&&Qhuq?8H(Z;oh9SUhklK3`kEGKLoe;kVwuWaNRX3 zUnB#Us?LGxc*Vt5)X`Yr`p3IC$)b*8#^~-qGTGroP!k|zn4)o>>!gO_`eNbhRBlHq zD8sSD?JiB=91?na zadc$-f!bMuL#7!LjKFc~gTvjF1A9HeMEmEc(kt1?t6Lav0i!_vjaoC}#IQKQR@Ya5 zkRlg$Ww8QTo^ksy#6DJU0F1Z5+L4lGcd=ffba{=Ihw%8_;(#TXj1R~|Jw;0tf&*{r zdhe;NQ{qZ6zph_As}izQeHVj!0(!Hhen70Sj6o+`n`Longi?VT{O$QTQp_Z9flz9~ zA46Q|FP5x$+j3L^p`sw=fUax1DS#)lDfYFxR?FE6M?GsarfWGGZ$M^&eYAp2QB zDF1kN4r}IZw51WU9<`me;>WS2m1p-rU1zTUptdVCwlq{C0-8eSB;0rf>5Cv| z6_4AH9I#)FoR#Le`c6CIMp)Oi%JNR?G%u~)Cc8_4_ORgzDq5`EJ+yM6H$ z{Z|ezHPGuQN%3&ruCBxXV$S+LN(7r-&DBEl@N~LY4Xn3 zOU#EAc9W4h?ow%bh=}&_3<_)Vs?DxeD3Q}W@tX9vR;P~yq^(NyY%VLwHFM)IOQ@L% znIP4ZnP>rR35KCgs8(sa3aB~YZug9&Xw^A}~kxT;|?=GXz3 zM*NlWTG0}{MP3Lcel%s|=_12qo08`VsP7jkc#A~L$$dSMc(?K;2=fGJg(Daf^7sHM zt%#JDPSxLS{d^XhB&zBIwn^9rw8UaetAn#~g&)s#;Ru+wMN~ULnc4fj9&0&jwYHI& z2?w(A#S=yC&&+pKFwgM-xwbbhcuKmmlnE8Dlv4G=NKWU7rf_B`I8f*-0C}+q@)LV= z+{UcFN6$vAPy86j5Q1g7JO;#2^0Z~g{o4;0_f}RCqIW)N9G<4?DeveNxk}EQw9LPz zBa`f3p}yNfeocj15s=;k>09|6rT+{^Q%#A{eXSTS?EQQibkejfn3fj)CM2&PS zoV+fYoiAj)ZHDrMjVkmd5chw4! z1uK%6XUzQ8&!CIVE49sMi5lft7Ofj4f?sASfsfs?L?d^V$MM}@W?5YC_K)v&`%&ES zXS%>Mg3t(X*FFC(LxS((WL;k-xtf~WUp|q?JedQW;?o_dy1A-6^A_QHK{UfFf8nKD+e8YE-a04z28zL0e?YJWS$(I)hU6C`K})2g21|)j;rM?wAJ-V3ev)HeWRU z#DZ5x=O4)tNa@`ymfZws`w9E_Ut^Mxf*zK)dJjFTDb|$o+w;&5{He2uu|TX)VL>$S ziI(JL-czI5f?deae{a-DWFT)#pDpDkO~yZ|8iyA456w?cX9ijyJ#Trmpvir+53?iZ z-1R5lUI6$7JOb|gPE8YPrmAiKUi}N8YJJy*3MqABVk#$J3CIqB0z2GRRiw`2(FH@z zX0-aF$1DIHE!SV{K}rE6>yM8bwKM23B?;f?@A?f^%G;)QXm&oJ!3=dLY++2qOs>X;7YtZWnEVTo{R@Nr zLHin+6H-^`7gxWKtX{qO!{QtyoTsT;eVgT*GON1Z+LEU{6Z?n1Hw8#)S=DT>Xbx7~ z*M-1{FI8<0O4cIQLeSpFK5glQ&W9ym6D}k4MT3CDtb;Mj$I>{OW%ut+IS0`9F6r$! zVh^_H*(w;=S)Ar&y!RGVccEIGK&|uxxy(B3QW1A)5GbDvJD**P3AzrFM0tXAfc}XC zH6_~r;IJJLn{9ZVaKL+ad>sp-uE>#GgT_)%jd;s$W%}NK_y?xyx{dbEJ|2;m(|W+y*TM ztxt6PZW8Rb<4m;CNVxkL#0B&D&mbb76x$F%O_QrgdzDma4XkELoxhjsMhpf%E9wTT zOT?WRM=IQCD7i5A&C{z+7HisHjCvCpSiVhT#{0v1<^<*~$$9L93GKYJ@D+&6#eQf; zDWyO6PjVj~!Q}R9wdlLXHk<3zT_^+ODYz4%v!=&QCZg+qaM+Qw*4uA2{PQb-W+09V z`^xLnUR?n7h*f zQXK=FLAY&tSXZ9qa=o+bxxZHyQ6aoQ7HM(p*P(*7M?U9DPW&bZ4R`Hp|XT3Ez zHlL8sV|%JP5XOeT%x-7nd%^J&g!H3H%CEF1Pw_?#|KA0vo|!?rKsH?@(Tqe5N_`ie zN5-TwEd3&uOzn4k(v<=J5cT2DZ{_N@xj4#c(MMq3^g4AMh>$XAh7ZeNC&z8xP%9iE@%mKeZuJGWIT47_#u@ z1osU`G>DoXXFx&Thu*yr4NP>ETWDE=J)A2{y0BNJL)l)p{S z;QrP@Iu;BQW;nq&@F!bPCsoqbj(LKJ?#-YMR6mO>tzT9YbVIafVK8Hc3_?6k{z~j; zmEt121lg~F=1)|D(Wbm8q1*v(hLvsF0^!cCU&X6JGP&pcUAg=~|UL$w>5y{lZpe$UsvXvZ%5MuI5M$oN6ps%S~q zr=sueKgti7Cr`?UdVad6w9FI-Zc{zD2tH-|)jM0oJsBUKZuN5~Ti`z!tBK*086lZb z3rCM{m>$m$CYl4i>@ul>JsJXtt8VSl+vMuUk|520QTr!Im-F&1ocQIxYwA(#HlzHV zX{hSZL{(FqYM*P>OrpfuG)>Ds zdXw?SA^pUqoi)ns6cDo0&9Iv9wQe(O8*ErdPrzl0;1S?Mkr)x;~a@kd&3GK6Yl13c>ll zBz8kF9h9!p_wrINK|AJ}bH+w$rv>9_te0p2)d%msBlJSbrhiSEfQMxf`8jydz?*a^ z%^3$%dAKhbda4@RZNl`+!mtWUuaw+)79DIBXP(%hLHG3X>e(6qb^y>etaBPdZM>uO zNH$hnJqhdFpqc22*J(A`cK<(+EgnrKnN3?aHS|&F_kHF4Q3bcHs6lbEInkGlWd~n+o!howikxZw7`ou9wN=m1U}bvfaOCycz_peB4PC_ z`#8|cM_ed(Ia|%CKUEdI*7y7d(k}VO7Gh5HDjYu&M6wso9i4=kgv@6F$E&|B^m_pcCZVIU^cCtoPyvxnG z#JuMCEmTrr`vu<1OV{xP%duBk$qN*DN;|+m`QMf7Yj&W(z=c@i;9ITT+@Li^q%xP! z@6ylM`5~P%@8<|D4DR+nyZH3zh;Xz=s&Kmjfk}HC3w4(TV)VX($d?+!t14-=qU(QX z+9ShcRp!!pEb5k0)xAca`0<4rCmiAAC%K+%r2aw1e9!Mb3xc_z^%<~``t`HU&P4_` zo~&Cj#O9}v@;9a8@*ZOP?#Tb1piSdWvf??LR2sV4%#PL2yr!VUctqwOU*9bz`a2j<+0el$1W zMr4po6#U&8K}9e$Dh45oJ^+7rog>S=-kxNuR=%E#z7VfpyeTIzblsD;?R%A;^ zKua19r4AgL2ek5C3`$^2&Va%6d*+oWQBL8VGqgD@Z=(h;2>0H#(W?lWJS=?4fF8hH z@!0g;SOTEDGXq$hKp7Z2s>L%FuTnp3z&2sdcXzLj$GD2mQJ9cBt9+YA zn&vh29_yz_rzMcAj+Kr)kcBfylxQyej{2f7I8KuAkY<>d1#=JeXv)F_=nQynZ(6&N&oz z=od-z^W*1Gkex{d2zj+o{1FzlS9L#$csRnL-92jiDJAk@Z*7>3LQa8N8U|=_qpB?yw&$*)Et4*)yrrn@_R3Hb+xOVsg%bvyp-VgKuP)$2g2kF~m}+5fZVc{v~_{ULweJ4@4x+C=_f$r)e~2WbxKJYJbV7 z2V>4aU50ej7fQ;5{;pG~&;TJcH02;m#`DnqsJm(cp1#p_$RC<9oNIJ&Ps+G%TG%-C zUnhAjyCf}$W{c^Y!?t7L4#jZ|;rm0@!R!`nb)*Wn7^j2PI9|^2?t2U;&_w%ZNkb99@F@75clgD z|NKWaL;md;j`(VZQvc077A^J-7Ks|~lSpk5KpRP#B^BjUT~k;m$~=GTYs}(N>`e=F z(&>UpqHDTTupZF|!Q_k4@^Dc5_lggiq2B@tgeqf!un>^5`|9I@35(;mj0S}dWH3OF zlt(>L-OR_oQl2{?8EU@oTY^Z&hXPPlJ{_HEznB` zCIY$AuL622!#kuTC=W{NC~zGzDdGmJa)C7F`3jh*d2ALC*AKQ~AfsfKS$7x0s zF}W~otl}X#@UENzsPz=)*DlJ^g_mln z)X-^Jya|4u0$ferfLQbq17cTfs1FApK;}kKClp3D=8+D(CyMypJDy2M*e4>0K_AOE)HyEkB&rQ`Bvn{mUza#A+M{J zr0nRQJXC6^H@(N{w4CoOe@CH4e-i?uLKt8}xco*P!Ty@md1$kXAAiL!=puUTVB!SJt=mIHX>;QvKu6D7+g6E*Z@QxtV$^#^?rIxgA;b_#}5u&8y=L?uKAUt z7`<=9!5I>B84aI*_H(_E@1*Mx;8Q)Q49An3#JtY}gbQ)Z{EzEDG&zF>aaDknu6LZl zhiRis@A26tHj?(2ZJEG9hUl7-itejKxk+XyDN16TJMosw$?JTsVjX}H%RyvkzP5Oq zqWneh-`);}QZ$ArhlQ}@YL43_dvJn_bmC~#(A7Z z^?VPcy|E?Dz~8ddwv4MmO41$#RT3i zMe9YVaq!s|e2*n&PV66I0?3wsBPK8YA|@z5{B7oJ$oT@4X$Wa{9k^fg$v@p{5K+@5 zXT=+Um4f63%ujsIYQe*T?lsbFbGrdRdX;PP1r!G0Idhy^$JYAlKPweB$EFrUO#`xm z_S(Q3>vLp#N# ziUcNGYj>EmN%%T$i!$+bRt_6KcBuFV?FZ0V=rITlIp=J(!40L%8-PVU#U#VK$K$h=P0DY^+A?4*=#ZE3Ur8&P8G?36rj8k(&8qI_dlbZyD4$rfmw@17Kyz`9J7;Llq zx}QGq+kN!oufp;4Q`=zf#SpBk1`38I5b6t}d$p>hYtbfsx945oV6U4iJUxfh_}CdN z-5}J6knjK~i#T2T|TiOym-@9E zj)b!wo9WJI_lx!ej9xX%cl*vmnBJEn6ZJN(o$*}Ook(ThIfV~&2S_|h){c4rq^9VYgcZ?@L3QXRw(4U^{-;fdBxZqaIof5JS@g8{TWq7V|ST)yyM5YxP;&iG9rlZB8?!8K0T_H5-mX$E!`8?;^qvM#LWU5IohY2%M7YC0j&S zbpo0y%vS1X?!Z-KO~RDA&bPZurK3&WtUx0eb@WaA3`yeKWYQ{EY!6wk3MA& zK^Lsm9lYBfx^h44*?7R!>Y4FuT3p^$abE@B)g~O|iwn}8m_@659g=9vP|Rr!P^$|F0*LCP*rr)S z>@XN-QNl~0h)7A-y*cQzg6p}+ZSIDs_iAc@@eIu~3%$~9dSzbV^P|yK*nHefPeQAD zy*;Xh`hk}EIkr=6sbw`AgWG4bVFC}}5&=0*@o0j6W4JFbn#H>3sgx;CxYJk*07lQH zyvmN}`~QT8eK_j%d5ZAJB^|$#c^vv)ZDmK=XW#u{_BJbPJ=bvP!BtI4cV6OAUAF95 zuxW#+~QZPtXn9*RXhl3IJ{!kcsl zwd9KC5|_Zh_Brn`eh;BH*3}u~6B?OMxsdfJ+#Tm1Xa`w?2oW&>~RB8yqy1vz1z#tN)-Y2Eb$(je3nGu`I zs=9fb>b*`%d2;V~hZ=YX?k#$_4-*0p9=0}{$|G-oecOmaxX^^$kn|Bp@PZCkxc3(F zgEJ4<#&~AnIFUVmWW;B2SF4q@$?*EQpIWYS{t=qQV0AvpOZUCt{K0-&Qq5S;);0AC z1szPL{%E7SnO)|2)ul}7luW9K4?3G;9rl(FtNDUach+m^j5f|!C%LLAtOk9_1`*e3 zbB?j-nrzFn%P|=?9~v-j^EVJ-s7wu62$%-j=y2!GU25;EP_id9mY0$*@%o+z)a@8aF#hB%=c1jbCDm zziTbyIlNmTgV z6AmA}C*86!x42(Od`_9XYPrHlfZuX$6gB%nedA;q-#l_D>N$!>KUA2E%IArvKksW5 zW1m%S+cxKP2UM;^UQ?(PauiH;W`{N`vcZ?0$GH5lwen#bc$NK`cARu>iWOcYJ>z$3 zu!&>07PB_?14Y!hataaY+y!b7utX+4{KLRmjWfj63vz=<$MvbNwEz zd-Me4q#1IcZTe;+-!t^Mu_`t{E9^o!1&}94g5VYi=+8jXW$o{4UP3sfZLe=NtCzHG zT-?3yDDK;3krogoxA}7uXKUPd06`!8nd@0C;!ySM0drydY_%Gk_bBu3d|^b(^K8oN zYb=c{8*>{&zXZ^kIPiYBF}`-Zm6}Q7TK1UO1Bmj*c7Eox>fSXH)|>mljrf^fH;lxD_?b>t`_yv2ovCVMWuD-`&K18ETV9ir5V7Md7vaTn`r+$N zt!`SG+}SFtcPo7vE^-&z)Wp;9hPwpgKk)%c_Kn=MmpSacpKqfB9JAi-hk2$(i1jOD8v;(Be7RI( zrpDW%l#iq|Bi91%TfSGs@n`46=K*%#Y(4O(y$d6&?pkJ=Gx730tNP&6*h|aY`%&$I zYZZkk9t4XJQsS7ysFuJ^I6ZSb2&SZl)VP^I#wsS_%e5ZES zNs@64-aS30)>Fl++#3&VU_KBhes0uqxjVbsBrdV-{IW*D;PNBvqbFUSwz3)emIPTq zqT>_k8(Wg6vn{iffKl(Y7~pC=yvDxmlrB#$E^o#9lw{+1H(-pI02k$5P(ap2$M<$I z3+w;~VB}Kqy98^}Mo(fPl4}{Vild_zOMu=^Rn~TylXLyDPrOnfRoJhdDKm!G1~i}d zwp<<;ckarDw4F~;0dlL%NW}LchiX@n<#P%MLDr|xMp8p{M-?DiO&=-f@u-_O?~%zk zvNK&fEbE8xs#g0{gnb}$3PR%r8Bp`EC)&+yd#AQmp^KDd8Oo$}RTQh`>BxVQ2JK*R z`a2?pOB~xMpH_e1F@LctCj=y6jNCjBSdb(n6XdpQf}Z5R?zk3>U9ReqJnd$Td+JF< zRhh_i_y+pvi$$CUb#NBH9YEje?&lEBV;H9qB6*uHs;a5F>jJx%+-gqzp=p9BbzrAS z>E(`3Jmt!ojyeCV{99#xGG_cux?EU+ptF%GSY z4qXX&(UF@g-R#%hAMrjZdK$2;3p$-jLUFBS-H{v8LlJHZ(Zx0K!LjjfS!)2zyH7{A z(&f8FZr#oM4dE!dJk9+9yFbv2;|Ig*7dLgC+%rKfBwnNWvpGz`%12QOeP49351MC~ z=$KU0U7uI?+?9*P-LwZWWkt z5TcsT3ebW*HEk-fhFTCeUHF-#ZPW|bqvQ|t-OhHv@ifO)O=C?P=gnksxPvO}#AVFO z)CuFAQ^#(O%sQT!Y15ifMS`lbcUWV<=Z#1ZRkn~Bu zH7htQbI$w1Ek*o4H+DO*wp-Ngg?KddimY(xYRhH+4TIhS+rnRTM=mHFihTSv_-g-nEVIo69mKTkpAR#_nz=++;Y($m`@r&})HvQ3`{z0+5naYG@fp&AP#c-Z=J<K;zvy0T-_cA-ok(cN1eRu zG}lYGG^HB7*aVF=V#BO1u4{yXj{_T`&$|zk`o34<&c|xR>Dgp?eSr>VoSgz3Ti#GT zFgYRy=`RoMv>&kP;5{8J#-i*vv0H|_*DeYsVB&{>wj{V50)Y$ZRGk*`w%f=}~ z?;Wyb(Fo`<12_|vm98(k0)tBHpN6&~>Cp6OAsBjo0d}FAQ7)?QxlQb%HN5K?t}QQr z8XAmuO)XdB+l##A_$B}6js&oo$wqmv>Eyhsq_l)Vr1@P8{n}UtU`OGnJ>5}Z%}Ylm z>C`|DIsf;TQPuypjO2`C46;o(qFy70yCpCH9WOqH|Lqzfww?am-SEEPFVXhiuRs`# zqVRjqGuaD)@A>n-CkSN))G_Nr>$cot?-49{&Iv~6>ACX1bD`^I)EZf4fKWc4odS8fAagB)V9>U;$!K&Bb z-(4l@0!OuOYDGb%xrHjaKnxOd>t;BoLpPR0lK0)Nu2kobXsV&J8C@F;{?pN-oZl(X zz1hm9@5=z+FyWjxrD8W7%_@S=H)E6NAl%-bjO=o7!Z7vToW-Hw&?Euo5P}v(m&?Ny zV?T6G7Jhdc6!|HS6=YEYu6qhX)bgh~u7Bc-IoQJ^h}#rMxUAF80|T{FEvvw?cgpAm zBuMn-+}(Q*h1#uw+cBojaP;%rgLv85hLemyxHH|6f6n`o_jq_M-xG`)eGwG<-6sut z8cR8s6=GsDRHx7yk8Z{;!G$@z^D43O0(Xk$wE$tO^87|N7|JhLDKbm~;j>+qm$YBb zHm*;v;}A(;7SO3%aKky-AvU<+1_0ztw@bW&&t~R|mBe|(_{oNtqDAw*a!gF?raPNK*1L7d0pBEmE2JI9-oRQ738OO(~ z#ypP|NG=K_ImRiuTTKI2Hyp`N{muGPwnAPm}p1UpK zxIeF}UptufwvhVWB}Fv!cEPebTpiFCQ&WksQ)-7_c70bjNKMf=MOKB0@mTVSvX z6*EKLmfiSSOrZmCtYi)qGIP$w*;XPs_c^WTU-e6W*66waw-4WQ=&m zVL;r}sx~UTHCsK4qiBS{>hGU4k7%%$H8Ne0(ST71R`Uhov;Zw(Ne1c*QDZxW`1zU@ z+D9V=dF6J1P2<~+8q`0)su(1;)}gd}0!Gqc7Wl}^<$h|rOIUL0=TrF(-SSfj3BOwU z3CQQVrDk(eRQ}gL&T*_A?P-e`FO}e83P|#S#Qyb{eTn~8m-4;mv(O4xdqMUH%0%5*ExG;JMuf& z9o&Za_C>x#4=N-S5B-jJ zvw#DtjA!L5&PONf8((D7(Mo>6Ak)}nRlhi$=DLx*;e&wjW9pNyXp=NkV|SXvFwX|S zt-fjek)kg(gx#N&6MI~&FiU?L%dqp?)xFzM-Po%hkvn_tbB`f-YkT7TaN*sQ&c22C zvIWC&{*=%QBpDmTea9NRg>_2)z8 zbFPXb^98qotUpLeA{evP5cV*f%w{q#L)GFmtqN>n9cE6~pYIR;4^?Lw76sUKYY_oK zI;25Kr8}fWT0o?wq$H)Ap+rFGkS^(F=!T)YJBMcIp@v~#fHS`D`ObHpzw=|R$>-U7 zt$VL+)dQ1vJJgLDbH{eq+r@`)J}{znnhtGMIoPa#ec`lrIFy$xfIc_Xs0%0L&c2_Tn{x*0DqkaFTb;*N zGNKm59{0@+ys789uudTkZMmA(M%BW@a?Hy65)Cl-K80)sK}xD=#Ea5T2eMx!EcG7^ z$&6(k|N03&*3VC*q^?reY*)3-EqB-hG8@|0XHw7>q0!vJuSFb&`??6Cr2rk6-e4Zu;z0K*j9mM$g?JQgjN|s6$}l zSZ9mO9(j8mKiUrF#U#jukw?I9!7yeG5o>s%_>9>7Xm%QGUVr7dEw7F2nfR2+*Dw)6 zHf8>!qK`@!o?a&lA0Kxd*VRuP&eS8Q$&Jm&pLMjuZ+{}dC9FkrpEI;(=$tvE< zyx{qZZQoLWv)ey!QT07`)~YHVTS`rR)n6%0C@UX5BCgl;pT)#?ux(nNK|ZctnVBJ4 z&9-KW1+`-tv8!rW*U@=7RXKfcE1(91`Fs8*_KBHt-d<$A`zwoLf?DVjT9`-8f? zQnzF7e|YQbTy_t;+zX`sF_$Gj=ycUXe~zj3&(vQgTW0!%r{|M=?Vy?1E2&(9n>$6& zLF!T2=7+cF@g|gmO8PaSW$EYZ){RfQ)tbcrtbGM3luQ*h+@EuTQ%$R01Bs}f6M}Mw zsm9Rx`WzD}wXdzGU^z$~0XEvKKf3gPJwE%VdUJ_V;XXx~QHwriocU1j+utuNCqErL zW#vNhiGE75ekd&$zpY_1vTqr>l}|{ZOllaQcRsdxBAp}o&z|!3uh0BTEw-&(ASKmW zHC#n7fw=K`zHa=K=P=YMcDs?eiauY7jdJy5y98eEX6{9COj z|DTopFUhBqR@)-ac<6eh#?UI$9oIN>M7uUFBXn+e-pJEPePlG1AC^WxPG^$7mvOkZn*e7!4{0|AkG)YsMEs4@8B{$dw-zOzs6cXL!x zTc~dnvXj_e+^PC%vHGXfubgONV2hEG4jusNmyQ%YU4xGeG^|C=2@W-&NIRWao+bp# z(cEmMMxOC)`$R*(om;Ktv50TRC(!OU4|kufahABjh-p;^`9w0w8_u$rTNli46R8Hq z`k#U?`HeTlU3Jo#(zzfyhFm-x8mgH>Mmos!iUIc+Ok)Y;)K8@gh`gcUsaJ@X=jL3%Hz$Wt^uU(`Y!PLZHM!-lx+6yO)yws+y240C zH@ZfK*zKx_Jzzf;6xIJE){zhT_Z?07zGOPng%flBKQH-{6#9~dr18=DGSEM!`d{)5 zA*CXJC;i0%LJpX{mf2H7z1p_Etmt=oYgoBZ>%Y8>s@AQx2*M&BVC+xt@Yr&IuE>(R zfGdO7{%x6?c`^Oe$iaYmZCxRwR;AnbB>e=S4xZ?8ldbrhz0vOp+9fQ*cFQYF4LOC5 zCV&7&^&s5ki8$yG$~MX2Fsqv^Z&+r#=tr*`I-Br+Ti_oGF!6=n6O6v{HW$eX+;img}~gc-CIk31S?T9IM(k zc(YA$cfC^RIRW1C4Jkay=IP+`kDq`e?>WhB?|-=Fg!;vY#~Ets*nrkN4AK%T<|HCd zc7T3z1q3?LB?+L=ch5BWgo_vCy9~>!A1Ap_Syw$E5{EVuYo3XX!{zBmo)z_>>2{UK z+Bsd3y_KC5FZde4sDP4>fxiEnj+#~T{g>kfR9oq9;qJx(u91|y06c!DzeXjl;Ww8B z?;l|_B&^DFOHwbVx($uYVWy`K<~1hd92t&aq7D}IRI^YZ?7AtHCVE5r2Jn z)QulE zR@}B>V;b3yAN6SMt==g!xQAuM%$i%DO?H%$qDx?5wwscef4Im>?2V?*_+%)y+r4>a z2zjT|I_yK8;{o(XvBfYIrhuP8gyhOio^k`HB0t%x8!>yE^s8s^WopXe&3t!EzUTNH z1p3XsGToYhRN4>~W-3R`HK_cX10Scx)~2y<>RP_#UXnOF4EJ~yko5OzDv*LaJZgNt zs)HAsi^t`(LHzn|#^Vro%D@LqW(q6Re?N~p)7DdVPOGh+DI|=tXT7%Hn02Km}YVkfjM?2v#pZ3IM@#Gco-{4^WN=~kv(;J;{ys$*3Fw|+T^Cc#AbJr@4 zqG*}OD*q%{VyUVgi#Z&ECEz0Q?zaijzUpClx^pb%qu)Gu!ftt&u^}`0@O$J{aP<;{ z^ANV;TKIby!pU`$C-Mt(Pp%(6*5=LIC$yfVM=zMnLMdN3fB6RYMZKCDw~tDoc{TMd zswX~)Taww<5L7UR4GP{vc`$6#+6gM;Wc6BNzj^gAD~vFYkk2Wv9g>Fd%FD-|NfMs* zQ#pWPrZm|Icr?zDkNy@e_ zXuH()&KO^jB0pi2`4u@(gg+!?Bl9VF&3<+r`S;8r|H;{|vlne(h*w>#40VfLG~X|1&lU_Jwkm8NzvEByWqG zQCewTtM6&Eg5M>Ii;AR+dYDB;p<4Rp*d0t&=cte` zvd9x^90D9K_-+*o*#B|&Vilf)JMjFzM(^y?b_+FhItR$D1pMcamI|zTEGR0mq#v)B z2KFwF0B?bgjVO+1l&*D{yCL}E%v^nBti8JA*_Y$w?~SXAU+YJze)7`Rt+S$nvIr#J zEwlER6)3gov?ToqV}(nK47WXYc{P_SRz5ik-}k#-9KwyIkIva3TWVZI1Udi(0VzxF zMH{{iwkO#OUd8ewL_MQyrqeV@CoO+6$zz@7Xt6{CSl@0`L&7o*34crxiSC5 zU^CH>$D0(0ER&KW>Pe>-h+f-3Wj2Kjn@N9r{qpp#W@p>D%EF@KhpMbpT^%m<9Cxe? zjIyy;5SDKLFIT?Y_e8R_d%|h!Y&mKo9#vXa;+t>(M{#(C3TLw>3B5x8@D0k{_M7;o z67ZNXx-f}aMX=H{v@Y2*njtZA41yc){pukY?(>}CY($-B*r$wXudjh+;iPek@{s_2WLBJg2Ss?5KA$jY+H4yKM6Zny1FKpj+dZ2`CE@g z3O4M}FnYk>rGxDi8#?rp*DFjpyQOxO!j0f_kXE|;JXH>RWd*OjAU*d=6fL}=`Fu}o zhpdp>&c$f)tcg27#@MRamup-@UEIj}0WomzTQCz}8)VoL`7>1?sBaK)lj85s2|^ z-kqYU6Q4Z+QG2bda`qFY{L_?cUa^ddiE9!`6&r%a2&S9}%h8lZfcVlw?321N^bJbc-Hfn8>>#@OJk<@M0e@wH99liFY3QQ3T zMAtWZeLUzx904`c-`vp*(GmOl9=et&sg%4F*I#xmObDZZo{H%hUaWv`Qv9Dzzc+@F z_fwAhT@Bs$`3(D>`gLv?^4Ztt?1kDr+B!wI@2e+|r1PxLe}f=*u=e<|PLI5sK94(+ zn_LJh<)@d!ZqPf%k>JKry6n}tm9EDdY?84ee3ADjRqq-6pWNA@hH7PF*jU}1PA3Aa z@mHam1=}5nhSpJU5F!{U&F@VzmDZyxY?blA=_kQ^)7OR*6a2g8NtbaY)DdUFgm`@D zXbm1MP&AUFo7m)tn9$zqQlJwKi02J~ctjo$fbK$8R}9u0&3y;4=zV&Tr@1OS4}XmG zBFzt2;k(@z?V4&rVMB;?K;!$+d-vHBREJckC~wXGqQic3WDb+s{uSt$w!G|v#XWrh z^1L+ndqc-_Aga&**0|;|6XxtCr!!@XE_Y%(^}TLN{#)xw!%QM?TIQrRqED7dg5Dy3 zSDApJV!~VqwawU!8z#UKh9&mg+4f9iZKs@; zkVw_HLGBT$_yl~l+cG>OIY+`3AL{tDZx64u{_q{VJA=Jo;%X}-0`(8zPzEPa@6+hX z+_hOS)v0XD5eLpKg1Y@tNdfKCayi}6M*ko^c%u1oL4L$b{8qP3AZHcdh}Kw3e@NlO zmwbyx#^1x_Zk!b4HqZ`ukiKPNA`%_*=dU045;FdNQZ^T)Lk-4V9FUs59v{7~N=2?e zWbe;fATq(2!?w7=%zJKM^=ZDTPNXuHVGM@zz1SP!{t{ZrLBL??P~-PIgIC3WjuDh0 zqk}{lwYDwp$zeVouRH{%@hOG`$qK$;p%N7pd8cw8us?i>1ep$f#Z2ON_9*9$Gki;c zp7DyDAOYpSFe6~2F7X+GPxMS8daH$vo31W@oG=T?XG;1+rV6=-BAk%V?&}F8SH0U@ zMeG#}8%N&}M!v{`OVvSmkZhg63zqRXi=tZ1<4J+88s&)}a(gG9>m+a4Rh}!{Gs`fR zH=Zm?#(9uMoZos2kY3iXj;AWMA1S#l=Na>;#x2m9k-ZRCG_>g>qv38a#x*JlwMka2 zdmbUhkx@h1+B3PZ@FO_7&IHlpy@MYRF^8X66Xg9Rw8WDbWSGw59G`Wm`^O}g8Kf5! z1T)ufpt!!F>E>QM<8clI7gzD~iH=xQi9OanNHi~IvJ!+F41+o5RX7$s{Nu) zI(^^U$U3g*JI8%rV%#!+%6t=^;^D5$yMQw7!tW}aQ&S0sUoVlSN|;t}TWaCcONt9# z@Oa~r{eySAa26kL2Bbz`htZh%UnFEx7#h@WtRiQ!A7(S$=k-h9n=P3eD?-*gf$-UE7=zOnG!vN0O z(lbex)+tQjWM648hu#|6Pyo51Q&8oNm2TU&3f}d{2e{^tuUExMV+V{=!oem)7wLcM znyvXu8$BDfN~jh#Z;&qEAH^hbHu|}jh?os@nF&qdY+Ri_20cCqoCsVjksHIzWu9sf zG;iu@oqgeC`uleyqy2Olg;tet0eBV%JwL8qSx)^FZG2DHam3)fmkF1gr7lwJm-+k| ze!6CD+}LXyU`ni}aiN;B(TJLa0?>Ocik@b|$Gi~6Aqy}DI>Ti0irHOiP7G>2O56g2 zh2yXiJjqp%PVMo1AE&1zX|_p~tvL~+o(z|-_$1bGEx|)Tp( zN}&%4Y}C^M@A>A4{;TA*Q0nay8{cH6tmA%Ru0_*1!df>-c@t*UAa~sa?doMO?+lHv zHwV=p)_T15NB=F8+WnBz?9PZQh%6acQ=`rH_8s|F!gu7PQA#>mAN&v ze7wIF&^7dq^47dac$XKz4 zQAWB|%EY6f`M%P(-N<8`JK4nY8_K!~CRpSqI@M}L59sd-CLqX<*}SqOli;{#?ZC(FcbBs>6Soa3_Vvw_@M|$i?-(+n zp;3nNtI!`eT(_6IK6FpyKd-)B7k|BAQsDg8NMGWx`?E!u96I3&>I1J-3M8GxQ+&?}LBwM?f?31i)95E?%NW#)t)3&2Ur&qR(@dl!Lj{+1gIGp#l-40wWy*-e; zN}(UHw?ABf->!?Pz&7cbDEwhZML$cm#M(+i{Wg(d??IPqDZGxc;^oA!7(EJ zkyPHT<;8DAF+VRpgr+<2uS;CouRl}r7xB!Ix*k#%*R*QYQM#Usvpk=P{Mg%+vpjn7 za0gA#kqtYHnneACTbs#5V`ow7`gQ$#$97nfzU!EvO6!1Z=jFKAL^;>e#ULiZ3%WpB z|C`Mm<$rC8y=yWU!-wgwA|Z%a}2SkcDiVGKg+ojhIp!41x&{lU*#O;zJ46SGL zBHS3TZ-|(9Owl!=&YSb1xGi%X|D=YZv%Fm=?g@^DnKltmf&Y@dWJl#&NP+HAI|Odd zFpvXJUKT*d7_BV_u>~OzT6nnoc8U5w+y^H@vwd>)^@+>DzWXX8 zb9d8@9OFy^^SQ(0CS8xG!c<-&OR0eL=}hgBly?nC+Z{~)*ZkSM!UT!cl`ptrD}Fw% z0c;Z8x)A?_2~ljm9#o*4C-M5+Sy`&@&SF;=-;VB#LL-i@4=d;$aRBwXS|hBwe$^=b%0JKo;; z-WO=0vjNszMg;EHR;}Q3>eF9}BtIUUB*E;{Z?~So0x+g^|$Y28GoxG&2{LnC& z;w{byj%-r0iHjdNv!`l=Q|4Cp0l?$m)_igJxbHG)+G<}elAZYq=qvx3-f+}5ozwMp zzT4Y-R3j|aw8z3Ym&A$yv2UI?7U0;T2{H`zs{nz&>`hJ_+v&vS>zFK#vu~6K#edfc zm5X6>D`W1Q4V%`V(xMK7;jYsfXN6{GxdvsqR^?x!pxKkPNqk{u2f}DN|KpO`j6TRC zmzV=PXcm!gkVU0>;(dMHdYW(vX7q6l#6P#~3MN=yv2TU#fupY@HwjDDQPZ}g?vKrL z;&;fAl55oQl8QLaxl=M}I<2~w9`EAYxkz<;92{6oRPJr;LFsgU@Q0rUyNdGptlr-p z96)yu7Ze5D_bD1rEAAxKXNEzxzX#Y0s#!?@6L^ius+y^`fk9 zSQ&G6g7`_#j>Nuc1U=pPfI?je4GK_Txz_NU*SAX5R8tDjkY6KG zpfRNjKssNhuOsDECnVs8sFP=*O5f-rQP)^$iQf0P`YmoC?<#6CO4QHl(?*!Q>GY%y znK7Q2wEyP7vkqNjYP*IN+k|fKi%kU*_ubN1`YR3fcq4;!Rnng^k~LejC=tec!`aX8 z{SBMR>~_%3-USIhx#mpm zjqwav;>tp6tZ}3Y&izn=c?nak6o-E3Y#3*&NT!wXXt z4SS9MP4@Ajc%!m9a$5{r=3e32`j1``YxxSol(x)@2czWOQif$$>Vhi4*QQ6Z63a!0``qj=w#4qrq2^>5u6yeKCDGLvt5XWY zu0G@o%XTq@@Q5ACPJzq3?5r|W94_{0x{27hJu~}d&bq>`_lzSKGJ%EFIG~4fk3F97 z=k=YREx~1HaE$HK^+a70E00xEgWBA3=`i{klT+eZow$W=x~j(_ra0?D@nE{w>#(fY zJD7)H8%f6n^IhAgTN5>7jS;c6EZ|o_jAA_TOE>8y-wE}>@Go}!RdD*U? z?Zg^~`m{rixKpdNPOtd|I4)aED{IdGGckd08fvP~+~0RI+Y&AQe2-XZ@DA22l?3%# z>%C+!`u)Psz6#v37kGco$2d<@36IM1evrGQ*7ts~d;17x;61MNxhN((CD7rQMO;bx zrqWt!O9RyDh3t!?p$%fE=R>~i9H6$!*ib9Fv&&c}7gOf?ycqHBn4V^>pkB#8!IJGU zOAhIryJE;Et}_*$O5K|hr646@bYU4jv(EGwf;)qD%u8N7UAy*cmKmHA)RN@^9P=_k zmSrnT)svG~H3Jzgy0hS@rD(bUYRFaKJ$(9^WYdS5gLb%-b%)Qf#zhk7`)>W9huTX- zHmf&-qO|1&*xwgwswQ%(@~xQT>gva60zGmck{0NB3gn)KG-gZeNN1Y+J2dekZ+}F% zzxbW4;Q#QcwyWi6u>0NG6@@&_uY**pllyFt$pe;Dz*@F-Tm79XHI3hn)tF@Wki9-$B1DA7K+!{1R9a_547Or2 zgHh}Ry;YF9FUT74hqX$j-Cr+hz8w9cgurQk+^qrGtpEqvGFqVx3CMjnZ`Z!{$G1v; zj~e%60ib?+(2365Ec)a`(Vm@lO^|!;(mhG5^*FB17&_=kQ_A46l(6YrSv2xi9 zI-Ri|Ha|U8=d6r?2}#I8{2xR{KMYFkI5&6a#g!uNo~907QM~y0;MLWf2Ka@{avi_R zL_q4VRq2wAk<`r)L9O*c(tbIqxoZ<67?sVyLon+^MEl0bydAK;5IEmmR$2p`eRqE~ z6Jn)w8~ZzGLAj!NFD-<(E}qN;8V~hmmQU(#D@}(1!q*%p{C0Lt5$= zj1A~pHs`dLYT9U+yat7wzCQH3%s1$adN~m;4%(|^krboWvjo0I+O_GKX>jheGUuhRR6aQqu24T*55%#_z-W^L>*u;}znm^32N z_usfOdp|p|hG22IZ?qMEtuI<1k(BSA*YVk8(Q)X0?M>3*obY&hSE4I;bH;N) z4teCQ$8}%Cq6>dGm@X3>A_4YUToCmC%xAli4@H7v|BBkyIfi4AvX6u`)YtO9YNcNs ze_{8|5p3w%MGu^OCQzpIpq7-V<_}u|rEGT~MQeXcHvO3=?nI3Tz8R@A&Z59HNJ%F* zLJI$ezu*&I(D2>;gx+|mIocCJv`$)} z>)ynSIgYr+i@jY0xuIFK*zI!!^2@?RL>xgB-($z-b$RoauCbq{xwyf2?M-BN`;_ukTo0T8m?X?&C1 zN>RQ~b|a7J_2TqopCNpGWmGnXX-GH{`|j*8VC@aNPn2*r*pr}Wo^E&50?r+O9V;HL zeOI%Ktz_1RqZ~%l)F`!X21Mz;3nSd~7uia`9P^nfIv(~6_Uatzq20Couoh~zft#RS zufX17rC#E})*5?>@g+G7_V5Q8k$1-NN^v)-Y#v^}*&>wcYGe6R?3v|zXPFk4=c3jy zA{>*O17)U7Zd>x7tNdDS|6r+(&z$1%JHOf5EyZ#%dc0XR&1zADdjv^e?gB?aMQj9s z0((Uq|F+-&%YrcR8>x*z^oQ*BqbF6lipx>Oe{&gRe&jPlB0r#^5s@;{vdK%2ie{p1 zQ%;fuNL*5$^ZY;~B_(BIC)WP+O7vIC^YE1Ck8o}Ev8&M+cS&np3nZ zaIKyLDr2a`hBAIRp>`UQSX*EA0_c z59^Oqfey3@arvWpkaq55;PpsJutoo;$~eX-xH0v;e|KvX9}$7E=C|;3^-C6*fN!=k z40#|dta(53T|wlfb<-c}#a+LV-B5+nmaD2HTx+-a!~M{qaK3SNbl~hy(M$4hG0HS^``l?&Z3 z4T z)m}R5#h_I5P{=GIduc4^2=u^Z(W$@)5?V}2(LN}S6`r!?Nn4>e-m&3O&0Z*_Vy^+GH^2FJ?(BFw%&omBWXpr}(5o@c@Or5R3GmL=WP>KY^}$3dES;(BUd>CF z2d&Kog4J=p(#i&|$BRt5ONpsCo{PV2tmsq>OQGx*-5~KTffmQyuN(xkU8nnyPqA7x)m&o_q5+H0lYeV}+p4&|pNJO{0>y^Ui z&LU*B$B#qjEJWu-F2b<7K#d~+rN_3%*N&I++3vqit>7%2A ziUcZ++61=n&I230hCXv&Rq=!KL+_hm{bj26KJMk>gACa~VF2Shtr}`zE6sY;?)?Ki z(A1cf0#5V8of~Z~SV!7t#$8vrHBGSi(dOIus(NqJMTDFR^1;q;F6>0XsePxAWtc0Y!_?cv~4>PuaE9 zr53NiHb6=1LF4(kxXmywQOO;4=2*Mu+OhAUBmVX{5x3;`7T7zZ`8&t#^zG|)b~D|eZ&)S)WHdpegarP?9rKh@87mzZ%Ld3{`-IToWd~4)gz8;G z#mBqxc0k%J91zpHO|Qknr;`P+C@%TbHU1d)h;TRdk-L&^ndgkT-krN;09}(5ghL7q4^MZc0#JWxZspLvT+iQp&kcq` zco#GWQmwMy+HU_yFzy_>8Vr#LT{laq7@+eVjilu72SgC289r&73iTU5geTgCWHpD;>hv1A zou+!d?RJqG28CF02!h$s)B~-qk1!mKcB_0Jt-PetY*@NyR^R%y=(ks^xM&~AkS-wG z)DhCvRuhH4MZ!a#?%IRVr_kJUng&=qzsu~4=f-a(sg-bYO88#|hWK%Q%yrEPG#=Xv zX!(R=LWyW7F=U1k(^%}MkP@&^0&M`-Akg>+v2vS8V;cjeFYYav5n z&7uI@9!fhe#QY-s1c7Nj7wX}G_mUN%0HymLW5boT*6S5^=wz{xvtb4^C;vhpa45I# z1(dzhDddk7dpqO8@m2gDS+>=;u^Z=@8nEU@W_&u-nNq#zoGaavnWQ)P*04A>XF$>C zf<0WWfXXZ9<(SG{7NNZ`?P=!Qm}Q~8$C^^ksf!P*%4B~?SjhzU1T&w(7jaaji(!MR zf^_*I=U-?Cy<3 zA=9zwYejF1s62arm(}^nT57lT2Eq+xNFhZgAD7!73=sn@LN)8vLOO+NdGl}27L*l@ zZfRFMQ(6D&3R3KjNI~VUaEI4?ppohJNgPI~cI`nAL?;0)IX8CasJ!r&ZakIhw05Py zJ)ywKkPiSzk_`;B?I&${N%tr{h1vif&bJ?GC&)JuLP{4ldf*B>Ed4qw;f|0n7D4d# zPUk_l6`?Bh#iZ%be8x8Tkp#7OlF|Rl(_Pag}GZQYB7f>Wb~> z#}M``=R=$BBJ~ivj`SKwFm|#-ez%}he6H%F?7aD%r~1u!`G!yBe4EzWweguFy7eP+ zP8^R(UR>@d6V&i@iPy2(1Hp6Gu4_pY%J~UaZ?_PNd-E@mbJJ6)7mDk%?~gnmV5V^_ zHY5JI&Vqy5mDT->q3wN9pmHJ`g85hs7Jn)>$Qn&ENo{s_eKNK)lu+<<{lnF>K|IYX zmj~T2_p^umPe3(yiH~X1f33Q@3;dh(sq%S7Sy)S-3)pLJnEmZA=a0#mb$tpl(~Ni; z^^VRqlCagKSI489d?k__Qqf z`p80;Y;D;D_e;*6J7Z?-iu6ARlCeXV#V=aDU!Uqwm^$=(e^z%A_;W$-QDY+=b!|P^ zjAv@62aT0IUK}a3jSkoSI-|-qSb=83)-eSzw!#rO4ev(tf=b~SL-yOp+9eHJe&`k3 ztQVG~C+3+ic&W2bpVcVqh*H#4ByspK7opLE5>h_D3@zU6=U*7fq8f7HD0tCcanfW1 zJ*B-@au_Qz&80eX=3>N|E7OVmhCS}+VqT7x?F$Mgo)L+;cMe);raGJ2pB4l&8p#c8 z7oLvK->Qgj%SXfW5-4DF7zAN$3NFI1#`W)!csE+k|5M}Wp(9Fz+5iJ1x-#Z_|B9jG z1S%(p{t}uPqit)q`Ga$k#-$*$T;c9b3TyL|o{9@@N>osXhG2%%QLGLG{i&KMK1Z_D zmNG62IFKRu=E4g>hO_!p|~PS{)x(=x3&dB@GC| zc*D$9vk>~v@kR)n^ZFx69KT+Pz^2)OwUjed@&b5=bJws=@m-r3#(Vg%()O@I+p0zv z%TCO<@pHTj4*E&*#fjw9EBcRF{S00J?vVP7h7MegZIhR`ijF_Ufn_>CURCO)s82|` zN`ll9r)eUAU7r&bMteG1jr?~Y+87B;yH*)c{X-q4hvrMs$-7s07_tz0guoxOtn~mI z_6!df?%$twk}kbho)HE2tEX^;hJfxLsQ0)Da1ZlbI=kpd;0Wlq=HWlOB`d5KqXem% zuqCK$ylZ962X__(xY<;{PMq~3J1n*Wk9t-R-emy(!F_Bf-ctq+jh(_p_{>?7r_imn zcIy273rrT?#Ma!Lelbz)Y8Q=^t7dVZ^1dIVg^RzkK%Fbkx$XV#Z@(nJjZ(GjSTBJN zLuU%#6rsJ20+g7X=O0W~pPn9unZK98r|pnY@+!YNn8Q1YP`$U)Tq*KY4xt9NbL3=< z_uXEJ6Ggij#}dx_uSDbO8oE@9V%}S%Hkc4D&_|u^*GIbw;sX5$y{Z9V5{=nOLR zB4-$Wk~3wJCm;27*MSpW^fC2ECLURVst$CzAi`O~&Z3hK51X9ixbJ-`ufN}_SZv22 zh6Of>Q^93ey#i$W!D+#CVirSq-Iy+{)BU^;l5H9NGl(YY=@nlIZeGSERu-o4AUq;y zq)okpu!o0hi)0@b-YHk2Hqek{{=hT3^VMs2ni@@xbW%%W!|;PI@_?KvEJ9J&P@IrX z9S3p6-Y%3fMTz;LWhFa{r6qe(16LHsOt&TQ0#04Glv~Ra^ipwn;ibXc16!8W+j5Z| zfgC}yDUwbSyk?H7B6J&4E4fCNk@KP}`~|8xwjQ2+3(b^wAG|dyYbi*w4j#kjY=TadRI!Q#R`$bqZcLn+tWxakZ97W1niy3aA{ zPLkJ0yIdZMHBF>#A|UWkJMPe~`OVIyDo*lkwVbNvhO!30ZKP-iuX-aF?b&BQq=|Y@ zs^rant(o{Gw&AIe&9j)-JlzH-kz>&V4HtqRw#%tsqjO7Y+x%#ma(hLqnCGSFb0T*; zdz3Yy>zrOAh*c*u7QQt~j;38T`?mehx$JLDd7pRIk#K^ajmG#uA92Uk`VZ!#!8RiD zI`$Jm%6iA9i+bl;!3?3V9;#P#C^FjFwaTi#>J@y(Ao@+2&5itg;oUKTl4GIXxq4H% zdl|LrLb8vSe^%9n9F{ry^WW1THupEB$PF`?CGmF%y~RJ|+XvE9Q0@aD8pnYcif?W=QhASpqC zGTG!Mqjz(oLG(l-I~V1zJM4buK-10%>P2-oW$xe4qSxUpi!pn{H^o+pR@zumFs}A9 zd)Jv0c!cxh&F`5}6l@kK!ca21Ku={o?Jlt=m+xO>zS7CR{jQcU%ftpLEjw4kbDWo7 zHvjC!4&YS#N_>(AN1dH@i*9bW=~~Up1l_UDl#;tsMEh@KvY}T}GvfNOmd~GOEW_hM z=?&K_`J((hI_su#a-M+&*n(lC!J=AhHFv{6Yo-r3ga3VglTgfczPO-+iG7`T52?jd zEuPlA&P%k6B^)3Byrw1DPhA(O zlgF!}CVe`lh7tP5K87&2a)5(s09D0~wTbSbpFfws(7%+NSPXQ0_dd{JjVjW_A@|Jz zWq_FKNREGmjpX|7zF#bOiqu*}al3<{%2~#AIm~raqP#u#?xQUTc=ACa#^~3UgTBXr z+!@CM`uZR)77>P@`tQ9NJ#L(QW+3uOC<6{$MCyyu#N{2m&-?NNFIZ1?J0RF?m?<<8J z2332(Q!3)=!8lV7O6)FL6;S>pwf9i>=bvxa8Oc6}G!>Ks)-7AP_C>e+w%I8p3WMos zGYj(~M*gWLzXS>Y2_+7%c{nft825Yd{usdCVV4^x`%Ix^ zG;c50P;mTHk>%ItNw~_S+zy~~hV_R0L`RVI9e%=3m!in}p52`Ho8u2tls$W@2KMG% zy`_Yjd)-LPf)1Lh_52|JeM+?;<6+%qjV#4-fCNKmMTK@P=8i1d?Ls45={fp4mA1|` z*GI--9J*e$xQj`|xk|+#pCcE91kc;1=5iqg49cSAs?YX0#gKK>T_ZZILBq}7p$@;* zmArA7cB-{hNfs(=jvGHw(4)O6?DhPkX|nQoKEQb+Y4lrk-cm2mAgX?}zT+xA$Ft^Y z7q0U4co8Mz1o_uLj_8zMl_Cnw7Wt!y|Qm1?sNjE~PVh=>h_b#$LmtSz_zB0SAI2cc}~& zJZ9jHl~6^Bl@-Ttbsv32ZN0R7!b#E1n)`dqD>Vl^I?(7P`$M6GXVU!##aKG4|r#>!gU5o(_+MisWt~e@0UFj@bH@%C`AhiP&Mzp?qkmRrlx+}`wKD4oLeVH5mN@vWFS;w@w#a@vY}Xd;Imuop0O05kWe}K-E_Q7K$jd;^flu0@2XIp=i}+4 z)vtkc$#yrV*X5`P_d=_QtPTp{HMazsnWNsb&7QuIOcM&x#iw~;#phY^Z8uvHW}ONd z{eV_GVnS5SfoKN!M{kzCgV-H!ySx;?L+#N0?fhjib%1`H$~DKBRT$zKtf&O z$EQN#Bw{&8&mEa;dzDK0gpV74sWkd~s^()8rvL-|r&Q42I9^4QKIQwHsfA{>JF}Xr zh4cAgE=(JvA9@7Vl+t~0IWx85=~r|GXdHO3>1IvEkdB%s#hlRR!SUT^YLJ*0G29Zq zQ-LvO%yguj0)OJqs@)MP!DaVwNUl=T#$f!JA#-QP=k|sSNQpxxm6>ACR+QHny&EZ> z5l7|-gvrY>|2%8nQ`7Y1z6miSRrBJ&5U{_PBlxBpP|KDWTolx^8*GM0M-@dqm&ih? zl4Z&vYd*u5#4o2p>$z%C=C3~X_CU*{AYy~=&_=ad;H{*R156u;u%)~u^K_U{6nw#qwrcJk{=jOfbFvE(uzLaeSBo32V(wy&_L83Fy&VJdUBWBv2D)_ioR-A_b6$-N3`%U8I zgE=*?hA{ZR!ZzZH+y&(di@-q7HSZ=1AD2Pr7Wzi~gU_33(0p;yZE>J6o8jf!A8pbU z?sw9;p3G$H1Dx1z{i-10r1ISpwrF^3O%$|$R7RX&(Q9V!>t385LfD$Ux@Q3G^fNDX zNLEQCv+jTEH{y%EZ}U0zYjP`zH%n35tf!lKcSfg#F4--g@mO<4(Szi>{qY-#5E!@D zQxI3eaa|>RF^v=ZoVq6`G!%=4Zy76t+0^?r;HHId*{AiR&1#SJ=%6o!r~H)c-;6hB z1ylUMw;PxCO+Q;birx!;^C6kI(d~WYeBKtKX^>^>AWn7Z zd8Z0TJf?0-o7#QoHhMs9MClK3eL~k_wfR`=YuSGSUOX8@g{}vTPy23G97V~t_GM%K z6cET~ZMDks?uT?ab*(aeFc)>8qZ0RW8jOtrqaReO2VGH_>0M42NU@%G|A_{3Za27WUW_5DU{h*~~B6e(3ZUD&GPVjKly?UKXA&?>aQ8#f6obw6=TQ zw{Po5j}h4_l&x!cJ9ILV^|o5kH7lH}>uxEzLQph%hE=!^1^%t;`?uEoV_d&Pn~$X- z_J;2~4P9$QTQ3ppxaa2o_m=!;Hux%|-@fef1X29=_KYQ`+F0$xPlfN$_P37g3#$l> zSvKGGEf-u?7ebU+^N)WhGPqzcb^Ii17B30;WX-V4g_Vj99|Q7UoKOlo`RRH zN7vMZlq}9_lfxTIPaiTgb-iG)sb7(TU#9X8)Sc@7p19(-6YhzR|MUc6(LDgw1FjE> z6nC&6)0OTvH5d6)KAp~BTvX#kSuAGXod(R>%Rl?8PR`RZOHTYyO)Z4gcI4(oL#kGxH;Aw}L_yL_tjQ>jOMv>Z~hg^^1_ zuxoa8v&CiVO|VZ&9Z)yp0zkFi@Z37hNYpqvt?O@e06UTDN%f5W?rL6x(?+K=Tgd7=HD*6L&^{`?0Z>C$dcw99JX<;Se!lcV7c(9=>00$6Zq>- z8Q`>DmoN?+=@Sh&mI^Yy@X3BRd8BdiQ$C@a1ZD)zp9hp2oO3^?Wven=d3+f~?6htHL zod!9h9n6kO+UUsVeBY-Xxv=+vZ>ckOrDVGAprG|g3f}kSoGW!Uj4xDix#dy>Y~qiv z$Fw*?rwbY-eb&m1+uwB)lEKPK6{MsXd(f$ff%ui zo$da@v~ENmJ@#tql)Y3%Vu%$aN^y60clQ<#!J$ZScY^E5xA$6OpC6DhGV&&Qo_WtXue%2!vYas_ zDYZ*u=GS%^drUo|SaWPW;CFGqVxZ2K`s+shxWTbo;w9?Yn#}K07?Epdc+?C$Z@S)l zYZOe$a&V{^A!8Pi)glFu45DCf&r^x)3`WiWh_u_E%3V>R4knm-G}k^NjOG=%WF^pc z<$1flAbr!a_Adoug4SRI6s5x(u6SS_jCTt89{yVqlCTi4jvi&4C)^kLz33MQSJ51r zE`*~FJ2&caW#(iFdq(u(N981MIu7u!Z33FVGk|cH>-Cg zHQHc6L|AFJnPtLf7o5}Hwg4e!wAM$7saF^(D=xFD#r`mZ;?1vWCDeiiNN|;Qr2{A* zQjl$%=^YF2g5C!j;sDN4*|(JTS+gQ*44#t`o$69h;+`S{y>U1ElovNr$`Etn9J--l z2P(=Nniu(|2cyCe+R(MG=hIhN&;g+gzjUetaV3+OvL0#qd-XwZ^zY`?Ud*TfM-)Fb z;))9X{xAR;ilZX00xEIj5{J#lNhPaMX*9eQsA15!dh70okMS2Wxkq}^$lO+Bkepo2_3LTbqrZZ_dUdANtoWJhR209jWeJf(Xc3%? zUY*8kn=E+0od8qFf^*424`xb~!zn$zvf$m(VyQ^NVx=r0v?Q%BB6eZn)M%23V({>f zN$qXU>l1vfUE(&q2>n_2qr-qxTTRbPyULc7sHO{2A~(M{j0?Nw zX#SO%d`z0i(PqbJ2&P-#?QD1Y_3k;Q*S5q5d=9LY)9~W_U8JexKL&_CBgdv33qgu{ z6Pt5+nPEDc8qw{k0u3lPamC~3PVSd($14~d6H)LVUBC3mv7bfgvMmgXKWC!&{~70t zYNYEO0IDA7?TxzFrpJHh+M*B7$BaHF+l@VLdwe%hjvm#Py^&q2f)nY32>J@5A~jKm zZh)=dU*0)iA$qhBV|BzrR3gz~$F-4U$tsB3-UNlevsZ9A>nx~+KX(oG-=kJ5jgjoW-lf%~w^^Y7Lg3h}PU^Gf9ImNslWDJ6?N z&u{gtF#lXhGtM^3I+rvbXx+iAQO)&A?i;4O+G-4}#O56;e!jfya-Z5+xHlJ>^j6S! zf?Yp3+-656mVM0pV&(5)?crOn^CzJxM_)KC*=My^PrY%WnpRM3LOE^bTJaOO@i}wa za&)0+OyW;{o}>eTX*V27+`= z!(MZ)undW==ICEyYLu`#bIZkhpZ~ldG;)<|uF$jsW)8$YN~cuR7itkb&~rMHf<<)f z%=QFcPF;FC&Ci^sYP81qTZM|=wJs7g{Rr!=c}M!3{YaT?!;|Xp$%8t=H%Zx`nCS?b zx$ApPh3muZvMqbzG^7QA+qM*yeLX`#OG@Xf&NAn{atmf_-G12E2xBSA&*BOk>hePz z8TE$#=qv{fRA6mR!eaLyu2Jc+KWzPR)U>bYde9_qi*3miCO|{cC*hW2a|-D~Gq>UN zS8+L6we^eUR>Sr3{k7BecU5)x!G~~ji|obubCB^TGQLt8w|YR0X2X!aunR*ROs3qEvRi1wCg`k!x&8y)&>0@zw6~S=y8p%!>G(ha|CnsD}E(b{Us? zH+*~q@scaRy`3|-C}jNS^RfLW8!vNi%UjXY9If4;r2%_j6_^^(km>e@eMcA<=*Y31 z+RF6ZUk+9E0^^6TMZ`svrs2gu7-6dOnR1om>(!amo{OM4cC+2OqDE)-uL$ zbp5lPYCIaqO_VLEsiq^Ne6Z1<`jodo6%@rQUw_$aP6bnam@K(i>ibTa}O>ZDpbIR%8YEtwv~5h z!1%>f)Fy*x}M}`d)*m|J0}*a^YjSsB$1jtmJAN-cI!8e0xqAn zESH-N!QXm?cTVHA3IIFNA|7N;&56n4%7$ zUW2Eb$cLrwN53m~W0(}kng`Dk=h3LoC03M565{YfR|#Dpm4WgrK}_KihT8Onj7}i| zZLa~__?6D2y4*?3wBsrvh-K5HXjgY)>Va_KeCX*LX#c_(XW@^e_sDw^*1dN>=i==n zzI@<5Ain+6dRYH?cZxnN@`8&Gw{$FhOkKgt0)&F&;-2tqs)cVDAF)HlL(P9pMOeh^`yAl5k38y z!&>e3udtfSNkP9gw^i|0#QFgTXCCbfZJB3|`6DRnNF9E^C=##sEylL8B;$?hMJ^s5{U0|( zcg3xMVTSFO&_{%>1sK z*bZ^)(k7&{rulIPNW_CD?~*HTrLA2i~_Ge?1)r8aA|OW2{XFXI|s81jU3J75hj zT6?KzyX1tyaQ@YIgSt`^5=&>5DUIt@)7zjHX`!gl=HfVNz@mTG;E`0~v^rsgC}6YOIj3@Tw-JCd8vH5x+?Or&Ef3N<@sXiR9!bizH7&2*x;c* zT_^3e3Wms-81Su#oggy^cB&oHtF_NQtY-N~icqICL^*W|w5wy>8G2fL%p5Om&DE@T;!0uXVw6mF@ za4K(mE@oDwA>Kqb>UzhB>b=r&&@O*IJv(_jEgqu=f4<<(^UI7)#Ziq zpjKtcwJ=rY`FenXNxHCULiS8{zy&XClexeQD&MME9cZlu&20 z&`X+D^)G%z!O#NO82$x%f~Bls&#Q64Z1Y}KO54U#bCgdKRRANDY`^-pyQc<) zIi{q*S16j;N;aUXjM06(4f-{pZmO%E@KP&2-3SAzVh#eyTzhHRQ0=r6T8X zX{I%lYfJ~DO4F!@1mUn9?=3eR&!PT|oKee>Xm{%AER~y*t>qdLvf7w8E8M4kiw<9xKemrr)%g;zw;=Ui@*GRxe{E0HR2407|X>~ zY9;mqe9AUEa=iU6?xmbO=uoEKbnQj~fJ+M~QHxCKrR!TFxpmoyV+RP)(3TV7>f~+v zhiQ5C5MnO7Gscq4z>)G^5c`){q~PZccv&4hn#3##K(em9g_Fh~GvRjH?tc`Cp_hwn z(e>wIpWD;WHGRkK`(k3ka;z&r>EL#H3wtpczXN)NezU^JyKJ)HPzMJq@ajdo`v>TE z0)et>10!Dc^!B|kXD}5~QfL7~@%XoSV4yX@b{CPU8%BNRnhspWS?9 z|8`KsZ+A3s5X!JF%BJQK^M&_ryVW2_k2C`q< z&Q&^`wjEjIESL_5ojw1cJW6Om4LJ8v`t~Fnmtrv~x@SSkTZ+qW-jen{nhsD|)T7r4(sfY4J0#?Y^%Gih+Ewm=VVc#@sL@BO` z0?`6`SkljC69#y8`e_*UKpjs@8ML00KW4y+3x?L-R9~Koh_Y!0qiTAdSg6G})5N>) z{rvr2{AF&Jf{a3hqr$6EfAE-feE*{C1N6QUU(D{e=>37vzXdPej3sM=(|N&gf`eGz z)}{dPf>R+G5$JQwY(!*^c%(6Ii}|(;EE_UIowUw=^)@(O8B-SltlSIa0((yLLrE>+ zAw84CEfL4T05&^}^km^=1i1gDH?jHf0n-vIcrNguLd_|Js zUtyzCux+)#!ixJD&2EZefTZ3rXR*V{;qJ%al?~qUWZJ(7JJ$1!2v{VfAOD;Q!5;QK z!KA^Hn`T^5Bo060aG1~AKM;Vh@-@*Kd{)W}`}E zx5%^4!ld&NE#RwtC<_7p`G|qy(sjEubKDi~Mdi~!&+@`3OqN>gsRq3}9b>#a^W){x zktN44A;5vlYiSHf0j6->KI@714vXuuc?R3d?(&w=Tgx+JMY09{lA9Aidsx};rOCURb*>P$g{uyw5_ zIVs=cEJi4YwE)l-N38O<*9q8Dfd-@j8;JJ0Qe_aE%6|LFIwpTkb>oJUnTlI=P!yhT6O<%=U?J71-?I$NWLk7W<>{R{uF*S=~eXf`4Hm~)I*?aG&fM%5uT z^U5JU@!sHrJEwF_&i*p`tm(4GI6Kw}nELX)aLJW|BD(|IBw3DMOH-TP7x-erbY&Mf z5tB+p0K3T(R}U)obS^1JSIsZx@KB~%V<5M*%F1BS*&1A09nrG*}6{noI@wxbUB3d7k4aDy0sczlGpekBf^SjLwyd3WBWf2DWnBL#w zvAd|$IJd>!Zwcmvlxvp1Q}_{zG5wKXG%49K&cDX5-iVV=kXKg6^l09`&y#|I*o7ZM z*Z*mx6X@{k-$7^2jIic)IYB|;SLaX^{qE`Rk{Q>Hhnk)wFvm#vDn%npiONhj3aYnp z%5DeEWjq>^&pzVZSgWJeNq~`e$ITvW@Qxev}qBXCnB-%N08;;UeGiTWBbB=G&)= zK7yGnN@_6bF7Zq5oS83FkfEv;9A(&KV?yA^HE5Lj3fR9!*c;YAIAk$Jrn` z9ZKRhy*$rn>lGl{Owy2PP{Rl5>BR zxo4uR>CL&1-fqwuE`P{HOKJef` zGsf62^UwBq%CX0mcre*X15VK_7DmL8dPv`3j|oP--r4QCrH>pEDN13m2P6b4?W2Gekg2 za9MrXbvi1850xiAhO)%B8=r2`qGLyH|9eEX=7$!E)X|o~C0TOzo(B}_tvTK>a<^}? z08(P-Y9_6qk!;70v|8wb{*5|3a{y`hQJf1u!X^){OTW7$jM{|I2USbmF%rwGh$dC6 z2+8cXMBXfJ!<2z$6QZ|VoU@o8Ft@sfa_|O#^OPv*7K)^S0A?KnpqNx!IGdQ(*2QQE zLKePShqxb3dw5282YDtnt*2^#A(zrSgrC^~y4iuI`Rl=};WONR3cmk=>St9cW@M5n2H zr18%jSl8up*+5Dqu{h=(yq9jkWv(76bS!{eZGJYM>q*tWZ>7yOimm&gfA*whE!*Dp zDmTwSI24NnE_%kAFAb!{pX=GDDcP@`S#mn=TR+5arRUgV;8+dpZ5<(c#Y3odC$6|z z*zoS+U3aa!fCEIyNQqFB@$=~SYNSk)G;2*9qiU(QBSZ$+MJ=DM^xSufkp8kVN4=t9 zMP45snbNR+BWjLz87^0!(DVD(_TD4@)0IM<7c2&W0`2S3f-77;Y1^286w_t@-9W`? z+BUSfnSE@~HhoW_PJ%zGHfw5$uHqd=wM@X+rGC){JCS%R&-|UCBoKoU<{OLQFB2SX z97n*`wr*$J`yqcuwQzV(jB8$9oe16!4< zZ;v}U!X6u<_jbqbrIFplYoh5)@*b@e!3$A$Wl5`|+PZ4rJbpXek&scHNB>;!!me!P z{Pd5g&#PdFUppL5?p4t-Kirieoe{N>L0ZwkDc9*HJ4GZfM}fbh}E7~8vR{}M?{P1{POD#K|(nTYl9&1 zlHrGwKLRMZ5lVbO5F46(eKh|^-q~*k7dq$Aae<+RD*ab=h-?`vu6WDk)8Gz!&kNJe z%OVl)`B3wpL8(AmMqt~ua9hWLzXZKuFY+DQElP@^^2MrhAK(P_T5Tiw&$JGy2F=8g ztSZr79jYjW^K^-Pmk=Q0eD}*+NjE&c&1@ydwB#5onrE z?#+7(__H&gU=(`Li1^KE{PS-Q@P~oQxcHARn7wYhb+<%b`^GaEZsw#Bvg(BM15W~K z47wDSo9zjb+%oWQvPnM4FOB?shZk$;3?b*XP~Z_pVAtj2$j+x1yRLut$xU*rm|KaA z*G9e32F`9qSC~w>s8zY3lNKTUTC~XlkkI^w#Mb|Q8A)Pv8Bwg01hu(a$+nO~ zUaTo6l4e=AII;bIQ5w_dg|ol7eCoWXKiT=tbe00rIL}Z8#xOg2x+uUlc`6ZB z-FjDY{2v7M&=mN7dbLqPwBN)eEq__aXjy!>WiFS(L(u8T?cM0RbM5(wrRgyHO3nGC z^K!5?R>ej_vTYb-f8}GrCt~fYn#MKx=42Ip#xEM~SifthYs}@^C7M0jMhX5%b&S3R zY*rg@v-FKRD|J8Wi5pp=bjB%$WLgd?XX2%=m}dbI6iH3g@J^B6lD=|{vq$|x7b(Fy z0eU>C!whIUTi*Q<`+sWgV+>F38Qy608OV2G>*>wfv&sL&Bn+&?f&L-<6HfE-9~)8( z%0WEe)2+HUKexHe#Vgz|l&U0sKUaDwAbFXV_2Pd$JgNcjb8d~tKEBYHZ&2Eo{RsA) ztv53Go6LPx&`-z7gL(MRck^K=)2R*H+?Lk&DERBlwy+)ZyvP2xV5ePczrS~}L$Chp zD#YnsVb>p$31ZtsNnfx@9%|E01O<#lAOC)93-;hxDbpaBAczexNr?_WHrvC@0o>%@ zpZYl5HPx5#bIzv;SeI*~)|~r>YSHv=U(N`Q!^Z`adj4R=-6j=N@StKQe?1h-aw6SU zRNn|Y)9Rz;GH0=52cHv`Y67^|`S0T3B0@Y&@sbwoP%Iw-bP8nDh*T@{Y@o{{T<=q&w!(v3U znf-i^NA|?=XE9qyz?ynxJafMMJ9Uu>^MAP+W!~93ZlxQMbi zx8G~rbTYQYmuxB1h>|u<>>iwbs$~Zr4pjpL>3{-M+{aigq=)ANhrP zoo~>C6U!*6^-eu4PL|wvcj5}hwS(C3NMGEn0)ATsrSH#dyYDbEPyzo7KuBymGGYH)zJCzr=10ZaID|$c}pz zeHL@w5p(2B(FZQVc|h_8q7~MWW%sC`5SoDX*Y_2 zbHv<@t%GNFu1F{4hN(de*`92#(@)&{{S|gRta(^Ew4_E6Pa!L>bvu98IX*8JB5Ye* z8d?#uXod3zo;dk8En)ecJG+9fTkf0sUAULCv4Jkjo=hXg^%pzStvfwWml=jvoANa@ z7)Kgp=5Z+At^A=olj#SKHR=kUCEe=FV*xjiFWr@uEG+4cU@k@$$!t#&7`} z8u0IP64UqT_-%`yEUa6;42AxyF-ka^L}pLak}$PRz)!>M_btc{8U4V#eEpC4EXX$l z=09BK)ni1&OeFjX`j=7OC9-QbPbV^~VL&*Ma^qNHbS1V#VQs36rpb2v;cXr;y~cKl!x4tNV}eZ_@bn z+c+BjwfS{z40lCmzBlVyAj?kuPV)QR4$#J#xpw&*#_6Il&Ec^Dsw=>zTWRYStI z6x;y1{&dPJBbc|bKX;rr6joRMX1Shq^e2l;gOS+EKyXU|PB(+BtlIlGT5Rrp+n%i^ zd)2&%*jPvX%v+pQ=~A=kHer(KBA<~>zQ?Hd#x!nG$*0ux1CM20Ip|b;FI0A+1IMsAC{QcKEv+cu zJxml>9e;556&~J@kjugZm2Hxg=;$B)r*akl@hbUcT^m zpIBwR)(!j(PwRHZ3_x02n4G4Ra!S%unEdv-tl^nvq*g-33&8OkV5nlIh??7=-KG6< zd4wb$dG~E!n8|Q|W)buJa4-+hvu(22+>{vl@796zsq5nA?aWrNsb5T`j^`X{2XowA z+&M7?Z#FgVf)j7WrV4s}#|fQqv9oW%121?*xzd>lyNXWfp7_iXEV<274!_l~lbq;01QzPad!7f|5{i}H+snYuyS-bSB)a6DI7L%8CNif z36oq0z9vHTw_DX72`$L6Cali?j%bqR;DPiDL}_#wnk)_gltJY3}+B+n{eOh|b`pv}i&O6lvhl}G z$=|{E(b*h52mFL-dsyK8cm}_=WQ-zy&4Y$yq$0EudJjPy*Kt^IV5sSn+B#ilf|3XT zZ=!N4gMg~ka0U}5`v7BW8(i_(18c=pG0fR|hyH9n)^TN*nA0fxQ=i-Y?m=wh>E=Wy zLs|4uPc8@a;58uDPf~Kj3GDC;xvhD>dz0~MUEQ;zq+Rl~CQtKD#XeY&+*O@NTKx_s z)Dm3FSBno9i$GYQMaGQMNtEd_on(oj)9Jt6VtB$|5Zf2UFaX)YA2U48P`k3O^7Vm? zn3R=UQ;%revRmXB-7>-A<;PSA<}p5ikHJu(XM5#v!+|E5!v6zUpit2>pII@kuXPjl;O;DZh-hB&F*mh>e?1osGE<22D8c z3_CLR+zz-8P)_v;6E!iQ>@> zH>N3pbQBK^c)AvaUeeyYvkhjtopB{1d;*)ae?8__;=`v?zj>|eUoaUIp(v*q?D zZKT{~_Z2BGe4gei9ND?dKuh8mH)}I#O|cJ_(a6>X_8{=`k0$f#3wd$?+ub>nm^IBx zF3%p8Wsaq{0=Juu#GTEI0aQ$ ziLj_#6PG>JtRDu+&-3Se|CjhDQAyU_7aWv0y~>-9sqYkj`Va)C{C0?m?(CY%rj<&@ zjd1%U|ABk0_g8u-+l<9H;a1{F?Y#aQB+AgHxv&y*<0{_;~|BE^GRFSBJB zM$pP1geF-O#}b(i$mF5{+z1Av(L|2cJgvUw+qzt7?42T^%>eg9%zd7n-DMZf(<|cI zYM{Qz(kJNhuHR+bF;1#*z)#}f?3KN@op*qr(^(eJ-rhbLh+Y^GLp+BTs}D7GrpB!l z8T1s6q&%hezntt$j5o4w>3mp8juZ1Nc^d;Mm(x=$!U~kaMUnK#=DdF`C$`~~il-Cw z9YmeuYt23$e&_YEE{p%9KmyWZ#AYY9QMiKBAMhNzF-2)H(g?IbK08e<1AB@OEMg{q zN))-V^(kocMDh{jrC4$UE4)^O(VmFcfrUaIvJ?zdE#l@8MIsMsd7Z1kzrLhH7eSHG zvE=@bzxEVG>b;UG-vYR4Pl0w1GOk98bF$mH{2h4bhi$8mZ(Yde{XDPct^pU!~Q%T)WwLA^h7pHVh0=pF)RcTI{Hczt^DGe{SsegMK4|BGcY zVZiQ3?=3aHu#g5%ljZ%ECrEid-?u9C+s~8g@5d<4Vl@(Kw71^vr@@N~P3ik_vN05v zmf(|PCAS6mBQ*e5E|#LHmw5-jYlrlOTf&&1>RpsgUv^o=OomnpHl>WyX>>vgA3)h@^()0xS@E+KL$N23$t(kf;>iaHxC!PQN}M%s zaDRa4{H(;7THh#{iXslMz`BpdKx?MdGy{PR`bz2we;2=za2lF*2ks>e<#_iwspebW zZYcyk_;wQx0*@`ZC)&d1%Bmom2M8g5#Dw2)Dcy@Fb$eaMZyK@wU1#4ZbM_LCq?RJD z$(}BejbqKdCfCcDVPP&ZNvkbU()+(trTyKrH#{ga?`jQAQz}9uqHemJw+1cXJ4=gg zzOR(TzlmBh@ILs&NFwaPQXN#kyPcpssI(?IK13N{N2i>fS*USY$)?{9x7WMo*x|cT z3)(J6nL`$4%ha&Dpi@7p;8z!m#ad$tnXS*>8^R}zGfpv8CZexTt{18)bIbQ%WJ)yc>T%IDCaJrD+Dn!2@MpmF(ZbMn zFJ!lEFPj^_n)u4=c7JL>42q8m?_zRMLY`S>Kh#(f6@R4NYK-Y$%eKs(rf4agUC8Q& z#48sMYEyz=W^cP&-bqUu_KSz*X<`tGPWoa7k&sGuOg5@etQ{q0Kuk$Lwbg!DqftQ; z^@bbMtD|#<=#0RhQ(MGc3C?)ymI|fQu1SbdWw}5>0j) zps$+avZdciL%4h!l;mvWBzZ!Ys zS~Re^t$In$?TiqSy3y817sHj>Aw;Zkzif?Vi&1=e*FQ6SY_IYP}W8=6$%J;xNz!21!c=4fs&9r7P9=>;_?D}rN`s&&w3c2CUPooEY~ zOaXcm^}^PtGsOM=CB^Hlzp}gZ-E?=CPp|EKLAQfl+4IiBkNy|bLTI7ZyJySAQ5qu- z0ssEOSdmI)<cQH5iU0maMY94j}u_cvm`zQUZxZBR|<20M(yB?vZudteqGi3ppDjgo~mK6{| zCt>&Rri&IQMFNaD&&QpXjSJ76_(zFC{ZSRPDI7niQIh)-%UibTYD0kFdP5jhm?7y! z=&<2GZrYd|cF4KxqYurm>v=ni2P)n!jnXzxB3zjjbE0$}Yia=N z{!~yl{fEIAN=LYH$x$+^e=@Jz64M)+OaBhp zd$ClrCOfxM`8J2mDR+tt9{=%}zndr;zUL$oDWGb0nfSEFOZ>~_++GsC>V5EfTpKNp zECfEa<)c{x(WjqmVcE(Ki1bfV@sDDnV&fPD$?y$TiR5J8Rd0vmJxaaw1h9IktGuLp?es*3=m`Lxz+a8|@?*Wc73ZiZi zQLZJgt#t|G?7~smd%@N#>5K7c`W(N1z8inzQVy;CPdhLl1_zAsMUGZI7J?-HAsB$Y zds9_}Wi?ybV|<0NiFDNo2*D{IgUfuP>b3`U&+w6GL;tm(mSeuZbE><9zWPv88FGWN ztwkgSJbTLwhE^B3o!VQ{e@XWDSkbT>GU$K<6q2wj>%cEHM3W?^?-p)9f1sfZJa@T$t3Q4$S?C zj5*6~)JW7t*(S2$v)JXS&wk6nT2hG>l7Vr@sfJ`f!w?I2`0@?yiCGVE4IUXYDJ^wR zh#J#%XvLyb3d=AaFdV?#qyChX*r~rI;Y|v3{Be6JkMUihR0B#abev@Se|n?dF$T7y z^yxSvk7jDPD3+B0uoKChY&8_XZgtnC$aZbP&234QE}RZ{b@?8j}Uu0!W|Mcr8E z-KZ63q1!>hULaLm2Cwy-_sc=GSSxv(Hag8nr; z@;)SgdBwz3?d9j^9UHE$^$N(ZA2Gof(BNjGN8jx;#gUZnYeEtD~?Z=s+qm zk7EX5An@vRe&j>mkQFXG31dm8*pQLP!%lBK`yd-ir8trb_^jK!7 znW9np5DC_`T#SdFR-Q2>kwQJvd|sHK1g~AAV6nN2a^}B_5k*1s#Xv8jXl2rWbThUN zcYDQfqQ_KkY~C`aX@B!ux)L?ktL@^fR)_xwt8_}K162n2%KU3Iujs5u7OmIPk=nWO z<6?!ia{Py_>?uTPoxd8O>lk!r=$^8ESA0a#=Udh?MM+u<%k%-fKDUUC>j=~?Mtf~y z5R7h+9eoUV;TQHgxHF-F9x(SiojBPwdLHyhkYKOXGc76J37$~$|ES8F0$fRNy?1U! zq+4M+IgG*kWA==Cdpp2^s#0$Dbs*}ipglXmGtQUnr61s?j7bQR@v$P@75^+#wy%HB zu#B=u^MR$cr|jDcgX>2=>hLt(Z!zaz@K@xAG1W@S&E(8e2tK*x-2cJbL2}7zHg7%Dn_en zI3FRmlCc};iD70XFDFvb2vaPh$P)h|Vzyx;cMB!nd;7_5s8=Abtqpwx!&bfGIEHKi z6yxyIv2|DCBGJS)7-;UqwY1v+N%mRTT8gj~HAyG|CrTAB8zo-HWa~fsX0rn=uiB-9 z{@idtX)t#N)Gs=}R{cgWM7<@A<$B`SQGQ4Sj$`qVo9&CZAoN?yc*=jH_d@u5CN}Tb{)hY&kS;u~F zipu47jUv)Y5|z$`M|Hc~J3{pedBvA~;&skdryR_J5|cZQkGIWL8EJj53fz3g!=UFm zqEVAF*+`U`q7PcM*Zc9C5=IAwBL9(k6r|pG@FCUiO@$M6zgWJ|Y;<1y%@dwj2J`EY zIu(d4aTY#qJgCRQb`vA3ts)LSe+#C0+H;_T3Qr@z$u45fLRj26Jo6mVt#ZS&@t-2Y z$}hX;n!!^p?D@jZh|7(<$Br4vV1=QZs-p4dJo3F9S5m!Z)Iom-AOL2NjC0p{XELP< z8;8-1hjH(D*)SPl3^=XB-SocJ%3V#*p`AGL6b|^18L+(3)@n3)aE?RJ$+F)NzV%A{ zdl|OwL0Wj%&~64KT6j8EyF|NN+%jvmXJTlXi{CJVcX38Lu{PfL-r4y1Q1-L2v6zkB zJKX_v3d7qT>OU8AVRaUW1DkAT$c0zf>aWFnA4+njXULbEi(XkwfIuShGoDi0=~^}h z^!)E<2E$oqiB!W2E?x$5-Qld)Aoliy2YKwM!N}f7mY*lB_EQ-w=G%ht$(@f->yoQc zjlsd#xa{iSb{=}!zZ_Rhp1XbyM+pxI7dqK?RPe;vQ> zTa)J|LI3pw(TIv2Y$Wn3qIK6Yp}yK4rw3Y>!qhFyc=VL?Bh;-d*JWqaB-{f|6$_91 zH(ewTR^(r$9V9P1Tq?i)+4r3sCo1C3OEQk$FVY6Y_fGh zs@fS#!6Z0}CgmOdZ|VKiVPJ!34+%Uxv*`$pc&|~V^BpwF5tDtZ`twrGqH{>-ct(cO z&+N2q51>}9*QRbaj!dNY#<7(-p7g9v)#KSB2Njz%82#s*iTc6ew0$C3xfRQOFB0yt zGfnIh*WtQZNnlTg8n zo>s^Zb$5K|K;jLDLON$-wQc86y_g`T}oJR*HffsP9v0hgEzO{XiNu@=T<@J~-IC(6NbIK_BMyr_cchmk0EO^u zu#9-lv#yT3#@!ID`&r$NOhQ=piB3%MNY*XY1HE1F8+_o_?+ zJ=f`IdzSA$QpV~-``;8v=lQR;zixGPCZ6T?qoTiRfHTr{v@8i~Vl-&2!xj>S0_#tT zQ~1^V3J)C}j%G3pco)Zc9Otzjq~Nca)#SPt&{S*jP!#OpfsUQs#9s_KkioWH2X+QM z2Y6Gc9*kLuO)dI5Y!wA?%8g0+D=iD zuh<okcB23Ui6MD)KYLpzp#INt~le)8-}>NhdB(vv%ZFZtQTN|3`X z;}I`HjqF4`DZW$_c~#x1wE540-DUJN@C%}_yGs8;K@o92=}TuNKjkLuvB*h#p^U~W z-!gB`f&T6$mGJ)~?7hR`+}gJBAR-7NAriet^c1}(B!n=!5G4ts_sAGVOM(!+Mkj>m zy)$}m5oMHNFnSw|GQ;4zW$)*C_x^tGcl`dyF*#z~Gg<3e=XGA^xz@EBeIei3!guzC z<7(X>R>4~pD7a($T9a78gIx7gwMR)?KARE75V@B6UV@1Z{cM_brUd}!?=suz!cNXe zcI7?9KXCoF%{XDpU)Zk26bcYeQMnv;Wa`lGA1^dE<7s7fmR2SgL`vOv-#!>eId4r> zc1VGF+cGxI4eD=YpysKQQjc$kRU1~?EUXC`hQ9}{Z-0omq*MC#=?HLj(hON+v!`(5 z-UDd?iz3~xEGfFK<}V%-*^irEoUT;l)hqaxtC<*+^H?&O0Ua1S1R4 zoXCBq9s0SG_D;1|?Gxdt`K#2MLl2j~x3DcuDh?64Z|@if&3hfx3mKap>KHi4WbYun ziG-k^-`-Ew{-k9fUm|ovtE&EpGnWyUp)dI)yYp@2`h#O`TVLjtF12Jft)1h|572r! zgYPxC|2Ue!M&qN=OUM*jfY?t{8#t;OP-{-|Ql&V|%dTHzA64WR%NEM?bJub;ZrFk*k;pgEpF)_LaQBg){-)g@Gt zhuGOC*q!qk``a*yJJ8zqwQFv@mw4(n7alvoT=gE7N%u`JdP!{#Fk>Hz&Y@aCyu~*l z*`-Zv92Ap%jv7UGIJ576d;B;e?ThVG)%eemhF7hKF3!O@=Wz=L`TM=(x5B~VU2J?C zwUmL_E7HCnG4lD!(&V z^lf5M(Y-(L!r%5RRarN#+dbn1!=}IztrcLFU8?yIyNZG=kHNc?sT(sNn1As0qrb=N z(~+AE2qr!Jh7B)1c%{f)W#-)#x>p?#Vssg{WD2xIWCin`o@_Hj{RU1f@=fPxOJj6? zgx%JXO0sb$7 z+FsO}s)`*W*%4}q{#QN={^5m>1?NiO1xAN@o(ZBBuSyBhY^2rAfCdOy;y0Y8kW*uW z7g+>zJ&j@Eu~A+0-Cv2}ETyYuv~L{A-|QXjv{6PnZsCRJ`pjtCHxGTUz-D(+4E%oG zZDPATwq7~G`2~DCEHb*TYG^zO%Y0qPf&F6o{|XnpfJpiks6#JGK<eB&@Er1K>e?$#-$-bng-)g08{-FbW_wTi1oR(;J3 zCz_Key_Zb4Nc))ryHL6hzOB03cs#_g0(7_#9+>{%kMXz9W^9c^dakHti#DK{u5$B9 z3-PB(*rwuE=6u-l*6-H#b2o|g#_(&t=*8GVT`)3l97X5)WMN<5wc-on&}z0q5jm);oht&QOv3!BgL?T z12u9Q#eUeAa@-OrQo zC#!_=)BAz4A3cA{@za13_UsHxRz#@bc)apcORL$$SDKXeEV;tBJ7l5=tBl6%*a274 zO@)-S7ycF?l-yZUOad42dykl%BC~Aq;w(&Z?>)Vi$SF;xkXIcEMMjl+T*84gr6wH+ z7m!*@n%i|&x@C&2T5OUKc?u=omG&gvWgO#$r0AelPkf5G0$D? zSmOcpjbydduv7DTp!iWbhOX;8g8fiV3{mgg6J7+hem7IHuFDif8nc_i0Y^>6(=!wA z+wqBsOPUc{ewJ#sQ^G*-K@#^ewt_gIOh#lhaEDF3V3iCHYNIZH-iwMw-DYwx{&n%S z&ExB2jmJx?;6jxZ;O9`X?gbkznyM4%>HqpL1NC+BI z(KdiTp>02s%enNP{gJwvS38OgR1}zUb*f_{B*IaXZNW8`dxaN9(mCY&en`G(8tJ~Z z^bU}0ulqSRN_aItbukvR(Exql6fi9{+)FU{;lkyLG_HP$YQ)22p9VLcaGMdyen7!a zwjV{`ReR{VGB{K3SxKb~&w76IK|sud)$Sx;4i1}$I2!#rJMH@WEtOohzp_cLKiOo6 znzOm|hVNd+!>dj|RHI#nk+#ZO9V0naS9}CgK4E@Lpzh^b^3QYqR*LsERu25S%K7Ej zkIvX%Wfb%kSV7@ctzTc-77q1yp{mwKTxydaz&v52sp`cDIf2RfR0rtseht{8;9>Vv zsvs>-h-ZN^Kf9_8LoZPBl)RQKY7im;U2?&!Dzeh@AFT=Q+%#xp=_s6H3)N5c_4GOf zfdHgRp$R0ki)z&RvH+lG<>L$V^yU)ht_3I?z0cst+;CnGYK>GDp|quyX?PZ(sjylsn#H60Y{nXJf{r4K#Q zRsmfP+Ipjf&=S9Vczkf&38f88zAC#NR9o-9??T~Z9~3$t9;|nDJ~6Fa2i-S?0rq)w z1CY!bP20u5qRHrACH)|FTYM>aGkPM=!I*{<>JT%CU?R-;;FQKj=^RQMg-412+%7NYLVX5o%*_W7cD+g1P$d<_3SEa}B+Qnapx+*Z#+Yo3l$-p+IpX z-t*w)8Jy!0$!p3YXhgtf~h!rK=jLX~Wa(x;=#Il^ zf%rhi6+pdx2iVK}vqvs3-NK_cS}R490ZsWc&7HDY$}0^t=6;MMvMF&n&+VUvhnqg2 z)~|EQ3t=Mo%o;2e(snK8K8tqAt?Lv#ABCk5Yy+7JV;{@3rRB8k`K%o)-*8$KTx0d@v_l4Y@J9nq*xyNvmN!ANM`fRXjLiaVe!R1RhKpr$ zr(0k-EIBfkH}27e*mDc3lBAh8qyM%8%wb&nE(NlEeWXWn6UI$Z(bdt%_wbpn2)yg~)7MXxY)kd8&r z2iD+EVN$PTzddYw5s)JzrXy&u^R$7Vd0a`%-fF5!*06Tj+Ga?MxP0e)vyrsKeoYry zjr787Y{O@4az%VUGuVaP{+wX`Inw`A%QwXrcY^|Ne>AqRTc8OdD*Be!uB*f4Ov~G4 zrrb;0lTAo`J9zlz3;)tjpV%&6Cw%anB2hE{$2NQ<$#i_@6xMiDeFPqyT}k#DY(y8% z03Ffr@M>7)_!06X*>n4Je1>GUSCJ?Y<%vA^Tki74BZZrG!;o)OI{WteTEacYxGv)|1VN+v#cik&wVI4yXrdUiKh$=h^L6Gwid4>O=fVo&4Rpx7woq8a{h zl^{dZ1URJO!F=GjreLLkp+RXVXbD^R_)b+OO+_>%ot~-7rfT|f@D*7as%ymC#>Czy zV4o&j?fFih%gS6=BHVj9r&q~+O^A8nxpxpJ5wF!tlxk~<)p^REypm@2_;u! zkAka82j>TE8>(iLiVKe)wUg@?Pj{g{z;QLk99lIfl@=GUkAC6XC$q7>Kft|Vfns?*oIrfxM?Ku^CJHEzs~xPVj0 z>4wCFJohzmH6oMArUF9_od)9sMJ#bjq#|KnN6LyORj;;1TBtq?7Z^Nn`Dlaa-3URK z8>yQ>PQRo)bjo$!RAsI0+fxZcI=5TRNrhQ**^m(*=3fF6OdgmLqrtS?9MeT&Mvk*} zW74?DoTOKKste?FgCbNPWW5XR5PY~E4gS1q#83P*SZS_ayv!}~{IW-ktj`C7`p`Fl zvr~dT!mgZYQM>8FgZ(F{1mmVOX~_Ds4q&cJ$ca#(|4L*Nj?obt(D7%Dbv4=z(uv^Fx9W4#S(y747D6gj$@?L4s#J>uR$|AlRXLl zr2r<8P(M7_*>u%sn8Em9TSyj9z{lMtaUVjQP*-HfcLtF7fq|si{kvALpZYl-XFJ-! zwx(f)b#GF|I+t3k+UO4_yD!sE*f5@4{k?Vzyq}*uQ)e(Le6Zt-z9wPJ6Mh&T@yq1N z{E6bNSFivx(;%AQ5uGrkcjT2v#c@)_nx1{}1$hJGKIDF0Tw1r)fx#!`1Vb8Q7@K6+ z`?!`OMx0hb^;HC=i`W4r_DYJcf-ib@UxgQB)6^_t-JI?OvV$A-J#gI-j~^5 zhL1v3zbYvF#vkJNW&BsS%4J)|lY1%xaj7`q(GCr6&03IxUeaVW!pHALEeUxk z!W>zUj)+%cJ8J=(#f_(119Hbo=RYC9#nX08SeTeP=2&-~D%vBy)e+z6*kFsU>k1IF zY39xpAGCjou$N@gPn zxri0`!XHkvvlP+2hMTs2bF}oxTUjx5@AFkSn#5*VTrpvhnP+Ix9D122p6voH%ZxU; z$u)!X$Q|Hax!9}5i==;+#A_-t{>u40H7E;YJ$NPvy_xW601*D^pUQN4Cp>yL-L}W^ z^q;^7E+iCQccc5V6@cJE^~SC+JO$dKd}W?0fPl<|4!p}=fu$*dHRh>6V9yU@54|J? z+zD|voqT0TEGMk4=><&e{TJN;!s&ZXO4|+KgDDZ)St&knM?Jrqo2pMMF-xZBHp1ZK z-i-ZvcUF1y2D!xSjm+opx4U))%gyUnA}OZ+%$eq=-dT^ zMTLujWDZ7UhDURd8q6N68Ox(8+i&157hfd>6y04ip!E=%b%5 zJgOY{YC~)>^-{gv{Z~pxBr@8zK;|ir{F;s^6O!=eCi%K`tbP(y?6;f3mZR3{>g6OU zUI@hZO7~jgdr8fY&qcyc#aSz>DBd|(c=5l~~+1h&{X)$!?N?YM{ zb)sdOS4--V5M);YM^!!GbggN#!p|}Kz?u3!xZC;#>bWLrVufQ!)G@}k8mcNttDLaq z-j|!YOW);^1T~8ZI(*ARpms_B7NP4KKnfc_lyS?@Nn!ZUYE zs|5wUASvi`PJNu6uvC!qN0uA8KllMu+oCZL2bQBJi?toft+cb$F+fp>f!Ep>Gc_F` zK3+y-Ui&%EdGkcxX0q?#@k6D!Kw21QRK8J>>qC?-KuAIk0|KV97U)gO3e-J3Pqa`I zQpyJ3=2Y2hdoiJ>;%|j_n7Ybui1-s`G|%KU%ZeJM)$Q43GIV$($2WDtK;8-YNga$E ziHGB%H@WZs3-L7l99(B7j_<(b^4zy;=@M;S;;+dF8b?W47{Pq(dikeiXF-F!_mY9x zoNMPxSAz~IYx{a97cs94rA)7Kz(5VxOMNw71AzAYD!j+&Hr0EH{NzWcjIOpSRAeuuVCE^T$S@ zs_z)meWQ5-btGYM*;Fv6?f7W5{I=$5@l5f5dI89bwmzV$x&6Ly!rF<^2|vzG5ys;n zlCG1bI_xYtRS-eT*hKqNKujVP+oGR2^s9IqKA$7ZY;?b8$iTNx@u-go#n`l_EFT_E zuDl;c^XuUH!A&q9^g#m$ggyZO~=$hF<8TczJEVhMs;3M8|aiA^S0y z_$2(W?qydaE1suhF}0-p65p5BsxGM;LpC7OIl+3J=uQ&EIIpfP>7IDsD;K=vJ*utO zCZT3%Hqr;<;)jGFwMDG`XRV|nwWbMZO-AVIPbTU5xmRX#P~NQ2-o3`=LEMO8$G}i^ z>X945l)&&@5){b^GX?DoL>|2CxZV;~{{t0~roNu53aF25V^ z!z-*5Zk86Jpb=JMOx@M2a((U)U1&{dt{I549++qBzaL z_fg>ztaVbl@w^+ecS0SMGerx<57lRhstV~Q0N~+B4KDpADv}Cx;IN1-nZ{#p{KZh6 z;P=uQ+NNr9n)=5m%ud1ctr92h#P*A3+O?;RkA2ocjtX1;hYnrffDPGuq?|Qs79(U7 zE47EshB*{$+@0cBY{Ax^pT@m$0yA8oTEof$B~K77mu3B^_A^%oT;-P zIN;8JfQEppO=LxNMkGdAN;c!+*4*VwZzCvPVxv~;f2R(KI(hLK@s8TR69_S5>T+zF zDAagF<^J$BT>IBSALEJPdy-|K2yVajCCTG>xh`zuw7gT|kRW(I=Do4GXQ&y=WA)6A zU3tBiKpBxqY|d77rAg3^jcJw?LFJ3;{Hey2La6vPs;>l6W6U>9)Fj#ED3XZ1HFWyW zk!a;o_LDA3B$>8@J*-rm7^G{V>yg}k8_3=_zP#p7?~FE9-DzWKIW?H0U&Tg%E$4Gg zJd8(cO^2`vmzD<(+OW5$_-#G*AWq@gL(m|0RQ|B7?Wv7)AseYyFVA!Qy$1;xap*O# zJ6l(wk2kw5$v<1Z@TYRm)2bXZSPhEBx&C4P@lSL>pu=@jaUb>V?>h~rPpceQH{|T+v`-+ooTMJc%6GDzME_1P z$7HJ})PW8^*X|yV<-->#^|m}^8WlQ;j!gF&xs$6~c+eaoXSdKC)jUGW0YG&0=*cHV z3?@sM$tl>l{vJpY>K=-)Ombb>r)t?h`f-+|dFu9d=NHPPfqCCk8k{$Z%v?Q!oHF%g zgatA;G(q+_K%O^|O!f;>j`cLVjG^$K?x8SGlT6cyG@%O3qt#WN%pqK~3nQN6Jt$aX zwC9TZ@^Pms(kZ9~HV>cxLGNKHP%II7qC^2s;K|UuQpwnauUM#maWbxvM{`~%k-sAF zEYwV{_KRV`CD@}F`K1H19HkpBGWK3|;%Xx1&LME|L+0<7ffCC5YF5=ep>7uxz%8GL#;OavAuCBZy#}Li-W^GB&dQ|W&9>lo=Isc2!D>6d>p}Je| z1Nr`fNZezAt3@u|X94n$tyo{C)_35$GI2P*a7W(n{UNH#$CrAbO@SF_RTU!M%4+i; ziwC0gXYB&Vdrj%TGRqQe&2d$e;Hl?Ok=pxtk@KX@dJ~fkv5xsTC}Wg2bQ)Osw)>fp z;)A`N{RWd?G)LdTMqJ>;rxqH*m<)~A|KxyP#1+=)Dg;CcX=*((8=54R!~kDPV-1|` zQ4#xvj#l&X@~P}Y4G!=Pj8%KJO-1AkB9KPz1Lv@DA;v$jb6Z>9#{qg+oBx4ZmPsy( zLjJUNHw6FqlT8IPH%<75f2s)&xh6pAt#zn`NV$(5)tN58-Z1 z-ebrupT%6h!!t#n>P(+skmG%8=LXL~zhcSpo5c_6dODqtOsS5`14Pk0>u6ik{a+T= zKjpq&>S7i6Zv*D&i$d) zMj-5G4*K!oUXe)xN=#m?!rM#z(pYEMXRfZHrqcH}s$;SELI_zjY81Qg=v?kWaD5os8I*jt{tamU0B6snJ=gi-DW*j0-z=?@1s-Y z94r^d=w+IY@fTP@qZ>jT15G@pFRhRZNw^d!6#npRF0C?x7wzG`OwK?0C97100@gW+ zaw#s(zemFbud=`Q$`;6e>9Q~SzyiFCP0#N=@bKQz6b9XyKY7wOuo}6@cF492AKt$l zh`zk!>3p&uwc_XPIG*!KcXVD)uV*00zfe8_@k%e$Oz78L)gox4)qNSf3fwZ6g|^6+ zhQvqI_FGiWUdhUSPjRmo^IG5DL(tefiNChYMI~g*>G*e0HK)>ud*xc+72cVtJtUE` zof`=?J9Q^M&1u4x*6$kEZ!-+03@pp=T(|yb&Z^m<`6WZ&NHj!@yWPko$E%OzP?uCk ziuDL2|0l*6B8B9fo}aBA6?h4+P}DLYD;dN#kqM7-3ME$iBZaRnnKsk9SPOA6#_V0rNJOMX&aCF7x z{EO|w{Z-!Ua0UI)3KaYS(=!(P?@fX}tnD+hLn@etSqO+>4fNZ2$L4^fVw<1~;c_%h zTU~Vj`3vcp)T&SIMC!4p9vJC>gm}ogpGnco)t!(%dDYuaK>0z+QzUZRov<$&2qY_7 z@a7Kb>HtCA+XSIA6rTLa=UM5)&x4Rd;VdJ)`Y!)CVXm#E-(or=`hx93gg zr4-hfOEKT}5W@~PRF@!+z_=yhtX*DE;^Ho(;pkCd_HS{XcH#Z@s>?}Yq58!_eDaw; zh%c&(p998vw#Qm1o)h-H{34GM6-#JkD{D`_LM`)si`3;7zMm{<>p@KO<-hXQ`&pJu zScmTR3P>Cpv-NS9EE_QU+%-GG2o<{+-0Bol8Z0OpOClA4Fof@}b=yc&7Nj7nD_%&~ zL_|}=7}>KpM*|IBuH3B{`o_Q0U~a-wP&DPvRU zue%UtX5K1f_za-=S(Zwl;?Uc9~v!To#tZiEm z$m@Cf6c~4zT0i9O!xPxI4gpl%GvBC+vRQ9a+m_KC$TSM$@oVd zw9EH}RjtXLPHLWCf02a6XS)5!0DYV`MvTlLwhlB+I1YgJB8zTijl-e%cZQpTBgIOO zx+ptq&OPqPs9f{g`uQF-O;0qE=3)=kr5&tl-^Ij#s~~_8c4#8w-}%&J^l$9yxJrhK4sqXj zO~mO-i+B#;%yOov&wKJuAf*HcYUwb&bfwg^YZGcO7{tUtTQ?EDsLMKZwu8?xwv6=| z*f5t(+sKWl3)hvGC)TBmUOZ*LV-5}fnY*cxfj>9jq3#q@7<;0>tvXW71-6*l&5G$? zH3gd?zFTX@+!8zk66d2A!(8D*XxHaNlcaHd2H6oLd+@BzRmmX&n-bcfKHr!9Sa(YwNiy#R zaIX8Z6sI=B4clo*eGTZA7x5uMw!Z~HOih8yKI4g4QnGEGh;<(szY14Xo2t0;AJTP3 zBQANqxBLNfSV`(=imS|q^w1R_On3SXe6xSsj`)v~`IkO+%JUjc&7C!Jlzc(20nw6= z7(1znCVPL7D15~jE>(BJ;%A*X5^Bs9ES}N)KuaE71y+~kJ!-|3cP(VPNbXH{JY*Y+CgD2rogg9TPQKAs zqyKPm!ZO|AcXi;bS9fi`k;_oQSu(L;JOL;`+Qf%L#OyAiJd&c zh6`56<>j|BlFn0+ctiV(T_8x_xwLKJ$>o#{__n6gjfE)4-Y5N%#>@ol#<@QW^p=Xi z2jc5PHFVvvk$J9iHfZddRw`MBzDvYM7`{&P!0LzCN2QgH-6M|-QSN1;lu*gJv&;MVS+NE9Rkol$@1F|UvI`0E^Er)K2!acU{rK{EYB>g@O-;FrGJqs z;2{1Yvbg05wsggR`?$cnIdCtnn#@v=7GUgdIrrd*KEXNOaOH6 zysvp~cyqev!rNX9ow z{QGNfv#v|Z{(yE?9y!t#Y|}8Kf%e!RE66n}RaL2{RdI8B`O*l%yYilZZQ ze0Yfou#;H*ZK8uL?>41#@k0!gMB7%+pTD!E+;rNkf7Z_7000eZ-<$N#(j$#8TyZ_` z05Q?4B0s5>_|(R+SR4Cc^@sIFNY|Qc+<-Wg*QS4)KrCMX-Zfx?n&5uzX`EYbt`46u z&w9Ds^3}~8Q%7Q`aP)Imy>@waUee#?U1tB*fKbiSZ*0%(is()4Jq=LU8Et05-o|wzTEM;9DLNs&V&VnHF*7DV6$*3zio+R_uX&q>!{%!qA%EjkE9$txwsv zsGT5L?fB7ir-TwQJyz3KCCwd7@^`>^YR;|tfl}jE!Vm*A11Gdsyxe6xknBEX5-JUaKGJ91}0NU6V ztX{+tf4^i?gu+s2TubR|JAxlbM*3#UqWFF>1?E(H)JYUJUQYRZt7Ng3{mS8&s9s94 zwBFrmrtgH*^MEM(Ods?%87QZ2C~prF-Z3IGxV{UBInw(lk)`+TGk0%gLAo&sI1yN& zzG<9kum9`5V&zeyP{?+Xz38ULIHS@&3Iq2wTWcdL(ob{2Vq<^AB?R zA7i?R_8YRfdV@^;rF4}ekJJ{j%AR!L{!~eTm+$P(H&pkhU8%)4jzl%4eIyhiBa#Sn z9=l4=Q@wCNPn4^@xBh$E^6Q12=*1ImLc%HRHm~}0DjKpZKq7hngWtwE8~pInU4-P+~6T=0mwOFP9DQ?+^AmKZC_HjR3c!>@5Q@eXlMU zJ&zSPlVAaCaUx!RfR+M!kNFR&Z}z6CaXk%MUY|aX;~ddHJm{L#KZ_r@Q+w0&FDm}u zw0D$BD3*rJUG4;oj&}bv8$sy$IWKt5b64@)iToed@Lz;V<8?I|fDIG}+SDq11R0ZHE{s9v>a9`Xx!J_mw+Bs?NqOKO;Q7^ynY^QLE=tpppy7zMV{Hxr&B+jatZQePFo|T-h1cPh+70+IC4oV z^=^?O=_mtt$?X9^c`Uqc`lW-XW~vS-m}eDa9$3TDW$95DV=xsuje3}Fb_RMzRzxHk zR{~fUpW~V_6Znt$g|>m=Z%s`s_yxI?8S9Dl*aMdn`a5b@tl|%++nX4C%-3Pi+`I>AW7xBHw1c#NIb~ zob)?EA4pVhcv~vR+N~j(tLxQV6-M^GdKsP--a9E7F!dF(3fAJgU4aZAIz2xmXfHif zLahLHjqFEB{k}4BvLJGg30g#Ur8jf-;%6|Ia%M}tbs%CzlM-_U!ROew?Zih_q)0i? zBZrOpS&I4_B_@i2%B?X=;&jKBo4av$7H#DqbMQ6#wC%w^Jrj+&Z4G%O(Yj|#ZXm2H zWVtcd#^3Z`$?9T-@Mp*){2WPj_1i~7EgB=Qkw|Ase*=>=Ee}wn8;7yVe&T9N*e$xsr$4-8t6A7K#;Tz` zc^WpFwN$0Pp|?KM*FTO^t6*CJoMq{wjmJZt7j#0A7d@fxf*@4 zkMz6p5^Z;>>y&T*wGd`$YEE10lWfu0xHX(QlZd6e-P4hXM*(Xh=)Zv4puPIx?ZF#Q%GSB!g2z7c0APOcJYF>o!!)40J<*}x5JMCaWKrlj**7Ug$7}(Z@RX zs`_0D6q1wOHhlsoQ7EuW1m;->kil(ZE`3Pk%*ch4)=aFSgvTN}9#OJ9C3^=fY!;HN z@v4@70RhvpUeS87Iv|}eYu)}^M1Bq@9}IjR20@?{*l03jUo~+=2*+-{==pja z;dEjD|4+<`r;;IRaW7mq00!ycy;{i7C?|c+;}|TrC!U%knn(7pHv={^&jm5;f(hkO z;Y!{V+P%(nw%5*b4Z{g*#W0>+&xIkr1E&0uU|B{O;ZCUL-8c$It{fKUge&5_@wAg` zCN1{EAuQaBb}?{UV9A9QMBUlpiZsd`i@wc`Ws5ZI{iqs-RFCw(Spo3#$bjn6431F2 z*`5FFwCLTV^!6GIu6h!u3hbP^5a}C@SU6qHAulgdRPT}`_ahTvvWAq~5@We(;n*k( zwt*VokT7!X#W+2B`n0-h>E+7Qt-3%+EYAgV2iJ598tF?ifimE4BbluNP?(G&hr0C zs456a9OqtX`GRH7wwF5(7zTc*(k1CP?$N4dPl<~%%wN9k^dN@9I+L-fQrzx~QPS&4 zN^t|pR|g`BOr1ai-)(8Z#r%Fx%NQaQa5?UCHnUNQot2R%eT0u-*~{fWp?Ckn*?75s zMZ|{5QC#85cJsIk{n+xGq=A)-AG6c_WqgQ!{>Vb4NU7pi-OYq|j|Fs=jYwr9IlwDj zSM^g3GbJ&&mkkW@<*0gofYlAi_gbj@p-qjSmaf|nmk}CeyWrowJ^LCeOeIVHLoZ2? zY?xJ|ch{C0{L0?aqX%BNp4c!niaI+Hr0U@XnTYBBo5_iUZXq!uhQ51 zjuQlX=HXde!PrE-SV5ZueoI78U_tz$&VTw-)CH0`KxB$g{nJck9_+0E{)ZODwTlD8 z$Q6OeBz{;!Q^vXgparCxY1f?~a7f`@vF%GlTfR`A&{8X4F2(H~x4>8ddJ1F+oN^l* z(l{_Vaz=!aeUZ{hpUDX0uMo%PQudM=diig zkM~Q(zsHALgP`#y1_#34#m)4u-F)d4^Kan1&_}_Ugcv!NXZ1fN14Yn0pyjxF0+R)8 zkg2uzx}|p!uF~|paCt2!&DsG8n@N`QDT4HTWwD;!!4=!fu9TZ_PB}{ zMC3)|Yjv!dy4@_zHsYDp2&x=+f22`+#(pKDu=3i!(k%^Qwkk9pRIIM{Qwd8GYB=aa zNKItDA7j;Th_Y2xha+phZSKTf^IFY|GunLapi}IE=k2aF_XbS!NM93Y!F$7TD`d{X z`yXh$R#7`C^GuDRQkDwhmA0KyJ5c)H?#sCq_BDbaDvCm9aY-ZFVY4r(HI=|zI4kLJK&C!8C3E>SMRw^lQIW4x^ zu6{ic5Q7pY@)~;3135N8Y2t}9*0xTj?{Z*1>u8h;uxE)Qm?u4jlJmIpeUdBRohHt{ z$;NZ)vEf{_5&Pwy;rk;p9tm%Oz4y%kp08xuP7X4xNITcaz98Z=ZCWk8wkN3ub$!?sp_CW5fd@eex1{Yb(ZWA)sZ$g84|FwNtEf{~6~H%aYdDA2$W;8wO89 zDNr{x5u5iPvwduR9yq2?p=v?eA=|!ySJn6}-pcn48BrmD()^1`H51T8P{1)P=b;0Y zNStZn^iU(_60u_M{~-PaX^YNo9`;Imjb_V)ptDkHe~SH!sIk)2)YE?RA%QWi?mC{$ z42dDnKRFuTS@SzV+nyyg;qxNErcc!FsR$NXYoUgL#qzcjlzV*dMp|-$+sI`lfkv#anZ-eblIkt-Q zL*EK2vR5<=XPMF(S(*R`^QWcnc(K<~Ih~fgQ2FV+S+B&N4L?T&>*{?*rk27V-{t<< zSy(WLmF-i^+dHO~vY7i%QjTS8LVa;2fuY>>zF#OMHQcl{UX`_nMLxH|3-#O}Z+^*t zj@=sBM9R@3m3(3*`3Sea<^PA?ekSzPczwo%t>RxdZ9K3{tUg;DTD~`xeiY<`EnR*p z9`;t>o@ zG~jh)EF9LT$Pgn$_(Y|#LtB}gE%gT24O*L#LlESF~Fs@!#yrDR;Gk+3nOL0Yu zW4XVapzBuksxN9#rh+WQ^|{4IchxZ2e#Ex?<)f&nT}n_{&WH_tqG?%`>34i=yUwK$E=TH{T!4((nJ~;BZsPzFm+e^M zWZY!=qh*M4(s}I0d%jrb>4eHcUGVUOP`Qe7;;*JXrRZCQ2z6!m&i&WTr9wTQrNUjAbrOz`u)kTMs!4`#UShI2xy_sN|3~d6W3lDmIheBjvw@ z0k}VObUECCZD;M?h``6LVY2;hOf*4#kTq&~IoInWgHy8a`r82sYXHy1Xpov=UIH#m z&lw4O7+-k>e$7?p{1k$w7KvrzjX6)dCfkaA_tS8-*US32a?z;CzUu5CE++b_LjBW` z?MDJe7qsWyx9_UVX~>nRi7-x<(IP3~zRp@IsKHjjMdiu_DcE6)e}3!}HJrFvho~6a zT?9BXdBt2A;n*Nm7qAJM%q=@^El_DS?X)z%o#UOH=`*0im6B#?>To<@xr@p3&R=fk z=J3Fh{rLF)YCK)K4lKXvF(gJ;q1PPREVZ<#DsP{By;Dg8zOa76+7P}`MAVtX22Ycl z<|=OdV(UGS7g4eZ2(=@74bmy@A)xlfxRExcoQ*`|$3uEfxw>*6B*Om2U$ZP1b=KUw)zV#*)Qxg+t5UJIty>T>wnKwh@J|E&hp^UJJc z-e3$KP__TQ!R*w5#;39jDooWJvH0|)m1RqbEF8@(ku0Jd&Y!s&XoOQicY>DbeAetC zj7=-4o-Lhjj=I`!^*3GIB;D4#J4=0bs8X%G(MQN>M2)k|q?@|b5Q`b+0J@k75^2H= zB5h^yi%J4QvJXq?%$k2RPggsNp`rMh(9Q^(ewL>GQmTl`ACM~M!rUc#RjNWBnbl0} z;U>VyuVi;Kn=ZF7sTS$xXf>O6f54KBW^8NCCuOX!?n1rOHqyhTfDJQ+nj$aAALw)- z=1M-Is#RlbbnMT$-S%r9u(TDuIEB07$I~x!Cd#ZkmA6$t2})j8eR%5ATpIpeZ3iMs znf4|1m+M|lD92+h#;%V^na^i=ScRHN-phIuPeIl%cKtlJnru+V#3zRt`rqAfwJdA| zwxhPbA6S+khP&8>*h)D!UfP%I1fD#Wk|rqlpq5FN>GXXSU|Tjm6?i(b2Vyb2)1$Y9 z9e8H&(y;F<(g`)%0~K$kFyyt-ewc8Vb6B8IQA$Qo%mcxzX?}_2QdF{c)b1Cz zJl%DdMcUeJc@=4U@fUId`;kV3Na>|6g`>tWHzx*(=dfof5KVg}?X$~~;qH$6MOr?XXb+t?qJDWULra$xU zhMuLFVwaX(Wj96!&_3AlCS-%tfsHG0g9gJ$IO4R^@l4ONz!Y{UZwCbXoH|;f>2<18{tY92|;1I=8}Lny`H1 z?KzkOhmS%&ldAkQv|2VyGv!QZ*XwhDGr1A6sf%eRFxH8|Fudt(;@zr#sY{fgpRQZ z_igZ5nEs9;NeaM4>b+Wq`GS~3_@_DI~|y@%F(p`A;BcI7q^ngMhA zYS(wbcwtH1jCe)vq9ltoH-Ee&KrUrC_mI>JQfM2xG1aa*GBrFuA@+oHJW1aPls#m5 zwBec*!~^-s9hfTeP7+u~Dc`+^)RtZ%D60v?`zEehXgLCPx5j=q{HtWR6JB!lO?1;l zsOrFkC*#Q_iKhPI-S^(wb!7;30~4P;S9a5z#ozACN-X1d{}x;4CU~6I`cwCNVV)Sxt19&Wnm(giTK~8+@_+Rwo7#*T-4I!aA{z;Lw}3&?Pb#cOGyCW z?zJ`X;dt{=zN4p-NgspBc~{|c5XHWn~23@p=bHk`K+8nDsa&awWJ5Q3AlSf_7~R$MzyB41VH^;<(4W<{=}O9?t>gF=6FWr~c+u(@?8L0ZHC) z+q=?k(dij+6Jhmtjfqe^zSiSRU9wC)cAAo;DH~lIZZZ`?ZM8oBLIG1o7P->JA_Tm2 z^=X5`sh~bdL09`T;r;9phBlATsOy{f$8nk^px}=oRwUj4SOs#nzXbB1v`8eXnp(x)$_6tZAo}EB z0lorxJwjqp^RXQIyY! ziPP_L!^x5>Vww}uhW-CMRQ4a|poy#>P)!v?#&W@4(dD?ba{z9%xh3EI1_nc#nIp(&pIjWen1*C$Hujt*cohjgUUDO|L?r@l>q0U}R% zVBEE$wucBUe36>mRLg;7{1lA3Q;90NIc5NYLUA*d{IudDPZvD11U(%_sLP#pGz?bU zMHXq|jbI)|<^F2_OyTTTGd&d82cv0b$eOSfiEkf%3?T>OK))K9DSwTK=P(Iz=aLf@ZcBVp8N zFlFq1RT~iu!-lqY4#n*r1|=E{zRa%j2-n<5P9r}Lgf!Z!+1*sk*8QX-oWS=m?P5hn z0CC0f>l{%p;@^;3*V6FxMgZC9RdT@-5B)2MG0Q})JvO5Psl798XvTOV2V4RSO8%YR zx?5SC&vvk3hjnKmXPt2Xe$ph27ktKP#I=tUnNytDJ)rvo(sMSyla{MPEyG1Xx<@$cpn{uuN z1&-aupAUi_WI$l;=wvd7)U zdo!~VN6u$zccUpDddD#aXYB$DgkDSA37u=0ZZ>zJ3spkvB8N-inno!DV?vuZqr}b?allsTj8*MIT?$k22 zOPb*9h|;Ynbuuz6(pDss?|)9&;i_NpHahpKhI~z+s*@{>QIxbfGrO5t$I?rmLwrF= zmnX;Jloa2oVSCN;1!ongs~evoxRJl}{><3ylrFHj-V_eifDS0vaqqlm2t6V{dXcOY zH=34Z9JWHh^m2D-h$JqfjVFsj^5m&DY@a}efV*bCfGTwFxS{1?*aDMkO`nmEdWkV^ zGR3c>f*iAuoE_JOo|oT*MSU#wMYxLxwkg#WEPoay7cn0j`zlG*XCx^q+#YCwzNX%z z)RiYpFM?JLV~(QN`@k2{1d-k^ zj&q$g@F7|t8X=7p_KKEck5#OwI21hR6Aw|6vMX=9=ZAvENUmoEZ$zb z*zyU^MaD>`ZJ+m*-yT+HHShMtiydF)mg?V@$v7M*4`^VuE8VOrjlLE-!StH+TH*7W z5UtZdZu-^RoLOtT)Nr9=E@*J;*}5$@v94EpB=2yXeI3&@)gRsLaY5RgJN))p@zlna zx_yc}d!c6+PZfpeAVrA%HMhFGt?&MWmy=}09S-#TuS_#P9lNZv`NaV0^x>x%Mq{LGd#%znb#@FDOys=3X|KVff*NDIz@}bcsIOY1}kO%-SyItEPfsRs6 zpY&9YX+-{mM+)4LfjUb8rUq9sFV-U-ydo=H_| z(247QDy=8ggs$w9`N?6hVJwI^&wGwV*_Hn=jFtJAH+#dB!^Y`Q+`wMnRw!J9BTylI z+ZrboO8%!R@xQU@|20_u%lAIR^5_oJgY<2~dlhTT->u=E5MzwI;4>Tq!F z7IP>bAJ4WET+^AwTrd=1qosdHrC(M7q|<;1{hM^!aU%~%r(rbI9%E(vVj0nKfRdEK zH}lr_@)yFNT2E2}|BHkAa}NJ6zZky!+sHW8IVeL*QQ-ir_2EKW$H@@OoBxP$w=2=jVVRq}*wmUVniS5y&Y-DJw5N;@3$JlorE;$zL>V*`J5q zyO8KEM?}O`)+-QTqz9y;e;z^7#LDCZ%|v7$V@Y4>m~+h+RRkRJ`;~M;KR-NC`q1`U zkUU^cZm*F|&dFi>b!(!ZNSk|`A}s7YA`SN)?G-1gBrhLKRJkbfwYCq!6}~Sq2cxMh$-Jcx*O{4+|wQBJ1E-ef zO#G;J-NF~GtdsM@^4F(#KsJn?MNp3rXgZ6&+9NwwOIhiB{b&g}CDg?G!_@`8QCXV~ z@%7iqX};Ah;zB!NEmCkDv5G|SxCS9f72 z@OfM2pCv>zz2BXO%h9Q8lf0pTY_qIfAMB2G{CsT_%K0)Jb^_