From a9958e82c606e3b8d1d0c81929ffcbc1ad4a162e Mon Sep 17 00:00:00 2001 From: jkaminski Date: Thu, 4 Jul 2024 17:43:53 +0300 Subject: [PATCH] added large bn experiments --- .../divided_bn.py | 465 ++ .../f1_lbn.png | Bin 0 -> 57902 bytes .../f1_undir_lbn.png | Bin 0 -> 67092 bytes .../golem/__init__.py | 1 + .../golem/api/__init__.py | 0 .../golem/api/api_utils/__init__.py | 0 .../golem/api/api_utils/api_params.py | 102 + .../golem/api/main.py | 139 + .../golem/core/__init__.py | 0 .../golem/core/adapter/__init__.py | 2 + .../golem/core/adapter/adapt_registry.py | 147 + .../golem/core/adapter/adapter.py | 186 + .../golem/core/adapter/nx_adapter.py | 117 + .../golem/core/constants.py | 10 + .../golem/core/dag/__init__.py | 0 .../golem/core/dag/convert.py | 29 + .../golem/core/dag/graph.py | 283 + .../golem/core/dag/graph_delegate.py | 82 + .../golem/core/dag/graph_node.py | 118 + .../golem/core/dag/graph_utils.py | 268 + .../golem/core/dag/graph_verifier.py | 43 + .../golem/core/dag/linked_graph.py | 226 + .../golem/core/dag/linked_graph_node.py | 76 + .../golem/core/dag/verification_rules.py | 63 + .../golem/core/log.py | 243 + .../golem/core/optimisers/__init__.py | 0 .../core/optimisers/adaptive/__init__.py | 0 .../core/optimisers/adaptive/agent_trainer.py | 198 + .../core/optimisers/adaptive/common_types.py | 11 + .../optimisers/adaptive/context_agents.py | 132 + .../optimisers/adaptive/experience_buffer.py | 142 + .../optimisers/adaptive/history_collector.py | 42 + .../adaptive/mab_agents/__init__.py | 0 .../mab_agents/contextual_mab_agent.py | 106 + .../adaptive/mab_agents/mab_agent.py | 116 + .../mab_agents/neural_contextual_mab_agent.py | 24 + .../core/optimisers/adaptive/neural_mab.py | 315 ++ .../optimisers/adaptive/operator_agent.py | 99 + .../core/optimisers/adaptive/reward_agent.py | 31 + .../golem/core/optimisers/advisor.py | 32 + .../golem/core/optimisers/archive/__init__.py | 2 + .../optimisers/archive/generation_keeper.py | 168 + .../archive/individuals_containers.py | 171 + .../optimisers/dynamic_graph_requirements.py | 11 + .../golem/core/optimisers/fitness/__init__.py | 2 + .../golem/core/optimisers/fitness/fitness.py | 170 + .../fitness/multi_objective_fitness.py | 117 + .../golem/core/optimisers/genetic/__init__.py | 0 .../core/optimisers/genetic/evaluation.py | 279 + .../core/optimisers/genetic/gp_operators.py | 136 + .../core/optimisers/genetic/gp_optimizer.py | 144 + .../core/optimisers/genetic/gp_params.py | 103 + .../optimisers/genetic/operators/__init__.py | 0 .../genetic/operators/base_mutations.py | 427 ++ .../optimisers/genetic/operators/crossover.py | 349 ++ .../optimisers/genetic/operators/elitism.py | 46 + .../genetic/operators/inheritance.py | 48 + .../optimisers/genetic/operators/mutation.py | 148 + .../optimisers/genetic/operators/operator.py | 43 + .../genetic/operators/regularization.py | 65 + .../genetic/operators/reproduction.py | 134 + .../optimisers/genetic/operators/selection.py | 247 + .../optimisers/genetic/parameters/__init__.py | 0 .../genetic/parameters/graph_depth.py | 35 + .../genetic/parameters/mutation_prob.py | 36 + .../genetic/parameters/operators_prob.py | 46 + .../genetic/parameters/parameter.py | 36 + .../genetic/parameters/population_size.py | 93 + .../golem/core/optimisers/graph.py | 5 + .../golem/core/optimisers/graph_builder.py | 136 + .../optimisers/initial_graphs_generator.py | 72 + .../golem/core/optimisers/meta/__init__.py | 0 .../optimisers/meta/surrogate_evaluator.py | 50 + .../core/optimisers/meta/surrogate_model.py | 23 + .../optimisers/meta/surrogate_optimizer.py | 58 + .../core/optimisers/objective/__init__.py | 2 + .../core/optimisers/objective/objective.py | 90 + .../optimisers/objective/objective_eval.py | 49 + .../core/optimisers/opt_graph_builder.py | 238 + .../opt_history_objects/__init__.py | 0 .../opt_history_objects/generation.py | 59 + .../opt_history_objects/individual.py | 142 + .../opt_history_objects/opt_history.py | 277 + .../opt_history_objects/parent_operator.py | 26 + .../golem/core/optimisers/opt_node_factory.py | 73 + .../optimisers/optimization_parameters.py | 94 + .../golem/core/optimisers/optimizer.py | 186 + .../core/optimisers/populational_optimizer.py | 182 + .../golem/core/optimisers/random/__init__.py | 0 .../random/random_mutation_optimizer.py | 67 + .../core/optimisers/random/random_search.py | 80 + .../core/optimisers/random_graph_factory.py | 70 + .../golem/core/optimisers/timer.py | 83 + .../golem/core/paths.py | 40 + .../golem/core/tuning/__init__.py | 0 .../golem/core/tuning/hyperopt_tuner.py | 174 + .../golem/core/tuning/iopt_tuner.py | 263 + .../golem/core/tuning/optuna_tuner.py | 135 + .../golem/core/tuning/search_space.py | 61 + .../golem/core/tuning/sequential.py | 220 + .../golem/core/tuning/simultaneous.py | 122 + .../golem/core/tuning/tuner_interface.py | 279 + .../golem/metrics/__init__.py | 0 .../golem/metrics/edit_distance.py | 117 + .../golem/metrics/graph_features.py | 59 + .../golem/metrics/graph_metrics.py | 179 + .../golem/metrics/mmd.py | 91 + .../golem/serializers/__init__.py | 2 + .../golem/serializers/any_serialization.py | 20 + .../golem/serializers/coders/__init__.py | 7 + .../serializers/coders/enum_serialization.py | 10 + .../coders/graph_node_serialization.py | 23 + .../serializers/coders/graph_serialization.py | 40 + .../coders/opt_history_serialization.py | 123 + .../coders/parent_operator_serialization.py | 19 + .../serializers/coders/uuid_serialization.py | 10 + .../golem/serializers/serializer.py | 380 ++ .../golem/structural_analysis/__init__.py | 0 .../structural_analysis/base_sa_approaches.py | 68 + .../structural_analysis/graph_sa/__init__.py | 0 .../graph_sa/edge_sa_approaches.py | 316 ++ .../graph_sa/edges_analysis.py | 77 + .../graph_sa/entities/__init__.py | 0 .../graph_sa/entities/edge.py | 17 + .../graph_sa/graph_structural_analysis.py | 297 + .../graph_sa/node_sa_approaches.py | 326 ++ .../graph_sa/nodes_analysis.py | 81 + .../graph_sa/postproc_methods.py | 72 + .../graph_sa/results/__init__.py | 0 .../results/base_sa_approach_result.py | 21 + .../results/deletion_sa_approach_result.py | 30 + .../graph_sa/results/object_sa_result.py | 77 + .../results/replace_sa_approach_result.py | 39 + .../graph_sa/results/sa_analysis_results.py | 135 + .../graph_sa/results/utils.py | 21 + .../graph_sa/sa_approaches_repository.py | 30 + .../graph_sa/sa_requirements.py | 54 + .../golem/utilities/__init__.py | 0 .../golem/utilities/data_structures.py | 317 ++ .../golem/utilities/grouped_condition.py | 42 + .../golem/utilities/memory.py | 57 + .../golem/utilities/profiler/__init__.py | 0 .../utilities/profiler/memory_profiler.py | 97 + .../golem/utilities/profiler/time_profiler.py | 64 + .../golem/utilities/random.py | 36 + .../utilities/requirements_notificator.py | 14 + .../golem/utilities/sequence_iterator.py | 91 + .../golem/utilities/serializable.py | 38 + .../golem/utilities/singleton_meta.py | 20 + .../golem/utilities/utilities.py | 36 + .../golem/visualisation/__init__.py | 0 .../golem/visualisation/graph_viz.py | 516 ++ .../visualisation/opt_history/__init__.py | 0 .../opt_history/arg_constraint_wrapper.py | 99 + .../visualisation/opt_history/diversity.py | 135 + .../visualisation/opt_history/fitness_box.py | 50 + .../visualisation/opt_history/fitness_line.py | 263 + .../opt_history/genealogical_path.py | 131 + .../opt_history/graphs_interactive.py | 92 + .../opt_history/history_visualization.py | 37 + .../opt_history/multiple_fitness_line.py | 155 + .../opt_history/operations_animated_bar.py | 196 + .../opt_history/operations_kde.py | 78 + .../golem/visualisation/opt_history/utils.py | 117 + .../golem/visualisation/opt_viz.py | 96 + .../golem/visualisation/opt_viz_extra.py | 295 + .../golem_integration.ipynb | 4775 +++++++++++++++++ .../log_time_lbn.png | Bin 0 -> 65886 bytes .../shd_lbn.png | Bin 0 -> 62150 bytes 169 files changed, 20848 insertions(+) create mode 100755 paper_experiments/large_bayesian_networks_experiments/divided_bn.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/f1_lbn.png create mode 100644 paper_experiments/large_bayesian_networks_experiments/f1_undir_lbn.png create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/__init__.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/api/__init__.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/api/api_utils/__init__.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/api/api_utils/api_params.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/api/main.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/__init__.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/adapter/__init__.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/adapter/adapt_registry.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/adapter/adapter.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/adapter/nx_adapter.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/constants.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/dag/__init__.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/dag/convert.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/dag/graph.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/dag/graph_delegate.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/dag/graph_node.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/dag/graph_utils.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/dag/graph_verifier.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/dag/linked_graph.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/dag/linked_graph_node.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/dag/verification_rules.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/log.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/__init__.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/__init__.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/agent_trainer.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/common_types.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/context_agents.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/experience_buffer.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/history_collector.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/mab_agents/__init__.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/mab_agents/contextual_mab_agent.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/mab_agents/mab_agent.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/mab_agents/neural_contextual_mab_agent.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/neural_mab.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/operator_agent.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/reward_agent.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/advisor.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/archive/__init__.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/archive/generation_keeper.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/archive/individuals_containers.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/dynamic_graph_requirements.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/fitness/__init__.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/fitness/fitness.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/fitness/multi_objective_fitness.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/__init__.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/evaluation.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/gp_operators.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/gp_optimizer.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/gp_params.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/operators/__init__.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/operators/base_mutations.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/operators/crossover.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/operators/elitism.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/operators/inheritance.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/operators/mutation.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/operators/operator.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/operators/regularization.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/operators/reproduction.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/operators/selection.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/parameters/__init__.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/parameters/graph_depth.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/parameters/mutation_prob.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/parameters/operators_prob.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/parameters/parameter.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/parameters/population_size.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/graph.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/graph_builder.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/initial_graphs_generator.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/meta/__init__.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/meta/surrogate_evaluator.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/meta/surrogate_model.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/meta/surrogate_optimizer.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/objective/__init__.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/objective/objective.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/objective/objective_eval.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/opt_graph_builder.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/opt_history_objects/__init__.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/opt_history_objects/generation.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/opt_history_objects/individual.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/opt_history_objects/opt_history.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/opt_history_objects/parent_operator.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/opt_node_factory.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/optimization_parameters.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/optimizer.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/populational_optimizer.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/random/__init__.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/random/random_mutation_optimizer.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/random/random_search.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/random_graph_factory.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/timer.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/paths.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/tuning/__init__.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/tuning/hyperopt_tuner.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/tuning/iopt_tuner.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/tuning/optuna_tuner.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/tuning/search_space.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/tuning/sequential.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/tuning/simultaneous.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/core/tuning/tuner_interface.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/metrics/__init__.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/metrics/edit_distance.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/metrics/graph_features.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/metrics/graph_metrics.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/metrics/mmd.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/serializers/__init__.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/serializers/any_serialization.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/serializers/coders/__init__.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/serializers/coders/enum_serialization.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/serializers/coders/graph_node_serialization.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/serializers/coders/graph_serialization.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/serializers/coders/opt_history_serialization.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/serializers/coders/parent_operator_serialization.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/serializers/coders/uuid_serialization.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/serializers/serializer.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/__init__.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/base_sa_approaches.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/__init__.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/edge_sa_approaches.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/edges_analysis.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/entities/__init__.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/entities/edge.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/graph_structural_analysis.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/node_sa_approaches.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/nodes_analysis.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/postproc_methods.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/results/__init__.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/results/base_sa_approach_result.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/results/deletion_sa_approach_result.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/results/object_sa_result.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/results/replace_sa_approach_result.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/results/sa_analysis_results.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/results/utils.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/sa_approaches_repository.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/sa_requirements.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/utilities/__init__.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/utilities/data_structures.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/utilities/grouped_condition.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/utilities/memory.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/utilities/profiler/__init__.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/utilities/profiler/memory_profiler.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/utilities/profiler/time_profiler.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/utilities/random.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/utilities/requirements_notificator.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/utilities/sequence_iterator.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/utilities/serializable.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/utilities/singleton_meta.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/utilities/utilities.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/visualisation/__init__.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/visualisation/graph_viz.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/__init__.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/arg_constraint_wrapper.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/diversity.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/fitness_box.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/fitness_line.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/genealogical_path.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/graphs_interactive.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/history_visualization.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/multiple_fitness_line.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/operations_animated_bar.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/operations_kde.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/utils.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_viz.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_viz_extra.py create mode 100644 paper_experiments/large_bayesian_networks_experiments/golem_integration.ipynb create mode 100644 paper_experiments/large_bayesian_networks_experiments/log_time_lbn.png create mode 100644 paper_experiments/large_bayesian_networks_experiments/shd_lbn.png diff --git a/paper_experiments/large_bayesian_networks_experiments/divided_bn.py b/paper_experiments/large_bayesian_networks_experiments/divided_bn.py new file mode 100755 index 0000000..3428b0f --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/divided_bn.py @@ -0,0 +1,465 @@ +import sys, os, ast +import prince +import pandas as pd +import numpy as np +import bamt.preprocessors as pp +import scipy.stats as stats +from bamt.networks.continuous_bn import ContinuousBN +from bamt.networks.discrete_bn import DiscreteBN +from bamt.networks.hybrid_bn import HybridBN +from collections import defaultdict +from kmodes.kmodes import KModes +from kmodes.kprototypes import KPrototypes +from varclushi import VarClusHi +from pgmpy.estimators import K2Score +from sklearn import preprocessing +from sklearn.feature_selection import mutual_info_regression +from sklearn.cluster import KMeans +from joblib import Parallel, delayed + + +def encode_categorical_features(data: pd.DataFrame) -> pd.DataFrame: + """ + Encode categorical features + :param data: data for encoding + :return: encoded data + """ + for col in data.columns: + if data[col].dtype == 'object': + le = preprocessing.LabelEncoder() + data[col] = le.fit_transform(data[col]) + return data + +def varclushi_clustering( + data: pd.DataFrame, + maxeigval2: int = 1, + maxclus: int = 4) -> dict: + data = encode_categorical_features(data) + vc = VarClusHi(data, maxeigval2=maxeigval2, maxclus=maxclus) + vc.varclus() + clusters_df = vc.rsquare + clusters_df = clusters_df[['Cluster', 'Variable']] + clusters = {} + for i in range(max(clusters_df['Cluster']) + 1): + clusters[i] = list( + clusters_df[clusters_df['Cluster'] == i]['Variable']) + return clusters + +def kmeans_clustering(data, n_clusters, max_cluster_size=50, scaling=True): + # Prepare the data + if scaling: + from sklearn.preprocessing import StandardScaler + scaler = StandardScaler() + data_scaled = scaler.fit_transform(data.T) + else: + data_scaled = data.T + + # Perform KMeans clustering on scaled data + kmeans = KMeans(n_clusters=n_clusters) + kmeans.fit(data_scaled) + + # Get cluster labels + cluster_labels = kmeans.labels_ + + # Create a dictionary with clusters + clustered_variables = defaultdict(list) + for j in range(len(data.columns)): + clustered_variables[cluster_labels[j]].append(data.columns[j]) + + cluster_centers = kmeans.cluster_centers_ + + distribute_extra_variables( + clustered_variables, + cluster_centers, + data_scaled, + max_cluster_size, + data) + + return clustered_variables + +def distribute_extra_variables( + clustered_variables, + cluster_centers, + coordinates, + max_cluster_size, + data): + overflow = [] + for key, value in clustered_variables.items(): + if len(value) > max_cluster_size: + overflow.extend((key, data.columns.get_loc(index)) + for index in value[max_cluster_size:]) + clustered_variables[key] = value[:max_cluster_size] + + if overflow: + for original_cluster, index in overflow: + var_coordinate = coordinates.iloc[index] + distances = [ + np.linalg.norm( + var_coordinate - + center) for center in cluster_centers] + # Prevent returning the variable to the same cluster + distances[original_cluster] = float('inf') + closest_cluster = np.argmin(distances) + + # Find a cluster that has space and is closest to the variable + while len( + clustered_variables[closest_cluster]) >= max_cluster_size: + distances[closest_cluster] = float('inf') + closest_cluster = np.argmin(distances) + + clustered_variables[closest_cluster].append(data.columns[index]) + +def mca_clustering(data, n_clusters, max_cluster_size=50): + # Perform MCA + mca = prince.MCA() + mca.fit(data) + + # Get transformed coordinates + coordinates = mca.column_coordinates(data) + + # Perform KMeans clustering on coordinates + kmeans = KMeans(n_clusters=n_clusters) + kmeans.fit(coordinates) + + # Get cluster labels + cluster_labels = kmeans.labels_ + + # Create a dictionary with clusters + clustered_variables = defaultdict(list) + for j in range(len(data.columns)): + clustered_variables[cluster_labels[j]].append(data.columns[j]) + + cluster_centers = kmeans.cluster_centers_ + + distribute_extra_variables( + clustered_variables, + cluster_centers, + coordinates, + max_cluster_size, + data) + + return clustered_variables + + +def famd_clustering(data, n_clusters, max_cluster_size=50): + # Perform FAMD + famd = prince.FAMD() + famd.fit(data) + + # Get transformed coordinates + coordinates = famd.column_coordinates(data) + + # Perform KMeans clustering on coordinates + kmeans = KMeans(n_clusters=n_clusters) + kmeans.fit(coordinates) + + # Get cluster labels + cluster_labels = kmeans.labels_ + + # Create a dictionary with clusters + clustered_variables = defaultdict(list) + for j in range(len(data.columns)): + clustered_variables[cluster_labels[j]].append(data.columns[j]) + + cluster_centers = kmeans.cluster_centers_ + + distribute_extra_variables( + clustered_variables, + cluster_centers, + coordinates, + max_cluster_size, + data) + + return clustered_variables + + +class DividedBN: + + def __init__(self, + data: pd.DataFrame, + data_type: str = 'mixed', + max_local_structures: int = 8, + hidden_nodes_clusters=None): + """ + :param data: data for clustering + :param cluster_number: number of clusters + :param max_var_number_in_cluster: maximum number of variables in cluster + """ + self.data = data + self.data_type = data_type + self.max_local_structures = max_local_structures + self.local_structures_nodes = {} + self.local_structures_edges = {} + self.hidden_nodes_clusters = hidden_nodes_clusters + self.hidden_nodes = {} + self.local_structures_info = {} + self.root_nodes = {} + self.child_nodes = {} + self.external_edges = {} + + def set_local_structures(self, + has_logit: bool = True, + use_mixture: bool = True, + maxeigval2: int = 1, + parallel_count: int = -1): + + def create_bn(has_logit, use_mixture): + if self.data_type == "mixed": + return HybridBN( + has_logit=has_logit, + use_mixture=use_mixture) + elif self.data_type == "discrete": + return DiscreteBN() + elif self.data_type == "continuous": + return ContinuousBN() + + def find_root_and_child_nodes(local_structure_info): + root_nodes = local_structure_info[local_structure_info['parents'].str.len( + ) == 0]['name'].tolist() + list_of_all_parents = sum( + local_structure_info['parents'].tolist(), []) + child_nodes = [ + node for node in local_structure_info['name'] if node not in list_of_all_parents] + return root_nodes, child_nodes + + def process_key(key): + data_cluster = self.data[self.local_structures_nodes[key]] + discretized_merged_data, info = self.preprocess_data(data_cluster) + bn = create_bn(has_logit, use_mixture) + bn.add_nodes(info) + bn.add_edges( + discretized_merged_data, + scoring_function=( + 'K2', + K2Score)) + local_structure_info = bn.get_info() + root_nodes, child_nodes = find_root_and_child_nodes( + local_structure_info) + return key, bn.edges, local_structure_info, root_nodes, child_nodes + + if self.data_type == "continuous": + self.local_structures_nodes = varclushi_clustering( + self.data, maxeigval2=maxeigval2, maxclus=self.max_local_structures) + elif self.data_type == "discrete": + self.local_structures_nodes = mca_clustering( + self.data, self.max_local_structures) + elif self.data_type == "mixed": + self.local_structures_nodes = famd_clustering( + self.data, self.max_local_structures) + + results = Parallel(n_jobs=parallel_count)( + delayed(process_key)(key) for key in self.local_structures_nodes) + + for key, edges, local_structure_info, root_nodes, child_nodes in results: + self.local_structures_edges[key] = edges + self.local_structures_info[key] = local_structure_info + self.root_nodes[key] = root_nodes + self.child_nodes[key] = child_nodes + + def preprocess_data(self, data_cluster): + encoder = preprocessing.LabelEncoder() + discretizer = preprocessing.KBinsDiscretizer( + n_bins=5, encode='ordinal', strategy='quantile') + p = pp.Preprocessor( + [('encoder', encoder), ('discretizer', discretizer)]) + discretized_merged_data, _ = p.apply(data_cluster) + return discretized_merged_data, p.info + + def find_optimal_clusters( + self, + data, + data_type, + max_clusters=10, + random_state=42): + if data_type == "mixed": + cat_columns = self.detect_categorical_columns(data) + costs = [] + for n_clusters in range(1, max_clusters + 1): + if data_type == "continuous": + model = KMeans( + n_clusters=n_clusters, + random_state=random_state) + elif data_type == "discrete": + model = KModes( + n_clusters=n_clusters, + init='Huang', + verbose=0, + n_jobs=-1) + elif data_type == "mixed": + model = KPrototypes( + n_clusters=n_clusters, + init='Cao', + verbose=0, + n_jobs=-1) + + model.fit(data) + costs.append(model.cost_ if data_type != + "continuous" else model.inertia_) + + # Find the elbow point + elbow_point = np.argmax(np.diff(np.diff(costs))) + 2 + return elbow_point + + def set_hidden_nodes(self, data): + for key in self.local_structures_nodes: + data_cluster = data[self.local_structures_nodes[key]] + + if self.hidden_nodes_clusters is None: + # Use the Elbow method to find the optimal number of clusters + n_clusters = self.find_optimal_clusters( + data_cluster, self.data_type) + print( + "Optimal number of clusters for local structure {} is {}".format( + key, n_clusters)) + else: + n_clusters = self.hidden_nodes_clusters + + if self.data_type == "continuous": + model = KMeans(n_clusters=n_clusters) + elif self.data_type == "discrete": + model = KMeans(n_clusters=n_clusters) + data_cluster = encode_categorical_features(data_cluster) + # model = KModes( + # n_clusters=n_clusters, + # init='Huang', + # verbose=0, + # n_jobs=-1) + elif self.data_type == "mixed": + cat_columns = self.detect_categorical_columns(data_cluster) + model = KPrototypes( + n_clusters=n_clusters, + init='Cao', + verbose=0, + n_jobs=-1) + + model.fit(data_cluster) + self.hidden_nodes[key] = model.fit_predict( + data_cluster).astype(np.int32) + + def detect_categorical_columns(self, data): + """ + Automatically detects categorical columns based on their data types. + """ + categorical_columns = [] + for i, col in enumerate(data.columns): + if not np.issubdtype( + data[col].dtype, + np.number) or np.issubdtype( + data[col].dtype, + np.integer): + categorical_columns.append(i) + return categorical_columns + + def connect_structures_hc(self, evolutionary_edges): + def process_meta_edge(meta_edge, self): + source_structure_key = int(str(meta_edge[0])) + target_structure_key = int(str(meta_edge[1])) + + source_nodes = self.local_structures_nodes[source_structure_key] + target_nodes = self.local_structures_nodes[target_structure_key] + + source_edges = self.local_structures_edges[source_structure_key] + target_edges = self.local_structures_edges[target_structure_key] + + united_edges = source_edges + target_edges + init_edges = [tuple(edge) for edge in united_edges] + + source_data = self.data[source_nodes] + target_data = self.data[target_nodes] + + merged_data = pd.concat([source_data, target_data], axis=1) + + if self.data_type == "mixed": + bn = HybridBN() + elif self.data_type == "discrete": + bn = DiscreteBN() + elif self.data_type == "continuous": + bn = ContinuousBN() + + discretized_merged_data, merged_info = self.preprocess_data(merged_data) + + bn.add_nodes(merged_info) + + all_possible_edges_between_s_t = [(i, j) for i in source_nodes for j in target_nodes] + all_possible_edges_between_t_s = [(i, j) for i in target_nodes for j in source_nodes] + + white_list = all_possible_edges_between_s_t + all_possible_edges_between_t_s + + params = { + 'white_list': white_list, + 'init_edges': init_edges, + 'remove_init_edges': False + } + + bn.add_edges(discretized_merged_data, scoring_function=('K2', K2Score), params=params) + + learned_edges = bn.edges + + learned_edges_list = [list(edge) for edge in learned_edges] + + return (meta_edge, learned_edges_list) + + external_edges = {} + + results = Parallel(n_jobs=-1)(delayed(process_meta_edge)(meta_edge, self) for meta_edge in evolutionary_edges) + + for meta_edge, learned_edges_list in results: + external_edges[meta_edge] = learned_edges_list + + self.external_edges = external_edges + return external_edges + + def connect_structures_simple(self, evolutionary_edges): + external_edges = [] + for meta_edge in evolutionary_edges: + source_structure = int(str(meta_edge[0])) + target_structure = int(str(meta_edge[1])) + + for source_node in self.child_nodes[source_structure]: + for target_node in self.root_nodes[target_structure]: + external_edges.append([source_node, target_node]) + + return external_edges + + def connect_structures_spearman(self, evolutionary_edges, percentile_threshold=95, hard_threshold=0.9): + correlations = [] + node_pairs = [] + meta_edges = [] + + if self.data_type == "discrete" or self.data_type == "mixed": + encoded_data = encode_categorical_features(self.data) + else: + encoded_data = self.data + + for source_structure, target_structure in evolutionary_edges: + for source_node in self.local_structures_nodes[int(str(source_structure))]: + for target_node in self.local_structures_nodes[int(str(target_structure))]: + source_data = encoded_data[str(source_node)] + target_data = encoded_data[str(target_node)] + + correlation, _ = stats.spearmanr(source_data, target_data) + # print(f"Correlation between {source_node} and {target_node}: {correlation}") + + correlations.append(correlation) + node_pairs.append((source_node, target_node)) + meta_edges.append((source_structure, target_structure)) + + if correlations: + threshold = np.percentile(correlations, percentile_threshold) + if threshold < hard_threshold: + threshold = hard_threshold + print(f"Threshold: {threshold}") + + external_edges = {} + for i, (source_node, target_node) in enumerate(node_pairs): + if correlations[i] >= threshold: + metaedge = meta_edges[i] + if metaedge not in external_edges: + external_edges[metaedge] = [] + external_edges[metaedge].append((source_node, target_node)) + print(f"Connected {source_node} and {target_node} with correlation {correlations[i]}") + else: + print("No valid correlations found") + external_edges = {} + + return external_edges diff --git a/paper_experiments/large_bayesian_networks_experiments/f1_lbn.png b/paper_experiments/large_bayesian_networks_experiments/f1_lbn.png new file mode 100644 index 0000000000000000000000000000000000000000..c0e198af9869f02116496dbf721a264866e41d66 GIT binary patch literal 57902 zcmb@u1yq%5+cnDGZgrb%I|&s~Lf8@_ZGwxIkPThjraThZ=7?+f5!RO*n5bw)_R`jj_aCpUh}%|D#}Z(TfK8N9Ua{|%IT9zbaa1p z)6p&C`+Eicr9UJ}1b+zGo>I3}wluJHxL~bMCwIa2vYDl=nejztdwpvgV@nG@?!(-? zT+BwcwwG;$d3enK^9F89YeSv`ecLbMOIBSztzkn)M}L9*|3{2Ov@sps=RnHIV=9i% zhT0rlJsp;c$EJn$AK$p={vEmDg9c}$X$_%i7Ndl$YRFCl2w#jUA+>orR%V$_AYsJOICA)I{?&{aqSK593bc{jB`lO&ms}GAOqnKupuw7Y-CjJ_u zU;e1SI`qV9dU`b_C2C$?Uf1w&BCmc~jQw!4Npo7P^dqh(`Zhhb#okPfajNlYmx|Xf zU%8S~SXi^Nvhv&5*cpEg)%pYtJ`9qyg@uI|w?O0fzHsk}-Vl%G`GRiSgPLiIii$of z*6obIjfmvz4t(#qD|)Q6I9fQEM^`*MH@E!l+Y?(j&#Fug&}bRvS~Ih=!q27Y=oqzc zSkuu()Ojq9Gc8}fT&!NmqBSd2u)|eZE7L;Gj+>ufB{Vct*t+Y^BOcwPZ{6jlGvB)D z(}SNeb+D~dX{T9_~_!ri?)mNPKlc7(&19x4WC|Lr-e(=uT)*5=OJj> zUSAO?xP_51NjX~H7+*5m6KGkVqA%k%)nv$_5>tI5aPG|Cd`v19S(0?G%yCy&&KcK6 zP~vbOPkZIql%Mvl-hAM>2`w)6{OfC)=_aa=8te?28)MGwvm0!X!vOnxdwVD87AE=a zS1{?V4AM+Bkb6`asH~l1OL3cFp1jT?VqlugSY9n?(G zyL;{Wbpi8c{^LTQxH-=z_=JRnu&}eUQyP=Bl`mg@F*V#0EwMDOeD&&8&3xx{%v<6$Q!H5hqJppfah%{yE}O%?VPl-hwS^VzkaTrYp-v7Ak(yw+myKx zBSIdTy>c6ux?<7&ZEE+o9-R1mn~8&&nYp~D^6tis8~Gi-o?pIVMcMg(`#QZP8aR{l zsBzl9drx(ZjwUzidT7^0D>S~jxrRZ}e1zZN!*uH=B9=lLgpq19jY?e{Q84A7ntROMKz?n8U0z-F^YaPV0PGVXlOV&+M$cFNEItv zX1zANCZ@n`ZlKMf!=L5U$&(*lw=&StoxbajWu>=CE$Q6L#b;Xr*PX*Gq#9N=crl1< z-n=;?P|z~&^5-{XjDnp9*wc;cBz1IjVrPrqzc-m0YSPTK2&xPciT?Oe-O1TG4ud^V z8+kw2@ylKQUYCK`Xch{e!3RaGGyY5O*Cy)ZYi{E{SDkL!sAPU;0~1s7@-d617R09Lmq(cfN6G}4yuP+v z*lysVMSET>l~pDV>tmoTHxs+d(B{D|K?@_?X7h^+*Vpep^4!PgA67QDGVET}__MUs zDsh*p=}1gSbG}Pvu*aekCSOu3!>k5jDfGnc_3XU7N_GSFFAD^xTlzki`Ex!!bm`V% z%l3rv-m1Z#ia-wa7taxXRI;u01&nJ?4!2}x&g2-5jnSu03 zg%I(l5*`JAE?btaSGqBOyiz1~@xiJ!YwB*T+m&Wq$3_Ni!-fsKJ3UEU!=y}nE9Ys+ zn-rKyrJBuuAB(+Km>cri{>!^n<{w`i!${OU;WMwe2Na)Qne1f{vG41T4yl|_FI-y8Etu)q)s|z& zSu`dSz};UPDbt)|7r%GkKIMhZk%|5~Y_<>t59Qs3t}`hO2^!%7rVV{fh9R^P-+li3 z6@vMD3!Nw9a%}sXzHntRdyauZ?wfXZ7xnemmPLcE5nEcAuK)j@1Iv@XJti9 z*RoP_9Y!;5J;Mk|^Kr=}V_o!l3bt2cQj~Xl-@8|fAJT1l9kU z`R!yelSjl6`-#3YZdioEuHRo2%zu-?6Ui>jPAWS(W~Emk5~n*)nwYm_$YE2S#i))7 zRkba>3Wz}P9Ox?b(|mdHru))@gOiI(^uRQtlL?+b*6a4|+ol-)<}9o5#o0lgg^4-^ zYkT_w!*vq#U$3*tKE7Oa*jUJIHqCizD7K`eWOMPpefzfT*bz%Yep8CR^Ow8(m6l6r zObU73zpq>sEQTp*-(2kHa9-T;>vKu3-A`f%bCo4K;t!iQ_r)g{)J_gIrW@7lb^H2y zrLkAOv?2TTK0N z;kn^VzReLW*7M)3#O8)lW4-pXv1#hI%#8P%wq{wG;Qu)L03O}5-i%@iRpGK*5iet6 z4g?9=NSZbzoarv}SIu{x+-#26Jb{Td#>}yM6&Du|ejn=;cAwAcsR~J6wTUfaU=3Y~ zHo+VvY>a>jBzD#g_coq7bt)EPZSCMtRec1gp+4qJI12@V;Ox@kf@-0=+m-{0M{8@-3JVXXMJq7fym_;!nHSXvZ3k^DMX%q2mh5j*3Ao4FIlE%9nAqE0-cm~+g@yBk+y zBV#WK*!0LUh&ihy=TALFR0%MsPQ|Tg=Q*B1EK^0w?C$N2#&s9IcY2dZMg}NKF3@D6 zug2QSs%E;wBiVU!pwx(;%vS^R`?D_`qDhFs-XixtJJEkB(Qc$Qdlvc6cA%bp-45Q9 zsRk9&>9yr8Em~sE6N=YYtW!x)KeUBI<+MSn1~7_dwsmArWl%iU44Ko0mM!K@1#Y=- z-n^OU@>3X~sp--#d5c9oe*9QvkCdkP(!9+nNy##V=M%Tqp22q$L>3Xj0uW?8+MYi< zm|UndSzcB(rl#u_SyFPE1TugRe=@8itwL_&6@tyFhTdDb&&6bzx5QEW*!pwWw0OsL zkW3WF`Xd3Ez`h`d_Lne7{S;ru(f(s}Dl#ukrk(E2=()i-S z3G9^Ps&Oi3LXY3IVfEjC6EtU*^l<*`Tl#eCZfPMQp;+vzfsZf4r5_#$C0h(bvs1)Ap)y!3 zzEQ_TPUaEU%gUf9rhn0ie&M2{>)ME@nxYb;_*yCS`0{dK(1Sp~FJ^RQWo2{Tw`AKy z+vNdo=u1W+_3KHob0tKtAG4C!@^s83AR7upNkuO5j)#hqU z)>TLy#@#B-i({n^=9xAo$!R!NMh>|kFhUBo6{{@vkp~c>v}BM$=qDM zg^MJ)lTuPGW~o-4l7s5E*E8fa>PLU86d6^YyLJ1vVa#Y-Lz1>BgDLjq;TG8-VOq06 z&xa?ml>n`+&WqS`X!RBj^&E%M;4j~? zUK20PFV2nbh~wduD8^K047KNr(6%2sbm$RAP%HTmALB$miQ|*4KwXzomjI#?%rh)I zff3Cp8<|$1*z&wv|_ZmRv=V!j$=TQo(io-1H zd6-Qgy7|vekMdv<2QIZ`3%_lD@6D7v(w3{Gme0b*HqciSuKD8p)m!Ts;t-1!*5Ob~dKTeQ$nEa#epqnqvc3J;__K*DZSGYaj-BfQtL98Q3i9prD}jw`pY6A7 z&ucLBYBT@z>S}|(5LU9Z&4V<4(*{{U(|ewtGyv*4Geh1d6PRrMfhVRE3Z)@r)IsEH z+d|4Wn^v%@K+c2O+S-YnuhugNKOC7gD;;uC1ycd~r1fO$-QULZh-C|p&bf8F4!b@V zt{bczZ3*glJK>DSWu^S_#~=6Z-!D(e8R_Zjioh3~md-AfPkHz5BxT3kJSCHffw^rl zDQeQ;E?|z1%dh~Y7JyOzr8+PFN;>A=PgAONbSb{BGvADn4F)<23(qEm$#IG4LUXNJD-vV-2*3qF0{2@R2 zEKx)ZV{RNwb}!e~``>yh;&%#|9AIN(tH{7&ZNS7c9r<3ub-ySKo0>v>DrlLM=QQ4I zX87;+`Y5?wo=g($vzpDTm+cZTVeu=~vw5(QRa)Ice$c*TerCM9yj;fD*Oxt5&>@p# zgtu?s_6UJu2w1kC!vbi;nBx(ZmK!3%KNWFEvL9)U#lGvu#%S>QA`hM?liDEd&r#ag zC|I#`G!op9Lo+P~aBLEAwWPRszjp3A!^z$wi51K&v9eEisTQOBv&@WL`Gf}&pf4AA z7@(QA$flPtU%)-B+TrJe7lwDbWr{3|O6w7|?ep2nrT+ZN6*@x65u6QDg-!)xUbM3= ze=ew!&kF(bFae__EE###Y=v9tF2~kb>+Z5yBsu~8vdtvO6Yx_zKf-n&(C6sUD-kW; z`?4BC%uDyU+uEja>*U3N2^e&xK$7_@bT=Kd8UgM*`HepH@z!eVTWuE{~WBU_dpx8}Pz+!J@mYx*@60@6mK}tI&7svfc7qd$T^N?B;7csi3g!9Y%lgLv z7zYwF+m*n%b=+t4Ns`7#)NtkcEUQcY+}gsbAqk7SJ2+Kj+g-bkso98?xX@l+l5$&^ zY{Vw00}LS8XD}NJjDwq7UNZbpkykN&OHjxvt@!gEz{mhDhqUA0j!e$vgk)tM0jfET z^i)wVI$AK7F+`g6W?~j`V z*@58TBV#qkK}Y9(1q(C=IR`5_SRlqPLL=26(#Qd$L!kx+YTbHYXmhG~VU{hVu#v+D zOC(pnhpBnhs#QBobn_go2CGW;(ML55Hlq#LLvkVF+TIe2 zSq~(`-@JXBez9n^+gQ;yVDfbEP7`f^y+O^CFE*~z1_$U*7<6#rNV4UiCO@aNT!4sLP#Xg`SclRkHd!CcGJyT$aAt=INI>(;OSdP-?Rpajre|H- zsvaM@lyJ3myhZTA#~1v)vD&#A1l2t{sKucWEQ$!zfSe(K$W8$E zKy~Q#!xpWN!xRioymMJTM0e$xCs71MFSff)YeTI%4UAalyH85#S$T*=A+YY~4*rkU zy%O`Ina4S1TqUm;>Xq)f0loq5?;m?;q_Z~IC)mZs zb%5j>5l-sPU3k2EQQQm>R^{!jb#}we0?YsU>i{$JQ78@#xZdV}{&@~4YlQp(tn-Lv z&fP0j+Mpv|4>C~=b|3+p45E&T0w(p<;GGGaCc2KFZ*nD0yN|4u9&Np=N;RkoKG<*n zi6G#>(CEaL2tKUP%2~75?hCX1`P1#GO(X6XJh0FvFmfbZ5Ook?EZw~2@NiSARJ43h z?Ug^4@!JonoBr$EJ2wub8HcB!cRl!Dqx7w-^isav=!jM?)$*z^-g@hOo{fG}e_ixw z(-in4HR58?J$)lDz0ynpoqT73qSS9GN}woR{r&9yQ>(*1BxRLiTqn2Ybq&hPd~>?~ z`h255CF7Zp4V!lFXr6e#M@S&}c+`#(#I%-!4`t3iKl%q~tE^@IXGZ3RRaI7JkK{N(OeT~%s? zwV%7`h^JVY)CqFlKr<{26#8iW<=tId&)$K7II7S~627KJln2KcAVE7#Rk8p)O z0BnONeeIlRR<3|Wt9tG5pgJ;d)cdRuyek&Fi)tZZX!hpDs{Rod!|w+FZcm7JBP+D0 zY$q}o9{n^GSm*02kpl4;kEI2{idTGl_8bE@^09R(stuy=RB&f($Upu8tj$0>^vMK> zjh4fv{P8J0mtRXr1cU{2ujK^b=LI)!%-R7%!`=iQ;dRPUvM(32>p+85cB!*?c#O2y zf0mz~b7dWU>XKa57t~Rg)?D=NU0;aDVs`I2<5-ngD)!QT_U_)Qwl;0t+e6#A{E~!; zbNlF+4Viz)6pz6Ukl>xFJYGoGVpI)HLF(+Z9XKKWA<}VEX2*(|MsxohMrzG*ow2^v+Yeo1 z^1@JOv3KKx9eg8A;usJ-O3m|Q*OhjULXeASp}(|*SZ_8Jgk&`-tmE7txwi2s4Vi+l z$;KXn!D%@*gKe3I*^>F!{QbA0ii%2H0W^{N2#Q}yR&#yQx$KKJi+Th~t=maeJ1IT2 zu-nl8LxGwDK}Aqc7_|%CwJ|VGIkS4KsZ9wO!~w`}0k(5XT#lWdf=cN!I;^?w3(kR6 zC(q?MzO-A}pT<(yF+I!oxpIqN#x+6AZgRnFSuktX1>Hm+EHXWlNoh%kZhT@C8ZN+E0*G>6yL&0*(Zv&TS=5qIwVo&WvI{ehbB+Ypsvk?+bO zl_1LrTYb4f)Ib7r!X-VG0QPzLrP}kH1l;ExC;RK9h>!}_n2rTNmNTS8-++Js=pLJQ z?@oY*VGOFVdi82)JCROh1+hrN>BoC2o(kK=A%;)l;o=>~zF_&J5Mqn~Dou?$n>g^* zM=FOwnp96fsZ>J5-BiL(u7a{*4665(&(JHPB~z?@;c13xqddT!Dge>!)Nnjx9W}&J zB{Q=Glv|Rq2sBXpfZ*hAl*uTQsm3`49U9u13hqD^f+pe4NCk#f!8A-kVn-p-?E6ef zg0XQIbI2H zS~&mheg(sg7HGgIrJTdA z!BFl2QQO1H%G!@PB>2~E)7@N4ZvcjL#Fv=D#aWpSkEQ&nk+%2%9cKz8V*kLvKmuez z@E5R8h$;ej?9Y)WQzh^xo%IS*@~Sn40KHSfJ+HzB-+<*isWvzY9X0R$gB^aMV}48< z88>bWEA`uNiU@9;F*!b7M=vpde%whNRJIYwVgi5x_}OL_QPAA$Oo8jn=0X=oMHzH3&141>>PTC^$cPVMbI!l^fvpQ8o+At6&yMx#)AWaR6(g}l5xk%GjbfAWV& zeIJNVCNh>Vh{D+f^$Y9%@$=`=)xq{kNs>>G#I*?>CUPFM*3G@YYIHKxTl7pxNT%#4 zq(M`y%d+ZJN;7Xv3fd`Xk$`)z?-9Lw_b#E%So|Rd2R(NR)cWi>p~S!>=A5>AD_0C) zsWC2#5|8rJOAk5Kxa$zT949uE{Xyp=w-$hys8EOyWIYFoyJ_y(vj^B(8JoSkyE_s) zAkJyLhXA#6=g!scCgcM3p_>Mk&j@QgaO8+ObTz~krC6oN2?$qO4r}w2A#J1qeO3gE zX$>|c)`3!N*|u%Jf`S5Ah!%lRn-oNSSsS%-Y>f|Uq@08XRsU4L)V8mBk9M~8sjc@< zVa1bb(?COFJXS}u^DYwuM`ve~p^q<#j*8(6MR4W7#X;UxgGHUl@+ZX`1@V~(6#ewa z?J9t4q7*{NWSK%!CZvmS7(qcn)dcl3K&i?gg^5+y%GMVw&YB9CeNfftrq=DH;{)-p+*v9Og0&+ zF;ERPCEg+_s}LkN&(%C`^K1+6oK9P;Rp)yS)ReAWyY^7fQpaxS<5(ySuu}(8hc2Ew z4r$$_J|-06c1dI7bnpVkogaaNsKjKTfGo)8{12Qtjx9#2eqqqiUA)FVUh?h-`meSu zT!;b-B{vXq)6pRp#lu0QAR6Y_uDfgtgy8t&H&cK>;j27e(cz>gs9TU<_z(Emig23ab#1CElQN(wAH`JWYNvnK@Kw0)pTkQ|JS8x5yPh{f; zAS0WTwL$hg6!Qb9Vb)r7(9FwEwDazVo$DJaUIK=)6&v% zxGh%=Wk3$)s0Yvy_wn!~uRMsND0yW?o8>Ack!ivb#EJ;xP>nMJECygOjw%*$o`^vV z-se{u0pz3#-OG=KAPRy6Ao}sUe!uL=AaXdK`?u|Zz2%lQDR7VF)_gy|t4JPlJ~kP6 zcFGECYS))9VW2!jRzoq|v+|ixcrl~P(AAzlf8NUoD?Y8k4paviKxMJRE~U+YSCm5g zahV#b!8+mHQg%vGM{O!^!}gz_nL7XP>Gt-Kev?4XvitT@qQ^3#=Zw9?1d^?=$9 zYlxlz>Jr=slai9C5vY0TnRrcb%l-B9>ei|MYJ#J9Iv=cCfOs@nadZ={CFT z>FFQAZ)us&6wHpoHeOdIc!hK5Tu7alr5_vFxIK91#*NglgFnCQ>kHgi5+tF9-ckl} z*N+B)U7bjv=~~)9e_s-W%k>ToC4i(*sK^h&1FrB`{Q2jfEEFW?G*k0m?{l-R=j$d+ z1`(M!3}#V0P<}QD36ALZ*roCKTH^AEXsLQKhfwPO@L>d`(ZQF+j5a6xe|(ExY(&IK zRJWu8{t!(8&6kb3y)8(;MR5dRZ0DhiHzub+icV9p)uVwFG`+oC@m%HhyfItqS8j3Cr^0wv4Xzdc9#2bNl`N^t3&thW=Fky z_m@<_pQ7KMJ$u&ObM`1Vw}pm;{Uh(krwEQ%wsL)hFyn75ylPed(g`B0#z9<;k`E%= zoE;n&{8bg`kIF9I!+W)g{TX?+MzOJF6z4YG4d7sEq*LrcUar{f;A)f!^ z$6Yzj0>`N-qL2CD5S>r>3+eSGn)JOQr$9P?ygu%L5clJC z-jKLt$m>eb&!9zjWw&uhffy<`Nh{V3jP@zjMpFE8@K=jlb2o4N6_39FAa$OISxvv@ z*S8!!eAr0QL%>)`4Jh8qmc8V-ychS+Z>6gpYFfjf{pZk~U*06g0exNwwq3^+S!Lg$gxzD;Z;@$TCtN zRcXm!NT{-J$q|@57#_fL+jwfI1L$+TcERo_psyISrec6j7rUYF&t%z@?@e*4Ek>RR zmy9uQS-(^CyE%fs27>8X$M5>c#M+(33e*S{n~|1GsS~&I3>bPVLs$SXv+o5{E4tqcIph)C9&5@11(SCQI3LFX^o@>IyQSmMD*0<>7*}oDzZiDqiWnkn z0#a^->&&^$n{6pAnU*zE!&>}$CArRHMN1K;39ttB-RHSgW_B2xtql5a@#YO1l4`;W z70-Xl=AQ%5IXPM&auQ;cf3oEUv9XhUZCM+_uUyFon0pSQ6tlEA?a~b-tlAN;7Ea5w zRA|exB8*(A=$^lMhD^`!aCnBfh*4t_Lyk48(&V57H{$;wqNsGl0G~iQDcG zsH+WR_v%z@S?2TXk8Y`vR;*^tpKLff+@5~|tQX>yWyC#nq}xHk%=zj3 zcBM$^!8(Q8rq$0E$xyWA7Vh;}N^>w=w{G2;qpx%-9v@PG(nDn-WM(i~3VNgdQ(paR z1xuhI1s)5N(O^Nb-?}+ZQ@?at4h(0mM0Q@-DVkMCTrxs7TjNe|kf11m8tS+tW+InG zuG`fRs~Vp%ozEEYZY2K4M*8IY^?RqM@^Wh`N{8$01}4IQAJSpFne&2Jq=aM@rW1lzM1dLWtV!rv1+DA3S{ga_QZF$!E!F# zBUHXf;gR9tONoO6G$SXM==F!0Jo2pKF2Muy!b>Yz=dDUFY6t9;dw1?QR@d6F{tA>| zPpe}ed+wUf|MKNat=XnsyKFkFAU?BD+}%%^2lFVsn;QH2^)f}NCM=_B*V+{`Gh1lT z_!G=WA2zsZ6t%SMFB-@f$9|YW+)2~B8)=hMGdFJ3$Ny>N z@hh@2w}`mUKQ5Rtc!=zWi0}e5h(otyZ&?6O2#^(gNePw(J6P`*QAZA~yOrh@ZT003 zSV5*k9|V3d`Y^9tHbR>|_Wo4!FURN6jLZvfg*CXg+KtRpQG@VT>E+SRc|7tGh1Y~N z+bTI(Sl;!_`1pL{pWQ-bfe0EYWN+Qkm2p(_j_@CJM>8()@9gT*vn}zE?0h@p@ecpg zkOg^w1$8DKTK4zvj^?dd;%%2`2^w0|G>f*KfxUT!J8u_M49qtlV3Je$u zW35@ZlG}9K_M0~%@|}k*7=%URA!m}Ep_yjne$Qql{YYbo2X|UiveWpZZ<7bZ?%cVI zIXiJLNcQ^m>-_HX-qZ+4O^0MGRS&AfykG%TNYyyZw%fY}SovtidH$VxyifTrR{XtO za(T1eNNsSWK*Hp|J3LoBsp$7uzYfOQ5c}canrq8#?7nravg`l$01W1!Uun#;l~Y7( z(axEdTC;WDH|gc-ZBw+lQ4avl+k>~63K~-MJ>q1aq};pncjN2pbCb0)JcHg622ste zui$3Ub(_@9pBwoH98pok!9p3KaPzKR@%b*(2lnjIK@?2_O*iaMN3~a3Gy5CQ!t9_r zv~w-6DBkUBO*$5LfQ|6U^j=#a+53ta>%Sa|``;=E=YTx;RAj*s^R4R%qDz?lXnQCM z74b)}UW?N$yy`qXl6-yTw4qU6Lt;zpjPqc7SI3Tw?KY z?6u`$N|W*Dvg4p}B{rpcG*$++q&tmEc9(W0U=hX11=G_nnhJ{^x z_}D08uob3Bf33`on|JKkYiTKgYYp{TGaARAJ=j>{`<9k9wrO_M!{f58(B&5oTr=1H z1VNqOWPci-eFksby2ZiEyO%bGryPVUk7_&jNaNFm&!jkvm(So(%XISl2Qx~rGzB2z zL5G6j1&U=0=Tr-F3{b2{mUNfi?@ieltC^ldlFzc0e{TNgg_@O>6`+#7^;|3Ww=c95D9y~TG9^zr)KANzGzbpPjU*#G6%ezyA;N`Ju=Uc($-L&~HT zW*H@B8Ao!~GME2CO+GT)Jy$q6W|iqV+gd!6k>-V?`f&O6>_ja=)5>Q%puIhWCd@*) zR4nfksO3J~HkS+v6}4%<7w3V#FhI^*>lX+aFQ;Y)^2g3C2@I7g7Q%POfrJ57!7oI& zhAVct*NLCFqn6q$CcUE$<1XY}TVRdb5I@}r975jTG#z$@o^TD#+g8EuPd^pAY#sn`W*8-v1? z>8mS(O3&hzB6FkzxZ@F4ViE3lip{PW?kKD}=D&CUZ=ZX%w7Pl>OYF(J)YwY6zsf6V z%$dI_S@xTU?3w47*F|mN87`6rK=c&_KW;vp`uri^Wy+Dk+ZT`AeTM`bHCo4G5j2Gs3||#sv2t$ic?t9fCi<5ZJ8fhrIcqM;Im5d$c&@yDUY2ukYatIt@(CRI3rEzhBN0);WS z!X0{n!iExx5^hqZaIsTbTDrEt&4t(FP|3f)cg{+9i?3I7nm4EUu`K^i(if;lF{sZf^HKijE8n zuu-H>-%-d_*Ax2nkt=2;7hme7gba5MB|Xjm#jfa%96f#7r=l}7^ffgC>PK`9&k@hg zJ=BOdH_fL{Op!Vtd3~aTlP9lt3=!v}XrV#HJ^tCmEJF!Bp5OFq{R_R)H!32*m4Sj< zus-?5#|H}AIyM*-!qJLyxB$BsOvCblhZ6%z^=jAfzMX zyO~W5)<(vYiABlb?ARApvPh6W9>EgT@LXz|EDQBBD*0ZTEO=}%z-^aB3p zDh-2-8n2ZZY|{GuE?43PdhgfNrbJPn<0s!u@;eCP@uvSPuO#|?-ajntaP#;U{=`B4 zQ>Tg!9I7#E%>IL}gd@{}^#CvL$@X^lGQZ*`gI$Q7L>Q3uuB=O-Qa_-I@89%6BIon= z{uv8azU?ebOw+cW<-eAs^{UjSu)aQZzyqlOs)a%7&7v%e?!$){0WHLc7ME8888 zC&h$Y5`V0VU#6HJKmXrwaXLEt-!zEs|CO}=FK>L}_q%0i>Js8M|IXdok~vQ*w&Fji zF#lQ4)~hf`3iR7{$gz>9?umfpQoP~Ap5i?}1uo`*e|N6qeJXoTecspN)H6F|Uj3A}Lg&9SPYO7@;B!9a#xoPiWtXLHLN151JjI4XO$WMLw{Gav!yP&0;m~mh24! zn;+zB>%ri&jY(~&qKKQeTzD&90|hwok7X+&cZgfGWSnAh|DJ(6er{aX6LADxHS59C z6tD|xJa;v!zIcAyq#+^EqRlx*DUw5D(Yed^H<(d7pj!||G7G>F@;epAAe2UBPE*l+ zHC)|Hc;?k*a|1(7eBZu)RY4Ga$P;ou>D=9Vqz++QqZE|4^&zrr%{>!wI1AZ89a>5$ zh8wht>gPv6Jfd}#ZbFZPGbB0+94P@T>6S|mP{+1}M?thg(W1Ldzahh6b~saaa&A-^ zPDnyiXJ)t7;LU073z`VaI`I46MNFIeoOeWM^KW3a<>_HJC6b%~ugR?o+q=qxYS`Lp zP$NKK`*F`EA7A(a*@+k3JAyJf@S0MB^L_8aSZ{G>DcoqkzuGJwSdEm_DVfU(4qYlB zl}6Z59A>-xE>I(oK$kooZ;g3-n;xKVfZgN{EIsfvKEwya7HoG7j*+}O-4R{VB$Nyl zVN@kq=fq$($}mPB9w(zB7>AFpIMEG+3X6=BV1V5=&+g%I4; zPY6=5{j;xKo%70%8;`=^3ne@nYCkuE%q9$wxf0av`?fmyMgJTeLu!)QoX4CI^7d{EAD)aZCNIMOh z;EPdMXQmh@amW^!3oxIr5;K*QZY}sBb>bR}i*~4fC-;Hlm~@qV21`L3)(x`4_6Hk9 zz4r_(Gn(&cj{Ee9nNo{RnIz~k^TC&-q@*-FxB+~CzSJO+VE)e}YPJ=~O?H)b0-Z)6 z3&pxGEG6DT4@=y}jbB)Qux~Hj1a&KllH>Po*>g@Kzx7I=b$N@k=f$R5*PsTIx}H(= zImB6`?y|x(>u%k+bJ_QR%noeZwv9q9y7-7iz^M8(e68T(V6;V}!z~g-_QJ&Tm>vko zlTMNgoPuJj;b#2eXQ>61k3e97xXPehGzdYPzf|l+q%~p=C*f^&ih-HJaqgp9Z`Deu z!iM#EPUUMW#$fC1Cv>nqe~E*OOGQVgT@&u(i2BT!tp^kpA!0xpfRo(^xiiEh@6#t# z2>=fazdPIT{g&~4(ft${DX3cyKD87RH?E!F3UUCF)qL@Z-pSE11~?x;6K049C}}0c z#Bf6Le}PSFY(9!Q56o0&NL(b)RKI)?vT(O*jN*m1JSXLCJUZ2gFR+G18U{Ow(_CqB zs%rhU`kS{q%A?x%5QbEU3HqrS#Y~n7YC8S#9UdWJXcss^i*I}gju<4t6a(F&T#py_ zHu1AfDc<`1rqsIsP^`yDnSRu+I^zf7G891@CF%!*`)xhG_7Z@cLdU|>o ziItXS6dYiyIVhuZyY=1qm+C{)rIoMu?oGO{wvq122ODeNvbs9C{NAaC(yxB@LAn16 zCok{-G)+P8!++!C&kRIa5Ud_eCzMJnL)zbH^u8-}eJDORw5F)IgX|Tl zfNNYwe~}q${}(S5DN1)YvML7+)`SNH#ZwCRp8C7fdgFf}6pMt`!6uk<`B5pLC?#qx z!jwa8GyPFP!RjX!6m%;>9aBu!SpZg;R52f&l!g2l_*7uffz%c*TnObr$0F8!3>uZh zDzYzKx&%G416JK!H0dNl4CH{INr=q# zScnkiKFNyBacBMlw{+RZXTEJg!}v-D(JWn17nOWxIkYewr>dmr8Rs_gS+q_+l~|&O z9H%VzlpltWX!Io7!#%(ahw^r?Q&H{CRs;U$sDrSX<}g~AqW*z?wzg~af9BDsd4eKfJ{y7V!X$*DfsbQCsD0I3nB>9$ zODulB55I(=*kQxUwF}?M)zP(o9B>W-U87qu>ZM`0XOz*T2Vqw6_g2_L$$Xo$VV^i{ zfW~PyH4a}Cd>Ldhp_{d#lk~V}c6c@5moVm^#8}>DpLLt*?j`1EWrg(G=#cdRsi~a%~kp14;L7fLR`G8zPz)7QN0AM z$FNgN+qDDgP7VrIHzWqu?e44&T}BF7Fu(yQv13b7%8(@Ba`sBL&Atco1 zbwAiG!xO!_W(^nqjSw5jul(%spUlt9!g9L1Tus(al%F4tVZ*dG)XqL{b~Afv$BnK7 zDANHoM)iUn^DGVy(<;5m=kChVc(S8>e0*F!Y;*3&c`Zp#?tzYO{Y&S`G8e{V?c7SY z;u!dRVC}*BOoUx7O4y@YI-;WU&>=yTWLebIGF2emBxkZLzbb!;T{TH8Yx_O zsrayVVesq`rQagnmIph-%~4Q2-eKD-U7ir96gGXu)3uL--k>O z1E6EaQ>$H>$v>1ipYn%OT*G}EYl}A6S#%Vr;?RMNMMhu-D!<(T3n*8IvukZcKfP#U zYL$I0`rDK6H08z7J|M5*e1`G_FLZj}E<+_qyEAWgz?9^Dm{`J==A-=(w+W3{&$yOA zc39X#I2`$&$PTp4sd_TZ zRQ>!9wBoChiZ}uwQQh%lxOF!Nq?4U!n0pTU;`poNGxERG0SL5*wI1rp*S=_YyIWg2 zhxLh{-eHN>b@U^f85kmQFhjI3GA0<)AHH5+Z2g14mLxI z7J&;ot}g@U=kJ*iso7!V83qQzt@L1!gLu77RsM5s?ckS9$!1N~s8mFu`uQBo#iaa^ zCqdD0!6Bx)(ZY*?h*65*{eG{Ddqw%%_Og8eckX0y>%fY{TUA(rh%ockkYri(ySkHp}RIR4v-174`J)+%r+9s+#>C;Vv0+@K>Z@-nOUf%@4yA%6Vmv z+_;Lys;mD&oL7W#1xGmSk^0M;UO9ul8%!%w$fl?jZvf?b64 zKx4IR-o8Bs1ulYa!3Kc>w@uc=RUGlNmt>jQ5Yu7TC&&K^ca>fTWjUVPQyg4#W@%?)Pna zIsol-+~=~MV=_(P!z7+*B=pqVu0KmA%=?dch$~{26&&V9+Fanxucc39bYWMoJ^@%s zPf7+}OcHT}jmez41+K7xTLaeJ#B@I$$zu~N&j@YB5RhXkvU76q8x0ho<5>DZcbvQZ z)sV4tNE7`m8gA=-xTO(g*b3i`$FRQ$B{(0#4eFKOV$`?$a^7az$=7Hc4WNqZ;fI6wKUgK3YtpsS+DK zX-CJ68^hAtV1te^)Hr=z-Q4)JQ(yb^CxQ9y`aEh&_bU8e;oB>>%g<*mO>^z*cx$$! zOl&O<4QRkn0;sG)v0n?ZIv>_byV(h2GOxsA4Wm~ye6pM{zU^S_XbIe64&WRC6Sm4o@hN7lxDaddHPAFX{kPTGT^ZA) z-I$ywMA|qZ87FIc;~bn5coaooTnHIfazVnXFfEo|uJHA3)PJd$wTq7Ky9b+J&F4^c z6_p2&deOaP4bWjQ4!H3$#);g;RvuIf}`bD#6twtKhPf>5HCvY()1mh+^^EfZqH#EX{$ zi|5q6&1cYY$V$NhI|_F0#L>2Dl`p^)!iasx*huTE&zOe8Ei6ntI(d!;(&UIMx=+66 z`PTJj{NdpMpt(QKOBPfpT63raoA(}VZ2aMpw8_Cv?1Br*)HhBlRJ zpI9(AB7#QP0W6M8MjrH(B7P-7Q8Df+@qNl~taR_IISTc}U(bU6@}DVl6Dc=3XO$T7 z5?aK!pOu|MSl)ndx_7T(X-4QdobdrZK@UW$%#%Msh1vZ8qVDj=mnUEz@GC4V1SqW7 z>_!eTLK_5(nR{rCq)P)&YuEE(2k98V1_d~cCjIwNe<{>pr=A^R;~)6!v&G31(+yP^ z@#FIS9|Lqbrskcb0&H}XBBf)$RfqEKT&KZUMidD&VpoGyZ{FqN!dLq8(*BZ)gN0Kg z3u^o2ch}wBRE(MiF_FRw6pAXjzVW)#T8Q^Kwoa0#?tM4@{HAcXcI63Q^Akc8(!WTy z8e%>a*zL&$8k5-ws%XGr`R3X zPa)yULA@bXc|Uqp*ek}-sP8Hy{5uZ;rE)vtnfJlBB$K?2}exZ~2cP8zuWB z#<!!1jr zPSgM0WXIok5GL?5#AK&Zzs1SPDPAMB2dzLMkj#Bdg5?4mOpCpUo7o2j9DkpGcrXw! z+)Ke@6^5S0=m9?KuFHv<>E}XRd1av`G)(j>+kSc_PQ0N+_Oy@aaPH8J zix(}(oZ~19Lg(_ozL7znyldP`LC9Q%i<)r&@}$8UD)>XF4X7S#>6bl}8=sVxm4&RA zitV}@j+xRT))3&hew?hK9IF(Y9171PH6l@K+Nkrr2ftlH5TW41@3()gJIdGyOsu{Q#b6wD0i0@N54Qz_j4TG z5lLpRtHiSY#f5!=>1%gQy+m#e!%~_2c&XcV=;H=bumsa+q?dT0mTn?oHWdiki3W!R z6PgLeY*#b``IxHA={I$S6UK?fj&S>{)_KoHoM`F6d3`Ni5p-u3B_Cvv^yI@j2Gu4VJ=Y+Xs-ygdMfG^ zCD12>3U-!Nhw0}xPCLctl`Ed4mh>1rU$gZC?Z&D)q>c2=>9T=Sfe>(dzbaNbF0uXC z?{p=r&zwvyH}{C>2A`LkeQZbzisM~h2IL#xlRAFOqrSXvww*%OSw!~WRNKo+*~a}WQ>X*&IqZI=4+pZ4L%>W1K0Hz0 zDLAqcY-1|eZWNYYaK|{Ed~%CssdB{3)>fe3@CX7ygm4emHY#(Eh;JahDcPzs>hK)v z%Ij!!m`+%@(D}<^L?_Y@zruRP)?_gk`@uGcmE};dz{eFUo~F}M^f^t990bwuZS19y zW^^V<)al&k&r4eD0dOlrzShJOJSw_KNwqXHlfBgo7`_2HhG@8Eg#RvIy;%j}4f}9e zx<#7~kXl2=>(|Gq5lGuwg6B`m*xIVZU78usjn0V_(5MN^G|d~&8SNU>de<}7W!TBe z62bG2oBD71Kg0sSj>WD^b|D5Axt*c@^V^lA5Javzbv=QJ_DyTYQd^s2cto{$C5ZXs0Zh0)WbZ6gCJNu zcTRf3&%r~%oKYiCe2mDBJ(=>sBE$D=kK=S=X4AS~3Fb-=iXOBa4(>yT&LH2_tLd&I z?lQ&k9);2liSD=*VqGjI7>EUy7*nwf_XS}}EivG+$vur6utB6>87E=c?rPGW7m4)< zedmV$$abIaR( z_x#vXN7wh_RFa+wtvL6EVpa-Z=4l!zi86{*Dsi1?xvVMmD@MnRYN~(&6$@DMlY$B(5wy+rih*DDc}1 zV4vbR_NR8%KaU3-_4IDQnT3AZ^-&J+n4`6*rs0Lmk1F$M!LEI%o*PtXMmXNm1SU%y zXYDx7K|!YzPCB^&7fhMTLTKn7bY-xj-NAPNC$*q$7QWf89{ag$bJV1qKi>@4ylq=F zy||;6VdXP<8u3D-ajzCGhQqzj#NF;y1~2GJpsuRx!4O{_@Kz=3MKX}5Z>*^mzlH7( zpvoLzbx^gehBe~;zJb;Pw|Inbvjwq#zdmCz9;m0F!cs8#J%UolC$I>E6nxC7{n?Gi&!sfkp!W_r3L8@H$h2+4WE#X;UHP? zspl|}BL?D3xmyt*vnSIVSKt_ww{1MiSBG;9N2rmbyj&@|QmRCI=* zXE{WBQr$Y6*c_)2;!fnodsAq+sp~4`n{2Tx>_dqXJTaA6RN)DV(7_2fs7aQpEa1$G zMxQ;SCa5M5Oz`KjOiD@#BrFJ0$)wPU1`77sBSJ#PA73Qb$Ej)^)Gatkj0XP~dv5}c z^}e$KyEmKK7RTzW?|C|NC9P>pHLVI#1Md#0_wCbgYOh zoeu~)3ZR;jNN~=PH;eUeRs~3Gp$_(P@n7#_G6C{qf)!wgV_3l~Jbty*Ce7aDjY4ID ztKQq+%W%R-$>_gF2gB`gN76M>>!hEJ^F{Zd2J$pVcU;otmA%%(ENqAvhQ|k zEZ4pA;y;nFW06dh7HB)D?s&c;H@8vs@v&S;LG8`#S59>p)XH{_!{EY08h+iox1h4} z0OCl+mXd7=n2nTtoki!tf8(}a=%!=xpV7~2rk`R1GKgd%d_+xOvp zn8A0XjfCE_Dn`C5!RkbZ?F!L@AjtdLZ;LD@+7+PpL`>tpHbIdi&zB*rFb?fS@4G5) zRGYG8G@7WiVo+e>XVb3XU^2EU|;#{A+D$?wP|D4l4$f6!8(=1r9Ooc zUnH71T{Px{0tHe6Ibaxin^eb#A{pp>o5LokBd&ov^9GHi1DjFSM)VI_i%WmrvQq^H!VD&H(^8x4b)i8iItsp_A~Yq`u?8S%3KlRXUakOJ z4@Hvv5Tr4!5=z^=){L#fcnQ8=C`jYlC2{)MiZ9$;EKBT>?;8mFZFLyjwUGb)S_Y>4 z4$qj?kNjXQkeAHX?f{Lx_KnH2jFGu@)~C8b(JLN=tQa}>!&P6tfB)Y9;Z}Bjb9VgA zyosJ49f6KfQ3n3hE}T~wX>vZ>T>bUX(8uoc;QHg|^VRw%#+gF;94hbr$XY7?`G6_t zKw!6Plmt1Ib!%tX2BP0$?Qo}AAff*buX)2arR!EV7QH4JZlpzilzM|eg1hDY((GJU z+)ymie4-kFAB-^^9KlZCzr0x>P=9@qU^w6q1tyRZv0Tl+`fU;nnot4SaD1xm)|(~L z`qOl4?U5HaHB5X!kdK#2rzYS{j-^tAqgzK`LgyGoDi!0vxbNIOyN?OV|L=_S)9)Ow z4;DzhOfjjfYCDUX{=mYPMm(b^z05079*Z|RJ9y0jV*+_YfO}u+so9FoMh%UCV($|{ zA3`B$4TD|I3vCbIv!ZTgTNq3LvpG|Jl$ zQ4w>@m%yO8*Y*vAxcttXoY~UNfdY-aE$t^p&k6%}@PBvo>nx6}tUYFfRyH z)11V3@eKc*Bl5>vo_I9d4w;8{tgKG?mm7MfpNG1W%yFk~x7oDtPM=M-$`A=)q>3GU z)C%lLRHFbB?OG^gFmjuQd2eRtcg*0Ld9;5}k;j@t5LZWMS0#{h?%ZPeVNTwExzqPV zTiPRUy1!?u9OIMCYYbKxAdvkw&fpq}j)I_efTkNvNcF#bvY43i9FwUd+I* zNL>-U)!05fuf$(0n2P_*iE)3Zj`*aQ)kZ4ZiT;R27d6}R+=Jf$C{$@|s>4JI2&9P= zk*Kd{Qaz}VmOA+z#qi_4T?RZBee zM8RFUW$^8{*khvF2BPiV+U;N=6)}vG?Q5G$ABOo$6qr>H@nJ_Gwkz6f1BDXYL&E}Y z;sPKmM{TwnWi42<%hV9V2YN>DjrRrjcGe9Ht2$vWdaFglZbI~Mr>ldcCtkgNeE_VC zH=ta^fz#_B62MC+87iqN@DPw!Fi=seBMMc2ENza)@Dbfav~v<$>U!IA7iScdBt#NZ z3n(v5WFe?Nfh?RXarZjQTDdYRZZBW5A+5ux(QVzT_vvT(K;7Q9c5Nt{1{Dq#rvm55 z7+0qWEQr2utQ*0=F8w>nf&rH+}Nq<(k91rPSHpTFV+n z_}>hQUEyBIHqWwkJ4j{+!QMjdz2o`^s+@}a$_*z4Alcc~d>vTyrNleWBX?vZ+?!>Y zNYEJ0Fv2gs4h*QF3oN7~o`A``eN5;nBw&z8>glTnx9NH4`NhGUD6PxO8?zjS^l-!a zWnF<6#jgb)B~n|c>dzUuuWGr2&B{Y5ta*XfJtt7 z*R3y*bp-RUZ(4e4V!jN-X>;zdN$gWreh@a{^fK`mR%#;QTF}C~cI}DWye7}wKTITe z1TX?mT#JGNtusLx zB0PlH&l+&K=WWO~98hG1)s`J=yu9|W*{ksbkKjKP3#@Oz_U}f+|KAPS|1W&~fA~Yz z;QVuQEO@!loZqedCu#d`MROqv*B&qC6^U(QP|mMetT zWU+2t(vvI6E7pXT5`W7zaBA7q${$g^uXrWc+L{aPq~9kk6KF~NM~t*huR*r|ORth? zYR&&#BUOn*Nsu{5QL&e-RWQm2-Wr2cZy=VajIMyNNE`a|m=xOuq)r|Tg*%`FAox54 zj0S$mPAx}P7jlO}t)%xd3Cl2}qPxGNqeH?%Xt$FjX4crCHhC4GA*Y6pZJD&*u3ZdPb0rl;N z$-zmIB8QF^8H9v86v5%a1YJ;*+Qf&X>5op4RtdNl5Uku(n85P4d8(b_6y5+?EM5kh zBdMeV1Gy(qA{b24C;=Q4VfqYRfRhDGK2q^;{{H19VcQr5#nCEYdYwg9?fo5XSJ1mi z)`~X@;Am7a?wv*>}V_vbP!$!z6se8kmwT*LQ=UW`En@zw6wMPBrl$93bL`)cnZp< z0T*T4dsX`YUN{2Z<~t9vN?X8=mIBl zjVtc8cxlJ>)`5+uPxx#5-jq45Fsk8YfZ}DG!qBVqX&BT)5*{bx0$Y}7ObesaGUJY* zwmQzV3VJ`B`o_?NCZRVqxH!zI?(XRcdWU{1kHeRKgUW4|Vp%iTnrE;Ldf`fAjUH>p z%0Na>6Xi5DsCl8Er7!{opMeIe5^z_%(D-Bmr@8Y^V%6$|E)RGm?U?1aU(Rhpb{)=g z2E)j$%C+IfC${H9pd6G;H{kz~0nopVy|-9&O68{j$r1)r_`oyttvXR+=$*TY0)Qpk z`1a|q{d{|3Yi|s&E-zjp88Su6W=EM?J)~!0b0S8-+<$DMK7FsiusXf z`J+#>2c)geijIm{9=@%+73*joS= zGia6zG0-AA7u^Rbp+oZWR|5hAX?a__*J5syQHB`|^B|C${du+8SzBy{4?dMqW}1+> zhCw$80N^p{6|iEMw;BVgP)irG8=8>bb_!TM^w!+<&b8iA{XotJ@Oy#Mv_B?O8y2+F zasnm_P773^He@eiljj6k8`+cW_?D$x9-R_nYdP&P(pO{fDrKS^B!z81)mnzKGiXnd zZZL$xIv@J#DT6fB3ZYVL6lA@o;0aZO;)a0`pw!T6kf6F}X$kZJ$@xMLs@kAP?gy_% z8Ioi+(4ix91m@P$Z=YKyJe%Pfi|MkKcO9#|N?SaKy^0vuJ+6Dz7OA-Z&|-m8Drkj} z@ICIn_RYx9r4!B3)DCtU^n*tcdCZV}EqVY*Ema}Z4#$!N)r`m{B`}r9{DJ%%kjYo0 z?rCjn3&#eJ0u~QIvOqTdH@2r@_~u@vVe?Go)@@q?-a6{bDI5+q%;CRa)n8dBT zRxNS~!%!I}2D2;=wA3EEs@o2^B@eGSboc>G1mf%a#;0JlDFgNItwrM-skYI6b?;eh zT0(yxyw4xN^jl@zo8z{D!Rc1<0L28kBC7U4O+?@ERjs`a4IeKySc`#DA+(z z*;+LKRf%ldRD?}a3g4BMX4P&2+7X7gLNHS`@v6F-Brg?8!y%(dWC}Ek!=ZY(df)Xu z#aKlDM6H2!mouQk;sT&@| zZ#USjRT+8cLl8~@%(n?Z4PE8(RQky#p%%VNBB^!*lx#;+W`murL9TK5;SnoPU8e!z zx6vja`>rZM+&2Y`dA?FTI`!{66z@bG?7g-&hAkTI z!f?W1iYP|@_9{~9(sLS61Nq0ukTrk0!^1P`pb%sUXIaXb@wbU@4sw`*&7Y0rY5B?n)-YmjeOiWm=s;Ua>d9-aO#&k;_(6TxQM#c zQ^A(4bz7HYEzvdJU-Tqil&IO+E=JsS`G6A>gORy}&yvS%9-> zN(8~cg3aSL-Vq}7RlzPJ9czh>W}G6jWDaWclSbi_7x-N0nNFqwLqYlck3t-N&1lg? zengVYvlx%|#N)6D&TBWHOvdKt^BVfn3y=IoE6OPDIUMc1o^$#7$#l7871kXZKtYu@ zlh^Y0^j}P#89RjD-t}*tx*x7OfA`)?+TWOUZv!G5$+2P#6yVhw233MEcA>#U2?^;p zpC{jqZHI?rb41*(cu+Egs!LBdfD8#^RU;+kQkG3i@m5$uS`bA|ISg+ix zi--W^mo(w=P)Ju5*%R3A=F*>VU=qLdB0Hx~>D{Q@v;T1Iis_lPFC0E)Mv#;-Ss~7z z?Zt~N7L9#q#(L0fVZWD$=m^990Ysg(0wzFx4#xe zKC1zsMFeO6{>7}+wPiQOK^U+hLGCD68@xa102%yrbVT9i8KsC$T^7&YSE}nM_FmA&T(K43ZC9Yg1PHjK-suIe8B`MP84z5JxyUo(pThPMz`!07?t7hI=6*hF~3 zed|6ZCm)KhgvBB}AO*0sP}1ZxcY)_Q)^J@sMmywXw`7Eu9aly%-6wHnya7;dJ!F7Y z2*9NA3f`F!wWP!9{db`Kqy@Pb=vnyakp#Zr61Ej-IQdJozlU`O2?pq2~jIc5%#_=+O3o%LH#h zNr@7&Yzz?JNsN_ed&EBdIlkb$ko9oBkYX+w-5^S+ z;jxo(1$fd`8#XSTdhuEBho%o3z~7$KvBejugxYjl!I4dvLbWA*)YzAB9SXsoI0-Bi zc^jijZK|&UvDC6;Wua}`bWjQ^+I%PRZrkobj>`};H@R+w;Gv8L4Mt>gHmqX}59LeX zTO#^J!2~kOVEpK?{Jus zkAWH@;A5O!feEUFQBw2c<1bZVneva^Y{I=4`=G{GDe>V-7&*8-7~p^WuJrfaHB z5Ztteye-KXo5Hwq@@K`Fj>G&K(W?At4yB?wcnE}aTpOe;cP~Wgbv*CpU`obhm<%GTGHA8Gs;sOu8U^bz24^Pnv7p_LMm<8Dgv!jEGn78dU2JR@@heL{GW862UgjTM=N3>15^nj6J#tm~B?MeX=uv0Kq8MFnM5YwjG>F!)KE6T)pNGW4uh`0jqm!B*ev(k@N$ zRvA9)AA)xAYBJ%cKl3a^=FdC^n!LML1h?+g?ab$^8DyPQe}S6+1mfxrxnhYES{YR4 zm2IrE>G|q*{1){igyHS)s?f*a=N31M7z50$h+o>B0%ugt@ngq?D_K)h`2|?7*ewTU zUHtvUMePkf`v;~MULqD=0qT9>t!IG{U=DLj1uM4aci=^&YYcWY38Mc3Vq0&HJ}O{B zLCSMkXNJU~bW!_A^|*VykW@U*naq-}X)KUDntXYBJBz;O(`ansl_@1TAeypmO003*VvcD@F_AP({#hYd z`vdD;c8JbjvM>_jG0SyljVEM|a@=~*P%rKracI}+0wz2c8I2Vv)4j8oc8V#+%h9YBf$NDzfoZ;}NG~JS5hFwPZX^Ki_Bl z&*#(1W^Aqn(>I6BYkIhvZOwFlj&1*6f--mi%2_x49f6)!NMZZGdqXXcsbMKxZHNBE zh%cf~1dKaW^&P)l#-jgpt{$RK-pD$@G}?OPxvRq|);;m$=&fyZeX<|Y;m{%T1ZD}q zWEFJS&6~3(?SYZ^fjFQ~Gf8&IJv862OiU4om++j`UfQfX6(;mjk3ZG6=dhsWqaBde zBNODTkxl_b-hFT*!|_HYv#?3>%4izLumlnt!vRd~&ew)8ApJCGdCYBv=b``u1iHl- z(G#rihkmp%TvnK2I5+sAjQOHo58BgYF+fKx*lh&?vsins!+W)C8n}|+tc@|lkl2eC zokAUq`57EGG>>I;vL5+uDX`4`H?o|dvg;Li53)`huG@Gq{5d8En;?bo*4huLalN3w z-lu1M9nipJ$&B{=&L*vAEBa4K!$B$_)DBn@sL%gDGF#XE?@PVud-Wr_{k1iu^&8l>;J^t$##F1YxkbHt0L}i zytjHU6u94kg{_X_1Je6`Mm2V+Li!%Jj~a{^9r)1Jrci3=b@i&Lf}&zUuhEW*ADrg> zX0v0t>(cwPJToJAn*1se%7Ke2$>kecY|)y3u-JYfl1`oILd^uvBq4tv5LdtwnQITv zWK4j6C^Bv_pCdbDv(Bx1PxDU2o%%mT&Tw7Imbh5tYQTl8y91U6lZ7cmSyNLpx-dY- zrOBf93DRvjCQX9@`YN$#!gu0hAo!-7BM)X&SX-J-Pp$?wp5=IbdsmkL1EH@CzpwRHcd#g$Z1LhHW{E^XIF=_R+L@Ky~ernuVItG%Ya=- z+OA!@%*t0|Eu4A3o1xsC<)jw*=5xHE?us-S(fgFUAAb2T%=Gix_<#6QR>5TY4`1=3mQ*5Z)0vCN63bm^=G!~Z+ zb+zSvA_&Koiw?*9qk#5NK`?gE&D{2!e#cDw^h=cY0$F~3xs-0H2wI2!{xNw@sah^> zd;QWs8ErQl__}&LyqEIfe0RKr$5)_DYrmXj;xCJfg9S&G7>eq<4Y+dx=Hmq&Z)yd? zdy^~+{s`}-q%c2j>IMJGy=QaPq3IVM#&u~M1&v$WuX~N~VzwRe&wnnX|A8;?-{Exs z7oQx?ZPq0Q&}NN~7YhGW{o1`N!g4kX#dXzwnw}mRso|WKo@QvL@|kV=aX!8L&v&r_ zC+pR0Yk%7qc<}x3#qi$p%x|)|ueEN=?);Z;Vm}94G5X#+ir=>RNN3kZge_5EY)&+6 zhJ?f7$xHty>x8T@r^HAs({`PkU6A|w0Q=el%H2^a;l1@qD5xDr%`sX{?Rv|FQ}=)w zR-+POJ&<48ZGnZG_9LhL)E)m%O6vaUW3K^YZaQkO4xTDWH_Z;w;dqbzw{fG=kHVAJ z>B`Q{`kk1VhWn^R(m2O0VVmaHj;}gf_kKUnxOVcdc3A9L=R0F;`HF|KClct;{T|DJG{XMdB zZu_fN_}9{LiSeDBK*!6-6CO=#e!yUuT~-BF3LY^Vqz1wqq`)n%1G^VIpJ+H`tOsaB z_B?u->EL`#d<0-v^k#5s$+%|R36)JFCoDK-5g!?d7vYGol%BK&0)jv;Lr@m@h&DNUDf6va%08i5p_oOBO8E^t%wrlV~{V9a`x0NebP7XptGEM;l zvqn|>rmmAM;MMJr2Ea6w2QrjFQ08tE0|Nr%P{annu;{0AE?HSw%rNLV?Ag7$p>O&I zK55B6D71B}>Ld#RV`bvH;Q+yH3!83C&M%`!hZDx8q@J~=%9yKPdP`Zg-6Xt4suCn} z{^P0Ljly$e^__FrvFYhn+MlH|J1hLn%@c>)mavQdT>}(?V zFIVr3P97xYMq0q5z;RfOA?*Wx!g~P3Wp$auHDmRQr>TQa)6&|ai`|w%M~q)Wf-l4W zheH=@uSq`It6vltc`@MpNa6O19mPAB8?5Z)X}`~rSTd%^E+o{EeOdr-T@~{dvii!p zdG?nBY-(>JKfXh4s>!zZigPWuGwM9<9Q_z1E779YzZ_p}vT9)KWb?FCTHcb+E{z#L zMJ8G%KApmlJV@0RT|NHo11Gd={S_FD(u_}8F+tJ}O}esEQNT6BbG57l90(c81_ogu zd-quT!B&GrpQG=Dm_xQkoy<5PVmSa)blL#VQuC(kEGp834PWI0qGk32sE{b)oHIui zcpEw*p(JkLkO2(78lIBTAgX5e6-z;f%Pu^b)oqhc!1G{`bJ@Q&CFiKGe|jdA+|h{m z8X$gq2diGKU9)%l$DK@c$P!R6V-o!`>Sf>Q!nZyJjJHy1)B{mGma4da|4nj|L!-Xt zKsLa5f@uK{-r-k~%gM>XTCh!ThDHSW@M;aRh<>iz%W$nIlxV6Qvww>COVCwd_OTFp zi9A%nVswpBZHtFzwzCm9a5A6=RbK$@>;8gbR6T$el1Vi=wx)h2Bk>=n6xW z=CqL*t9*VcAYnVLk^Xk)w2UL$zZl^XxhL+kzuno6!?X76KjgDGQ7tg-Vle;Kv0QKF zvB>Fsb^-F)-zc92kfCUQxA3`2OL>Y{1hLQy;;Hi=e8uX_4 zsVOYgct!q-XqrXCml2kY%Mr1afhiI11ID5W-@bjDV}1%$L^?~*YEn%x(?UB_5!_DD zIc~3T>WAjP&H5X9)O5wqX%&x5<$rs|Mt-^V5Qn$~6lEJdNa3pGB%QPw*=W1vaZ0a>pV;6a~ zNM(Fm+qm)iJL@+vUa5qK>ughYX->M}aLtXE;q%*~R;!W|omm3*YTmpHWA$|7hl?6h zPvkBe6*?6EsG;wvU;3#8=1j8*+t=qypW$|=Y&)I%x+l+n=kmKR`z8{uq+|^lZJ8r0 zXXJGD#19_qZ6O?%ddp95xIe3isbae00Y~JFzJu%@kuy5qS{R09q%VYfzLn^`@iMZ<2K`@rhsz@kJ_5Rg!-7qEN-f9&s;Ov*ixBCVSpiaVhB=E<;VFdI zdxfE&EziUNkrg;EHQ{@A?Gpc=VkvO&Rb z_5CnjB3>h9bEu+jED}uYQJMjIq&wFZIozjm&?A?pq=E>Z`6X`|uyl2N5$G#O5P3L4DOeqq8iku$3#+NB zoGtM>@CqJ5uq;WIiKL)FxDfz#5G7OvDljx-*vJ-6w%(!T_Fl~1&xEhbiC>TypFD|4 zsr3)xU-Xe^qHG$LU#XSN22yi0saL)w&AGMPI*s*=(o}1%lh?`2ck8 z+LVA8Z~;02YY726)Xx=y;?ZIYK(%^2l9>%gDi=NfR5-0 zbFST>!c}6cM1x}zg?Xad24%Lhj~vk#gfgfS9|M)JG6)wo-##B5gLoblP!tkDC<;$^ zh7x~kM@Pq|Ob}xH>WZmY0K!fsHlg!T^|ROvGSLSj4o1}Wt@E`oWu*1Mx}%K5yikyq zWfsE-0rCr4l?nR#;pUJSyVn zgQ#ze+Yq(=yBUPWAWr2N{)C;)Q?yv)wAjzBo_s`V#h0HGH=hV|w)!Kda^04BO3&h4 z(QTdX=H>>z)<grn*zDZ;7B9esJrYV z8)&SRDk8`NLWt-xFyMB>vdza*Ma!GX3**!xi-!3z6}_d|HgkNTc4+$N)p-ypNdwh5 z4jh55&|BM;F+Wl6w3d$#l0d_q7&@fI8pfCpj`arJ;|7yN9w_Uf5ZovuBZJCphto(u zU3h?yi|Oc$88bu-@_(h)4WxYY`@fv5#+* zOmuHN3-S=0dDSssf_;`;CKB`@CMG6Mv3zOs&z{{c<+RXRMG+Ve8X2im5@8E~#0Trl zCwV7fTw6&bDtM&Vi1z1zdqh;ljB%$JZ|?=PxKU51g1AH)nNeUB+V_|4B@g+C<3aCp zjxV`FOlFAduEW5kMH51-*%11iok-(c^|aJEc_A*TeBm zVHa;#4m0g10C)B~%+22p3RXG$N=GPvIn)*B`iq!`6UhlwXT|C?n>p<@HJW2X*%N9Q zYE{j!HHX5F4%Ualz_YzfQL~!+bW&j3q91!c$a$p0dLRNVY|_oUB|Uf-7V`-h+KEQe zGSWOT9)W3gw^j*@mhIU0-!-zx@ zaDfKeZUe|AyKqR@nJLK?`-k0hV&{UPGrrkc(h7@xAs&`0b^5IEAWNbjnM4aOqqQCWzL=zDkV<0iG2*FWKsX#52)0e;bs2W>ZsW45?%{`qChkBu zAT)juBM9IGDM$EOAl}RbxXgM?V2lqWxDX!>tO?@D5IYJweKmr?Oy&bLG56r^-9fyd z)q`fIDK@bE`)g|}0Ens%A;7GM-@F34v_wZIJ}zm?pn(caDbje1hcJPpt{pu z;UrGVp@$9~ifXdG5Wp9jY&FoS28n?iG#`cY8TxGIMB3Tw!CgutHR!FHf}W-Lat3M2 zAzDD;|0s5=JTOEe9XO8`4#203v=k{L0DMclJ~(;CKtjh9NdOX`hGo5ldM$R&z7F*rB1d zg(?Lc%C7`n(p*Jq4rkgY6K@}l?)tXIr95VUrVyYWk4<1Jl8_@A+sGbILRLjb8;yx0 z$spCo`}cVWnspfJt;Bp&&y}&OG~xsnt3+eaz!phfUfxP4@FY}mqY0M6WKji{;aHdc zMEY^q6e6`y(~-r!rHM7br_MC4woKIQ4cH_wiqRw~*^YD+TojRrZGC^|3@Bmg5@izT zHp(WUIL*g6bQdOSpFrqN=_7@MNL&O-gX%dC8%jgcD=MXrxZL8W@8d0K9FGDxbm^&Z z0oi0e>~L4YAvmOQT->5X==}Nf6fHr?QbU$Sae{Q6aBxC9w`tC)Q%Luq6_*G%yg1}t zM0=*u4@h*xz`8g4aJF;iq79m!opyiR3=%W!my*Y=Agf6bKA?~tKH{{KNoxYI@&SH+ zei(m+gNGQ3Lzs#w7b!M3F~12Ps2FgRXm zE@!`fUgyQJ$4eI7$UO-*%VSugXPmUl7fuVpBo4{(zf*=B_iOT0=qRzmj`w zGcMUw>=Se{>Hnab{qP8L{pNFSW zO^IiA+uPf~A4=axLvbJIeNcXMS7~E)9(}BN6X^vPzE#ozrTzQ2f-iie9`?l&81FlT z-T9)Vq~zzHe}3d|hXqPEfL;z-WZRQ5d=pK)MS1nrfDQ}OcF>B%@Le{a{e0vDRvH9s zU;CfX{#3O+!)esEejM|HM*^fBszf(#-`=zYd|Ucxv@pt!x*+Y}48mog;?Tf=2#zd+ zKXY>>%(L-95I#Nd0RgzfL1nxd-GsUL+NjKzuyd{`dXt{-6zdd+Xq|@=p{EUk(~Ifi zuq@sl+!*CFuH+}2`$%(x9CAE&(1~ion-2@%n(tyD#JmYoP4s3@Oe$af%*knY@&j_< z?oJNQ6eLGYcvT+I-tO=rP+KCwbS_0U7K#JfHn==0?zr^IVH^xS;msowQka^^D5*vA z%!MaE`7AMRSQmF9FsM%A1({Nz|2#A#hQz{fl5-O@@Vyy-Um`seFmL|+b<&L~;dLfBC0XNK@r-`qH;{M5aX zQz^S;IMK`{U%EH&;kledK%?h^5~kj1eC4*YZ%#HmeH>TPH0~stT=ECJeI&*LZrr@N zHOQe?g07tyX)nHKjP1|z|HNbLh&+!otF8;=CYN_c`5iwT1rc%Y^LX@TxnoY}?sK=m>>D*hNyOxDJslBKs#OMdXS) zR{)JkiqGjFP8rm?Vl1t!z~R^2K)DZJ!Mns^_({kyw_1NZnDONk1RcQ!x~rNn;)jC{ zD%AauMDE1IY>GdP*%1Cy_WJP=3-~6^&(Lo;Gm$Uf!wvR%_Qd$)s4t>&492Z&y9i%G zYE*^0g7J+t5u-Hk+gP)J+$umqqlie&Je;Vuh4Ags1J(RYjUtiFv21V@gpXXc-MDch zDx~eQ8s>UWPw#|ujt?)5AgJOvEz1O#^go+CL*{G(ZX>5U7auhKg{ludG9}bSjf#Ci>(ub z7)C}DfX2-F;tC}*;b7(szDK2qY%%FquX2*euun!EPlE1OQrrTf zknt$KREtGk1L=m)KmYtw^_1WMME_9Rs69Hev;GNe9mt5@vqf_yPIP5U%S41_qc}p6%xZ-~)WVjg>3(PTU;^1$2``z)5i&qZ{Ev8F&@dWxgH5bHAd=S-v$_w?AQ5*CR)}JEa zZYg#Wwc8kCc^ZWlNK?+Nvu30`@2RS`LE>I7Xjk>7R~>7{M*R6cF^hA_&Sx9#{R`tz z#8B%PDVpT+l{|5U=Wiz&hS{PVT=!xV#N`As<`-#G3dCH%&E35z0d3-x$IC6$Fy=*& zCDZKqw}wt+YQ)mSuxmK}<(d6${-@Rh?ggQa>+s5(Kq9+T>eI8(&Byaj;)$n}ynm9VbfRU;uwM4Xur|~YG^U4J)7~$2n-|k8n#^t? z?nln!N`L@dzJGB8J=CYyscq3se&5*-2lsg-pFG7$mSqWeLl|%hSDH7C&o;OHJUD18 z**KDJ;Z$?cX2%F}R*1#?s90?!DzqD&+MeWaQT5WELbTiKEC5cl_C8%<8Gf_EW8hK` z%7ij(W$!lI1f8_^n_%5wsVHmIVbuI=Eaw;Ctq`KIf@kpE&Mf7a%A5U#NcU-o9gpQs z0#tRN9sqKzSYy|WDEADn!_BsNBQpi#vbxOlKy(nX=~N|P2ueMVDL;lIk1|#Lq+kAU zpRekz*=ILVzK>D5QXI*!MB1axR{iXp6)0NLu^fLno zCh-)MF9y*)-WEZ@h$Xtc6{!?0kgyU{xv?IbJ`8PQlhNA^e*gHqvHn3ofT9#R5Lne} zuM8q6po5THf~b%$6hIINk$G}>5hz@|_xuo@cr&lLBc&b-Fz0Gafit~Hfm?@%y(w6O93(x!pR3VDano8O-`tY)#~#|wP|wR zXJ+4MFIA4m5`ueJP+qPALDv*Cz#jnE;RjSi>YEg(fJ{j={AhTm$(akeE=_}>&;&vV z;pT+ECgIr)H!4P=wnSa7Hhv8$x=>^i;fUeJsDv@XQHK8u`O2kKZev6txje%CT@DH5 zYp<)uC?RbRnz2H`PXqEec?3{`BI&Sm>Wa3uYv-pE6zBVbUw^$1(EtKXsiHt~ws7gv z^$ZvhT3)%b;?b-lLtp>wzF84z{bA=0Btl8hfs4XRkqpV>L{-7mFhmo#WI|p#rGCK~ z7fICLdAC+>ga-)G9AFY1M?`TP$yhZXft|d8+lWe!lu^LTePSgghf~a=e&bUJOY4r4 z$+K*Dp1EYZ<5Q0n($R>NC-EG}h69-TI-u}i2sCVNN%zw3S|5{jQFG`_1pCegSjv&( z4b(wp`w{x+s&MQR$`4-}8XP}~E{ARE-`YhlS;`Z>yl~~p1RBQ!n#Iu|u>T13CAm;@sRl!2D7go2%6eOUtO=yQyP%d*JPo z7Kbqo=F+b1K(8R!?mdUSgVr6O4BkfWU*41918TV^j>DFaj96V>XP3;0K7M`_d&=T=KS!)5~(KJ?h$Wlb$jhO5lg4sLiu?oqI+y3gN90&2_0N zDoATdcMs|5nYwHh|DdJFXXdWluEV5(KI;UxQGIm!prw3(0EWVi+1^K&*oOEIG-F04 z!uC^B93oXD!cJ`^aC2b$fXLPYRPTt22zvDEn1zJeU6yEGR>#>Y;jcGIC z8RfqpRqoP`dR!$3(19DSPGaZPiN-LGBkXHAYfgGqK6Q3h&m_j)q&8xuVW z>u|<1DmX*9%M3Def5&tyro>V@3JNA?sfD+S8z`0oCS!;Nt+GNXJlXo8#M3rmNFC(_ z;Zxhcf0iZCxYtKL2C$96NCA8AMnrw`9R+?sB|t=W3beC2F_!mOy0Xg_g{Y#cnF-cq#`fA$1LqhA(f^x_CP1f$ym1 zslBv%_-gf?)Cu;&JR*%l;!o7Fot?_BJg7nL@S?|=ylUu_#Qd!ZB2tO~XDs9cO^2Rn z#-OoEFkXuDI2<;@)0=NrMGm}TR^;tNDU@Ihtl6ytaz?4Te7hPOn`4`~d^YM^h>2FA z4>W)-aF~5%X(K?=V0491_pXQ8Se``_Zc9+7ie9keR|ZX#;v=dMH9uN&C*T6=f_eoh zyGB|bhGL@&$pZxbuQb|a;+}sjSCV=CW)O+l21MWG5|)3yv$Da;hdKE;?hAHa#u$e( z&@iwoKjPI<9()tHPKER@6%HpBb1u0Of~f{N>Qao!y=CC8fzysX7Avg+>osnNBa~uH z?=U(Khk#8$hGI2s3_1aj!xB=vG3(o^4F7W)R9cX(3*k}6Z%M;bcNwWQ5GX37_bd`p zv@eZfZ(uC6Yj z=k<$Sww%&+>P>5>nip9eAV_|UA`=}Bbc_VOOf!8?3z7|>(J>4e-2qq~vg`5!78(ZF z8Gs%oTlW%Hhl)^RlFkJv{6n3(J{)ln+H-rL66FctbNg7oiqq#DcBpM%hrm!Y3>((t zvvAy!QWRP?@n|y`4%V~Jhs}g8T2}|rW5uZ&fpAVjj6et})IyIq6haLKb>nczh@w=H zK8{#_NJgTO<6&)kQ#Fe@9PznQ0Hp_zCPBnEX9V3Z>O2pB`xc2qiQWtdfDHl?6`xAm zhr@V1mZcp)PPP?FFR)9QO?gx@2ItTzf@r$3lwGeXusTw6X#15b6=R!sX|tS``J4Y7!eGY z4M(t>X_XV}0CGqbF5`VGsUujK6|kbJu$PYp-He6&>p7$4g8}k&hPbULop&8#|A zUFhV2s{Ig@o{x|cQ9=B|PC*$>LP%0m^WeD|bG>7m6#!pk~?czcLtUo-ZRwQ*r za6Z#|#jsK|GW$pbuMu<*&4) zTt{p0xVZBMpCP+}I3S^xAYiynl9MHVqv>zg(JFKwVU(~Ix+&=2bO9hB^}L?Jt5_L^ zcJqnE0^=gcM>?4zFi-~zsyWp4K`j^!CNWiqK`!Ix+`cTv-?|1|*o;4({PEfxJ~ijh z_K$OA1TVlub|;-lICF?ck#P=jqY|rFMHl_BbkxI)Xv$;zF|KHegjmw~`%|nXD3u>V zT&FvP0mR}b)kj6U9FljAnMS~$MFc(HsAKoGFPOKCg*;>Es24)bbVC7(`&4fwfa09%oU3{hRu ziCerKfQHquVat<)sTqlV;=@aa01Omga{EZ220-Dr5%i&T=&T^u(Ew%$!MBB({bl>O z9F9sX5V3SAh(3Ov7XYRjLI&AMl5PYYfxvP}NGN!_6DsGV{SM2Gk_Hoy@cL(cLhPq5 zCPE)gt{!Nf7+as5wPODKP(+85Xk^f}k+&K;?*POU2((3WGQc?qoeEP z@29StYt)5$2HJ(|=-ExCpmJ9*RS%*49t(_#@m_*vsQr2l00>S~|bMVPnzr z$2=!qUAKCG^BdFJjkXq6xNfd1w>#537ut9x_eMzapj%8LgrwS&T&=gy#t-EfJj%=571T7hr^e$ASyG!uIP?Lia?od&0+qPeo zV=jUwK}kYXd@9gDYC!@l<>({sKcVI*7Y|N}^3P|KVYh8{{(csoacV1_`fuy#rpXUm z9Avj!D;+$nuJ1H(2ONggSPF&)`L)O?Nt)75=M(}Ky6ijCQ*Yc1krNgY@R-T81`0Tr zTl^rpjt5a+(T+yB0&Ca(m~SU`=clfNLQ+J|LarS_U0w8-jqI#?mqiT>Y|Q>%wn!0< zBw8eTf^~qCY#DEqtZj>s1m-Mmm?4vUqgK}J+L0GyZDcBc z2+w?a{{~N_Kyr)WrZBJqbV<`Da=Y5oOQF>fdLQlfIP%l(+8>W>_a1Z5z*fGhMYd}}4 z!YL7l4m#StBfcYpJ^lp=>+J4x2n%Hf4>PS1%W;Pr=eR&#l&y}Up@Py{Km4I(78Dbt}|mv1>t zjX!_>yx);8?FJk_iFAwXFUnk3TjS!Tp2 z2mmA(Yj#vAb3cq6JI%MP-K1h?URY`Z@t5!g$R1#gV!+)Hr)O9(R2zpY2Claodg9XM z0Kx^V-`Cmn)TtqGwnmq3*x??gug*Oi)YoG=S&mwWTGepGZQW$H6K54_ndFR9;KG=n}ULyr74`tYBero%VQ; z)1X$wace$4z7BAoCiU5{Z~?{TqgKJ!q%k+;%sJnXI*_oRZFfxwXg>8l0yAdKkxz=l zou=c-dAM~kSxQnGq!#T>pdVO9c#Ha@fxr??fMpXu^GzqM8U9 zl^TlS@?@n>oOIe;9bIS=t3mJ-`V!2LdS2A#@hGsmux5%=V$>q%nY36KR6c%-(}9o! zIwX)$8XR^wwcNLzIxXM(O5hyk40l6af|MbSL9G!fT`?FHP&`12Mk&g}u&_w%W~wu3 zBQNAv{Ba0BD$ ziangHKys##AB@C-9F0Q5_PAH$_;4BcKeS8;SVG#=+J8gP;;C;|Z)fLvBnNNFZehrBWYj~ zG)UaLijL97SgkFT9bgx{e!OzMgsxoY+ezgIw9)t^CNB42D>GqI6C>rw@Cw7DjRIGP z?#9WJC(~sm1UlcY>Ev-LtFPw7^Fenk8XQHqRqsttbwS&ay4>X23e(@4Yntbqw5%~{ zDNgLve1syMk|Mflx&aq1JOObNz;q;r3Q#Hgy@+$<`hsoSg;RxEL$rxd{h2_Q2${V% z_!`Tze$t8{co^GRl?Gtny$eCoXwut|Mh24o=;esd(MtR~y&4vN#9qUUAhqTkuHN24 zbR!fU)D6U;OLA{CYk)eQ^kzsVo6cP{xXikWf~2*H)QrJYKv)Sl_{hQ%0~;iRtqL&5 z?O-v4rIDFY1foR%LR7684FrCWL%b5)33n8dZ$5wd zaz=9q?Rqh!i2fmicy84x<3!YILyA<1f8LB6YJBPP<=WSyK%gdhC#Ra5#6i0^w?~P% z8D|oaDUhLy0ClwRU%YbVDKy*bh${iQh&u)z>z|=Pyp-_t=SwbqPX*=NTXK^2^Cv3p(-`O%RWFnb`!K0 zkNG=;On4j+DcuI_3By!HV1tJMrqazs)}G{_YSX1o45q=u7Su=U?`^E7Z-~>w9a1gz z&k%3u3(^}P@}2MpqroN=XMr3Ba&CTjc}wb9FYu2g2 z<%ZLLK7Jb>yM1lCnq0|cOpv(d=vdC{y0gug?@wx$xlD}e1EVF z7aIZ0v_5V=C_we&NMesJd*h5g9lVQz0La2e^dRM#IPbYEhF);W#8RGiz!YaZiP7Q= zsNYys=APq=Q4VcJ8F0sL=i!0RpWlG>r21g@-(b-zN454j{1+*5rlzLK-CS}s!$6Zs zZPiXS9+J&Rn2Se;(HPhcVV#ZNQASN0egnTX@OK6f1=IHY4xTd1U)e1Qe(_3j)&Xqq z^5LgR{S`Sx7QjyCd$^q=7Z?+7;}V(8B2{abYWu$anHAQ`t8w^+BHw{o?roykr>ABD zlOr&IZY9VQy=ifN=a;5kTbh;>E4gU*?%lA+0adu!WkQ!q9o)Gmh|sSQKL{N{U)ub` zOhZ*F&>%RTVT_?<;<7G4v4`2`n($zhy@RSPn98c|O)33x<}V*JhniHdxxl9}8*=p_ z8ZJO86|4~M_-AxSvGiJ>nEQ*JQb%)=Kt?3`bV#P0Bd~Mxu}fGFl&AnQmrp;FyxY<7 z88Il3nTVSup)!dYcmLSlVI}6In|`J;bim{uSaHIQvI~e}0zOkXlgX?=jnb%Xzh`R z0|dqqiwIk^3o)x2U^8kc3BUz8gEiIyA0}4LgN#umqUXu)-@e#?3~?hQklOm#QFI?D zSi9{!R{mkoOG%*1tusz(Q$3a>8Y->$>F4EvT`fkOlaN^6)zk|1>?4LnrrpHg%jeqg zwmo#^C>V9>wrhh^6Pv<+!%g*XgT52CqdT{RGH@XIq307-*}4$E0%fZ#nWOt*1cyZ` z26^CvMD&^0DcS~+7yTnI_NS@ngYXy(x!T!86R6x+TMUMwIvBhvKs1mOJSsYA5`JIK z;(=W6hL~G3u^Wh9O4u*HJ#`7l*O(^xQLMspZx0}2Y?r{1VI0tAl+VPb>#W(Ji+O7#>OVu$uDuR4=D##ni%GOgua1KTioN+r173u zmwn=PVoqZ?!AYo?G8QzR+_W2&y>OoirIz;+Pt3LFS>NFQsL znIJvnvc@LuGaH^>!@-5R#+xg0#@@6zjwLEWi_`1}qIn%7T+?h~ne{}!VUAskvH7*G z{-x?;qKy+F56>v01BIrK^#_es>;wMS)n_m{P@mm=u6;R$8!#E)07oFg5DiOUMu%`4 zNHG$|u$M%Vo&%6M#cn3*mIg7Ppu5WN2k!z~vipI*Z;gYpIE43#e= zoDA3Ypz~g0!J~%?Ov93f0Y+@&MH2x@h z;&Tc?J)u#CdD};eMlDaq{0O<*jo@!L&AE?L=G^~95d}J7rF{sn`0ag`&yq5Ul zYl&Tj$Luvn&!M_E(|@as@M=?^jh}?eG^qmN z^GFVEanE;e@3FE$fN-(9mp6E7ce;Z3$B(0p(v4*^ghFGx8lCUlLN*LCfx(h1s0RXq zE**ZU`Pd`sjvC4mzK*;ewW!D8eW}~4YDYdHcM#WID=As`d#kU-_%H9E9)WuDL}VPk zz6q$d821qD$lEB6phj&Dk)wb71biyz-B{xITriNB@AevexN@^9hu}ISUWC$+7I1W- z*2~S}xNL0N-p?_XI8cCvn53Gihj8#bOh~|BWsQZ*>Aa(-0#IDw+flArcX56{#)ALbgVm7D<{Ya$SUEKHp1c&Y2hUVqT0loRhfszx;mB z?|Ht<^HAZ9FG8lSvA?^}<$PYHFk)SXzp`F)|oa`p!;lNzKn}dB=2*IYPF>ALGX*ex>$|-Xt)B zsj3s;$z2@`TgKIXn zzU|_!K-_E>tz!LK&TCofIOFGJiwYs{?LSlqdU%EkfX%=uACt*P#IsAMVfjcMJqCW3 z&rd-pwMROMACJ`?JHEzV!5qKKMuSDlv3eQKlQ+l2?FDJ|evQdg@a6NAG`0J|!R&O0Q($5_6c|bnfNVvaU}mIllEPD$c0@(_$5Y{i|&}dX)CN zDr4OF%{^_QuB3+q*wRBJIRi^)j_hy}!;3SEVH5L(rbCLm=oNzjkJ#KJ2`WQL8?i$k z%2AnTX2AYQu?HF?8`K`bFEnBjE(JO!LJ}Ry>e5EP>c8+_Oy4zs=Z@w#5w6saI~$2U zmP3m2)MJ~A)qDw$qAUjzr~pd{`9hM3r#PP-6G?sr|F9bhT6QwDp|xzk>{+RaR4w>~t16F*wgVbG<*5dZzC!WF{WO!H*w@#WXA#AL^bI+znvDfj zMWU}#mPCoR1BF5r9~yaP)>V>Nc%MY2f>1YBbs8iV(|?gY1|3e(QIvKF1;?>!(E}&4 zSEVYE37#@iemZ0VzzwdP8fp2T8%T*PvzHMDGL5Buy>mg99c9G4C1E{cvfRA3NxAaw zPRdgayNU4?dMMxE{Du>>E-T(-M;Ukp?s{NC74xIc_&!ahe3ivl{VP;6K_6?L5t;%w z>O;VE%{vPPZ8P6Lkh_+px?$687s=eFH)J>u z7JXBcyI+iz`4VS687up8GHaRhwH5Qca)xycisSnNs(fMjj zhbOT*{VdX~DU$2uHeSm-W!c=dD5!QWlsNQklj+uBiW*z_mR1M=Os~Fmi-FLe%;lP7 zCpiSOXS9&uFa?lu7t#hFNooQm5nVxXBE{6{T8!$&PA=JNJ}7W|zEb0@n}j*gP?vi8 zm=ItLDX0?mpmdP`F4yb9!69f8O`?&ZB6L?e$6lb&A!ON_Z`Qvc6) zAod!G4ZLXq!M8`{j&(iK@{YVN6sh@~M!GrKE+s0KqY<41?Hmf;7(R(<98?0xM6$$c zsiwTPw%0nFWltB9EavoeElaLv*paYIKzQ=`^J(U)!2t=!msTa8x_q`KD?fQhNpjV$ zK30=q`+pmf$=8cdb(=9`yYeK?%kC^iPft&AA~5yaZJa{3JPcT{u4(4bNpY>FXZoM1 z$h?F&d3N2sdqH`P1O05sy+`9TfnO|!$y<2$%#E8FP6vgRr7yDKnqt(}-cNHfwsxYn)__GV;0{Eq^nj<&5x!lr? zqo(ti0rx&X5`5R!;ga2}963e^v#Xq1QtX5_JTog`_dWDFmk|8jCV&1i_XkH{6v$Ma z)k5LKl`^%buvG2Y<4>Kjd$n_xwGrKSNR{}6Ld-QigU!(n zEAU>%7d-^q^yZ6_G%v=kiDkgSfI&*AQRyiC=Caer3Xis6^Seg0Mo}WUD`-8Ezi=J? zQhu_a(yQ$;`AYWm^2-}&?IRW~U>O<4HCXLr@RSPFT9%1O5X``HbKn;EqD@O5{nSkC zN&01Q#e7_qtHQ&DjcQt-z%6cH00WS>)s7P4gwi)#)Q8-v8!SR3edLjU&7+pm{=hqF zi`dHO5SGE-2NN1f&H9Bfdvg&l?try=_?>J5QZbBzUy)elR&+gP5AW;^f43^X7~5~* z5KUMTPu=kJ_Q3Rn3}=>{;)i-ivHd5c^q}efQT+ns$Ax^;t?X_H077oZ21|P`Qw1$J z_g|_*fhxHq#fl;$Lj2TABM~4Gn@XKY4HC&~x>K_A%VVLTeCnk6lR%9l-6EIc>bn@J zkaN+vNLVLb88~QGbOx=LJ6^DpUg%_#vmq0w;w18!N(78?ek37cNX4!wgOPP~`S;_} zvr$?XHT92>(o@|V>2ex;qXrRd7b9PlM*ArZ@+9#)6#WpU2lvu=GCwKRd@@piE2UuF zK*gddsJ(hKGUc$B-!98{zlmBlqWzyz$aq06M>{V5eLE|hWUU){?wkls$}2NDuM7)Q z_Ck#v?v#e#*cEcT>A}KVDOszbGRs#5DH{bD$)YjBcdQ zs_UXm9!gQsga7XB3Ag=~=+e_znVXQ)p?DG4jk*r|d0BejE?$n>nYA?{0xUS*dQ8ty ziFL7tfFr42g=A_dPi36p;r0tH@(wvlAeNdzY?m}3%ulBVKB%>os zR$G&2Fw>ZH#sfjiiNeoi{1oYnNZ3$%^nMCQCUVVD2c0GUdGA4GFXZ~PBL&%!J)tB}_#a0rgM^mZhr08)Ou#4#NbwKA) zB@_;7g&`p(+1+Gx%?2ZYi%k9qSPcUQp2(WZ+{Zz#r&nqKlg(?`T(=Sh4M|{cQl}aU zha3TqcHB$*5{-sPC_d;}y*V;6va+6vM4r)b^bBN8Q2_%NpJF?5ixUwY7=8#kw2jL5 zwFq?Wy90b|k5%)Uv!9QwHupbmCvMa9TcNt&OduH&T3q}T!hA%8e4{otzR__pF?&_L zz?8dYc>G(m?t1PUTarro!AOX)7MA$_h~m+nrU(he(`f_?bac~B;I2fD7Nt>o1(9k3 z$xxqV^`9x#11<2IV&+9nO`Wx>G>ueuxI+A`v<^Il!XM7u%)G}x`g^`O@*Z=Jv1}va zu@jKQbhEe5dt<8{pqxL#iRV|bgDmD@YFSpze~RN@{p)tb@k1n1<3&|JeDAYIevAv1 zf`UYKEP7>w5tZU*!=-Hl%Yp(uK}*y_K_>!tFDrFxHlDa%exj9WAToGD_E$|{vXV|A zDvL=7*&vMp{yI|{YPpH=2>Oj|m1yGzQ_sY>eLNPyB}ag!rD!C7|3Cs-%og*jAT5mq zX}L}*jnyp)wd47+va+Cg9G8QFEZw~ix0o_C(?j$+ACE`ufZ72mbBEDgc%&ZE$kOi* z97dG`CAGX1YA_;hM2$NVc1$DANjiqqRbmY)!AJw*kQ?b2IcrVGFDh|}sci-<5N%Fc zIGOU?G~aY+w>CTmfrdO4NH8}(hbdGOP{b<@qc{kDGALYS4Ee7Pam5&3nZ)6&dg%ph zmOh$EwS6PG6`w?x6^RyE^Yr2!hX~W2oMKHwOJ05NmVb1KozeU3iy6@W_wJJi8k%hH XRGZ>5`1M#WsgZG*tINJ2Gs6A}3@hpp literal 0 HcmV?d00001 diff --git a/paper_experiments/large_bayesian_networks_experiments/f1_undir_lbn.png b/paper_experiments/large_bayesian_networks_experiments/f1_undir_lbn.png new file mode 100644 index 0000000000000000000000000000000000000000..d64d85398b5a6c6037220ca4c63975220c580b74 GIT binary patch literal 67092 zcmc$`1yEM&+c&x>K@db*Qo2DzBn6}erKChcy1S8-kVYgG5h*DZK~hpe1XKiRkQ6}y z>8^7vH=Hcpy-Zv!kn1My&h}33_BN*HJS|+^Y@8eg`7ZHY;5ldQ z?(Xa+#?SBYpI_i}a<$^;n5Qy@i=1>;y6J`>r%lj*FbbsdZ4e9uxhi)>%lq--S1-LA zgU7flUL;$@#AK})wqaacQ3Kjm3#~WF?F>Ea`o)SW2WoS|}?^#PZvyVdnuWPjt~X? z`@ebFmR#C1e_sjV0-TC+e_y#K*E;|G;EWAbqU*mO$Wg!hUw_N9oFyb8qQGADu}CmO z;6bUJ$7s1tNd28{F52kmXjy4#gkI7w;@)WnCJqj~hYuejLFC*zw^t_LF4N}Z9r|?ON-PFZok5Ga+-`narZ1XOY{EjzqY)xaAI`6o>oU_bMtXQkV~& zlLq%^{Wm22j>hZnP@27q(GYuCTueYpN~%??Lr6${QTofX>#M_7nORp8+4LhGL_}~A zu+2t9NU9tAZ<1$cXP48Ru(_gfwSBw)XMb_2Sv!#u3rW~aWMrh$wQF+L)*KR-FPA!g z6&q{tr|B=&9n{QS#*LJ_D|LK8D(<-`t*RQQ>0@utV`^qLH9H#yBVs!LIcsONox$Yx z?WbYZkz9=j2Y$J^x#w6}abP}IHa1$r327!57h~k3DYAUlzmpGa@8AF6r_0bRM558( z3UjML{>o!Pr`++YzC(=?i{>qz7y3dM9Y)A3EiJ3PS54(Z6}7S|?T7C=j8dRGB!5$WN}GD^K@IZfZvWYui`a1xku;%)gZrp`!-@`HltZ_ zv)ua3;?j~vV}L&%Hs)CUoz$x}BziTj7Lm`UA6S}OS((8UA&8Zg)#BG$%Dufke2?cZ zUi3V>PIE2&65-L&(fJD(dKC(m3BA0$d}Ll4HwNf6_|^2{Rl|hM%*0E*OA|12-u$V3 zaIoGzg0~rR(R*dm^`=n}*AsX(cU!_*7#bRSdh@j%Y&i+B3~VNEJ;K! zfrnU6G+fpxF)~~{{5f7P4!aA(&CQKaPD#tdeR1ea`Xz_EadAYgrUEJa#xF18WJW|3 zU5ZO$X3E}f9*s_CXJ&Rj$@^DN17SwN>{ki!KUu z<#BRyzA7(A|Ngv5Gq#9`2-$FDfrs6AoWP|^6yjdLT4QO1i76@XJ$_6JgCK=}V$RnJEOAmyo(*GcFAep?Rz2U1!MeJ)a?WY28NQVYItxk#?Qf$_$Qa| zn0|YoaQ@;&Et(uXo_ATY_nfC&Pf7$H(K03AZEbB)i8!BAh@sN>wn?s7xrzj7##3ms z3JQ{<3k`02W#h*Ub93{i=4MRTJql0-rhfj6IzdRyQIV*eClYuh@#(Q>$kV6ia`W=u zwzgu&#l<;$c(mPTlzp0$v)5?oRI=)Ibl^iP6<}~A#vtPSw0Hi4mk+)CaD%@%B7Khd z_K)5?R@ITwp`ila_4W0ZU@Y9pg@x$QQ0)1Ig&fDPZ{`PH(7cN|n{(;m_SPL&9z&Jnf_D`5qBG*@dFk4Y=z`G zju*WW#qZxMxi62?X%>>f5@^#k_SgPab7yNYG9m(R@I0VK z?4G!G!ReR>8chiNzT;SR@csM5ENpBt9AwR%o%k>GE1KUYFi$NmKHu^+FUk6nuQszf z)2?*=dRRk)WRY%3oS4VYr?U5m=0~ge?HwGnD+71eShs$CjUKIXVipsl#=*gPRavQk zZVtyWk-YASG6Uu_8WzoHo2>VGBIzFmmt*GRQw`wZ=RYm%G#*-2C9Lz(Ao5Dci9-83 zu&O^i6vXxM@mVt~EiD!C_({fR_(sdoxB=~n&@~u^gtQ$pqAB@Jr<%jwfBK{&LJwaU zcIWis@ar^DH@0VLnJ>%yuU$I@e~5#X8a4wLgZr0PQo`F>o;X$&zi4T0etTIa|rAq5#J4tN~ejqHy#UI72O>jZ<9E0IB)+l44VlK4#qomsud;>OILUBLc1uD|RucjXdh&CY>`<8nf$hNa!pguxt*GV3WBPoxOr2YEhM9^QWv^emtbf0bgc+MI zEiI|^A34CnDAsw={3$~`*_4ooNWt0pJbab->*<%?wzcKGcgv9UrZ+M}IkrRV785hoUQ9US57` z@@$;n|btt}H1##7$<`qaI>y~OnNM6kj7p5G*eex_4!!>GjUsWUz-=UVc(uzUB+ z*1vzO4mcDeq7_;BbakUwlKZ9%Z2ns!0SDd*S8D=~4=JHTBSDO^;dM72vkJJ+tJ8|Q zVk6EA1H5^;W2dDKf{JxZxGGeXl$tgd23-#KJRV=ZLjhYQ4;8a~8C2iMoBi-I>Ix?9pmkD#5t_48Q<1(dpT)RM^?U>Kv`1CrOh=mWN;C!f0!H zXJUm#zJby*-5N={Gn;CBaqU_}#Ko|yqD)M_%hxi*$e~~FELBZ>erYH^|KrEh+YsEH z?!cpvv@|NTPaD2*4-N|plc|O3(Lw4;K|!(et9JQwjhof{{QM*Uq@Cr!<7jm1H48T~ z%nr6jodBjF&{y5(KV$g%`r_f^-+lP-EHe2nmL^TmiYQl;6Kx1CGw-sXeV#vm{(d}} zh}-v510}|%X=rHj_!)ov_(3ITdCI84?_`!bp0Kd+De8-8)xWQthj;q)z4-Xk&=nb- z*1Px{Z?8_b%@39^Was1vSag!*@$0;K^JcA)l8o&3+P4m9+88l0G3Q}2(85$<+mALa zKUnnb41wn)Bxr9uyY?1lJXyh-7q;h<`wRWWlx(VxBVhk9@$nI%?F5En6+pp-iMbTs zTg^>PNQrUdI{=&}=H{VOE%%D)!m?Cv-#&-FtFBHW?a?FC=FpQn&@nY$TVAgVIHXp4 zA`uKV3ea5g$ft*bicjAE?q2cqyp8Y7+T%=?$LsNkh~D)T-f-DjF$ueK@NA|%o*8GFNlbVnZkOs9x3N6 zD=S-pO=$+b=KY5ct>{jLZqn4-tC{rzdOB>cd(hyr0uJ|Hm6hGi%3_?JnL%N=VXfyB z^Z;~4Im4H)*yiEoC4?;{3lK`c?fW$!&5rKwlGbg5l>0P*v!H{A!rHl_t4n$EGySq`qo3Q7{xpnBPe+Fn?^>cQ&Li({wsKUi^S22-t9`|H~aWV$a<_=K+Nmc^Uqtb zY1cvxCK~)_GtEw%IH9Dh+ziXZWobkxIkE-Hq3Q10ED{8e!vp}6v%7of%a?r3&CRbI zMlX5$_?Yx$%R`rz-``s5@{5&|OO8s4h!|ICk7q~~6A@9OqovIqEHO?67EpAMn4GMg zz5W%7d_V(yM5;^Ao$`(#WPWh@Cxv(6SJ?Jm{vXr<@ zHeuwgao%|L`+9KqiQ(blT*>Y4-YJW_&!3ZOhv`rLc>tHv1giC;KqDtArWA^P@BF-V z^5E{?rk#(J+uWyQWfqd;O&}($9?s79p^F*q!H>eqpt`@`7YjV9>)g^T)OwVeS>pa2 zZp5}^`8qv4J$gx-XbRrkw{PEaEyyV=rn zxR4Opgw=xA&{@LV0P`x26S~4u?1kQ=f=+9d z(}WHe{93(4IG}0eFJHd&KUw(L{jkb^wT<%kl{S8SQ6V?>X^cDGTi{QgyYGJ@aPLQc zUXD zr8uZWFD{P8A)qL* zmvR`n2%A3Vg8*h&$#G`j(M=QVKG?7>Ko!uo=j`UzVwLHC;`Hg$c{Uo-Q|_g&UY$pp z05DTNMV>x&@?de2w#i)u+cAyFSjV)AB&DWzP{? zyFbdc=V~Y_%xQxaNPxoX#YOvM=B3UrFz*=BE{Sw!hwJ&4toC5l-cL`D9;vV^Huyru zVKC=9UgrZHj9f)di~KyQP%URl# zt-Lx=IEeAX-gkCt?kX}Ryf2yi$dXf1QX(-WfHL&RZ3lo`pao$f<#vOr4l%$|);_^h3l;!0{M2f-LM_-*zl)Do068A^+GeectZaY4568im86$2bKh8zb|<2_Fj?(`lr^*FKLO1 zD7!KP05LyQ%KS*k?u4SE;%+9eDGbD@&O58}bg=Y_YNJvDtDPFCZh}^ylIC|)w->5` z$=K!-H0;tw`%<*|q*~2>|9;}>3Z)PfcU@cV{V+w44V3HoW@_&r9haT_8E}=9h9+9- zXb0tpicm0T8$%x)Ca_jS`}!wq0h$Ro+O^tPo?rkTB?EMQ_OR|D9x$Xl8S&wT?yfEZ zXsa6f{MvNGR?hJ5g!J^{t=AvN#ta&G+D03_nVEczU9O8VWpmvI=x#k)`J%Grb6U1S z@cV(!zWc&E2?+@R(S%4kVe>wF_6*90cRp40-i7UhT{om@X|yWZaMI}eM-4frfHRuH zrauikOtJA5-zSC2F(oVq;YmM{9}6B{dF{rTLkQd&|H+0Nb`N}j4+XYoOZ z0DyiE#H-stuhu-vgq9W;7nLhc+_JP}15}SlOWswYyIG^fCGYND-h6`mg43_g{mM0O z`Zgc8+^O1zBTy(Y$O4VWX-m28)YrWz*xpY+$pU+)V)N9gQ^?czA4%bYQ3iK@ezc2> zin^Pan0PP&ASvy<$=&9;7%};;w+INLy+$6oL+z>d-Q+?kRN#>xkKr3KbW8yapm1A6 zT%4QLzjNQipl|)bW!$IpRvM4Jd~^tDF3ABZo0^-u2aTBf-~pS2gu#=ieH)gTWrJb% zUwecN>wU@B=eqed9xND6)B^5k18}NbUiov>KNczryXs?UppKvr#Q`MBfjwjW^^MR7 z$g|KMifQY4_x;3OrbC;>3gn@la%x;=%I=Iq2|>|$AOpa))J5f$$Zyt9KQl6XFqN&zX{0_v#39U$={u@*d9Yy^d)i+Lf{GW zAYZ-lT8@KNZwkWz=v~D18{YSi>Fmb#2A)zXG>SSpDF@4K75=XeK6*NF{S>-E4?Wb0 z4QJvSW=hy1b=OQo(#9blS;Hvy{b{3or0z?gsuKCc z?qZg!3?7|(&&|ii#wKCF(}kTtj=b`uIwkgjppcL}$SR$*#x}Eab6uowoY;jxQCt1$ zHPMmp6-OgX1eh|kqC)5tA>r&*9bH}B)_3nHcK5hGg%MD`@>#b;!k8ZeB!~m%ljFPj z^YfQ4A=2lhwX~A_OJbd1oIU0XP#i+D8baNQj3=LP!}l}*A-Va<`#||UVoN*!*N_F1(ndC9ppJ4 ze^+|+{5em>myhFZ>Dvn+ORQDwDc^|$USztQ|Av2iwh*w)Jk)P!=5ZkQp(-2b`2s=5 zU%PwXty;O6`5G@5TCQtzbooFBkjfYz_ zKpso2dzgG^U~++Mn~hbw*bXwVG8Gk7KLfo-n`&506$%-5rK6G26F=&y^aZ)(RAEI7 za&k2TgE8^=2i$S?whN;Rt4V$7+jGDtXGy+{jz13IzQdQtdTa1S^_=m8K}8>LH|5dv zFXv%uCw3Fhou%IBf>t&If)2cT%Ha0#NX>(soJC6Z_ z+&&R_Oeiicjx;^Hp8jpHsAJH@hjZce0|CH7jRA+qfs-t?lao_xdpi~qPDH2wWQszi zvggFpx=Y!Mf(<~;Y)M^vdU|FbBqeV41IDOeR#Vv%qllEV9(aBhl*lkpTBiZquZ7M) zdn8XW_a4lQgibDqdLaeP?>%K=qN2Dw^UG<39pC4dRKk87GXwQ6??B}p+}exEabRzq z^~+Tq4epnH*5szT%&r1OSC$m}^{pc@&yhc!-k>j49}xEL#UiM7sc%kC26lt)vv2D) z@S&>Mp7dDVWq7lNAi z^-~g_!uTqi_U2wi@Xd_gxZ6bLQ@v|z<7B}xU!H=>f{lYy=)gXDAk9L;8B5$-4uB@i zFm8fv-yTHK6sKnx2Try=8q4g)#uFqjgmn`@JQT1WI*)|qJV<=-0I}ZkFff90GR`u( zGPS!tx3)QpH1+h9zW;c&Gd(ji^T6#OCtgxhd9gEqk79FA3}m$g|DO z23bHP*GHl>Rtp=ONKhLQ1dv~@tDKUbU!C9OjHrvZn-JI}SGbZ*-`>S(i~9g7WpXp6 z1?aZ`h=G80G;$PY+ZH9*Z;K5V5Nl&&2K)Jr0y}pBX@SX%EV@mZx!7F11>6fWfJa=T z4kP71GCWN-HaFWqhKS1eL_tQza-8kxBkh`)83AYwl&WsHW1zXfT@6nk<37I{&tXG z)Xv&$mo}R(Dxd-j$K-fy^xW~D(RKZL--^z`2l4Uo5)Y-EC#C0q{_N~B&+&tf>oWI= zF0T@-N4-k>n>g~%3#v8p%Mz5pLV)VIHuK1rE;YYQqF5Mk!Zpb-f2hUPo=en0QKLQj z$-pqD(w!QvT)Beb?(Y5y%0phPhlL7mheI9DKCJ~l-@)PG zY#<BAVk?x&=LS67RGygV~#9M}d97TE}NiMh1|Uev3JmpY07(gZR+c~(}I zvzJ#YI}1?vZ_LS^(;#kNJ%nbmW+w(pGYnVQi(6Gp;CO*11*40OYPn|y;ABlh z_X21}BSSp>WuU(#Fh*y{$U?z~M`dsX-?+-jkt+-B3mgltebaf) zqA~}wloSJqs8r`~OV_kLOijH9=)v5=f)VT)SXHQ02zJ+bFfU^mr0Buq``q8()|;o& z)8Kz6&#n5+8%#v=#ti|m6j6B%;HCHGPjb)Y@mL_$Z-<8T7&|sE4O<;SVL%l>Fts|Q z)+cChvMnqCf&l4^4>YAvfb*fDp|{(j$o0ytH4?-(R3BYD32>$b03Hg9!D&PN?~517 z0*`jtQRVpQiqOXQ{kJ0{`VH?MgWQTbOrYX1GBIJoYEx2D0!oa4TM`1dQTW)%px2It zsmLBx9eQwog4&5%WSwQOT1~(>UJDF_+GDFIMm`6MVF);oAbb#0QAG^Dwkm#}pOX^; zrCoPEwH}m4Q;;fASsTWdoZ`_51oT3HKtzp4dv=k+`5eV?$1|{cn^CF1pnwCQ(6PoX zUFa@L*4DPRQGoW$0GN9p>E2OxX8(q^RZAX`2??=cn!8}f( zisEKgU;iN#q<%K8Du>Zh98Q56^%iz;f7}RX zBmhCytdf#=NLZMG&OB6MM`>zmx-xNmRP{cfVecpRE>X$_)WTF&jyfRN164J7g9BRK(B1Ru<7aRbI}S53AL43be#vD1Ww-cJ%#{v4odtm z^!whudv^um0QmJMfPzo-UC98{Xx8z7Rk>YgQ;hGlg};syGzlc&_|OZL+>zg=)NCMps zMkSu9^-5S2=srbmgM)%Ox5)6IiNXY9)@iy&sU~=@KG2qkRdD;b?@BM9S$ws-qw45J&0aTvnmLNjr zM=H2q8r4Uk?kWrts%`uFin)7uz`7^AXxoPe^jZ-}jaI5?ZR1e} zmvwS!i4aQNSYsdq1WXjb>4N^vND`@z=2Yy5h~ci=QBBFtZh@(L6ROlqQ!o~49Yb%t zpOHbRqN37Qq)m_^?xg@_PkTHEbrwK*LyaX^JoleGAp(*Jjzx2SKRKu+$t6A;eB+xw zNl8j9&ENo@2Z{SJ*pH~58ZVusocHS0tDuk&EHU@_AkYdqS-%A}pNWtK3GWr@qc^B0t`JWj3v9{8T(<*w-{jWPX!V_KJ}|)|TU+J8f@uEm;S|ct ze7AqaqAv5bBAYc219}u}YQZR>&G_i7RsxL#)QM&&v_}v7%583d?`0?%R>`4~!gB?T z6_@$XjG^J-GE51m_W;B74pwy3R~C6SwbP(@x4;&kT3U)lmH98dgoK2pUdu-I6Ag5f zd3ZvEx*8;ON4k|MgnYVp?WT?C;d3@mJw>z7P zUwgID20~2e>I77Z4YZ@qdlmES*|VPq2O)JOL@n$X^H->}Xh7kS!a3d__zk}UYT6&} z;t)7P;4ALgt~4+BDPYT?u@Xp8_5Tu620VOb=f=I?@i$44nwlEajtOeEpiaBb%*1qk zMvWf)Dki4ANr-dMLN5UMP=BKCE`)NfC7WsB{r({n;rp(x7MNBvmW71X#*K`O*otz& zyNLhg=NPa-iwg@(Pfs@kxTPNz4i|8L+yEaBF9tFK0s=oMwIYB(BRy?dK}x|J^H2Vl z$x~_((Tw}A|4DdB+WP)dvqQG(|T3z7%>jzF_24`9)q0L$=w@NG*JnPyKln z$cg;-i~irP^IyLeaq+iE@b?o4&Hg_>`oF*Bn&K#i-}}~SOvtJ^+`A&D9V&e;NGTU9 zSo-Ob`rl8dy4GoPMOBretgKf<{eC|DeL>V2ywJ~I1cM3-MFA&Q&=Ghjf{~zv9h9RVF>iR4F+u5mW@hx`$2=^oZOzTuY`|M9=&1g_Q{5)f z6);^pfGT1DE0BGVxU+T2w3O<#<;Ui7n=hSyMhpxL-rnAmfUw?obOgV6!Ha^hjzspB zp`mKp66uQP&)w&LoP{f&$WJ??QS)U`I5a^+h=si!KnxVlTXy0se+EZ0tCwvS?Yo!n zY$Yv-dM%BF0XTw$&r?A6kd6w4_{}A!@fZ*#(1KH7RPS4}waZpojo{$;oENr|iHqQh zQ|AaRE#>U(eQ=H<`Z!RI@6Y$pW=p=rIX5{O1SK{Hlnm6IMH6-)>!5U`m=Z{j5=A#@q1 zP47?NX&Ky>gh)&b=$(Mn97c%p^_E7+RjV4@=B^Zu$SW$EtxN{IDlH7Vk^f0o&RXlL z5(_3?n2eqtRZCNo^yn1v?+@;|B^ixt zj$^oqcM|Jr=UEyWtRO!_QEhPKF{GC^2BwwrI4*JO8i;CKU785IHp=+%$BCN{2A^X9^FQ+GEd$^_NaLJDn!E;=q^3s+6;d`qm%6El_$ zk^)p7JUJ;H(Pla{+~0nmvE_5Vnw0yZp2+pS(ftO^+MOK{AP+PkN-FsI(E^6(d3r?_ zYCq-S{^`4rA}t7mRQ{cutxt8Gslwqd zI9F5C!rbx>X89#SaC)y`SnzbqUq|i@ofu>Jm-U+oO@~F+qVuj2gTeMwnT0F3T}S$ z0Yet3eiu~!^L7Ig)is2vsk{49O$WQLwJN)y^-i}D07eVH5)uxPw%yM?oPM>C<}%${ zw5j%5zU299EDG`k+!foas|v+B4wJtc@ZY^_D{-^7vWm|P(2qJR$OdullQ|0hFI;nQ z9}W#s!RVWIKWQn@?IuE?E|=Q&yMkD7Lfq5--6J6y?Mf*4ERZpXR*0oVkeZqj0jp1^ zB>d*%ofy-0H4FQVv)IqcOv;>#C}%DAb44E;^{7iflqq z5i7D@8VxK;x{nnEq6!o;6EyY)^AUBqyT&a)Au%x){+^rV8}d}1fD%GcS{CAJ-Ng~H zvBAmOgdZM@>O$!c8mU;gy|>Y9KE67m2&QXspOK;>?nGeYE4h<3Wsff3X^>#VML+{e zWLLvdPjlu0w`bU~Ika#slopv^pqcFASHwiXK8PBAJqLj=G$J1d^MWA1CK`2r4PNYq@N)se(Ej(aV1oS4V;^uP2PD7#5)YYGfZ0~6kRZjTLuOwfZDzTUAL=l zol9HQ#-cyYD9?FsfaUS7^XS>io`XEQ%f+&?AWN#{7%2I;-w+5pU;`D9;8Y zs9S3JB&axfYkY>1Xf-o8&zpz(x&h-5LNilc*|4)Zhp=RalLT2C`Hgp6Gj%GP~-ZCd!biFi(Fv=%Ygbj|CwLF_1mEG zY(hebvfoQ;AzR1};w3MTVS;H`{>W1ZL3(;LY`b~wxQkJ{0evwL5!q#Nv4cg0+;%Wj zDdZ6rriAs*y;}1);}#|YjPrTf0jO7n660kS+(;9caW7n%@6iZLfZ0pL%na1)5}%7i zTTg#Kj(T+>*O{JuB@QxaMa6<-DH)Ttm;g2uk7zx2xw|PH5rK!N;g7fFh202QrYulsqYlw*FK=*l<>}Zk4szhJ-%g;e4 zSWwt#5#ev7jSRuv=5la+nxB6U>@VRD8RDdH`k^I5ylV1W9~sD`E1UK4b#(>+Xz6F# z)%XVOmPa<-fQM70UwVNO>)R$K7{Ss(4gR=KZwk)2E=5Na)SV2&j0k1osjWLT#W_X2 zG&WMG__fp2wE$wm=H%pL#CW}7Pszl&3h}2Y+OS1dTTVveK#0K3!~wTr zV;oQIq@uMP!rpR!btkP51gWqj!Fmetfe#lvd{sXjHqV2!Xh-mA3W zTt~rj51k0ORlMXJ@omRPcA1%uj**8aM&Q?Hky!RwT3eAOvua#T%`$cMb3N7*=pxnf zz{3toPPU2s>3Ur>+Z4=LaQHNS?6llF+F9@RK=AWFko<*F$z=H@B;+%1Ljp@(XG?bG za}*%%4Ff5srA78cl9#q^>gNapL|?zSCZSh`*M_8tW?TUk0Gmnp5e7!U%KExIq)B0< zK`f!!SQ1Ar`)&+)M*R78xs4~kf8Wz{3o3u<-!Bhd9#^Q{|HeaD|Fd&jX>pZU=PYEh z5Sc9LQIn)JIVQla{hm~RZ&hxYkZqRhY?eCW=EfW$x7>YBp;Yf~U*Dpu+6SLIccz+x zFMID@JT*Lg6Zs8;hla2l8jhh(wtKI!V8g;Wm#kdsy~;n|-&J8?R=|VK9#nRdsb*GZ zk0pMo2D5ewZ*Th+SbA{OBnf{(ij9rh(Glym*$smBxC~zH(KxzWSV&#{G9dQ9_q5FV z_m&crBII!$r{(<~Rkm>I96qw#>5fn(#>bg`lte&q=dOHo!#ypouxJ%K&=9YjBZiPB zK3KbG*X(@3k1ja$b=^$!f%pLA!2^kp;YN)FR_2jghbSC}4w*YlBq~0G( zL@Xc%00HFQ_yO`i%~Ptddw-w5r~VFq+QWx-;$(j=#>9ogHd~+>6&;O(^u4&HR%pL5 zUrHVKZ0&-u@IBBU{%fu^O*P}V$cL$IF8un{1)${=@;N(<00Ig6Qvylmhviwa`w;X! zp&0+<3N#C@`1V#gvt^^xfZ<|*kqwQGwtQUT1{qFmRulq!D7*#(8Z3spVe;rP7;^sl6+8>T^#Iion4)V1eUd*9w2tj;Z|l*1r=XMOxHC zu5c3LaAhS+Dm$C0=lMfH@Hhw%z*$DMZrExAN)I#z-8K^*dr)OS$>c2IHh0f!Tu^Mi`woy>ZQS1-gMsvtv!@tyN zg73LO&56u_k*fjMF8MmleR=_&+4+r4!fzI!l*^cZQvnd)>eTomEHd(BP;-wP=xcYY zs&pSt1GM=sn98>7sewTHdu7jq2t}OnlD60gadaDCMnVW($Ki=pNz6^L3bclxbMf3Nm_*YpF z*lMT6#qR)jWMcVNdf;tfzypg2 ze*q!Py9_AWXJ}|bK-?{~7j~Y!9R}}>asdN@ zGRo^Bj?8}@t$6?cxN=ZMMQrEHyLa!9Aowo3rNBrL^`Z0A4waQZyP`DvM7x?>Tgkx9 zg~*}+8%t|pWS(b2dq3--vK#;JZU|{-N>JOAgXNew%M3W4(>n2jK@+>pnVeFQ@iRPm z*643XNM;WhW%Z4=_v%F>jI$812dSH<>Imq|$*0bwI3A{^eW^B@mX;?jT)3c>(qa1i z=IdvV);N;e&CGHz4wfbV2n%QY@j#*RjjpM|0lu67lqM<3Ja4Vm%So1pBK20_r%!yu zc-Rn>0T^Aa>7%E|@#HcAch%8&|7_P>ava2rW=1L+F%SqAX%7Da>MBpKJNQ>QEZ~B2 zGS3T>=j`ct5*d2qk=U%&|LT=Gc%DTD`>#t&ugrZi1d8>_cM~5(*v_urFJDS6KPp$; z5U+rPUv%C_2ZaX{kXZ+ap$Lo~y1zmyUuL1b*Uvua+s?*QLsKpLt2-{l5DtRg-h>~U z{+CT(=;`r9dEN(*qsspaKp z@WvPg1TYvG-vZGQ28T2Hv7^5Jg#R|y6G_65%uN2Vu@l)ADwIljO@Nh5AOj1#p=ETm zcx;cG8*nZl#&9fGJ#sG20s-lb1&h2^O5w@eV6%Z-^Keob>dx z6t{m}7I2vs26Q3SSU_IMzup8Q*R$s}4w8Q8_!MSXs z%i`cQU_t@A0@Tc5)U#=*l=SoShp$hRfGmx^l=iWEJ1_G8x}1`hrm^@kad+28NjWMw z&w>}k{le$WWca|!uoLDFAKv7lP`Ur;5n*H~KWla;2ui|o$pC(dDJb4id0ErSkje;+ zRLK(JdY{^s2C)hsKWg0M_oOP(fyi8T0Yv7wq+)aMBQFVO+kTf-Wzs6zn1?L>Vn50w`)IwZti_& zV$hQ+IjYz$u=>pw{PAziW+>mbidOq`Srkp(MdNUPU;jg;`+q2o(>dTu!-s0!jF21I z_0Ri*nUsG6&0M-`o!T=0!BbR?O2j~DUcS-rP--)U#afVj#A+&_sp+<}4(BZ*=CPV{ z4Or1hGjYVrQp#OYSB^1efxq{+DssXPRtjxzJc?S3z>D2-Ft@q~q*#`NY{iNbR&Nh( zD2%=WFeMHW#?@BLZ|stTo+53%6}Cw6TW;=e(QxdLl8AOFBwwusY%m-|uQ5O{CieQ| z!an~K37;5tH5p>^O?)_b7fpG5*Qnm0ODMb(j*R5Np>Nv1THM+Uh$6xF+Aqb>2%iK6 zuJALJn5R7mDlUBnjE{p4!?X zoSfeg0Bq3;np)Guac~M#|4T;M2D$BD^|vi8U;c9-A_~aTl5&(e`S>Uh^(x29d1V0M zY(*Y*c8!hEe*gY`gec_ix`kCNTBK!em)ijXWk~J*1w=ZUf)8ANyRddRTS^GK8dZtS z+CLRN7j(A{1hN2@yI_HKm(J{JTAKCZ*M;Er_>hFuRNOmv?o`+KqbHaPC9HgXsR3QJ zbajyeka_j0AP$O_*>r26t(kV!v#~G&t0{nau)bTWCO*NSdXBxB4TG*IA}Yi^xDOeR z2>*T0sZVqakh+rrwEE!#EpS)a(Mr1WrmfcG4kXSilSX4TGa(>h;e|1QTb`qq*#Ni$ z&)xlKyXT@srwHgxrayXTLBUMzq>5vZim9**RLoa31k`F9Rt86oX21!=BlN#JSmLH$ z|Atk+Vha94AbdZNwCx*;oPczQVFNsw)Qg-kONTxySldIZfR*ptzi;A9Ho$h z-#T$>2XlRnAFQ_fspMp294|IglaeAb10KbLMEoJ67J-A$;p!aN>S@vkkg#%rLyYLc zbeUnpKpaI=862q7^fsM0~4gXNpKKUl*G zQ$q>NxCs!rdK>a}rHxV+ed~Ea!S-zVE;w%=0vUJE^f3{rTc7i~tnMqZz+KmAo`MO* zKsJ8P7M)+(oP6t#z%wb>J3){<2aq=4z#I%Mc&x}-fTZ7Q$1wl2fb0t>>Bg+fxSv@1 zy7)8#1Cz=?fVUDN#}pjW($1A%c**u#1R#@S&Nv`?aEveGLIrOm+x5o?g0jX~nn5U| zhUSN(RrMH1rvJbV3m+I*jeJ!!J~r83oVqqsW>W|}>PCSv5J=!HoP8KU080T#Up1*L=q8A2EgupuPh;M6e#|UZ;KjZWof*7x@ zO!DG#3fxleL6u%zZuD}Q?UaYe@;0p9S5+nPmLK(>-m2>E0b+o^38xbf1gsODtucse zti@9O)%U%3FJQuPOY4CTXVDYJ*WsR@UQK)k@8@?)zkeS=NBxGl^5=od^EV%+rKxuo z|Ghz9um6DLgO4^3M{-L_jI-t-e6zDVXDr~mAu`sm6bs55_K6ddeT8Olo}Ehau$*-4 ztF9~y2?oOQpqaDKur`25R}jhu#e?F_=4+6<8PhHTg^*w^GAo?NDcV?Why?YQvV59Qjj0XV3M zpn1RN&#xut%$nzHJ%4VSwCDSeAZX80eUjCROiRB7CJp6}A2txI*Wqfsvwf-6)TL%~ zZgCM4gg5}6a&n>E$@jv-Qfz(}Rl;LO;4IMz$Xc>>Ldlx7DEq2WVz^)1H&+2byjUj_ zJ#ZlG=smqQ8x3r4zIRqI?4v+X^S4ZU-g*McPn;PlU}R0Vy+#nq)PcOXus>UH-VhMb z1_d1*9gs_G#Qy7>t0qOZz!L^u8fw2h9EOKSxXrzUJ)&3di&Iv1Vb%g74SKIF^ZeL* zJHJo2x#5dFnSa3pJ8muZ z^_MhI;~+!zcWPx!)j=ZO{zV7LA>g%~d)vp^EUCX!4%AF!_)l|~`#Kr2)SoVOv1gyg z%P-N#UQRBMx>3UetDuwdY>_LQFh)@3oa$ z^TY%t9C49>`X?iE7t9bRFl#78zm}Ib?Ek`*@TWc@C1ng25uAbHyRS^(bwWr`P!@C| z2&QN|9wt-r^Q(ckO{SwG^tm7b>)Q2VMrLL@SoU!8ST5uQp}_5?;|E4UcKVP~J~r*i zUPez-OoE1G{rM>)029y^VCE^&+721Dp5oME%|HlLhJmNBvp0^VUY*c3ntt{DDOwTi z;Na6PyK6Yt9*a=(>X+YzM`A#6U0Jih21l~gydw}To&v(ZqbJJOImnvEnu(dRl(-M0 zSxs}FeS1GB!^Tvk1W@q;NhBFbDV0u$uB~_yNn%7B^qepiqCl!_3bF!Jvkdxo! z^9PL37EXAp)NPIedORemh&&cXOa`9&P+hdf1G1E)A_YlO9pHVG072e@5(|9=?tl(X zW3;wLPE5kNZ;rvE&aRcI9QEQ9UQ8?ori&M`VSN51opfaf7bZ{U6j0Vl(K*A~HrJ#PTA1M z%u;WHr)b!OO6%X8Ypb{Vf0Kb7E@!E)7!jtYDCHhx$!m2aZLEhE00lh*!Ck5uqF7mL z8S7QTPL0#|==DbIIQ`)9HkJjJgR2)^tV4B1tC(3?32<=amw#QC)9SZbw+jBlelYdpnqQp;V!wf+BRp-Obt~z=|(QZQwvdmj!RCVW&uk$-fd1b zx%z11$7>TuM02j4)a}BimiZ#C`uFkLv{(BCW)+h8A zKepuwf#7K;9;4jWUm*n;#!mdy-zIjNdhovog#LR<=YPmu{%3D-FQ-E|Nnw`oaYl%8 zT7d(ykQDw$y=3xr0^Zz#7lxU3i%N!t%m$V&sY5;HJpQoJlb> z3SUWbcDQ$g=MKRgo*xz1GFeZ@5?#l~jZi`iq+iz4lLCP!6MTfD6u~9Sjox4vJdrqg zMXQe+*aMHz0fFzPEAJo`S2T*-pq4@O^CXN+SNt;g81(d`@GKS(szg)Ev&WtJDNp7K%uxKSZT9PW{nWZ+BG*nxeZl; zFwq8wQJ?1TA8DM|zY{v#`|HBVX<1(D)X4ib>p=%<5Y=U0+Gp1uzEmH` zy(%~~DjbwE(~cr4Zk+?nz++tfD6QJ?Q@bRv72#nQHI=d&c3!VH!;z`0X#zfbyUdwxM$FvakM)l*bZ#F-_ z7@c!=+sB6n%rz<*ctnO$On~!fI2Sa)D~$nI@F)~#$g97)W;gM7+a-ftq*Ztk02=I_ zsVUh&z*MM`3sG1~1iYzw&k*h~NRZx!?&R#eLj}(&Q?$2-cqeA5->xRG1hb)6XYe=S zjLCV25f(XxhEOQ!aM<|_RA1ef2F+ho4d(hq`<*lNQbc$2=d@8aR$(WcZS&UiT1;LwTr}@Sv<)aC`w$ON0=j_shQ{^GDLq z(nptBZ;hL-`>!!_NSeuOX|e2WGi(_Bs;l{56`>Q%|9^?l$t0#lNt=)8=xEH=El;Bk zz?3({hg0|rBjAwWBs@`pO29Oxjz@S~C}7t@z}3wSp3uetk5nU-$s$G|qV4wK(Iz28 z;XB>c{@|I2PMHFDPGC_suz4+|_ZTQB1Xc^NYf?tW_}TWEX;XN}-AMpgKI;KV?D-HI z6B7K8d2IG@2G-Ks?FoLK)X>Lp)|2!bum%9bU>;DQLk|SmpfTXZkXv1s(CFx(uJU{G z%HnXyEMOtvb8{v6_#TA-@kT2|U!frSW^flU8jgHrN(D&&thi)$cyL+~;>7;Vd~xw7 zaF2c_^NHU?C;eV_Ih zhIW#ISxOG6T!mQ|h$e=zk=f|2uayh@T{Yz;Tl%tHn1xE8`>?+GF|?%2t2L`-Ls&m zL(WQCXW)21NJUMp{gDv&EDhEF#om`dW7)24KaEPHK_jA~42dKeGBgQE#v)S#Dw&cY zWN0GF^kgVwrVJS}XI9>XqJc=J%1kJc5dP!pdD`!OzhQsh|NZ-4|62dr*4k_BSC9L- z@9VyX^E}SuI8M#^;N^kwez&zo-l&2%kZ&`d{O$slS$}=@RnX6|rNI@` z`MQXur4Y=a zKJ#h`k_+Bo7s9-QE9O?5dsl-SY+-p;_9R-8z$%{0OCCe~p_V!o7%0b8DpZmT@~@K> zQh~X1e>^TNU4mTKec|%^(IBMo$Q;#4bLe_8^y=Zq1JIkVMnv2)l-n!`6-6+F0@6J@?A z!LCOfwpD4uVXx-ho}L9@y4*VNQNCVuy=IK_m{rf^xT2sZ_Mi3{xtrQ?9A54reKB(nb!4!0-#)EyG1ryibyjZAtXN#uEq^VwbOA*`ptE|Z)<=b3)qv>XU@G-WSeV(n z_w`4_L5SCY7!`Dz)YPf$rE6*1tFs&ShG>>2paKYA=+Y!XQ;NAc91=I8{0|Q(Y_f!l zx>}^;pt*-4ZMktSKuK2VCbouR=K>`?*SduChnbma4=?gxb)&e3N!cg;F$%>Z^MwoF z7;hfX=zE)4_TjMw8Bm}TB<-!JSPi`nre4wOk{Pa^o-Z5yX$O($)i$|ArGrXJ5Jh$? z>6ILgbkUOGTeV^|NpsBT=Z{LQ4Sc7KwdSBcb@gu{@N8r%TIaWJP5C(BY}j-{lOK05K!oXWeNk}f+w78`Kfnb6(}B6Y{1ALlG_qE&pGE3}a0An%$G)ag?6h%C z`h(wA-2w(!?=nxb0D&Zy8gPdA2+1ALksZWk7r7crZ~;Af5Gts++fnd{bt!HoB{Nd;DxK^9QfoO0x=asi32QMPJri*fdt@&r{`lRel_bILL(v!JSR-7_ZIwpedg6WN&RixURzyY zqp*b@6Sw=ETqU{J<2woQVe)wOShktgf0pVT813I7EzOJ!5wWCaM+Y~l%>pq(ZB{co zJxNz%@X7T|D|birTlVRM=`m)4*8JhA0RiBl>-m<&U1T&tWc>sNDF9=3P=P^z0^>&GRo2;W>asiO ziz|%ZSbZyuUgb_)Nk*}Szn}O;!x9Sjyj4v-6@2=OBRxyO@^Ob^Mc|MjAlR)4%+>hB zoPPN2_>Z!=i3Z* zrT4<+^1Hd;T+Y03kSOb34#@yf`-nd!ry0eblMB}?w-SkIjcjZPzNUh zMX~w(f<+AYvU!{VDW5 zXW*XSE->3mahN=`Ph-FqDPnQ`3`-ctO#7Jyc-b@DypvRv{_t4xDL*S0h~wZcI)3?g3(!FL|Ra(Wse?V*+y!9jsn?Wwm(ov@*1vitn* z){b*xNriT{b7#AJUs81VhYNgX*nc=E?=#j4dFiWk_m?(`0`Mg%1+Fm3s=4CONMZhM zuTR#Q18FT@+wwf;B zNAqT8_*XK%ZF`VWC+N8|E$-1b&wYCWZJV3jW8gE!_-sDYYCM}g$%apySWn8Pavvug zkcT4pa=`SaSgQ^m9IUGm-&)pstKqXGH)lj}E+n|v?15VT)O zNrZCm)H4;OhI1AM%!0?n1=w$orlncB&0FMxDyV^zZ+f)D#(YmqnV`{So6u-cI8ua7+b?x9fNdn9ob3Ok3ad$#SN zxM9%h621%pd&T;E4wt_9E>myyql72`ELfpjU;xbhb8^U>&?enJojcw2LLxz<+jZlR zjSbURR4&hV`>!`Yf=ToY6(Wc&+Hf7J) zA5K~!`8?}a&J3|X4L{ss3%QP3=XtR1;I54`GOVx}iezM>K0e-Rxv53!1>i14ohXyP z_Q^ZyUBswIX6b5wdN(7+SDvWURU)SDhT``HF3lk%MEDlQUx)O&k_&&+Jp0 z%01Vx6sQ5%n0sN=9I``0xeq?TPa4M7Auoj%Wut33m~MpHgxa7wWSDjJ{@#90PIGTc5|1j_;dQ-?^z~f6xylzt-|WcT%HcA- z6qd5s22a8rCC*2Oe}B4S-lwX>KGob=Zl6a(f`d29uH@uopeThh6;?w1@-pIjprlW9 zjm}zl4nM1nKSAn~_Jcp7KpfyW5_)CS3%97ZU; zuUzAszcfQkEGfQ_ZOxifAD={i9GqT+BlsGW+jew_bqsdy+ZU@3dBuW53up{^&Bx>sq2Q&s!M(rGaa5^uaWZ6=vndGQ zS2z_zD?i@$7CuGhUr=N|(swb@)St%`7NAMN(woXmk8AVLG(S(F{;R;CIyf})MOs)zcR z#)uGWKpLIVDYGlkTH75edVp1W=fHu!pwu}w)OG;2fxxZ};p-dhB&?37U8?#K3X0^?HRFtKefy5Vr4AiEE;lzf44*=X^6t@B7E0;8 z-B(xK4}0IZBK45{=XdL`e*;EL>>2|krolBJ*~E2mBV;M+UTu#*p0XL3SOfgX5wreCQUVtk=Wt*dE46^Wl9* ztYpH%!+|4RCarU}BwJm3={9SQ@bOZLx1QnoTc1B?aqRItvrz@d?C1B{`qM*SzO16S zLGJVkO0D5`-P}R`3!KO?_d>f0XWfOau7-rbgV9*9a3RQmMS#(DJ8Wv-ObixqegTaM zOofkKVoL;d^X2&Gt=sj81x776ymdM&1P;N5hXMXN93Y;aIDM28RVJeRg3N~yEwCC8 z6dW;aD6lsKriY@?Wef(-@UZn)psXDo9rsYlfl;T|o+*@N;gHSw(0S72qNBleC_uoN z0gaUfWuX(s#@9q<1p>0IM9AK%c2JjXV&InmouVA5#_h1WaS*)ga&>A`yc$oeH zr2&GC1u_!}OR>i?gMRkz-5Jms-0>)%+gzDoJiwm?36=!3=J0i&y(;)H`q{X7uuS>i zxkEv5PP_1;PGZ_+8m^NRq!b!3#|_?{XpS(CVKKW~WF#lD8r3udX=S(}5sf9uqhEHy zV*tt6ug)IB!%y&n{b1UJJ0LFRt3`|!=;`YVd@)s$-{8Mz!^#*K=F<@(?q${99ZH7| z5ui1$T}{m&lnBk&(IZHSA#zN4mv>%)43KFouP%ZvTMPx4%Im!JuVXj$g@zk=>zD9v8m>!lxCIWB# zqgFutcR4J(0_OZ`Ha0~4jQB=0iO~+#(?xHpxTL8XEkG)#H< zOzpd2?17(X77w|k0f+Ni;PA<(H<@&IcdzH;n;#UEjY%OB#mmYF^Peh-Cte%<6?TLK z#8g@0=pm##5cl}J-8WYA_I5L?8Ar!#ymtNiwsMZO@n6F?JBg2!zA-rXQR@5#k-a+# zeBwR#Zp?aL%)!T}gKWsr-ZJTZ*7!QBgpU;8kLxaRQ0u=s%k3MIqVXAuH}H=uD4fSI2Jma~s;ZXv*Qmd)`d#>Kt}39sR==gyP9roBbJ$npD+PdB4_8Z7U< z7ei^vYHC83C`J((LzQzJu$(uW)Z=P%ahdVhlqx$g^l`2}MjpIm-8B2e#QZbO@36(I z$e}kA+N>pwguvds_~#FJ7Z70@8nG?>`ZZm6XRaI=@=%wslClbF>ZlSv^Ja^Hc1dou zFX)#ZaN2P;G=H*dNyIeTXWU%W501PQHxCaHpAhnSu0w*begDSK;u584Cel8yjsQX<`7*1a4?~p@S@c7%I>IEgDFbjU}zZ z-&2OZj9UE28TUh`MJOwT-{Ly6o3X&VSO^&E^+17!54DG-Dt#;E(*~OJRuJ-caq;Cs zv+58n&|Mdv?hZnuo@|u!XQ_%iTC^7Gp!XBM2D^+*>lp+bVB^AIG4@5nZE(l~1zfMg zw-73X$H9g^0l_ULt}Jb?vIoqY{tL4XKA~^li!USKe++2yT)mlGrb*3k27q`R!?Iqt z#*k)_xp}}REGR8^6|{oDGRV+Fcgl!W>w!eJwDjH=2^fK$stPzyl=1MfWI(BWe4PDH zUP)9Wh8A|{OOO&gPCw-V`NN$J?KJ3l19hpX6eT+848*r9mRVDX;2J3(;dd%joErc7 zpYYHHeBSdjgE))+HBao7DrS}D!7~G*7%Wdl&_7^-&J4nSa*p}r_eg#om}nplT)pe! zY6MjQcD0BBmQaqv8-aYIFjy_37R|907r}UR`FTG5$^6yWQC>kZPvL?UD{z6J%^j2v z?Y^P+ARz$$027%sD;cNB2_?cb0pAj+4-4hXNZ~OfbOHSJV~Ha;n(>R>9+h%{%196V z{gw#Hu3Yi~%^znUcvk)=JN;v0=0o*oq`reTRT_PEmS{3Jj{=7`U1JBdK|4RvvpajB z_Ki9LS%Bhz|2hZd28vrnMKHw;p=pOLe6p9U{(%Zij7Q%zDiz+)vb_C%+T0ujSs8Un z1Gw92o^ilo9LY43*-uq$E5?F=s7SZb0p4WR}!MH(4XcaalbY1CPOD{HhVGgU2 z!iJhdGj>%9LQOi*!N?|GSY2PAqpX-mP>u}llQ6n}I}$S?PJzGPGB&YfIEVSXmxPLZ zxFkL<4i_JrdHF!ng)KBX?O@B)PEk?*jOL1jhkErTFQ8&cxB6JPp`ixOQ0lsLvdM_p z-*_uqfgarb_`Y?iD0jy;mzfq0@Q+}xCkR&hb|74&Pb4Ts#9e5tt)(-jT1knEH?^>b zG0s`$=E^BI3|b6;UG#2b0^$POu(?oGR=j+9AlG8xpe<>4Nq&erc!}pVM$eTx>idaX zMSRa(C2+qt1kOJBb%tt|(W01%G{0U5G(UCf@JM+E)!b?c8(U0{it?=qkwXtNlCbox zVBhy(i?6qqb2H@1@I4|i>Ba*tDVponj4Sm*SuTN$o5buG&_E5x9S*XX8((2K1Ho@u z%alF1cP?gx9CIq2b^xlqB{V|D*|U~}uk_w^{$p0ni$Z0J+{q&wp4=|GX*QYv!Pc|C z_|V~42HovX6aD^5nW%$Kpz7jd4FL)SvS1F)A z#KOy9p>bc+rT2|P-~f&;RCVjK1F8UY6OcQuoybv9WC82x=hPUFa>TAfi8atk9CTV} z925~T9~#39b+Q-;){RfXmKR~nWaWV~%KPQ5g~&DNFs z%@Z|?@lT)ni>D3*&AoIR?Jj?rj6i-FE3CC+%NAB39Q+OaIscj9wMzZ)&=$+73Fd22 zQA=Sb%iFd_s|z+%I1CT`wv)dh=PH0uxYWh4FJTITuX=lJ$r7wW3rhp^BjRdW&^w9=xCl*MI(^$kE@vm$|9@Y8YPzM;tq8+B%r-*`q(eVZ(z9`Ol zfLr(n^uk)3I{zzUl#=(;#Mh6;1#@UMSm=K9+cPq?WL2*Z>o3PCO=ar1HBcm3s2{S)E1s z`HrUjaC2F3{i&Idj;8|m8+(pkFR0u^4toQgJtCh*p@^tH#E}nSiAVk>cPta0b7b!DLg5j1QA;H(Kpy&w`29vFLnuzM~5R4^VIRIHM`c_PQkF zMQ{`x2leC%7!~aZ!64Ry`nf23^?KTpR%=8|zAy02l5*)im=df@eQ%IXLx=6xTKMT4|R(?Ubqw!h)fadn@htn%uR;|*&%^>-V~ z{NcS05I`B|DSx8|P2Bqj?4R}#E!w4`XErZ8d-}82-^9a2Ied1W>>1C5lMAP3PRZwF z3X}0$k@D}g!(i`BxC!&c@7{Cbqop1WW*ggVcBZ9T=IZz4)ZLP=Y%xDTSRNw=!f$t8 zm*alYZs@w-Zr?TeNwL3Ca=^c$U1Ed(VjG(gNZS@VJMV>Z$=m}RFMn3Qt`#2?l@??y z%E;y*V)@rjJW?BG-;Fu}9j%NN7)5)c?5yX9DWyo@uJ5Fu-w_+fgI0vI;Q6RwyooD!2qEv6ZNM=P& zFZ$lHSe3zZe1ih_+M4;x62GJl&GFQ{@}iLG+I{yY0`-d$9TGd)xS*Lvajx`9^O8OO zcEtZJ83$|z!6BGG&E-s-*WUP*i=wHstGV=-A znBv$ycA2%}cE@@_A!c=pO?#mYAfw~Iz;p9TF4?bn81x?r8@I@WCcc*Oypb@>_}lX6 z9^)UeVQ85l`9bK!EEpJ}wCooW>GdP*BE;SnCl}9|GpD2DhMH>WT)wluVr>j4K5UWw zDlvSD^IXR@wqsPfGEzk<&2v2m$%|ms*$i$p9uy=l0gdB--ntfU9iq5Bu*1UPpPW;J0=8dR z*e&?r2kj91eW_@E#rQ4WXjS0`Gr^OM8O9eUM))R1dgo9`6+%^Y$yS5H8>lioMoSaW z$+tAP{q12}?TwYH-lDB8MrAYn?Jt8qbP}XFuneG06oGvP699Ce*eBpHPkeVBp8d|F zu;mG8s4u3-jCU+24In_Xegb0Ik~M1#35z_T%MrI}>9(Dj0Eh#uZy{mCrA%|bAH4_%2mIp{$wacB5A z^R8g%3vW{SK>OJJCCiTWTuTaIj#}=wY023uKOegB@^a~`&T`O9S}@FYeD{*e2a>t& zZP>6s&V57C=B#TQxD*yfabKfS_`Xd)E)X~ z@1}~$BKhHJy1*ac1S9HEG*@|m zDp!cah5P6j5`}(1@%Bx$54o9DCdQ}DpF3CEHA2!h3Ejj)PU_sQj75vs(@C>vpUL2f z-y=~bEE2ng_TPTX#aC2u$XpUHYNCIlW3@^(ifzAsN*%6Q)Bc~^O$pn{r2P2YO4SI#ZC`TFGR9{FGQ;6HYcKXTQ_h)GL z^5I06Chr(A#+Q^lCRF(4N~`mjerHX1lgX5>Wl^g;Ptk*K;A8l-%esR`%jpnC4jv+lIM&q(`ct|n{#fCt*?(VzqQ z_i0&Slcip+PZPmO5h+O`tJ?ZD-XA{XFZZ`K`~M9e^X=Yf$0QUa;=zMcS9i<#cNDZfuAo}1OTEN5kng+)BS zxcIxq9QvgisTxtu;B06ns>f+YCie{vZp~~=#;zdVB2K#wmlQ_*rk9sGrqrNrs)GN9 z{7d5*+qG!_N@)zzh|+_Hip|F-*C&Gj{$438Dhf}WP5bAHoLym!33!(GnWufR9fDAO zIu{$%HRRIh>z2ZBmXkuU+O~7&?dE3Xt!WJ{;T}(L|6JWx`~k>b!(jxLI&04SGhmPK zAInOpojtEL)P4vD$&b85~{_Pq?MRzLomU1E+<@5Oq+bgSE z^n+J2AN|SGAd}2z|FoR{-6g++{bV|e+@!yM^ziUry=@O%%CS=hD&LpN&0Ad_?iPh= zWQh3r=v}+Ql4e_Z?vc4}#J!2%b^68KNnyTiYOjxeTF^x?lK2{|!aMzQeQv=4SmWtG$KYx-o1zM?k68dW&q2cTlD466ED0XY1 zZId)Q?eqb21erVheK)$3WLbUO!y#@ZheNR{SSYe8P#_+CF3y#@&FXz)v5S4!q4RcS zi@ry>x5np0jjbt7@p=;T3dbVw8t7Y8hcL^fHM;0vRV(OL(17M7J5JZIJ6yb&fVn&T z_3LXOvGAnfPBAPofEXa$y1QoJ%QSp?Q_#cx4F0SNMQxl*aU1q~^`IvSS~=h8stG^H z8@%^vgk)U*ftjk$Cch?$8~cXmt-ARp)oZKK_FG0q1}Jny(vcK?MK8=c;GWswiXRG> z)Pjxjg^Q6xMPm^lP$t6J+!SW){F(`Vw)zk`eIIIz989B6{zpgiUsy!?`P~~To>UPk z`I!;n*vB^t!xyyMzI4wSmf9NBk-i6M$L2^Vk}IIq%0@-|m@;{zE}2 zpt4#zGt{52=jN}w5xB=W`|izcHcbyf{F zkGmbZWaWNB+jZ-aGQ^K9B=Y`6WM>*S}n{beg$d#@erM` zGGUf*W_rcq-?;kft=&f6=j9A-A}Tw6T-U|j;mpNyQp#x-84E3HI>y6iSbPb)s8zyx z`%JxgAhiUaT=RT|wa<$=_V#xXYiQ4rSOi{+ZC{-tI@@N}NCzKrU%y8}+`IU8_8*3l zQ9r(U=pBnc@bQ5DF+-()KO%R&vWNVUU{pzEWMOH*>@g3r#A3i(LKC^H0)4cT^c6*G^L!2u0%J@N`eAamQmMt&x zcOg0hRV9$y7Ea3J!&6cY23A%NaZtpdDct5JL!?QX_vB__H`$G$%+<*y3mfg4sIf;4 z9G)hAr;nnKbIhjU2-A{lLDa6s5iHjL~SXfPS_Nidi8F*gBm(Rf(glOMN{y}a~r^3m!j z`Y%76{4;c4T1DR)x61!VA6$30g+h_5|1cOK(N@dNvj!Azk+{KRz;fIObe{KDC%cfZ z=@7-TgkOqP{%j-_SQp|&(iE;!D~9>FH*QlK|K@aBX-d)4)2n#d0b%=9_{&Sq&I>BY z{h7UpQXNDOXS?7~GJRJYu7kkF!)X3iZ{Ow|k6=NvMef%{++Hlf2KwnGr}f$Jb=Ns<&pKD^QfW!JgxU61H)Ah%i(DuC`f&`l#}C_mT?7JAW+ub-~gdNHWREiCzi3 z;^Nx){tR908L)D(j6|t-QjYoBIbly8+PC*3KO7xw(M1WM9`tCkS4e+*v0d!2AvuJz ztvi+ODsRFEr1aTt-FnO&^y{xEsqLD)4aqv&nK@q}|olJYNvpaT& z+t+~`Yjc)Y@2Vn1Yve~H0#PdWC~A~&5vs!q#;Lb8T-Md#SDGpP{oN_tTqr-bBMbB+ z{`HfwD1NVm;_%p50|VzJKLd06G`F6+&S@d-6OZ1_(dRq8+8yT135cxGNag{`)_w2N zW4QVF`IQiD^6Ta#H#znItt(#-^8oS?) zQ{_xMEL(TAtYxD8$*#{~`Oc$1$G?$y7C28LgaxJ*e(Ob;%I+(p-*@N7k5PD(`v>>Z zZqM!WlSEYuWC$cmlJ{XxM7*btSVQGVc~g@v-f2A=F|K>$PrJID-6u`nT=dcl6f!hO zD;!oz>QoBkk3&2m`Bs?r5v~GcJS3BOVe-xSIbMf;iA&2}e%e2}nLSTbtwD~5wv=On z_Q1ea@T!*n{lCVig;bkB0zk&&I^&8A-7YL55~{Cr{rQ!_l#5)aG#i1o^@Z(ug$DNa z@u_UH+&oAkw*BQU(vj^fD7X{ooY^@rV4gXRKzpr}T{3!Cj=b3ZGM4_>YhG4Y$6hIJ zwZ35g^7DO)Tn=*h254KR&_I)fiX`Age7=urgNg0@
    KC1T8Yyskt9rpS88jmVY% zeX0|;d}|6MOTSYcw-*?z?!m#-yW3Bjf-HZm_)?zBc;_lbb;OcS%F1{Y4}+EipOiZ; zg1=_aBjulQ%e`b>E8=V&SzYSlMrX;HU@ULP?s6FJ0e<)v63tb7Seo zus`!mtihE+2FX~1&AEDrzk{~ud{SOP`{Q4^cXt8`%z1gnXXw8y-Me}3@=U}Le3q?+ zYjo%<;LEG?KX$XAAf7uugT7g}1v5dJtxG@kj1|fv44t(0BKoW|+jJ-@tkMWGni`8v zdA*p*i5Bex11Z2tryd{sjWKLr>|t$L_|<(!K{| z63F=-zIE|&aY6r+axnb`R5YyUJOUV0>MsdBcK8$osj-_QE^c@#LXu(Z38=^3Kq&F~ zFQZOEF8JqsEQa^`BQC@|r;})ZOUV$#y<)Ew5Eaz~Zp;@1%v1HunS>W6nSaGF5aIF3 z$QXbP`wHpG?Ua&>U!y@#NdR*O+rZ#uHu~0Nw0T{PSsmJ|;!IUQb0s97f#_+di6CFm ze%4V^~=!8eZj@ZmVNa3#d5RHN1I+=+jm z%+sGcCrstVa98yMsJ?sA4j3`mZ7i+Lio@T&X^@6rK&@=fannBR7ryI@u^HavCO3N6 z^QJVBL7F}aR;_ccTa&W@O_a$-hd|YIt%P-~Z3~VpQHNopg2SijQ`1@`-qLUi-9zK? zwK|-ixhlK5FO3iPVoe=E*76qTGs8I^+I8XLBYnXlSy@R0I^E`y<6d%=Xca%w zGGwHwXa<*vP)p1hYSgGb*BN^uB;*dD-DAZk#&K4v|DA-N<3-!!repuux8Kn7`L@Sl z*rY&HK@hvu8GRf{rsdU%BP1E?NaFVddn7NDc1a|!p#RmQ^Io>+Bic1m!)CWS9Y%7u zC$Elu*F{w%j0!vaZvR4~;J%;ePFqu@qbCoCzZs2jMY5A29dEJB$J<-$7wj~M=09TJ zUeG>9u@VU&lMT-jITPl2A&ziN27H7OCKveI2W~oa_+DEF~{=5h&7g84pQuMoa;lksrSS zgcI3kUhnbUHGG4lmWt|VTCVl*_+uOM)%1_(18MFO6VqLb3+Gqfrd z(WP+ff?`sF0Hfx zvfiQhoc0Mfvnm|vKxukger+{(U$nF&Hkz-~R;NbAa&Oe=n@{1&#FGOy6`}#{T(?>O zqqv!YQlW-Did@x*bG&f4vSToHZ+ki4_d34d`hmZjml6~?pFO{Vc{wVpQ(r(T=afKZ z=q&v9gDUS%l)MlI6phyF@?+j%llW*g5BmdLpsn&R({5SLu3eQk_u|)+BzzlaI=bJB z^)+BGj#M{Q`fm9U>FZ9kp;c6W9lhUtnm)8#1MG~QzaqsM`l}JI5zM4-!pIT+%sM1& zCds)Oo?2=o((Apm)ymO{_SIYk=fwlI-&IC{WRh2Dq+}~qCcNO+Gb2}U0(!%|T9~Ub z18v&QpcgRMyl+VUg#O0|sgRgFKxqJ$MY^WIOA!eL77Na&U3Q(d%5`h(%;X;sjVD46 zWQeCq*Kr*mHYTYZN>cGq+%=4xqvyAN%;KZ>JkWo)3abbG&(3DZsZbeLZB0HbJEe&* zH3We6oSn3&nNoqa8l7kwnfz>>lQQNBDgwj9B-zXczV%l;A5mpKPBl~nup&|5u>DI# zKmer%R0N_R%Z<~9wdc+E;7I~n3$vi7Zhjn?y*MgVfI^X&vZYA_xiU%VVUVqY?!itU z&Ce+?IOY-`-3fUx!sh<&;zQ_VZKi)ZD5_&m*FwYEC%(D=|=vOiapu9~b4O(Qh7Si9X+|HER;- z|0hHg1+vRw2trG2BD+0$NC3v>pY0NfC8=MJwTnKnhg0-4`)tyCJX1{qA;P|1t| z<~bs^BZy=JAnP2l`1|-kQ16i{nyybE=F}>2nHsTJPWuwS6bVcO%#QB&|CX{#v(dz1Yr-+Nu(2PX`@y=?Qto_b|s-W!Q>H z_6b|SW2l21fmId?3!8WDoLtj(%2)ck++T`LxOkYcvl z0l$L;9Lf_YTyZNRTvi(5r;oVDF?VTavc)6TNCIj;BR8p&e)2l5eV;5vpxkdr9)f1< z$G(33%FR0s>{z zH?f{5_@G@Sw=H@rW9ryAwqYRyJY!{vrr*kQ<1@U$etuj48sl^HGflja#cf2_T0!lH za&0b?DF}$}qfAt3CUy?uI&2?FwdRm7gdxmKM!+RiM8B2NHEY&ZUH9;Kfc6$D=292B zloj8QHH`j@$AVWOk2qC+%by!7`Y`&AAN*E)_3G78gbfeif0U#aK}RSacWoN0dCBzbO}z#vtN+1*Sfn>QKVR)0_2`i}naP-Z^;l9{VMkz#W3Sp9;Ak-S_{2mK{T9E> zWowS>032`>I4QsMyXn0s~ru20_&9mT()-(2OpO-{1Eix%oqzI8SE z^4DO8)pK+h+PS_7FV0w8zProe-LuQT!EQiHSUGWlo%}H7ctr4jKZ5_`Y{+(*ax5Bo zb1=!U#AQNsc%6Hchh)IsuIM3hnMnYU)1EWjLJj$*}sos_HA}v9XewDWRvf^hCJs;fcM~Fag;eN)I-ggLO@Ia`B)+UNASQEHExw zl)2T{$EW^X`Y9eoU<{~z%&0`HVrNfAjWGeiX{z~e3d4MzVI>Jg)5eTAvM1zS=q5}Y z(^vN{U%GS&#sp&eMUGn7*B!rA?=;WJvl|E@wXT+8XQ|uA=(E4sFo?O6o4^=2tm=S~ zC%k&4%EQf_fcetbl$!>oeH64g#&*kTF^`+*^Ebe%P=h}LXu>wZ40ND6!1 zw1alzFywe34M0U)S%q_qhg#CH+#d zoPWMO1i#qxV?cN@vH3G3z+M4UvkB~g1F23x^hZEWi9Z)ZyA!w1FQOQvV@`K8wife~cGeF&mZl{kucdKADr2@HoJyt^n1 zr946Rps7hJnVvX^02W1KHN+Pqt}|n*k=zws{(Ve@cUrP13=g$PbW!LpsefkEf*cy z^(#JRnDOTR_h>R_ToLwZA$=vqJ5;ou=l`tm-$l5o8W{AffS5W080FqDD8?671pOLHJbh5sp=x{BCo*y1Fv25!px0x-@Q$ z7A^4&2k964Tt$t7ds-~w7cNC)C;Azq2$%wa^%Hgwl8Ieb&e*s8xd{vgXp5zgK7d?1 z5H>CPw;^%${m)mSNDAPttgY23E-wE2k6Wwc0o70->?rju8UO)9Gim_ALE+}jov4#( zQfXmn8&EkSS4|ONc3!13&w1(b-yt5$n7G2hrc5JV`}T5XMQU!Yy_Yf4xKkSSIza!A z0*FSmGlF*@RTK#(h_tGd+Y5SV%jH?TK;hOLw`~;R#T35N-dDG*smiMYm?nx+7R*=9 z;_M&gTtAt6A#T+1ux`za94*kB+hA-L{PnED~ek`P*5XU$1oDcKRf{5%aB* z7iFs}Bv&)s^*HZmE4*0bWtq%k2`LF+RfDRCc9% z6vN_9w-<6H`;Y84oQv4|6a~+!UbBWlXLvUYCM=a31m0NfE)>?pjB8Jh_Pr%l!vL_B zW~RqVNCof0?=oPRAJ5)AoUIB}FdCvU98H23zI@oNsE)XxHfso_>YkL`R&~h1!1{D9 zf`FT)UYqvzxx6!sc#q;uXr&qGQ%9b0x^nBHMT`Z$n-|1R5H+kXmY_6Ii`plt>=RH4 zQeTzazwxTb^K$=hF(2BHib4I%g#hym_(i|*h6+*gvnIip>nmha6j0DdJE*nH>7oa$jXM-r+W*%AP`lG+af*n&~;Vl)AS zR6T6?o#8jCs!);b#sP08j66nWW;DFJQWYfl08H>CY$Hejg7;2(-n^MP-xkvK2IL?b zTcqFz^E*N@O4!0trKtYz2nsocfJT3zB0K3(wUts(`0og(BK2PB_Z*C07|(P(CVt10EZxf~^odrV08?Eg=g4j5}Q_ zK#wfUjC%z-%Y8Z19$#e;;O5?kGNLL#QqTvZ;5$buY$y|$gnZ8hi-r&vK`>WGBt>E% zax6p1;@EZfv<+-XnH+g&*DNi`_#9}BKGm;L)+E6g;+;fPj;XOg_6Tr^3gT;YRskVT z*yB3^*TD)9SigQZTr*q=H%XBkgQT1ayz-D`$y5#@NZeJv&*Y>}`v$q7B6Y$}0S(X4 z`R&>YMRgQgMNM9;R(u1Zw+)`x19>hKTTia^v3<`e_rM&-E{bj57ccs;)70lR1Y#dZX-MvuzM&i>gflb(`# z0Abp?YnL9iqrd-wh;h|BK@AicLNgNcP>QrI7JqKXl0lEhKJ3P42mcMIXl9C<(YdKS zP&@6OYxh|jxzY#nY6X?W5zISx?#iYnsiz{~5U2C?S4^un#ryjD#!4N1D^XT`NQ5a~ z?9hGd%00t1o$qB2<((YofzQs|ijKR(L(&^BvN~u|jkcaVV&U7oHKicgQb$p}RVlC| zs43faGN4Gq-d}yZ)Bkx(obBYw>QhsVf{Wz8ZMw~;vF_2QL6`2}$KqBTku$D4vO4XD zOx;gZD5UKiVk-SwYP*_2$CBM@%W6ENgc=|3UKB)sEE+x8|IViz^7ZwMBdw z_Avxf=OxU;sgc_)A~2wQjfa2x==>??$${L&K{SfrCsZ^-_yV@RnfXqOB%c;DHYV~^mskoe%@7`(Si3%M!EH=}I?ZYhu zkN=hgYis06MZ*k<0^=>V5`TFcsi@FVV?iP9K_w95Z?O$j1;T8FLJJgw8j}KsMo!w5 zzi|DRk=U6?A`3e2iqgX?)PZ>vaWU(hM%v%I?Zw}3R{Vzu;&^gDI3t9^hNU9iI1OL* zt>Ac=Lmmk0kB3_igF6Gj(RQ>?i^PM#k!uoy=|vZXs_4Up**`v9Wf8R>)N^ojOri@5 zK)$!ZAIzIS-xS`xRn83?9akamSh`U@0dxb+1yA-Jbh5Ni985b#-?l4wxsZM!K{ zd?(d>ZbnF*%uNalf85K=Ugu(Jt?bnfRvkwbiHhz~H=`7GM&S z(h7bWXjL~^IYT^e?dA1gQUkRLUX^+g@N2ZZ0K`6A+UW=?IQ4ke+bJ_WF}|#;R>dP@ zyMJ0^Rx>OTe9_D=5f!C={{!pNx8lDW;Px#FUTV=;s}nHnj)nCVOA2S&PkgD@wM+yR zng2+#cK&BeNK#0J4d&?gv3Dfkk*VBe!XyvG|2+hXNF5$V{nC#{u*0$73Cl{%fcw+J7kYSMf@#6VJ${yQ^Lx|8N(ij zSxeNRD24)v5TRHD{#6HzN=}UeV_y!b`=9{nAsst{Qx!bg9Yl)=!=G2fozDj>ZU;HA zqt3pbB>rq-%G4~79YP%rZ&A-bFf%Bj2X-ux{`5c#cNEtpY1g5WxLI46O**QMz0#9zIXNZAxw` zaV^O%nf_A7Qki3p)%|m7>KBVXBcLmwpny|;|NgVcc9Bu5<4#y{vjhzQmQ>j%4$=X4 zB2?Fv^n35hSSxOI z51a38i?msHPeV96fgSa2`l$o`c^AQNsw;7I#UB55P|W`2M|M0i;dkKdAU86CPZBBv zU&NTg8yqTcv(FtRnoq=IuR+)Utm^9J>G>MVJQZ`6tjQw?^$6zH?>aLXiSv>g*!{@= z8@rU9GG>W78ITxxX2RYe1!3&>8qJ&Hz6ByWBU^y;mSR>Mu;(shu3TTkGnkxHz#l+ z)nU^i69-6#rmCtRjALxj3Ub`ryq;^FR>X&92;>#Gc2 z;T|JT>I)o6egzpmMnsSho^iAJDA*AY5al(~$?pS1r`Tc+XeJRw&IjNVf(_^o5`uc+ zz+9b!$DY+-mmPh7F6}H<1%W>CiZve|-%a=lP?EYH~J*+Fld4)!Eyd z2uuY`8z|l+N1P-G0+@(qFN*^iKsZi78NGEP>SxdEnD02g@uJ#Abb;H41Vl#oyJ=1!I|3tsd6GDFhu`Ha4y1+LTH9%xjy%2Cnf0k zIqWz@>Vp5B)3>*Gdx+X6>h@13W&nxx<5ln@G6?B1xe4MbZkfPYxMjjo0{fJl_;|6{ z@g^!w-osi^Tak)}87)&ftubX!o~S2ZK*bnKK@D3Y&WQUs!igfUz*3ikBPQs=BTOX7 zkPnjkuTH;*K$AN7b1}Z<0la#l0@WeA4<-Oag`-$Z{M)vvA(kt&Ebz-d6tPQ3%I>o& zVgxOen?XD3N3ckQKIqAi=D+bk5)vi5+#<{avKtXicA~}sgivp=2(pbTmyx6bM+Yjz z<7PQ`rJZs{FBJgeBE)_|$KTm{aW61x<@9i~ZI+0{E~L3`52UA>W%*)&Mp2qlm?bG#8|j zvjDIO51#0Iu-Z32eR=Jyw53x2X+IFjc*VqyAi<;h`X2U*T*cjsgQNj{gA!t6xxvFB z6cn7dR6-2H+&c%&@LE)fvwz|nMp63f$4Vy}^$^jsa99%pMR~Ue3+EAS?K@D&hj+D| z%{j^SOgOaKz0phzGKq7avBR6RV5BYXRGYL{ZjMyWX@U0q7O`b$97bz)HoUMMpmL^L zw#kIrLL?%F|9A~ehEWq1$O~Mm9MP$H&(DDi2wCR&XabbojC3@i*rD+ zU;`7i-u8eG`!*aC#`waGAjz5(APrn&lG%6$3-mlTf&O7)+YFGt-nq3Y0wE$B&b#d} z0I*0tmR)7if}rRYSU%)W;Fz4=@;$WL}S_1K7? z=eM*t+hLm3Pk@MQ9475lpFh}D9a;`bI&)^DSs}3A%DzR57mH!VoV0hqtht4myXO4) z^L9;^Vxivltq#u~+4cvWDhZe&(Ghu699-Vmnd8ptcI*l&K7zga6gFtF{$hAjmzjyN zkyKsGK;3aBQuxG`6SxR6fVHq0ukWJ8SUxSVb zp5*J=TxVKLq@E84)T{v;MY`Vw6o3<%HP)%dLv7eaE&o?*XCBpa-v50w7_%70SjLiN zMj^69B1_i6Hxy-QQMN)+mbOvKHW(u$DYRKyq=<^rB0EW;C~31L?b1f6?#HX&b)9pc z`=9&V=iKMaHJ3A5KHty#{eCUa=kxV^`%c=LUZI`g>}E_DO_i18U>>~?4}9FEcW*1l ziIbK!f34Q@sb6(~k(hy_Mm-Uo@Mc7OKK5SK+xY}F+_ix<{YiTnOLUWBO@0ay#QN$) zyAST%?yv$KRW_y!ds5?Ai{A7P4pRh1lUgqHpj>>hA`=k+U(^BcEn2G!q!qtA8;h-Wh9DOy>1+e|0eN7klY zA3U+Tm-3azl|2wtX~+_htouxKtcW&}d7br|Ts6?C^bG`zt`lNQIou)US?c345!rn)&b2rbkR@i^>X`pq5d^m*K+}pZ5g%8ChI# zZAHa%KzyH#xm6$Mq`c5_{TeGeC%iu_(*qSlm&IO6b)aZnz>AS^MS1n|!p09zPHWm4 zU|B1y)~C9U?aGhDRp$kq;jcnrPAbo7dpRz>y+h)cStnpj-3l7udBD7it z%XB=3J4~n z=R$*ck6f;M`9iV!tjaZ^Df!RCz?cAluB1m#F`qBlv1oC)Y5Y|}=b6u+KX=mQYiWv4R$k1t+zZ4z@jsw`Dog9^{<84KgO}-}9WXl? zaC?+^H28FV6wmVMc|aST++a#{x?npo_rj*vw?r+YY*h5|nhEr14>iM6GDmcQwNW zY;oPI*XblPZxq^a4wvG`5lzu0#RyM8aCH+7Q^n-b>(4JG6GSA%f6;ku1k$fucIl+JM|)e>%Yp9uSIUm05>y8GNLYqEMO;MrcztCq&V%Xzhj>nltej$4eOoUp2j~vw{V6>P1?lDb<9fxz|6_0`|Gcn zlmWta(>4I-4k_LGPtfB)yMFrgBJn%T*qtgW{rh`0WlucptzU85ODjS^n)uRcbi_my z)!%==IU!;ovG8tsjQov{YI`;P5|jF)$B&;wN@j_%gA=m{V^Q!kJDJ;H=KHd!I67hQ zln-m&Juw!xq`48NioC}&Jr*%ru86$0>Bgt#DdKEF0Ru9&DK^b_msa7HQ=Zq8FL!+D zygQI%WO?VGE{q%M{M(pN-{<9{SJ;|3UMsZEYr51SKpe+VeoS-(tWu!!EysFq8WikO|d}+l_si0XyI-w#(K;8BvAF z2)mU2s=&oG2n$K}O8$-+e+(~D^Tb0 zHU(AcYZqR=KJ#LiTopEYR;}UV>nXy=7N#``A4tXpF$?udY<#T{ub`_Nvd>Tin1a{P zS;x~5oT9)y$2u=l9X$P-QgQx91B0$$|1Dfr2E`D}x||g(gYE7GSd^QaD;T}n81>c0 zX5LXA9veKXKzyY*G9!TJI_i3_5Jopfh#q4VR6j3H4rvv-4agX3!^rKWG4N zeE?@Npx|eU zDn)5Pmo4+nh`Fef<`>egeS1rGi5T19;(7+b%HnXtg`BCy;x1AQP^QRu*4C>{*);#- zeqQQB^S1u-gd*!a@R1eeoph@NFmmh}naACjSnE}*`R62G30t6!Da;DV-l0tzfpVTE zC51r+9yODe&Y*1K#qo!|n$RbYm32=kkW zb)G?gRQB1D_c-tE(6idR4NGDRb`Pk)f+<+Iq3tcr4b*b-IMisE<>=vD5I^g0L^N`VdtjmO3+OQ#_Jkwan0a%Wmx^q8{UHSy!EpOU?pL@-JB z8`hA1YfI4#tEF{;_j@KXryzjKg_v>D#EP_7GPj_3)J2!IFLdu5KymbNR7Y{30B4wg zReN&UjEW_otxde#3C?xlwEdnNTy!d1TwPrS7Qyyso#XkF&o<>g=%rTl8*upL_-6LK zWt>h00|SB<$bb;7QOfPxn!7f?tF|-FKia=$%p4sx`{VNl%|AZs_ta@l(#vA=rD)M@ zUxg-TJx_^tbH6|Bm)-i(+k3SN-G`>M3#qfo8Pjn_QcH7un2+ZO9nT>v{W_GjT6D-1 zwZq^XuIh>L|HowdFE{+r)$7D{-H_vB^3heyEI2W z9P*r^PS73P<}-Rf=%RDmi2iuK{TB|g3K8{KOiT=8uw`|Xo?an$m=EA2dnhx1_~3Q< z$B)C9tOX&r!ywEB$uuR~AiJoqU)k$c03@OhkvV*$)MLu5(|inyBnFu|6$fcp$79K% zXi2MHt!I4yTw&S{CYblh-K|01W=;xv1ZJqyY3P;mdl>0i75y#)KWQ`k%+!uP3aK%I zp;K}2Hc5M&{&tf8+gWwy#U|vU=!q)xbX;Ew_BGvYpT#62ee@E1q+C19$|9?`SsJN9RmA#)4aF}l}mX)<6f%hJ|& z>(@^_7354OXBzF=r>g-*4Kkct2>%OVT! zAMPJxrgAIRIo3#fjG9s8oi#Q%y4e&%g6LJ9AGnqmG@tP~W7_2y3*&ye8kVGWP4Sd2k32fyD2Tz?+xKMB0YkM|V# z5jE9PqMEdaYiX!lvpaG{_@DIz%dRtPN1g%%qBz!NKTNn3+_as z4dO0J+=1t!IatV*hCpKxtYFTg%B_&WO+`~$F6)8}7ln|E<)`7e6N6h(xo*~uvJ2F4 zq8MeX3>WvL_nJE6#!1X7Xt-H$EaqZe$@97F`q4hA3?@9M;1o?8_z@48-0)4Y4JYOa z3W!Ar;!mWmkzW3{5zC&?Bz5oFRm7}6_nn~eJnKlN`b(g3nLd)!EH-JfSIFFqVlO50 z$sn@}=3Zj8M;SHXKhPR@DE0mm=?q|0g6ktXk6^r`gCO`6vO--F0*5gLR!&x8RP5QX zh=em*w(kQ?T(C8i38NYyw{^^hV; zAoA;azR);!(`n&<7_k$~7o+r6SGWbban=8nk(K3ZGJiP0&Cf!ki+QLZ7S)rwKX4Vh z12NO1z{r9N3C1=Zn^Qwn9t-7lh-ez#$f4*-6{Q$;YpDE6fH+#fMU$e0IO_Y*9Ci4i>u4*{w|AzK@(ZA zsH8{TV!1q2J_Bm0YJi&4rx z5!w8zqH~T+UcxOpwGR#6DaqmjY>Fvg{7Uf~9HZhX^Cx1thoeg|8%&N*SG`H6A&X6> ze9RTAqJMEIpeB~eme7QfK}H1iCf9nJp#^HxofD9$-f*X@-iO_twV%M01yZO=>6h*| za=lu0Z&|bso%<1z;=7Ui1W4yBlT?oyN3W}Tm851qH0609A1i~zB<&LOf+diw(v(L7 zirPg)F{M;9Dd$om!+@UblOzd6WQ(~G2`vGibtQP8B;`2&jmK) z{b(4er&^`M4?JRB5L-=(CK?Aaxk)?}@Cx88lX!R`X7;wvulJqEn~6;3#t2dIetaTf zh|FyN_48)NUmx(9nCX265O)EXaj?(!E`05Sq5s=>()<5PNB;M5hWDN+JK=LgP9rH1 zuqN6lD`Bgx@RoP2(Wr$2ECj|%DJxE|C=gCVJjhn!zk5P5%G8{wXPo<9T=OLl9B&B? zs)J>oKulte!>OaHD+sb*yHxeyZjP(K81;Gi#BYQm$zFq1RAlLevlEt4AwCwu`ijCE z+ot0BKSqD^_SNg7FP)ncVPeHQc|CoEM__-~&(kKvgh6iQfrMIg_nNsvTJ6Hfh=D62k$Ctcc}VB;)Q(B@*Dp^@U8e1l&P1 z0Rj)@ne7WRYRIDIU1sLUO=8j8TU^P^I(!!UtxcW7JR2HljA+K!(6K1k;?L=-j-hb} z9S&uix8{rbqpYfR!F5yFo1!^n+IM~b&K)~0qLzey?Hh0Xg`*@tMM4}k)idzCV-p>1 zBw|ZzMHD`drIGoI%3IyX_b{K|YqM^*9zB9N*d~vyiTWzHN9j-cEpmmDdH6m{rC%Nv zPUW;*md;WVzN(ic9vkdskKrQ`nf;rCJGU8zRhi(L!mYw&Kg79;QQ((}&M|ylzmf$b z#{8@_et~p0+Mi#wP4$~NF3RWV@IZ}DA~hGDF*c8X3&l)~8f@YX{6*ci9+f{}_YzL8 zvVvMEgXa2*ed!Wu>1=G41(tQZx6@W>>*E-1YTJ0lvC<5=K#;}5Z1TU5pY?eapd9Jc z1xooN)Ay=-IP}`A<{rq^ndb>P64?>Qi=h5me7p>&_R-uRqN@J#xQVnCGE^YUF7qa9 z9BljbR;{fiJ>ToCJBj)ei`!5(QFq=Ot5-}u&7uzXIJi${UJ2hG%B;H%c&I0XCt|KH z55SMKPlHG?V8bR6AA&aOA?cdz_LEb$r z=TUTi_AWpqH1J;r%zyk;g6D5$e@&wx&Hx2PXnNq0P5O-CS2i!87wZ<$5vhllP>c#* z#p=jf_|WI1St$pb&~;?~(g-u_^rg^SeBW9f+4w0FlFs<<-0bdf=z7uIbl938a+(aY zOh6`ce(C%IT#x69A(&Xsaaa(-D&LI+u>(HS=amV);KGkICWxLi>#BdRr^3f=LXTx6 za$!Xv3uI^m1StFc&|;>LOMsY*KkIDq;=#iBHO6p-B6y%NUF&c(GT~W691z$4MIS}6 zT^L?EdD5gBxYaNeY64E$Ut8JpB=oWN*N-z?q<*I|=9jEBC?eXBXSK*=FV$oySu{(4 zR_L;{(+SuamEP{VH>q73c{Xi7#|_26P@~7zQUm}l<;P5R+wE36>R{UuSr1lTZ)tW` z)am`v|Blogv~*&6AZ*h70fS{^{m1XWpOZDgws%Uwqfg`huJ-NQ0aQ$AcVPMdA(rOS zLldX(v02DrwOy)>Rz!4?)% zWBrfSvwq9PO3_`=i?&un6voTOp8Tfgz(Xnjtlnu?q~j(-CHUmh*?W7;H*u$^15tbW zW$z4BA)YIH*4TCk%FJq{h82pk9Wr1v5QumLYLj*Vk_)FGDnwjjck%J^t_8)D|3yZm zoZa*$i1sUa*Y^rB^2&EA9j-{{8UN7w@eaZU0+VYsw>7`t-L zl?zbbvh!RU%g0G|B0>R*U(U`MP{fqSk4I)}@(j<3MH2Jx-nsLDqF;b^I3hr`!E$;u z3)^xLH+>)K$r+S+OR3$JS}bmrOuKiJT5+htiN3P1u+SoE1g2xeb*k`dXnjh}bHKli zB~6Z47JG_UEO2P2=gH$&D2*RB+{J~DO;?)%jGb=ha|hiZw(AoTtM2tV;c(^0)qktM zag!59{mv3EWq*hlm)-ixyy{Op?<_GElVN_IKa^cj0#1Sry;gkAy}Ng%YLUKPm`eCn zX;`J(lp+ZJD?W!Z$Vy&zykmr9Tuw*{y0`0<9bQUj16Kcxb1ET95Ic&FMWA-ikW@em zjmp*(^$%5GxcJkN7o^!p%{8yTo&>lXy5i<88JZU%LjCUH6@R&3uXz7nw5L)G(Om0g zT&4L!{9b z93B(3Tk_!u6B{eyH4714UVuZe1X_So6`WEyNqIlK!UlG24LMtMb(%R+lhlh`ILDRe^#$+5Mr0!|?gW^X&n^+&5p+oj8x0l&_^ z`u%(yd#%6(<&5POIFhC8XxtEB4NAElYHn}YtAis9x-YKvBa)lPiHm`UGMPy=d!L3v zSSHXKxjkafzI_vw9vY46)O>gRCbzRLO%(0I^PrCn2NukK-87ru^YwET?RrT;(sn0} zFFQxgYQEd^S`zyV(ccVSwAMLYr=l|;Ra3I|0K!tp(C~L zJn(<;V2%(`0%?$VrFo_z8Uj@$Iz54;#5e+)DAStD(k>DJ#8QpoOuXu3%7cl(buid~ zI;$cVq|rH3t7NbgQg-oWX~hMW4n-T5$F$CV27Ni*J8gqwQ-i&9bm@KrUiMP@srl~Z zSIc|3=z9A1?4Y$tspkWAiQA%aARo`YJP%u-j)*Vn|107covkSiq^Ko;0ecIA!EL1h z8*&6$ACK7goNg*wQ%AtGBm1 zPZ4#@Q*^+kpNOx$IIE+03=5?xu9LD$wQ3O}Z1BvwLF8QS5X z=3~u!Tb0*B6J+i8#Fa`DCBIj{t?#-J9>Nf1F?_0L8_zCPNU^rYh}uJDQ8OfgOfPyu;ulB*jgUU$oTJY@htSZ(ok-;YM3? zrQDQJ9R&!~e^|W$B5Fpu-N9sWfl^{qO`kz!5b$c)%q;p)KiR%1%)I}yqt8(f`Ajph!Q*GSlPpP zR#&WT9MB1545D!#f^Di04FtstrIRL2(lCGU3*Em|6B6d^imdjNq{OTVd~;i*i`US_ zL+>VaY06bktFbRmTYT6n%u+N$`$C4uAe2NSF};Bi4>c>cWgiYVOEBcTKL^RNhsY<) z>C!r%hTZcHt~1`G_W8Hf@0R!4s#|kPMw4Kk-*NaoJWCS#U|_WnuX{<3B3orNSndrF z2|jyPN5pdXgH3w2v~+nYnHWnjILHeEnF$^!U>G7@zOk4bmpo{{{hh?Y0X@9NJ&zAq z;easc?!9|pHCYvk9WYsB{9(mu!w2=?>fTAU&kP(Y^OEY-KlY%zb6?q0+P2i>qCB;j z(7%CbyhSE06bva$1WWq{H^+P}C;v<3x9{rC;j{1EOcb0Lf&qcZune0cE0w+;*!=2v zV{I+7MWqyp(w^#nb+>*@uU-q=a-JfI)nDPHG1m0IX-~TccU^|jbxLh7QWo(y1G74b zsV>x~_y|&{hFr_5FD;U{VZqB1eJmMh)wl(!W(lz*SG8oGuhTpxE=U=8 zf_66eP`}CN8KnC_dQe>11}Gtp%YBj6(`ocZs)rIWu%$7nMR;B%o65ZLJ-!}TeKTov ziSLlfqdIw{SiaqXyu`gUD`KwxzkXB3;&C7)qL2g}g&ZYcv@XoL<@*>y=+y!wa*6d~ zDa-!t+9l*pzVhMpq=>f!%S;QeamV9}hF%jF3zcmDSmQyb6CHKCAK+CDVZRD=ocamr zGHlL;PlQ_om%f--X9ggBaN#%U$l1NFQ#@pZ)Hy(`7DKqfON zqdmch?5wMIuWh*nEp5<0>Q)1;eqOe_1#AY5;BfeK;I<;z1Bqo7{Rr4tPR6cyX=&)>(7_k@1~V)^jhg z)p`3uJIiu$cwHc>Wy(_Ou@3-ImnSZ*NR-Zn;T{T=`9iAEST=TRai~XGu!#35c7rlu zAl>C?^M9niq4%B`-T5cdz#GGl5pLrL>uhAEaLdE-13Xiv31QESLaC3wv5`lubHEw9 z`m+HBP938yE}13#rJhfCeOD`$ccnyvZ4Cdi81BM|xfnK+Q8GDIm2m=Lj@r~eA z;Pg*`&1%b{9uLa7pSRHpslHyw(j!rU_Daj(1=wgF`rH4GE1cS+^(KwRz6tub$AA9q z?G5$N+FF|84}7=OtQ0FJHY64%Hn1oQkas`BU^WrYyt?&YJqf7GDsN8;|anX|j z1|G@b^N$BQ-0k^==+q2Ow_@w|`i~#yRLxj;VzdRg*ZXH$3)i?9+Wj3AkQHihzu}t^ zDg>p-=4FiPQF7>tPbmFV5KxUZm$AdoJrhkzuQ6-R$&5o)cRZ0ds9PrBV)36&~wS zG_%+K&6{TH_UiEW7OgqNd~WTq@M}Bbf?7_V^D#Xh4%c72ogrSz{r@(8Z+Q7p=&^<1 z%!7*4$``9%4y<;Zl9~Of8U=NkjV*PlxoT2PWy~*_p7oyWWOwpq2^RFHP7UBPaYbD9 z&uMfm8ZHfXVgx8%bX{ZQ*Pj{OaDPAB{OzIWL%T&=MAp}*DW=%o-7UrFz4~YAj_GC0 z3x*cpgR;G;)_Z=o+pY0C-We_j61kf1y5g{kd3^nCx|j{;b7WRxu&(bMlijBOsPCc+ z5VOE^zp%aZGB)*XcZ8+wS8IBn?p>}34G0*&`H#C*N7A>D}AeXRd;O@!j00`b!PwX3|g5a}t z_lFxwQPC>NA6eYj;zU4w{?_me5H{a|10U}n{o~sK*UBbzinRIrU%RzGd`q=a@vP44 z%PZLXl~D-{aR?HmES+5P)dSoGkl%TG<<;W<3M<)g0??V4qU#AIGtTuW$%@!$YO{6Q zCwkp-ZMRh-Y$#n%B2orOwb%6iX5v7;Rtg3gfB{vYgB@+uYwkJQ%D3lU$3`D&n^LYL zcF?sy?g6$4;|b!#=0o{h4~b)7E3EyR>I-_aRwzx?NsQ>wuH6S%4%Ox;-%T=qm3hrS zaALuz;H!QXo!pU=7>am}<8NAHm9e*VHE_DS?etSBh>ORCB*fN`hv61u(pMjzJc#@v zznbX+-K)(+UBMWY8=FM;91~yASX<5KH7@GoutdhfDLC*dH|)~2>(TIz)D?$nYNg`f zPzd@m*4_49W(F_;Z0sV!MAUa%>f?1jxuQJ@Z>j|*+Iq?*uXF1vr0pO5E5|?ELeS>{ zUeoOA|K9ItB9a7ycs1#NktE>sZ0`{W1vhBe7z{}i=?tg`(Igss4^~!g#p2-bBpWd8 z_Ild#S?3v~5g=ZXPGMnw^z;4u_t(>;N~10rnG|G_b>;t19qb=SLOeh6`sZ6VV*0a? zH5ZT?Nyv+)-~o(qh$h&6*Hacv#t+{2+j@XJUdmZVpCH~V9v7%5^c%hUg~k_*kIfNA zxTc2}J8en3E*eUaktQw{7%aHaO|t?m2B%-=8QE#7`IBHP(z!x#+nX}< zQix9g!HoTXe@%c;P(WhknOWEjNZ!PDFALRWSzR7%peW6tzTNGL9=btM->n#K;Z>bR zMtHbfU~ia@#|2-588Nhxl^7-_2ys}RTVR|^THa!(>MDg#zK>bjRR`mSj>IzIf#7IPyD|?EVOi+KiJelMbj*xTO3MG^A0Y>EF64VbeioE_*HWP{QgJ*gT>tX`@ zq@I+Z^vd<^D;uxm*$^*P-HN`8<1Ob!Gle8Z?TAaRrtOSkoJ%eQfVdI}KBH()ZilyR zEQ?Do{t71n7h~MgLWG>-;oAIVZ4-QYW+BB>Oup`sg*^vqnat|<6XLdW!ZrXsKNW2Q z4V*LvqBTMYw65Ib&^-SZQ3GBe2d`t8N%{_g*+@}3Ai$;iVuUOjcQEs)`M@HzH#_uS zhwe}Y|5|M%Qr8IJgc3yxLI!C4!bh;@Q7q0NWM?4KV)S4rdQjQKw2N7sS9^5YWK0=_ zlwmB0E8)!!_MMnHVUTjMj2U9E#h9@B?=w7_T+|#_G817GFyN=gimWWgH#WFD9>OWm zZvMPFz7Z$yLvnj+X-iS6&)URaA$dnP6tpz2b&RLJ$$6AyPai8LVA7oclL#HcdeO1z zBI`h$rce2Wo3Uqyt=XO48oG}$f2Z{n@YgbXza)}<&9)SsgaJ+CP1GNYST7Q|_dE-KK zOX5E)j60ZvaFJB^>j3UK65jr_@8AM|-`)dg}YjW$3z7(O}O05sda4`BlmH4i`p|(oNG%~v>aQF43Xr*_qZ*RN-r_3~QMBcb{ zUtKvu1)=g$wLC#edj^(qu7^}U1PrN1NB9K#Y;H5VF>V<5)R;26HTUXQ0+MFuE?e#n zEba9_TQzWE*InZ_yjUPJo1A$&uuQ?_X`IE6HE(0}nCURTW4AYNgIioGA`fzzn1j&} zl!7A)B3-8Yu4OwijiGc?BJ^SEc`1W#nXoqJIWDcO^bD*|CpHWe9e2&b%gB;7SHBC`* zUE{+;?9$5XV_)Rwx;Vy9^o)0ofA;=0ID-HDAxlxDi$?0HFuDNZl62{XMGPXy38Ub0 z+2;u7xN$_y-zif>vFV>9HlUd!`&Zrt>(5O!pY$QH^IzwL50r~=Wwnum4v+HLti9L;uy(`tDKp&~!Vz zF9QZde2Xy*^q-nGP5xr_P8YY`zJ)7$ONIM{-kYm}$|IehRlO3h6@8ds<|pzzH|*a0 z@#9LR@%yg(+m4!*eyFPI;|cCvl#L1o$qfge7V!K^NhUT5@CvHWJI&R{{o&MJI=;(S#_IN zzyFF$2m8PDT9Lm2NLoLanuWp$$$67tM7W+WcW`;f;e0R)hCdT-)%3ll}af?b_6UDa(mZ$$?97Zh7VU zpC?#5`h?8W%c{t|8bT_QBP|@F_;Kz(9$ndU)|zqR^b#8NNR=BM?wBuqm}ku<6Kf0= z1r13U$lD+V>EXCU2!dG&5=6O{8J&_`$0IT{#4lH}AA>qxVJf6ssTL@ivXC@#$&%cz z1|<>I<^JZciD&nH(s&s~iDn&!NS+1UT!!GrC- z{YKxwAo`S_x}M|ERlo@#2I*!-gGr^JW$5l$)QcwqI{I8mZkF|HRnX zt7vAQ}3=&@fq7fOjH6PS^akq^?+2C1m*2@W14 zmn>_ye6^KP7fQ_EmJc2|@^F5>*&j!~KfT}|4)MbJu!xAF6KnlfGX4j2d?4&%+`nmQ zx>OI3_=fC6=W&oS%EN~bH_J#*pRQ2+keZr0&qwnhvV+Y}|NQMKZp|((ldoL4GI07e z(#t1G-}I+XZvydOtB5Q8P+3{TxW#H@c?3i)7YOP6geJf<{$(+;wjx;A@$%M$X+n~An?3thQUdw_+7MJ~d|svMKb zxY7{-um8;aBQPk)?{bvdz=69XBS&9MtTX<0=aeM3PmB*n0ap(lI`l8<6eFgTw%HIz zHO7sj=&1h)D&XPi*)5{(WpEsBb+o%TI0e5|VQsM-E@T%oeUVE9lUFw@QpXrBM>deSHL8nfwFdWcf6tY`}-);ag%nr}V%>3>KDu})9x3syr!kz1v zR;~sk8$@&phb15=ZG2&7X=y2Q#O-!Z%O6?%x}{B<_V$XI(-wi2ejR;4SrND3h3a61 zhsHhM3HCp(nEC2?M#hZAi@V#~+uvEG{@r)q?a1(3*9Ib|TC38?)O5Z^@1W4o{p5$g z#`oy>9b%()ef##E=R3a34p5RPhFHPh?mT(&3)-V)Tas1QE{l(h{B;LK=AjAOd7Wc` zIhbz&ZD|W%G@+=-U0I>ija%?M0@VlgIPlG#8KZWPu>U6P{9NkasF#Q&8K zuSY?kHKY{SJ38Kf9K{N2HFO-IhcNsdchKlHXI1-- z-+T+xNM+MsI<~%N_tU3;FDWT$!;b$>x4>AGf={xt?h`>`%dh4(Y5p`RG&HoRM#)22v40JoPaYb37cE)B8TiJx z)16m#-piIR?~mO3HCCg&7gj^>`(~IWCMGUgyjULWs&yY#Rn>v-O-xK48%{wc3tQgS zw^P?7N(4!uc-qeW^UwA?Wt+f|kc=l!ZUC=78k@DuR(I_&Wko|s)tL_Rw$Frx9e^MD zX2-2t)3#|en2Jrr-%-=>T ztEenou;2z-gf?&TTsJjL|9%H;WWaOdwa%{m?CNBB1)@tv|YLSSc)?%E2~+ijzko;sj-R4?4}-hOkF%=6t=jmEbz>k zRWEX3Zc?cTW}7nnv5Wcp!#yi2D&Avc_w5P@i>fM8+z}6t=|)DqXxl=PlD5w*r%d__ zZgt;5?z8T`1E|@}sgpdPs^tUcw<-#c<|>;3yR{r-COjoiN~UypWrZ+YX3 z)rY(6pE*ZQPl6{dPd#_Qd3M@Cp#94+pK=)YobL$j&z?Qo#8g7pgR>aofDtYGec-^h zI~ei41QIzSzHBt>GDo2}9Tj!7q{J#NE{;tboRHwlVtm#@|2k*95@$voW!%NsH$T!Y zOm}n~2X#`&tFoZJOdtSMoy^Q!!tH$1H*Ors;FIDa(mL_z(Y%Y-8?3nE`XP1g>>DLy1HvX_(V}wQ&Y2v zmwUMP!z(c{QP9X#5wqvc{i$u+wr#!=wyFx`-suEPx0EJ3dEUG^jgLP>RrNI~r!d&Z zrgxWzf1X_^OdV^#&dTacVxr%(XNyLU9&LDaLodhN&G*m!HX-`T6&XM>FfvjQF`Am1 z<}u5KQ#5_@wL&kG<7ApkC)a79$}>54;X)8wnn=l1-8@Un;)0gJb3BxrI=3bXKBVG@ z_pPCpXdBpWjOS7QpMQRRBzsllG^*>UWZEeH M)SY>8`ifou3#p7 GPAlgorithmParameters: + default_gp_algorithm_params_dict = dict(list(vars(GPAlgorithmParameters()).items())) + k_pop = [] + for k, v in self._input_params.items(): + if k in default_gp_algorithm_params_dict: + default_gp_algorithm_params_dict[k] = self._input_params[k] + k_pop.append(k) + for k in k_pop: + self._input_params.pop(k) + return GPAlgorithmParameters(**default_gp_algorithm_params_dict) + + def get_graph_generation_parameters(self) -> GraphGenerationParams: + default_graph_generation_params_dict = self.get_default_graph_generation_params() + k_pop = [] + for k, v in self._input_params.items(): + if k in default_graph_generation_params_dict: + default_graph_generation_params_dict[k] = self._input_params[k] + k_pop.append(k) + for k in k_pop: + self._input_params.pop(k) + ggp = GraphGenerationParams(**default_graph_generation_params_dict) + return ggp + + def get_graph_requirements(self) -> GraphRequirements: + default_graph_requirements_params_dict = dict(list(vars(GraphRequirements()).items())) + # if there are any custom domain specific graph requirements params + is_custom_graph_requirements_params = \ + any([k not in default_graph_requirements_params_dict for k in self._input_params]) + for k, v in self._input_params.items(): + # add all parameters except common left unused after GPAlgorithmParameters and GraphGenerationParams + # initialization, since it can be custom domain specific params + if k not in self._default_common_params: + default_graph_requirements_params_dict[k] = self._input_params[k] + if is_custom_graph_requirements_params: + return DynamicGraphRequirements(default_graph_requirements_params_dict) + else: + return GraphRequirements(**default_graph_requirements_params_dict) + + def get_actual_common_params(self) -> Dict[str, Any]: + for k, v in self._input_params.items(): + if k in self._default_common_params: + self._default_common_params[k] = v + return self._default_common_params diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/api/main.py b/paper_experiments/large_bayesian_networks_experiments/golem/api/main.py new file mode 100644 index 0000000..326bb91 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/api/main.py @@ -0,0 +1,139 @@ +import logging +from typing import Optional + +from golem.api.api_utils.api_params import ApiParams +from golem.core.constants import DEFAULT_API_TIMEOUT_MINUTES +from golem.core.log import Log, default_log +from golem.utilities.utilities import set_random_seed + + +class GOLEM: + """ + Main class for GOLEM API. + + Args: + :param timeout: timeout for optimization. + :param seed: value for a fixed random seed. + :param logging_level: logging levels are the same as in `logging `_. + + .. details:: Possible options: + + - ``50`` -> critical + - ``40`` -> error + - ``30`` -> warning + - ``20`` -> info + - ``10`` -> debug + - ``0`` -> nonset + :param n_jobs: num of ``n_jobs`` for parallelization (set to ``-1`` to use all cpu's). Defaults to ``-1``. + :param graph_requirements_class: class to specify custom graph requirements. + Must be inherited from GraphRequirements class. + + :param crossover_prob: crossover probability (chance that two individuals will be mated). + + ``GPAlgorithmParameters`` parameters + :param mutation_prob: mutation probability (chance that an individual will be mutated). + :param variable_mutation_num: flag to apply mutation one or few times for individual in each iteration. + :param max_num_of_operator_attempts: max number of unsuccessful evo operator attempts before continuing. + :param mutation_strength: strength of mutation in tree (using in certain mutation types) + :param min_pop_size_with_elitism: minimal population size with which elitism is applicable + :param required_valid_ratio: ratio of valid individuals on next population to continue optimization. + + Used in `ReproductionController` to compensate for invalid individuals. See the class for details. + + :param adaptive_mutation_type: enables adaptive Mutation agent. + :param context_agent_type: enables graph encoding for Mutation agent. + + Adaptive mutation agent uses specified algorithm. 'random' type is the default non-adaptive version. + Requires crossover_types to be CrossoverTypesEnum.none for correct adaptive learning, + so that fitness changes depend only on agent's actions (chosen mutations). + ``MutationAgentTypeEnum.bandit`` uses Multi-Armed Bandit (MAB) learning algorithm. + ``MutationAgentTypeEnum.contextual_bandit`` uses contextual MAB learning algorithm. + ``MutationAgentTypeEnum.neural_bandit`` uses contextual MAB learning algorithm with Deep Neural encoding. + + Parameter `context_agent_type` specifies implementation of graph/node encoder for adaptive + mutation agent. It is relevant for contextual and neural bandits. + + :param decaying_factor: decaying factor for Multi-Armed Bandits for managing the profit from operators + The smaller the value of decaying_factor, the larger the influence for the best operator. + :param window_size: the size of sliding window for Multi-Armed Bandits to decrease variance. + The window size is measured by the number of individuals to consider. + + + :param selection_types: Sequence of selection operators types + :param crossover_types: Sequence of crossover operators types + :param mutation_types: Sequence of mutation operators types + :param elitism_type: type of elitism operator evolution + + :param regularization_type: type of regularization operator + + Regularization attempts to cut off the subtrees of the graph. If the truncated graph + is not worse than the original, then it enters the new generation as a simpler solution. + Regularization is not used by default, it must be explicitly enabled. + + :param genetic_scheme_type: type of genetic evolutionary scheme + + The `generational` scheme is a standard scheme of the evolutionary algorithm. + It specifies that at each iteration the entire generation is updated. + + In the `steady_state` individuals from previous populations are mixed with the ones from new population. + UUIDs of individuals do not repeat within one population. + + The `parameter_free` scheme is same as `steady_state` for now. + + ``GraphGenerationParams`` parameters + :param adapter: instance of domain graph adapter for adaptation + between domain and optimization graphs + :param rules_for_constraint: collection of constraints for graph verification + :param advisor: instance providing task and context-specific advices for graph changes + :param node_factory: instance for generating new nodes in the process of graph search + :param remote_evaluator: instance of delegate evaluator for evaluation of graphs + + ``GraphRequirements`` parameters + :param start_depth: start value of adaptive tree depth + :param max_depth: max depth of the resulting graph + :param min_arity: min number of parents for node + :param max_arity: max number of parents for node + + Also, custom domain specific parameters can be specified here. These parameters can be then used in + ``DynamicGraphRequirements`` as fields. + """ + def __init__(self, + timeout: Optional[float] = DEFAULT_API_TIMEOUT_MINUTES, + seed: Optional[int] = None, + logging_level: int = logging.INFO, + n_jobs: int = -1, + **all_parameters): + set_random_seed(seed) + self.log = self._init_logger(logging_level) + + self.api_params = ApiParams(input_params=all_parameters, + n_jobs=n_jobs, + timeout=timeout) + self.gp_algorithm_parameters = self.api_params.get_gp_algorithm_parameters() + self.graph_generation_parameters = self.api_params.get_graph_generation_parameters() + self.graph_requirements = self.api_params.get_graph_requirements() + + def optimise(self, **custom_optimiser_parameters): + """ Method to start optimisation process. + `custom_optimiser_parameters` parameters can be specified additionally to use it directly in optimiser. + """ + common_params = self.api_params.get_actual_common_params() + optimizer_cls = common_params['optimizer'] + objective = common_params['objective'] + initial_graphs = common_params['initial_graphs'] + + self.optimiser = optimizer_cls(objective, + initial_graphs, + self.graph_requirements, + self.graph_generation_parameters, + self.gp_algorithm_parameters, + **custom_optimiser_parameters) + + found_graphs = self.optimiser.optimise(objective) + return found_graphs + + @staticmethod + def _init_logger(logging_level: int): + # reset logging level for Singleton + Log().reset_logging_level(logging_level) + return default_log(prefix='GOLEM logger') diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/__init__.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/adapter/__init__.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/adapter/__init__.py new file mode 100644 index 0000000..f85dc38 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/adapter/__init__.py @@ -0,0 +1,2 @@ +from .adapter import BaseOptimizationAdapter, DirectAdapter, IdentityAdapter +from .adapt_registry import AdaptRegistry, register_native diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/adapter/adapt_registry.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/adapter/adapt_registry.py new file mode 100644 index 0000000..850c241 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/adapter/adapt_registry.py @@ -0,0 +1,147 @@ +from copy import copy +from functools import partial +from typing import Callable + +from golem.utilities.singleton_meta import SingletonMeta + + +class AdaptRegistry(metaclass=SingletonMeta): + """Registry of callables that require adaptation of argument/return values. + AdaptRegistry together with :py:class:`golem.core.adapter.adapter.BaseOptimizationAdapter` + enables automatic transformation between internal and domain graph representations. + + **Short description of the use-case** + + Operators & verification rules that operate on internal representation + of graphs must be marked as native with decorator + :py:func:`golem.core.adapter.adapt_registry.register_native`. + + Usually this is the case when users of the framework provide custom + operators for internal optimization graphs. When custom operators + operate on domain graphs, nothing is required. + + **Extended description** + + Optimiser operates with generic graph representation. + Because of this any domain function requires adaptation + of its graph arguments. Adapter can automatically adapt + arguments to generic form in such cases. + + Important notions: + + * 'Domain' functions operate with domain-specific graphs. + * 'Native' functions operate with generic graphs used by optimiser. + * 'External' functions are functions defined by users of optimiser. + + Most notably, custom mutations and custom verifier rules. + + * 'Internal' functions are those defined by graph optimiser. + + Most notably, the default set of mutations and verifier rules. + All internal functions are native. + + Adaptation registry usage and behavior: + + * Domain functions are adapted by default. + * Native functions don't require adaptation of their arguments. + * External functions are considered 'domain' functions by default. + + Hence, their arguments are adapted, unless users of optimiser + exclude them from the process of automatic adaptation. + It can be done by registering them as 'native'. + + AdaptRegistry can be safely used with multiprocessing + insofar as all relevant functions are registered as native + in the main process before child processes are started. + """ + + _native_flag_attr_name_ = '_adapter_is_optimizer_native' + + def __init__(self): + self._registered_native_callables = [] + + def register_native(self, fun: Callable) -> Callable: + """Registers callable object as an internal function + that can work with internal graph representation. + Hence, it doesn't require adaptation when called by the optimiser. + + Implementation details. + Works by setting a special attribute on the object. + This attribute then is checked by ``is_native`` used by adapters. + + Args: + fun: function or callable to be registered as native + + Returns: + Callable: same function with special private attribute set + """ + original_function = AdaptRegistry._get_underlying_func(fun) + setattr(original_function, AdaptRegistry._native_flag_attr_name_, True) + self._registered_native_callables.append(original_function) + return fun + + def unregister_native(self, fun: Callable) -> Callable: + """Unregisters callable object. See ``register_native``. + + Args: + fun: function or callable to be unregistered as native + + Returns: + Callable: same function with special private attribute unset + """ + original_function = AdaptRegistry._get_underlying_func(fun) + if hasattr(original_function, AdaptRegistry._native_flag_attr_name_): + delattr(original_function, AdaptRegistry._native_flag_attr_name_) + self._registered_native_callables.remove(original_function) + return fun + + @staticmethod + def is_native(fun: Callable) -> bool: + """Tests callable object for a presence of specific attribute + that tells that this function must not be restored with Adapter. + + Args: + fun: tested Callable (function, method, functools.partial, or any callable object) + + Returns: + bool: True if the callable was registered as native, False otherwise. + """ + original_function = AdaptRegistry._get_underlying_func(fun) + is_native = getattr(original_function, AdaptRegistry._native_flag_attr_name_, False) + return is_native + + def clear_registered_callables(self): + # copy is to avoid removing elements from list while iterating + for f in copy(self._registered_native_callables): + self.unregister_native(f) + + @staticmethod + def _get_underlying_func(obj: Callable) -> Callable: + """Recursively unpacks 'partial' and 'method' objects to get underlying function. + + Args: + obj: callable to try unpacking + + Returns: + Callable: unpacked function that underlies the callable, or the unchanged object itself + """ + while True: + if isinstance(obj, partial): # if it is a 'partial' + obj = obj.func + elif hasattr(obj, '__func__'): # if it is a 'method' + obj = obj.__func__ + else: + return obj # return the unpacked underlying function or the original object + + +def register_native(fun: Callable) -> Callable: + """Out-of-class version of the ``register_native`` + function that's intended to be used as a decorator. + + Args: + fun: function or callable to be registered as native + + Returns: + Callable: same function with special private attribute set + """ + return AdaptRegistry().register_native(fun) diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/adapter/adapter.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/adapter/adapter.py new file mode 100644 index 0000000..ede40e1 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/adapter/adapter.py @@ -0,0 +1,186 @@ +from __future__ import annotations + +from abc import abstractmethod +from copy import deepcopy +from typing import TYPE_CHECKING, TypeVar, Generic, Type, Optional, Dict, Any, Callable, Tuple, Sequence, Union + +from golem.core.dag.graph import Graph +from golem.core.log import default_log +from golem.core.optimisers.graph import OptGraph, OptNode +from golem.core.adapter.adapt_registry import AdaptRegistry +from golem.core.optimisers.opt_history_objects.individual import Individual + +if TYPE_CHECKING: + from golem.core.optimisers.genetic.operators.operator import PopulationT + +DomainStructureType = TypeVar('DomainStructureType') + + +class BaseOptimizationAdapter(Generic[DomainStructureType]): + def __init__(self, base_graph_class: Type[DomainStructureType] = Graph): + self._log = default_log(self) + self.domain_graph_class = base_graph_class + self.opt_graph_class = OptGraph + + def restore_func(self, fun: Callable) -> Callable: + """Wraps native function so that it could accept domain graphs as arguments. + + Behavior: ``restore( f(Graph)->Graph ) => f'(DomainGraph)->DomainGraph`` + + Implementation details. + The method wraps callable into a function that transforms its args & return value. + Arguments are transformed by ``adapt`` (that maps domain graphs to internal graphs). + Return value is transformed by ``restore`` (that maps internal graphs to domain graphs). + + Args: + fun: native function that accepts native args (i.e. optimization graph) + + Returns: + Callable: domain function that can accept domain graphs + """ + return _transform(fun, f_args=self.adapt, f_ret=self.restore) + + def adapt_func(self, fun: Callable) -> Callable: + """Wraps domain function so that it could accept native optimization graphs + as arguments. If the function was registered as native, it is returned as-is. + ``AdaptRegistry`` is responsible for function registration. + + Behavior: ``adapt( f(DomainGraph)->DomainGraph ) => f'(Graph)->Graph`` + + Implementation details. + The method wraps callable into a function that transforms its args & return value. + Arguments are transformed by ``restore`` (that maps internal graphs to domain graphs). + Return value is transformed by ``adapt`` (that maps domain graphs to internal graphs). + + Args: + fun: domain function that accepts domain graphs + + Returns: + Callable: native function that can accept opt graphs + and be used inside Optimizer + """ + if AdaptRegistry.is_native(fun): + return fun + return _transform(fun, f_args=self.restore, f_ret=self.adapt) + + def adapt(self, item: Union[DomainStructureType, Sequence[DomainStructureType]]) \ + -> Union[Graph, Sequence[Graph]]: + """Maps domain graphs to internal graph representation used by optimizer. + Performs mapping only if argument has a type of domain graph. + + Args: + item: a domain graph or sequence of them + + Returns: + Graph | Sequence: mapped internal graph or sequence of them + """ + if type(item) is self.domain_graph_class: + return self._adapt(item) + elif isinstance(item, Sequence) and type(item[0]) is self.domain_graph_class: + return [self._adapt(graph) for graph in item] + else: + return item + + def restore(self, item: Union[Graph, Individual, PopulationT, Sequence[Graph]]) \ + -> Union[DomainStructureType, Sequence[DomainStructureType]]: + """Maps graphs from internal representation to domain graphs. + Performs mapping only if argument has a type of internal representation. + + Args: + item: an internal graph representation or sequence of them + + Returns: + Graph | Sequence: mapped domain graph or sequence of them + """ + if type(item) is self.opt_graph_class: + return self._restore(item) + elif isinstance(item, Individual): + return self._restore(item.graph, item.metadata) + elif isinstance(item, Sequence) and isinstance(item[0], Individual): + return [self._restore(ind.graph, ind.metadata) for ind in item] + elif isinstance(item, Sequence) and isinstance(item[0], self.opt_graph_class): + return [self._restore(graph) for graph in item] + else: + return item + + @abstractmethod + def _adapt(self, adaptee: DomainStructureType) -> Graph: + """Implementation of ``adapt`` for single graph.""" + raise NotImplementedError() + + @abstractmethod + def _restore(self, opt_graph: Graph, metadata: Optional[Dict[str, Any]] = None) -> DomainStructureType: + """Implementation of ``restore`` for single graph.""" + raise NotImplementedError() + + +class IdentityAdapter(BaseOptimizationAdapter[DomainStructureType]): + """Identity adapter that performs no transformation, returning same graphs.""" + + def _adapt(self, adaptee: DomainStructureType) -> Graph: + return adaptee + + def _restore(self, opt_graph: Graph, metadata: Optional[Dict[str, Any]] = None) -> DomainStructureType: + return opt_graph + + +class DirectAdapter(BaseOptimizationAdapter[DomainStructureType]): + """Naive optimization adapter for arbitrary class that just overwrites __class__.""" + + def __init__(self, + base_graph_class: Type[DomainStructureType] = OptGraph, + base_node_class: Type = OptNode): + super().__init__(base_graph_class) + self.domain_node_class = base_node_class + + def _adapt(self, adaptee: DomainStructureType) -> Graph: + opt_graph = deepcopy(adaptee) + opt_graph.__class__ = self.opt_graph_class + for node in opt_graph.nodes: + node.__class__ = OptNode + return opt_graph + + def _restore(self, opt_graph: Graph, metadata: Optional[Dict[str, Any]] = None) -> DomainStructureType: + obj = deepcopy(opt_graph) + obj.__class__ = self.domain_graph_class + for node in obj.nodes: + node.__class__ = self.domain_node_class + return obj + + +def _transform(fun: Callable, f_args: Callable, f_ret: Callable) -> Callable: + """Wraps function by transforming its arguments and return value: + ``f_args`` is called on each of the function arguments, + ``f_ret`` is called on the return value of original function. + + This is a helper function used for adaption of callables by + :py:class:`golem.core.adapter.adapter.BaseOptimizationAdapter`. + + Args: + fun: function to be transformed + f_args: argument transformation function + f_ret: return value transformation function + + Returns: + Callable: wrapped transformed function + """ + + if not isinstance(fun, Callable): + raise ValueError(f'Expected Callable, got {type(fun)}') + + def adapted_fun(*args, **kwargs): + adapted_args = (f_args(arg) for arg in args) + adapted_kwargs = dict((kw, f_args(arg)) for kw, arg in kwargs.items()) + + result = fun(*adapted_args, **adapted_kwargs) + + if result is None: + adapted_result = None + elif isinstance(result, Tuple): + # In case when function returns not only Graph + adapted_result = tuple(f_ret(result_item) for result_item in result) + else: + adapted_result = f_ret(result) + return adapted_result + + return adapted_fun diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/adapter/nx_adapter.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/adapter/nx_adapter.py new file mode 100644 index 0000000..183e445 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/adapter/nx_adapter.py @@ -0,0 +1,117 @@ +from copy import deepcopy +from typing import Optional, Dict, Any, Iterable + +import networkx as nx + +from golem.core.adapter import BaseOptimizationAdapter +from golem.core.dag.graph_node import GraphNode +from golem.core.optimisers.graph import OptGraph, OptNode + + +class BaseNetworkxAdapter(BaseOptimizationAdapter[nx.DiGraph]): + """Base class for adaptation of networkx.DiGraph to optimization graph. + Allows to use NetworkX directed graphs with our optimizers. + + For custom networkx adapters overwrite methods responsible for + transformation of single nodes (`_node_adapt` & `_node_restore`). + """ + + def __init__(self): + super().__init__(base_graph_class=nx.DiGraph) + + def _node_restore(self, node: GraphNode) -> Dict: + """Transforms GraphNode to dict of NetworkX node attributes. + Override for custom behavior.""" + parameters = {} + if hasattr(node, 'parameters'): + parameters = deepcopy(node.parameters) + + if node.name: + parameters['name'] = node.name + + return parameters + + def _node_adapt(self, data: Dict) -> OptNode: + """Transforms a dict of NetworkX node attributes to GraphNode. + Override for custom behavior.""" + data = deepcopy(data) + name = data.pop('name', None) + return OptNode(content={'name': name, 'params': data}) + + def _adapt(self, adaptee: nx.DiGraph) -> OptGraph: + mapped_nodes = {} + + def map_predecessors(node_id) -> Iterable[OptNode]: + for pred_id in adaptee.predecessors(node_id): + yield mapped_nodes[pred_id] + + # map nodes + for node_id, node_data in adaptee.nodes.items(): + # transform node + node = self._node_adapt(node_data) + mapped_nodes[node_id] = node + + # map parent nodes + for node_id, node in mapped_nodes.items(): + # append its parent edges + node.nodes_from = map_predecessors(node_id) + + return OptGraph(mapped_nodes.values()) + + def _restore(self, opt_graph: OptGraph, metadata: Optional[Dict[str, Any]] = None) -> nx.DiGraph: + nx_graph = nx.DiGraph() + nx_node_data = {} + + # add nodes + for node in opt_graph.nodes: + nx_node_data[node.uid] = self._node_restore(node) + nx_graph.add_node(node.uid) + + # add edges + for node in opt_graph.nodes: + for parent in node.nodes_from: + nx_graph.add_edge(parent.uid, node.uid) + + # add nodes ad labels + nx.set_node_attributes(nx_graph, nx_node_data) + + return nx_graph + + +_NX_NODE_KEY = 'data' + + +class DumbNetworkxAdapter(BaseNetworkxAdapter): + """Simple version of networkx adapter that just stores + `OptNode` as an attribute of NetworkX graph node.""" + + def _node_restore(self, node: GraphNode) -> Dict: + return {_NX_NODE_KEY: node} + + def _node_adapt(self, data: Dict) -> OptNode: + return data[_NX_NODE_KEY] + + +class BanditNetworkxAdapter(BaseNetworkxAdapter): + """ Classic networkx adapter with nodes indexes in names instead of uids. + It is needed since some frameworks (e.g. karateclub) have asserts in which node + names should consist only of its indexes. + """ + def _restore(self, opt_graph: OptGraph, metadata: Optional[Dict[str, Any]] = None) -> nx.DiGraph: + nx_graph = nx.DiGraph() + nx_node_data = {} + + # add nodes + for node in opt_graph.nodes: + nx_node_data[node.uid] = self._node_restore(node) + nx_graph.add_node(opt_graph.nodes.index(node)) + + # add edges + for node in opt_graph.nodes: + for parent in node.nodes_from: + nx_graph.add_edge(opt_graph.nodes.index(parent), opt_graph.nodes.index(node)) + + # add nodes ad labels + nx.set_node_attributes(nx_graph, nx_node_data) + + return nx_graph diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/constants.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/constants.py new file mode 100644 index 0000000..c9fad3b --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/constants.py @@ -0,0 +1,10 @@ +import numpy as np + +MAX_GRAPH_GEN_ATTEMPTS = 1000 +MAX_TUNING_METRIC_VALUE = np.inf +MIN_TIME_FOR_TUNING_IN_SEC = 3 +# Max number of evaluations attempts to collect the next pop; See usages. +EVALUATION_ATTEMPTS_NUMBER = 5 +# Min pop size to avoid getting stuck in local maximum during optimization. +MIN_POP_SIZE = 5 +DEFAULT_API_TIMEOUT_MINUTES = 5.0 diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/dag/__init__.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/dag/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/dag/convert.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/dag/convert.py new file mode 100644 index 0000000..67474bf --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/dag/convert.py @@ -0,0 +1,29 @@ +from typing import Tuple, Dict, TYPE_CHECKING +from uuid import uuid4 + +import networkx as nx + +if TYPE_CHECKING: + from golem.core.dag.graph import Graph + from golem.core.dag.graph_node import GraphNode + + +def graph_structure_as_nx_graph(structural_graph: 'Graph') -> Tuple[nx.DiGraph, Dict[uuid4, 'GraphNode']]: + """ Convert graph into networkx graph object """ + nx_graph = nx.DiGraph() + node_labels = {} + new_node_indices = {} + for node in structural_graph.nodes: + unique_id = uuid4() + node_labels[unique_id] = node + new_node_indices[node] = unique_id + nx_graph.add_node(unique_id) + + def add_edges(nx_graph, structural_graph, new_node_indices): + for node in structural_graph.nodes: + for parent in node.nodes_from: + nx_graph.add_edge(new_node_indices[parent], + new_node_indices[node]) + + add_edges(nx_graph, structural_graph, new_node_indices) + return nx_graph, node_labels diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/dag/graph.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/dag/graph.py new file mode 100644 index 0000000..239bb23 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/dag/graph.py @@ -0,0 +1,283 @@ +from abc import ABC, abstractmethod +from enum import Enum +from os import PathLike +from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple, TypeVar, Union + +import networkx as nx + +from golem.core.dag.graph_node import GraphNode +from golem.visualisation.graph_viz import GraphVisualizer, NodeColorType + +NodeType = TypeVar('NodeType', bound=GraphNode, covariant=False, contravariant=False) + + +class ReconnectType(Enum): + """Defines allowed kinds of removals in Graph. Used by mutations.""" + none = 'none' # do not reconnect predecessors + single = 'single' # reconnect a predecessor only if it's single + all = 'all' # reconnect all predecessors to all successors + + +class Graph(ABC): + """Defines abstract graph interface that's required by graph optimisation process. + """ + + @abstractmethod + def add_node(self, node: GraphNode): + """Adds new node to the graph together with its parent nodes. + + Args: + node: graph nodes + """ + raise NotImplementedError() + + @abstractmethod + def update_node(self, old_node: GraphNode, new_node: GraphNode): + """Replaces ``old_node`` node with ``new_node`` + + Args: + old_node: node to be replaced + new_node: node to be placed instead + """ + raise NotImplementedError() + + @abstractmethod + def update_subtree(self, old_subtree: GraphNode, new_subtree: GraphNode): + """Changes ``old_subtree`` subtree to ``new_subtree`` + + Args: + old_subtree: node and its subtree to be removed + new_subtree: node and its subtree to be placed instead + """ + raise NotImplementedError() + + @abstractmethod + def delete_node(self, node: GraphNode, reconnect: ReconnectType = ReconnectType.single): + """Removes ``node`` from the graph. + If ``node`` has only one child, then connects all of the ``node`` parents to it. + + Args: + node: node of the graph to be deleted + reconnect: defines how to treat left edges between parents and children + """ + raise NotImplementedError() + + @abstractmethod + def delete_subtree(self, subtree: GraphNode): + """Deletes given node with all its parents. + Deletes all edges from removed nodes to remaining graph nodes + + Args: + subtree: node to be deleted with all of its parents + and their connections amongst the remaining graph nodes + """ + raise NotImplementedError() + + @abstractmethod + def node_children(self, node: GraphNode) -> Sequence[Optional[GraphNode]]: + """Returns all children of the ``node`` + + Args: + node: for getting children from + + Returns: children of the ``node`` + """ + raise NotImplementedError() + + @abstractmethod + def connect_nodes(self, node_parent: GraphNode, node_child: GraphNode): + """Adds edge between ``parent`` and ``child`` + + Args: + node_parent: acts like parent in graph connection relations + node_child: acts like child in graph connection relations + """ + raise NotImplementedError() + + @abstractmethod + def disconnect_nodes(self, node_parent: GraphNode, node_child: GraphNode, + clean_up_leftovers: bool = False): + """Removes an edge between two nodes + + Args: + node_parent: where the removing edge comes out + node_child: where the removing edge enters + clean_up_leftovers: whether to remove the remaining invalid vertices with edges or not + """ + raise NotImplementedError() + + @abstractmethod + def get_edges(self) -> Sequence[Tuple[GraphNode, GraphNode]]: + """Gets all available edges in this graph + + Returns: + pairs of parent_node -> child_node + """ + raise NotImplementedError() + + def get_nodes_by_name(self, name: str) -> List[GraphNode]: + """Returns list of nodes with the required ``name`` + + Args: + name: name to filter by + + Returns: + list: relevant nodes (empty if there are no such nodes) + """ + + appropriate_nodes = filter(lambda x: x.name == name, self.nodes) + + return list(appropriate_nodes) + + def get_node_by_uid(self, uid: str) -> Optional[GraphNode]: + """Returns node with the required ``uid`` + + Args: + uid: uid of node to filter by + + Returns: + Optional[Node]: relevant node (None if there is no such node) + """ + + appropriate_nodes = list(filter(lambda x: x.uid == uid, self.nodes)) + + return appropriate_nodes[0] if appropriate_nodes else None + + @abstractmethod + def __eq__(self, other_graph: 'Graph') -> bool: + """Compares this graph with the ``other_graph`` + + Args: + other_graph: another graph + + Returns: + is it equal to ``other_graph`` in terms of the graphs + """ + raise NotImplementedError() + + def root_nodes(self) -> Sequence[GraphNode]: + raise NotImplementedError() + + @property + def root_node(self) -> Union[GraphNode, Sequence[GraphNode]]: + """Gets the final layer node(s) of the graph + + Returns: + the final layer node(s) + """ + roots = self.root_nodes() + if len(roots) == 1: + return roots[0] + return roots + + @property + @abstractmethod + def nodes(self) -> List[GraphNode]: + """Return list of all graph nodes + + Returns: + graph nodes + """ + raise NotImplementedError() + + @nodes.setter + @abstractmethod + def nodes(self, new_nodes: List[GraphNode]): + raise NotImplementedError() + + @property + @abstractmethod + def depth(self) -> int: + """Gets this graph depth from its sink-node to its source-node + + Returns: + length of a path from the root node to the farthest primary node + """ + raise NotImplementedError() + + @property + def length(self) -> int: + """Return size of the graph (number of nodes) + + Returns: + graph size + """ + + return len(self.nodes) + + def show(self, save_path: Optional[Union[PathLike, str]] = None, engine: Optional[str] = None, + node_color: Optional[NodeColorType] = None, dpi: Optional[int] = None, + node_size_scale: Optional[float] = None, font_size_scale: Optional[float] = None, + edge_curvature_scale: Optional[float] = None, + title: Optional[str] = None, + node_names_placement: Optional[Literal['auto', 'nodes', 'legend', 'none']] = None, + nodes_labels: Dict[int, str] = None, edges_labels: Dict[int, str] = None, + nodes_layout_function: Optional[Callable[[nx.DiGraph], Dict[Any, Tuple[float, float]]]] = None): + """Visualizes graph or saves its picture to the specified ``path`` + + Args: + save_path: optional, save location of the graph visualization image. + engine: engine to visualize the graph. Possible values: 'matplotlib', 'pyvis', 'graphviz'. + node_color: color of nodes to use. + node_size_scale: use to make node size bigger or lesser. Supported only for the engine 'matplotlib'. + font_size_scale: use to make font size bigger or lesser. Supported only for the engine 'matplotlib'. + edge_curvature_scale: use to make edges more or less curved. Supported only for the engine 'matplotlib'. + dpi: DPI of the output image. Not supported for the engine 'pyvis'. + title: title for plot + node_names_placement: variant of node names displaying. Defaults to ``auto``. + + Possible options: + + - ``auto`` -> empirical rule by node size + + - ``nodes`` -> place node names on top of the nodes + + - ``legend`` -> place node names at the legend + + - ``none`` -> do not show node names + + nodes_labels: labels to display near nodes + edges_labels: labels to display near edges + nodes_layout_function: any of `Networkx layout functions \ + `_ . + """ + GraphVisualizer(graph=self) \ + .visualise(save_path=save_path, engine=engine, node_color=node_color, dpi=dpi, + node_size_scale=node_size_scale, font_size_scale=font_size_scale, + edge_curvature_scale=edge_curvature_scale, node_names_placement=node_names_placement, + title=title, nodes_layout_function=nodes_layout_function, + nodes_labels=nodes_labels, edges_labels=edges_labels) + + @property + def graph_description(self) -> Dict: + """Return summary characteristics of the graph + + Returns: + dict: containing information about the graph + """ + return { + 'depth': self.depth, + 'length': self.length, + 'nodes': self.nodes, + } + + @property + def descriptive_id(self) -> str: + """Returns human-readable identifier of the graph. + + Returns: + str: text description of the content in the node and its parameters + """ + if self.root_nodes: + return self.root_node.descriptive_id + else: + return sorted(self.nodes, key=lambda x: x.uid)[0].descriptive_id + + def __str__(self): + return str(self.graph_description) + + def __repr__(self): + return self.__str__() + + def __len__(self): + return self.length diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/dag/graph_delegate.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/dag/graph_delegate.py new file mode 100644 index 0000000..765ab19 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/dag/graph_delegate.py @@ -0,0 +1,82 @@ +from typing import Union, Sequence, List, Optional, Tuple, Type + +from golem.core.dag.graph import Graph, ReconnectType +from golem.core.dag.graph_node import GraphNode +from golem.core.dag.linked_graph import LinkedGraph + + +class GraphDelegate(Graph): + """ + Graph that delegates calls to another Graph implementation. + + The class purpose is for cleaner code organisation: + - avoid inheriting from specific Graph implementations + - hide Graph implementation details from inheritors. + """ + + def __init__(self, *args, delegate_cls: Type[Graph] = LinkedGraph, **kwargs): + self.operator = delegate_cls(*args, **kwargs) + + def add_node(self, node: GraphNode): + self.operator.add_node(node) + + def update_node(self, old_node: GraphNode, new_node: GraphNode): + self.operator.update_node(old_node, new_node) + + def update_subtree(self, old_subtree: GraphNode, new_subtree: GraphNode): + self.operator.update_subtree(old_subtree, new_subtree) + + def delete_node(self, node: GraphNode, reconnect: ReconnectType = ReconnectType.single): + self.operator.delete_node(node, reconnect) + + def delete_subtree(self, subtree: GraphNode): + self.operator.delete_subtree(subtree) + + def node_children(self, node: GraphNode) -> Sequence[Optional[GraphNode]]: + return self.operator.node_children(node=node) + + def connect_nodes(self, node_parent: GraphNode, node_child: GraphNode): + self.operator.connect_nodes(node_parent, node_child) + + def disconnect_nodes(self, node_parent: GraphNode, node_child: GraphNode, + clean_up_leftovers: bool = False): + self.operator.disconnect_nodes(node_parent, node_child, clean_up_leftovers) + + def get_edges(self) -> Sequence[Tuple[GraphNode, GraphNode]]: + return self.operator.get_edges() + + def __eq__(self, other) -> bool: + return self.operator.__eq__(other) + + def __str__(self): + return self.operator.__str__() + + def __repr__(self): + return self.operator.__repr__() + + def root_nodes(self) -> Sequence[GraphNode]: + return self.operator.root_nodes() + + @property + def root_node(self) -> Union[GraphNode, Sequence[GraphNode]]: + return self.operator.root_node + + @property + def nodes(self) -> List[GraphNode]: + return self.operator.nodes + + @nodes.setter + def nodes(self, new_nodes: List[GraphNode]): + self.operator.nodes = new_nodes + + @property + def descriptive_id(self): + return self.operator.descriptive_id + + @property + def length(self) -> int: + return self.operator.length + + @property + def depth(self) -> int: + return self.operator.depth diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/dag/graph_node.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/dag/graph_node.py new file mode 100644 index 0000000..edc68e6 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/dag/graph_node.py @@ -0,0 +1,118 @@ +from abc import ABC, abstractmethod +from copy import copy +from typing import List, Optional, Iterable +from uuid import uuid4 + + +class GraphNode(ABC): + """Definition of the node in directed graph structure. + + Provides interface for getting and modifying the parent nodes + and recursive description based on all preceding nodes. + """ + def __init__(self): + self.uid = str(uuid4()) + + @property + @abstractmethod + def nodes_from(self) -> List['GraphNode']: + """Gets all parent nodes of this graph node + + Returns: + List['GraphNode']: all the parent nodes + """ + pass + + @nodes_from.setter + @abstractmethod + def nodes_from(self, nodes: Optional[Iterable['GraphNode']]): + """Changes value of parent nodes of this graph node + + Args: + nodes: new sequence of parent nodes + """ + pass + + @property + @abstractmethod + def name(self) -> str: + """ Str name of this graph node """ + pass + + @abstractmethod + def __str__(self) -> str: + """Returns short node type description + + Returns: + str: text graph node representation + """ + pass + + def __repr__(self) -> str: + """Returns full node description + + Returns: + str: text graph node representation + """ + return self.__str__() + + def description(self) -> str: + """Returns full node description + for use in recursive id. + + Returns: + str: text graph node representation + """ + + return self.__str__() + + @property + def descriptive_id(self) -> str: + """Returns structural identifier of the subgraph starting at this node + + Returns: + str: text description of the content in the node and its parameters + """ + return descriptive_id_recursive(self) + + +def descriptive_id_recursive(current_node: GraphNode, visited_nodes=None) -> str: + """ Returns descriptive id with nodes names. """ + if visited_nodes is None: + visited_nodes = [] + + node_label = current_node.description() + + full_path_items = [] + if current_node in visited_nodes: + return 'ID_CYCLED' + visited_nodes.append(current_node) + if current_node.nodes_from: + previous_items = [] + for parent_node in current_node.nodes_from: + previous_items.append(f'{descriptive_id_recursive(parent_node, copy(visited_nodes))};') + previous_items.sort() + previous_items_str = ';'.join(previous_items) + + full_path_items.append(f'({previous_items_str})') + full_path_items.append(f'/{node_label}') + full_path = ''.join(full_path_items) + return full_path + + +def descriptive_id_recursive_nodes(current_node: GraphNode, visited_nodes=None) -> List[GraphNode]: + """ Returns descriptive id with nodes, not with its names. """ + if visited_nodes is None: + visited_nodes = [] + + full_path_items = [] + if current_node in visited_nodes: + return [] + visited_nodes.append(current_node) + if current_node.nodes_from: + previous_items = [] + for parent_node in current_node.nodes_from: + previous_items.extend(descriptive_id_recursive_nodes(parent_node, copy(visited_nodes))) + full_path_items.extend(previous_items) + full_path_items.append(current_node) + return full_path_items diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/dag/graph_utils.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/dag/graph_utils.py new file mode 100644 index 0000000..c2068b1 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/dag/graph_utils.py @@ -0,0 +1,268 @@ +from typing import Sequence, List, TYPE_CHECKING, Callable, Union, Optional + +from golem.utilities.data_structures import ensure_wrapped_in_sequence + +if TYPE_CHECKING: + from golem.core.dag.graph import Graph + from golem.core.dag.graph_node import GraphNode + + +def distance_to_root_level(graph: 'Graph', node: 'GraphNode') -> int: + """Gets distance to the final output node + + Args: + graph: graph for finding the distance + node: search starting point + + Return: + int: distance to root level + """ + + def child_height(parent_node: 'GraphNode') -> int: + height = 0 + for _ in range(graph.length): + node_children = graph.node_children(parent_node) + if node_children: + height += 1 + parent_node = node_children[0] + else: + return height + + if graph_has_cycle(graph): + return -1 + height = child_height(node) + return height + + +def distance_to_primary_level(node: 'GraphNode') -> int: + depth = node_depth(node) + return depth - 1 if depth > 0 else -1 + + +def nodes_from_layer(graph: 'Graph', layer_number: int) -> Sequence['GraphNode']: + """Gets all the nodes from the chosen layer up to the surface + + Args: + graph: graph with nodes + layer_number: max height of diving + + Returns: + all nodes from the surface to the ``layer_number`` layer + """ + + def get_nodes(roots: Sequence['GraphNode'], current_height: int) -> Sequence['GraphNode']: + """Gets all the parent nodes of ``roots`` + + :param roots: nodes to get all subnodes from + :param current_height: current diving step depth + + :return: all parent nodes of ``roots`` in one sequence:69 + """ + nodes = [] + if current_height == layer_number: + nodes.extend(roots) + else: + for root in roots: + nodes.extend(get_nodes(root.nodes_from, current_height + 1)) + return nodes + + nodes = get_nodes(graph.root_nodes(), current_height=0) + return nodes + + +def ordered_subnodes_hierarchy(node: 'GraphNode') -> List['GraphNode']: + """Gets hierarchical subnodes representation of the graph starting from the bounded node + + Returns: + List['GraphNode']: hierarchical subnodes list starting from the bounded node + """ + started = {node} + visited = set() + + def subtree_impl(node): + nodes = [node] + for parent in node.nodes_from: + if parent in visited: + continue + elif parent in started: + raise ValueError('Can not build ordered node hierarchy: graph has cycle') + started.add(parent) + nodes.extend(subtree_impl(parent)) + visited.add(parent) + return nodes + + return subtree_impl(node) + + +def node_depth(nodes: Union['GraphNode', Sequence['GraphNode']]) -> int: + """Gets the maximal depth among the provided ``nodes`` in the graph + + Args: + nodes: nodes to calculate the depth for + + Returns: + int: maximal depth + """ + nodes = ensure_wrapped_in_sequence(nodes) + final_depth = {} + subnodes = set() + for node in nodes: + max_depth = 0 + # if node is a subnode of another node it has smaller depth + if node.uid in subnodes: + continue + depth = 1 + visited = [] + if node in visited: + return -1 + visited.append(node) + stack = [(node, depth, iter(node.nodes_from))] + while stack: + curr_node, depth_now, parents = stack[-1] + try: + parent = next(parents) + subnodes.add(parent.uid) + if parent not in visited: + visited.append(parent) + if parent.uid in final_depth: + # depth of the parent has been already calculated + stack.append((parent, depth_now + final_depth[parent.uid], iter([]))) + else: + stack.append((parent, depth_now + 1, iter(parent.nodes_from))) + else: + return -1 + except StopIteration: + _, depth_now, _ = stack.pop() + visited.pop() + max_depth = max(max_depth, depth_now) + final_depth[node.uid] = max_depth + return max(final_depth.values()) + + +def map_dag_nodes(transform: Callable, nodes: Sequence) -> Sequence: + """Maps nodes in dfs-order while respecting node edges. + + Args: + transform: node transform function (maps node to node) + nodes: sequence of nodes for mapping + + Returns: + Sequence: sequence of transformed links with preserved relations + """ + mapped_nodes = {} + + def map_impl(node): + already_mapped = mapped_nodes.get(id(node)) + if already_mapped: + return already_mapped + # map node itself + mapped_node = transform(node) + # remember it to avoid recursion + mapped_nodes[id(node)] = mapped_node + # map its children + mapped_node.nodes_from = list(map(map_impl, node.nodes_from)) + return mapped_node + + return list(map(map_impl, nodes)) + + +def graph_structure(graph: 'Graph') -> str: + """ Returns structural information about the graph - names and parameters of graph nodes. + Represents graph info in easily readable way. + + Returns: + str: graph structure + """ + return '\n'.join([str(graph), *(f'{node.name} - {node.parameters}' for node in graph.nodes)]) + + +def graph_has_cycle(graph: 'Graph') -> bool: + """ Returns True if the graph contains a cycle and False otherwise. Implements Depth-First Search.""" + + visited = {node.uid: False for node in graph.nodes} + stack = [] + on_stack = {node.uid: False for node in graph.nodes} + for node in graph.nodes: + if visited[node.uid]: + continue + stack.append(node) + while len(stack) > 0: + cur_node = stack[-1] + if not visited[cur_node.uid]: + visited[cur_node.uid] = True + on_stack[cur_node.uid] = True + else: + on_stack[cur_node.uid] = False + stack.pop() + for parent in cur_node.nodes_from: + if not visited[parent.uid]: + stack.append(parent) + elif on_stack[parent.uid]: + return True + return False + + +def get_all_simple_paths(graph: 'Graph', source: 'GraphNode', target: 'GraphNode') \ + -> List[List[List['GraphNode']]]: + """ Returns all simple paths from one node to another ignoring edge direction. + Args: + graph: graph in which to search for paths + source: the first node of the path + target: the last node of the path """ + paths = [] + nodes_children = {source.uid: graph.node_children(source)} + target = {target} + visited = dict.fromkeys([source]) + node_neighbors = set(source.nodes_from).union(nodes_children[source.uid]) + stack = [iter(node_neighbors)] + + while stack: + neighbors = stack[-1] + neighbor = next(neighbors, None) + if neighbor is None: # current path does not contain target + stack.pop() + visited.popitem() + else: + if neighbor in visited: # path is not simple + continue + if neighbor in target: # target node was reached + path = list(visited) + [neighbor] + pairs_list = [[path[i], path[(i + 1)]] for i in range(len(path) - 1)] + paths.append(pairs_list) + else: # target node was not reached + visited[neighbor] = None + children = nodes_children[neighbor.uid] if neighbor.uid in nodes_children \ + else nodes_children.setdefault(neighbor.uid, graph.node_children(neighbor)) # lazy setdefault + node_neighbors = set(neighbor.nodes_from).union(children) + stack.append(iter(node_neighbors)) + return paths + + +def get_connected_components(graph: 'Graph', nodes: Optional[List['GraphNode']]) -> List[set]: + """ Returns list of connected components of the graph. + Each connected component is represented as a set of its nodes. + Args: + graph: graph to divide into connected components + nodes: if provided, only connected components containing these nodes are returned + Returns: + List of connected components""" + def _bfs(graph: 'Graph', source: 'GraphNode'): + seen = set() + nextlevel = {source} + while nextlevel: + thislevel = nextlevel + nextlevel = set() + for v in thislevel: + if v not in seen: + seen.add(v) + nextlevel.update(set(v.nodes_from).union(set(graph.node_children(v)))) + return seen + visited = set() + nodes = nodes or graph.nodes + components = [] + for node in nodes: + if node not in visited: + c = _bfs(graph, node) + visited.update(c) + components.append(c) + return components diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/dag/graph_verifier.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/dag/graph_verifier.py new file mode 100644 index 0000000..11cd9b5 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/dag/graph_verifier.py @@ -0,0 +1,43 @@ +from typing import Sequence, Optional, Callable + +from golem.core.adapter import BaseOptimizationAdapter +from golem.core.adapter.adapter import IdentityAdapter +from golem.core.dag.graph import Graph +from golem.core.log import default_log + +# Validation rule can either return False or raise a ValueError to signal a failed check +VerifierRuleType = Callable[..., bool] + + +class VerificationError(ValueError): + pass + + +class GraphVerifier: + def __init__(self, rules: Sequence[VerifierRuleType] = (), + adapter: Optional[BaseOptimizationAdapter] = None, + raise_on_failure: bool = False): + self._adapter = adapter or IdentityAdapter() + self._rules = rules + self._log = default_log(self) + self._raise = raise_on_failure + + def __call__(self, graph: Graph) -> bool: + return self.verify(graph) + + def verify(self, graph: Graph) -> bool: + # Check if all rules pass + adapt = self._adapter.adapt_func + for rule in self._rules: + try: + if adapt(rule)(graph) is False: + return False + except ValueError as err: + msg = f'Graph verification failed with error <{err}> '\ + f'for rule={rule} on graph={graph.descriptive_id}.' + if self._raise: + raise VerificationError(msg) + else: + self._log.debug(msg) + return False + return True diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/dag/linked_graph.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/dag/linked_graph.py new file mode 100644 index 0000000..789c3f6 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/dag/linked_graph.py @@ -0,0 +1,226 @@ +from copy import deepcopy +from typing import Any, Dict, List, Optional, Tuple, Union, Callable, Sequence + +from networkx import graph_edit_distance, set_node_attributes + +from golem.core.dag.convert import graph_structure_as_nx_graph +from golem.core.dag.graph import Graph, ReconnectType +from golem.core.dag.graph_node import GraphNode +from golem.core.dag.graph_utils import ordered_subnodes_hierarchy, node_depth, graph_has_cycle +from golem.core.paths import copy_doc +from golem.utilities.data_structures import ensure_wrapped_in_sequence, Copyable, remove_items + +NodePostprocessCallable = Callable[[Graph, Sequence[GraphNode]], Any] + + +class LinkedGraph(Graph, Copyable): + """Graph implementation based on linked graph node + that directly stores its parent nodes. + + Args: + nodes: nodes of the Graph + postprocess_nodes: nodes postprocessing function used after their modification + """ + + def __init__(self, nodes: Union[GraphNode, Sequence[GraphNode]] = (), + postprocess_nodes: Optional[NodePostprocessCallable] = None): + self._nodes = [] + for node in ensure_wrapped_in_sequence(nodes): + self.add_node(node) + self._postprocess_nodes = postprocess_nodes or self._empty_postprocess + + @staticmethod + def _empty_postprocess(*args): + pass + + @copy_doc(Graph.delete_node) + def delete_node(self, node: GraphNode, reconnect: ReconnectType = ReconnectType.single) -> object: + node_children_cached = self.node_children(node) + + self._nodes.remove(node) + for node_child in node_children_cached: + node_child.nodes_from.remove(node) + + if reconnect == ReconnectType.single: + # if removed node had a single child + # then reconnect it to preceding parent nodes. + if node.nodes_from and len(node_children_cached) == 1: + child = node_children_cached[0] + child.nodes_from.extend(node.nodes_from) + elif reconnect == ReconnectType.all: + if node.nodes_from: + for child in node_children_cached: + child.nodes_from.extend(node.nodes_from) + elif reconnect == ReconnectType.none: + pass + + self._postprocess_nodes(self, self._nodes) + + @copy_doc(Graph.delete_subtree) + def delete_subtree(self, subtree: GraphNode): + subtree_nodes = ordered_subnodes_hierarchy(subtree) + self._nodes = remove_items(self._nodes, subtree_nodes) + # prune all edges coming from the removed subtree + for subtree in self._nodes: + subtree.nodes_from = remove_items(subtree.nodes_from, subtree_nodes) + + @copy_doc(Graph.update_node) + def update_node(self, old_node: GraphNode, new_node: GraphNode): + self.actualise_old_node_children(old_node, new_node) + new_node.nodes_from.extend(old_node.nodes_from) + self._nodes.remove(old_node) + self.add_node(new_node) + self.sort_nodes() + self._postprocess_nodes(self, self._nodes) + + @copy_doc(Graph.update_subtree) + def update_subtree(self, old_subtree: GraphNode, new_subtree: GraphNode): + new_subtree = deepcopy(new_subtree) + self.actualise_old_node_children(old_subtree, new_subtree) + self.delete_subtree(old_subtree) + self.add_node(new_subtree) + self.sort_nodes() + + @copy_doc(Graph.add_node) + def add_node(self, node: GraphNode): + if node not in self._nodes: + self._nodes.append(node) + for n in node.nodes_from: + self.add_node(n) + + def actualise_old_node_children(self, old_node: GraphNode, new_node: GraphNode): + """Changes parent of ``old_node`` children to ``new_node`` + + :param old_node: node to take children from + :param new_node: new parent of ``old_node`` children + """ + old_node_offspring = self.node_children(old_node) + for old_node_child in old_node_offspring: + updated_index = old_node_child.nodes_from.index(old_node) + old_node_child.nodes_from[updated_index] = new_node + + def sort_nodes(self): + """ Layer by layer sorting """ + if not isinstance(self.root_node, Sequence) and not graph_has_cycle(self): + self._nodes = ordered_subnodes_hierarchy(self.root_node) + + @copy_doc(Graph.node_children) + def node_children(self, node: GraphNode) -> List[Optional[GraphNode]]: + """ Returns list of children of specified node. """ + return [other_node for other_node in self._nodes + if other_node.nodes_from and + node in other_node.nodes_from] + + @copy_doc(Graph.connect_nodes) + def connect_nodes(self, node_parent: GraphNode, node_child: GraphNode): + if node_child in self.node_children(node_parent): + return + node_child.nodes_from.append(node_parent) + + def _clean_up_leftovers(self, node: GraphNode): + """Removes nodes and edges that do not affect the result of the graph. + Leftovers are edges and nodes that remain after the removal of the edge / node + and do not affect the result of the graph. + + :param node: node to be deleted with all of its parents + """ + + if not self.node_children(node): + self._nodes.remove(node) + for node in node.nodes_from: + self._clean_up_leftovers(node) + + @copy_doc(Graph.disconnect_nodes) + def disconnect_nodes(self, node_parent: GraphNode, node_child: GraphNode, + clean_up_leftovers: bool = False): + if node_parent not in node_child.nodes_from: + return + if node_parent not in self._nodes or node_child not in self._nodes: + return + node_child.nodes_from.remove(node_parent) + if clean_up_leftovers: + self._clean_up_leftovers(node_parent) + self._postprocess_nodes(self, self._nodes) + + def root_nodes(self) -> Sequence[GraphNode]: + return [node for node in self._nodes if not any(self.node_children(node))] + + @property + def nodes(self) -> List[GraphNode]: + return self._nodes + + @nodes.setter + def nodes(self, new_nodes: List[GraphNode]): + self._nodes = new_nodes + + @copy_doc(Graph.__eq__) + def __eq__(self, other_graph: Graph) -> bool: + return \ + set(rn.descriptive_id for rn in self.root_nodes()) == \ + set(rn.descriptive_id for rn in other_graph.root_nodes()) + + @copy_doc(Graph.descriptive_id) + @property + def descriptive_id(self) -> str: + if self.length == 0: + return 'EMPTY' + elif self.root_nodes(): + return ''.join([r.descriptive_id for r in self.root_nodes()]) + else: + return sorted(self.nodes, key=lambda x: x.uid)[0].descriptive_id + + @copy_doc(Graph.depth) + @property + def depth(self) -> int: + if not self._nodes: + return 0 + elif not self.root_nodes() or graph_has_cycle(self): + return -1 + else: + depths = node_depth(self.root_nodes()) + return max(ensure_wrapped_in_sequence(depths)) + + @copy_doc(Graph.get_edges) + def get_edges(self) -> Sequence[Tuple[GraphNode, GraphNode]]: + edges = [] + for node in self._nodes: + if node.nodes_from: + for parent_node in node.nodes_from: + edges.append((parent_node, node)) + return edges + + +def get_distance_between(graph_1: Graph, graph_2: Graph) -> int: + """ + Gets edit distance from ``graph_1`` graph to the ``graph_2`` + + :param graph_1: left object to compare + :param graph_2: right object to compare + + :return: graph edit distance (aka Levenstein distance for graphs) + """ + + def node_match(node_data_1: Dict[str, GraphNode], node_data_2: Dict[str, GraphNode]) -> bool: + """Checks if the two given nodes are identical + + :param node_data_1: nx_graph format for the first node to compare + :param node_data_2: nx_graph format for the second node to compare + + :return: is the first node equal to the second + """ + node_1, node_2 = node_data_1.get('node'), node_data_2.get('node') + + operations_do_match = str(node_1) == str(node_2) + params_do_match = node_1.content.get('params') == node_2.content.get('params') + nodes_do_match = operations_do_match and params_do_match + return nodes_do_match + + graphs = (graph_1, graph_2) + nx_graphs = [] + for graph in graphs: + nx_graph, nodes = graph_structure_as_nx_graph(graph) + set_node_attributes(nx_graph, nodes, name='node') + nx_graphs.append(nx_graph) + + distance = graph_edit_distance(*nx_graphs, node_match=node_match) + return int(distance) diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/dag/linked_graph_node.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/dag/linked_graph_node.py new file mode 100644 index 0000000..20a19ac --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/dag/linked_graph_node.py @@ -0,0 +1,76 @@ +from typing import Union, Optional, Iterable, List +from golem.core.dag.graph_node import GraphNode +from golem.utilities.data_structures import UniqueList + + +class LinkedGraphNode(GraphNode): + """Class for node definition in the directed graph structure + that directly stores its parent nodes. + + Args: + nodes_from: parent nodes which information comes from + content: ``dict`` for the content in the node + + Notes: + The possible parameters are: + - ``name`` - name (str) or object that performs actions in this node + - ``params`` - dictionary with additional information that is used by + the object in the ``name`` field (e.g. hyperparameters values) + """ + + def __init__(self, content: Union[dict, str], + nodes_from: Optional[Iterable['LinkedGraphNode']] = None): + # Wrap string into dict if it is necessary + if isinstance(content, str): + content = {'name': content} + + self.content: dict = content + self._nodes_from = UniqueList(nodes_from or ()) + + super().__init__() + + @property + def nodes_from(self) -> List['LinkedGraphNode']: + return self._nodes_from + + @nodes_from.setter + def nodes_from(self, nodes: Optional[Iterable['LinkedGraphNode']]): + self._nodes_from = UniqueList(nodes) + + @property + def name(self) -> str: + name = self.content.get('name') + return str(name) if name is not None else '' + + @property + def parameters(self) -> dict: + return self.content.get('params', {}) + + @parameters.setter + def parameters(self, new_parameters): + if self.content.get('params'): + self.content['params'].update(new_parameters) + else: + self.content['params'] = new_parameters + + def __hash__(self) -> int: + return hash(self.uid) + + def __str__(self) -> str: + return str(self.content.get('name', self.uid)) + + def __repr__(self) -> str: + return self.__str__() + + def description(self) -> str: + label = self.name or self.uid + # TODO: possibly unify with __repr__ & don't duplicate Operation.description + if not self.parameters: + node_label = f'n_{label}' + elif isinstance(label, str): + # If there is a string: name of operation (as in json repository) + node_label = f'n_{label}_{self.parameters}' + else: + # If instance of Operation is placed in 'name' + node_label = label.description(self.parameters) + return node_label diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/dag/verification_rules.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/dag/verification_rules.py new file mode 100644 index 0000000..cbe6fda --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/dag/verification_rules.py @@ -0,0 +1,63 @@ +import networkx as nx +from networkx import isolates + +from golem.core.adapter import register_native +from golem.core.dag.convert import graph_structure_as_nx_graph +from golem.core.dag.graph import Graph +from golem.core.dag.graph_node import GraphNode +from golem.core.dag.graph_utils import graph_has_cycle + +ERROR_PREFIX = 'Invalid graph configuration:' + + +@register_native +def has_root(graph: Graph): + if graph.root_node: + return True + + +@register_native +def has_one_root(graph: Graph): + if isinstance(graph.root_node, GraphNode): + return True + + +@register_native +def has_no_cycle(graph: Graph): + if graph_has_cycle(graph): + raise ValueError(f'{ERROR_PREFIX} Graph has cycles') + + return True + + +@register_native +def has_no_isolated_nodes(graph: Graph): + nx_graph, _ = graph_structure_as_nx_graph(graph) + isolated = list(isolates(nx_graph)) + if len(isolated) > 0 and graph.length != 1: + raise ValueError(f'{ERROR_PREFIX} Graph has isolated nodes') + return True + + +@register_native +def has_no_self_cycled_nodes(graph: Graph): + if any([node for node in graph.nodes if node.nodes_from and node in node.nodes_from]): + raise ValueError(f'{ERROR_PREFIX} Graph has self-cycled nodes') + return True + + +@register_native +def has_no_isolated_components(graph: Graph): + nx_graph, _ = graph_structure_as_nx_graph(graph) + ud_nx_graph = nx.Graph() + ud_nx_graph.add_nodes_from(nx_graph) + ud_nx_graph.add_edges_from(nx_graph.edges) + if ud_nx_graph.number_of_nodes() == 0: + raise ValueError(f'{ERROR_PREFIX} Graph is null, connectivity not defined') + if not nx.is_connected(ud_nx_graph): + raise ValueError(f'{ERROR_PREFIX} Graph has isolated components') + return True + + +DEFAULT_DAG_RULES = [has_root, has_no_cycle, has_no_isolated_components, + has_no_self_cycled_nodes, has_no_isolated_nodes] diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/log.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/log.py new file mode 100644 index 0000000..bcf4471 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/log.py @@ -0,0 +1,243 @@ +import json +import logging +import multiprocessing +import os +import pathlib +import sys +from logging.config import dictConfig +from logging.handlers import RotatingFileHandler +from typing import Optional, Tuple, Union + +from typing_extensions import Literal + +from golem.core.paths import default_data_dir +from golem.utilities.singleton_meta import SingletonMeta + +DEFAULT_LOG_PATH = pathlib.Path(default_data_dir(), 'log.log') + + +class Log(metaclass=SingletonMeta): + """Log object to store logger singleton and log adapters + + Args: + config_json_file: ``json`` file from which to collect the logger if specified + output_logging_level: logging levels are the same as in standard python module 'logging' + log_file: file to write logs in + """ + + __log_adapters = {} + + def __init__(self, + config_json_file: str = 'default', + output_logging_level: int = logging.INFO, + log_file: Optional[Union[str, pathlib.Path]] = None, + use_console: bool = True): + self.log_file = log_file or DEFAULT_LOG_PATH + self.logger = self._get_logger(config_file=config_json_file, + logging_level=output_logging_level, + use_console=use_console) + + @staticmethod + def setup_in_mp(logging_level: int, logs_dir: pathlib.Path): + """ + Preserves logger level and its records in a separate file for each process only if it's a child one + + Args: + logging_level: level of the logger from the main process + logs_dir: path to the logs directory + """ + + cur_proc = multiprocessing.current_process().name + log_file_name = logs_dir.joinpath(f'log_{cur_proc}.log') + Log(output_logging_level=logging_level, log_file=log_file_name, use_console=False) + + def get_parameters(self) -> Tuple[int, pathlib.Path]: + return self.logger.level, pathlib.Path(self.log_file).parent + + def reset_logging_level(self, logging_level: int): + """ Resets logging level for logger and its handlers """ + # Resets logging level is needed because before initialization with API params Singleton + # can be initialized somewhere else with default ones + self.logger.setLevel(logging_level) + for handler in self.handlers: + handler.setLevel(logging_level) + for adapter in self.__log_adapters.values(): + adapter.logging_level = logging_level + + def get_adapter(self, prefix: str) -> 'LoggerAdapter': + """ Get adapter to pass contextual information to log messages + + Args: + prefix: prefix to log messages with this adapter. Usually, the prefix is the name of the class + where the log came from + """ + + if prefix not in self.__log_adapters.keys(): + self.__log_adapters[prefix] = LoggerAdapter(self.logger, + {'prefix': prefix}) + return self.__log_adapters[prefix] + + def _get_logger(self, config_file: str, logging_level: int, use_console: bool = True) -> logging.Logger: + """ Get logger object """ + logger = logging.getLogger() + if config_file != 'default': + self._setup_logger_from_json_file(config_file) + else: + logger = self._setup_default_logger(logger=logger, logging_level=logging_level, use_console=use_console) + return logger + + def _setup_default_logger(self, logger: logging.Logger, logging_level: int, + use_console: bool = True) -> logging.Logger: + """ Define console and file handlers for logger + """ + + if use_console: + console_handler = logging.StreamHandler(sys.stdout) + console_handler.setLevel(logging_level) + console_formatter = logging.Formatter('%(asctime)s - %(message)s') + console_handler.setFormatter(console_formatter) + logger.addHandler(console_handler) + + file_handler = RotatingFileHandler(self.log_file, maxBytes=100000000, backupCount=1) + file_handler.setLevel(logging_level) + file_handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')) + logger.addHandler(file_handler) + + logger.setLevel(logging_level) + + return logger + + @staticmethod + def _setup_logger_from_json_file(config_file): + """ Setup logging configuration from file + """ + + try: + with open(config_file, 'rt') as file: + config = json.load(file) + dictConfig(config) + except Exception as ex: + raise Exception(f'Can not open the log config file because of {ex}') + + @property + def handlers(self): + return self.logger.handlers + + def release_handlers(self): + """ This function closes handlers of logger + """ + + for handler in self.handlers: + handler.close() + + def getstate(self): + """ Define the attributes to be pickled via deepcopy or pickle + + Returns: + dict: ``dict`` of state + """ + + state = dict(self.__dict__) + del state['logger'] + return state + + def __str__(self): + return f'Log object for {self.logger.name} module' + + def __repr__(self): + return self.__str__() + + +class LoggerAdapter(logging.LoggerAdapter): + """ This class looks like logger but used to pass contextual information + to the output along with logging event information + """ + + def __init__(self, logger: logging.Logger, extra: dict): + super().__init__(logger=logger, extra=extra) + self.logging_level = logger.level + self.setLevel(self.logging_level) + + def process(self, msg, kwargs): + self.logger.setLevel(self.logging_level) + return '%s - %s' % (self.extra['prefix'], msg), kwargs + + def message(self, msg: str, **kwargs): + """ Record the message to user. + Message is an intermediate logging level between info and warning + to display main info about optimization process """ + level = 45 + self.log(level, msg, **kwargs) + + def log_or_raise( + self, level: Union[int, Literal['debug', 'info', 'warning', 'error', 'critical', 'message']], + exc: Union[BaseException, object], + **log_kwargs): + """ Logs the given exception with the given logging level or raises it if the current + session is a test one. + + The given exception is logged with its traceback. If this method is called inside an ``except`` block, + the exception caught earlier is used as a cause for the given exception. + + Args: + + level: the same as in :py:func:`logging.log`, but may be specified as a lower-case string literal + for convenience. For example, the value ``warning`` is equivalent for ``logging.WARNING``. + This includes a custom "message" logging level that equals to 45. + exc: the exception/message to log/raise. Given a message, an ``Exception`` instance is initialized + based on the message. + log_kwargs: keyword arguments for :py:func:`logging.log`. + """ + _, recent_exc, _ = sys.exc_info() # Catch the most recent exception + if not isinstance(exc, BaseException): + exc = Exception(exc) + try: + # Raise anyway to combine tracebacks + raise exc from recent_exc + except type(exc) as exc_info: + # Raise further if test session + if is_test_session(): + raise + # Log otherwise + level_map = { + 'debug': logging.DEBUG, + 'info': logging.INFO, + 'warning': logging.WARNING, + 'error': logging.ERROR, + 'critical': logging.CRITICAL, + 'message': 45, + } + if isinstance(level, str): + level = level_map[level] + self.log(level, exc, + exc_info=log_kwargs.pop('exc_info', exc_info), + stacklevel=log_kwargs.pop('stacklevel', 2), + **log_kwargs) + + def __str__(self): + return f'LoggerAdapter object for {self.extra["prefix"]} module' + + def __repr__(self): + return self.__str__() + + +def is_test_session(): + return 'PYTEST_CURRENT_TEST' in os.environ + + +def default_log(prefix: Optional[object] = 'default') -> 'LoggerAdapter': + """ Default logger + + Args: + prefix: adapter prefix to add it to log messages + + Returns: + :obj:`LoggerAdapter`: :obj:`LoggerAdapter` object + + """ + + # get log prefix + if not isinstance(prefix, str): + prefix = prefix.__class__.__name__ + + return Log().get_adapter(prefix=prefix) diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/__init__.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/__init__.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/agent_trainer.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/agent_trainer.py new file mode 100644 index 0000000..80e9b6c --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/agent_trainer.py @@ -0,0 +1,198 @@ +import operator +from copy import deepcopy +from functools import reduce +from typing import Sequence, Optional, Any, Tuple, List, Iterable + +import numpy as np + +from golem.core.dag.graph import Graph +from golem.core.log import default_log +from golem.core.optimisers.adaptive.common_types import TrajectoryStep, GraphTrajectory +from golem.core.optimisers.adaptive.experience_buffer import ExperienceBuffer +from golem.core.optimisers.adaptive.operator_agent import OperatorAgent +from golem.core.optimisers.fitness import Fitness +from golem.core.optimisers.genetic.operators.mutation import Mutation +from golem.core.optimisers.objective import Objective +from golem.core.optimisers.opt_history_objects.individual import Individual +from golem.core.optimisers.opt_history_objects.opt_history import OptHistory +from golem.core.optimisers.opt_history_objects.parent_operator import ParentOperator +from golem.utilities.data_structures import unzip + + +class AgentTrainer: + """Utility class providing fit/validate logic for adaptive Mutation agents. + Works in tandem with `HistoryReader`. + + How to use offline training: + + 1. Collect histories to some directory using `ExperimentLauncher` + 2. Create optimizer & Pretrain mutation agent on these histories using `HistoryReader` and `AgentTrainer` + 3. Optionally, validate the Agent on validation set of histories + 4. Run optimization with pretrained agent + """ + + def __init__(self, + objective: Objective, + mutation_operator: Mutation, + agent: Optional[OperatorAgent] = None, + ): + self._log = default_log(self) + self.agent = agent if agent is not None else mutation_operator.agent + self.mutation = mutation_operator + self.objective = objective + self._adapter = self.mutation.graph_generation_params.adapter + + def fit(self, histories: Iterable[OptHistory], validate_each: int = -1) -> OperatorAgent: + """ + Method to fit trainer on collected histories. + param histories: histories to use in training. + param validate_each: validate agent once in validate_each generation. + """ + # Set mutation probabilities to 1.0 + initial_req = deepcopy(self.mutation.requirements) + self.mutation.requirements.mutation_prob = 1.0 + + for i, history in enumerate(histories): + # Preliminary validity check + # This allows to filter out histories with different objectives automatically + if history.objective.metric_names != self.objective.metric_names: + self._log.warning(f'History #{i+1} has different objective! ' + f'Expected {self.objective}, got {history.objective}.') + continue + + # Build datasets + experience = ExperienceBuffer.from_history(history) + val_experience = None + if validate_each > 0 and i % validate_each == 0: + experience, val_experience = experience.split(ratio=0.8, shuffle=True) + + # Train + self._log.info(f'Training on history #{i+1} with {len(history.generations)} generations') + self.agent.partial_fit(experience) + + # Validate + if val_experience: + reward_loss, reward_target = self.validate_agent(experience=val_experience) + self._log.info(f'Agent validation for history #{i+1} & {experience}: ' + f'Reward target={reward_target:.3f}, loss={reward_loss:.3f}') + + # Reset mutation probabilities to default + self.mutation.update_requirements(requirements=initial_req) + return self.agent + + def validate_on_rollouts(self, histories: Sequence[OptHistory]) -> float: + """Validates rollouts of agent vs. historic trajectories, comparing + their mean total rewards (i.e. total fitness gain over the trajectory).""" + + # Collect all trajectories from all histories; and their rewards + trajectories = concat_lists(map(ExperienceBuffer.unroll_trajectories, histories)) + + mean_traj_len = int(np.mean([len(tr) for tr in trajectories])) + traj_rewards = [sum(reward for _, reward, _ in traj) for traj in trajectories] + mean_baseline_reward = np.mean(traj_rewards) + + # Collect same number of trajectories of the same length; and their rewards + agent_trajectories = [self._sample_trajectory(initial=tr[0][0], length=mean_traj_len) + for tr in trajectories] + agent_traj_rewards = [sum(reward for _, reward, _ in traj) for traj in agent_trajectories] + mean_agent_reward = np.mean(agent_traj_rewards) + + # Compute improvement score of agent over baseline histories + improvement = mean_agent_reward - mean_baseline_reward + return improvement + + def validate_history(self, history: OptHistory) -> Tuple[float, float]: + """Validates history of mutated individuals against optimal policy.""" + history_trajectories = ExperienceBuffer.unroll_trajectories(history) + return self._validate_against_optimal(history_trajectories) + + def validate_agent(self, + graphs: Optional[Sequence[Graph]] = None, + experience: Optional[ExperienceBuffer] = None) -> Tuple[float, float]: + """Validates agent policy against optimal policy on given graphs.""" + if experience: + agent_steps = experience.retrieve_trajectories() + elif graphs: + agent_steps = [self._make_action_step(Individual(g)) for g in graphs] + else: + self._log.warning('Either graphs or history must not be None for validation!') + return 0., 0. + return self._validate_against_optimal(trajectories=[agent_steps]) + + def _validate_against_optimal(self, trajectories: Sequence[GraphTrajectory]) -> Tuple[float, float]: + """Validates a policy trajectories against optimal policy + that at each step always chooses the best action with max reward.""" + reward_losses = [] + reward_targets = [] + for trajectory in trajectories: + inds, actions, rewards = unzip(trajectory) + _, best_actions, best_rewards = self._apply_best_action(inds) + reward_loss = self._compute_reward_loss(rewards, best_rewards) + reward_losses.append(reward_loss) + reward_targets.append(np.mean(best_rewards)) + reward_loss = float(np.mean(reward_losses)) + reward_target = float(np.mean(reward_targets)) + return reward_loss, reward_target + + @staticmethod + def _compute_reward_loss(rewards, optimal_rewards, normalized=False) -> float: + """Returns difference (or deviation) from optimal reward. + When normalized, 0. means actual rewards match optimal rewards completely, + 0.5 means they on average deviate by 50% from optimal rewards, + and 2.2 means they on average deviate by more than 2 times from optimal reward.""" + reward_losses = np.subtract(optimal_rewards, rewards) # always positive + if normalized: + reward_losses = reward_losses / np.abs(optimal_rewards) \ + if np.count_nonzero(optimal_rewards) == optimal_rewards.size else reward_losses + means = np.mean(reward_losses) + return float(means) + + def _apply_best_action(self, inds: Sequence[Individual]) -> TrajectoryStep: + """Returns greedily optimal mutation for given graph and associated reward.""" + candidates = [] + for ind in inds: + for mutation_id in self.agent.available_actions: + try: + values = self._apply_action(mutation_id, ind) + candidates.append(values) + except Exception as e: + self._log.warning(f'Eval error for mutation <{mutation_id}> ' + f'on graph: {ind.graph.descriptive_id}:\n{e}') + continue + best_step = max(candidates, key=lambda step: step[-1]) + return best_step + + def _apply_action(self, action: Any, ind: Individual) -> TrajectoryStep: + new_graph, applied = self.mutation._adapt_and_apply_mutation(ind.graph, action) + fitness = self._eval_objective(new_graph) if applied else None + parent_op = ParentOperator(type_='mutation', operators=applied, parent_individuals=ind) + new_ind = Individual(new_graph, fitness=fitness, parent_operator=parent_op) + + prev_fitness = ind.fitness or self._eval_objective(ind.graph) + if prev_fitness and fitness: + reward = prev_fitness.value - fitness.value + elif prev_fitness and not fitness: + reward = -1. + else: + reward = 0. + return new_ind, action, reward + + def _eval_objective(self, graph: Graph) -> Fitness: + return self._adapter.adapt_func(self.objective)(graph) + + def _make_action_step(self, ind: Individual) -> TrajectoryStep: + action = self.agent.choose_action(ind.graph) + return self._apply_action(action, ind) + + def _sample_trajectory(self, initial: Individual, length: int) -> GraphTrajectory: + trajectory = [] + past_ind = initial + for i in range(length): + next_ind, action, reward = self._make_action_step(past_ind) + trajectory.append((next_ind, action, reward)) + past_ind = next_ind + return trajectory + + +def concat_lists(lists: Iterable[List]) -> List: + return reduce(operator.add, lists, []) diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/common_types.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/common_types.py new file mode 100644 index 0000000..528ebcc --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/common_types.py @@ -0,0 +1,11 @@ +from typing import Union, Hashable, Tuple, Sequence + +from golem.core.dag.graph import Graph +from golem.core.optimisers.opt_history_objects.individual import Individual + +ObsType = Union[Individual, Graph] +ActType = Hashable +# Trajectory step includes (past observation, action, reward) +TrajectoryStep = Tuple[Individual, ActType, float] +# Trajectory is a sequence of applied mutations and received rewards +GraphTrajectory = Sequence[TrajectoryStep] diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/context_agents.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/context_agents.py new file mode 100644 index 0000000..f8c9bca --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/context_agents.py @@ -0,0 +1,132 @@ +from enum import Enum + +from typing import List, Callable, Any + +import numpy as np + +from golem.utilities.requirements_notificator import warn_requirement +from golem.core.adapter.nx_adapter import BanditNetworkxAdapter +from golem.core.optimisers.opt_history_objects.individual import Individual + +try: + from karateclub import FeatherGraph +except ModuleNotFoundError: + warn_requirement('karateclub', 'other_requirements/adaptive.txt') + + +def adapter_func_to_networkx(func): + """ Decorator function to adapt observation to networkx graphs. """ + def wrapper(obs, available_operations): + nx_graph = BanditNetworkxAdapter().restore(obs) + embedding = func(nx_graph, available_operations) + return embedding + return wrapper + + +def adapter_func_to_graph(func): + """ Decorator function to adapt observation to networkx graphs. """ + def wrapper(obs, available_operations): + if isinstance(obs, Individual): + graph = obs.graph + else: + graph = obs + return func(graph, available_operations) + return wrapper + + +def encode_operations(operations: List[str], available_operations: List[str], mode: str = 'label'): + """ Encoding of operations. + :param operations: operations to encode + :param available_operations: list of all available operations + :param mode: mode of encoding. Available type: 'OHE' and 'label', default -- 'label' + """ + encoded = [] + for operation in operations: + if mode == 'label': + encoding = available_operations.index(operation) + else: + encoding = [0] * len(available_operations) + encoding[available_operations.index(operation)] = 1 + encoded.append(encoding) + return encoded + + +@adapter_func_to_networkx +def feather_graph(obs: Any, available_operations: List[str]) -> List[float]: + """ Returns embedding based on an implementation of `"FEATHER-G" `_. + The procedure uses characteristic functions of node features with random walk weights to describe + node neighborhoods. These node level features are pooled by mean pooling to + create graph level statistics. """ + descriptor = FeatherGraph() + descriptor.fit([obs]) + emb = descriptor.get_embedding().reshape(-1, 1) + embd = [i[0] for i in emb] + return embd + + +@adapter_func_to_graph +def nodes_num(obs: Any, available_operations: List[str]) -> List[int]: + """ Returns number of nodes in graph. """ + return [len(obs.nodes)] + + +@adapter_func_to_graph +def labeled_edges(obs: Any, available_operations: List[str]) -> List[int]: + """ Encodes graph with its edges with nodes labels. """ + operations = [] + for node in obs.nodes: + for node_ in node.nodes_from: + operations.append(node_.name) + operations.append(node.name) + return encode_operations(operations=operations, available_operations=available_operations) + + +@adapter_func_to_graph +def operations_quantity(obs: Any, available_operations: List[str]) -> List[int]: + """ Encodes graphs as vectors with quantity of each operation. """ + encoding = [0] * len(available_operations) + for node in obs.nodes: + encoding[available_operations.index(node.name)] += 1 + return encoding + + +@adapter_func_to_graph +def adjacency_matrix(obs: Any, available_operations: List[str]) -> List[int]: + """ Encodes graphs as flattened adjacency matrix. """ + matrix = np.zeros((len(available_operations), len(available_operations))) + for node in obs.nodes: + operation_parent_idx = available_operations.index(node.name) + for node_ in node.nodes_from: + operation_child_idx = available_operations.index(node_.name) + matrix[operation_parent_idx][operation_child_idx] += 1 + return matrix.reshape(1, -1)[0].astype(int).tolist() + + +def none_encoding(obs: Any, available_operations: List[str]) -> List[int]: + """ Empty encoding. """ + return obs + + +class ContextAgentTypeEnum(Enum): + feather_graph = 'feather_graph' + nodes_num = 'nodes_num' + labeled_edges = 'labeled_edges' + operations_quantity = 'operations_quantity' + adjacency_matrix = 'adjacency_matrix' + none_encoding = 'none_encoding' + + +class ContextAgentsRepository: + """ Repository of functions to encode observations. """ + _agents_implementations = { + ContextAgentTypeEnum.feather_graph: feather_graph, + ContextAgentTypeEnum.nodes_num: nodes_num, + ContextAgentTypeEnum.labeled_edges: labeled_edges, + ContextAgentTypeEnum.operations_quantity: operations_quantity, + ContextAgentTypeEnum.adjacency_matrix: adjacency_matrix, + ContextAgentTypeEnum.none_encoding: none_encoding + } + + @staticmethod + def agent_class_by_id(agent_id: ContextAgentTypeEnum) -> Callable: + return ContextAgentsRepository._agents_implementations[agent_id] diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/experience_buffer.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/experience_buffer.py new file mode 100644 index 0000000..6b4852d --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/experience_buffer.py @@ -0,0 +1,142 @@ +from collections import deque +from typing import List, Iterable, Tuple, Optional + +import numpy as np + +from golem.core.optimisers.adaptive.common_types import ObsType, ActType, TrajectoryStep, GraphTrajectory +from golem.core.optimisers.opt_history_objects.individual import Individual +from golem.core.optimisers.opt_history_objects.opt_history import OptHistory + + +class ExperienceBuffer: + """Buffer for learning experience of ``OperatorAgent``. + Keeps (State, Action, Reward) lists until retrieval.""" + + def __init__(self, window_size: Optional[int] = None, inds=None, actions=None, rewards=None): + self.window_size = window_size + self._prev_pop = set() + self._next_pop = set() + + if inds and not (len(inds) == len(actions) == len(rewards)): + raise ValueError('lengths of buffers do not match') + self._individuals = deque(inds) if inds else deque(maxlen=self.window_size) + self._actions = deque(actions) if actions else deque(maxlen=self.window_size) + self._rewards = deque(rewards) if rewards else deque(maxlen=self.window_size) + + @staticmethod + def from_history(history: OptHistory) -> 'ExperienceBuffer': + exp = ExperienceBuffer() + exp.collect_history(history) + return exp + + def _reset(self): + self._prev_pop = set() + self._next_pop = set() + + # if window size was not specified than there is no need to store these values for reuse. + # Otherwise, if the window_size is specified, then storages will be updated automatically in queue + if self.window_size is None: + self._individuals = deque(maxlen=self.window_size) + self._actions = deque(maxlen=self.window_size) + self._rewards = deque(maxlen=self.window_size) + + @staticmethod + def unroll_action_step(result: Individual) -> TrajectoryStep: + """Unrolls individual's history to get its source individual, action and resulting reward.""" + if not result.parent_operator or result.parent_operator.type_ != 'mutation': + return None, None, np.nan + source_ind = result.parent_operator.parent_individuals[0] + action = result.parent_operator.operators[0] + # we're minimising the fitness, that's why less is better + reward = (source_ind.fitness.value - result.fitness.value) / abs(source_ind.fitness.value)\ + if source_ind.fitness and source_ind.fitness.value != 0. else 0. + return source_ind, action, reward + + @staticmethod + def unroll_trajectories(history: OptHistory) -> List[GraphTrajectory]: + """Iterates through history and find continuous sequences of applied operator actions.""" + trajectories = [] + seen_uids = set() + for terminal_individual in history.final_choices: + trajectory = [] + next_ind = terminal_individual + while True: + seen_uids.add(next_ind.uid) + source_ind, action, reward = ExperienceBuffer.unroll_action_step(next_ind) + if source_ind is None or source_ind.uid in seen_uids: + break + # prepend step to keep historical direction + trajectory.insert(0, (source_ind, action, reward)) + next_ind = source_ind + trajectories.append(trajectory) + return trajectories + + def collect_history(self, history: OptHistory): + seen = set() + # We don't need the initial assumptions, as they have no parent operators, hence [1:] + for generation in history.generations[1:]: + for ind in generation: + if ind.uid not in seen: + seen.add(ind.uid) + self.collect_result(ind) + + def collect_results(self, results: Iterable[Individual]): + for ind in results: + self.collect_result(ind) + + def collect_result(self, result: Individual): + if result.uid in self._prev_pop: + # avoid collecting results from individuals that didn't change + return + self._next_pop.add(result.uid) + + source_ind, action, reward = self.unroll_action_step(result) + if action is None: + return + self.collect_experience(source_ind, action, reward) + + def collect_experience(self, obs: Individual, action: ActType, reward: float): + self._individuals.append(obs) + self._actions.append(action) + self._rewards.append(reward) + + def retrieve_experience(self, as_graphs: bool = True) -> Tuple[List[ObsType], List[ActType], List[float]]: + """Get all collected experience and clear the experience buffer. + Args: + as_graphs: if True (by default) returns observations as graphs, otherwise as individuals. + Return: + Unzipped trajectories (tuple of lists of observations, actions, rewards). + """ + individuals, actions, rewards = self._individuals, self._actions, self._rewards + observations = [ind.graph for ind in individuals] if as_graphs else individuals + next_pop = self._next_pop + self._reset() + self._prev_pop = next_pop + return list(observations), list(actions), list(rewards) + + def retrieve_trajectories(self) -> GraphTrajectory: + """Same as `retrieve_experience` but in the form of zipped trajectories that consist from steps.""" + trajectories = list(zip(*self.retrieve_experience(as_graphs=False))) + return trajectories + + def split(self, ratio: float = 0.8, shuffle: bool = False + ) -> Tuple['ExperienceBuffer', 'ExperienceBuffer']: + """Splits buffer in 2 parts, useful for train/validation split.""" + mask_train = np.full_like(self._individuals, False, dtype=bool) + num_train = int(len(self._individuals) * ratio) + mask_train[-num_train:] = True + if shuffle: + np.random.default_rng().shuffle(mask_train) + buffer_train = ExperienceBuffer(inds=np.array(self._individuals)[mask_train].tolist(), + actions=np.array(self._actions)[mask_train].tolist(), + rewards=np.array(self._rewards)[mask_train].tolist()) + buffer_val = ExperienceBuffer(inds=np.array(self._individuals)[~mask_train].tolist(), + actions=np.array(self._actions)[~mask_train].tolist(), + rewards=np.array(self._rewards)[~mask_train].tolist()) + return buffer_train, buffer_val + + def __len__(self): + return len(self._individuals) + + def __str__(self): + return f'{self.__class__.__name__}({len(self)})' diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/history_collector.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/history_collector.py new file mode 100644 index 0000000..b33c219 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/history_collector.py @@ -0,0 +1,42 @@ +import os +from pathlib import Path +from typing import Optional, Iterable + +from golem.core.log import default_log +from golem.core.optimisers.opt_history_objects.opt_history import OptHistory + + +class HistoryReader: + """Simplifies reading a bunch of histories from single directory.""" + + def __init__(self, save_path: Optional[Path] = None): + self.log = default_log(self) + self.save_path = save_path or Path("results") + self.save_path.mkdir(parents=True, exist_ok=True) + + def load_histories(self) -> Iterable[OptHistory]: + """Iteratively loads saved histories one-by-ony.""" + num_histories = 0 + total_individuals = 0 + for history_path in HistoryReader.traverse_histories(self.save_path): + history = OptHistory.load(history_path) + num_histories += 1 + total_individuals += sum(map(len, history.generations)) + yield history + + if num_histories == 0 or total_individuals == 0: + raise ValueError(f'Could not load any individuals.' + f'Possibly, path {self.save_path} does not exist or is empty.') + else: + self.log.info(f'Loaded {num_histories} histories ' + f'with {total_individuals} individuals in total.') + + @staticmethod + def traverse_histories(path) -> Iterable[Path]: + if path.exists(): + # recursive traversal of the save directory + for root, dirs, files in os.walk(path): + for history_filename in files: + if history_filename.startswith('history'): + full_path = Path(root) / history_filename + yield full_path diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/mab_agents/__init__.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/mab_agents/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/mab_agents/contextual_mab_agent.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/mab_agents/contextual_mab_agent.py new file mode 100644 index 0000000..b476b1f --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/mab_agents/contextual_mab_agent.py @@ -0,0 +1,106 @@ +import random +from functools import partial +from typing import Union, Sequence, Optional, List, Callable + +import numpy as np +from mabwiser.mab import MAB, LearningPolicy, NeighborhoodPolicy +from scipy.special import softmax + +from golem.core.dag.graph import Graph +from golem.core.dag.graph_node import GraphNode +from golem.core.optimisers.adaptive.context_agents import ContextAgentsRepository, ContextAgentTypeEnum +from golem.core.optimisers.adaptive.mab_agents.mab_agent import MultiArmedBanditAgent +from golem.core.optimisers.adaptive.operator_agent import ActType, ObsType, ExperienceBuffer + + +class ContextualMultiArmedBanditAgent(MultiArmedBanditAgent): + """ Contextual Multi-Armed bandit. Observations can be encoded with simple context agent without + using NN to guarantee convergence. + + :param actions: types of mutations + :param context_agent_type: function to convert observation to its embedding. Can be specified as + ContextAgentTypeEnum or as Callable function. + :param available_operations: available operations + :param n_jobs: n_jobs + :param enable_logging: bool logging flag + """ + + def __init__(self, actions: Sequence[ActType], + context_agent_type: Union[ContextAgentTypeEnum, Callable], + available_operations: List[str], + n_jobs: int = 1, + enable_logging: bool = True, + decaying_factor: float = 1.0): + super().__init__(actions=actions, n_jobs=n_jobs, enable_logging=enable_logging, + decaying_factor=decaying_factor, is_initial_fit=False) + self._agent = MAB(arms=self._indices, + learning_policy=LearningPolicy.UCB1(alpha=1.25), + neighborhood_policy=NeighborhoodPolicy.Clusters(), + n_jobs=n_jobs) + self._context_agent = context_agent_type if isinstance(context_agent_type, Callable) else \ + partial(ContextAgentsRepository.agent_class_by_id(context_agent_type), + available_operations=available_operations) + self._is_fitted = False + + def _initial_fit(self, obs: ObsType): + """ Initial fit for Contextual Multi-Armed Bandit. + At this step, all hands are assigned the same weights with the very first context + that is fed to the bandit. """ + # initial fit for mab + n = len(self._indices) + uniform_rewards = [1. / n] * n + contexts = self.get_context(obs=obs) + self._agent.fit(decisions=self._indices, rewards=uniform_rewards, contexts=np.tile(contexts, (n, 1))) + self._is_fitted = True + + def choose_action(self, obs: ObsType) -> ActType: + if not self._is_fitted: + self._initial_fit(obs=obs) + contexts = self.get_context(obs=obs) + arm = self._agent.predict(contexts=contexts.reshape(1, -1)) + action = self.actions[arm] + return action + + def get_action_values(self, obs: Optional[ObsType] = None) -> Sequence[float]: + if not self._is_fitted: + self._initial_fit(obs=obs) + contexts = self.get_context(obs) + prob_dict = self._agent.predict_expectations(contexts=contexts.reshape(1, -1)) + prob_list = [prob_dict[i] for i in range(len(prob_dict))] + return prob_list + + def get_action_probs(self, obs: Optional[ObsType] = None) -> Sequence[float]: + return softmax(self.get_action_values(obs=obs)) + + def choose_nodes(self, graph: Graph, num_nodes: int = 1) -> Union[GraphNode, Sequence[GraphNode]]: + subject_nodes = random.sample(graph.nodes, k=num_nodes) + return subject_nodes[0] if num_nodes == 1 else subject_nodes + + def partial_fit(self, experience: ExperienceBuffer): + """Continues learning of underlying agent with new experience.""" + obs, arms, processed_rewards = self._get_experience(experience) + contexts = self.get_context(obs=obs) + self._agent.partial_fit(decisions=arms, rewards=processed_rewards, contexts=contexts) + + def _get_experience(self, experience: ExperienceBuffer): + """ Get experience from ExperienceBuffer, process rewards and log. """ + obs, actions, rewards = experience.retrieve_experience() + arms = [self._arm_by_action[action.__name__] for action in actions] + # there is no need to process rewards as in MAB, since this processing unifies rewards for all contexts + self._dbg_log(obs, actions, rewards) + return obs, arms, rewards + + def get_context(self, obs: Union[List[ObsType], ObsType]) -> np.array: + """ Returns contexts based on specified context agent. """ + if not isinstance(obs, list): + obs = [obs] + contexts = [] + for ob in obs: + if isinstance(ob, list) or isinstance(ob, np.ndarray): + # to unify type to list + contexts.append(np.array(ob).flatten()) + else: + context = np.array(self._context_agent(ob)) + # some external context agents can wrap context in an additional array + contexts.append(context.flatten()) + return np.array(contexts) diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/mab_agents/mab_agent.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/mab_agents/mab_agent.py new file mode 100644 index 0000000..59e3279 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/mab_agents/mab_agent.py @@ -0,0 +1,116 @@ +import os.path +import _pickle as pickle +import random +import re +from functools import partial +from typing import Union, Sequence, Optional, Callable + +from mabwiser.mab import MAB, LearningPolicy +from scipy.special import softmax + +from golem.core.dag.graph import Graph +from golem.core.dag.graph_node import GraphNode +from golem.core.optimisers.adaptive.operator_agent import OperatorAgent, ActType, ObsType, ExperienceBuffer +from golem.core.optimisers.adaptive.reward_agent import FitnessRateRankRewardTransformer +from golem.core.paths import default_data_dir + + +class MultiArmedBanditAgent(OperatorAgent): + def __init__(self, + actions: Sequence[ActType], + n_jobs: int = 1, + enable_logging: bool = True, + decaying_factor: float = 1.0, + path_to_save: Optional[str] = None, + is_initial_fit: bool = True): + super().__init__(actions=actions, enable_logging=enable_logging) + self.actions = list(actions) + self._indices = list(range(len(actions))) + # str because parent operator for mutation is stored as string for custom mutations serialisation + self._arm_by_action = dict(map(lambda x, y: (self._get_callable_name(x), y), actions, self._indices)) + self._agent = MAB(arms=self._indices, + learning_policy=LearningPolicy.EpsilonGreedy(epsilon=0.4), + n_jobs=n_jobs) + self._reward_agent = FitnessRateRankRewardTransformer(decaying_factor=decaying_factor) + if is_initial_fit: + self._initial_fit() + self._path_to_save = path_to_save + + @staticmethod + def _get_callable_name(action: Callable): + if isinstance(action, partial): + return action.func.__name__ + else: + try: + return action.__name__ + except AttributeError: + return str(action) + + def _initial_fit(self): + n = len(self.actions) + uniform_rewards = [1. / n] * n + self._agent.fit(decisions=self._indices, rewards=uniform_rewards) + + def choose_action(self, obs: ObsType) -> ActType: + arm = self._agent.predict() + action = self.actions[arm] + return action + + def get_action_values(self, obs: Optional[ObsType] = None) -> Sequence[float]: + prob_dict = self._agent.predict_expectations() + prob_list = [prob_dict[i] for i in range(len(prob_dict))] + return prob_list + + def get_action_probs(self, obs: Optional[ObsType] = None) -> Sequence[float]: + return softmax(self.get_action_values()) + + def choose_nodes(self, graph: Graph, num_nodes: int = 1) -> Union[GraphNode, Sequence[GraphNode]]: + subject_nodes = random.sample(graph.nodes, k=num_nodes) + return subject_nodes[0] if num_nodes == 1 else subject_nodes + + def partial_fit(self, experience: ExperienceBuffer): + """Continues learning of underlying agent with new experience.""" + _, arms, processed_rewards = self._get_experience(experience) + self._agent.partial_fit(decisions=arms, rewards=processed_rewards) + + def _get_experience(self, experience: ExperienceBuffer): + """ Get experience from ExperienceBuffer, process rewards and log. """ + obs, actions, rewards = experience.retrieve_experience() + arms = [self._arm_by_action[action] for action in actions] + processed_rewards = self._reward_agent.get_rewards_for_arms(rewards, arms) + self._dbg_log(obs, actions, processed_rewards) + return obs, arms, processed_rewards + + def save(self, path_to_save: Optional[str] = None): + """ Saves bandit to specified file. """ + + if not path_to_save: + path_to_save = self._path_to_save + + # if path was not specified at all + if not path_to_save: + path_to_save = os.path.join(default_data_dir(), 'MAB') + + if not path_to_save.endswith('.pkl'): + os.makedirs(path_to_save, exist_ok=True) + mabs_num = [int(name.split('_')[0]) for name in os.listdir(path_to_save) + if re.fullmatch(r'\d_mab.pkl', name)] + if not mabs_num: + max_saved_mab = 0 + else: + max_saved_mab = max(mabs_num) + 1 + path_to_file = os.path.join(path_to_save, f'{max_saved_mab}_mab.pkl') + else: + path_to_dir = os.path.dirname(path_to_save) + os.makedirs(path_to_dir, exist_ok=True) + path_to_file = path_to_save + with open(path_to_file, 'wb') as f: + pickle.dump(self, f) + self._log.info(f"MAB was saved to {path_to_file}") + + @staticmethod + def load(path: str): + """ Loads bandit from the specified file. """ + with open(path, 'rb') as f: + mab = pickle.load(f) + return mab diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/mab_agents/neural_contextual_mab_agent.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/mab_agents/neural_contextual_mab_agent.py new file mode 100644 index 0000000..1265ca6 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/mab_agents/neural_contextual_mab_agent.py @@ -0,0 +1,24 @@ +from typing import Sequence, List + +from golem.core.optimisers.adaptive.mab_agents.contextual_mab_agent import ContextualMultiArmedBanditAgent +from golem.core.optimisers.adaptive.neural_mab import NeuralMAB +from golem.core.optimisers.adaptive.context_agents import ContextAgentTypeEnum +from golem.core.optimisers.adaptive.common_types import ActType + + +class NeuralContextualMultiArmedBanditAgent(ContextualMultiArmedBanditAgent): + """ Neural Contextual Multi-Armed bandit. + Observations can be encoded with the use of Neural Networks, but still there are some restrictions + to guarantee convergence. """ + def __init__(self, + actions: Sequence[ActType], + context_agent_type: ContextAgentTypeEnum, + available_operations: List[str], + n_jobs: int = 1, + enable_logging: bool = True, + decaying_factor: float = 1.0): + super().__init__(actions=actions, n_jobs=n_jobs, + context_agent_type=context_agent_type, available_operations=available_operations, + enable_logging=enable_logging, decaying_factor=decaying_factor) + self._agent = NeuralMAB(arms=self._indices, + n_jobs=n_jobs) diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/neural_mab.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/neural_mab.py new file mode 100644 index 0000000..bc27a73 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/neural_mab.py @@ -0,0 +1,315 @@ +import copy +import math +from typing import List, Any, Union, Dict + +from golem.utilities.requirements_notificator import warn_requirement + +try: + import torch +except ModuleNotFoundError: + warn_requirement('torch', 'other_requirements/adaptive.txt') + +import numpy as np +from mabwiser.mab import MAB, LearningPolicy, NeighborhoodPolicy +from mabwiser.utils import Arm, Constants, Num + +from golem.core.log import default_log + +import warnings + +warnings.filterwarnings("ignore") + + +class NeuralMAB(MAB): + """ + Neural Multi-Armed Bandit. + The main concept is explained in the article: https://arxiv.org/abs/2012.01780. + Deep representation is formed with NN and Contextual Multi-Armed Bandit is integrated to choose arm. + + NB! Neural MABs can be used with 1.8.0 version of torch since some methods are deprecated in later versions, + however, python of version 3.10 is not supported in this version of torch. + """ + + def __init__(self, arms: List[Arm], + learning_policy: Any = LearningPolicy.UCB1(alpha=1.25), + neighborhood_policy: Any = NeighborhoodPolicy.Clusters(), + seed: int = Constants.default_seed, + n_jobs: int = 1): + + super().__init__(arms, learning_policy, neighborhood_policy, seed, n_jobs) + self.nn_with_se = NNWithShallowExploration(context_size=1, arms_count=len(arms)) + self.arms = arms + self.seed = seed + self.n_jobs = n_jobs + self.log = default_log('NeuralMAB') + # to track when GNN needs to be updated + self.iter = 0 + self._indices = list(range(len(arms))) + self._mab = MAB(arms=self._indices, + learning_policy=learning_policy, + neighborhood_policy=neighborhood_policy, + n_jobs=n_jobs) + self.is_fitted = False + + def _initial_fit_mab(self, context: Any): + """ Initial fit for Contextual Multi-Armed Bandit. + At this step, all hands are assigned the same weights with the very first context + that is fed to the bandit. """ + # initial fit for mab + n = len(self.arms) + uniform_rewards = [1. / n] * n + deep_context = self._get_deep_context(context=context) + self._mab.fit(decisions=self._indices, rewards=uniform_rewards, contexts=n * [deep_context]) + self.is_fitted = True + + def partial_fit(self, decisions: List[Any], rewards: List[float], contexts: List[Any] = None): + + # get deep contexts, calculate regret and update weights for NN (once in _H_q iters) + deep_contexts = self.nn_with_se.partial_fit(iter=self.iter, decisions=decisions, + rewards=rewards, contexts=contexts) + self.iter += 1 + + # update contextual mab with deep contexts + self._mab.partial_fit(decisions=decisions, contexts=deep_contexts, rewards=rewards) + + def predict(self, contexts: Any = None) -> Union[Arm, List[Arm]]: + """ Predicts which arm to pull to get maximum reward. """ + if not self.is_fitted: + self._initial_fit_mab(context=contexts) + deep_context = self._get_deep_context(context=contexts) + a_choose = self._mab.predict(contexts=[deep_context]) + return a_choose + + def predict_expectations(self, contexts: Any = None) -> Union[Dict[Arm, Num], List[Dict[Arm, Num]]]: + """ Returns expected reward for each arm. """ + if not self.is_fitted: + self._initial_fit_mab(context=contexts) + deep_context = self._get_deep_context(context=contexts) + return self._mab.predict_expectations(contexts=[deep_context]) + + def _get_deep_context(self, context: Any) -> List[Any]: + """ Returns deep representation of context. """ + temp = self.nn_with_se.transfer(context, 0, len(self.arms)) + feat = self.nn_with_se.feature_extractor(temp, self.nn_with_se.W).numpy().squeeze() + return list(feat) + + +class NNWithShallowExploration: + """ Neural Network with shallow exploration which means that weights are updated every _H_q iterations. """ + + def __init__(self, context_size: int, arms_count: int): + """ + Initial fit for NN. + beta -- parameter for UCB exploration + H_q -- how many time steps to update NN + interT -- internal steps for GD + """ + self._beta = 0.02 + self._lambd = 1 + self._lr = 0.001 + self._H_q = 5 + self._interT = 1000 + self._hidden_dim = [1000, 1000] + hid_dim_lst = self._hidden_dim + dim_second_last = self._hidden_dim[-1] * 2 + + dim_for_init = [context_size + arms_count] + hid_dim_lst + [1] + self.arms_count = arms_count + self.W0, total_dim = self._initialization(dim_for_init) + self.LAMBDA = self._lambd * torch.eye(dim_second_last, dtype=torch.double) + self.bb = torch.zeros(self.LAMBDA.size()[0], dtype=torch.double).view(-1, 1) + + theta = np.random.randn(dim_second_last, 1) / np.sqrt(dim_second_last) + self.theta = torch.from_numpy(theta) + + self.THETA_action = torch.tensor([]) + self.CONTEXT_action = torch.tensor([]) + self.REWARD_action = torch.tensor([]) + self.result_neuralucb = [] + self.W = copy.deepcopy(self.W0) + self.summ = 0 + self.log = default_log('NNWithShallowExploration') + + def partial_fit(self, iter: int, + decisions: List[Any], rewards: List[float], contexts: List[Any] = None): + deep_contexts = [] + + # update NN and calculate reward + for decision, context, reward in zip(decisions, contexts, rewards): + + # calculate reward + temp = self.transfer(context, decision, self.arms_count) + feat = self.feature_extractor(temp, self.W) + deep_contexts.append(list(feat.numpy().squeeze())) + expected_reward = torch.mm(self.theta.view(1, -1), feat) + self._beta * self.UCB(self.LAMBDA, feat) + + self.summ += (expected_reward - reward) + self.result_neuralucb.append(self.summ) + + # gather dataset for next NN training (context_action and reward_action) + if np.mod(iter, self._H_q) == 0: + context_action = temp + reward_action = torch.tensor([reward], dtype=torch.double) + else: + context_action = torch.cat((self.CONTEXT_action, temp), 1) + reward_action = torch.cat((self.REWARD_action, torch.tensor([reward], dtype=torch.double)), 0) + + # update LAMBDA and bb + self.LAMBDA += torch.mm(self.feature_extractor(temp, self.W), + self.feature_extractor(temp, self.W).t()) + self.bb += reward * self.feature_extractor(temp, self.W) + theta, _ = torch.solve(self.bb, self.LAMBDA) + + if np.mod(iter, self._H_q) == 0: + theta_action = theta.view(-1, 1) + else: + theta_action = torch.cat((self.THETA_action, theta.view(-1, 1)), 1) + + # update weight of NN + if np.mod(iter, self._H_q) == self._H_q - 1: + self.log.info(f'Current regret: {self.summ}') + self.W = self.train_with_shallow_exploration(context_action, reward_action, self.W0, + self._interT, self._lr, theta_action, self._H_q) + return deep_contexts + + @staticmethod + def UCB(A, phi): + """ Ucb term. """ + try: + tmp, _ = torch.solve(phi, A) + except Exception: + tmp = torch.Tensor(np.linalg.solve(A, phi)) + + return torch.sqrt(torch.mm(torch.transpose(phi, 0, 1).double(), tmp.double())) + + @staticmethod + def transfer(c, a, arm_size): + """ + Transfer an array context + action to new context with dimension 2*(__context__ + __armsize__). + """ + action = np.zeros(arm_size) + action[a] = 1 + c_final = np.append(c, action) + c_final = torch.from_numpy(c_final) + c_final = c_final.view((len(c_final), 1)) + c_final = c_final.repeat(2, 1) + return c_final + + def train_with_shallow_exploration(self, X, Y, W_start, T, et, THETA, H): + """ Gd-based model training with shallow exploration + Dataset X, label Y. """ + W = copy.deepcopy(W_start) + X = X[:, -H:] + Y = Y[-H:] + THETA = THETA[:, -H:] + + prev_loss = 1000000 + prev_loss_1k = 1000000 + for i in range(0, T): + grad = self._gradient_loss(X, Y, W, THETA) + for j in range(0, len(W) - 1): + W[j] = W[j] - et * grad[j] + + curr_loss = self._loss(X, Y, W, THETA) + if i % 100 == 0: + print('------', curr_loss) + if curr_loss > prev_loss_1k: + et = et * 0.1 + print('lr/10 to', et) + + prev_loss_1k = curr_loss + + # early stopping + if abs(curr_loss - prev_loss) < 1e-6: + break + prev_loss = curr_loss + return W + + @staticmethod + def _initialization(dim): + """ Initialization. + dim consists of (d1, d2,...), where dl = 1 (placeholder, deprecated). """ + w = [] + total_dim = 0 + for i in range(0, len(dim) - 1): + if i < len(dim) - 2: + temp = np.random.randn(dim[i + 1], dim[i]) / np.sqrt(dim[i + 1]) + temp = np.kron(np.eye(2, dtype=int), temp) + temp = torch.from_numpy(temp) + w.append(temp) + total_dim += dim[i + 1] * dim[i] * 4 + else: + temp = np.random.randn(dim[i + 1], dim[i]) / np.sqrt(dim[i]) + temp = np.kron([[1, -1]], temp) + temp = torch.from_numpy(temp) + w.append(temp) + total_dim += dim[i + 1] * dim[i] * 2 + + return w, total_dim + + @staticmethod + def feature_extractor(x, W): + """ Functions feature extractor. + x is the input, dimension is d; W is the list of parameter matrices. """ + depth = len(W) + output = x + for i in range(0, depth - 1): + output = torch.mm(W[i], output) + output = output.clamp(min=0) + + output = output * math.sqrt(W[depth - 1].size()[1]) + return output + + def _gradient_loss(self, X, Y, W, THETA): + """ Return a list of grad, satisfying that W[i] = W[i] - grad[i] for single context x. """ + depth = len(W) + num_sample = Y.shape[0] + loss = [] + grad = [] + relu = [] + output = X + loss.append(output) + for i in range(0, depth - 1): + output = torch.mm(W[i], output) + relu.append(output) + output = output.clamp(min=0) + loss.append(output) + + THETA_t = torch.transpose(THETA, 0, 1).view(num_sample, 1, -1) + output_t = torch.transpose(output, 0, 1).view(num_sample, -1, 1) + output = torch.bmm(THETA_t, output_t).squeeze().view(1, -1) + + loss.append(output) + + feat = self.feature_extractor(X, W) + feat_t = torch.transpose(feat, 0, 1).view(num_sample, -1, 1) + output_t = torch.bmm(THETA_t, feat_t).squeeze().view(1, -1) + + # backward gradient propagation + back = output_t - Y + back = back.double() + grad_t = torch.mm(back, loss[depth - 1].t()) + grad.append(grad_t) + + for i in range(1, depth): + back = torch.mm(W[depth - i].t(), back) + back[relu[depth - i - 1] < 0] = 0 + grad_t = torch.mm(back, loss[depth - i - 1].t()) + grad.append(grad_t) + + grad1 = [] + for i in range(0, depth): + grad1.append(grad[depth - 1 - i] * math.sqrt(W[depth - 1].size()[1]) / len(X[0, :])) + + return grad1 + + def _loss(self, X, Y, W, THETA): + # total loss + num_sample = len(X[0, :]) + output = self.feature_extractor(X, W) + THETA_t = torch.transpose(THETA, 0, 1).view(num_sample, 1, -1) + output_t = torch.transpose(output, 0, 1).view(num_sample, -1, 1) + output_y = torch.bmm(THETA_t, output_t).squeeze().view(1, -1) + + summ = (Y - output_y).pow(2).sum() / num_sample + return summ diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/operator_agent.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/operator_agent.py new file mode 100644 index 0000000..7b10345 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/operator_agent.py @@ -0,0 +1,99 @@ +import random +from abc import ABC, abstractmethod +from enum import Enum +from typing import Union, Sequence, Optional + +import numpy as np + +from golem.core.dag.graph import Graph +from golem.core.dag.graph_node import GraphNode +from golem.core.log import default_log +from golem.core.optimisers.adaptive.common_types import ObsType, ActType +from golem.core.optimisers.adaptive.experience_buffer import ExperienceBuffer + + +class MutationAgentTypeEnum(Enum): + default = 'default' + random = 'random' + bandit = 'bandit' + contextual_bandit = 'contextual_bandit' + neural_bandit = 'neural_bandit' + + +class OperatorAgent(ABC): + def __init__(self, actions: Sequence[ActType], enable_logging: bool = True): + self.actions = list(actions) + self._enable_logging = enable_logging + self._log = default_log(self) + + @property + def available_actions(self) -> Sequence[ActType]: + return self.actions + + @abstractmethod + def partial_fit(self, experience: ExperienceBuffer): + raise NotImplementedError() + + @abstractmethod + def choose_action(self, obs: Optional[ObsType]) -> ActType: + raise NotImplementedError() + + @abstractmethod + def choose_nodes(self, graph: ObsType, num_nodes: int = 1) -> Union[GraphNode, Sequence[GraphNode]]: + raise NotImplementedError() + + @abstractmethod + def get_action_probs(self, obs: Optional[ObsType]) -> Sequence[float]: + raise NotImplementedError() + + @abstractmethod + def get_action_values(self, obs: Optional[ObsType]) -> Sequence[float]: + raise NotImplementedError() + + def _dbg_log(self, obs, actions, rewards): + if self._enable_logging: + prec = 4 + rr = np.array(rewards).round(prec) + nonzero = rr[rr.nonzero()] + msg = f'len={len(rr)} nonzero={len(nonzero)} ' + if len(nonzero) > 0: + msg += (f'avg={nonzero.mean():.3f} std={nonzero.std():.3f} ' + f'min={nonzero.min():.3f} max={nonzero.max():.3f} ') + + self._log.info(msg) + self._log.info(f'actions/rewards: {list(zip(actions, rr))}') + + action_values = list(map(self.get_action_values, obs)) + action_probs = list(map(self.get_action_probs, obs)) + action_values = np.round(np.mean(action_values, axis=0), prec) + action_probs = np.round(np.mean(action_probs, axis=0), prec) + + self._log.info(f'exp={action_values} ' + f'probs={action_probs}') + + +class RandomAgent(OperatorAgent): + def __init__(self, + actions: Sequence[ActType], + probs: Optional[Sequence[float]] = None, + enable_logging: bool = True): + super().__init__(actions, enable_logging) + self._probs = probs or [1. / len(actions)] * len(actions) + + def choose_action(self, obs: Graph) -> ActType: + action = np.random.choice(self.actions, p=self.get_action_probs(obs)) + return action + + def choose_nodes(self, graph: Graph, num_nodes: int = 1) -> Union[GraphNode, Sequence[GraphNode]]: + subject_nodes = random.sample(graph.nodes, k=num_nodes) + return subject_nodes[0] if num_nodes == 1 else subject_nodes + + def partial_fit(self, experience: ExperienceBuffer): + obs, actions, rewards = experience.retrieve_experience() + self._dbg_log(obs, actions, rewards) + + def get_action_probs(self, obs: Optional[Graph] = None) -> Sequence[float]: + return self._probs + + def get_action_values(self, obs: Optional[Graph] = None) -> Sequence[float]: + return self._probs diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/reward_agent.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/reward_agent.py new file mode 100644 index 0000000..08f17dc --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/adaptive/reward_agent.py @@ -0,0 +1,31 @@ +from typing import List, Tuple + + +class FitnessRateRankRewardTransformer: + """ + Agent to process raw fitness values. + The original idea is that the use of raw fitness values as rewards affects algorithm's robustness, + therefore fitness values must be processed. + The article with explanation -- https://ieeexplore.ieee.org/document/6410018 + """ + def __init__(self, decaying_factor: float = 1.): + self._decaying_factor = decaying_factor + + def get_rewards_for_arms(self, rewards: List[float], arms: List[int]) -> List[float]: + unique_arms, decay_values = self.get_decay_values_for_arms(rewards, arms) + frr_per_arm = self.get_fitness_rank_rate(decay_values) + frr_values = [frr_per_arm[unique_arms.index(arm)] for arm in arms] + return frr_values + + def get_decay_values_for_arms(self, rewards: List[float], arms: List[int]) -> Tuple[List[int], List[float]]: + decays = dict.fromkeys(set(arms), 0.0) + for i, reward in enumerate(rewards): + decays[arms[i]] += reward + decays.update((key, value * self._decaying_factor) for key, value in decays.items()) + return list(decays.keys()), list(decays.values()) + + @staticmethod + def get_fitness_rank_rate(decay_values: List[float]) -> List[float]: + # abs() is used to save the initial sign of each decay value + total_decay_sum = abs(sum(decay_values)) + return [decay / total_decay_sum for decay in decay_values] if total_decay_sum != 0 else [0.]*len(decay_values) diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/advisor.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/advisor.py new file mode 100644 index 0000000..f53d160 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/advisor.py @@ -0,0 +1,32 @@ +from typing import List, Any, TypeVar, Generic + +from golem.utilities.data_structures import ComparableEnum as Enum + +NodeType = TypeVar('NodeType') + + +class RemoveType(Enum): + """Defines allowed kinds of removals in Graph. Used by mutations.""" + forbidden = 'forbidden' + node_only = 'node_only' + node_rewire = 'node_rewire' + with_direct_children = 'with_direct_children' + with_parents = 'with_parents' + + +class DefaultChangeAdvisor(Generic[NodeType]): + """ + Class for advising of graph changes during evolution + """ + + def __init__(self, task=None): + self.task = task + + def propose_change(self, node: NodeType, possible_operations: List[Any]) -> List[Any]: + return possible_operations + + def can_be_removed(self, node: NodeType) -> RemoveType: + return RemoveType.node_rewire + + def propose_parent(self, node: NodeType, possible_operations: List[Any]) -> List[Any]: + return possible_operations diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/archive/__init__.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/archive/__init__.py new file mode 100644 index 0000000..8928b8a --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/archive/__init__.py @@ -0,0 +1,2 @@ +from .generation_keeper import GenerationKeeper +from .individuals_containers import HallOfFame, ParetoFront diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/archive/generation_keeper.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/archive/generation_keeper.py new file mode 100644 index 0000000..7095e68 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/archive/generation_keeper.py @@ -0,0 +1,168 @@ +import datetime +from abc import ABC, abstractmethod +from typing import Dict, Iterable, Sequence, Optional, Any, Callable + +import numpy as np + +from golem.core.optimisers.fitness import is_metric_worse +from golem.core.optimisers.genetic.operators.operator import PopulationT +from golem.core.optimisers.objective.objective import Objective +from golem.core.optimisers.opt_history_objects.individual import Individual +from .individuals_containers import HallOfFame, ParetoFront + +PARETO_MAX_POP_SIZE_MULTIPLIER = 5 + + +class ImprovementWatcher(ABC): + """Interface that allows to check if optimization progresses or stagnates.""" + + @property + def stagnation_iter_count(self) -> int: + """Returns number of generations for which any metrics has not improved.""" + raise NotImplementedError() + + @property + def stagnation_time_duration(self) -> float: + """Returns time duration for which any metrics has not improved.""" + raise NotImplementedError() + + @property + @abstractmethod + def is_any_improved(self) -> bool: + """Check if any of the metrics has improved.""" + raise NotImplementedError() + + @abstractmethod + def is_metric_improved(self, metric_id) -> bool: + """Check if specified metric has improved.""" + raise NotImplementedError() + + @property + @abstractmethod + def is_quality_improved(self) -> bool: + """Check if any of the quality metrics has improved.""" + raise NotImplementedError() + + @property + @abstractmethod + def is_complexity_improved(self) -> bool: + """Check if any of the complexity metrics has improved.""" + raise NotImplementedError() + + +def _individuals_same(ind1: Individual, ind2: Individual) -> bool: + return (ind1.fitness == ind2.fitness and + ind1.native_generation == ind2.native_generation and + ind1.graph == ind2.graph) + + +class GenerationKeeper(ImprovementWatcher): + """Generation keeper that primarily tracks number of generations and stagnation duration. + + Args: + objective: Objective that specifies metrics and if it's multi objective optimization. + keep_n_best: How many best individuals to keep from all generations. + NB: relevant only for single-objective optimization. + initial_generation: Optional first generation; + NB: if None then keeper is created in inconsistent state and requires an initial .append(). + similarity_criteria: a function that in the case of multi-objective optimization + tells the Pareto front whether two individuals are similar, optional. + """ + + def __init__(self, + objective: Optional[Objective] = None, + keep_n_best: int = 1, + initial_generation: PopulationT = None, + similarity_criteria: Callable = _individuals_same): + self._generation_num = 0 # 0 means state before initial generation is added + self._stagnation_counter = 0 # Initialized in non-stagnated state + self._stagnation_start_time = datetime.datetime.now() + + self._objective = objective + self._metrics_improvement: Dict[Any, bool] = {} + self._reset_metrics_improvement() + + if objective.is_multi_objective: + self.archive = ParetoFront(maxsize=keep_n_best * PARETO_MAX_POP_SIZE_MULTIPLIER, + similar=similarity_criteria) + else: + self.archive = HallOfFame(maxsize=keep_n_best) + + if initial_generation is not None: + self.append(initial_generation) + + @property + def stagnation_start_time(self): + return self._stagnation_start_time + + @property + def best_individuals(self) -> Sequence[Individual]: + return self.archive.items + + @property + def generation_num(self) -> int: + return self._generation_num + + @property + def stagnation_iter_count(self) -> int: + return self._stagnation_counter + + @property + def stagnation_time_duration(self) -> float: + return (datetime.datetime.now() - self._stagnation_start_time).seconds / 60 + + @property + def is_any_improved(self) -> bool: + return any(self._metrics_improvement.values()) + + def is_metric_improved(self, metric_id) -> bool: + return self._metrics_improvement[metric_id] + + @property + def is_quality_improved(self) -> bool: + return any(self._metrics_improvement[metric_id] + for metric_id in self._objective.quality_metrics) + + @property + def is_complexity_improved(self) -> bool: + return any(self._metrics_improvement[metric_id] + for metric_id in self._objective.complexity_metrics) + + @property + def _metric_ids(self) -> Iterable[Any]: + return self._objective.metric_names + + def append(self, population: PopulationT): + previous_archive_fitness = self._archive_fitness() + self.archive.update(population) + self._update_improvements(previous_archive_fitness) + + def _archive_fitness(self) -> Dict[Any, Sequence[float]]: + archive_pop_metrics = (ind.fitness.values for ind in self.archive.items) + archive_fitness_per_metric = zip(*archive_pop_metrics) # transpose nested array + archive_fitness_per_metric = dict(zip(self._metric_ids, archive_fitness_per_metric)) + return archive_fitness_per_metric + + def _update_improvements(self, previous_metric_archive): + self._reset_metrics_improvement() + current_metric_archive = self._archive_fitness() + for metric in self._metric_ids: + # NB: Assuming we perform minimisation, so worst==max + previous_worst = np.max(previous_metric_archive.get(metric, np.inf)) + current_worst = np.max(current_metric_archive.get(metric, np.inf)) + # archive metric has improved if metric of its worst individual has improved + if is_metric_worse(previous_worst, current_worst): + self._metrics_improvement[metric] = True + + self._generation_num += 1 # becomes 1 on first population + self._stagnation_start_time = datetime.datetime.now() \ + if self.is_any_improved or self._generation_num == 1 else self._stagnation_start_time + self._stagnation_counter = 0 if self.is_any_improved else self._stagnation_counter + 1 + + def _reset_metrics_improvement(self): + self._metrics_improvement = {metric_id: False for metric_id in self._metric_ids} + + def __str__(self) -> str: + ff = self._objective.format_fitness + fitnesses = [ff(ind.fitness) for ind in self.best_individuals] + return f'{self.archive.__class__.__name__} archive fitness ({len(fitnesses)}): {fitnesses}' diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/archive/individuals_containers.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/archive/individuals_containers.py new file mode 100644 index 0000000..3a5b418 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/archive/individuals_containers.py @@ -0,0 +1,171 @@ +# This code is modified part of DEAP library (Library URL: https://github.com/DEAP/deap). +from bisect import bisect_right +from operator import eq +from typing import Callable, Optional + +from golem.core.optimisers.genetic.operators.operator import PopulationT +from golem.core.optimisers.opt_history_objects.individual import Individual + + +class HallOfFame: + """ + The hall of fame contains the best individual that ever lived in the + population during the evolution. It is lexicographically sorted at all + time so that the first element of the hall of fame is the individual that + has the best first fitness value ever seen, according to the weights + provided to the fitness at creation time. + + The insertion is made so that old individuals have priority on new + individuals. A single copy of each individual is kept at all time, the + equivalence between two individuals is made by the operator passed to the + *similar* argument. + + :param maxsize: The maximum number of individual to keep in the hall of + fame. + :param similar: An equivalence operator between two individuals, optional. + It defaults to operator :func:`operator.eq`. + + The class :class:`HallOfFame` provides an interface similar to a list + (without being one completely). It is possible to retrieve its length, to + iterate on it forward and backward and to get an item or a slice from it. + """ + + def __init__(self, maxsize: Optional[int], similar: Callable = eq): + self.maxsize = maxsize or 0 + self.keys = list() + self.items = list() + self.similar = similar + + def update(self, population: PopulationT): + """ + Update the hall of fame with the *population* by replacing the + worst individuals in it by the best individuals present in + *population* (if they are better). The size of the hall of fame is + kept constant. + + :param population: A list of individual with a fitness attribute to + update the hall of fame with. + """ + if not population: + return + for ind in population: + if len(self) == 0 and self.maxsize != 0: + # Working on an empty hall of fame is problematic for the loop + self.insert(population[0]) + continue + if ind.fitness > self[-1].fitness or len(self) < self.maxsize: + for hofer in self: + # Loop through the hall of fame to check for any similar individual + if self.similar(ind, hofer): + break + else: + # The individual is unique and strictly better than the worst + if len(self) >= self.maxsize: + self.remove(-1) + self.insert(ind) + + def insert(self, item: Individual): + """ + Insert a new individual in the hall of fame using the + :func:`~bisect.bisect_right` function. The inserted individual is + inserted on the right side of an equal individual. Inserting a new + individual in the hall of fame also preserve the hall of fame's order. + This method **does not** check for the size of the hall of fame, in a + way that inserting a new individual in a full hall of fame will not + remove the worst individual to maintain a constant size. + + :param item: The individual with a fitness attribute to insert in the + hall of fame. + """ + i = bisect_right(self.keys, item.fitness) + self.items.insert(len(self) - i, item) + self.keys.insert(i, item.fitness) + + def remove(self, index: int): + """ + Remove the specified *index* from the hall of fame. + + :param index: An integer giving which item to remove. + """ + del self.keys[len(self) - (index % len(self) + 1)] + del self.items[index] + + def clear(self): + """Clear the hall of fame.""" + del self.items[:] + del self.keys[:] + + def __len__(self): + return len(self.items) + + def __getitem__(self, i): + return self.items[i] + + def __iter__(self): + return iter(self.items) + + def __reversed__(self): + return reversed(self.items) + + def __str__(self): + return str(self.items) + + +class ParetoFront(HallOfFame): + """ + The Pareto front hall of fame contains all the non-dominated individuals + that ever lived in the population. That means that the Pareto front hall of + fame can contain an infinity of different individuals. + + :param similar: A function that tells the Pareto front whether or not two + individuals are similar, optional. + + The size of the front may become very large if it is used for example on + a continuous function with a continuous domain. In order to limit the number + of individuals, it is possible to specify a similarity function that will + return :data:`True` if the genotype of two individuals are similar. In that + case only one of the two individuals will be added to the hall of fame. By + default the similarity function is :func:`operator.eq`. + + ParetoFront also supports hard-limiting maxsize, in which cases the element + with worst primary metric is removed. By default ParetoFront is unbounded. + + Since, the Pareto front hall of fame inherits from the :class:`HallOfFame`, + it is sorted lexicographically at every moment. + """ + + def __init__(self, maxsize: Optional[int] = None, similar: Callable = eq): + HallOfFame.__init__(self, maxsize, similar) + + def update(self, population: PopulationT): + """ + Update the Pareto front hall of fame with the *population* by adding + the individuals from the population that are not dominated by the hall + of fame. If any individual in the hall of fame is dominated it is + removed. + + :param population: A list of individual with a fitness attribute to + update the hall of fame with. + """ + for ind in population: + is_dominated = False + dominates_one = False + has_twin = False + to_remove = [] + for i, hof_member in enumerate(self): # hall of fame member + if not dominates_one and hof_member.fitness.dominates(ind.fitness): + is_dominated = True + break + elif ind.fitness.dominates(hof_member.fitness): + dominates_one = True + to_remove.append(i) + elif ind.fitness == hof_member.fitness and self.similar(ind, hof_member): + has_twin = True + break + + for i in reversed(to_remove): # Remove the dominated hofer + self.remove(i) + if not is_dominated and not has_twin: + if len(self) >= self.maxsize > 0: + self.remove(-1) + self.insert(ind) diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/dynamic_graph_requirements.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/dynamic_graph_requirements.py new file mode 100644 index 0000000..67145da --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/dynamic_graph_requirements.py @@ -0,0 +1,11 @@ +from dataclasses import dataclass + +from golem.core.optimisers.optimization_parameters import GraphRequirements + + +@dataclass +class DynamicGraphRequirements(GraphRequirements): + """ Class for using custom domain specific graph requirements. """ + def __init__(self, attributes: dict): + for attribute, value in attributes.items(): + setattr(self, attribute, value) diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/fitness/__init__.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/fitness/__init__.py new file mode 100644 index 0000000..d54e4eb --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/fitness/__init__.py @@ -0,0 +1,2 @@ +from .fitness import Fitness, SingleObjFitness, null_fitness, is_metric_worse +from .multi_objective_fitness import MultiObjFitness diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/fitness/fitness.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/fitness/fitness.py new file mode 100644 index 0000000..eeb2b3f --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/fitness/fitness.py @@ -0,0 +1,170 @@ +from abc import abstractmethod +from typing import Sequence, Any, Optional, Tuple + +import numpy as np + +from golem.utilities.data_structures import Comparable + + +class Fitness(Comparable): + """Abstracts comparable fitness values that can be in invalid state. + + Importantly, Fitness comparison is semantic: `more-than` means `better-than`. + Fitness can be compared using standard operators ``>``, ``<``, ``>=``, etc. + + Fitness comparison handles invalid fitness: invalid fitness is never better + than any other fitness. Fitness implementations must ensure this contract. + + Default Fitness comparison is lexicographic (even for multi-objective fitness, + to ensure total ordering). For proper comparison of multi-objective fitness + use method `dominates`. + """ + + @property + def value(self) -> Optional[float]: + """Return primary fitness value""" + return self.values[0] + + @property + @abstractmethod + def values(self) -> Sequence[float]: + """Return individual metric values. + Returned values are already weighted, if weights are used.""" + raise NotImplementedError() + + @values.setter + @abstractmethod + def values(self, new_values: Optional[Sequence[float]]): + """Assign individual metric values. Accepts unweighted values.""" + raise NotImplementedError() + + @values.deleter + @abstractmethod + def values(self): + """Clear internal metric values.""" + raise NotImplementedError() + + @property + @abstractmethod + def weights(self) -> Sequence[float]: + """Return weights used for weighting individual metrics.""" + raise NotImplementedError() + + @property + @abstractmethod + def valid(self) -> bool: + """Assess if a fitness is valid or not.""" + raise NotImplementedError() + + def dominates(self, other: 'Fitness', selector: Any = None) -> bool: + """Implementation-specific test for fitness domination. + By the default behaves same as less-than operator for valid fitness. + Less means worse; so the better fitness is a dominating one. + + :param other: another fitness for dominates test. + :param selector: optionally specifies which objectives of the fitness to use. + """ + return self > other + + def reset(self): + del self.values + + def __lt__(self, other: 'Fitness') -> bool: + """'Less-than' for fitness means 'worse-than'. + NB: in the case of both invalid the other takes precedence + """ + if not self.valid: + # invalid self is worse + return True + elif not other.valid: + # valid self is NOT worse than invalid other + return False + # if both are valid then compare normally + return is_metric_worse(self.values, other.values) # lexicographic comparison + + def __hash__(self) -> int: + # try to avoid numeric precision errors in hash comparisons + vals = tuple(np.round(value, 8) if value is not None else None + for value in self.values) + return hash(vals) + + def __str__(self): + """Return the values of the Fitness object.""" + return str(self.values if self.valid else tuple()) + + def __repr__(self): + """Return the Python code to build a copy of the object.""" + return "%s.%s%r" % (self.__module__, self.__class__.__name__, + tuple(self.values) if self.valid else tuple()) + + def __eq__(self, other: 'Fitness') -> bool: + return (isinstance(other, self.__class__) and + self.valid and other.valid and + self.allclose(self.values, other.values)) + + def __bool__(self) -> bool: + return self.valid + + @staticmethod + def allclose(values1, values2) -> bool: + return np.allclose(values1, values2, rtol=1e-8, atol=1e-10) + + +class SingleObjFitness(Fitness): + """Single-objective fitness with optional supplementary values + for distinguishing cases when primary fitness values are equal. + This fitness implements lexicographic comparison on its fitness values. + + :param primary_value: Primary fitness metric, may be None. It determines if fitness is valid. + :param supplementary_values: Define supplementary metrics, must not be None. + """ + + def __init__(self, primary_value: Optional[float] = None, *supplementary_values: float): + self._values: Tuple = (primary_value, *supplementary_values) + + @property + def values(self) -> Sequence[float]: + return self._values + + @values.setter + def values(self, new_values: Optional[Sequence[float]]): + if new_values is None: + self.reset() + if any(secondary_value is None for secondary_value in new_values[1:]): + raise ValueError('Secondary values must not be None for prioritized fitness') + self._values = tuple(new_values) + + @values.deleter + def values(self): + self._values = (None,) + + @property + def weights(self) -> Sequence[float]: + # Return default weights + return (1.,) * len(self.values) + + @property + def valid(self) -> bool: + return self._values[0] is not None + + def __hash__(self) -> int: + # __hash__ required explicit super() call + return super().__hash__() + + def __str__(self) -> str: + # For single objective return only the primary value + return str(round(self.value, 4)) if self.value is not None else 'null_fitness' + + +def null_fitness() -> SingleObjFitness: + """Alias for creating default-initialised single-value fitness.""" + return SingleObjFitness(primary_value=None) + + +def is_metric_worse(left_value, right_value) -> bool: + if isinstance(left_value, Fitness): + # Fitness object already handles metric comparison in the right way + return left_value < right_value + else: + # Less is better -- minimisation task on raw metric values + return left_value > right_value diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/fitness/multi_objective_fitness.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/fitness/multi_objective_fitness.py new file mode 100644 index 0000000..cd1ac31 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/fitness/multi_objective_fitness.py @@ -0,0 +1,117 @@ +# This code is modified part of DEAP library (Library URL: https://github.com/DEAP/deap). +import sys +from numbers import Real +from typing import Sequence, Union +from operator import mul, truediv + +import numpy as np + +from golem.core.optimisers.fitness.fitness import Fitness, is_metric_worse + + +class MultiObjFitness(Fitness): + """The fitness is a measure of quality of a solution. If *values* are + provided as a tuple, the fitness is initialized using those values, + otherwise it is empty (or invalid). + + :param values: The initial values of the fitness as a tuple, optional. + :param weights: Optional weights for fitness values. 1.0 by default. + :param wvalues: Allows to provide values by giving already weighted values. + If provided, then :param values: is ignored. This is used by deserialization. + + Comparison between Fitnesses is made lexicographically. + Maximization and minimization are taken care off by a multiplication + between the :attr:`weights` and the fitness :attr:`values`. The comparison + can be made between fitnesses of different size, if the fitnesses are + equal until the extra elements, the longer fitness will be superior to the + shorter. + """ + + def __init__(self, values: Sequence[Real] = (), + weights: Union[Sequence[Real], Real] = 1, + *, wvalues: Sequence[Real] = None): + if wvalues is not None: + # This branch is mainly for deserialization from .wvalues attribute + values = tuple(np.true_divide(wvalues, weights)) + + if isinstance(weights, Real): + # Single value provided or default 1.0 weights + weights = (weights,) * len(values) + elif isinstance(weights, Sequence): + self._check_length(values, weights) + else: + raise TypeError("Attribute weights of %r must be a sequence or a number." % self.__class__) + + self._weights = weights + self.values = values + + @property + def weights(self) -> Sequence[Real]: + return self._weights + + def getValues(self): + return self.wvalues + + def setValues(self, values): + if values is None: + self.reset() + else: + self._check_length(values) + try: + self.wvalues = tuple(map(mul, values, self.weights)) + except TypeError: + _, _, traceback = sys.exc_info() + raise TypeError("Both weights and assigned values must be a " + "sequence of numbers when assigning to values of " + "%r. Currently assigning value(s) %r of %r to a " + "fitness with weights %s." + % (self.__class__, values, type(values), + self.weights)).with_traceback(traceback) + + def delValues(self): + self.wvalues = () + + values = property(getValues, setValues, delValues, + ("Fitness values. Use directly ``individual.fitness.values = values`` " + "in order to set the fitness and ``del individual.fitness.values`` " + "in order to clear (invalidate) the fitness.")) + + def dominates(self, other: 'MultiObjFitness', selector=slice(None)): + """Return true if each objective of *self* is not strictly worse than + the corresponding objective of *other* and at least one objective is + strictly better. + + :param other: Other multi-objective fitness for comparison + :param selector: Slice indicating on which objectives the domination is + tested. The default value is `slice(None)`, representing + every objectives. + """ + not_equal = False + for self_wvalue, other_wvalue in zip(self.wvalues[selector], other.wvalues[selector]): + if is_metric_worse(other_wvalue, self_wvalue): + not_equal = True + elif is_metric_worse(self_wvalue, other_wvalue): + return False + return not_equal + + @property + def valid(self): + """Assess if a fitness is valid or not.""" + return len(self.wvalues) != 0 + + def __hash__(self): + return hash(self.wvalues) + + def __eq__(self, other): + return (isinstance(other, self.__class__) and + self.valid and other.valid and + self.allclose(self.wvalues, other.wvalues)) + + def __str__(self): + return str(tuple(round(wval, 4) for wval in self.wvalues)) + + def _check_length(self, values, weights=None): + if weights is None: + weights = self.weights + if len(values) != len(weights): + raise TypeError("Attribute weights for all values must be provided.") diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/__init__.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/evaluation.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/evaluation.py new file mode 100644 index 0000000..14ee73f --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/evaluation.py @@ -0,0 +1,279 @@ +import gc +import logging +import pathlib +import timeit +from abc import ABC, abstractmethod +from datetime import datetime +from functools import partial +from typing import List, Optional, Sequence, Tuple, TypeVar, Dict + +from joblib import Parallel, delayed + +from golem.core.adapter import BaseOptimizationAdapter +from golem.core.dag.graph import Graph +from golem.core.log import default_log, Log +from golem.core.optimisers.fitness import Fitness +from golem.core.optimisers.genetic.operators.operator import EvaluationOperator, PopulationT +from golem.core.optimisers.graph import OptGraph +from golem.core.optimisers.objective import GraphFunction, ObjectiveFunction +from golem.core.optimisers.opt_history_objects.individual import GraphEvalResult +from golem.core.optimisers.timer import Timer, get_forever_timer +from golem.utilities.serializable import Serializable +from golem.utilities.memory import MemoryAnalytics +from golem.utilities.utilities import determine_n_jobs + +# the percentage of successful evaluations, +# at which evolution is not threatened with stagnation at the moment +STAGNATION_EVALUATION_PERCENTAGE = 0.5 + +EvalResultsList = List[GraphEvalResult] +G = TypeVar('G', bound=Serializable) + + +class DelegateEvaluator: + """Interface for delegate evaluator of graphs.""" + + @property + @abstractmethod + def is_enabled(self) -> bool: + return False + + @abstractmethod + def compute_graphs(self, graphs: Sequence[G]) -> Sequence[G]: + raise NotImplementedError() + + +class ObjectiveEvaluationDispatcher(ABC): + """Builder for evaluation operator. + Takes objective function and decides how to evaluate it over population: + - defines implementation-specific evaluation policy (e.g. sequential, parallel, async); + - saves additional metadata (e.g. computation time, intermediate metrics values). + """ + + @abstractmethod + def dispatch(self, objective: ObjectiveFunction, timer: Optional[Timer] = None) -> EvaluationOperator: + """Return mapped objective function for evaluating population. + + Args: + objective: objective function that accepts single individual + timer: optional timer for stopping the evaluation process + + Returns: + EvaluationOperator: objective function that accepts whole population + """ + raise NotImplementedError() + + def set_graph_evaluation_callback(self, callback: Optional[GraphFunction]): + """Set or reset (with None) post-evaluation callback + that's called on each graph after its evaluation. + + Args: + callback: callback to be called on each evaluated graph + """ + pass + + @staticmethod + def split_individuals_to_evaluate(individuals: PopulationT) -> Tuple[PopulationT, PopulationT]: + """Split individuals sequence to evaluated and skipped ones.""" + individuals_to_evaluate = [] + individuals_to_skip = [] + for ind in individuals: + if ind.fitness.valid: + individuals_to_skip.append(ind) + else: + individuals_to_evaluate.append(ind) + return individuals_to_evaluate, individuals_to_skip + + @staticmethod + def apply_evaluation_results(individuals: PopulationT, + evaluation_results: EvalResultsList) -> PopulationT: + """Applies results of evaluation to the evaluated population. + Excludes individuals that weren't evaluated.""" + evaluation_results = {res.uid_of_individual: res for res in evaluation_results if res} + individuals_evaluated = [] + for ind in individuals: + eval_res = evaluation_results.get(ind.uid) + if not eval_res: + continue + ind.set_evaluation_result(eval_res) + individuals_evaluated.append(ind) + return individuals_evaluated + + +class BaseGraphEvaluationDispatcher(ObjectiveEvaluationDispatcher): + """Base class for dispatchers that evaluate objective function on population. + + Usage: call `dispatch(objective_function)` to get evaluation function. + + Args: + adapter: adapter for graphs + n_jobs: number of jobs for multiprocessing or 1 for no multiprocessing. + graph_cleanup_fn: function to call after graph evaluation, primarily for memory cleanup. + delegate_evaluator: delegate graph fitter (e.g. for remote graph fitting before evaluation) + """ + + def __init__(self, + adapter: BaseOptimizationAdapter, + n_jobs: int = 1, + graph_cleanup_fn: Optional[GraphFunction] = None, + delegate_evaluator: Optional[DelegateEvaluator] = None): + self._adapter = adapter + self._objective_eval = None + self._cleanup = graph_cleanup_fn + self._post_eval_callback = None + self._delegate_evaluator = delegate_evaluator + + self.timer = None + self.logger = default_log(self) + self._n_jobs = n_jobs + self.evaluation_cache = None + self._reset_eval_cache() + + def dispatch(self, objective: ObjectiveFunction, timer: Optional[Timer] = None) -> EvaluationOperator: + """Return handler to this object that hides all details + and allows only to evaluate population with provided objective.""" + self._objective_eval = objective + self.timer = timer or get_forever_timer() + return self.evaluate_population + + def set_graph_evaluation_callback(self, callback: Optional[GraphFunction]): + self._post_eval_callback = callback + + def population_evaluation_info(self, pop_size: int, evaluated_pop_size: int): + """ Shows the amount of successfully evaluated individuals and total number of individuals in population. + If there are more that 50% of successful evaluations than it's more likely + there is no problem in optimization process. """ + if pop_size == 0 or evaluated_pop_size / pop_size <= STAGNATION_EVALUATION_PERCENTAGE: + success_rate = evaluated_pop_size / pop_size if pop_size != 0 else 0 + self.logger.warning(f"{evaluated_pop_size} individuals out of {pop_size} in previous population " + f"were evaluated successfully. {success_rate}% " + f"is a fairly small percentage of successful evaluation.") + else: + self.logger.message(f"{evaluated_pop_size} individuals out of {pop_size} in previous population " + f"were evaluated successfully.") + + @abstractmethod + def evaluate_population(self, individuals: PopulationT) -> PopulationT: + raise NotImplementedError() + + def evaluate_single(self, graph: OptGraph, uid_of_individual: str, with_time_limit: bool = True, + cache_key: Optional[str] = None, + logs_initializer: Optional[Tuple[int, pathlib.Path]] = None) -> GraphEvalResult: + + graph = self.evaluation_cache.get(cache_key, graph) + + if with_time_limit and self.timer.is_time_limit_reached(): + return None + if logs_initializer is not None: + # in case of multiprocessing run + Log.setup_in_mp(*logs_initializer) + + adapted_evaluate = self._adapter.adapt_func(self._evaluate_graph) + start_time = timeit.default_timer() + fitness, graph = adapted_evaluate(graph) + end_time = timeit.default_timer() + eval_time_iso = datetime.now().isoformat() + + eval_res = GraphEvalResult( + uid_of_individual=uid_of_individual, fitness=fitness, graph=graph, metadata={ + 'computation_time_in_seconds': end_time - start_time, + 'evaluation_time_iso': eval_time_iso + } + ) + return eval_res + + def _evaluate_graph(self, domain_graph: Graph) -> Tuple[Fitness, Graph]: + fitness = self._objective_eval(domain_graph) + + if self._post_eval_callback: + self._post_eval_callback(domain_graph) + if self._cleanup: + self._cleanup(domain_graph) + gc.collect() + + return fitness, domain_graph + + def evaluate_with_cache(self, population: PopulationT) -> PopulationT: + reversed_population = list(reversed(population)) + self._remote_compute_cache(reversed_population) + evaluated_population = self.evaluate_population(reversed_population) + self._reset_eval_cache() + return evaluated_population + + def _reset_eval_cache(self): + self.evaluation_cache: Dict[str, Graph] = {} + + def _remote_compute_cache(self, population: PopulationT): + self._reset_eval_cache() + if self._delegate_evaluator and self._delegate_evaluator.is_enabled: + self.logger.info('Remote fit used') + restored_graphs = self._adapter.restore(population) + computed_graphs = self._delegate_evaluator.compute_graphs(restored_graphs) + self.evaluation_cache = {ind.uid: graph for ind, graph in zip(population, computed_graphs)} + + +class MultiprocessingDispatcher(BaseGraphEvaluationDispatcher): + """Evaluates objective function on population using multiprocessing pool + and optionally model evaluation cache with RemoteEvaluator. + + Usage: call `dispatch(objective_function)` to get evaluation function. + + Args: + adapter: adapter for graphs + n_jobs: number of jobs for multiprocessing or 1 for no multiprocessing. + graph_cleanup_fn: function to call after graph evaluation, primarily for memory cleanup. + delegate_evaluator: delegate graph fitter (e.g. for remote graph fitting before evaluation) + """ + + def __init__(self, + adapter: BaseOptimizationAdapter, + n_jobs: int = 1, + graph_cleanup_fn: Optional[GraphFunction] = None, + delegate_evaluator: Optional[DelegateEvaluator] = None): + + super().__init__(adapter, n_jobs, graph_cleanup_fn, delegate_evaluator) + + def dispatch(self, objective: ObjectiveFunction, timer: Optional[Timer] = None) -> EvaluationOperator: + """Return handler to this object that hides all details + and allows only to evaluate population with provided objective.""" + super().dispatch(objective, timer) + return self.evaluate_with_cache + + def evaluate_population(self, individuals: PopulationT) -> PopulationT: + individuals_to_evaluate, individuals_to_skip = self.split_individuals_to_evaluate(individuals) + # Evaluate individuals without valid fitness in parallel. + n_jobs = determine_n_jobs(self._n_jobs, self.logger) + + parallel = Parallel(n_jobs=n_jobs, verbose=0, pre_dispatch="2*n_jobs") + eval_func = partial(self.evaluate_single, logs_initializer=Log().get_parameters()) + evaluation_results = parallel(delayed(eval_func)(ind.graph, ind.uid) for ind in individuals_to_evaluate) + individuals_evaluated = self.apply_evaluation_results(individuals_to_evaluate, evaluation_results) + # If there were no successful evals then try once again getting at least one, + # even if time limit was reached + successful_evals = individuals_evaluated + individuals_to_skip + self.population_evaluation_info(evaluated_pop_size=len(successful_evals), + pop_size=len(individuals)) + if not successful_evals: + for single_ind in individuals: + evaluation_result = eval_func(single_ind.graph, single_ind.uid, with_time_limit=False) + successful_evals = self.apply_evaluation_results([single_ind], [evaluation_result]) + if successful_evals: + break + MemoryAnalytics.log(self.logger, + additional_info='parallel evaluation of population', + logging_level=logging.INFO) + return successful_evals + + +class SequentialDispatcher(BaseGraphEvaluationDispatcher): + """Evaluates objective function on population in sequential way. + + Usage: call `dispatch(objective_function)` to get evaluation function. + """ + + def evaluate_population(self, individuals: PopulationT) -> PopulationT: + individuals_to_evaluate, individuals_to_skip = self.split_individuals_to_evaluate(individuals) + evaluation_results = [self.evaluate_single(ind.graph, ind.uid) for ind in individuals_to_evaluate] + individuals_evaluated = self.apply_evaluation_results(individuals_to_evaluate, evaluation_results) + evaluated_population = individuals_evaluated + individuals_to_skip + return evaluated_population diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/gp_operators.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/gp_operators.py new file mode 100644 index 0000000..cd481a5 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/gp_operators.py @@ -0,0 +1,136 @@ +import itertools +from copy import deepcopy +from typing import Any, List, Tuple, Optional + +from golem.core.dag.graph_node import descriptive_id_recursive_nodes +from golem.core.dag.graph_utils import distance_to_primary_level + + +def equivalent_subtree(graph_first: Any, graph_second: Any, with_primary_nodes: bool = False) \ + -> List[Tuple[Any, Any]]: + """Finds the similar subtrees in two given trees. + With `with_primary_nodes` primary nodes are considered too. + Due to a lot of common subgraphs consisted only of single primary nodes, these nodes can be + not considered with `with_primary_nodes=False`.""" + + pairs_list = [] + all_nodes = graph_first.nodes + graph_second.nodes + all_descriptive_ids = [set(descriptive_id_recursive_nodes(node)) for node in graph_first.nodes] +\ + [set(descriptive_id_recursive_nodes(node)) for node in graph_second.nodes] + all_recursive_ids = dict(zip(all_nodes, all_descriptive_ids)) + for node_first in graph_first.nodes: + for node_second in graph_second.nodes: + if (node_first, node_second) in pairs_list: + continue + equivalent_pairs = structural_equivalent_nodes(node_first=node_first, node_second=node_second, + recursive_ids=all_recursive_ids) + pairs_list.extend(equivalent_pairs) + + pairs_list = list(set(pairs_list)) + if with_primary_nodes: + return pairs_list + # remove nodes with no children + result = [] + for pair in pairs_list: + if len(pair[0].nodes_from) != 0: + result.append(pair) + return result + + +def replace_subtrees(graph_first: Any, graph_second: Any, node_from_first: Any, node_from_second: Any, + layer_in_first: int, layer_in_second: int, max_depth: int): + node_from_graph_first_copy = deepcopy(node_from_first) + + summary_depth = layer_in_first + distance_to_primary_level(node_from_second) + 1 + if summary_depth <= max_depth and summary_depth != 0: + graph_first.update_subtree(node_from_first, node_from_second) + + summary_depth = layer_in_second + distance_to_primary_level(node_from_first) + 1 + if summary_depth <= max_depth and summary_depth != 0: + graph_second.update_subtree(node_from_second, node_from_graph_first_copy) + + +def num_of_parents_in_crossover(num_of_final_inds: int) -> int: + return num_of_final_inds if not num_of_final_inds % 2 else num_of_final_inds + 1 + + +def filter_duplicates(archive, population) -> List[Any]: + filtered_archive = [] + for ind in archive.items: + has_duplicate_in_pop = False + for pop_ind in population: + if ind.fitness == pop_ind.fitness: + has_duplicate_in_pop = True + break + if not has_duplicate_in_pop: + filtered_archive.append(ind) + return filtered_archive + + +def structural_equivalent_nodes(node_first: Any, + node_second: Any, + recursive_ids: Optional[dict] = None, + seen: Optional[List[Any]] = None) -> List[Tuple[Any, Any]]: + """ Returns the list of nodes from which subtrees are structurally equivalent. + :param node_first: node from first graph from which to start the search. + :param node_second: node from second graph from which to start the search. + :param recursive_ids: dict with recursive descriptive id of node with nodes as keys. + :param seen: list of already visited nodes to avoid infinite recursion. + Descriptive ids can be obtained with `descriptive_id_recursive_nodes`. + """ + + nodes = [] + is_same_type = type(node_first) == type(node_second) + seen = seen or [] + + if node_first in seen or node_second in seen: + return [] + seen.append(node_first) + seen.append(node_second) + # check if both nodes are primary or secondary + if hasattr(node_first, 'is_primary') and hasattr(node_second, 'is_primary'): + is_same_graph_node_type = node_first.is_primary == node_second.is_primary + is_same_type = is_same_type and is_same_graph_node_type + + for node1_child, node2_child in itertools.product(node_first.nodes_from, node_second.nodes_from): + nodes_set = structural_equivalent_nodes(node_first=node1_child, node_second=node2_child, + recursive_ids=recursive_ids, seen=seen) + nodes.extend(nodes_set) + if is_same_type and len(node_first.nodes_from) == len(node_second.nodes_from) \ + and are_subtrees_the_same(match_set=nodes, + node_first=node_first, node_second=node_second, + recursive_ids=recursive_ids): + nodes.append((node_first, node_second)) + return nodes + + +def are_subtrees_the_same(match_set: List[Tuple[Any, Any]], + node_first: Any, node_second: Any, recursive_ids: dict = None) -> bool: + """ Returns `True` if subtrees of specified root nodes are the same, otherwise returns `False`. + :param match_set: pairs of nodes to checks subtrees from. + :param node_first: first node from which to compare subtree + :param node_second: second node from which to compare subtree + :param recursive_ids: dict with recursive descriptive id of node with nodes as keys. + Descriptive ids can be obtained with `descriptive_id_recursive_nodes`.""" + matched = [] + if not recursive_ids: + first_recursive_id = set(descriptive_id_recursive_nodes(node_first)) + second_recursive_id = set(descriptive_id_recursive_nodes(node_second)) + else: + first_recursive_id = recursive_ids[node_first] + second_recursive_id = recursive_ids[node_second] + + # 1. Number of exact children must be the same + # 2. All children from one node must have a match from other node children + # 3. Protection from cycles when lengths of descriptive ids are the same due to cycles + if len(node_first.nodes_from) != len(node_second.nodes_from) or \ + len(match_set) == 0 and len(node_first.nodes_from) != 0 or \ + len(first_recursive_id) != len(second_recursive_id): + return False + + for node, node2 in itertools.product(node_first.nodes_from, node_second.nodes_from): + if (node, node2) or (node2, node) in match_set: + matched.append((node, node2)) + if len(matched) >= len(node_first.nodes_from): + return True + return False diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/gp_optimizer.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/gp_optimizer.py new file mode 100644 index 0000000..36bc1db --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/gp_optimizer.py @@ -0,0 +1,144 @@ +from copy import deepcopy +from random import choice +from typing import Sequence, Union, Any + +from golem.core.constants import MAX_GRAPH_GEN_ATTEMPTS +from golem.core.dag.graph import Graph +from golem.core.optimisers.genetic.gp_params import GPAlgorithmParameters +from golem.core.optimisers.genetic.operators.crossover import Crossover +from golem.core.optimisers.genetic.operators.elitism import Elitism +from golem.core.optimisers.genetic.operators.inheritance import Inheritance +from golem.core.optimisers.genetic.operators.mutation import Mutation +from golem.core.optimisers.genetic.operators.operator import PopulationT, EvaluationOperator +from golem.core.optimisers.genetic.operators.regularization import Regularization +from golem.core.optimisers.genetic.operators.reproduction import ReproductionController +from golem.core.optimisers.genetic.operators.selection import Selection +from golem.core.optimisers.genetic.parameters.graph_depth import AdaptiveGraphDepth +from golem.core.optimisers.genetic.parameters.operators_prob import init_adaptive_operators_prob +from golem.core.optimisers.genetic.parameters.population_size import init_adaptive_pop_size, PopulationSize +from golem.core.optimisers.objective.objective import Objective +from golem.core.optimisers.opt_history_objects.individual import Individual +from golem.core.optimisers.optimization_parameters import GraphRequirements +from golem.core.optimisers.optimizer import GraphGenerationParams +from golem.core.optimisers.populational_optimizer import PopulationalOptimizer + + +class EvoGraphOptimizer(PopulationalOptimizer): + """ + Multi-objective evolutionary graph optimizer named GPComp + """ + + def __init__(self, + objective: Objective, + initial_graphs: Sequence[Union[Graph, Any]], + requirements: GraphRequirements, + graph_generation_params: GraphGenerationParams, + graph_optimizer_params: GPAlgorithmParameters, + **custom_optimizer_params + ): + super().__init__(objective, initial_graphs, requirements, + graph_generation_params, graph_optimizer_params, **custom_optimizer_params) + # Define genetic operators + self.regularization = Regularization(graph_optimizer_params, graph_generation_params) + self.selection = Selection(graph_optimizer_params) + self.crossover = Crossover(graph_optimizer_params, requirements, graph_generation_params) + self.mutation = Mutation(graph_optimizer_params, requirements, graph_generation_params) + self.inheritance = Inheritance(graph_optimizer_params, self.selection) + self.elitism = Elitism(graph_optimizer_params) + self.operators = [self.regularization, self.selection, self.crossover, + self.mutation, self.inheritance, self.elitism] + self.reproducer = ReproductionController(graph_optimizer_params, self.selection, self.mutation, self.crossover) + + # Define adaptive parameters + self._pop_size: PopulationSize = init_adaptive_pop_size(graph_optimizer_params, self.generations) + self._operators_prob = init_adaptive_operators_prob(graph_optimizer_params) + self._graph_depth = AdaptiveGraphDepth(self.generations, + start_depth=requirements.start_depth, + max_depth=requirements.max_depth, + max_stagnation_gens=graph_optimizer_params.adaptive_depth_max_stagnation, + adaptive=graph_optimizer_params.adaptive_depth) + + # Define initial parameters + self.requirements.max_depth = self._graph_depth.initial + self.graph_optimizer_params.pop_size = self._pop_size.initial + self.initial_individuals = [Individual(graph, metadata=requirements.static_individual_metadata) + for graph in self.initial_graphs] + + def _initial_population(self, evaluator: EvaluationOperator): + """ Initializes the initial population """ + # Adding of initial assumptions to history as zero generation + self._update_population(evaluator(self.initial_individuals), 'initial_assumptions') + pop_size = self.graph_optimizer_params.pop_size + + if len(self.initial_individuals) < pop_size: + self.initial_individuals = self._extend_population(self.initial_individuals, pop_size) + # Adding of extended population to history + self._update_population(evaluator(self.initial_individuals), 'extended_initial_assumptions') + + def _extend_population(self, pop: PopulationT, target_pop_size: int) -> PopulationT: + verifier = self.graph_generation_params.verifier + extended_pop = list(pop) + pop_graphs = [ind.graph for ind in extended_pop] + + # Set mutation probabilities to 1.0 + initial_req = deepcopy(self.requirements) + initial_req.mutation_prob = 1.0 + self.mutation.update_requirements(requirements=initial_req) + + for iter_num in range(MAX_GRAPH_GEN_ATTEMPTS): + if len(extended_pop) == target_pop_size: + break + new_ind = self.mutation(choice(pop)) + if new_ind: + new_graph = new_ind.graph + if new_graph not in pop_graphs and verifier(new_graph): + extended_pop.append(new_ind) + pop_graphs.append(new_graph) + else: + self.log.warning(f'Exceeded max number of attempts for extending initial graphs, stopping.' + f'Current size {len(pop)}, required {target_pop_size} graphs.') + + # Reset mutation probabilities to default + self.mutation.update_requirements(requirements=self.requirements) + return extended_pop + + def _evolve_population(self, evaluator: EvaluationOperator) -> PopulationT: + """ Method realizing full evolution cycle """ + + # Defines adaptive changes to algorithm parameters + # like pop_size and operator probabilities + self._update_requirements() + + # Regularize previous population + individuals_to_select = self.regularization(self.population, evaluator) + # Reproduce from previous pop to get next population + new_population = self.reproducer.reproduce(individuals_to_select, evaluator) + + # Adaptive agent experience collection & learning + # Must be called after reproduction (that collects the new experience) + experience = self.mutation.agent_experience + experience.collect_results(new_population) + self.mutation.agent.partial_fit(experience) + + # Use some part of previous pop in the next pop + new_population = self.inheritance(self.population, new_population) + new_population = self.elitism(self.generations.best_individuals, new_population) + + return new_population + + def _update_requirements(self): + if not self.generations.is_any_improved: + self.graph_optimizer_params.mutation_prob, self.graph_optimizer_params.crossover_prob = \ + self._operators_prob.next(self.population) + self.log.info( + f'Next mutation proba: {self.graph_optimizer_params.mutation_prob}; ' + f'Next crossover proba: {self.graph_optimizer_params.crossover_prob}') + self.graph_optimizer_params.pop_size = self._pop_size.next(self.population) + self.requirements.max_depth = self._graph_depth.next() + self.log.info( + f'Next population size: {self.graph_optimizer_params.pop_size}; ' + f'max graph depth: {self.requirements.max_depth}') + + # update requirements in operators + for operator in self.operators: + operator.update_requirements(self.graph_optimizer_params, self.requirements) diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/gp_params.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/gp_params.py new file mode 100644 index 0000000..b977fec --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/gp_params.py @@ -0,0 +1,103 @@ +from dataclasses import dataclass +from typing import Sequence, Union, Any, Optional, Callable + +from golem.core.optimisers.adaptive.operator_agent import MutationAgentTypeEnum +from golem.core.optimisers.adaptive.mab_agents.neural_contextual_mab_agent import ContextAgentTypeEnum +from golem.core.optimisers.genetic.operators.base_mutations import MutationStrengthEnum, MutationTypesEnum, \ + simple_mutation_set +from golem.core.optimisers.optimizer import AlgorithmParameters +from golem.core.optimisers.genetic.operators.crossover import CrossoverTypesEnum +from golem.core.optimisers.genetic.operators.elitism import ElitismTypesEnum +from golem.core.optimisers.genetic.operators.inheritance import GeneticSchemeTypesEnum +from golem.core.optimisers.genetic.operators.regularization import RegularizationTypesEnum +from golem.core.optimisers.genetic.operators.selection import SelectionTypesEnum + + +@dataclass +class GPAlgorithmParameters(AlgorithmParameters): + """ + Defines parameters of evolutionary operators and the algorithm of genetic optimizer. + + :param crossover_prob: crossover probability (chance that two individuals will be mated). + :param mutation_prob: mutation probability (chance that an individual will be mutated). + :param variable_mutation_num: flag to apply mutation one or few times for individual in each iteration. + :param max_num_of_operator_attempts: max number of unsuccessful evo operator attempts before continuing. + :param mutation_strength: strength of mutation in tree (using in certain mutation types) + :param min_pop_size_with_elitism: minimal population size with which elitism is applicable + :param required_valid_ratio: ratio of valid individuals on next population to continue optimization. + + Used in `ReproductionController` to compensate for invalid individuals. See the class for details. + + :param adaptive_mutation_type: Experimental feature! Enables adaptive Mutation agent. + :param context_agent_type: Experimental feature! Enables graph encoding for Mutation agent. + + Adaptive mutation agent uses specified algorithm. 'random' type is the default non-adaptive version. + Requires crossover_types to be CrossoverTypesEnum.none for correct adaptive learning, + so that fitness changes depend only on agent's actions (chosen mutations). + ``MutationAgentTypeEnum.bandit`` uses Multi-Armed Bandit (MAB) learning algorithm. + ``MutationAgentTypeEnum.contextual_bandit`` uses contextual MAB learning algorithm. + ``MutationAgentTypeEnum.neural_bandit`` uses contextual MAB learning algorithm with Deep Neural encoding. + + Parameter `context_agent_type` specifies implementation of graph/node encoder for adaptive + mutation agent. It is relevant for contextual and neural bandits. + + :param selection_types: Sequence of selection operators types + :param crossover_types: Sequence of crossover operators types + :param mutation_types: Sequence of mutation operators types + :param elitism_type: type of elitism operator evolution + + :param regularization_type: type of regularization operator + + Regularization attempts to cut off the subtrees of the graph. If the truncated graph + is not worse than the original, then it enters the new generation as a simpler solution. + Regularization is not used by default, it must be explicitly enabled. + + :param genetic_scheme_type: type of genetic evolutionary scheme + + The `generational` scheme is a standard scheme of the evolutionary algorithm. + It specifies that at each iteration the entire generation is updated. + + In the `steady_state` scheme at each iteration only one individual is updated. + + The `parameter_free` scheme is an adaptive variation of the `steady_state` scheme. + It specifies that the population size and the probability of mutation and crossover + change depending on the success of convergence. If there are no improvements in fitness, + then the size and the probabilities increase. When fitness improves, the size and the + probabilities decrease. That is, the algorithm choose a more stable and conservative + mode when optimization seems to converge. + + :param decaying_factor: decaying factor for Multi-Armed Bandits for managing the profit from operators + The smaller the value of decaying_factor, the larger the influence for the best operator. + :param window_size: the size of sliding window for Multi-Armed Bandits to decrease variance. + The window size is measured by the number of individuals to consider. + """ + + crossover_prob: float = 0.8 + mutation_prob: float = 0.8 + variable_mutation_num: bool = True + max_num_of_operator_attempts: int = 100 + mutation_strength: MutationStrengthEnum = MutationStrengthEnum.mean + min_pop_size_with_elitism: int = 5 + required_valid_ratio: float = 0.9 + + adaptive_mutation_type: MutationAgentTypeEnum = MutationAgentTypeEnum.default + context_agent_type: Union[ContextAgentTypeEnum, Callable] = ContextAgentTypeEnum.nodes_num + + selection_types: Optional[Sequence[Union[SelectionTypesEnum, Any]]] = None + crossover_types: Sequence[Union[CrossoverTypesEnum, Any]] = \ + (CrossoverTypesEnum.one_point,) + mutation_types: Sequence[Union[MutationTypesEnum, Any]] = simple_mutation_set + elitism_type: ElitismTypesEnum = ElitismTypesEnum.keep_n_best + regularization_type: RegularizationTypesEnum = RegularizationTypesEnum.none + genetic_scheme_type: GeneticSchemeTypesEnum = GeneticSchemeTypesEnum.generational + + decaying_factor: float = 1.0 + window_size: Optional[int] = None + + def __post_init__(self): + if not self.selection_types: + self.selection_types = (SelectionTypesEnum.spea2,) if self.multi_objective \ + else (SelectionTypesEnum.tournament,) + if self.multi_objective: + # TODO add possibility of using regularization in MO alg + self.regularization_type = RegularizationTypesEnum.none diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/operators/__init__.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/operators/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/operators/base_mutations.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/operators/base_mutations.py new file mode 100644 index 0000000..ea19276 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/operators/base_mutations.py @@ -0,0 +1,427 @@ +from copy import deepcopy +from functools import partial +from random import choice, randint, random, sample, shuffle +from typing import TYPE_CHECKING, Optional + +import numpy as np + +from golem.core.adapter import register_native +from golem.core.dag.graph import ReconnectType +from golem.core.dag.graph_node import GraphNode +from golem.core.dag.graph_utils import distance_to_root_level, distance_to_primary_level, graph_has_cycle +from golem.core.optimisers.advisor import RemoveType +from golem.core.optimisers.graph import OptGraph, OptNode +from golem.core.optimisers.opt_node_factory import OptNodeFactory +from golem.core.optimisers.optimization_parameters import GraphRequirements +from golem.core.optimisers.optimizer import GraphGenerationParams, AlgorithmParameters +from golem.utilities.data_structures import ComparableEnum as Enum + +if TYPE_CHECKING: + from golem.core.optimisers.genetic.gp_params import GPAlgorithmParameters + + +class MutationStrengthEnum(Enum): + weak = 0.2 + mean = 1.0 + strong = 5.0 + + +class MutationTypesEnum(Enum): + simple = 'simple' + growth = 'growth' + local_growth = 'local_growth' + tree_growth = 'tree_growth' + reduce = 'reduce' + single_add = 'single_add' + single_change = 'single_change' + single_drop = 'single_drop' + single_edge = 'single_edge' + + none = 'none' + + @property + def __name__(self): + return self.name + + +def get_mutation_prob(mut_id: MutationStrengthEnum, node: Optional[GraphNode], + default_mutation_prob: float = 0.7) -> float: + """ Function returns mutation probability for certain node in the graph + + :param mut_id: MutationStrengthEnum mean weak or strong mutation + :param node: root node of the graph + :param default_mutation_prob: mutation probability used when mutation_id is invalid or graph has cycles + :return mutation_prob: mutation probability + """ + mutation_prob = default_mutation_prob + graph_cycled = node is None + if node: + graph_cycled = distance_to_primary_level(node) < 0 + if mut_id in list(MutationStrengthEnum) and not graph_cycled: + mutation_strength = mut_id.value + mutation_prob = mutation_strength / (distance_to_primary_level(node) + 1) + return mutation_prob + + +@register_native +def simple_mutation(graph: OptGraph, + requirements: GraphRequirements, + graph_gen_params: GraphGenerationParams, + parameters: 'GPAlgorithmParameters' + ) -> OptGraph: + """ + This type of mutation is passed over all nodes of the tree started from the root node and changes + nodes’ operations with probability - 'node mutation probability' + which is initialised inside the function + + :param graph: graph to mutate + """ + exchange_node = graph_gen_params.node_factory.exchange_node + visited_nodes = set() + + def replace_node_to_random_recursive(node: OptNode) -> OptGraph: + if node not in visited_nodes and random() < node_mutation_probability: + new_node = exchange_node(node) + if new_node: + graph.update_node(node, new_node) + # removed node must not be visited because it's outdated + visited_nodes.add(node) + # new node must not mutated if encountered further during traverse + visited_nodes.add(new_node) + for parent in node.nodes_from: + replace_node_to_random_recursive(parent) + + root_nodes = graph.root_nodes() + root_node = choice(root_nodes) if root_nodes else None + node_mutation_probability = get_mutation_prob(mut_id=parameters.mutation_strength, + node=root_node) + + root_node = root_node or choice(graph.nodes) + replace_node_to_random_recursive(root_node) + + return graph + + +@register_native +def single_edge_mutation(graph: OptGraph, + requirements: GraphRequirements, + graph_gen_params: GraphGenerationParams, + parameters: 'GPAlgorithmParameters' + ) -> OptGraph: + """ + This mutation adds new edge between two random nodes in graph. + + :param graph: graph to mutate + """ + + def nodes_not_cycling(source_node: OptNode, target_node: OptNode): + parents = source_node.nodes_from + while parents: + if target_node not in parents: + grandparents = [] + for parent in parents: + grandparents.extend(parent.nodes_from) + parents = grandparents + else: + return False + return True + + for _ in range(parameters.max_num_of_operator_attempts): + if len(graph.nodes) < 2 or graph.depth > requirements.max_depth: + return graph + + source_node, target_node = sample(graph.nodes, 2) + if source_node not in target_node.nodes_from: + if graph_has_cycle(graph): + graph.connect_nodes(source_node, target_node) + break + else: + if nodes_not_cycling(source_node, target_node): + graph.connect_nodes(source_node, target_node) + break + return graph + + +@register_native +def add_intermediate_node(graph: OptGraph, + node_factory: OptNodeFactory) -> OptGraph: + nodes_with_parents = [node for node in graph.nodes if node.nodes_from] + if len(nodes_with_parents) > 0: + shuffle(nodes_with_parents) + for node_to_mutate in nodes_with_parents: + # add between node and parent + new_node = node_factory.get_parent_node(node_to_mutate, is_primary=False) + if not new_node: + continue + + # rewire old children to new parent + new_node.nodes_from = node_to_mutate.nodes_from + node_to_mutate.nodes_from = [new_node] + + # add new node to graph + graph.add_node(new_node) + break + return graph + + +@register_native +def add_separate_parent_node(graph: OptGraph, + node_factory: OptNodeFactory) -> OptGraph: + node_idx = np.arange(len(graph.nodes)) + shuffle(node_idx) + for idx in node_idx: + node_to_mutate = graph.nodes[idx] + # add as separate parent + new_node = node_factory.get_parent_node(node_to_mutate, is_primary=True) + if not new_node: + # there is no possible operators + continue + if node_to_mutate.nodes_from: + node_to_mutate.nodes_from.append(new_node) + else: + node_to_mutate.nodes_from = [new_node] + graph.nodes.append(new_node) + break + return graph + + +@register_native +def add_as_child(graph: OptGraph, + node_factory: OptNodeFactory) -> OptGraph: + node_idx = np.arange(len(graph.nodes)) + shuffle(node_idx) + for idx in node_idx: + node_to_mutate = graph.nodes[idx] + # add as child + old_node_children = graph.node_children(node_to_mutate) + new_node_child = choice(old_node_children) if old_node_children else None + new_node = node_factory.get_node(is_primary=False) + if not new_node: + continue + graph.add_node(new_node) + graph.connect_nodes(node_parent=node_to_mutate, node_child=new_node) + if new_node_child: + graph.connect_nodes(node_parent=new_node, node_child=new_node_child) + graph.disconnect_nodes(node_parent=node_to_mutate, node_child=new_node_child, + clean_up_leftovers=True) + break + return graph + + +@register_native +def single_add_mutation(graph: OptGraph, + requirements: GraphRequirements, + graph_gen_params: GraphGenerationParams, + parameters: AlgorithmParameters + ) -> OptGraph: + """ + Add new node between two sequential existing modes + + :param graph: graph to mutate + """ + if graph.depth >= requirements.max_depth: + # add mutation is not possible + return graph + + new_graph = deepcopy(graph) + single_add_strategies = [add_as_child, add_separate_parent_node, add_intermediate_node] + shuffle(single_add_strategies) + for strategy in single_add_strategies: + new_graph = strategy(new_graph, graph_gen_params.node_factory) + # maximum three equality check + if new_graph == graph: + continue + break + return new_graph + + +@register_native +def single_change_mutation(graph: OptGraph, + requirements: GraphRequirements, + graph_gen_params: GraphGenerationParams, + parameters: AlgorithmParameters + ) -> OptGraph: + """ + Change node between two sequential existing modes. + + :param graph: graph to mutate + """ + node_idx = np.arange(len(graph.nodes)) + shuffle(node_idx) + for idx in node_idx: + node = graph.nodes[idx] + new_node = graph_gen_params.node_factory.exchange_node(node) + if not new_node: + continue + graph.update_node(node, new_node) + break + return graph + + +@register_native +def single_drop_mutation(graph: OptGraph, + requirements: GraphRequirements, + graph_gen_params: GraphGenerationParams, + parameters: AlgorithmParameters + ) -> OptGraph: + """ + Drop single node from graph. + + :param graph: graph to mutate + """ + if len(graph.nodes) < 2: + return graph + node_to_del = choice(graph.nodes) + node_name = node_to_del.name + removal_type = graph_gen_params.advisor.can_be_removed(node_to_del) + if removal_type == RemoveType.with_direct_children: + # TODO refactor workaround with data_source + graph.delete_node(node_to_del) + nodes_to_delete = \ + [n for n in graph.nodes + if n.descriptive_id.count('data_source') == 1 and node_name in n.descriptive_id] + for child_node in nodes_to_delete: + graph.delete_node(child_node, reconnect=ReconnectType.all) + elif removal_type == RemoveType.with_parents: + graph.delete_subtree(node_to_del) + elif removal_type == RemoveType.node_rewire: + graph.delete_node(node_to_del, reconnect=ReconnectType.all) + elif removal_type == RemoveType.node_only: + graph.delete_node(node_to_del, reconnect=ReconnectType.none) + elif removal_type == RemoveType.forbidden: + pass + else: + raise ValueError("Unknown advice (RemoveType) returned by Advisor ") + return graph + + +@register_native +def tree_growth(graph: OptGraph, + requirements: GraphRequirements, + graph_gen_params: GraphGenerationParams, + parameters: AlgorithmParameters, + local_growth: bool = True) -> OptGraph: + """ + This mutation selects a random node in a tree, generates new subtree, + and replaces the selected node's subtree. + + :param graph: graph to mutate + :param local_growth: if true then maximal depth of new subtree equals depth of tree located in + selected random node, if false then previous depth of selected node doesn't affect to + new subtree depth, maximal depth of new subtree just should satisfy depth constraint in parent tree + """ + node_idx = np.arange(len(graph.nodes)) + shuffle(node_idx) + for idx in node_idx: + node_from_graph = graph.nodes[idx] + if local_growth: + max_depth = distance_to_primary_level(node_from_graph) + is_primary_node_selected = (not node_from_graph.nodes_from) or (node_from_graph != graph.root_node and + randint(0, 1)) + else: + max_depth = requirements.max_depth - distance_to_root_level(graph, node_from_graph) + is_primary_node_selected = \ + distance_to_root_level(graph, node_from_graph) >= requirements.max_depth and randint(0, 1) + if is_primary_node_selected: + new_subtree = graph_gen_params.node_factory.get_node(is_primary=True) + if not new_subtree: + continue + else: + new_subtree = graph_gen_params.random_graph_factory(requirements, max_depth).root_node + + graph.update_subtree(node_from_graph, new_subtree) + break + return graph + + +@register_native +def growth_mutation(graph: OptGraph, + requirements: GraphRequirements, + graph_gen_params: GraphGenerationParams, + parameters: AlgorithmParameters, + local_growth: bool = True + ) -> OptGraph: + """ + This mutation adds new nodes to the graph (just single node between existing nodes or new subtree). + + :param graph: graph to mutate + :param local_growth: if true then maximal depth of new subtree equals depth of tree located in + selected random node, if false then previous depth of selected node doesn't affect to + new subtree depth, maximal depth of new subtree just should satisfy depth constraint in parent tree + """ + + if random() > 0.5: + # simple growth (one node can be added) + return single_add_mutation(graph, requirements, graph_gen_params, parameters) + else: + # advanced growth (several nodes can be added) + return tree_growth(graph, requirements, graph_gen_params, parameters, local_growth) + + +@register_native +def reduce_mutation(graph: OptGraph, + requirements: GraphRequirements, + graph_gen_params: GraphGenerationParams, + parameters: AlgorithmParameters, + ) -> OptGraph: + """ + Selects a random node in a tree, then removes its subtree. If the current arity of the node's + parent is more than the specified minimal arity, then the selected node is also removed. + Otherwise, it is replaced by a random primary node. + + :param graph: graph to mutate + """ + if len(graph.nodes) == 1: + return graph + + nodes = [node for node in graph.nodes if node is not graph.root_node] + shuffle(nodes) + for node_to_del in nodes: + children = graph.node_children(node_to_del) + is_possible_to_delete = all([len(child.nodes_from) - 1 >= requirements.min_arity for child in children]) + if is_possible_to_delete: + graph.delete_subtree(node_to_del) + else: + primary_node = graph_gen_params.node_factory.get_node(is_primary=True) + if not primary_node: + continue + graph.update_subtree(node_to_del, primary_node) + break + return graph + + +@register_native +def no_mutation(graph: OptGraph, *args, **kwargs) -> OptGraph: + return graph + + +base_mutations_repo = { + MutationTypesEnum.none: no_mutation, + MutationTypesEnum.simple: simple_mutation, + MutationTypesEnum.growth: partial(growth_mutation, local_growth=False), + MutationTypesEnum.local_growth: partial(growth_mutation, local_growth=True), + MutationTypesEnum.tree_growth: tree_growth, + MutationTypesEnum.reduce: reduce_mutation, + MutationTypesEnum.single_add: single_add_mutation, + MutationTypesEnum.single_edge: single_edge_mutation, + MutationTypesEnum.single_drop: single_drop_mutation, + MutationTypesEnum.single_change: single_change_mutation +} + +simple_mutation_set = ( + MutationTypesEnum.tree_growth, + MutationTypesEnum.single_add, + MutationTypesEnum.single_change, + MutationTypesEnum.single_drop, + MutationTypesEnum.single_edge, + # join nodes + # flip edge + # cycle edge +) + +rich_mutation_set = ( + MutationTypesEnum.simple, + MutationTypesEnum.reduce, + MutationTypesEnum.growth, + MutationTypesEnum.local_growth +) diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/operators/crossover.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/operators/crossover.py new file mode 100644 index 0000000..47df146 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/operators/crossover.py @@ -0,0 +1,349 @@ +from copy import deepcopy +from itertools import chain +from math import ceil +from random import choice, random, sample, randrange +from typing import Callable, Union, Iterable, Tuple, TYPE_CHECKING + +from golem.core.adapter import register_native +from golem.core.dag.graph_utils import nodes_from_layer, node_depth, get_all_simple_paths, get_connected_components +from golem.core.optimisers.genetic.gp_operators import equivalent_subtree, replace_subtrees +from golem.core.optimisers.genetic.operators.operator import PopulationT, Operator +from golem.core.optimisers.graph import OptGraph, OptNode +from golem.core.optimisers.opt_history_objects.individual import Individual +from golem.core.optimisers.opt_history_objects.parent_operator import ParentOperator +from golem.core.optimisers.optimization_parameters import GraphRequirements +from golem.core.optimisers.optimizer import GraphGenerationParams +from golem.utilities.data_structures import ComparableEnum as Enum + +if TYPE_CHECKING: + from golem.core.optimisers.genetic.gp_params import GPAlgorithmParameters + + +class CrossoverTypesEnum(Enum): + subtree = 'subtree' + one_point = "one_point" + none = 'none' + subgraph = 'subgraph_crossover' + exchange_edges = 'exchange_edges' + exchange_parents_one = 'exchange_parents_one' + exchange_parents_both = 'exchange_parents_both' + + +CrossoverCallable = Callable[[OptGraph, OptGraph, int], Tuple[OptGraph, OptGraph]] + + +class Crossover(Operator): + def __init__(self, + parameters: 'GPAlgorithmParameters', + requirements: GraphRequirements, + graph_generation_params: GraphGenerationParams): + super().__init__(parameters, requirements) + self.graph_generation_params = graph_generation_params + + def __call__(self, population: PopulationT) -> PopulationT: + if len(population) == 1: + new_population = population + else: + new_population = [] + for ind_1, ind_2 in Crossover.crossover_parents_selection(population): + new_population += self._crossover(ind_1, ind_2) + return new_population + + @staticmethod + def crossover_parents_selection(population: PopulationT) -> Iterable[Tuple[Individual, Individual]]: + return zip(population[::2], population[1::2]) + + def _crossover(self, ind_first: Individual, ind_second: Individual) -> Tuple[Individual, Individual]: + crossover_type = choice(self.parameters.crossover_types) + + if self._will_crossover_be_applied(ind_first.graph, ind_second.graph, crossover_type): + crossover_func = self._get_crossover_function(crossover_type) + for _ in range(self.parameters.max_num_of_operator_attempts): + first_object = deepcopy(ind_first.graph) + second_object = deepcopy(ind_second.graph) + new_graphs = crossover_func(first_object, second_object, max_depth=self.requirements.max_depth) + are_correct = all(self.graph_generation_params.verifier(new_graph) for new_graph in new_graphs) + if are_correct: + parent_individuals = (ind_first, ind_second) + new_individuals = self._get_individuals(new_graphs, parent_individuals, crossover_type) + return new_individuals + + self.log.debug('Number of crossover attempts exceeded. ' + 'Please check optimization parameters for correctness.') + + return ind_first, ind_second + + def _get_crossover_function(self, crossover_type: Union[CrossoverTypesEnum, Callable]) -> Callable: + if isinstance(crossover_type, Callable): + crossover_func = crossover_type + else: + crossover_func = self._crossover_by_type(crossover_type) + return self.graph_generation_params.adapter.adapt_func(crossover_func) + + def _crossover_by_type(self, crossover_type: CrossoverTypesEnum) -> CrossoverCallable: + crossovers = { + CrossoverTypesEnum.subtree: subtree_crossover, + CrossoverTypesEnum.one_point: one_point_crossover, + CrossoverTypesEnum.exchange_edges: exchange_edges_crossover, + CrossoverTypesEnum.exchange_parents_one: exchange_parents_one_crossover, + CrossoverTypesEnum.exchange_parents_both: exchange_parents_both_crossover, + CrossoverTypesEnum.subgraph: subgraph_crossover + } + if crossover_type in crossovers: + return crossovers[crossover_type] + else: + raise ValueError(f'Required crossover type is not found: {crossover_type}') + + def _get_individuals(self, new_graphs: Tuple[OptGraph, OptGraph], parent_individuals: Tuple[Individual, Individual], + crossover_type: Union[CrossoverTypesEnum, Callable]) -> Tuple[Individual, Individual]: + operator = ParentOperator(type_='crossover', + operators=str(crossover_type), + parent_individuals=parent_individuals) + metadata = self.requirements.static_individual_metadata + return tuple(Individual(graph, operator, metadata=metadata) for graph in new_graphs) + + def _will_crossover_be_applied(self, graph_first, graph_second, crossover_type) -> bool: + return not (graph_first is graph_second or + random() > self.parameters.crossover_prob or + crossover_type is CrossoverTypesEnum.none) + + +@register_native +def subtree_crossover(graph_1: OptGraph, graph_2: OptGraph, + max_depth: int, inplace: bool = True) -> Tuple[OptGraph, OptGraph]: + """Performed by the replacement of random subtree + in first selected parent to random subtree from the second parent""" + + if not inplace: + graph_1 = deepcopy(graph_1) + graph_2 = deepcopy(graph_2) + else: + graph_1 = graph_1 + graph_2 = graph_2 + + random_layer_in_graph_first = choice(range(graph_1.depth)) + min_second_layer = 1 if random_layer_in_graph_first == 0 and graph_2.depth > 1 else 0 + random_layer_in_graph_second = choice(range(min_second_layer, graph_2.depth)) + + node_from_graph_first = choice(nodes_from_layer(graph_1, random_layer_in_graph_first)) + node_from_graph_second = choice(nodes_from_layer(graph_2, random_layer_in_graph_second)) + + replace_subtrees(graph_1, graph_2, node_from_graph_first, node_from_graph_second, + random_layer_in_graph_first, random_layer_in_graph_second, max_depth) + + return graph_1, graph_2 + + +@register_native +def one_point_crossover(graph_first: OptGraph, graph_second: OptGraph, max_depth: int) -> Tuple[OptGraph, OptGraph]: + """Finds common structural parts between two trees, and after that randomly + chooses the location of nodes, subtrees of which will be swapped""" + pairs_of_nodes = equivalent_subtree(graph_first, graph_second) + if pairs_of_nodes: + node_from_graph_first, node_from_graph_second = choice(pairs_of_nodes) + + layer_in_graph_first = graph_first.depth - node_depth(node_from_graph_first) + layer_in_graph_second = graph_second.depth - node_depth(node_from_graph_second) + + replace_subtrees(graph_first, graph_second, node_from_graph_first, node_from_graph_second, + layer_in_graph_first, layer_in_graph_second, max_depth) + return graph_first, graph_second + + +@register_native +def subgraph_crossover(graph_first: OptGraph, graph_second: OptGraph, **kwargs) -> Tuple[OptGraph, OptGraph]: + """ A random edge is chosen and all paths between these nodes are disconnected. + This way each graph is divided into two subgraphs. + The subgraphs are exchanged between the graphs and connected randomly at the points of division. + Suitable for graphs with cycles. Does not guarantee not exceeding maximal depth. """ + first_subgraphs, first_div_points = get_subgraphs(graph_first) + second_subgraphs, second_div_points = get_subgraphs(graph_second) + graph_first = connect_subgraphs(first_subgraphs[0], second_subgraphs[1], first_div_points, second_div_points) + graph_second = connect_subgraphs(first_subgraphs[1], second_subgraphs[0], first_div_points, second_div_points) + + return graph_first, graph_second + + +def get_subgraphs(graph): + edges = graph.get_edges() + if not edges: + return deepcopy([graph.nodes, graph.nodes]), {*deepcopy(graph.nodes)} + + target, source = choice(edges) + graph.disconnect_nodes(target, source) + + simple_paths = get_all_simple_paths(graph, source, target) + simple_paths.sort(key=len) + division_points = {source, target} + + while len(simple_paths) > 0: + node_first, node_second = choice(simple_paths[0]) + graph.disconnect_nodes(node_first, node_second) if node_first in node_second.nodes_from \ + else graph.disconnect_nodes(node_second, node_first) + division_points.union([node_first, node_second]) + + simple_paths = get_all_simple_paths(graph, source, target) + simple_paths.sort(key=len) + + subgraphs = get_connected_components(graph, [source, target]) + return subgraphs, division_points + + +def connect_subgraphs(first_subgraph, second_subgraph, first_div_points, second_div_points): + first_points = list(first_div_points.intersection(first_subgraph)) + second_points = list(second_div_points.intersection(second_subgraph)) + connections_num = min(len(first_points), len(second_points)) + new_graph = OptGraph([*first_subgraph, *second_subgraph]) + + for _ in range(connections_num): + first_idx, second_idx = randrange(len(first_points)), randrange(len(second_points)) + first_node, second_node = first_points.pop(first_idx), second_points.pop(second_idx) + + if random() > 0.5: + new_graph.connect_nodes(first_node, second_node) + else: + new_graph.connect_nodes(second_node, first_node) + return new_graph + + +@register_native +def exchange_edges_crossover(graph_first: OptGraph, graph_second: OptGraph, max_depth): + """Parents exchange a certain number of edges with each other. The number of + edges is defined as half of the minimum number of edges of both parents, rounded up""" + + def find_edges_in_other_graph(edges, graph: OptGraph): + new_edges = [] + for parent, child in edges: + parent_new = graph.get_nodes_by_name(str(parent)) + if parent_new: + parent_new = parent_new[0] + else: + parent_new = OptNode(str(parent)) + graph.add_node(parent_new) + child_new = graph.get_nodes_by_name(str(child)) + if child_new: + child_new = child_new[0] + else: + child_new = OptNode(str(child)) + graph.add_node(child_new) + new_edges.append((parent_new, child_new)) + return new_edges + + edges_1 = graph_first.get_edges() + edges_2 = graph_second.get_edges() + count = ceil(min(len(edges_1), len(edges_2)) / 2) + choice_edges_1 = sample(edges_1, count) + choice_edges_2 = sample(edges_2, count) + + for parent, child in choice_edges_1: + child.nodes_from.remove(parent) + for parent, child in choice_edges_2: + child.nodes_from.remove(parent) + + old_edges1 = graph_first.get_edges() + old_edges2 = graph_second.get_edges() + + new_edges_2 = find_edges_in_other_graph(choice_edges_1, graph_second) + new_edges_1 = find_edges_in_other_graph(choice_edges_2, graph_first) + + for parent, child in new_edges_1: + if (parent, child) not in old_edges1: + child.nodes_from.append(parent) + for parent, child in new_edges_2: + if (parent, child) not in old_edges2: + child.nodes_from.append(parent) + + return graph_first, graph_second + + +@register_native +def exchange_parents_one_crossover(graph_first: OptGraph, graph_second: OptGraph, max_depth: int): + """For the selected node for the first parent, change the parent nodes to + the parent nodes of the same node of the second parent. Thus, the first child is obtained. + The second child is a copy of the second parent""" + + def find_nodes_in_other_graph(nodes, graph: OptGraph): + new_nodes = [] + for node in nodes: + new_node = graph.get_nodes_by_name(str(node)) + if new_node: + new_node = new_node[0] + else: + new_node = OptNode(str(node)) + graph.add_node(new_node) + new_nodes.append(new_node) + return new_nodes + + edges = graph_second.get_edges() + nodes_with_parent_or_child = list(set(chain(*edges))) + if nodes_with_parent_or_child: + + selected_node = choice(nodes_with_parent_or_child) + parents = selected_node.nodes_from + + node_from_first_graph = find_nodes_in_other_graph([selected_node], graph_first)[0] + + node_from_first_graph.nodes_from = [] + old_edges1 = graph_first.get_edges() + + if parents: + parents_in_first_graph = find_nodes_in_other_graph(parents, graph_first) + for parent in parents_in_first_graph: + if (parent, node_from_first_graph) not in old_edges1: + node_from_first_graph.nodes_from.append(parent) + + return graph_first, graph_second + + +@register_native +def exchange_parents_both_crossover(graph_first: OptGraph, graph_second: OptGraph, max_depth: int): + """For the selected node for the first parent, change the parent nodes to + the parent nodes of the same node of the second parent. Thus, the first child is obtained. + The second child is formed in a similar way""" + + parents_in_first_graph = [] + parents_in_second_graph = [] + + def find_nodes_in_other_graph(nodes, graph: OptGraph): + new_nodes = [] + for node in nodes: + new_node = graph.get_nodes_by_name(str(node)) + if new_node: + new_node = new_node[0] + else: + new_node = OptNode(str(node)) + graph.add_node(new_node) + new_nodes.append(new_node) + return new_nodes + + edges = graph_second.get_edges() + nodes_with_parent_or_child = list(set(chain(*edges))) + if nodes_with_parent_or_child: + + selected_node2 = choice(nodes_with_parent_or_child) + parents2 = selected_node2.nodes_from + if parents2: + parents_in_first_graph = find_nodes_in_other_graph(parents2, graph_first) + + selected_node1 = find_nodes_in_other_graph([selected_node2], graph_first)[0] + parents1 = selected_node1.nodes_from + if parents1: + parents_in_second_graph = find_nodes_in_other_graph(parents1, graph_second) + + for p in parents1: + selected_node1.nodes_from.remove(p) + for p in parents2: + selected_node2.nodes_from.remove(p) + + old_edges1 = graph_first.get_edges() + old_edges2 = graph_second.get_edges() + + for parent in parents_in_first_graph: + if (parent, selected_node1) not in old_edges1: + selected_node1.nodes_from.append(parent) + + for parent in parents_in_second_graph: + if (parent, selected_node2) not in old_edges2: + selected_node2.nodes_from.append(parent) + + return graph_first, graph_second diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/operators/elitism.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/operators/elitism.py new file mode 100644 index 0000000..e462659 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/operators/elitism.py @@ -0,0 +1,46 @@ +from random import shuffle + +from golem.core.optimisers.genetic.operators.operator import PopulationT, Operator +from golem.utilities.data_structures import ComparableEnum as Enum + + +class ElitismTypesEnum(Enum): + keep_n_best = 'keep_n_best' + replace_worst = 'replace_worst' + none = 'none' + + +class Elitism(Operator): + def __call__(self, best_individuals: PopulationT, new_population: PopulationT) -> PopulationT: + elitism_type = self.parameters.elitism_type + if elitism_type is ElitismTypesEnum.none or not self._is_elitism_applicable(): + return new_population + elif elitism_type is ElitismTypesEnum.keep_n_best: + return self.keep_n_best_elitism(best_individuals, new_population) + elif elitism_type is ElitismTypesEnum.replace_worst: + return self.replace_worst_elitism(best_individuals, new_population) + else: + raise ValueError(f'Required elitism type not found: {elitism_type}') + + def _is_elitism_applicable(self) -> bool: + if self.parameters.multi_objective: + return False + return self.parameters.pop_size >= self.parameters.min_pop_size_with_elitism + + @staticmethod + def keep_n_best_elitism(best_individuals: PopulationT, new_population: PopulationT) -> PopulationT: + final_population = [] + final_population += best_individuals + new_unique_inds = [ind for ind in new_population if ind not in best_individuals] + if new_unique_inds: + shuffle(new_unique_inds) + remain_n = len(new_population) - len(best_individuals) + final_population += new_unique_inds[:remain_n] + return final_population + + @staticmethod + def replace_worst_elitism(best_individuals: PopulationT, new_population: PopulationT) -> PopulationT: + population = best_individuals + new_population + # sort in descending order (Fitness(10) > Fitness(11)) + sorted_ascending_population = sorted(population, key=lambda individual: individual.fitness, reverse=True) + return sorted_ascending_population[:len(new_population)] diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/operators/inheritance.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/operators/inheritance.py new file mode 100644 index 0000000..2895e74 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/operators/inheritance.py @@ -0,0 +1,48 @@ +from typing import TYPE_CHECKING + +from golem.core.optimisers.genetic.operators.operator import PopulationT, Operator +from golem.core.optimisers.genetic.operators.selection import Selection +from golem.utilities.data_structures import ComparableEnum as Enum + +if TYPE_CHECKING: + from golem.core.optimisers.genetic.gp_params import GPAlgorithmParameters + + +class GeneticSchemeTypesEnum(Enum): + steady_state = 'steady_state' + generational = 'generational' + parameter_free = 'parameter_free' + + +class Inheritance(Operator): + def __init__(self, parameters: 'GPAlgorithmParameters', selection: Selection): + super().__init__(parameters=parameters) + self.selection = selection + + def __call__(self, previous_population: PopulationT, new_population: PopulationT) -> PopulationT: + gp_scheme = self.parameters.genetic_scheme_type + if gp_scheme == GeneticSchemeTypesEnum.generational: + # Previous population is completely substituted + next_population = self.direct_inheritance(new_population) + elif gp_scheme == GeneticSchemeTypesEnum.steady_state: + # Previous population is mixed with new one + next_population = self.steady_state_inheritance(previous_population, new_population) + elif gp_scheme == GeneticSchemeTypesEnum.parameter_free: + # Same as steady-state + next_population = self.steady_state_inheritance(previous_population, new_population) + else: + raise ValueError(f'Unknown genetic scheme {gp_scheme}!') + return next_population + + def steady_state_inheritance(self, + prev_population: PopulationT, + new_population: PopulationT + ) -> PopulationT: + # use individuals with non-repetitive uid + not_repetitive_inds = [ind for ind in prev_population if ind not in new_population] + full_population = list(new_population) + list(not_repetitive_inds) + return self.selection(full_population, + pop_size=self.parameters.pop_size) + + def direct_inheritance(self, new_population: PopulationT) -> PopulationT: + return new_population[:self.parameters.pop_size] diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/operators/mutation.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/operators/mutation.py new file mode 100644 index 0000000..ebe4842 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/operators/mutation.py @@ -0,0 +1,148 @@ +from copy import deepcopy +from random import random +from typing import Callable, Union, Tuple, TYPE_CHECKING, Mapping, Hashable, Optional + +import numpy as np + +from golem.core.dag.graph import Graph +from golem.core.optimisers.adaptive.mab_agents.contextual_mab_agent import ContextualMultiArmedBanditAgent +from golem.core.optimisers.adaptive.mab_agents.mab_agent import MultiArmedBanditAgent +from golem.core.optimisers.adaptive.mab_agents.neural_contextual_mab_agent import NeuralContextualMultiArmedBanditAgent +from golem.core.optimisers.adaptive.operator_agent import \ + OperatorAgent, RandomAgent, MutationAgentTypeEnum +from golem.core.optimisers.adaptive.experience_buffer import ExperienceBuffer +from golem.core.optimisers.genetic.operators.base_mutations import \ + base_mutations_repo, MutationTypesEnum +from golem.core.optimisers.genetic.operators.operator import PopulationT, Operator +from golem.core.optimisers.opt_history_objects.individual import Individual +from golem.core.optimisers.opt_history_objects.parent_operator import ParentOperator +from golem.core.optimisers.optimization_parameters import GraphRequirements, OptimizationParameters +from golem.core.optimisers.optimizer import GraphGenerationParams, AlgorithmParameters + +if TYPE_CHECKING: + from golem.core.optimisers.genetic.gp_params import GPAlgorithmParameters + +MutationFunc = Callable[[Graph, GraphRequirements, GraphGenerationParams, AlgorithmParameters], Graph] +MutationIdType = Hashable +MutationRepo = Mapping[MutationIdType, MutationFunc] + + +class Mutation(Operator): + def __init__(self, + parameters: 'GPAlgorithmParameters', + requirements: GraphRequirements, + graph_gen_params: GraphGenerationParams, + mutations_repo: Optional[MutationRepo] = None, + ): + super().__init__(parameters, requirements) + self.graph_generation_params = graph_gen_params + self.parameters = parameters + self._mutations_repo = mutations_repo or base_mutations_repo + self._operator_agent = self._init_operator_agent(graph_gen_params, parameters, requirements) + self.agent_experience = ExperienceBuffer(window_size=parameters.window_size) + + @staticmethod + def _init_operator_agent(graph_gen_params: GraphGenerationParams, + parameters: 'GPAlgorithmParameters', + requirements: OptimizationParameters): + kind = parameters.adaptive_mutation_type + if kind == MutationAgentTypeEnum.default or kind == MutationAgentTypeEnum.random: + agent = RandomAgent(actions=parameters.mutation_types) + elif kind == MutationAgentTypeEnum.bandit: + agent = MultiArmedBanditAgent(actions=parameters.mutation_types, + n_jobs=requirements.n_jobs, + path_to_save=requirements.agent_dir, + decaying_factor=parameters.decaying_factor) + elif kind == MutationAgentTypeEnum.contextual_bandit: + agent = ContextualMultiArmedBanditAgent( + actions=parameters.mutation_types, + context_agent_type=parameters.context_agent_type, + available_operations=graph_gen_params.node_factory.get_all_available_operations(), + n_jobs=requirements.n_jobs, + decaying_factor=parameters.decaying_factor) + elif kind == MutationAgentTypeEnum.neural_bandit: + agent = NeuralContextualMultiArmedBanditAgent( + actions=parameters.mutation_types, + context_agent_type=parameters.context_agent_type, + available_operations=graph_gen_params.node_factory.get_all_available_operations(), + n_jobs=requirements.n_jobs) + # if agent was specified pretrained (with instance) + elif isinstance(parameters.adaptive_mutation_type, OperatorAgent): + agent = kind + else: + raise TypeError(f'Unknown parameter {kind}') + return agent + + @property + def agent(self) -> OperatorAgent: + return self._operator_agent + + def __call__(self, population: Union[Individual, PopulationT]) -> Union[Individual, PopulationT]: + if isinstance(population, Individual): + population = [population] + + final_population, application_attempts = tuple(zip(*map(self._mutation, population))) + + # drop individuals to which mutations could not be applied + final_population = [ind for ind, init_ind, attempt in zip(final_population, population, application_attempts) + if not(attempt and ind.graph == init_ind.graph)] + + if len(population) == 1: + return final_population[0] if final_population else final_population + + return final_population + + def _mutation(self, individual: Individual) -> Tuple[Individual, bool]: + """ Function applies mutation operator to graph """ + mutation_type = self._operator_agent.choose_action(individual.graph) + is_applied = self._will_mutation_be_applied(mutation_type) + if is_applied: + for _ in range(self.parameters.max_num_of_operator_attempts): + new_graph = deepcopy(individual.graph) + + new_graph = self._apply_mutations(new_graph, mutation_type) + is_correct_graph = self.graph_generation_params.verifier(new_graph) + if is_correct_graph: + # str for custom mutations serialisation + parent_operator = ParentOperator(type_='mutation', + operators=mutation_type.__name__, + parent_individuals=individual) + individual = Individual(new_graph, parent_operator, + metadata=self.requirements.static_individual_metadata) + break + else: + # Collect invalid actions + self.agent_experience.collect_experience(individual, mutation_type, reward=-1.0) + + self.log.debug(f'Number of attempts for {mutation_type} mutation application exceeded. ' + 'Please check optimization parameters for correctness.') + return individual, is_applied + + def _sample_num_of_mutations(self, mutation_type: Union[MutationTypesEnum, Callable]) -> int: + # most of the time returns 1 or rarely several mutations + is_custom_mutation = isinstance(mutation_type, Callable) + if self.parameters.variable_mutation_num and not is_custom_mutation: + num_mut = max(int(round(np.random.lognormal(0, sigma=0.5))), 1) + else: + num_mut = 1 + return num_mut + + def _apply_mutations(self, new_graph: Graph, mutation_type: Union[MutationTypesEnum, Callable]) -> Graph: + """Apply mutation 1 or few times iteratively""" + for _ in range(self._sample_num_of_mutations(mutation_type)): + mutation_func = self._get_mutation_func(mutation_type) + new_graph = mutation_func(new_graph, requirements=self.requirements, + graph_gen_params=self.graph_generation_params, + parameters=self.parameters) + return new_graph + + def _will_mutation_be_applied(self, mutation_type: Union[MutationTypesEnum, Callable]) -> bool: + return random() <= self.parameters.mutation_prob and mutation_type is not MutationTypesEnum.none + + def _get_mutation_func(self, mutation_type: Union[MutationTypesEnum, Callable]) -> Callable: + if isinstance(mutation_type, Callable): + mutation_func = mutation_type + else: + mutation_func = self._mutations_repo[mutation_type] + adapted_mutation_func = self.graph_generation_params.adapter.adapt_func(mutation_func) + return adapted_mutation_func diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/operators/operator.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/operators/operator.py new file mode 100644 index 0000000..da36fa7 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/operators/operator.py @@ -0,0 +1,43 @@ +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Callable, Optional, Sequence + +from golem.core.log import default_log +from golem.core.optimisers.optimization_parameters import GraphRequirements +from golem.core.optimisers.opt_history_objects.individual import Individual + +if TYPE_CHECKING: + from golem.core.optimisers.optimizer import AlgorithmParameters + +PopulationT = Sequence[Individual] +EvaluationOperator = Callable[[PopulationT], PopulationT] + + +class Operator(ABC): + """ Base abstract functional interface for genetic operators. + Specific signatures are: + - Selection: Population -> Population + - Inheritance: [Population, Population] -> Population + - Regularization: [Population, EvaluationOperator] -> Population + - Mutation: Union[Individual, Population] -> Union[Individual, Population] + - Crossover: Population -> Population + - Elitism: [Population, Population] -> Population + """ + + def __init__(self, + parameters: Optional['AlgorithmParameters'] = None, + requirements: Optional[GraphRequirements] = None): + self.requirements = requirements + self.parameters = parameters + self.log = default_log(self) + + @abstractmethod + def __call__(self, *args, **kwargs): + pass + + def update_requirements(self, + parameters: Optional['AlgorithmParameters'] = None, + requirements: Optional[GraphRequirements] = None): + if requirements: + self.requirements = requirements + if parameters: + self.parameters = parameters diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/operators/regularization.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/operators/regularization.py new file mode 100644 index 0000000..7d8e7db --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/operators/regularization.py @@ -0,0 +1,65 @@ +from copy import deepcopy +from typing import TYPE_CHECKING + +from golem.core.dag.graph_utils import ordered_subnodes_hierarchy +from golem.core.optimisers.genetic.operators.operator import PopulationT, EvaluationOperator, Operator +from golem.core.optimisers.graph import OptGraph, OptNode +from golem.core.optimisers.opt_history_objects.individual import Individual +from golem.core.optimisers.opt_history_objects.parent_operator import ParentOperator +from golem.core.optimisers.optimizer import GraphGenerationParams +from golem.utilities.data_structures import ComparableEnum as Enum + +if TYPE_CHECKING: + from golem.core.optimisers.genetic.gp_params import GPAlgorithmParameters + + +class RegularizationTypesEnum(Enum): + none = 'none' + decremental = 'decremental' + + +class Regularization(Operator): + def __init__(self, parameters: 'GPAlgorithmParameters', + graph_generation_params: GraphGenerationParams): + super().__init__(parameters=parameters) + self.graph_generation_params = graph_generation_params + + def __call__(self, population: PopulationT, evaluator: EvaluationOperator) -> PopulationT: + regularization_type = self.parameters.regularization_type + if regularization_type is RegularizationTypesEnum.decremental: + return self._decremental_regularization(population, evaluator) + elif regularization_type is RegularizationTypesEnum.none: + return population + else: + raise ValueError(f'Required regularization type not found: {regularization_type}') + + def _decremental_regularization(self, population: PopulationT, evaluator: EvaluationOperator) -> PopulationT: + size = self.parameters.pop_size + additional_inds = [] + prev_nodes_ids = set() + for ind in population: + prev_nodes_ids.add(ind.graph.descriptive_id) + parent_operator = ParentOperator(type_='regularization', + operators='decremental_regularization', + parent_individuals=ind) + subtree_inds = [Individual(OptGraph(deepcopy(ordered_subnodes_hierarchy(node))), parent_operator, + metadata=self.requirements.static_individual_metadata) + for node in ind.graph.nodes + if Regularization._is_fitted_subtree(self.graph_generation_params.adapter.restore(node)) and + node.descriptive_id not in prev_nodes_ids] + + additional_inds.extend(subtree_inds) + prev_nodes_ids.update(subtree.graph.root_node.descriptive_id for subtree in subtree_inds) + + additional_inds = [ind for ind in additional_inds if self.graph_generation_params.verifier(ind.graph)] + evaluator(additional_inds) + additional_inds.extend(population) + if len(additional_inds) > size: + additional_inds = sorted(additional_inds, key=lambda ind: ind.fitness, reverse=True)[:size] + + return additional_inds + + # TODO: remove this hack (e.g. provide smth like FitGraph with fit/unfit interface) + @staticmethod + def _is_fitted_subtree(node: OptNode) -> bool: + return node.nodes_from and hasattr(node, 'fitted_operation') and node.fitted_operation diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/operators/reproduction.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/operators/reproduction.py new file mode 100644 index 0000000..bbaacde --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/operators/reproduction.py @@ -0,0 +1,134 @@ +from typing import Optional + +import numpy as np + +from golem.core.constants import MIN_POP_SIZE, EVALUATION_ATTEMPTS_NUMBER +from golem.core.log import default_log +from golem.core.optimisers.genetic.gp_params import GPAlgorithmParameters +from golem.core.optimisers.genetic.operators.crossover import Crossover +from golem.core.optimisers.genetic.operators.mutation import Mutation +from golem.core.optimisers.genetic.operators.operator import PopulationT, EvaluationOperator +from golem.core.optimisers.genetic.operators.selection import Selection +from golem.core.optimisers.populational_optimizer import EvaluationAttemptsError +from golem.utilities.data_structures import ensure_wrapped_in_sequence + + +class ReproductionController: + """ + Task of the Reproduction Controller is to reproduce population + while keeping population size as specified in optimizer settings. + + It implements a simple proportional controller that compensates for + invalid results each generation by computing average ratio of valid results. + Invalid results include cases when Operators, Evaluator or GraphVerifier + return output population that's smaller than the input population. + + Example. + Let's say we need a population of size 50. Let's say about 20% of individuals + are *usually* evaluated with an error. If we take select only 50 for the new population, + we will get about 40 valid ones. Not enough. Therefore, we need to take more. + How much more? Approximately by `target_pop_size / mean_success_rate = 50 / 0.8 ~= 62'. + Here `mean_success_rate` estimates number of successfully evaluated individuals. + Then we request 62, then approximately 62*0.8~=50 of them are valid in the end, + and we achieve target size more reliably. This runs in a loop to control stochasticity. + + Args: + parameters: genetic algorithm parameters. + selection: operator used in reproduction. + mutation: operator used in reproduction. + crossover: operator used in reproduction. + window_size: size in iterations of the moving window to compute reproduction success rate. + """ + + def __init__(self, + parameters: GPAlgorithmParameters, + selection: Selection, + mutation: Mutation, + crossover: Crossover, + window_size: int = 10, + ): + self.parameters = parameters + self.selection = selection + self.mutation = mutation + self.crossover = crossover + + self._minimum_valid_ratio = parameters.required_valid_ratio * 0.5 + self._window_size = window_size + self._success_rate_window = np.full(self._window_size, 1.0) + + self._log = default_log(self) + + @property + def mean_success_rate(self) -> float: + """Returns mean success rate of reproduction + evaluation, + fraction of how many individuals were reproduced and mutated successfully. + Computed as average fraction for the last N iterations (N = window size param)""" + return float(np.mean(self._success_rate_window)) + + def reproduce_uncontrolled(self, + population: PopulationT, + evaluator: EvaluationOperator, + pop_size: Optional[int] = None, + ) -> PopulationT: + """Reproduces and evaluates population (select, crossover, mutate). + Doesn't implement any additional checks on population. + """ + # If operators can return unchanged individuals from previous population + # (e.g. both Mutation & Crossover are not applied with some probability) + # then there's a probability that duplicate individuals can appear + + # TODO: it can't choose more than len(population)! + # It can be faster if it could. + selected_individuals = self.selection(population, pop_size) + new_population = self.crossover(selected_individuals) + new_population = ensure_wrapped_in_sequence(self.mutation(new_population)) + new_population = evaluator(new_population) + return new_population + + def reproduce(self, + population: PopulationT, + evaluator: EvaluationOperator + ) -> PopulationT: + """Reproduces and evaluates population (select, crossover, mutate). + Implements additional checks on population to ensure that population size + follows required population size. + """ + total_target_size = self.parameters.pop_size # next population size + collected_next_population = {} + for i in range(EVALUATION_ATTEMPTS_NUMBER): + # Estimate how many individuals we need to complete new population + # based on average success rate of valid results + residual_size = total_target_size - len(collected_next_population) + residual_size = max(MIN_POP_SIZE, + int(residual_size / self.mean_success_rate)) + residual_size = min(len(population), residual_size) + + # Reproduce the required number of individuals that equals residual size + partial_next_population = self.reproduce_uncontrolled(population, evaluator, residual_size) + # Avoid duplicate individuals that can come unchanged from previous population + collected_next_population.update({ind.uid: ind for ind in partial_next_population}) + + # Keep running average of transform success rate (if sample is big enough) + if len(partial_next_population) >= MIN_POP_SIZE: + valid_ratio = len(partial_next_population) / residual_size + self._success_rate_window = np.roll(self._success_rate_window, shift=1) + self._success_rate_window[0] = valid_ratio + + # Successful return: got enough individuals + if len(collected_next_population) >= total_target_size * self.parameters.required_valid_ratio: + self._log.info(f'Reproduction achieved pop size {len(collected_next_population)}' + f' using {i+1} attempt(s) with success rate {self.mean_success_rate:.3f}') + return list(collected_next_population.values())[:total_target_size] + else: + # If number of evaluation attempts is exceeded return a warning or raise exception + helpful_msg = ('Check objective, constraints and evo operators. ' + 'Possibly they return too few valid individuals.') + + if len(collected_next_population) >= total_target_size * self._minimum_valid_ratio: + self._log.warning(f'Could not achieve required population size: ' + f'have {len(collected_next_population)},' + f' required {total_target_size}!\n' + helpful_msg) + return list(collected_next_population.values()) + else: + raise EvaluationAttemptsError('Could not collect valid individuals' + ' for next population.' + helpful_msg) diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/operators/selection.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/operators/selection.py new file mode 100644 index 0000000..c2f1ab7 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/operators/selection.py @@ -0,0 +1,247 @@ +import functools +import math +from copy import copy +from random import choice, randint, sample +from typing import Callable, List, Optional + +from golem.core.optimisers.genetic.operators.operator import PopulationT, Operator +from golem.utilities.data_structures import ComparableEnum as Enum + + +class SelectionTypesEnum(Enum): + tournament = 'tournament' + spea2 = 'spea2' + + +class Selection(Operator): + def __call__(self, population: PopulationT, pop_size: Optional[int] = None) -> PopulationT: + """ + Selection of individuals based on specified type of selection + :param population: A list of individuals to select from. + :param pop_size: Optional custom population_size. + Taken from algorithm parameters if not specified. + """ + pop_size = pop_size or self.parameters.pop_size + selection_type = choice(self.parameters.selection_types) + return self._selection_by_type(selection_type)(population, pop_size) + + @staticmethod + def _selection_by_type(selection_type: SelectionTypesEnum) -> Callable[[PopulationT, int], PopulationT]: + selections = { + SelectionTypesEnum.tournament: tournament_selection, + SelectionTypesEnum.spea2: spea2_selection + } + if selection_type in selections: + return selections[selection_type] + elif isinstance(selection_type, Callable): + return selection_type + else: + raise ValueError(f'Required selection not found: {selection_type}') + + +def default_selection_behaviour(selection_func: Optional[Callable] = None, *, ensure_unique: bool = True, + populate_by_single: bool = True): + def func_wrapper(func: Callable): + @functools.wraps(func) + def wrapper(individuals: PopulationT, pop_size: int, *args, **kwargs): + if ensure_unique: + individuals = list({ind.uid: ind for ind in individuals}.values()) + else: + individuals = copy(individuals) + + if populate_by_single and len(individuals) == 1: + return individuals * pop_size + + if len(individuals) <= pop_size: + return individuals + + return func(individuals, pop_size, *args, **kwargs) + return wrapper + + if selection_func: + return func_wrapper(selection_func) # Allows to decorate without args. + return func_wrapper # Allows to decorate with args but no selection_func specified. + + +@default_selection_behaviour +def tournament_selection(individuals: PopulationT, pop_size: int, fraction: float = 0.1) -> PopulationT: + """ Having the size of *individuals* equals to *n* will have no effect other + than lax ordering of *individuals*. """ + individuals = list(individuals) # don't modify original + group_size = math.ceil(len(individuals) * fraction) + min_group_size = min(2, len(individuals)) + group_size = max(group_size, min_group_size) + chosen = [] + iterations_limit = pop_size * 10 + for _ in range(iterations_limit): + if len(chosen) >= pop_size: + break + group = sample(individuals, min(group_size, len(individuals))) + best = max(group, key=lambda ind: ind.fitness) + individuals.remove(best) + chosen.append(best) + return chosen + + +@default_selection_behaviour +def random_selection(individuals: PopulationT, pop_size: int) -> PopulationT: + return sample(individuals, pop_size) + + +# Code of spea2 selection is modified part of DEAP library (Library URL: https://github.com/DEAP/deap). +@default_selection_behaviour +def spea2_selection(individuals: PopulationT, pop_size: int) -> PopulationT: + """ + Apply SPEA-II selection operator on the *individuals*. Usually, the + size of *individuals* will be larger than *n* because any individual + present in *individuals* will appear in the returned list at most once. + Having the size of *individuals* equals to *n* will have no effect other + than sorting the population according to a strength Pareto scheme. The + list returned contains references to the input *individuals*. + + :param individuals: A list of individuals to select from. + :returns: A list of selected individuals + """ + inds_len = len(individuals) + fitness_len = len(individuals[0].fitness.values) + inds_len_sqrt = math.sqrt(inds_len) + strength_fits = [0] * inds_len + fits = [0] * inds_len + dominating_inds = [list() for _ in range(inds_len)] + + for i, ind_i in enumerate(individuals): + for j, ind_j in enumerate(individuals[i + 1:], i + 1): + if ind_i.fitness.dominates(ind_j.fitness): + strength_fits[i] += 1 + dominating_inds[j].append(i) + elif ind_j.fitness.dominates(ind_i.fitness): + strength_fits[j] += 1 + dominating_inds[i].append(j) + + for i in range(inds_len): + for j in dominating_inds[i]: + fits[i] += strength_fits[j] + + # Choose all non-dominated individuals + chosen_indices = [i for i in range(inds_len) if fits[i] < 1] + + if len(chosen_indices) < pop_size: # The archive is too small + for i in range(inds_len): + distances = [0.0] * inds_len + for j in range(i + 1, inds_len): + dist = 0.0 + for idx in range(fitness_len): + val = \ + individuals[i].fitness.values[idx] - \ + individuals[j].fitness.values[idx] + dist += val * val + distances[j] = dist + kth_dist = _randomized_select(distances, 0, inds_len - 1, inds_len_sqrt) + density = 1.0 / (kth_dist + 2.0) + fits[i] += density + + next_indices = [(fits[i], i) for i in range(inds_len) + if i not in chosen_indices] + next_indices.sort() + # print next_indices + chosen_indices += [i for _, i in next_indices[:pop_size - len(chosen_indices)]] + + elif len(chosen_indices) > pop_size: # The archive is too large + inds_len = len(chosen_indices) + distances = [[0.0] * inds_len for _ in range(inds_len)] + sorted_indices = [[0] * inds_len for _ in range(inds_len)] + for i in range(inds_len): + for j in range(i + 1, inds_len): + dist = 0.0 + for idx in range(fitness_len): + val = \ + individuals[chosen_indices[i]].fitness.values[idx] - \ + individuals[chosen_indices[j]].fitness.values[idx] + dist += val * val + distances[i][j] = dist + distances[j][i] = dist + distances[i][i] = -1 + + # Insert sort is faster than quick sort for short arrays + for i in range(inds_len): + for j in range(1, inds_len): + idx = j + while idx > 0 and distances[i][j] < distances[i][sorted_indices[i][idx - 1]]: + sorted_indices[i][idx] = sorted_indices[i][idx - 1] + idx -= 1 + sorted_indices[i][idx] = j + + size = inds_len + to_remove = [] + while size > pop_size: + # Search for minimal distance + min_pos = 0 + for i in range(1, inds_len): + for j in range(1, size): + dist_i_sorted_j = distances[i][sorted_indices[i][j]] + dist_min_sorted_j = distances[min_pos][sorted_indices[min_pos][j]] + + if dist_i_sorted_j < dist_min_sorted_j: + min_pos = i + break + elif dist_i_sorted_j > dist_min_sorted_j: + break + + # Remove minimal distance from sorted_indices + for i in range(inds_len): + distances[i][min_pos] = float("inf") + distances[min_pos][i] = float("inf") + + for j in range(1, size - 1): + if sorted_indices[i][j] == min_pos: + sorted_indices[i][j] = sorted_indices[i][j + 1] + sorted_indices[i][j + 1] = min_pos + + # Remove corresponding individual from chosen_indices + to_remove.append(min_pos) + size -= 1 + + for index in reversed(sorted(to_remove)): + del chosen_indices[index] + + return [individuals[i] for i in chosen_indices] + + +# Auxiliary algorithmic functions for spea2_selection +# This code is a part of DEAP library (Library URL: https://github.com/DEAP/deap). +def _randomized_select(array: List[float], begin: int, end: int, i: float) -> float: + """ + Allows to select the ith smallest element from array without sorting it. + Runtime is expected to be O(n). + """ + if begin == end: + return array[begin] + q = _randomized_partition(array, begin, end) + k = q - begin + 1 + if i < k: + return _randomized_select(array, begin, q, i) + else: + return _randomized_select(array, q + 1, end, i - k) + + +def _randomized_partition(array: List[float], begin: int, end: int) -> int: + i = randint(begin, end) + array[begin], array[i] = array[i], array[begin] + return _partition(array, begin, end) + + +def _partition(array: List[float], begin: int, end: int) -> int: + x = array[begin] + i = begin - 1 + j = end + 1 + while True: + j -= 1 + while array[j] > x: + j -= 1 + i += 1 + while array[i] < x: + i += 1 + if i < j: + array[i], array[j] = array[j], array[i] + else: + return j diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/parameters/__init__.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/parameters/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/parameters/graph_depth.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/parameters/graph_depth.py new file mode 100644 index 0000000..03dc78e --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/parameters/graph_depth.py @@ -0,0 +1,35 @@ +from golem.core.optimisers.archive.generation_keeper import ImprovementWatcher +from golem.core.optimisers.genetic.operators.operator import PopulationT +from golem.core.optimisers.genetic.parameters.parameter import AdaptiveParameter + + +class AdaptiveGraphDepth(AdaptiveParameter[int]): + """Adaptive graph depth parameter. Max allowed graph depth changes + during optimisation depending on observed improvements in the population. + If there are no improvements, then graph depth is incremented. + If there is an improvement, then graph depth remains the same. + Can also play a role of static value if :param adaptive: is False.""" + + def __init__(self, improvements: ImprovementWatcher, + start_depth: int = 1, max_depth: int = 10, + max_stagnation_gens: int = 1, + adaptive: bool = True): + self._improvements = improvements + self._start_depth = start_depth + self._max_depth = max_depth + self._current_depth = start_depth + self._max_stagnation_gens = max_stagnation_gens + self._adaptive = adaptive + + @property + def initial(self) -> int: + return self._start_depth + + def next(self, population: PopulationT = None) -> int: + if not self._adaptive: + return self._max_depth + if self._current_depth >= self._max_depth: + return self._current_depth + if self._improvements.stagnation_iter_count >= self._max_stagnation_gens: + self._current_depth += 1 + return self._current_depth diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/parameters/mutation_prob.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/parameters/mutation_prob.py new file mode 100644 index 0000000..fc8a209 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/parameters/mutation_prob.py @@ -0,0 +1,36 @@ +import numpy as np + +from golem.core.optimisers.genetic.operators.operator import PopulationT +from golem.core.optimisers.genetic.parameters.parameter import AdaptiveParameter + + +class AdaptiveMutationProb(AdaptiveParameter[float]): + + def __init__(self, default_prob: float = 0.5): + self._current_std = 0. + self._max_std = 0. + self._min_proba = 0.1 + self._default_prob = default_prob + + @property + def initial(self) -> float: + return self._default_prob + + def next(self, population: PopulationT) -> float: + self._update_std(population) + + if len(population) < 2: + mutation_prob = 1.0 + elif self._max_std == 0: + mutation_prob = self._default_prob + else: + mutation_prob = max(1. - (self._current_std / self._max_std), self._min_proba) + return mutation_prob + + def _update_std(self, population: PopulationT): + self._current_std = self._calc_std(population) + self._max_std = max(self._current_std, self._max_std) + + @staticmethod + def _calc_std(population: PopulationT) -> float: + return float(np.std([ind.fitness.value for ind in population])) diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/parameters/operators_prob.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/parameters/operators_prob.py new file mode 100644 index 0000000..b0fc2dc --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/parameters/operators_prob.py @@ -0,0 +1,46 @@ +from typing import Tuple + +from golem.core.optimisers.genetic.gp_params import GPAlgorithmParameters +from golem.core.optimisers.genetic.operators.inheritance import GeneticSchemeTypesEnum +from golem.core.optimisers.genetic.operators.operator import PopulationT +from golem.core.optimisers.genetic.parameters.mutation_prob import AdaptiveMutationProb +from golem.core.optimisers.genetic.parameters.parameter import AdaptiveParameter, ConstParameter + +VariationOperatorProb = AdaptiveParameter[Tuple[float, float]] + + +class AdaptiveVariationProb(VariationOperatorProb): + """Adaptive parameter for variation operators. + Specifies mutation and crossover probabilities.""" + + def __init__(self, mutation_prob: float = 0.5, crossover_prob: float = 0.5): + self._mutation_prob_param = AdaptiveMutationProb(mutation_prob) + self._mutation_prob = self._mutation_prob_param.initial + self._crossover_prob_init = crossover_prob + self._crossover_prob = crossover_prob + + @property + def mutation_prob(self): + return self._mutation_prob + + @property + def crossover_prob(self): + return self._crossover_prob + + @property + def initial(self) -> Tuple[float, float]: + return self._mutation_prob_param.initial, self._crossover_prob_init + + def next(self, population: PopulationT) -> Tuple[float, float]: + self._mutation_prob = self._mutation_prob_param.next(population) + self._crossover_prob = 1. - self._mutation_prob + return self._mutation_prob, self._crossover_prob + + +def init_adaptive_operators_prob(parameters: GPAlgorithmParameters) -> VariationOperatorProb: + """Returns static or adaptive parameter for mutation & crossover probabilities depending on genetic type scheme.""" + if parameters.genetic_scheme_type == GeneticSchemeTypesEnum.parameter_free: + operators_prob = AdaptiveVariationProb() + else: + operators_prob = ConstParameter((parameters.mutation_prob, parameters.crossover_prob)) + return operators_prob diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/parameters/parameter.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/parameters/parameter.py new file mode 100644 index 0000000..2f16043 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/parameters/parameter.py @@ -0,0 +1,36 @@ +from abc import ABC, abstractmethod +from typing import Generic, TypeVar + +from golem.core.optimisers.genetic.operators.operator import PopulationT + +T = TypeVar('T') + + +class AdaptiveParameter(ABC, Generic[T]): + """Abstract interface for parameters of evolutionary algorithm and its Operators. + Adaptive parameter is defined by `initial` value and subsequent `next` values. + Parameter can potentially change on each call to next(). + The specific policy of adaptivity is determined by implementations.""" + + @property + @abstractmethod + def initial(self) -> T: + raise NotImplementedError() + + @abstractmethod + def next(self, population: PopulationT) -> T: + raise NotImplementedError() + + +class ConstParameter(AdaptiveParameter[T]): + """Stub implementation of AdaptiveParameter for constant parameters.""" + + def __init__(self, value: T): + self._value = value + + @property + def initial(self) -> T: + return self._value + + def next(self, population: PopulationT) -> T: + return self._value diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/parameters/population_size.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/parameters/population_size.py new file mode 100644 index 0000000..275f058 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/genetic/parameters/population_size.py @@ -0,0 +1,93 @@ +import math +from typing import Optional + +from golem.core.constants import MIN_POP_SIZE +from golem.core.optimisers.archive.generation_keeper import ImprovementWatcher +from golem.core.optimisers.genetic.gp_params import GPAlgorithmParameters +from golem.core.optimisers.genetic.operators.inheritance import GeneticSchemeTypesEnum +from golem.core.optimisers.genetic.operators.operator import PopulationT +from golem.core.optimisers.genetic.parameters.parameter import AdaptiveParameter +from golem.utilities.data_structures import BidirectionalIterator +from golem.utilities.sequence_iterator import fibonacci_sequence, SequenceIterator + +PopulationSize = AdaptiveParameter[int] + + +class ConstRatePopulationSize(PopulationSize): + def __init__(self, pop_size: int, offspring_rate: float, max_pop_size: Optional[int] = None): + self._offspring_rate = offspring_rate + self._initial = pop_size + self._max_pop_size = max_pop_size + + @property + def initial(self) -> int: + return self._initial + + def next(self, population: PopulationT) -> int: + # to prevent stagnation + pop_size = max(len(population), self._initial) + if not self._max_pop_size or pop_size < self._max_pop_size: + pop_size += math.ceil(pop_size * self._offspring_rate) + if self._max_pop_size: + pop_size = min(pop_size, self._max_pop_size) + return pop_size + + +class AdaptivePopulationSize(PopulationSize): + def __init__(self, + improvement_watcher: ImprovementWatcher, + progression_iterator: BidirectionalIterator[int], + max_pop_size: Optional[int] = None): + self._improvements = improvement_watcher + self._iterator = progression_iterator + self._max_pop_size = max_pop_size + self._initial = self._iterator.next() if self._iterator.has_next() else self._iterator.prev() + + @property + def initial(self) -> int: + return self._initial + + def next(self, population: PopulationT) -> int: + pop_size = len(population) + too_many_fitness_eval_errors = pop_size / self._iterator.current() < 0.5 + + if too_many_fitness_eval_errors or not self._improvements.is_any_improved: + if self._iterator.has_next(): + pop_size = self._iterator.next() + elif self._improvements.is_quality_improved and self._improvements.is_complexity_improved and pop_size > 0: + if self._iterator.has_prev(): + pop_size = self._iterator.prev() + + pop_size = max(pop_size, MIN_POP_SIZE) + if self._max_pop_size: + pop_size = min(pop_size, self._max_pop_size) + + return pop_size + + +def init_adaptive_pop_size(requirements: GPAlgorithmParameters, + improvement_watcher: ImprovementWatcher) -> PopulationSize: + genetic_scheme_type = requirements.genetic_scheme_type + if genetic_scheme_type == GeneticSchemeTypesEnum.steady_state: + pop_size = ConstRatePopulationSize( + pop_size=requirements.pop_size, + offspring_rate=1.0, + max_pop_size=requirements.max_pop_size, + ) + elif genetic_scheme_type == GeneticSchemeTypesEnum.generational: + pop_size = ConstRatePopulationSize( + pop_size=requirements.pop_size, + offspring_rate=requirements.offspring_rate, + max_pop_size=requirements.max_pop_size, + ) + elif genetic_scheme_type == GeneticSchemeTypesEnum.parameter_free: + pop_size_progression = SequenceIterator(sequence_func=fibonacci_sequence, + start_value=requirements.pop_size, + min_sequence_value=1, + max_sequence_value=requirements.max_pop_size) + pop_size = AdaptivePopulationSize(improvement_watcher=improvement_watcher, + progression_iterator=pop_size_progression, + max_pop_size=requirements.max_pop_size) + else: + raise ValueError(f"Unknown genetic type scheme {genetic_scheme_type}") + return pop_size diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/graph.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/graph.py new file mode 100644 index 0000000..b6e862f --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/graph.py @@ -0,0 +1,5 @@ +from golem.core.dag.graph_delegate import GraphDelegate +from golem.core.dag.linked_graph_node import LinkedGraphNode + +OptNode = LinkedGraphNode +OptGraph = GraphDelegate diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/graph_builder.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/graph_builder.py new file mode 100644 index 0000000..6c0a0ff --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/graph_builder.py @@ -0,0 +1,136 @@ +from copy import deepcopy +from typing import Union, Tuple, Optional, List, Dict + +from golem.core.adapter.adapter import DomainStructureType +from golem.core.dag.graph_node import GraphNode + + +class GraphBuilder: + """ Builder for incremental construction of directed acyclic graphs. + Semantics: + - Forward-only & addition-only (can't prepend or delete nodes). + - Doesn't throw, doesn't fail: methods always have a way to interpret input given current graph state. + - Is not responsible for the validity of resulting graph (e.g. correct order, valid operations). + - Builds always new graphs (on copies of nodes), preserves its state between builds. State doesn't leak outside. + """ + + OperationType = Union[str, Tuple[str, dict]] + + def __init__(self, graph_adapter: Optional[DomainStructureType] = None, *initial_nodes: Optional[GraphNode]): + """ + Create builder with prebuilt nodes as origins of the branches. + + Args: + graph_adapter: adapter to adapt built graph to particular graph type. + initial_nodes: variadic positional argument with initial graph nodes + """ + self.graph_adapter = graph_adapter + self.heads: List[GraphNode] = list(filter(None, initial_nodes)) + + @property + def _iend(self) -> int: + return len(self.heads) + + def reset(self): + """ Reset builder state. """ + self.heads = [] + + def to_nodes(self) -> List[GraphNode]: + """ + Return list of final nodes and reset internal state. + :return: list of final nodes, possibly empty. + """ + return deepcopy(self.heads) + + def add_node(self, operation_type: Optional[str], branch_idx: int = 0, params: Optional[Dict] = None): + """ Add single node to graph branch of specified index. + If there are no heads => adds single Node. + If there is single head => adds single Node using head as input. + If there are several heads => adds single Node using as input the head indexed by branch_idx. + If input is None => do nothing. + If branch_idx is out of bounds => appends new Node. + + :param operation_type: new operation, possibly None + :param branch_idx: index of the head to use as input for the new node + :param params: parameters dictionary for the specific operation + :return: self + """ + raise NotImplementedError() + + def add_sequence(self, *operation_type: OperationType, branch_idx: int = 0): + """ Same as .node() but for many operations at once. + + :param operation_type: operations for new nodes, either as an operation name + or as a tuple of operation name and operation parameters. + :param branch_idx: index of the branch for branching its tip + """ + raise NotImplementedError() + + def grow_branches(self, *operation_type: Optional[OperationType]): + """ Add single node to each branch. + + Argument position means index of the branch to grow. + None operation means don't grow that branch. + If there are no nodes => creates new branches. + If number of input nodes is bigger than number of branches => extra operations create new branches. + + :param operation_type: operations for new nodes, either as an operation name + or as a tuple of operation name and operation parameters. + :return: self + """ + raise NotImplementedError() + + def add_branch(self, *operation_type: Optional[OperationType], branch_idx: int = 0): + """ Create branches at the tip of branch with branch_idx. + + None operations are filtered out. + Number of new branches equals to number of provided operations. + If there are no heads => will add several nodes. + If there is single head => add several nodes using head as the previous. + If there are several heads => branch head indexed by branch_idx. + If branch_idx is out of bounds => adds nodes as new heads at the end. + If no not-None operations are provided, nothing is changed. + + :param operation_type: operations for new nodes, either as an operation name + or as a tuple of operation name and operation parameters. + :param branch_idx: index of the branch for branching its tip + :return: self + """ + raise NotImplementedError() + + def add_skip_connection_edge(self, branch_idx_first: int, branch_idx_second: int, + node_idx_in_branch_first: int, node_idx_in_branch_second: int): + """ Joins two nodes which are not placed sequential in one branch. + Can be used only in the very end of graph building but before 'join_branches'. + Edge is directed from the first node to the second. + + :param branch_idx_first: index of the first branch which to take the first node + :param branch_idx_second: index of the second branch which to take the second node + :param node_idx_in_branch_first: index of the node in its branch + :param node_idx_in_branch_second: index of the node in its branch + """ + raise NotImplementedError() + + def join_branches(self, operation_type: Optional[str], params: Optional[Dict] = None): + """ Joins all current branches with provided operation as ensemble node. + + If there are no branches => does nothing. + If there is single branch => adds single node using it as input. + If there are several branches => adds single node using all heads as inputs. + + :param operation_type: operation to use for joined node + :param params: parameters dictionary for the specific operation + :return: self + """ + raise NotImplementedError() + + @staticmethod + def _unpack_operation(operation: Optional[OperationType]) -> Tuple[Optional[str], Optional[Dict]]: + if isinstance(operation, str) or operation is None: + return operation, None + else: + return operation + + @staticmethod + def _pack_params(name: str, params: Optional[dict]) -> Optional[dict]: + return {'name': name, 'params': params} if params else {'name': name} diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/initial_graphs_generator.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/initial_graphs_generator.py new file mode 100644 index 0000000..2ec8fbd --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/initial_graphs_generator.py @@ -0,0 +1,72 @@ +from functools import partial +from typing import Callable, Optional, Sequence, Union, Iterable + +from golem.core.optimisers.optimization_parameters import GraphRequirements +from golem.core.constants import MAX_GRAPH_GEN_ATTEMPTS +from golem.core.dag.graph import Graph +from golem.core.log import default_log +from golem.core.optimisers.graph import OptGraph +from golem.core.optimisers.optimizer import GraphGenerationParams + +GenerationFunction = Callable[[], Graph] +InitialGraphsGenerator = Callable[[], Sequence[OptGraph]] + + +class InitialPopulationGenerator(InitialGraphsGenerator): + """Generates initial population using three approaches. + One is with initial graphs. + Another is with initial graphs generation function which generates a graph + that will be added to initial population. + The third way is random graphs generation according to GraphGenerationParameters and OptimizationParameters. + The last approach is applied when neither initial graphs nor initial graphs generation function were provided.""" + + def __init__(self, + population_size: int, + generation_params: GraphGenerationParams, + requirements: GraphRequirements): + self.pop_size = population_size + self.requirements = requirements + self.graph_generation_params = generation_params + self.generation_function: Optional[GenerationFunction] = None + self.initial_graphs: Optional[Sequence[Graph]] = None + self.log = default_log(self) + + def __call__(self) -> Sequence[OptGraph]: + pop_size = self.pop_size + adapter = self.graph_generation_params.adapter + + if self.initial_graphs: + if len(self.initial_graphs) > pop_size: + self.initial_graphs = self.initial_graphs[:pop_size] + return adapter.adapt(self.initial_graphs) + + if not self.generation_function: + self.generation_function = partial(self.graph_generation_params.random_graph_factory, self.requirements) + + population = [] + for iter_num in range(MAX_GRAPH_GEN_ATTEMPTS): + if len(population) == pop_size: + break + new_graph = self.generation_function() + if new_graph not in population and self.graph_generation_params.verifier(new_graph): + population.append(new_graph) + else: + self.log.warning(f'Exceeded max number of attempts for generating initial graphs, stopping.' + f'Generated {len(population)} instead of {pop_size} graphs.') + return population + + def with_initial_graphs(self, initial_graphs: Union[Graph, Sequence[Graph]]): + """Use initial graphs as initial population.""" + if isinstance(initial_graphs, Graph): + self.initial_graphs = [initial_graphs] + elif isinstance(initial_graphs, Iterable): + self.initial_graphs = list(initial_graphs) + else: + raise ValueError(f'Incorrect type of initial_assumption: ' + f'Sequence[Graph] or Graph needed, but has {type(initial_graphs)}') + return self + + def with_custom_generation_function(self, generation_func: GenerationFunction): + """Use custom graph generation function to create initial population.""" + self.generation_function = generation_func + return self diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/meta/__init__.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/meta/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/meta/surrogate_evaluator.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/meta/surrogate_evaluator.py new file mode 100644 index 0000000..cf7a70c --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/meta/surrogate_evaluator.py @@ -0,0 +1,50 @@ +import pathlib +import timeit +from datetime import datetime +from typing import Optional, Tuple + +from golem.core.adapter import BaseOptimizationAdapter +from golem.core.log import Log +from golem.core.optimisers.genetic.evaluation import DelegateEvaluator, SequentialDispatcher +from golem.core.optimisers.graph import OptGraph +from golem.core.optimisers.meta.surrogate_model import SurrogateModel, RandomValuesSurrogateModel +from golem.core.optimisers.objective.objective import to_fitness, GraphFunction +from golem.core.optimisers.opt_history_objects.individual import GraphEvalResult + + +class SurrogateDispatcher(SequentialDispatcher): + """Evaluates objective function with surrogate model. + Usage: call `dispatch(objective_function)` to get evaluation function. + Additionally, we need to pass surrogate_model object + """ + + def __init__(self, + adapter: BaseOptimizationAdapter, + n_jobs: int = 1, + graph_cleanup_fn: Optional[GraphFunction] = None, + delegate_evaluator: Optional[DelegateEvaluator] = None, + surrogate_model: SurrogateModel = RandomValuesSurrogateModel()): + super().__init__(adapter, n_jobs, graph_cleanup_fn, delegate_evaluator) + self._n_jobs = 1 + self.surrogate_model = surrogate_model + + def evaluate_single(self, graph: OptGraph, uid_of_individual: str, with_time_limit: bool = True, + cache_key: Optional[str] = None, + logs_initializer: Optional[Tuple[int, pathlib.Path]] = None) -> GraphEvalResult: + graph = self.evaluation_cache.get(cache_key, graph) + if logs_initializer is not None: + # in case of multiprocessing run + Log.setup_in_mp(*logs_initializer) + + start_time = timeit.default_timer() + fitness = to_fitness(self.surrogate_model(graph, objective=self._objective_eval)) + end_time = timeit.default_timer() + + eval_res = GraphEvalResult( + uid_of_individual=uid_of_individual, fitness=fitness, graph=graph, metadata={ + 'computation_time_in_seconds': end_time - start_time, + 'evaluation_time_iso': datetime.now().isoformat(), + 'surrogate_evaluation': True + } + ) + return eval_res diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/meta/surrogate_model.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/meta/surrogate_model.py new file mode 100644 index 0000000..4d3dd97 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/meta/surrogate_model.py @@ -0,0 +1,23 @@ +from abc import abstractmethod +from typing import Any + +import numpy as np + +from golem.core.dag.graph import Graph + + +class SurrogateModel: + """ + Model for evaluating fitness function without time-consuming fitting pipeline + """ + @abstractmethod + def __call__(self, graph: Graph, **kwargs: Any): + raise NotImplementedError() + + +class RandomValuesSurrogateModel(SurrogateModel): + """ + Model for evaluating fitness function based on returning random values for any model + """ + def __call__(self, graph: Graph, **kwargs: Any): + return np.random.random(1) diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/meta/surrogate_optimizer.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/meta/surrogate_optimizer.py new file mode 100644 index 0000000..0b1b40d --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/meta/surrogate_optimizer.py @@ -0,0 +1,58 @@ +from typing import Sequence + +from golem.core.optimisers.genetic.gp_optimizer import EvoGraphOptimizer +from golem.core.optimisers.genetic.gp_params import GPAlgorithmParameters +from golem.core.optimisers.graph import OptGraph +from golem.core.optimisers.meta.surrogate_evaluator import SurrogateDispatcher +from golem.core.optimisers.meta.surrogate_model import RandomValuesSurrogateModel +from golem.core.optimisers.objective import Objective, ObjectiveFunction +from golem.core.optimisers.optimization_parameters import GraphRequirements +from golem.core.optimisers.optimizer import GraphGenerationParams +from golem.core.optimisers.populational_optimizer import EvaluationAttemptsError, _try_unfit_graph + + +class SurrogateEachNgenOptimizer(EvoGraphOptimizer): + """ + Surrogate optimizer that uses surrogate model for evaluating part of individuals + + Additionally, we need to pass surrogate_model object + """ + def __init__(self, + objective: Objective, + initial_graphs: Sequence[OptGraph], + requirements: GraphRequirements, + graph_generation_params: GraphGenerationParams, + graph_optimizer_params: GPAlgorithmParameters, + surrogate_model=RandomValuesSurrogateModel(), + surrogate_each_n_gen=5 + ): + super().__init__(objective, initial_graphs, requirements, graph_generation_params, graph_optimizer_params) + self.surrogate_model = surrogate_model + self.surrogate_each_n_gen = surrogate_each_n_gen + self.surrogate_dispatcher = SurrogateDispatcher(adapter=graph_generation_params.adapter, + n_jobs=requirements.n_jobs, + graph_cleanup_fn=_try_unfit_graph, + delegate_evaluator=graph_generation_params.remote_evaluator, + surrogate_model=surrogate_model) + + def optimise(self, objective: ObjectiveFunction) -> Sequence[OptGraph]: + # eval_dispatcher defines how to evaluate objective on the whole population + evaluator = self.eval_dispatcher.dispatch(objective, self.timer) + # surrogate_dispatcher defines how to evaluate objective with surrogate model + surrogate_evaluator = self.surrogate_dispatcher.dispatch(objective, self.timer) + + with self.timer, self._progressbar: + self._initial_population(evaluator) + while not self.stop_optimization(): + try: + if self.generations.generation_num % self.surrogate_each_n_gen == 0: + new_population = self._evolve_population(surrogate_evaluator) + else: + new_population = self._evolve_population(evaluator) + except EvaluationAttemptsError as ex: + self.log.warning(f'Composition process was stopped due to: {ex}') + break + # Adding of new population to history + self._update_population(new_population) + self._update_population(self.best_individuals, 'final_choices') + return [ind.graph for ind in self.best_individuals] diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/objective/__init__.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/objective/__init__.py new file mode 100644 index 0000000..60beaa5 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/objective/__init__.py @@ -0,0 +1,2 @@ +from .objective import Objective, GraphFunction, ObjectiveFunction +from .objective_eval import ObjectiveEvaluate diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/objective/objective.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/objective/objective.py new file mode 100644 index 0000000..4895a06 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/objective/objective.py @@ -0,0 +1,90 @@ +import itertools +from dataclasses import dataclass +from numbers import Real +from typing import Any, Optional, Callable, Sequence, TypeVar, Dict, Tuple, Union, Protocol + +from golem.core.dag.graph import Graph +from golem.core.log import default_log +from golem.core.optimisers.fitness import Fitness, SingleObjFitness, null_fitness, MultiObjFitness + +G = TypeVar('G', bound=Graph, contravariant=True) +R = TypeVar('R', covariant=True) + + +class GraphFunction(Protocol[G, R]): + def __call__(self, graph: G) -> R: + ... + + +ObjectiveFunction = GraphFunction[G, Fitness] + + +@dataclass +class ObjectiveInfo: + """Keeps information about used metrics.""" + is_multi_objective: bool = False + metric_names: Sequence[str] = () + + def __str__(self): + return f'{self.__class__.__name__}(multi={self.is_multi_objective}, metrics={self.metric_names})' + + def format_fitness(self, fitness: Union[Fitness, Sequence[float]]) -> str: + """Returns formatted fitness string. + Example for 3 metrics: ``""" + values = fitness.values if isinstance(fitness, Fitness) else fitness + fitness_info_str = [f'{name}={value:.3f}' + if value is not None + else f'{name}=None' + for name, value in zip(self.metric_names, values)] + return f"<{' '.join(fitness_info_str)}>" + + +class Objective(ObjectiveInfo, ObjectiveFunction): + """Represents objective function for computing metric values + on Graphs and keeps information about metrics used.""" + + def __init__(self, + quality_metrics: Union[Callable, Dict[Any, Callable]], + complexity_metrics: Optional[Dict[Any, Callable]] = None, + is_multi_objective: bool = False, + ): + self._log = default_log(self) + if isinstance(quality_metrics, Callable): + quality_metrics = {'metric': quality_metrics} + self.quality_metrics = quality_metrics + self.complexity_metrics = complexity_metrics or {} + metric_names = [str(metric_id) for metric_id, _ in self.metrics] + ObjectiveInfo.__init__(self, is_multi_objective, metric_names) + + def __call__(self, graph: Graph, **metrics_kwargs: Any) -> Fitness: + evaluated_metrics = [] + for metric_id, metric_func in self.metrics: + try: + metric_value = metric_func(graph, **metrics_kwargs) + evaluated_metrics.append(metric_value) + except Exception as ex: + self._log.error(f'Objective evaluation error for graph {graph} on metric {metric_id}: {ex}') + return null_fitness() # fail right away + return to_fitness(evaluated_metrics, self.is_multi_objective) + + @property + def metrics(self) -> Sequence[Tuple[Any, Callable]]: + return list(itertools.chain(self.quality_metrics.items(), self.complexity_metrics.items())) + + def get_info(self) -> ObjectiveInfo: + return ObjectiveInfo(self.is_multi_objective, self.metric_names) + + +def to_fitness(metric_values: Optional[Sequence[Real]], multi_objective: bool = False) -> Fitness: + if metric_values is None: + return null_fitness() + elif multi_objective: + return MultiObjFitness(values=metric_values, weights=1.) + else: + return SingleObjFitness(*metric_values) + + +def get_metric_position(metrics, metric_type): + for num, metric in enumerate(metrics): + if isinstance(metric, metric_type): + return num diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/objective/objective_eval.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/objective/objective_eval.py new file mode 100644 index 0000000..c62410c --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/objective/objective_eval.py @@ -0,0 +1,49 @@ +from abc import ABC +from typing import TypeVar, Generic + +from golem.core.dag.graph import Graph +from golem.core.optimisers.fitness import Fitness +from .objective import Objective + +G = TypeVar('G', bound=Graph, covariant=True) + + +class ObjectiveEvaluate(ABC, Generic[G]): + """Defines how Objective must be evaluated on Graphs. + + Responsibilities: + - Graph-specific evaluation policy: typically, Graphs require some kind of evaluation + before Objective could be estimated on them. E.g. Machine-learning pipelines must be + fit on train data before they could be evaluated on the test data. + - Objective-specific estimation: typically objectives require additional parameters + besides Graphs for estimation, e.g. test data for estimation of prediction quality. + - Optionally, compute additional statistics for Graphs (intermediate metrics). + + Default implementation is just a closure that calls :param objective: with + redirected keyword arguments :param objective_kwargs: + """ + + def __init__(self, objective: Objective, eval_n_jobs: int = 1, **objective_kwargs): + self._objective = objective + self._objective_kwargs = objective_kwargs + self._eval_n_jobs = eval_n_jobs + + def __call__(self, graph: G) -> Fitness: + """Provides functional interface for ObjectiveEvaluate.""" + return self.evaluate(graph) + + @property + def eval_n_jobs(self) -> int: + return self._eval_n_jobs + + @eval_n_jobs.setter + def eval_n_jobs(self, n_jobs: int): + self._eval_n_jobs = n_jobs + + def evaluate(self, graph: G) -> Fitness: + """Evaluate graph and compute its fitness.""" + return self._objective(graph, **self._objective_kwargs) + + def evaluate_intermediate_metrics(self, graph: G): + """Compute intermediate metrics for each graph node and store it there.""" + pass diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/opt_graph_builder.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/opt_graph_builder.py new file mode 100644 index 0000000..c4472cf --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/opt_graph_builder.py @@ -0,0 +1,238 @@ +from inspect import signature +from typing import Union, Tuple, Optional, Dict + +from golem.core.adapter import BaseOptimizationAdapter +from golem.core.adapter.adapter import DomainStructureType +from golem.core.dag.linked_graph import LinkedGraph +from golem.core.optimisers.graph import OptNode, OptGraph +from golem.core.optimisers.graph_builder import GraphBuilder + + +class OptGraphBuilder(GraphBuilder): + """ Builder for incremental construction of directed acyclic OptGraphs. + Semantics: + - Forward-only & addition-only (can't prepend or delete nodes). + - Doesn't throw, doesn't fail: methods always have a way to interpret input given current graph state. + - Is not responsible for the validity of resulting OptGraph (e.g. correct order, valid operations). + - Builds always new graphs (on copies of nodes), preserves its state between builds. State doesn't leak outside. + """ + + OperationType = Union[str, Tuple[str, dict]] + + def __init__(self, graph_adapter: Optional[BaseOptimizationAdapter] = None, *initial_nodes: Optional[OptNode]): + super().__init__(graph_adapter, *initial_nodes) + + def add_node(self, operation_type: Optional[str], branch_idx: int = 0, params: Optional[Dict] = None): + """ Add single node to graph branch of specified index. + If there are no heads => adds single OptNode. + If there is single head => adds single OptNode using head as input. + If there are several heads => adds single OptNode using as input the head indexed by branch_idx. + If input is None => do nothing. + If branch_idx is out of bounds => appends new OptNode. + + :param operation_type: new operation, possibly None + :param branch_idx: index of the head to use as input for the new node + :param params: parameters dictionary for the specific operation + :return: self + """ + if operation_type is None: + return self + params = self._pack_params(operation_type, params) + if branch_idx < len(self.heads): + input_node = self.heads[branch_idx] + self.heads[branch_idx] = OptNode(content=params, nodes_from=[input_node]) + else: + self.heads.append(OptNode(content=params)) + return self + + def add_sequence(self, *operation_type: OperationType, branch_idx: int = 0): + """ Same as .node() but for many operations at once. + + :param operation_type: operations for new nodes, either as an operation name + or as a tuple of operation name and operation parameters. + :param branch_idx: index of the branch for branching its tip + """ + for operation in operation_type: + operation, params = self._unpack_operation(operation) + self.add_node(operation, branch_idx, params) + return self + + def grow_branches(self, *operation_type: Optional[OperationType]): + """ Add single node to each branch. + + Argument position means index of the branch to grow. + None operation means don't grow that branch. + If there are no nodes => creates new branches. + If number of input nodes is bigger than number of branches => extra operations create new branches. + + :param operation_type: operations for new nodes, either as an operation name + or as a tuple of operation name and operation parameters. + :return: self + """ + for i, operation in enumerate(operation_type): + operation, params = self._unpack_operation(operation) + self.add_node(operation, i, params) + return self + + def add_branch(self, *operation_type: Optional[OperationType], branch_idx: int = 0): + """ Create branches at the tip of branch with branch_idx. + + None operations are filtered out. + Number of new branches equals to number of provided operations. + If there are no heads => will add several nodes. + If there is single head => add several nodes using head as the previous. + If there are several heads => branch head indexed by branch_idx. + If branch_idx is out of bounds => adds nodes as new heads at the end. + If no not-None operations are provided, nothing is changed. + + :param operation_type: operations for new nodes, either as an operation name + or as a tuple of operation name and operation parameters. + :param branch_idx: index of the branch for branching its tip + :return: self + """ + operations = list(filter(None, operation_type)) + if not operations: + return self + if branch_idx < len(self.heads): + input_node = self.heads.pop(branch_idx) + for i, operation in enumerate(operations): + operation, params = self._unpack_operation(operation) + self.heads.insert(branch_idx + i, OptNode(content=self._pack_params(operation, params), + nodes_from=[input_node])) + else: + for operation in operations: + operation, params = self._unpack_operation(operation) + self.add_node(operation, self._iend, params) + return self + + def add_skip_connection_edge(self, branch_idx_first: int, branch_idx_second: int, + node_idx_in_branch_first: int, node_idx_in_branch_second: int): + """ Joins two nodes which are not placed sequential in one branch. + Edge is directed from the first node to the second. + + :param branch_idx_first: index of the first branch which to take the first node + :param branch_idx_second: index of the second branch which to take the second node + :param node_idx_in_branch_first: index of the node in its branch + :param node_idx_in_branch_second: index of the node in its branch + """ + if branch_idx_first >= len(self.heads) or branch_idx_second >= len(self.heads): + return + first_node = self._get_node_from_branch_with_idx(branch_idx=branch_idx_first, + node_idx_in_branch=node_idx_in_branch_first) + second_node = self._get_node_from_branch_with_idx(branch_idx=branch_idx_second, + node_idx_in_branch=node_idx_in_branch_second) + # to avoid cyclic graphs + if second_node not in first_node.nodes_from and first_node not in second_node.nodes_from: + second_node.nodes_from.append(first_node) + + return self + + def _get_node_from_branch_with_idx(self, branch_idx: int, node_idx_in_branch: int): + head_node = self.heads[branch_idx] + branch_graph = OptGraph(head_node) + return branch_graph.nodes[node_idx_in_branch] + + def join_branches(self, operation_type: Optional[str], params: Optional[Dict] = None): + """ Joins all current branches with provided operation as ensemble node. + + If there are no branches => does nothing. + If there is single branch => adds single node using it as input. + If there are several branches => adds single node using all heads as inputs. + + :param operation_type: operation to use for joined node + :param params: parameters dictionary for the specific operation + :return: self + """ + if self.heads and operation_type: + content = self._pack_params(operation_type, params) + new_head = OptNode(content=content, nodes_from=self.heads) + self.heads = [new_head] + return self + + def build(self) -> Optional[DomainStructureType]: + """ Adapt resulted graph to required graph class. """ + if not self.to_nodes(): + return None + result_opt_graph = OptGraph(self.to_nodes()) + if self.graph_adapter: + return self.graph_adapter.restore(result_opt_graph) + return result_opt_graph + + def merge_with(self, following_builder) -> Optional['OptGraphBuilder']: + return merge_opt_graph_builders(self, following_builder) + + +def _merge_adapters(adapter1: Optional[BaseOptimizationAdapter], adapter2: Optional[BaseOptimizationAdapter]) -> \ + Optional[BaseOptimizationAdapter]: + if adapter1 is None: + return adapter2 + elif adapter2 is None: + return adapter1 + merged_dct = {} + for (k, v1), v2 in zip(vars(adapter1).items(), vars(adapter2).values()): + if type(v1) != type(v2): + raise TypeError('adapters have different types of values') + elif isinstance(v1, bool): + merged_dct[k] = v1 and v2 + else: + merged_dct[k] = v2 + true_adapter_type = type(adapter1) + init_params = signature(true_adapter_type.__init__).parameters + init_args = {k: merged_dct[k] for k in merged_dct.keys() & init_params} + merged_adapter = true_adapter_type(**init_args) + cls_fields = { + k: merged_dct[k] + for k in merged_dct.keys() ^ init_params + if k in merged_dct and k in vars(merged_adapter)} + vars(merged_adapter).update(cls_fields) + return merged_adapter + + +def merge_opt_graph_builders(previous: OptGraphBuilder, following: OptGraphBuilder) -> Optional[OptGraphBuilder]: + """ Merge two builders. + + Merging is defined for cases one-to-many and many-to-one nodes, + i.e. one final node to many initial nodes and many final nodes to one initial node. + Merging is undefined for the case of many-to-many nodes and None is returned. + Merging of the builder with itself is well-defined and leads to duplication of the graph. + + If one of the builders is empty -- the other one is returned, no merging is performed. + State of the passed builders is preserved as they were, after merging new builder is returned. + + :return: GraphBuilder if merging is well-defined, None otherwise. + """ + + if not following.heads: + return previous + elif not previous.heads: + return following + + if type(following.graph_adapter) is not type(previous.graph_adapter): + raise ValueError('Adapters do not match: cannot perform merge') + + merged_graph_adapter = _merge_adapters(previous.graph_adapter, following.graph_adapter) + + lhs_nodes_final = previous.to_nodes() + rhs_tmp_graph = LinkedGraph(following.to_nodes()) + rhs_nodes_initial = list(filter(lambda node: not node.nodes_from, rhs_tmp_graph.nodes)) + + # If merging one-to-one or one-to-many + if len(lhs_nodes_final) == 1: + final_node = lhs_nodes_final[0] + for initial_node in rhs_nodes_initial: + rhs_tmp_graph.update_node(initial_node, + OptNode(content={'name': initial_node.name}, nodes_from=[final_node])) + # If merging many-to-one + elif len(rhs_nodes_initial) == 1: + initial_node = rhs_nodes_initial[0] + rhs_tmp_graph.update_node(initial_node, + OptNode(content={'name': initial_node.name}, nodes_from=lhs_nodes_final)) + # Merging is not defined for many-to-many case + else: + return None + + # Check that Graph didn't mess up with node types + if not all(map(lambda n: isinstance(n, OptNode), rhs_tmp_graph.nodes)): + raise ValueError("Expected Graph only with nodes of type 'OptNode'") + merged_builder = OptGraphBuilder(merged_graph_adapter, *rhs_tmp_graph.root_nodes()) + return merged_builder diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/opt_history_objects/__init__.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/opt_history_objects/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/opt_history_objects/generation.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/opt_history_objects/generation.py new file mode 100644 index 0000000..3f385e2 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/opt_history_objects/generation.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +from collections import UserList +from copy import deepcopy, copy +from typing import TYPE_CHECKING, Any, Dict, Iterable, Optional, Union + +from golem.utilities.data_structures import ensure_wrapped_in_sequence + +if TYPE_CHECKING: + from golem.core.optimisers.opt_history_objects.individual import Individual + + +class Generation(UserList): + """ + List of evolution individuals considered as a single generation. + Allows to provide additional information about the generation. + Responsible for setting generation-related info to the individuals. + """ + + def __init__(self, iterable: Union[Iterable[Individual], Generation], generation_num: int, + label: Optional[str] = None, metadata: Optional[Dict[str, Any]] = None): + self.generation_num = generation_num + self.label: str = label or '' + self.metadata: Dict[str, Any] = metadata or {} + super().__init__(iterable) + self._set_native_generation(self.data) + + def __setitem__(self, index, item: Union[Individual, Iterable[Individual]]): + super().__setitem__(index, item) + self._set_native_generation(item) + + def copy(self) -> Generation: + return copy(self) + + def __copy__(self): + cls = self.__class__ + result = cls.__new__(cls) + result.__dict__.update(self.__dict__) + result.data = copy(self.data) + return result + + def __deepcopy__(self, memo): + cls = self.__class__ + result = cls.__new__(cls) + memo[id(self)] = result + for k, v in self.__dict__.items(): + object.__setattr__(result, k, deepcopy(v, memo)) + return result + + def _set_native_generation(self, individuals: Union[Individual, Iterable[Individual]]): + individuals = ensure_wrapped_in_sequence(individuals) + for individual in individuals: + individual.set_native_generation(self.generation_num) + + def __repr__(self): + gen_num = f'Generation {self.generation_num}' + label = f' ({self.label}): ' if self.label else ': ' + data = super().__repr__() + return ''.join([gen_num, label, data]) diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/opt_history_objects/individual.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/opt_history_objects/individual.py new file mode 100644 index 0000000..f43ae66 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/opt_history_objects/individual.py @@ -0,0 +1,142 @@ +from __future__ import annotations + +import os +from copy import deepcopy +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union +from uuid import uuid4 + +from golem.core.dag.graph import Graph +from golem.core.log import default_log +from golem.core.optimisers.fitness.fitness import Fitness, null_fitness +from golem.core.optimisers.graph import OptGraph +from golem.serializers.serializer import default_load, default_save + +if TYPE_CHECKING: + from golem.core.optimisers.opt_history_objects.parent_operator import ParentOperator + + +INDIVIDUAL_COPY_RESTRICTION_MESSAGE = ('`Individual` instance was copied.\n' + 'Normally, you don\'t want to do that to keep uid-individual uniqueness.\n' + 'If this happened during the optimization process, this misusage ' + 'should be fixed.') + + +@dataclass(frozen=True) +class Individual: + graph: Graph + parent_operator: Optional[ParentOperator] = field(default=None) + metadata: Dict[str, Any] = field(default_factory=dict) + native_generation: Optional[int] = None + fitness: Fitness = field(default_factory=null_fitness) + uid: str = field(default_factory=lambda: str(uuid4())) + + def set_native_generation(self, native_generation): + if self.native_generation is None: + super().__setattr__('native_generation', native_generation) + + def _set_fitness_and_graph(self, fitness: Fitness, updated_graph: Optional[OptGraph] = None): + super().__setattr__('fitness', fitness) + if updated_graph is not None: + super().__setattr__('graph', updated_graph) + + def set_evaluation_result(self, eval_result: Union[GraphEvalResult, Fitness], + updated_graph: Optional[OptGraph] = None): + if self.fitness.valid: + raise ValueError('The individual has valid fitness and can not be evaluated again.') + + if isinstance(eval_result, Fitness): + self._set_fitness_and_graph(eval_result, updated_graph) + return + + self._set_fitness_and_graph(eval_result.fitness, eval_result.graph) + self.metadata.update(eval_result.metadata) + + @property + def has_native_generation(self) -> bool: + return self.native_generation is not None + + @property + def parents(self) -> List[Individual]: + if not self.parent_operator: + return [] + return list(self.parent_operator.parent_individuals) + + @property + def parents_from_prev_generation(self) -> List[Individual]: + parents_from_prev_generation = [] + next_parents = self.parents + for _ in range(1_000_000): + if not next_parents or all(p.has_native_generation for p in next_parents): + break + parents = next_parents + next_parents = [] + for p in parents: + next_parents += p.parents + else: # After the last iteration. + raise ValueError(f'The individual {self.uid} has invalid inheritance data.') + + parents_from_prev_generation += next_parents + return parents_from_prev_generation + + @property + def operators_from_prev_generation(self) -> List[ParentOperator]: + if not self.parent_operator: + return [] + parents_from_prev_generation = self.parents_from_prev_generation + operators = [self.parent_operator] + next_parents = self.parents + for _ in range(1_000_000): + if next_parents == parents_from_prev_generation: + break + parents = next_parents + next_parents = [] + for p in parents: + next_parents += p.parents + operators.append(p.parent_operator) + else: # After the last iteration. + raise ValueError(f'The individual {self.uid} has invalid inheritance data.') + + operators.reverse() + return operators + + def save(self, json_file_path: Union[str, os.PathLike] = None) -> Optional[str]: + return default_save(obj=self, json_file_path=json_file_path) + + @staticmethod + def load(json_str_or_file_path: Union[str, os.PathLike] = None) -> Individual: + return default_load(json_str_or_file_path) + + def __repr__(self): + return (f'') + + def __eq__(self, other: Individual): + return self.uid == other.uid + + def __copy__(self): + default_log(self).log_or_raise('warning', INDIVIDUAL_COPY_RESTRICTION_MESSAGE) + cls = self.__class__ + result = cls.__new__(cls) + result.__dict__.update(self.__dict__) + return result + + def __deepcopy__(self, memo): + default_log(self).log_or_raise('warning', INDIVIDUAL_COPY_RESTRICTION_MESSAGE) + cls = self.__class__ + result = cls.__new__(cls) + memo[id(self)] = result + for k, v in self.__dict__.items(): + object.__setattr__(result, k, deepcopy(v, memo)) + return result + + +@dataclass +class GraphEvalResult: + uid_of_individual: str + fitness: Fitness + graph: OptGraph # For the case if evaluation needs to assign some values to the graph. + metadata: Dict[str, Any] = field(default_factory=dict) + + def __bool__(self): + return self.fitness.valid diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/opt_history_objects/opt_history.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/opt_history_objects/opt_history.py new file mode 100644 index 0000000..dafc84c --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/opt_history_objects/opt_history.py @@ -0,0 +1,277 @@ +from __future__ import annotations + +import csv +import io +import itertools +import os +import shutil +from pathlib import Path +from typing import Any, Dict, List, Optional, Sequence, Union, TYPE_CHECKING + +from golem.core.log import default_log +from golem.core.optimisers.objective.objective import ObjectiveInfo +from golem.core.optimisers.opt_history_objects.generation import Generation + +from golem.core.paths import default_data_dir +from golem.serializers.serializer import default_load, default_save +from golem.visualisation.opt_viz import OptHistoryVisualizer + +if TYPE_CHECKING: + from golem.core.dag.graph import Graph + from golem.core.optimisers.opt_history_objects.individual import Individual + + +class OptHistory: + """ + Contains optimization history, saves history to csv. + Can be used for any type of graph that is serializable with Serializer. + + Args: + objective: information about metrics (metric names and if it's multi-objective) + default_save_dir: default directory used for saving history when not explicit path is provided. + """ + + def __init__(self, + objective: Optional[ObjectiveInfo] = None, + default_save_dir: Optional[os.PathLike] = None): + self._objective = objective or ObjectiveInfo() + self._generations: List[Generation] = [] + self.archive_history: List[List[Individual]] = [] + self._tuning_result: Optional[Graph] = None + + # init default save directory + if default_save_dir: + default_save_dir = Path(default_save_dir) + if not default_save_dir.is_absolute(): + # if path is not absolute, treat it as relative to data dir + default_save_dir = Path(default_data_dir()) / default_save_dir + else: + default_save_dir = default_data_dir() + self._default_save_dir = str(default_save_dir) + + @property + def objective(self): + return self._objective + + def is_empty(self) -> bool: + return not self.generations + + def add_to_history(self, individuals: Sequence[Individual], generation_label: Optional[str] = None, + generation_metadata: Optional[Dict[str, Any]] = None): + generation = Generation(individuals, self.generations_count, generation_label, generation_metadata) + self.generations.append(generation) + + def add_to_archive_history(self, individuals: Sequence[Individual]): + self.archive_history.append(list(individuals)) + + def to_csv(self, save_dir: Optional[os.PathLike] = None, file: os.PathLike = 'history.csv'): + save_dir = save_dir or self._default_save_dir + if not os.path.isdir(save_dir): + os.mkdir(save_dir) + + with open(Path(save_dir, file), 'w', newline='') as file: + writer = csv.writer(file, quoting=csv.QUOTE_ALL) + + # Write header + metric_str = 'metric' + if self.objective.is_multi_objective: + metric_str += 's' + header_row = ['index', 'generation', metric_str, 'quantity_of_operations', 'depth', 'metadata'] + writer.writerow(header_row) + + # Write history rows + idx = 0 + for gen_num, gen_inds in enumerate(self.generations): + for ind_num, ind in enumerate(gen_inds): + row = [idx, gen_num, ind.fitness.values, ind.graph.length, ind.graph.depth, ind.metadata] + writer.writerow(row) + idx += 1 + + def save_current_results(self, save_dir: Optional[os.PathLike] = None): + # Create folder if it's not exists + save_dir = save_dir or self._default_save_dir + if not os.path.isdir(save_dir): + os.makedirs(save_dir) + self._log.info(f"Created directory for saving optimization history: {save_dir}") + + try: + last_gen_id = self.generations_count - 1 + last_gen = self.generations[last_gen_id] + for individual in last_gen: + ind_path = Path(save_dir, str(last_gen_id), str(individual.uid)) + ind_path.mkdir(exist_ok=True, parents=True) + individual.save(json_file_path=ind_path / f'{str(individual.uid)}.json') + except Exception as ex: + self._log.exception(ex) + + def save(self, json_file_path: Union[str, os.PathLike] = None, is_save_light: bool = False) -> Optional[str]: + """ Saves history to specified path. + Args: + json_file_path: path to json file where to save history. + is_save_light: bool parameter to specify whether there is a need to save full history or a light version. + NB! For experiments and etc. full histories must be saved. However, to make the analysis of results faster + (show fitness plots, for example) the light version of histories can be saved too. + """ + history_to_save = lighten_history(self) if is_save_light else self + return default_save(obj=history_to_save, json_file_path=json_file_path) + + @staticmethod + def load(json_str_or_file_path: Union[str, os.PathLike] = None) -> OptHistory: + return default_load(json_str_or_file_path) + + @staticmethod + def clean_results(dir_path: Optional[str] = None): + """Clearn the directory tree with previously dumped history results.""" + if dir_path and os.path.isdir(dir_path): + shutil.rmtree(dir_path, ignore_errors=True) + os.mkdir(dir_path) + + @property + def historical_fitness(self) -> Sequence[Sequence[Union[float, Sequence[float]]]]: + """Return sequence of histories of generations per each metric""" + if self.objective.is_multi_objective: + historical_fitness = [] + num_metrics = len(self.generations[0][0].fitness.values) + for objective_num in range(num_metrics): + # history of specific objective for each generation + objective_history = [[ind.fitness.values[objective_num] for ind in generation] + for generation in self.generations] + historical_fitness.append(objective_history) + else: + historical_fitness = [[ind.fitness.value for ind in pop] + for pop in self.generations] + return historical_fitness + + @property + def all_historical_fitness(self) -> List[float]: + historical_fitness = self.historical_fitness + if self.objective.is_multi_objective: + all_historical_fitness = [] + for obj_num in range(len(historical_fitness)): + all_historical_fitness.append(list(itertools.chain(*historical_fitness[obj_num]))) + else: + all_historical_fitness = list(itertools.chain(*historical_fitness)) + return all_historical_fitness + + def all_historical_quality(self, metric_position: int = 0) -> List[float]: + """ + Return fitness history of population for specified metric. + + Args: + metric_position: Index of the metric for multi-objective optimization. + By default, choose first metric, assuming it is primary quality metric. + + Returns: + List: all historical fitness + """ + if self.objective.is_multi_objective: + all_historical_quality = self.all_historical_fitness[metric_position] + else: + all_historical_quality = self.all_historical_fitness + return all_historical_quality + + @property + def show(self): + return OptHistoryVisualizer(self) + + def get_leaderboard(self, top_n: int = 10) -> str: + """ + Prints ordered description of the best solutions in history + :param top_n: number of solutions to print + """ + # Take only the first graph's appearance in history + individuals_with_positions \ + = list({ind.graph.descriptive_id: (ind, gen_num, ind_num) + for gen_num, gen in enumerate(self.generations) + for ind_num, ind in reversed(list(enumerate(gen)))}.values()) + + top_individuals = sorted(individuals_with_positions, + key=lambda pos_ind: pos_ind[0].fitness, reverse=True)[:top_n] + + output = io.StringIO() + separator = ' | ' + header = separator.join(['Position', 'Fitness', 'Generation', 'Graph']) + print(header, file=output) + for ind_num, ind_with_position in enumerate(top_individuals): + individual, gen_num, ind_num = ind_with_position + positional_id = f'g{gen_num}-i{ind_num}' + print(separator.join([f'{ind_num:>3}, ' + f'{str(individual.fitness):>8}, ' + f'{positional_id:>8}, ' + f'{individual.graph.descriptive_id}']), file=output) + + # add info about initial assumptions (stored as zero generation) + for i, individual in enumerate(self.generations[0]): + ind = f'I{i}' + positional_id = '-' + print(separator.join([f'{ind:>3}' + f'{str(individual.fitness):>8}', + f'{positional_id}', + f'{individual.graph.descriptive_id}']), file=output) + return output.getvalue() + + @property + def initial_assumptions(self) -> Optional[Generation]: + if not self.generations: + return None + for gen in self.generations: + if gen.label == 'initial_assumptions': + return gen + + @property + def final_choices(self) -> Optional[Generation]: + if not self.generations: + return None + for gen in reversed(self.generations): + if gen.label == 'final_choices': + return gen + + @property + def generations_count(self) -> int: + return len(self.generations) + + @property + def tuning_result(self): + if hasattr(self, '_tuning_result'): + return self._tuning_result + else: + return None + + @tuning_result.setter + def tuning_result(self, val): + self._tuning_result = val + + @property + def generations(self): + return self._generations + + @generations.setter + def generations(self, value): + self._generations = value + + @property + def individuals(self): + self._log.warning( + '"OptHistory.individuals" is deprecated and will be removed later. ' + 'Please, use "OptHistory.generations" to access generations.') + return self.generations + + @individuals.setter + def individuals(self, value): + self.generations = value + + @property + def _log(self): + return default_log(self) + + +def lighten_history(history: OptHistory) -> OptHistory: + """ Keeps the most informative field in OptHistory object to show most of the visualizations + without excessive memory usage. """ + light_history = OptHistory() + light_history._generations = \ + [Generation(iterable=gen, generation_num=i) for i, gen in enumerate(history.archive_history)] + light_history.archive_history = history.archive_history + light_history._objective = history.objective + light_history._tuning_result = history.tuning_result + return light_history diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/opt_history_objects/parent_operator.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/opt_history_objects/parent_operator.py new file mode 100644 index 0000000..55e944b --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/opt_history_objects/parent_operator.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Hashable, Sequence +from uuid import uuid4 + +from golem.core.optimisers.opt_history_objects.individual import Individual +from golem.utilities.data_structures import ensure_wrapped_in_sequence + + +@dataclass(frozen=True) +class ParentOperator: + type_: str + operators: Sequence[Hashable] + parent_individuals: Sequence[Individual] = field() + uid: str = field(default_factory=lambda: str(uuid4()), init=False) + + def __post_init__(self): + operators = ensure_wrapped_in_sequence(self.operators) + object.__setattr__(self, 'operators', tuple(operators)) + parent_individuals = ensure_wrapped_in_sequence(self.parent_individuals) + object.__setattr__(self, 'parent_individuals', tuple(parent_individuals)) + + def __repr__(self): + return (f'') diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/opt_node_factory.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/opt_node_factory.py new file mode 100644 index 0000000..c23af6d --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/opt_node_factory.py @@ -0,0 +1,73 @@ +import random +from abc import abstractmethod, ABC +from random import choice +from typing import Optional, Iterable, List + +from golem.core.optimisers.graph import OptNode + + +class OptNodeFactory(ABC): + @abstractmethod + def exchange_node(self, + node: OptNode) -> Optional[OptNode]: + """ + Returns new node based on a current node using information about node and advisor. + + :param node: current node that must be changed. + """ + pass + + @abstractmethod + def get_parent_node(self, node: OptNode, **kwargs) -> Optional[OptNode]: + """ + Returns new parent node for the current node + based on the content of the current node and using advisor. + + :param node: current node for which a parent node is generated + """ + pass + + @abstractmethod + def get_node(self, **kwargs) -> Optional[OptNode]: + """ + Returns new node based on the requirements for a node. + """ + pass + + @abstractmethod + def get_all_available_operations(self) -> List[str]: + """ + Returns all available models and data operations. + """ + pass + + +class DefaultOptNodeFactory(OptNodeFactory): + """Default node factory that either randomly selects + one node from the provided lists of available nodes + or returns a node with a random numeric node name + in the range from 0 to `num_node_types`.""" + + def __init__(self, + available_node_types: Optional[Iterable[str]] = None, + num_node_types: Optional[int] = None): + self.available_nodes = tuple(available_node_types) if available_node_types else None + self._num_node_types = num_node_types or 1000 + + def get_all_available_operations(self) -> Optional[List[str]]: + """ + Returns all available models and data operations. + """ + return self.available_nodes + + def exchange_node(self, node: OptNode) -> OptNode: + return self.get_node() + + def get_parent_node(self, node: OptNode, **kwargs) -> OptNode: + return self.get_node(**kwargs) + + def get_node(self, **kwargs) -> OptNode: + chosen_node_type = choice(self.available_nodes) \ + if self.available_nodes \ + else random.randint(0, self._num_node_types) + return OptNode(content={'name': chosen_node_type}) diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/optimization_parameters.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/optimization_parameters.py new file mode 100644 index 0000000..cd6d360 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/optimization_parameters.py @@ -0,0 +1,94 @@ +import dataclasses +import datetime +from dataclasses import dataclass, field +from numbers import Number +from typing import Optional + +from golem.core.paths import default_data_dir +from golem.utilities.utilities import determine_n_jobs + + +@dataclass +class OptimizationParameters: + """Defines general algorithm-independent parameters of the composition process + (like stop condition, validation, timeout, logging etc.) + + Options related to stop condition: + + :param num_of_generations: maximum number of optimizer generations + :param timeout: max time in minutes available for composition process + :param early_stopping_iterations: for early stopping. + + Optional max number of stagnating + iterations for early stopping. If both early_stopping options are None, + then do not use early stopping. + + :param early_stopping_timeout: for early stopping. + + Optional duration (in minutes) of stagnating + optimization for early stopping. If both early_stopping options are None, + then do not use early stopping. + + Infrastructure options (logging, performance) + + :param keep_n_best: number of the best individuals of previous generation to keep in next generation + :param max_graph_fit_time: time constraint for evaluation of each graph (datetime.timedelta) + :param n_jobs: num of n_jobs + :param show_progress: bool indicating whether to show progress using tqdm or not + :param collect_intermediate_metric: save metrics for intermediate (non-root) nodes in graph + :param parallelization_mode: identifies the way to parallelize population evaluation + + History options: + + :param keep_history: if True, then save generations to history; if False, don't keep history. + :param history_dir: directory for saving optimization history, optional. + + If the path is relative, then save relative to `default_data_dir`. + If absolute -- then save directly by specified path. + If None -- do not save the history to disk and keep it only in-memory. + """ + + num_of_generations: Optional[int] = None + timeout: Optional[datetime.timedelta] = datetime.timedelta(minutes=5) + early_stopping_iterations: Optional[int] = 50 + early_stopping_timeout: Optional[float] = 5 + + keep_n_best: int = 1 + max_graph_fit_time: Optional[datetime.timedelta] = None + n_jobs: int = 1 + show_progress: bool = True + collect_intermediate_metric: bool = False + parallelization_mode: str = 'populational' + static_individual_metadata: dict = field(default_factory=lambda: { + 'use_input_preprocessing': True + }) + + keep_history: bool = True + history_dir: Optional[str] = field(default_factory=default_data_dir) + agent_dir: Optional[str] = field(default_factory=default_data_dir) + + +@dataclass +class GraphRequirements(OptimizationParameters): + """Defines restrictions and requirements on final graphs. + + Restrictions on final graphs: + + :param start_depth: start value of adaptive tree depth + :param max_depth: max depth of the resulting graph + :param min_arity: min number of parents for node + :param max_arity: max number of parents for node + """ + + start_depth: int = 3 + max_depth: int = 10 + min_arity: int = 2 + max_arity: int = 4 + + def __post_init__(self): + # check and convert n_jobs to non-negative + self.n_jobs = determine_n_jobs(self.n_jobs) + + for field_name, field_value in dataclasses.asdict(self).items(): + if isinstance(field_value, Number) and field_value < 0: + raise ValueError(f'Value of {field_name} must be non-negative') diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/optimizer.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/optimizer.py new file mode 100644 index 0000000..86adf1a --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/optimizer.py @@ -0,0 +1,186 @@ +from abc import abstractmethod +from dataclasses import dataclass +from typing import Any, Callable, Optional, Sequence, Union + +from tqdm import tqdm + +from golem.core.adapter import BaseOptimizationAdapter, IdentityAdapter +from golem.core.dag.graph import Graph +from golem.core.dag.graph_verifier import GraphVerifier, VerifierRuleType +from golem.core.dag.verification_rules import DEFAULT_DAG_RULES +from golem.core.log import default_log +from golem.core.optimisers.advisor import DefaultChangeAdvisor +from golem.core.optimisers.optimization_parameters import OptimizationParameters +from golem.core.optimisers.genetic.evaluation import DelegateEvaluator +from golem.core.optimisers.genetic.operators.operator import PopulationT +from golem.core.optimisers.graph import OptGraph +from golem.core.optimisers.objective import GraphFunction, Objective, ObjectiveFunction +from golem.core.optimisers.opt_history_objects.opt_history import OptHistory +from golem.core.optimisers.opt_node_factory import DefaultOptNodeFactory, OptNodeFactory +from golem.core.optimisers.random_graph_factory import RandomGraphFactory, RandomGrowthGraphFactory +from golem.utilities.random import RandomStateHandler + +STRUCTURAL_DIVERSITY_FREQUENCY_CHECK = 5 + + +def do_nothing_callback(*args, **kwargs): + pass + + +@dataclass +class AlgorithmParameters: + """Base class for definition of optimizers-specific parameters. + Can be extended for custom optimizers. + + :param multi_objective: defines if the optimizer must be multi-criterial + :param offspring_rate: offspring rate used on next population + :param pop_size: initial population size + :param max_pop_size: maximum population size; optional, if unspecified, then population size is unbound + :param adaptive_depth: flag to enable adaptive configuration of graph depth + :param adaptive_depth_max_stagnation: max number of stagnating populations before adaptive depth increment + """ + + multi_objective: bool = False + offspring_rate: float = 0.5 + pop_size: int = 20 + max_pop_size: Optional[int] = 55 + adaptive_depth: bool = False + adaptive_depth_max_stagnation: int = 3 + structural_diversity_frequency_check: int = STRUCTURAL_DIVERSITY_FREQUENCY_CHECK + + +@dataclass +class GraphGenerationParams: + """ + This dataclass is for defining the parameters using in graph generation process + + :param adapter: instance of domain graph adapter for adaptation + between domain and optimization graphs + :param rules_for_constraint: collection of constraints for graph verification + :param advisor: instance providing task and context-specific advices for graph changes + :param node_factory: instance for generating new nodes in the process of graph search + :param remote_evaluator: instance of delegate evaluator for evaluation of graphs + """ + adapter: BaseOptimizationAdapter + verifier: GraphVerifier + advisor: DefaultChangeAdvisor + node_factory: OptNodeFactory + random_graph_factory: RandomGraphFactory + remote_evaluator: Optional[DelegateEvaluator] = None + + def __init__(self, adapter: Optional[BaseOptimizationAdapter] = None, + rules_for_constraint: Sequence[VerifierRuleType] = tuple(DEFAULT_DAG_RULES), + advisor: Optional[DefaultChangeAdvisor] = None, + node_factory: Optional[OptNodeFactory] = None, + random_graph_factory: Optional[RandomGraphFactory] = None, + available_node_types: Optional[Sequence[Any]] = None, + remote_evaluator: Optional[DelegateEvaluator] = None, + ): + self.adapter = adapter or IdentityAdapter() + self.verifier = GraphVerifier(rules_for_constraint, self.adapter) + self.advisor = advisor or DefaultChangeAdvisor() + self.remote_evaluator = remote_evaluator + if node_factory: + self.node_factory = node_factory + elif available_node_types: + self.node_factory = DefaultOptNodeFactory(available_node_types) + else: + self.node_factory = DefaultOptNodeFactory() + self.random_graph_factory = random_graph_factory or RandomGrowthGraphFactory(self.verifier, + self.node_factory) + + +class GraphOptimizer: + """ + Base class of graph optimizer. It allows to find the optimal solution using specified metric (one or several). + To implement the specific optimisation method, + the abstract method 'optimize' should be re-defined in the ancestor class + (e.g. PopulationalOptimizer, RandomSearchGraphOptimiser, etc). + + :param objective: objective for optimisation + :param initial_graphs: graphs which were initialized outside the optimizer + :param requirements: implementation-independent requirements for graph optimizer + :param graph_generation_params: parameters for new graph generation + :param graph_optimizer_params: parameters for specific implementation of graph optimizer + + Additional custom params can be specified with `custom_optimizer_params`. + """ + + def __init__(self, + objective: Objective, + initial_graphs: Optional[Sequence[Union[Graph, Any]]] = None, + # TODO: rename params to avoid confusion + requirements: Optional[OptimizationParameters] = None, + graph_generation_params: Optional[GraphGenerationParams] = None, + graph_optimizer_params: Optional[AlgorithmParameters] = None, + **custom_optimizer_params): + self.log = default_log(self) + self._objective = objective + initial_graphs = graph_generation_params.adapter.adapt(initial_graphs) if initial_graphs else None + self.initial_graphs = [graph for graph in initial_graphs if graph_generation_params.verifier(graph)] \ + if initial_graphs else None + self.requirements = requirements or OptimizationParameters() + self.graph_generation_params = graph_generation_params or GraphGenerationParams() + self.graph_optimizer_params = graph_optimizer_params or AlgorithmParameters() + self._iteration_callback: IterationCallback = do_nothing_callback + self._history = OptHistory(objective.get_info(), requirements.history_dir) \ + if requirements and requirements.keep_history else None + # Log random state for reproducibility of runs + RandomStateHandler.log_random_state() + + @property + def objective(self) -> Objective: + """Returns Objective of this optimizer with information about metrics used.""" + return self._objective + + @property + def history(self) -> Optional[OptHistory]: + """Returns optimization history""" + return self._history + + @abstractmethod + def optimise(self, objective: ObjectiveFunction) -> Sequence[OptGraph]: + """ + Method for running of optimization using specified algorithm. + :param objective: objective function that specifies optimization target + :return: sequence of the best graphs + """ + pass + + def set_iteration_callback(self, callback: Optional[Callable]): + """Set optimisation callback that is called at the end of + each iteration, with the next generation passed as argument. + Resets the callback if None is passed.""" + self._iteration_callback = callback or do_nothing_callback + + def set_evaluation_callback(self, callback: Optional[GraphFunction]): + """Set or reset (with None) post-evaluation callback + that's called on each graph after its evaluation.""" + pass + + @property + def _progressbar(self): + if self.requirements.show_progress: + bar = tqdm(total=self.requirements.num_of_generations, desc='Generations', unit='gen', initial=0) + else: + # disable call to tqdm.__init__ to avoid stdout/stderr access inside it + # part of a workaround for https://github.com/nccr-itmo/FEDOT/issues/765 + bar = EmptyProgressBar() + return bar + + +IterationCallback = Callable[[PopulationT, GraphOptimizer], Any] + + +class EmptyProgressBar: + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + return True + + def close(self): + return + + def update(self): + return diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/populational_optimizer.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/populational_optimizer.py new file mode 100644 index 0000000..8a8704d --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/populational_optimizer.py @@ -0,0 +1,182 @@ +from abc import abstractmethod +from random import choice +from typing import Any, Optional, Sequence, Dict + +from golem.core.constants import MIN_POP_SIZE +from golem.core.dag.graph import Graph +from golem.core.optimisers.archive import GenerationKeeper +from golem.core.optimisers.genetic.evaluation import MultiprocessingDispatcher, SequentialDispatcher +from golem.core.optimisers.genetic.operators.operator import PopulationT, EvaluationOperator +from golem.core.optimisers.objective import GraphFunction, ObjectiveFunction +from golem.core.optimisers.objective.objective import Objective +from golem.core.optimisers.opt_history_objects.individual import Individual +from golem.core.optimisers.optimization_parameters import GraphRequirements +from golem.core.optimisers.optimizer import GraphGenerationParams, GraphOptimizer, AlgorithmParameters +from golem.core.optimisers.timer import OptimisationTimer +from golem.utilities.grouped_condition import GroupedCondition + + +class PopulationalOptimizer(GraphOptimizer): + """ + Base class of populational optimizer. + PopulationalOptimizer implements all basic methods for optimization not related to evolution process + to experiment with other kinds of evolution optimization methods + It allows to find the optimal solution using specified metric (one or several). + To implement the specific evolution strategy, implement `_evolution_process`. + + Args: + objective: objective for optimization + initial_graphs: graphs which were initialized outside the optimizer + requirements: implementation-independent requirements for graph optimizer + graph_generation_params: parameters for new graph generation + graph_optimizer_params: parameters for specific implementation of graph optimizer + + Additional custom params can be specified with `custom_optimizer_params`. + """ + + def __init__(self, + objective: Objective, + initial_graphs: Sequence[Graph], + requirements: GraphRequirements, + graph_generation_params: GraphGenerationParams, + graph_optimizer_params: Optional['AlgorithmParameters'] = None, + **custom_optimizer_params + ): + super().__init__(objective, initial_graphs, requirements, + graph_generation_params, graph_optimizer_params, **custom_optimizer_params) + self.population = None + self.generations = GenerationKeeper(self.objective, keep_n_best=requirements.keep_n_best) + self.timer = OptimisationTimer(timeout=self.requirements.timeout) + + dispatcher_type = MultiprocessingDispatcher if self.requirements.parallelization_mode == 'populational' else \ + SequentialDispatcher + + self.eval_dispatcher = dispatcher_type(adapter=graph_generation_params.adapter, + n_jobs=requirements.n_jobs, + graph_cleanup_fn=_try_unfit_graph, + delegate_evaluator=graph_generation_params.remote_evaluator) + + # early_stopping_iterations and early_stopping_timeout may be None, so use some obvious max number + max_stagnation_length = requirements.early_stopping_iterations or requirements.num_of_generations + max_stagnation_time = requirements.early_stopping_timeout or self.timer.timeout + self.stop_optimization = \ + GroupedCondition(results_as_message=True).add_condition( + lambda: self.timer.is_time_limit_reached(self.current_generation_num - 1), + 'Optimisation stopped: Time limit is reached' + ).add_condition( + lambda: (requirements.num_of_generations is not None and + self.current_generation_num >= requirements.num_of_generations + 1), + 'Optimisation stopped: Max number of generations reached' + ).add_condition( + lambda: (max_stagnation_length is not None and + self.generations.stagnation_iter_count >= max_stagnation_length), + 'Optimisation finished: Early stopping iterations criteria was satisfied' + ).add_condition( + lambda: self.generations.stagnation_time_duration >= max_stagnation_time, + 'Optimisation finished: Early stopping timeout criteria was satisfied' + ) + # in how many generations structural diversity check should be performed + self.gen_structural_diversity_check = self.graph_optimizer_params.structural_diversity_frequency_check + + @property + def current_generation_num(self) -> int: + return self.generations.generation_num + + def set_evaluation_callback(self, callback: Optional[GraphFunction]): + # Redirect callback to evaluation dispatcher + self.eval_dispatcher.set_graph_evaluation_callback(callback) + + def optimise(self, objective: ObjectiveFunction) -> Sequence[Graph]: + + # eval_dispatcher defines how to evaluate objective on the whole population + evaluator = self.eval_dispatcher.dispatch(objective, self.timer) + + with self.timer, self._progressbar as pbar: + + self._initial_population(evaluator) + + while not self.stop_optimization(): + try: + new_population = self._evolve_population(evaluator) + if self.gen_structural_diversity_check != -1 \ + and self.generations.generation_num % self.gen_structural_diversity_check == 0 \ + and self.generations.generation_num != 0: + new_population = self.get_structure_unique_population(new_population, evaluator) + pbar.update() + except EvaluationAttemptsError as ex: + self.log.warning(f'Composition process was stopped due to: {ex}') + break + # Adding of new population to history + self._update_population(new_population) + pbar.close() + self._update_population(self.best_individuals, 'final_choices') + return [ind.graph for ind in self.best_individuals] + + @property + def best_individuals(self): + return self.generations.best_individuals + + @abstractmethod + def _initial_population(self, evaluator: EvaluationOperator): + """ Initializes the initial population """ + raise NotImplementedError() + + @abstractmethod + def _evolve_population(self, evaluator: EvaluationOperator) -> PopulationT: + """ Method realizing full evolution cycle """ + raise NotImplementedError() + + def _extend_population(self, pop: PopulationT, target_pop_size: int) -> PopulationT: + """ Extends population to specified `target_pop_size`. """ + n = target_pop_size - len(pop) + extended_population = list(pop) + extended_population.extend([Individual(graph=choice(pop).graph) for _ in range(n)]) + return extended_population + + def _update_population(self, next_population: PopulationT, label: Optional[str] = None, + metadata: Optional[Dict[str, Any]] = None): + self.generations.append(next_population) + if self.requirements.keep_history: + self._log_to_history(next_population, label, metadata) + self._iteration_callback(next_population, self) + self.population = next_population + + self.log.info(f'Generation num: {self.current_generation_num} size: {len(next_population)}') + self.log.info(f'Best individuals: {str(self.generations)}') + if self.generations.stagnation_iter_count > 0: + self.log.info(f'no improvements for {self.generations.stagnation_iter_count} iterations') + self.log.info(f'spent time: {round(self.timer.minutes_from_start, 1)} min') + + def _log_to_history(self, population: PopulationT, label: Optional[str] = None, + metadata: Optional[Dict[str, Any]] = None): + self.history.add_to_history(population, label, metadata) + self.history.add_to_archive_history(self.generations.best_individuals) + if self.requirements.history_dir: + self.history.save_current_results(self.requirements.history_dir) + + def get_structure_unique_population(self, population: PopulationT, evaluator: EvaluationOperator) -> PopulationT: + """ Increases structurally uniqueness of population to prevent stagnation in optimization process. + Returned population may be not entirely unique, if the size of unique population is lower than MIN_POP_SIZE. """ + unique_population_with_ids = {ind.graph.descriptive_id: ind for ind in population} + unique_population = list(unique_population_with_ids.values()) + + # if size of unique population is too small, then extend it to MIN_POP_SIZE by repeating individuals + if len(unique_population) < MIN_POP_SIZE: + unique_population = self._extend_population(pop=unique_population, target_pop_size=MIN_POP_SIZE) + return evaluator(unique_population) + + +# TODO: remove this hack (e.g. provide smth like FitGraph with fit/unfit interface) +def _try_unfit_graph(graph: Any): + if hasattr(graph, 'unfit'): + graph.unfit() + + +class EvaluationAttemptsError(Exception): + """ Number of evaluation attempts exceeded """ + + def __init__(self, *args): + self.message = args[0] or None + + def __str__(self): + return self.message or 'Too many fitness evaluation errors.' diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/random/__init__.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/random/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/random/random_mutation_optimizer.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/random/random_mutation_optimizer.py new file mode 100644 index 0000000..b6adc70 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/random/random_mutation_optimizer.py @@ -0,0 +1,67 @@ +from typing import Union, Optional, Sequence + +from golem.core.dag.graph import Graph +from golem.core.optimisers.genetic.gp_params import GPAlgorithmParameters +from golem.core.optimisers.genetic.operators.mutation import Mutation +from golem.core.optimisers.genetic.operators.operator import EvaluationOperator, PopulationT +from golem.core.optimisers.objective import Objective +from golem.core.optimisers.opt_history_objects.individual import Individual +from golem.core.optimisers.optimization_parameters import GraphRequirements +from golem.core.optimisers.optimizer import GraphGenerationParams +from golem.core.optimisers.populational_optimizer import PopulationalOptimizer +from golem.core.optimisers.random.random_search import RandomSearchOptimizer +from golem.utilities.data_structures import ensure_wrapped_in_sequence + + +class PopulationalRandomMutationOptimizer(PopulationalOptimizer): + """ + Populational random search-based graph models optimizer. + Implemented as a baseline to compare with evolutionary algorithm. + """ + + def __init__(self, + objective: Objective, + initial_graphs: Union[Graph, Sequence[Graph]], + requirements: Optional[GraphRequirements] = None, + graph_generation_params: Optional[GraphGenerationParams] = None, + graph_optimizer_params: Optional[GPAlgorithmParameters] = None): + requirements = requirements or GraphRequirements() + graph_optimizer_params = graph_optimizer_params or GPAlgorithmParameters() + super().__init__(objective, initial_graphs, requirements, graph_generation_params, graph_optimizer_params) + self.initial_individuals = [Individual(graph, metadata=requirements.static_individual_metadata) + for graph in self.initial_graphs] + self.mutation = Mutation(self.graph_optimizer_params, self.requirements, self.graph_generation_params) + + def _evolve_population(self, evaluator: EvaluationOperator) -> PopulationT: + new_population = ensure_wrapped_in_sequence(self.mutation(self.population)) + new_population = evaluator(new_population) + return new_population + + def _initial_population(self, evaluator: EvaluationOperator): + self._update_population(evaluator(self.initial_individuals), 'initial_assumptions') + pop_size = self.graph_optimizer_params.pop_size + + if len(self.initial_individuals) < pop_size: + self.initial_individuals = self._extend_population(self.initial_individuals, pop_size) + # Adding of extended population to history + self._update_population(evaluator(self.initial_individuals), 'extended_initial_assumptions') + + +class RandomMutationOptimizer(RandomSearchOptimizer): + """ + Random search-based graph models optimizer + """ + + def __init__(self, + objective: Objective, + initial_graphs: Union[Graph, Sequence[Graph]], + requirements: Optional[GraphRequirements] = None, + graph_generation_params: Optional[GraphGenerationParams] = None, + graph_optimizer_params: Optional[GPAlgorithmParameters] = None): + requirements = requirements or GraphRequirements() + graph_optimizer_params = graph_optimizer_params or GPAlgorithmParameters() + super().__init__(objective, initial_graphs, requirements, graph_generation_params, graph_optimizer_params) + self.mutation = Mutation(self.graph_optimizer_params, self.requirements, self.graph_generation_params) + + def _generate_new_individual(self) -> Individual: + return self.mutation(self.best_individual) diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/random/random_search.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/random/random_search.py new file mode 100644 index 0000000..0ca3301 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/random/random_search.py @@ -0,0 +1,80 @@ +from random import choice +from typing import Optional, Sequence + +from golem.core.dag.graph import Graph +from golem.core.optimisers.archive import GenerationKeeper +from golem.core.optimisers.genetic.evaluation import SequentialDispatcher +from golem.core.optimisers.genetic.gp_params import GPAlgorithmParameters +from golem.core.optimisers.genetic.operators.operator import EvaluationOperator +from golem.core.optimisers.graph import OptGraph +from golem.core.optimisers.objective import Objective, ObjectiveFunction +from golem.core.optimisers.opt_history_objects.individual import Individual +from golem.core.optimisers.optimization_parameters import GraphRequirements +from golem.core.optimisers.optimizer import GraphOptimizer, GraphGenerationParams +from golem.core.optimisers.timer import OptimisationTimer +from golem.utilities.grouped_condition import GroupedCondition + + +class RandomSearchOptimizer(GraphOptimizer): + + def __init__(self, + objective: Objective, + initial_graphs: Sequence[Graph] = None, + requirements: Optional[GraphRequirements] = None, + graph_generation_params: Optional[GraphGenerationParams] = None, + graph_optimizer_params: Optional[GPAlgorithmParameters] = None): + super().__init__(objective, initial_graphs, requirements, graph_generation_params, graph_optimizer_params) + self.timer = OptimisationTimer(timeout=self.requirements.timeout) + self.generations = GenerationKeeper(self.objective, keep_n_best=requirements.keep_n_best) + self.current_iteration_num = 0 + self.best_individual = None + self.stop_optimization = \ + GroupedCondition(results_as_message=True).add_condition( + lambda: self.timer.is_time_limit_reached(self.current_iteration_num), + 'Optimisation stopped: Time limit is reached' + ).add_condition( + lambda: requirements.num_of_generations is not None and + self.current_iteration_num >= requirements.num_of_generations, + 'Optimisation stopped: Max number of iterations reached') + + def optimise(self, objective: ObjectiveFunction) -> Sequence[OptGraph]: + + dispatcher = SequentialDispatcher(self.graph_generation_params.adapter) + evaluator = dispatcher.dispatch(objective, self.timer) + + self.current_iteration_num = 0 + + with self.timer, self._progressbar as pbar: + self.best_individual = self._eval_initial_individual(evaluator) + self._update_best_individual(self.best_individual, 'initial_assumptions') + while not self.stop_optimization(): + new_individual = self._generate_new_individual() + evaluator([new_individual]) + self.current_iteration_num += 1 + self._update_best_individual(new_individual) + pbar.update() + self._update_best_individual(self.best_individual, 'final_choices') + pbar.close() + return [self.best_individual.graph] + + def _update_best_individual(self, new_individual: Individual, label: Optional[str] = None): + if new_individual.fitness >= self.best_individual.fitness: + self.best_individual = new_individual + + self.generations.append([new_individual]) + + self.log.info(f'Spent time: {round(self.timer.minutes_from_start, 1)} min') + self.log.info(f'Iteration num {self.current_iteration_num}: ' + f'Best individuals fitness {str(self.generations)}') + + self.history.add_to_history([new_individual], label) + self.history.add_to_archive_history(self.generations.best_individuals) + + def _eval_initial_individual(self, evaluator: EvaluationOperator) -> Individual: + init_ind = Individual(choice(self.initial_graphs)) if self.initial_graphs else self._generate_new_individual() + evaluator([init_ind]) + return init_ind + + def _generate_new_individual(self) -> Individual: + new_graph = self.graph_generation_params.random_graph_factory(self.requirements) + return Individual(new_graph) diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/random_graph_factory.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/random_graph_factory.py new file mode 100644 index 0000000..8d46e60 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/random_graph_factory.py @@ -0,0 +1,70 @@ +from random import randint, choices +from typing import Optional, Callable + +from golem.core.constants import MAX_GRAPH_GEN_ATTEMPTS +from golem.core.dag.graph import Graph +from golem.core.dag.graph_utils import distance_to_root_level +from golem.core.dag.graph_verifier import GraphVerifier +from golem.core.optimisers.graph import OptGraph, OptNode +from golem.core.optimisers.opt_node_factory import OptNodeFactory +from golem.core.optimisers.optimization_parameters import GraphRequirements + +RandomGraphFactory = Callable[[GraphRequirements, int], Graph] + + +class RandomGrowthGraphFactory(RandomGraphFactory): + """ Default realisation of random graph factory. Generates DAG graph using random growth. """ + def __init__(self, + verifier: GraphVerifier, + node_factory: OptNodeFactory): + self.node_factory = node_factory + self.verifier = verifier + + def __call__(self, requirements: GraphRequirements, max_depth: Optional[int] = None) -> OptGraph: + return random_graph(self.verifier, self.node_factory, requirements, max_depth) + + +def random_graph(verifier: GraphVerifier, + node_factory: OptNodeFactory, + requirements: GraphRequirements, + max_depth: Optional[int] = None, + growth_proba: float = 0.3) -> OptGraph: + max_depth = max_depth if max_depth else requirements.max_depth + is_correct_graph = False + graph = None + n_iter = 0 + + while not is_correct_graph: + graph = OptGraph() + graph_root = node_factory.get_node() + graph.add_node(graph_root) + if requirements.max_depth > 1: + graph_growth(graph, graph_root, node_factory, requirements, max_depth, growth_proba) + + is_correct_graph = verifier(graph) + n_iter += 1 + if n_iter > MAX_GRAPH_GEN_ATTEMPTS: + raise ValueError(f'Could not generate random graph for {n_iter} ' + f'iterations with requirements {requirements}') + return graph + + +def graph_growth(graph: OptGraph, + node_parent: OptNode, + node_factory: OptNodeFactory, + requirements: GraphRequirements, + max_depth: int, + growth_proba: float): + """Function create a graph and links between nodes""" + offspring_size = randint(requirements.min_arity, requirements.max_arity) + + for offspring_node in range(offspring_size): + node = node_factory.get_node() + node_parent.nodes_from.append(node) + graph.add_node(node) + height = distance_to_root_level(graph, node) + is_max_depth_exceeded = height >= max_depth - 1 + if not is_max_depth_exceeded: + # lower proba of further growth reduces time of graph generation + if choices([0, 1], weights=[1 - growth_proba, growth_proba])[0]: + graph_growth(graph, node, node_factory, requirements, max_depth, growth_proba) diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/timer.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/timer.py new file mode 100644 index 0000000..34cae03 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/optimisers/timer.py @@ -0,0 +1,83 @@ +import datetime +from abc import ABC + +from golem.core.log import default_log + + +class Timer(ABC): + def __init__(self, timeout: datetime.timedelta = None): + self.process_terminated = False + self.log = default_log(self) + self.timeout = timeout + + def __enter__(self): + self.start = datetime.datetime.now() + return self + + @property + def start_time(self): + return self.start + + @property + def spent_time(self) -> datetime.timedelta: + return datetime.datetime.now() - self.start_time + + @property + def minutes_from_start(self) -> float: + return self.spent_time.total_seconds() / 60. + + @property + def seconds_from_start(self) -> float: + return self.spent_time.total_seconds() + + def is_time_limit_reached(self) -> bool: + self.process_terminated = False + if self.timeout is not None: + if datetime.datetime.now() - self.start >= self.timeout: + self.process_terminated = True + return self.process_terminated + + def __exit__(self, *args): + return self.process_terminated + + +class OptimisationTimer(Timer): + def __init__(self, timeout: datetime.timedelta = None): + super().__init__(timeout=timeout) + self.init_time = 0 + + def _is_next_iteration_possible(self, time_constraint: float, iteration_num: int = None) -> bool: + minutes = self.minutes_from_start + if iteration_num is not None and iteration_num != 0: + evo_proc_minutes = minutes - self.init_time + possible = time_constraint > (minutes + (evo_proc_minutes / iteration_num)) + else: + possible = time_constraint > minutes + if not possible: + self.process_terminated = True + return possible + + def is_time_limit_reached(self, iteration_num: int = None) -> bool: + if self.timeout: + timeout = 0 if self.timeout.total_seconds() < 0 else self.timeout.total_seconds() / 60. + if timeout: + reached = not self._is_next_iteration_possible(iteration_num=iteration_num, + time_constraint=timeout) + else: + self.process_terminated = True + reached = True + else: + reached = False + return reached + + def set_init_time(self, init_time: float): + self.init_time = init_time + + def __exit__(self, *args): + self.log.info(f'Composition time: {round(self.minutes_from_start, 3)} min') + if self.process_terminated: + self.log.info('Algorithm was terminated due to processing time limit') + + +def get_forever_timer() -> Timer: + return Timer(timeout=None) diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/paths.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/paths.py new file mode 100644 index 0000000..5bdbd18 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/paths.py @@ -0,0 +1,40 @@ +import os +import platform +import tempfile + +from pathlib import Path +from typing import Callable + + +def copy_doc(source_func: Callable) -> Callable: + """ + Copies a docstring from the provided ``source_func`` to the wrapped function + + :param source_func: function to copy the docstring from + + :return: wrapped function with the same docstring as in the given ``source_func`` + """ + def wrapper(func: Callable) -> Callable: + func.__doc__ = source_func.__doc__ + return func + return wrapper + + +def project_root() -> Path: + """Returns project root folder.""" + return Path(__file__).parent.parent.parent + + +def default_data_dir() -> str: + """ Returns the folder where all the output data is recorded to + (logs, history, visualisations). Default is: `/tmp/GOLEM` + """ + name = 'GOLEM' + temp_folder = Path("/tmp" if platform.system() == "Darwin" else tempfile.gettempdir()) + + default_data_path = os.path.join(temp_folder, name) + + if name not in os.listdir(temp_folder): + os.mkdir(default_data_path) + + return default_data_path diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/tuning/__init__.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/tuning/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/tuning/hyperopt_tuner.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/tuning/hyperopt_tuner.py new file mode 100644 index 0000000..b8a7c5a --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/tuning/hyperopt_tuner.py @@ -0,0 +1,174 @@ +from abc import ABC +from datetime import timedelta +from typing import Callable, Dict, Optional, Tuple, Any + +import numpy as np +from hyperopt import hp, tpe, fmin, Trials +from hyperopt.early_stop import no_progress_loss +from hyperopt.pyll import Apply, scope +from hyperopt.pyll_utils import validate_label + +from golem.core.adapter import BaseOptimizationAdapter +from golem.core.dag.linked_graph_node import LinkedGraphNode +from golem.core.log import default_log +from golem.core.optimisers.objective import ObjectiveFunction +from golem.core.tuning.search_space import SearchSpace, get_node_operation_parameter_label +from golem.core.tuning.tuner_interface import BaseTuner + + +@validate_label +def hp_randint(label, *args, **kwargs): + return scope.int(scope.hyperopt_param(label, scope.randint(*args, **kwargs))) + + +hp.randint = hp_randint + + +class HyperoptTuner(BaseTuner, ABC): + """Base class for hyperparameters optimization based on hyperopt library + + Args: + objective_evaluate: objective to optimize + adapter: the function for processing of external object that should be optimized + search_space: SearchSpace instance + iterations: max number of iterations + early_stopping_rounds: Optional max number of stagnating iterations for early stopping. If ``None``, will be set + to ``max(100, int(np.sqrt(iterations) * 10))``. + timeout: max time for tuning + n_jobs: num of ``n_jobs`` for parallelization (``-1`` for use all cpu's) + deviation: required improvement (in percent) of a metric to return tuned graph. + By default, ``deviation=0.05``, which means that tuned graph will be returned + if it's metric will be at least 0.05% better than the initial. + algo: algorithm for hyperparameters optimization with signature similar to :obj:`hyperopt.tse.suggest` + """ + + def __init__(self, objective_evaluate: ObjectiveFunction, + search_space: SearchSpace, + adapter: Optional[BaseOptimizationAdapter] = None, + iterations: int = 100, + early_stopping_rounds: Optional[int] = None, + timeout: timedelta = timedelta(minutes=5), + n_jobs: int = -1, + deviation: float = 0.05, + algo: Callable = tpe.suggest, **kwargs): + early_stopping_rounds = early_stopping_rounds or max(100, int(np.sqrt(iterations) * 10)) + super().__init__(objective_evaluate, + search_space, + adapter, + iterations, + early_stopping_rounds, + timeout, + n_jobs, + deviation, **kwargs) + + self.early_stop_fn = no_progress_loss(iteration_stop_count=self.early_stopping_rounds) + self.algo = algo + self.log = default_log(self) + + def _search_near_initial_parameters(self, + objective, + search_space: dict, + initial_parameters: dict, + trials: Trials, + remaining_time: float, + show_progress: bool = True) -> Tuple[Trials, int]: + """ Method to search using the search space where parameters initially set for the graph are fixed. + This allows not to lose results obtained while composition process + + Args: + graph: graph to be tuned + search_space: dict with parameters to be optimized and their search spaces + initial_parameters: dict with initial parameters of the graph + trials: Trials object to store all the search iterations + show_progress: shows progress of tuning if True + + Returns: + trials: Trials object storing all the search trials + init_trials_num: number of iterations made using the search space with fixed initial parameters + """ + try_initial_parameters = initial_parameters and self.iterations > 1 + if not try_initial_parameters: + init_trials_num = 0 + return trials, init_trials_num + + is_init_params_full = len(initial_parameters) == len(search_space) + if self.iterations < 10 or is_init_params_full: + init_trials_num = 1 + else: + init_trials_num = min(int(self.iterations * 0.1), 10) + + # fmin updates trials with evaluation points tried out during the call + fmin(objective, + search_space, + trials=trials, + algo=self.algo, + max_evals=init_trials_num, + show_progressbar=show_progress, + early_stop_fn=self.early_stop_fn, + timeout=remaining_time) + return trials, init_trials_num + + +def get_parameter_hyperopt_space(search_space: SearchSpace, + operation_name: str, + parameter_name: str, + label: str = 'default') -> Optional[Apply]: + """ + Function return hyperopt object with search_space from search_space dictionary + + Args: + search_space: SearchSpace with parameters per operation + operation_name: name of the operation + parameter_name: name of hyperparameter of particular operation + label: label to assign in hyperopt pyll + + Returns: + parameter range + """ + + # Get available parameters for current operation + operation_parameters = search_space.parameters_per_operation.get(operation_name) + + if operation_parameters is not None: + parameter_properties = operation_parameters.get(parameter_name) + hyperopt_distribution = parameter_properties.get('hyperopt-dist') + sampling_scope = parameter_properties.get('sampling-scope') + if hyperopt_distribution == hp.loguniform: + sampling_scope = [np.log(x) for x in sampling_scope] + return hyperopt_distribution(label, *sampling_scope) + else: + return None + + +def get_node_parameters_for_hyperopt(search_space: SearchSpace, node_id: int, node: LinkedGraphNode) \ + -> Tuple[Dict[str, Apply], Dict[str, Any]]: + """ + Function for forming dictionary with hyperparameters of the node operation for the ``HyperoptTuner`` + + Args: + search_space: SearchSpace with parameters per operation + node_id: number of node in graph.nodes list + node: node from the graph + + Returns: + parameters_dict: dictionary-like structure with labeled hyperparameters + and their range per operation + """ + + # Get available parameters for current operation + operation_name = node.name + parameters_list = search_space.get_parameters_for_operation(operation_name) + + parameters_dict = {} + initial_parameters = {} + for parameter_name in parameters_list: + node_op_parameter_name = get_node_operation_parameter_label(node_id, operation_name, parameter_name) + + # For operation get range where search can be done + space = get_parameter_hyperopt_space(search_space, operation_name, parameter_name, node_op_parameter_name) + parameters_dict.update({node_op_parameter_name: space}) + + if parameter_name in node.parameters: + initial_parameters.update({node_op_parameter_name: node.parameters[parameter_name]}) + + return parameters_dict, initial_parameters diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/tuning/iopt_tuner.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/tuning/iopt_tuner.py new file mode 100644 index 0000000..ade0fc6 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/tuning/iopt_tuner.py @@ -0,0 +1,263 @@ +from copy import deepcopy +from dataclasses import dataclass, field +from datetime import timedelta +from typing import List, Dict, Generic, Tuple, Any, Optional + +import numpy as np +from iOpt.output_system.listeners.console_outputers import ConsoleOutputListener +from iOpt.problem import Problem +from iOpt.solver import Solver +from iOpt.solver_parametrs import SolverParameters +from iOpt.trial import Point, FunctionValue + +from golem.core.adapter import BaseOptimizationAdapter +from golem.core.optimisers.genetic.evaluation import determine_n_jobs +from golem.core.optimisers.graph import OptGraph +from golem.core.optimisers.objective import ObjectiveEvaluate +from golem.core.tuning.search_space import SearchSpace, get_node_operation_parameter_label, convert_parameters +from golem.core.tuning.tuner_interface import BaseTuner, DomainGraphForTune +from golem.utilities.data_structures import ensure_wrapped_in_sequence + + +@dataclass +class IOptProblemParameters: + float_parameters_names: List[str] = field(default_factory=list) + discrete_parameters_names: List[str] = field(default_factory=list) + lower_bounds_of_float_parameters: List[float] = field(default_factory=list) + upper_bounds_of_float_parameters: List[float] = field(default_factory=list) + discrete_parameters_vals: List[List[Any]] = field(default_factory=list) + + @staticmethod + def from_parameters_dicts(float_parameters_dict: Optional[Dict[str, List]] = None, + discrete_parameters_dict: Optional[Dict[str, List]] = None): + float_parameters_dict = float_parameters_dict or {} + discrete_parameters_dict = discrete_parameters_dict or {} + + float_parameters_names = list(float_parameters_dict.keys()) + discrete_parameters_names = list(discrete_parameters_dict.keys()) + + lower_bounds_of_float_parameters = [bounds[0] for bounds in float_parameters_dict.values()] + upper_bounds_of_float_parameters = [bounds[1] for bounds in float_parameters_dict.values()] + discrete_parameters_vals = [values_set for values_set in discrete_parameters_dict.values()] + + return IOptProblemParameters(float_parameters_names, + discrete_parameters_names, + lower_bounds_of_float_parameters, + upper_bounds_of_float_parameters, + discrete_parameters_vals) + + +class GolemProblem(Problem, Generic[DomainGraphForTune]): + def __init__(self, graph: DomainGraphForTune, + objective_evaluate: ObjectiveEvaluate, + problem_parameters: IOptProblemParameters, + objectives_number: int = 1): + super().__init__() + self.objective_evaluate = objective_evaluate + self.graph = graph + + self.number_of_objectives = objectives_number + self.number_of_constraints = 0 + + self.discrete_variable_names = problem_parameters.discrete_parameters_names + self.discrete_variable_values = problem_parameters.discrete_parameters_vals + self.number_of_discrete_variables = len(self.discrete_variable_names) + + self.float_variable_names = problem_parameters.float_parameters_names + self.lower_bound_of_float_variables = problem_parameters.lower_bounds_of_float_parameters + self.upper_bound_of_float_variables = problem_parameters.upper_bounds_of_float_parameters + self.number_of_float_variables = len(self.float_variable_names) + + self._default_metric_value = np.inf + + def calculate(self, point: Point, function_value: FunctionValue) -> FunctionValue: + new_parameters = self.get_parameters_dict_from_iopt_point(point) + BaseTuner.set_arg_graph(self.graph, new_parameters) + graph_fitness = self.objective_evaluate(self.graph) + metric_value = graph_fitness.value if graph_fitness.valid else self._default_metric_value + function_value.value = metric_value + return function_value + + def get_parameters_dict_from_iopt_point(self, point: Point) -> Dict[str, Any]: + """Constructs a dict with all hyperparameters """ + float_parameters = dict(zip(self.float_variable_names, point.float_variables)) \ + if point.float_variables is not None else {} + discrete_parameters = dict(zip(self.discrete_variable_names, point.discrete_variables)) \ + if point.discrete_variables is not None else {} + + parameters_dict = {**float_parameters, **discrete_parameters} + return parameters_dict + + +class IOptTuner(BaseTuner): + """ + Base class for hyperparameters optimization based on hyperopt library + + Args: + objective_evaluate: objective to optimize + adapter: the function for processing of external object that should be optimized + iterations: max number of iterations + search_space: SearchSpace instance + n_jobs: num of ``n_jobs`` for parallelization (``-1`` for use all cpu's) + eps: The accuracy of the solution of the problem. Less value - higher search accuracy, less likely to stop + prematurely. + r: Reliability parameter. Higher r is slower convergence, higher probability of finding a global minimum. + evolvent_density: Density of the evolvent. By default :math:`2^{-10}` on hypercube :math:`[0,1]^N`, + which means, that the maximum search accuracy is :math:`2^{-10}`. + eps_r: Parameter that affects the speed of solving the task. epsR = 0 - slow convergence + to the exact solution, epsR>0 - quick converge to the neighborhood of the solution. + refine_solution: if true, then the solution will be refined with local search. + deviation: required improvement (in percent) of a metric to return tuned graph. + By default, ``deviation=0.05``, which means that tuned graph will be returned + if it's metric will be at least 0.05% better than the initial. + """ + + def __init__(self, objective_evaluate: ObjectiveEvaluate, + search_space: SearchSpace, + adapter: Optional[BaseOptimizationAdapter] = None, + iterations: int = 100, + timeout: timedelta = timedelta(minutes=5), + n_jobs: int = -1, + eps: float = 0.001, + r: float = 2.0, + evolvent_density: int = 10, + eps_r: float = 0.001, + refine_solution: bool = False, + deviation: float = 0.05, **kwargs): + super().__init__(objective_evaluate, + search_space, + adapter, + iterations=iterations, + timeout=timeout, + n_jobs=n_jobs, + deviation=deviation, **kwargs) + self.n_jobs = determine_n_jobs(self.n_jobs) + self.solver_parameters = SolverParameters(r=np.double(r), + eps=np.double(eps), + iters_limit=iterations, + evolvent_density=evolvent_density, + eps_r=np.double(eps_r), + refine_solution=refine_solution, + number_of_parallel_points=self.n_jobs, + timeout=round(timeout.total_seconds()/60) if self.timeout else -1) + + def _tune(self, graph: DomainGraphForTune, show_progress: bool = True) -> DomainGraphForTune: + problem_parameters, initial_parameters = self._get_parameters_for_tune(graph) + + has_parameters_to_optimize = (len(problem_parameters.discrete_parameters_names) > 0 or + len(problem_parameters.float_parameters_names) > 0) + self.objectives_number = len(ensure_wrapped_in_sequence(self.init_metric)) + is_multi_objective = self.objectives_number > 1 + + if self._check_if_tuning_possible(graph, has_parameters_to_optimize, supports_multi_objective=True): + if initial_parameters: + initial_point = Point(**initial_parameters) + self.solver_parameters.start_point = initial_point + + problem = GolemProblem(graph, self.objective_evaluate, problem_parameters, self.objectives_number) + solver = Solver(problem, parameters=self.solver_parameters) + + if show_progress: + console_output = ConsoleOutputListener(mode='full') + solver.add_listener(console_output) + + solver.solve() + solution = solver.get_results() + if not is_multi_objective: + best_point = solution.best_trials[0].point + best_parameters = problem.get_parameters_dict_from_iopt_point(best_point) + tuned_graphs = self.set_arg_graph(graph, best_parameters) + self.was_tuned = True + else: + tuned_graphs = [] + for best_trial in solution.best_trials: + best_parameters = problem.get_parameters_dict_from_iopt_point(best_trial.point) + tuned_graph = self.set_arg_graph(deepcopy(graph), best_parameters) + tuned_graphs.append(tuned_graph) + self.was_tuned = True + else: + tuned_graphs = graph + + return tuned_graphs + + def _get_parameters_for_tune(self, graph: OptGraph) -> Tuple[IOptProblemParameters, dict]: + """ Method for defining the search space + + Args: + graph: graph to be tuned + + Returns: + parameters_dict: dict with operation names and parameters + initial_parameters: dict with initial parameters of the graph + """ + float_parameters_dict = {} + discrete_parameters_dict = {} + has_init_parameters = any(len(node.parameters) > 0 for node in graph.nodes) + initial_parameters = {'float_variables': [], 'discrete_variables': []} if has_init_parameters else None + for node_id, node in enumerate(graph.nodes): + operation_name = node.name + + # Assign unique prefix for each model hyperparameter + # label - number of node in the graph + float_node_parameters, discrete_node_parameters = get_node_parameters_for_iopt( + self.search_space, + node_id, + operation_name) + if has_init_parameters: + # Set initial parameters for search + for parameter, bounds in convert_parameters(float_node_parameters).items(): + # If parameter is not set use parameter minimum possible value + initial_value = node.parameters.get(parameter) or bounds[0] + initial_parameters['float_variables'].append(initial_value) + + for parameter, values in convert_parameters(discrete_node_parameters).items(): + # If parameter is not set use the last value + initial_value = node.parameters.get(parameter) or values[-1] + initial_parameters['discrete_variables'].append(initial_value) + + float_parameters_dict.update(float_node_parameters) + discrete_parameters_dict.update(discrete_node_parameters) + parameters_dict = IOptProblemParameters.from_parameters_dicts(float_parameters_dict, discrete_parameters_dict) + return parameters_dict, initial_parameters + + +def get_node_parameters_for_iopt(search_space: SearchSpace, node_id: int, operation_name: str) \ + -> Tuple[Dict[str, List], Dict[str, List]]: + """ + Method for forming dictionary with hyperparameters of node operation for the ``IOptTuner`` + + Args: + search_space: SearchSpace with parameters per operation + node_id: number of node in graph.nodes list + operation_name: name of operation in the node + + Returns: + float_parameters_dict: dictionary-like structure with labeled float hyperparameters + and their range per operation + discrete_parameters_dict: dictionary-like structure with labeled discrete hyperparameters + and their range per operation + """ + # Get available parameters for operation + parameters_dict = search_space.parameters_per_operation.get(operation_name, {}) + + discrete_parameters_dict = {} + float_parameters_dict = {} + categorical_parameters_dict = {} + + for parameter_name, parameter_properties in parameters_dict.items(): + node_op_parameter_name = get_node_operation_parameter_label(node_id, operation_name, parameter_name) + + parameter_type = parameter_properties.get('type') + if parameter_type == 'discrete': + discrete_parameters_dict.update({node_op_parameter_name: list(range(*parameter_properties + .get('sampling-scope')))}) + elif parameter_type == 'continuous': + float_parameters_dict.update({node_op_parameter_name: parameter_properties + .get('sampling-scope')}) + elif parameter_type == 'categorical': + categorical_parameters_dict.update({node_op_parameter_name: parameter_properties + .get('sampling-scope')[0]}) + + # IOpt does not distinguish between discrete and categorical parameters + discrete_parameters_dict = {**discrete_parameters_dict, **categorical_parameters_dict} + return float_parameters_dict, discrete_parameters_dict diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/tuning/optuna_tuner.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/tuning/optuna_tuner.py new file mode 100644 index 0000000..aac3990 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/tuning/optuna_tuner.py @@ -0,0 +1,135 @@ +from copy import deepcopy +from datetime import timedelta +from functools import partial +from typing import Optional, Tuple, Union, Sequence + +import optuna +from optuna import Trial, Study +from optuna.trial import FrozenTrial + +from golem.core.adapter import BaseOptimizationAdapter +from golem.core.optimisers.graph import OptGraph +from golem.core.optimisers.objective import ObjectiveFunction +from golem.core.tuning.search_space import SearchSpace, get_node_operation_parameter_label +from golem.core.tuning.tuner_interface import BaseTuner, DomainGraphForTune +from golem.utilities.data_structures import ensure_wrapped_in_sequence + + +class OptunaTuner(BaseTuner): + def __init__(self, objective_evaluate: ObjectiveFunction, + search_space: SearchSpace, + adapter: Optional[BaseOptimizationAdapter] = None, + iterations: int = 100, + early_stopping_rounds: Optional[int] = None, + timeout: timedelta = timedelta(minutes=5), + n_jobs: int = -1, + deviation: float = 0.05, **kwargs): + super().__init__(objective_evaluate, + search_space, + adapter, + iterations, + early_stopping_rounds, + timeout, + n_jobs, + deviation, **kwargs) + self.study = None + + def _tune(self, graph: DomainGraphForTune, show_progress: bool = True) -> \ + Union[DomainGraphForTune, Sequence[DomainGraphForTune]]: + predefined_objective = partial(self.objective, graph=graph) + + self.objectives_number = len(ensure_wrapped_in_sequence(self.init_metric)) + is_multi_objective = self.objectives_number > 1 + + self.study = optuna.create_study(directions=['minimize'] * self.objectives_number) + + init_parameters, has_parameters_to_optimize = self._get_initial_point(graph) + remaining_time = self._get_remaining_time() + if self._check_if_tuning_possible(graph, + has_parameters_to_optimize, + remaining_time, + supports_multi_objective=True): + # Enqueue initial point to try + if init_parameters: + self.study.enqueue_trial(init_parameters) + + verbosity_level = optuna.logging.INFO if show_progress else optuna.logging.WARNING + optuna.logging.set_verbosity(verbosity_level) + + self.study.optimize(predefined_objective, + n_trials=self.iterations, + n_jobs=self.n_jobs, + timeout=remaining_time, + callbacks=[self.early_stopping_callback] if not is_multi_objective else None, + show_progress_bar=show_progress) + + if not is_multi_objective: + best_parameters = self.study.best_trials[0].params + tuned_graphs = self.set_arg_graph(graph, best_parameters) + self.was_tuned = True + else: + tuned_graphs = [] + for best_trial in self.study.best_trials: + best_parameters = best_trial.params + tuned_graph = self.set_arg_graph(deepcopy(graph), best_parameters) + tuned_graphs.append(tuned_graph) + self.was_tuned = True + else: + tuned_graphs = graph + return tuned_graphs + + def objective(self, trial: Trial, graph: OptGraph) -> Union[float, Sequence[float, ]]: + new_parameters = self._get_parameters_from_trial(graph, trial) + new_graph = BaseTuner.set_arg_graph(graph, new_parameters) + metric_value = self.get_metric_value(new_graph) + return metric_value + + def _get_parameters_from_trial(self, graph: OptGraph, trial: Trial) -> dict: + new_parameters = {} + for node_id, node in enumerate(graph.nodes): + operation_name = node.name + + # Get available parameters for operation + tunable_node_params = self.search_space.parameters_per_operation.get(operation_name, {}) + + for parameter_name, parameter_properties in tunable_node_params.items(): + node_op_parameter_name = get_node_operation_parameter_label(node_id, operation_name, parameter_name) + + parameter_type = parameter_properties.get('type') + sampling_scope = parameter_properties.get('sampling-scope') + if parameter_type == 'discrete': + new_parameters.update({node_op_parameter_name: + trial.suggest_int(node_op_parameter_name, *sampling_scope)}) + elif parameter_type == 'continuous': + new_parameters.update({node_op_parameter_name: + trial.suggest_float(node_op_parameter_name, *sampling_scope)}) + elif parameter_type == 'categorical': + new_parameters.update({node_op_parameter_name: + trial.suggest_categorical(node_op_parameter_name, *sampling_scope)}) + return new_parameters + + def _get_initial_point(self, graph: OptGraph) -> Tuple[dict, bool]: + initial_parameters = {} + has_parameters_to_optimize = False + for node_id, node in enumerate(graph.nodes): + operation_name = node.name + + # Get available parameters for operation + tunable_node_params = self.search_space.parameters_per_operation.get(operation_name) + + if tunable_node_params: + has_parameters_to_optimize = True + tunable_initial_params = {get_node_operation_parameter_label(node_id, operation_name, p): + node.parameters[p] for p in node.parameters if p in tunable_node_params} + if tunable_initial_params: + initial_parameters.update(tunable_initial_params) + return initial_parameters, has_parameters_to_optimize + + def early_stopping_callback(self, study: Study, trial: FrozenTrial): + if self.early_stopping_rounds is not None: + current_trial_number = trial.number + best_trial_number = study.best_trial.number + should_stop = (current_trial_number - best_trial_number) >= self.early_stopping_rounds + if should_stop: + self.log.debug('Early stopping rounds criteria was reached') + study.stop() diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/tuning/search_space.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/tuning/search_space.py new file mode 100644 index 0000000..bfb29fb --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/tuning/search_space.py @@ -0,0 +1,61 @@ +from typing import Dict, Callable, List, Union + +OperationParametersMapping = Dict[str, Dict[str, Dict[str, Union[Callable, List, str]]]] + + +class SearchSpace: + """ + Args: + search_space: dictionary with parameters and their search_space + {'operation_name': {'param_name': {'hyperopt-dist': hyperopt distribution function, + 'sampling-scope': [sampling scope], + 'type': 'discrete', 'continuous' or 'categorical'}, ...}, ...}, + + e.g. ``{'operation_name': {'param1': {'hyperopt-dist': hp.uniformint, + 'sampling-scope': [2, 21]), + 'type': 'discrete'}, + 'param2': {'hyperopt-dist': hp.loguniform, + 'sampling-scope': [0.001, 1]), + 'type': 'continuous'}, + 'param3': {'hyperopt-dist': hp.choice, + 'sampling-scope': [['svd', 'lsqr', 'eigen']), + 'type': 'categorical'}...}, ..} + """ + + def __init__(self, search_space: OperationParametersMapping): + self.parameters_per_operation = search_space + + def get_parameters_for_operation(self, operation_name: str) -> List[str]: + parameters_list = list(self.parameters_per_operation.get(operation_name, {}).keys()) + return parameters_list + + +def get_node_operation_parameter_label(node_id: int, operation_name: str, parameter_name: str) -> str: + # Name with operation and parameter + op_parameter_name = ''.join((operation_name, ' | ', parameter_name)) + + # Name with node id || operation | parameter + node_op_parameter_name = ''.join((str(node_id), ' || ', op_parameter_name)) + return node_op_parameter_name + + +def convert_parameters(parameters): + """ + Function removes labels from dictionary with operations + + Args: + parameters: labeled parameters + + Returns: + new_parameters: dictionary without labels of node_id and operation_name + """ + + new_parameters = {} + for operation_parameter, value in parameters.items(): + # Remove right part of the parameter name + parameter_name = operation_parameter.split(' | ')[-1] + + if value is not None: + new_parameters.update({parameter_name: value}) + + return new_parameters diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/tuning/sequential.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/tuning/sequential.py new file mode 100644 index 0000000..2ac2229 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/tuning/sequential.py @@ -0,0 +1,220 @@ +from copy import deepcopy +from datetime import timedelta +from functools import partial +from typing import Callable, Optional + +from hyperopt import tpe, fmin, space_eval, Trials + +from golem.core.adapter import BaseOptimizationAdapter +from golem.core.constants import MIN_TIME_FOR_TUNING_IN_SEC +from golem.core.optimisers.graph import OptGraph +from golem.core.optimisers.objective import ObjectiveFunction +from golem.core.tuning.hyperopt_tuner import HyperoptTuner, get_node_parameters_for_hyperopt +from golem.core.tuning.search_space import SearchSpace +from golem.core.tuning.tuner_interface import DomainGraphForTune + + +class SequentialTuner(HyperoptTuner): + """ + Class for hyperparameters optimization for all nodes sequentially + """ + + def __init__(self, objective_evaluate: ObjectiveFunction, + search_space: SearchSpace, + adapter: Optional[BaseOptimizationAdapter] = None, + iterations: int = 100, + early_stopping_rounds: Optional[int] = None, + timeout: timedelta = timedelta(minutes=5), + n_jobs: int = -1, + deviation: float = 0.05, + algo: Callable = tpe.suggest, + inverse_node_order: bool = False, **kwargs): + super().__init__(objective_evaluate, + search_space, + adapter, + iterations, + early_stopping_rounds, timeout, + n_jobs, + deviation, + algo, **kwargs) + + self.inverse_node_order = inverse_node_order + + def _tune(self, graph: DomainGraphForTune, **kwargs) -> DomainGraphForTune: + """ Method for hyperparameters tuning on the entire graph + + Args: + graph: graph which hyperparameters will be tuned + """ + remaining_time = self._get_remaining_time() + if self._check_if_tuning_possible(graph, parameters_to_optimize=True, remaining_time=remaining_time): + # Calculate amount of iterations we can apply per node + nodes_amount = graph.length + iterations_per_node = round(self.iterations / nodes_amount) + iterations_per_node = int(iterations_per_node) + if iterations_per_node == 0: + iterations_per_node = 1 + + # Calculate amount of seconds we can apply per node + if remaining_time is not None: + seconds_per_node = round(remaining_time / nodes_amount) + seconds_per_node = int(seconds_per_node) + else: + seconds_per_node = None + + # Tuning performed sequentially for every node - so get ids of nodes + nodes_ids = self.get_nodes_order(nodes_number=nodes_amount) + final_graph = deepcopy(self.init_graph) + best_metric = self.init_metric + for node_id in nodes_ids: + node = graph.nodes[node_id] + + # Get node's parameters to optimize + node_params, init_params = get_node_parameters_for_hyperopt(self.search_space, node_id, node) + if not node_params: + self.log.info(f'"{node.name}" operation has no parameters to optimize') + else: + # Apply tuning for current node + graph, metric = self._optimize_node(node_id=node_id, + graph=graph, + node_params=node_params, + init_params=init_params, + iterations_per_node=iterations_per_node, + seconds_per_node=seconds_per_node) + if metric <= best_metric: + final_graph = deepcopy(graph) + best_metric = metric + self.was_tuned = True + return final_graph + + def get_nodes_order(self, nodes_number: int) -> range: + """ Method returns list with indices of nodes in the graph + + Args: + nodes_number: number of nodes to get + """ + + if self.inverse_node_order is True: + # From source data to output + nodes_ids = range(nodes_number - 1, -1, -1) + else: + # From output to source data + nodes_ids = range(0, nodes_number) + + return nodes_ids + + def tune_node(self, graph: DomainGraphForTune, node_index: int) -> DomainGraphForTune: + """ Method for hyperparameters tuning for particular node + + Args: + graph: graph which contains a node to be tuned + node_index: Index of the node to tune + + Returns: + Graph with tuned parameters in node with specified index + """ + graph = self.adapter.adapt(graph) + + with self.timer: + self.init_check(graph) + + node = graph.nodes[node_index] + + # Get node's parameters to optimize + node_params, init_params = get_node_parameters_for_hyperopt(self.search_space, + node_id=node_index, + node=node) + + remaining_time = self._get_remaining_time() + if self._check_if_tuning_possible(graph, len(node_params) > 1, remaining_time): + # Apply tuning for current node + graph, _ = self._optimize_node(graph=graph, + node_id=node_index, + node_params=node_params, + init_params=init_params, + iterations_per_node=self.iterations, + seconds_per_node=remaining_time + ) + + self.was_tuned = True + + # Validation is the optimization do well + final_graph = self.final_check(graph) + else: + final_graph = graph + self.obtained_metric = self.init_metric + final_graph = self.adapter.restore(final_graph) + return final_graph + + def _optimize_node(self, graph: OptGraph, + node_id: int, + node_params: dict, + init_params: dict, + iterations_per_node: int, + seconds_per_node: float) -> OptGraph: + """ + Method for node optimization + + Args: + graph: Graph which node is optimized + node_id: id of the current node in the graph + node_params: dictionary with parameters for node + iterations_per_node: amount of iterations to produce + seconds_per_node: amount of seconds to produce + + Returns: + updated graph with tuned parameters in particular node + """ + remaining_time = self._get_remaining_time() + trials = Trials() + trials, init_trials_num = self._search_near_initial_parameters(partial(self._objective, + graph=graph, + node_id=node_id, + unchangeable_parameters=init_params), + node_params, + init_params, + trials, + remaining_time) + + remaining_time = self._get_remaining_time() + if remaining_time > MIN_TIME_FOR_TUNING_IN_SEC: + fmin(partial(self._objective, graph=graph, node_id=node_id), + node_params, + trials=trials, + algo=self.algo, + max_evals=iterations_per_node, + early_stop_fn=self.early_stop_fn, + timeout=seconds_per_node) + + best_params = space_eval(space=node_params, hp_assignment=trials.argmin) + is_best_trial_with_init_params = trials.best_trial.get('tid') in range(init_trials_num) + if is_best_trial_with_init_params: + best_params = {**best_params, **init_params} + # Set best params for this node in the graph + graph = self.set_arg_node(graph=graph, node_id=node_id, node_params=best_params) + return graph, trials.best_trial['result']['loss'] + + def _objective(self, + node_params: dict, + graph: OptGraph, + node_id: int, + unchangeable_parameters: Optional[dict] = None) -> float: + """ Objective function for minimization problem + + Args: + node_params: dictionary with parameters for node + graph: graph to evaluate + node_id: id of the node to which parameters should be assigned + + Returns: + value of objective function + """ + # replace new parameters with parameters + if unchangeable_parameters: + node_params = {**node_params, **unchangeable_parameters} + + # Set hyperparameters for node + graph = self.set_arg_node(graph=graph, node_id=node_id, node_params=node_params) + + metric_value = self.get_metric_value(graph=graph) + return metric_value diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/tuning/simultaneous.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/tuning/simultaneous.py new file mode 100644 index 0000000..797036d --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/tuning/simultaneous.py @@ -0,0 +1,122 @@ +from functools import partial +from typing import Tuple, Optional + +from hyperopt import Trials, fmin, space_eval + +from golem.core.constants import MIN_TIME_FOR_TUNING_IN_SEC +from golem.core.optimisers.graph import OptGraph +from golem.core.tuning.hyperopt_tuner import HyperoptTuner, get_node_parameters_for_hyperopt +from golem.core.tuning.tuner_interface import DomainGraphForTune + + +class SimultaneousTuner(HyperoptTuner): + """ + Class for hyperparameters optimization for all nodes simultaneously + """ + + def _tune(self, graph: DomainGraphForTune, show_progress: bool = True) -> DomainGraphForTune: + """ Function for hyperparameters tuning on the entire graph + + Args: + graph: graph which hyperparameters will be tuned + show_progress: shows progress of tuning if True + + Returns: + Graph with tuned hyperparameters + """ + parameters_dict, init_parameters = self._get_parameters_for_tune(graph) + remaining_time = self._get_remaining_time() + + if self._check_if_tuning_possible(graph, len(parameters_dict) > 0, remaining_time): + trials = Trials() + + try: + # try searching using initial parameters + # (uses original search space with fixed initial parameters) + trials, init_trials_num = self._search_near_initial_parameters( + partial(self._objective, + graph=graph, + unchangeable_parameters=init_parameters), + parameters_dict, + init_parameters, + trials, + remaining_time, + show_progress) + remaining_time = self._get_remaining_time() + if remaining_time > MIN_TIME_FOR_TUNING_IN_SEC: + fmin(partial(self._objective, graph=graph), + parameters_dict, + trials=trials, + algo=self.algo, + max_evals=self.iterations, + show_progressbar=show_progress, + early_stop_fn=self.early_stop_fn, + timeout=remaining_time) + else: + self.log.message('Tunner stopped after initial search due to the lack of time') + + best = space_eval(space=parameters_dict, hp_assignment=trials.argmin) + # check if best point was obtained using search space with fixed initial parameters + is_best_trial_with_init_params = trials.best_trial.get('tid') in range(init_trials_num) + if is_best_trial_with_init_params: + best = {**best, **init_parameters} + + final_graph = self.set_arg_graph(graph=graph, parameters=best) + + self.was_tuned = True + + except Exception as ex: + self.log.warning(f'Exception {ex} occurred during tuning') + final_graph = graph + else: + final_graph = graph + return final_graph + + def _get_parameters_for_tune(self, graph: OptGraph) -> Tuple[dict, dict]: + """ Method for defining the search space + + Args: + graph: graph to be tuned + + Returns: + parameters_dict: dict with operation names and parameters + initial_parameters: dict with initial parameters of the graph + """ + + parameters_dict = {} + initial_parameters = {} + for node_id, node in enumerate(graph.nodes): + # Assign unique prefix for each model hyperparameter + # label - number of node in the graph + tunable_node_params, initial_node_params = get_node_parameters_for_hyperopt(self.search_space, + node_id=node_id, + node=node) + parameters_dict.update(tunable_node_params) + initial_parameters.update(initial_parameters) + + return parameters_dict, initial_parameters + + def _objective(self, parameters_dict: dict, graph: OptGraph, unchangeable_parameters: Optional[dict] = None) \ + -> float: + """ + Objective function for minimization problem + + Args: + parameters_dict: dict which contains new graph hyperparameters + graph: graph to optimize + unchangeable_parameters: dict with parameters that should not be changed + + Returns: + metric_value: value of objective function + """ + + # replace new parameters with parameters + if unchangeable_parameters: + parameters_dict = {**parameters_dict, **unchangeable_parameters} + + # Set hyperparameters for every node + graph = self.set_arg_graph(graph, parameters_dict) + + metric_value = self.get_metric_value(graph=graph) + + return metric_value diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/core/tuning/tuner_interface.py b/paper_experiments/large_bayesian_networks_experiments/golem/core/tuning/tuner_interface.py new file mode 100644 index 0000000..9563e43 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/core/tuning/tuner_interface.py @@ -0,0 +1,279 @@ +from abc import abstractmethod +from copy import deepcopy +from datetime import timedelta +from typing import TypeVar, Generic, Optional, Union, Sequence + +import numpy as np + +from golem.core.adapter import BaseOptimizationAdapter +from golem.core.adapter.adapter import IdentityAdapter +from golem.core.constants import MAX_TUNING_METRIC_VALUE, MIN_TIME_FOR_TUNING_IN_SEC +from golem.core.dag.graph_utils import graph_structure +from golem.core.log import default_log +from golem.core.optimisers.fitness import SingleObjFitness, MultiObjFitness +from golem.core.optimisers.graph import OptGraph +from golem.core.optimisers.objective import ObjectiveEvaluate, ObjectiveFunction +from golem.core.optimisers.timer import Timer +from golem.core.tuning.search_space import SearchSpace, convert_parameters +from golem.utilities.data_structures import ensure_wrapped_in_sequence + +DomainGraphForTune = TypeVar('DomainGraphForTune') + + +class BaseTuner(Generic[DomainGraphForTune]): + """ + Base class for hyperparameters optimization + + Args: + objective_evaluate: objective to optimize + adapter: the function for processing of external object that should be optimized + search_space: SearchSpace instance + iterations: max number of iterations + early_stopping_rounds: Optional max number of stagnating iterations for early stopping. + timeout: max time for tuning + n_jobs: num of ``n_jobs`` for parallelization (``-1`` for use all cpu's) + deviation: required improvement (in percent) of a metric to return tuned graph. + By default, ``deviation=0.05``, which means that tuned graph will be returned + if it's metric will be at least 0.05% better than the initial. + """ + + def __init__(self, objective_evaluate: ObjectiveFunction, + search_space: SearchSpace, + adapter: Optional[BaseOptimizationAdapter] = None, + iterations: int = 100, + early_stopping_rounds: Optional[int] = None, + timeout: timedelta = timedelta(minutes=5), + n_jobs: int = -1, + deviation: float = 0.05, **kwargs): + self.iterations = iterations + self.adapter = adapter or IdentityAdapter() + self.search_space = search_space + self.n_jobs = n_jobs + if isinstance(objective_evaluate, ObjectiveEvaluate): + objective_evaluate.eval_n_jobs = self.n_jobs + self.objective_evaluate = self.adapter.adapt_func(objective_evaluate) + self.deviation = deviation + + self.timeout = timeout + self.timer = Timer() + self.early_stopping_rounds = early_stopping_rounds + + self._default_metric_value = MAX_TUNING_METRIC_VALUE + self.was_tuned = False + self.init_graph = None + self.init_metric = None + self.obtained_metric = None + self.log = default_log(self) + self.objectives_number = 1 + + def tune(self, graph: DomainGraphForTune, **kwargs) -> Union[DomainGraphForTune, Sequence[DomainGraphForTune]]: + """ + Function for hyperparameters tuning on the graph + + Args: + graph: domain graph for which hyperparameters tuning is needed + + Returns: + Graph with optimized hyperparameters + or pareto front of optimized graphs in case of multi-objective optimization + """ + graph = self.adapter.adapt(graph) + self.was_tuned = False + with self.timer: + + # Check source metrics for data + self.init_check(graph) + final_graph = self._tune(graph, **kwargs) + # Validate if optimisation did well + final_graph = self.final_check(final_graph, self.objectives_number > 1) + + final_graph = self.adapter.restore(final_graph) + return final_graph + + @abstractmethod + def _tune(self, graph: DomainGraphForTune, **kwargs): + raise NotImplementedError + + def init_check(self, graph: OptGraph) -> None: + """ + Method get metric on validation set before start optimization + + Args: + graph: graph to calculate objective + multi_obj: If optimization was multi objective. + """ + self.log.info('Hyperparameters optimization start: estimation of metric for initial graph') + + # Train graph + self.init_graph = deepcopy(graph) + + self.init_metric = self.get_metric_value(graph=self.init_graph) + self.log.message(f'Initial graph: {graph_structure(self.init_graph)} \n' + f'Initial metric: ' + f'{list(map(lambda x: round(abs(x), 3), ensure_wrapped_in_sequence(self.init_metric)))}') + + def final_check(self, tuned_graphs: Union[OptGraph, Sequence[OptGraph]], multi_obj: bool = False) \ + -> Union[OptGraph, Sequence[OptGraph]]: + """ + Method propose final quality check after optimization process + + Args: + tuned_graphs: Tuned graph to calculate objective + multi_obj: If optimization was multi objective. + """ + self.log.info('Hyperparameters optimization finished') + + if multi_obj: + return self._multi_obj_final_check(tuned_graphs) + else: + return self._single_obj_final_check(tuned_graphs) + + def _single_obj_final_check(self, tuned_graph: OptGraph): + self.obtained_metric = self.get_metric_value(graph=tuned_graph) + + prefix_tuned_phrase = 'Return tuned graph due to the fact that obtained metric' + prefix_init_phrase = 'Return init graph due to the fact that obtained metric' + + if np.isclose(self.obtained_metric, self._default_metric_value): + self.obtained_metric = None + + # 0.05% deviation is acceptable + deviation_value = (self.init_metric / 100.0) * self.deviation + init_metric = self.init_metric + deviation_value * (-np.sign(self.init_metric)) + if self.obtained_metric is None: + self.log.info(f'{prefix_init_phrase} is None. Initial metric is {abs(init_metric):.3f}') + final_graph = self.init_graph + final_metric = self.init_metric + elif self.obtained_metric <= init_metric: + self.log.info(f'{prefix_tuned_phrase} {abs(self.obtained_metric):.3f} equal or ' + f'better than initial (+ {self.deviation}% deviation) {abs(init_metric):.3f}') + final_graph = tuned_graph + final_metric = self.obtained_metric + else: + self.log.info(f'{prefix_init_phrase} {abs(self.obtained_metric):.3f} ' + f'worse than initial (+ {self.deviation}% deviation) {abs(init_metric):.3f}') + final_graph = self.init_graph + final_metric = self.init_metric + self.obtained_metric = final_metric + self.log.message(f'Final graph: {graph_structure(final_graph)}') + if final_metric is not None: + self.log.message(f'Final metric: {abs(final_metric):.3f}') + else: + self.log.message('Final metric is None') + return final_graph + + def _multi_obj_final_check(self, tuned_graphs: Sequence[OptGraph]) -> Sequence[OptGraph]: + self.obtained_metric = [] + final_graphs = [] + for tuned_graph in tuned_graphs: + obtained_metric = self.get_metric_value(graph=tuned_graph) + for e, value in enumerate(obtained_metric): + if np.isclose(value, self._default_metric_value): + obtained_metric[e] = None + if not MultiObjFitness(self.init_metric).dominates(MultiObjFitness(obtained_metric)): + self.obtained_metric.append(obtained_metric) + final_graphs.append(tuned_graph) + if final_graphs: + metrics_formatted = [str([round(x, 3) for x in metrics]) for metrics in self.obtained_metric] + metrics_formatted = '\n'.join(metrics_formatted) + self.log.message('Return tuned graphs with obtained metrics \n' + f'{metrics_formatted}') + else: + self.log.message('Initial metric dominates all found solutions. Return initial graph.') + final_graphs = [self.init_graph] + self.obtained_metric = [self.init_metric] + return final_graphs + + def get_metric_value(self, graph: OptGraph) -> Union[float, Sequence[float]]: + """ + Method calculates metric for algorithm validation + + Args: + graph: Graph to evaluate + + Returns: + value of loss function + """ + graph_fitness = self.objective_evaluate(graph) + + if isinstance(graph_fitness, SingleObjFitness): + metric_value = graph_fitness.value + if not graph_fitness.valid: + return self._default_metric_value + return metric_value + + elif isinstance(graph_fitness, MultiObjFitness): + metric_values = graph_fitness.values + for e, value in enumerate(metric_values): + if value is None: + metric_values[e] = self._default_metric_value + return metric_values + + @staticmethod + def set_arg_graph(graph: OptGraph, parameters: dict) -> OptGraph: + """ Method for parameters setting to a graph + + Args: + graph: graph to which parameters should be assigned + parameters: dictionary with parameters to set + + Returns: + graph: graph with new hyperparameters in each node + """ + # Set hyperparameters for every node + for node_id, node in enumerate(graph.nodes): + node_params = {key: value for key, value in parameters.items() + if key.startswith(f'{str(node_id)} || {node.name}')} + + if node_params is not None: + BaseTuner.set_arg_node(graph, node_id, node_params) + + return graph + + @staticmethod + def set_arg_node(graph: OptGraph, node_id: int, node_params: dict) -> OptGraph: + """ Method for parameters setting to a graph + + Args: + graph: graph which contains the node + node_id: id of the node to which parameters should be assigned + node_params: dictionary with labeled parameters to set + + Returns: + graph with new hyperparameters in each node + """ + + # Remove label prefixes + node_params = convert_parameters(node_params) + + # Update parameters in nodes + graph.nodes[node_id].parameters = node_params + + return graph + + def _check_if_tuning_possible(self, graph: OptGraph, + parameters_to_optimize: bool, + remaining_time: Optional[float] = None, + supports_multi_objective: bool = False) -> bool: + if len(ensure_wrapped_in_sequence(self.init_metric)) > 1 and not supports_multi_objective: + self._stop_tuning_with_message(f'{self.__class__.__name__} does not support multi-objective optimization.') + return False + elif not parameters_to_optimize: + self._stop_tuning_with_message(f'Graph "{graph.graph_description}" has no parameters to optimize') + return False + elif remaining_time is not None: + if remaining_time <= MIN_TIME_FOR_TUNING_IN_SEC: + self._stop_tuning_with_message('Tunner stopped after initial assumption due to the lack of time') + return False + return True + + def _stop_tuning_with_message(self, message: str): + self.log.message(message) + self.obtained_metric = self.init_metric + + def _get_remaining_time(self) -> Optional[float]: + if self.timeout is not None: + remaining_time = self.timeout.seconds - self.timer.seconds_from_start + return remaining_time + else: + return None diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/metrics/__init__.py b/paper_experiments/large_bayesian_networks_experiments/golem/metrics/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/metrics/edit_distance.py b/paper_experiments/large_bayesian_networks_experiments/golem/metrics/edit_distance.py new file mode 100644 index 0000000..677285e --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/metrics/edit_distance.py @@ -0,0 +1,117 @@ +from datetime import timedelta, datetime +from itertools import zip_longest +from typing import Optional, Callable, Dict, Sequence + +import networkx as nx +import numpy as np +import zss +from networkx import graph_edit_distance, is_tree + +from examples.synthetic_graph_evolution.generators import generate_labeled_graph +from golem.core.optimisers.optimization_parameters import GraphRequirements +from golem.metrics.graph_metrics import min_max +from libs.netcomp import edit_distance + + +def _label_dist(label1: str, label2: str) -> int: + return int(label1 != label2) + + +def tree_edit_dist(target_graph: nx.DiGraph, graph: nx.DiGraph) -> float: + """Compares nodes by their `name` (if present) or `uid` attribute. + Nodes with the same name/id are considered the same.""" + if not (is_tree(target_graph) and is_tree(graph)): + raise ValueError('Both target graphs must be trees') + target_tree_root = _nx_to_zss_tree(target_graph) + cmp_tree_root = _nx_to_zss_tree(graph) + dist = zss.simple_distance(target_tree_root, cmp_tree_root, label_dist=_label_dist) + return dist + + +def graph_size(target_graph: nx.DiGraph, graph: nx.DiGraph) -> int: + return abs(target_graph.number_of_nodes() - graph.number_of_nodes()) + + +def _nx_to_zss_tree(graph: nx.DiGraph) -> zss.Node: + # Root is the node without successors + root = _get_root_node(graph) + # that's why we first reverse the tree to get proper DFS traverse + tree = graph.reverse() + # Add nodes with appropriate labels for comparison + nodes_dict = {} + for node_id, node_data in tree.nodes(data=True): + label = node_data.get('name', node_id) + nodes_dict[node_id] = zss.Node(label) + # Add edges + for edge in tree.edges(): + nodes_dict[edge[0]].addkid(nodes_dict[edge[1]]) + return nodes_dict[root] + + +def _get_root_node(nxgraph: nx.DiGraph) -> Sequence: + source = [n for (n, d) in nxgraph.out_degree() if d == 0][0] + return source + + +def get_edit_dist_metric(target_graph: nx.DiGraph, + timeout=timedelta(seconds=60), + upper_bound: Optional[int] = None, + requirements: Optional[GraphRequirements] = None, + ) -> Callable[[nx.DiGraph], float]: + def node_match(node_content_1: Dict, node_content_2: Dict) -> bool: + operations_do_match = node_content_1.get('name') == node_content_2.get('name') + return True or operations_do_match + + if requirements: + upper_bound = upper_bound or int(np.sqrt(requirements.max_depth * requirements.max_arity)), + timeout = timeout or requirements.max_graph_fit_time + + def metric(graph: nx.DiGraph) -> float: + ged = graph_edit_distance(target_graph, graph, + node_match=node_match, + upper_bound=upper_bound, + timeout=timeout.total_seconds() if timeout else None, + ) + return float(ged) or upper_bound + + return metric + + +def matrix_edit_dist(target_graph: nx.DiGraph, graph: nx.DiGraph) -> float: + target_adj = nx.adjacency_matrix(target_graph) + adj = nx.adjacency_matrix(graph) + nmin, nmax = min_max(target_adj.shape[0], adj.shape[0]) + if nmin != nmax: + shape = (nmax, nmax) + target_adj.resize(shape) + adj.resize(shape) + value = edit_distance(target_adj, adj) + return value + + +def try_tree_edit_distance(sizes1=None, sizes2=None, node_types=None, + print_trees=False): + if not sizes1: + sizes1 = list(range(5, 100, 5)) + if not sizes2: + sizes2 = sizes1 + if not node_types: + node_types = ['X'] + + for i, (n1, n2) in enumerate(zip_longest(sizes1, sizes2)): + g1 = generate_labeled_graph('tree', n1, node_types) + g2 = generate_labeled_graph('tree', n2, node_types) + + start_time = datetime.now() + dist = tree_edit_dist(g1, g2) + duration = datetime.now() - start_time + + print(f'iter {i} with sizes={(n1, n2)} dist={dist}, ' + f't={duration.total_seconds():.3f}s') + if print_trees: + print(nx.forest_str(g1)) + print(nx.forest_str(g2)) + + +if __name__ == "__main__": + try_tree_edit_distance(print_trees=False, node_types=list('XYZWQ')) diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/metrics/graph_features.py b/paper_experiments/large_bayesian_networks_experiments/golem/metrics/graph_features.py new file mode 100644 index 0000000..88c3be7 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/metrics/graph_features.py @@ -0,0 +1,59 @@ +from typing import Callable, Sequence, Tuple + +import networkx as nx +import numpy as np + +from golem.metrics import mmd +from golem.metrics.mmd import compute_mmd + + +def compute_all_stats(graph_prediction: Sequence[nx.Graph], + graph_target: Sequence[nx.Graph]) -> Tuple[float, float]: + mmd_degree = degree_stats(graph_prediction, graph_target) + mmd_clustering = clustering_stats(graph_prediction, graph_target) + + return mmd_degree, mmd_clustering + + +def degree_stats(graph_prediction: Sequence[nx.Graph], + graph_target: Sequence[nx.Graph]) -> float: + return mmd_stats(nx.degree_histogram, graph_prediction, graph_target, normalize=True) + + +def clustering_stats(graph_prediction: Sequence[nx.Graph], + graph_target: Sequence[nx.Graph]) -> float: + bins = 100 + return mmd_stats(clustering_stats_graph, graph_prediction, graph_target, + sigma=0.1, distance_scaling=bins, normalize=False) + + +def mmd_stats(stat_function: Callable[[nx.Graph], np.ndarray], + graph_prediction: Sequence[nx.Graph], + graph_target: Sequence[nx.Graph], + **kwargs): + sample_predict = list(map(stat_function, graph_prediction)) + sample_target = list(map(stat_function, graph_target)) + return compute_mmd(sample_target, sample_predict, **kwargs) + + +def mmd_stats_impl(stat_function: Callable[[nx.Graph], np.ndarray], + graph_prediction: Sequence[nx.Graph], + graph_target: Sequence[nx.Graph], + kernel: Callable = mmd.gaussian_emd, + sigma: float = 1.0, + distance_scaling: float = 1.0, + normalize: bool = False) -> float: + + sample_predict = list(map(stat_function, graph_prediction)) + sample_target = list(map(stat_function, graph_target)) + + mmd_dist = compute_mmd(sample_target, sample_predict, + normalize=normalize, kernel=kernel, + sigma=sigma, distance_scaling=distance_scaling) + return mmd_dist + + +def clustering_stats_graph(graph: nx.Graph, bins: int = 100) -> np.ndarray: + clustering_coeffs = list(nx.clustering(graph).values()) + hist, _ = np.histogram(clustering_coeffs, bins=bins, range=(0.0, 1.0), density=True) + return hist diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/metrics/graph_metrics.py b/paper_experiments/large_bayesian_networks_experiments/golem/metrics/graph_metrics.py new file mode 100644 index 0000000..3616534 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/metrics/graph_metrics.py @@ -0,0 +1,179 @@ +from typing import Sequence + +import networkx as nx +import numpy as np + +from golem.metrics.graph_features import degree_stats +from libs.netcomp import _eigs, normalized_laplacian_eig +from libs.netcomp import laplacian_matrix + + +def nxgraph_stats(graph: nx.Graph): + degrees = nx.degree_histogram(graph) + degrees_norm = np.divide(degrees, np.linalg.norm(degrees)).round(2) + stats = dict( + num_nodes=graph.number_of_nodes(), + num_edges=graph.number_of_edges(), + avg_clustering=np.round(nx.average_clustering(graph), 3), + degrees_hist=degrees, + degrees_hist_norm=degrees_norm, + ) + return stats + + +def degree_distance_kernel(target_graph: nx.DiGraph, graph: nx.DiGraph) -> float: + return degree_stats([graph], [target_graph]) + + +def degree_distance(target_graph: nx.DiGraph, + graph: nx.DiGraph, + normalized: bool = False) -> float: + """This is a heuristic metric for graphs where central + nodes are more important than peripheral ones. The "heavier" + the nodes (i.e. the higher their degree), the more significant + the difference in number of such nodes between two graphs.""" + # Compute histogram of node degrees + degrees_t = np.array(nx.degree_histogram(target_graph), dtype=float) + degrees_g = np.array(nx.degree_histogram(graph), dtype=float) + return degree_dist_weighted_compute(degrees_t, degrees_g, normalized) + + +def degree_dist_weighted_compute(degrees_t: Sequence[float], + degrees_g: Sequence[float], + normalized: bool = False) -> float: + degrees_t = np.asarray(degrees_t) + degrees_g = np.asarray(degrees_g) + + # Extend arrays to the same length with zeros + common_len = max(len(degrees_t), len(degrees_g)) + degrees_t.resize(common_len, refcheck=False) + degrees_g.resize(common_len, refcheck=False) + + # Compute weights as normalized degrees + weights = np.arange(1, common_len + 1).astype(float) + weights /= np.sum(weights) + + # Normalize + if normalized: + degrees_t /= np.sum(degrees_t) + degrees_g /= np.sum(degrees_g) + + # Compute distance between node degrees weighted by degree + dist = np.linalg.norm(weights * (degrees_t - degrees_g)) + return dist + + +def size_diff(target_graph: nx.DiGraph, graph: nx.DiGraph) -> float: + nodes_diff = abs(target_graph.number_of_nodes() - graph.number_of_nodes()) + edges_diff = abs(target_graph.number_of_edges() - graph.number_of_edges()) + return nodes_diff + np.sqrt(edges_diff) + + +def spectral_dist(target_graph: nx.DiGraph, graph: nx.DiGraph, + k: int = 20, kind: str = 'laplacian', + size_diff_penalty: float = 0.2, + match_size: bool = False, + ) -> float: + target_adj = nx.adjacency_matrix(target_graph) + adj = nx.adjacency_matrix(graph) + + # compute spectral distance + value = lambda_dist(target_adj, adj, kind=kind, k=k, match_size=match_size) + + if size_diff_penalty > 1e-5: + value += size_diff_penalty * size_diff(target_graph, graph) + return value + + +def spectral_dists_all(target_graph: nx.DiGraph, graph: nx.DiGraph, + k: int = 20, match_size: bool = True) -> dict: + target_adj = nx.adjacency_matrix(target_graph) + adj = nx.adjacency_matrix(graph) + + print(f'computing metrics for {k} spectral values between {target_adj.shape} & {adj.shape}') + + vals = {} + for kind in ('adjacency', 'laplacian_norm', 'laplacian'): + value = lambda_dist(target_adj, adj, kind=kind, k=k, match_size=match_size) + vals[kind] = np.round(value, 3) + vals['nodes_diff'] = size_diff(target_graph, graph) + return vals + + +def min_max(a, b): + return (a, b) if a <= b else (b, a) + + +def lambda_dist(A1, A2, k=None, p=2, kind='laplacian', match_size=True): + """The lambda distance between graphs, which is defined as + + d(G1,G2) = norm(L_1 - L_2) + + where L_1 is a vector of the top k eigenvalues of the appropriate matrix + associated with G1, and L2 is defined similarly. + + Parameters + ---------- + A1, A2 : NumPy matrices + Adjacency matrices of graphs to be compared + + k : Integer + The number of eigenvalues to be compared + + p : non-zero Float + The p-norm is used to compare the resulting vector of eigenvalues. + + kind : String , in {'laplacian','laplacian_norm','adjacency'} + The matrix for which eigenvalues will be calculated. + + Returns + ------- + dist : float + The distance between the two graphs + + Notes + ----- + The norm can be any p-norm; by default we use p=2. If p<0 is used, the + result is not a mathematical norm, but may still be interesting and/or + useful. + + If k is provided, then we use the k SMALLEST eigenvalues for the Laplacian + distances, and we use the k LARGEST eigenvalues for the adjacency + distance. This is because the corresponding order flips, as L = D-A. + + References + ---------- + + See Also + -------- + netcomp.linalg._eigs + normalized_laplacian_eigs + + """ + # check sizes & determine number of eigenvalues (k) + nmin, nmax = min_max(A1.shape[0], A2.shape[0]) + if match_size: + shape = (nmax, nmax) + A1.resize(shape) + A2.resize(shape) + else: + k = min(k, nmin) + + if kind == 'laplacian': + # form matrices + L1, L2 = [laplacian_matrix(A) for A in [A1, A2]] + # get eigenvalues, ignore eigenvectors + evals1, evals2 = [_eigs(L)[0] for L in [L1, L2]] + elif kind == 'laplacian_norm': + # use our function to graph evals of normalized laplacian + evals1, evals2 = [normalized_laplacian_eig(A)[0] for A in [A1, A2]] + elif kind == 'adjacency': + evals1, evals2 = [_eigs(A)[0] for A in [A1, A2]] + # reverse, so that we are sorted from large to small, since we care + # about the k LARGEST eigenvalues for the adjacency distance + evals1, evals2 = [evals[::-1] for evals in [evals1, evals2]] + else: + raise AttributeError(f"Invalid type {kind}, choose from 'laplacian', " + f"'laplacian_norm', and 'adjacency'.") + dist = np.linalg.norm(evals1[:k] - evals2[:k], ord=p) + return dist diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/metrics/mmd.py b/paper_experiments/large_bayesian_networks_experiments/golem/metrics/mmd.py new file mode 100644 index 0000000..66611a7 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/metrics/mmd.py @@ -0,0 +1,91 @@ +import itertools +import numpy as np +from scipy.linalg import toeplitz +from scipy.stats import wasserstein_distance + + +def emd(x, y, distance_scaling=1.0): + """Earth's mover distance (or Wasserstein metric) + between 2 probability distributions.""" + support_size = max(len(x), len(y)) + d_mat = toeplitz(range(support_size)).astype(np.float64) + distance_mat = d_mat / distance_scaling + + # convert histogram values x and y to float, and make them equal len + x = x.astype(np.float64) + y = y.astype(np.float64) + if len(x) < len(y): + x = np.hstack((x, [0.0] * (support_size - len(x)))) + elif len(y) < len(x): + y = np.hstack((y, [0.0] * (support_size - len(y)))) + + emd_value = wasserstein_distance(x, y, distance_mat) + return emd_value + + +def l2(x, y): + dist = np.linalg.norm(x - y, 2) + return dist + + +def gaussian_emd(x, y, sigma=1.0, distance_scaling=1.0): + ''' Gaussian kernel with squared distance in exponential term replaced by EMD + Args: + x, y: 1D pmf of two distributions with the same support + sigma: standard deviation + ''' + emd_val = emd(x, y, distance_scaling=distance_scaling) + return np.exp(-emd_val ** 2 / (2 * sigma ** 2)) + + +def gaussian(x, y, sigma=1.0): + dist = np.linalg.norm(x - y, 2) + return np.exp(-dist ** 2 / (2 * sigma ** 2)) + + +def discrepancy(samples1, samples2, kernel, *args, **kwargs) -> float: + d = sum(kernel(s1, s2, *args, **kwargs) for s1, s2 in itertools.product(samples1, samples2)) + d /= len(samples1) * len(samples2) + return d + + +def compute_mmd(samples1, samples2, kernel=gaussian_emd, normalize=True, *args, **kwargs) -> float: + ''' MMD between two samples + ''' + # normalize histograms into pmf + if normalize: + samples1 = [s1 / np.sum(s1) for s1 in samples1] + samples2 = [s2 / np.sum(s2) for s2 in samples2] + return discrepancy(samples1, samples1, kernel, *args, **kwargs) + \ + discrepancy(samples2, samples2, kernel, *args, **kwargs) - \ + 2 * discrepancy(samples1, samples2, kernel, *args, **kwargs) + + +def compute_emd(samples1, samples2, kernel, normalize=True, *args, **kwargs) -> float: + ''' EMD between average of two samples + ''' + if normalize: + samples1 = [np.mean(samples1)] + samples2 = [np.mean(samples2)] + return discrepancy(samples1, samples2, kernel, *args, **kwargs) + + +def test_mmd_compute(): + s1 = np.array([0.2, 0.8]) + s2 = np.array([0.3, 0.7]) + samples1 = [s1, s2] + + s3 = np.array([0.25, 0.75]) + s4 = np.array([0.35, 0.65]) + samples2 = [s3, s4] + + s5 = np.array([0.8, 0.2]) + s6 = np.array([0.7, 0.3]) + samples3 = [s5, s6] + + print('between samples1 and samples2: ', compute_mmd(samples1, samples2, kernel=gaussian_emd, sigma=1.0)) + print('between samples1 and samples3: ', compute_mmd(samples1, samples3, kernel=gaussian_emd, sigma=1.0)) + + +if __name__ == '__main__': + test_mmd_compute() diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/serializers/__init__.py b/paper_experiments/large_bayesian_networks_experiments/golem/serializers/__init__.py new file mode 100644 index 0000000..347c9cc --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/serializers/__init__.py @@ -0,0 +1,2 @@ +from .serializer import CLASS_PATH_KEY, INSTANCE_OR_CALLABLE, MODULE_X_NAME_DELIMITER, Serializer +from .any_serialization import any_to_json, any_from_json diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/serializers/any_serialization.py b/paper_experiments/large_bayesian_networks_experiments/golem/serializers/any_serialization.py new file mode 100644 index 0000000..1851303 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/serializers/any_serialization.py @@ -0,0 +1,20 @@ +from copy import deepcopy +from inspect import signature +from typing import Any, Dict, Type, TypeVar, Callable + + +INSTANCE_OR_CALLABLE = TypeVar('INSTANCE_OR_CALLABLE', object, Callable) + + +def any_to_json(obj: INSTANCE_OR_CALLABLE) -> Dict[str, Any]: + return {k: v for k, v in sorted(vars(obj).items()) if not _is_log_var(k)} + + +def any_from_json(cls: Type[INSTANCE_OR_CALLABLE], json_obj: Dict[str, Any]) -> INSTANCE_OR_CALLABLE: + obj = cls.__new__(cls) + vars(obj).update(json_obj) + return obj + + +def _is_log_var(varname: str) -> bool: + return varname.strip('_').startswith('log') diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/serializers/coders/__init__.py b/paper_experiments/large_bayesian_networks_experiments/golem/serializers/coders/__init__.py new file mode 100644 index 0000000..d62145f --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/serializers/coders/__init__.py @@ -0,0 +1,7 @@ +# flake8: noqa +from .enum_serialization import enum_from_json, enum_to_json +from .graph_node_serialization import graph_node_to_json +from .graph_serialization import graph_from_json +from .opt_history_serialization import opt_history_from_json, opt_history_to_json +from .parent_operator_serialization import parent_operator_from_json, parent_operator_to_json +from .uuid_serialization import uuid_from_json, uuid_to_json diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/serializers/coders/enum_serialization.py b/paper_experiments/large_bayesian_networks_experiments/golem/serializers/coders/enum_serialization.py new file mode 100644 index 0000000..9e2d2ba --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/serializers/coders/enum_serialization.py @@ -0,0 +1,10 @@ +from enum import Enum +from typing import Any, Dict, Type + + +def enum_to_json(obj: Enum) -> Dict[str, Any]: + return { 'value': obj.value } + + +def enum_from_json(cls: Type[Enum], json_obj: Dict[str, Any]) -> Enum: + return cls(json_obj['value']) diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/serializers/coders/graph_node_serialization.py b/paper_experiments/large_bayesian_networks_experiments/golem/serializers/coders/graph_node_serialization.py new file mode 100644 index 0000000..8d45b49 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/serializers/coders/graph_node_serialization.py @@ -0,0 +1,23 @@ +from typing import Any, Dict + +from golem.core.dag.linked_graph_node import LinkedGraphNode +from .. import any_to_json + + +def graph_node_to_json(obj: LinkedGraphNode) -> Dict[str, Any]: + """ + Uses regular serialization but excludes "_operator" field to rid of circular references + """ + encoded = { + k: v + for k, v in any_to_json(obj).items() + if k not in ['_operator', '_fitted_operation', '_node_data', '_parameters'] + } + if 'name' in encoded['content']: + encoded['content']['name'] = str(encoded['content']['name']) + if encoded['_nodes_from']: + encoded['_nodes_from'] = [ + node.uid + for node in encoded['_nodes_from'] + ] + return encoded diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/serializers/coders/graph_serialization.py b/paper_experiments/large_bayesian_networks_experiments/golem/serializers/coders/graph_serialization.py new file mode 100644 index 0000000..c032c64 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/serializers/coders/graph_serialization.py @@ -0,0 +1,40 @@ +from typing import Any, Dict, Type, Sequence, Union + +from golem.core.dag.graph import Graph +from golem.core.dag.graph_delegate import GraphDelegate +from golem.core.dag.linked_graph_node import LinkedGraphNode + + +def graph_from_json(cls: Type[Graph], json_obj: Dict[str, Any]) -> Graph: + obj = cls() + + nodes_key = 'nodes' if 'nodes' in json_obj else '_nodes' + if not issubclass(cls, GraphDelegate): + nodes = json_obj[nodes_key] + _reassign_edges_by_node_ids(nodes) + obj.nodes = nodes + + # GraphDelegate case is handled by this + vars(obj).update(**{k: v for k, v in json_obj.items() if k != nodes_key}) + + return obj + + +def _reassign_edges_by_node_ids(nodes: Sequence[Union[LinkedGraphNode, dict]]): + """ + Assigns each from "nodes_from" to equal from "nodes" + (cause each node from "nodes_from" in fact should point to the same node from "nodes") + """ + lookup_dict = {} + for node in nodes: + if isinstance(node, dict): + lookup_dict[node['uid']] = node + else: + lookup_dict[node.uid] = node + + for node in nodes: + nodes_from = node['_nodes_from'] if isinstance(node, dict) else node.nodes_from + if not nodes_from: + continue + for parent_node_idx, parent_node_uid in enumerate(nodes_from): + nodes_from[parent_node_idx] = lookup_dict.get(parent_node_uid, None) diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/serializers/coders/opt_history_serialization.py b/paper_experiments/large_bayesian_networks_experiments/golem/serializers/coders/opt_history_serialization.py new file mode 100644 index 0000000..8e12db2 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/serializers/coders/opt_history_serialization.py @@ -0,0 +1,123 @@ +from itertools import chain +from typing import Any, Dict, List, Sequence, Type, Union + +from golem.core.optimisers.graph import OptGraph +from golem.core.optimisers.opt_history_objects.individual import Individual +from golem.core.optimisers.opt_history_objects.generation import Generation +from golem.core.optimisers.opt_history_objects.opt_history import OptHistory +from .. import any_from_json, any_to_json +from ...core.optimisers.objective.objective import ObjectiveInfo + +MISSING_INDIVIDUAL_ARGS = { + 'metadata': {'MISSING_INDIVIDUAL': 'This individual could not be restored during `OptHistory.load()`'} +} + + +def _flatten_generations_list(generations_list: List[List[Individual]]) -> List[Individual]: + def extract_intermediate_parents(ind: Individual): + for parent in ind.parents: + if not parent.has_native_generation: + parents_map[parent.uid] = parent + extract_intermediate_parents(parent) + + generations_map = {ind.uid: ind for ind in chain(*generations_list)} + parents_map = {} + for individual in generations_map.values(): + extract_intermediate_parents(individual) + parents_map.update(generations_map) + individuals_pool = list(parents_map.values()) + return individuals_pool + + +def _generations_list_to_uids(generations_list: List[Generation]) -> List[Generation]: + new_list = [] + for generation in generations_list: + new_gen = generation.copy() + new_gen.data = [individual.uid for individual in generation] + new_list.append(new_gen) + return new_list + + +def _archive_to_uids(generations_list: List[List[Individual]]) -> List[List[str]]: + return [[individual.uid for individual in generation] for generation in generations_list] + + +def opt_history_to_json(obj: OptHistory) -> Dict[str, Any]: + serialized = any_to_json(obj) + serialized['individuals_pool'] = _flatten_generations_list(serialized['_generations']) + serialized['_generations'] = _generations_list_to_uids(serialized['_generations']) + serialized['archive_history'] = _archive_to_uids(serialized['archive_history']) + return serialized + + +def _uids_to_individuals(uid_sequence: Sequence[Union[str, Individual]], + uid_to_individual_map: Dict[str, Individual]) -> List[Individual]: + def get_missing_individual(uid: str) -> Individual: + individual = Individual(OptGraph(), **MISSING_INDIVIDUAL_ARGS, uid=uid) + return individual + + def uid_to_individual_mapper(uid: Union[str, Individual]) -> Individual: + return uid_to_individual_map.get(uid, get_missing_individual(uid)) if isinstance(uid, str) else uid + + return list(map(uid_to_individual_mapper, uid_sequence)) + + +def _deserialize_generations_list(generations_list: List[Union[Generation, List[Union[str, Individual]]]], + uid_to_individual_map: Dict[str, Individual]): + """The operation is executed in-place""" + for gen_num, generation in enumerate(generations_list): + individuals = _uids_to_individuals(uid_sequence=generation, + uid_to_individual_map=uid_to_individual_map) + if isinstance(generations_list[gen_num], Generation): + generations_list[gen_num].data = individuals + else: + generations_list[gen_num] = individuals + + +def _deserialize_parent_individuals(individuals: List[Individual], + uid_to_individual_map: Dict[str, Individual]): + """The operation is executed in-place""" + + def deserialize_intermediate_parents(ind): + parent_op = ind.parent_operator + if not parent_op: + return + parent_individuals = _uids_to_individuals(uid_sequence=parent_op.parent_individuals, + uid_to_individual_map=uid_to_individual_map) + object.__setattr__(parent_op, 'parent_individuals', tuple(parent_individuals)) + for parent in parent_individuals: + if any(isinstance(i, str) for i in parent.parents): + deserialize_intermediate_parents(parent) + + for individual in individuals: + deserialize_intermediate_parents(individual) + + +def opt_history_from_json(cls: Type[OptHistory], json_obj: Dict[str, Any]) -> OptHistory: + # backward compatibility with history._is_multi_objective field + if '_is_multi_objective' in json_obj: + json_obj['_objective'] = ObjectiveInfo(json_obj['_is_multi_objective']) + del json_obj['_is_multi_objective'] + + # OptHistory.individuals are now OptHistory._generations + if 'individuals' in json_obj: + json_obj['_generations'] = json_obj.pop('individuals') + + history = any_from_json(cls, json_obj) + # Read all individuals from history. + individuals_pool = history.individuals_pool + uid_to_individual_map = {ind.uid: ind for ind in individuals_pool} + # The attributes `individuals` and `archive_history` at the moment contain uid strings that must be converted + # to `Individual` instances. + _deserialize_generations_list(history.generations, uid_to_individual_map) + _deserialize_generations_list(history.archive_history, uid_to_individual_map) + # Process histories with zero generations. + if len(history.generations) > 0: + # Process older histories to wrap generations into the new class. + if isinstance(history.generations[0], list): + history.generations = [Generation(gen, gen_num) for gen_num, gen in enumerate(history.generations)] + # Deserialize parents for all generations. + _deserialize_parent_individuals(list(chain(*history.generations)), uid_to_individual_map) + # The attribute is used only for serialization. + del history.individuals_pool + return history diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/serializers/coders/parent_operator_serialization.py b/paper_experiments/large_bayesian_networks_experiments/golem/serializers/coders/parent_operator_serialization.py new file mode 100644 index 0000000..216b9b0 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/serializers/coders/parent_operator_serialization.py @@ -0,0 +1,19 @@ +from typing import Any, Dict, Type + +from golem.core.optimisers.opt_history_objects.parent_operator import ParentOperator +from .. import any_from_json, any_to_json + + +def parent_operator_to_json(obj: ParentOperator) -> Dict[str, Any]: + serialized_op = any_to_json(obj) + serialized_op['parent_individuals'] = [ + parent_ind.uid + for parent_ind in serialized_op['parent_individuals'] if parent_ind is not None + ] + return serialized_op + + +def parent_operator_from_json(cls: Type[ParentOperator], json_obj: Dict[str, Any]) -> ParentOperator: + deserialized = any_from_json(cls, json_obj) + object.__setattr__(deserialized, 'parent_individuals', deserialized.parent_individuals) + return deserialized diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/serializers/coders/uuid_serialization.py b/paper_experiments/large_bayesian_networks_experiments/golem/serializers/coders/uuid_serialization.py new file mode 100644 index 0000000..617851f --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/serializers/coders/uuid_serialization.py @@ -0,0 +1,10 @@ +from typing import Any, Dict, Type +from uuid import UUID + + +def uuid_to_json(obj: UUID) -> Dict[str, Any]: + return { 'hex': obj.hex } + + +def uuid_from_json(cls: Type[UUID], json_obj: Dict[str, Any]) -> UUID: + return cls(json_obj['hex']) diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/serializers/serializer.py b/paper_experiments/large_bayesian_networks_experiments/golem/serializers/serializer.py new file mode 100644 index 0000000..6308a48 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/serializers/serializer.py @@ -0,0 +1,380 @@ +import json +import os +from importlib import import_module +from inspect import isclass, isfunction, ismethod, signature +from json import JSONDecoder, JSONEncoder +from typing import Any, Callable, Dict, Optional, Type, TypeVar, Union + +from golem.core.dag.linked_graph import LinkedGraph +from golem.core.dag.linked_graph_node import LinkedGraphNode +from golem.core.log import default_log + +# NB: at the end of module init happens registration of default class coders + +INSTANCE_OR_CALLABLE = TypeVar('INSTANCE_OR_CALLABLE', object, Callable) +EncodeCallable = Callable[[INSTANCE_OR_CALLABLE], Dict[str, Any]] +DecodeCallable = Callable[[Type[INSTANCE_OR_CALLABLE], Dict[str, Any]], INSTANCE_OR_CALLABLE] + +MODULE_X_NAME_DELIMITER = '/' +CLASS_PATH_KEY = '_class_path' + +# Mapping between class paths for backward compatibility for renamed/moved classes +LEGACY_CLASS_PATHS = { + 'fedot.core.optimisers.gp_comp.individual/Individual': + 'golem.core.optimisers.opt_history_objects.individual/Individual', + 'fedot.core.optimisers.gp_comp.individual/ParentOperator': + 'golem.core.optimisers.opt_history_objects.parent_operator/ParentOperator', + 'fedot.core.optimisers.opt_history/OptHistory': + 'golem.core.optimisers.opt_history_objects.opt_history/OptHistory', + + 'fedot.core.dag.graph_node/GraphNode': + 'golem.core.dag.linked_graph_node/LinkedGraphNode', + 'fedot.core.dag.graph_operator/GraphOperator': + 'golem.core.dag.linked_graph/LinkedGraph', + 'fedot.core.dag.graph_operator/GraphOperator._empty_postprocess': + 'golem.core.dag.linked_graph/LinkedGraph._empty_postprocess', + + # for fedot->golem transition + 'fedot.core.optimisers.objective.objective/Objective': + 'golem.core.optimisers.objective.objective/Objective', + 'fedot.core.optimisers.fitness.fitness/Fitness': + 'golem.core.optimisers.fitness.fitness/Fitness', +} + +# for fedot->golem transition +# NB: must be ordered from specific to top-level modules +LEGACY_MODULE_PATHS = { + 'fedot.core.optimisers.gp_comp.individual': 'golem.core.optimisers.opt_history_objects.individual', + 'fedot.core.optimisers.opt_history_objects': 'golem.core.optimisers.opt_history_objects', + 'fedot.core.optimisers.gp_comp': 'golem.core.optimisers.genetic', + 'fedot.core.optimisers.graph': 'golem.core.optimisers.graph', + 'fedot.core.optimisers.objective.objective': 'golem.core.optimisers.objective.objective', + 'fedot.core.optimisers.fitness': 'golem.core.optimisers.fitness', + + 'fedot.core.log': 'golem.core.log', + 'fedot.core.adapter': 'golem.core.adapter', + 'fedot.core.dag': 'golem.core.dag', + 'fedot.core.utilities': 'golem.core.utilities', +} + + +class Serializer(JSONEncoder, JSONDecoder): + _to_json = 'to_json' + _from_json = 'from_json' + + CODERS_BY_TYPE = {} + + __default_coders_initialized = False + + def __init__(self, *args, **kwargs): + for base_class, coder_name in [(JSONEncoder, 'default'), (JSONDecoder, 'object_hook')]: + base_kwargs = {k: kwargs[k] for k in kwargs.keys() & signature(base_class.__init__).parameters} + base_kwargs[coder_name] = getattr(self, coder_name) + base_class.__init__(self, **base_kwargs) + + if not self.__default_coders_initialized: + Serializer._register_default_coders() + self.__default_coders_initialized = True + + @staticmethod + def _register_default_coders(): + from uuid import UUID + + from golem.core.dag.graph import Graph + from golem.core.dag.linked_graph_node import LinkedGraphNode + from golem.core.optimisers.opt_history_objects.individual import Individual + from golem.core.optimisers.opt_history_objects.generation import Generation + from golem.core.optimisers.opt_history_objects.opt_history import OptHistory + from golem.core.optimisers.opt_history_objects.parent_operator import ParentOperator + from golem.core.optimisers.fitness.fitness import Fitness + from golem.core.optimisers.objective.objective import ObjectiveInfo + from golem.utilities.data_structures import ComparableEnum + + from .any_serialization import any_from_json, any_to_json + + from .coders import ( + enum_from_json, + enum_to_json, + graph_from_json, + graph_node_to_json, + opt_history_from_json, + opt_history_to_json, + parent_operator_from_json, + parent_operator_to_json, + uuid_from_json, + uuid_to_json + ) + + _to_json = Serializer._to_json + _from_json = Serializer._from_json + basic_serialization = {_to_json: any_to_json, _from_json: any_from_json} + + default_coders = { + Fitness: basic_serialization, + ObjectiveInfo: basic_serialization, + Individual: basic_serialization, + Generation: basic_serialization, + LinkedGraphNode: {_to_json: graph_node_to_json, _from_json: any_from_json}, + Graph: {_to_json: any_to_json, _from_json: graph_from_json}, + OptHistory: {_to_json: opt_history_to_json, _from_json: opt_history_from_json}, + ParentOperator: {_to_json: parent_operator_to_json, _from_json: parent_operator_from_json}, + UUID: {_to_json: uuid_to_json, _from_json: uuid_from_json}, + ComparableEnum: {_to_json: enum_to_json, _from_json: enum_from_json}, + } + Serializer.CODERS_BY_TYPE.update(default_coders) + + @staticmethod + def register_coders(cls: Type[INSTANCE_OR_CALLABLE], + to_json: Optional[EncodeCallable[INSTANCE_OR_CALLABLE]] = None, + from_json: Optional[DecodeCallable[INSTANCE_OR_CALLABLE]] = None, + overwrite: bool = False, + ) -> Type[INSTANCE_OR_CALLABLE]: + """Registers classes as json-serializable so that they can be used by `Serializer`. + + Supports 3 alternative usages: + + - Default serialization is used (functions `any_to_json` and `any_from_json`). + - Custom serialization functions `to_json` & `from_json` are provided explicitly as arguments. + - Custom serialization functions `to_json` & `from_json` are defined in the class. + + Args: + cls: registered class + to_json: custom encoding function that returns json Dict. + Optional, if None then default is used. + from_json: custom decoding function that returns class given a Dict. + Optional, if None then default is used. + overwrite: flag that allows to overwrite existing registered coders. + False by default: method raises AttributeError in attempt to overwrite coders. + + Returns: + cls: class that is registered in serializer + """ + if cls is None: + raise TypeError('Class must not be None.') + from .any_serialization import any_from_json, any_to_json + + # get provided coders or coders defined in the class itself or default universal coders + coders = {Serializer._to_json: to_json or getattr(cls, Serializer._to_json, any_to_json), + Serializer._from_json: from_json or getattr(cls, Serializer._from_json, any_from_json)} + + if cls not in Serializer.CODERS_BY_TYPE or overwrite: + Serializer.CODERS_BY_TYPE[cls] = coders + else: + raise AttributeError(f'Object {cls} already has serializer coders registered.') + + return cls + + @staticmethod + def _get_field_checker(obj: Union[INSTANCE_OR_CALLABLE, Type[INSTANCE_OR_CALLABLE]]) -> Callable[..., bool]: + if isclass(obj): + return issubclass + return isinstance + + @staticmethod + def _get_base_type(obj: Union[INSTANCE_OR_CALLABLE, Type[INSTANCE_OR_CALLABLE]]) -> Optional[Type]: + contains = Serializer._get_field_checker(obj) + for k_type in Serializer.CODERS_BY_TYPE: + if contains(obj, k_type): + return k_type + return None + + @staticmethod + def _get_coder_by_type(coder_type: Type, coder_aim: str): + return Serializer.CODERS_BY_TYPE[coder_type][coder_aim] + + @staticmethod + def dump_path_to_obj(obj: INSTANCE_OR_CALLABLE) -> Dict[str, str]: + """ + Dumps the full path (module + name) to the input object into the dict + + :param obj: object which path should be resolved (class, function or method) + + :return: dict[str, str] with path to the object + """ + if isclass(obj) or isfunction(obj) or ismethod(obj): + obj_name = obj.__qualname__ + else: + obj_name = obj.__class__.__qualname__ + + if getattr(obj, '__module__', None) is not None: + obj_module = obj.__module__ + else: + obj_module = obj.__class__.__module__ + return { + CLASS_PATH_KEY: f'{obj_module}{MODULE_X_NAME_DELIMITER}{obj_name}' + } + + def default(self, obj: INSTANCE_OR_CALLABLE) -> Dict[str, Any]: + """ + Tries to encode objects that are not simply json-encodable to JSON-object + + :param obj: object to be encoded (class, function or method) + + :return: dict[str, Any] which is in fact json object + """ + if isfunction(obj) or ismethod(obj): + return Serializer.dump_path_to_obj(obj) + base_type = Serializer._get_base_type(obj) + if base_type is not None: + encoded = Serializer._get_coder_by_type(base_type, Serializer._to_json)(obj) + if CLASS_PATH_KEY not in encoded: + encoded.update(Serializer.dump_path_to_obj(obj)) + return encoded + + return JSONEncoder.default(self, obj) + + @staticmethod + def _get_class(json_obj: dict) -> Optional[Type[INSTANCE_OR_CALLABLE]]: + """ + Gets the object type from the class_path + + :param class_path: full path (module + name) of the class + + :return: class, function or method type + """ + class_path = json_obj[CLASS_PATH_KEY] + class_path = LEGACY_CLASS_PATHS.get(class_path, class_path) + module_name, class_name = class_path.split(MODULE_X_NAME_DELIMITER) + module_name = Serializer._legacy_module_map(module_name) + + try: + obj_cls = import_module(module_name) + except ImportError as ex: + obj_cls = Serializer._import_as_base_class(json_obj) + if not obj_cls: + default_log('Serializer').info( + f'Object was not decoded and will be stored as a dict ' + f'because of an ImportError: {ex}.') + else: + default_log('Serializer').info( + f'Object was decoded as {obj_cls} and not as an original class ' + f'because of an ImportError: {ex}.') + return obj_cls + + for sub in class_name.split('.'): + obj_cls = getattr(obj_cls, sub) + return obj_cls + + @staticmethod + def _legacy_module_map(module_path: str) -> str: + for legacy_prefix, new_prefix in LEGACY_MODULE_PATHS.items(): + if module_path.startswith(legacy_prefix): + return module_path.replace(legacy_prefix, new_prefix) + return module_path + + @staticmethod + def _is_bound_method(method: Callable) -> bool: + return hasattr(method, '__self__') + + @staticmethod + def _import_as_base_class(json_obj: dict) \ + -> Optional[Union[Type[LinkedGraph], Type[LinkedGraphNode]]]: + linked_graph_keys = {'_nodes', '_postprocess_nodes'} + linked_node_keys = {'content', '_nodes_from', 'uid'} + if linked_graph_keys.issubset(json_obj.keys()): + return LinkedGraph + elif linked_node_keys.issubset(json_obj.keys()): + return LinkedGraphNode + else: + return None + + @staticmethod + def object_hook(json_obj: Dict[str, Any]) -> Union[INSTANCE_OR_CALLABLE, dict]: + """ + Decodes every JSON-object to python class/func object or just returns dict + + :param json_obj: dict[str, Any] to be decoded into Python class, function or + method object only if it has some special fields + + :return: Python class, function or method object OR input if it's just a regular dict + """ + if CLASS_PATH_KEY in json_obj: + obj_cls = Serializer._get_class(json_obj) + del json_obj[CLASS_PATH_KEY] + base_type = Serializer._get_base_type(obj_cls) + if isclass(obj_cls) and base_type is not None: + coder = Serializer._get_coder_by_type(base_type, Serializer._from_json) + # call with a right num of arguments + if Serializer._is_bound_method(coder): + return coder(json_obj) + else: + return coder(obj_cls, json_obj) + elif isfunction(obj_cls) or ismethod(obj_cls): + return obj_cls + else: + return json_obj + return json_obj + + +def default_save(obj: Any, json_file_path: Optional[Union[str, os.PathLike]] = None) -> Optional[str]: + """ Default save to json using Serializer """ + if json_file_path is None: + return json.dumps(obj, indent=4, cls=Serializer) + with open(json_file_path, mode='w') as json_file: + json.dump(obj, json_file, indent=4, cls=Serializer) + return None + + +def default_load(json_str_or_file_path: Union[str, os.PathLike]) -> Any: + """ Default load from json using Serializer """ + + def load_as_file_path(): + with open(json_str_or_file_path, mode='r') as json_file: + return json.load(json_file, cls=Serializer) + + def load_as_json_str(): + return json.loads(json_str_or_file_path, cls=Serializer) + + if isinstance(json_str_or_file_path, os.PathLike): + return load_as_file_path() + + try: + return load_as_json_str() + except json.JSONDecodeError: + return load_as_file_path() + + +def register_serializable(cls: Optional[Type[INSTANCE_OR_CALLABLE]] = None, + to_json: Optional[EncodeCallable] = None, + from_json: Optional[DecodeCallable] = None, + overwrite: bool = False, + add_save_load: bool = False, + ) -> Type[INSTANCE_OR_CALLABLE]: + """Decorator for registering classes as json-serializable. + Relies on :py:class:`golem.serializers.serializer.Serializer.register_coders`. + Optionally adds `save` and `load` methods to the class for json (de)serialization. + + Args: + cls: decorated class + to_json: custom encoding function that returns json Dict. + Optional, if None then default is used. + from_json: custom decoding function that returns class given a Dict. + Optional, if None then default is used. + overwrite: flag that allows to overwrite existing registered coders. + False by default: method raises AttributeError in attempt to overwrite coders. + add_save_load: if True, then `save` and `load` methods are added to the class + that (de)serialize the class (from)to the file using `Serializer`. + See `default_save` & `default_load` functions. + + Returns: + cls: class that is registered in serializer + """ + _save = 'save' + _load = 'load' + + def make_serializable(cls): + Serializer.register_coders(cls, to_json, from_json, overwrite) + if add_save_load: + if hasattr(cls, _save) or hasattr(cls, _load): + raise ValueError(f'Class {cls} already have `save` and/or `load` methods, can not overwrite them.') + setattr(cls, _save, default_save) + setattr(cls, _load, default_load) + return cls + + # See if we're being called as @decorator() (with parens). + if cls is None: + # We're called with parens. + return make_serializable + + # We're called as `@decorator` without parens. + return make_serializable(cls) diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/__init__.py b/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/base_sa_approaches.py b/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/base_sa_approaches.py new file mode 100644 index 0000000..0afa40a --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/base_sa_approaches.py @@ -0,0 +1,68 @@ +from abc import abstractmethod, ABC +from typing import Union, List, Optional, Sequence + +from golem.core.dag.graph import Graph +from golem.core.dag.graph_node import GraphNode +from golem.core.optimisers.objective import Objective +from golem.structural_analysis.graph_sa.entities.edge import Edge +from golem.structural_analysis.graph_sa.sa_requirements import StructuralAnalysisRequirements + + +class BaseAnalyzeApproach(ABC): + """ + Base class for analysis approach. + :param graph: Graph containing the analyzed Node + :param objective: objective functions for computing metric values + """ + + def __init__(self, graph: Graph, objective: Objective, + requirements: StructuralAnalysisRequirements = None): + self._graph = graph + self._objective = objective + self._origin_metrics = list() + self._requirements = \ + StructuralAnalysisRequirements() if requirements is None else requirements + + @abstractmethod + def analyze(self, entity: Union[GraphNode, Edge], **kwargs) -> Union[List[dict], List[float]]: + """ Creates the difference metric(scorer, index, etc) of the changed + graph in relation to the original one. + :param entity: entity to analyze. + """ + pass + + @abstractmethod + def sample(self, *args) -> Union[List[Graph], Graph]: + """ Changes the graph according to the approach. """ + pass + + @staticmethod + def _compare_with_origin_by_metric(origin_metric: Optional[float], modified_metric: Optional[float]) -> float: + """ Calculates one metric. """ + if not modified_metric or not origin_metric: + return -1.0 + + if modified_metric < 0.0: + numerator = modified_metric + denominator = origin_metric + else: + numerator = origin_metric + denominator = modified_metric + + if denominator == 0: + return -1.0 + + return numerator / denominator + + def _compare_with_origin_by_metrics(self, modified_graph: Graph) -> List[float]: + """ Returns all relative metrics calculated. """ + modified_graph_metrics = self._objective(modified_graph).values + + if not self._origin_metrics: + self._origin_metrics = self._objective(self._graph).values + + res = [] + for i in range(len(modified_graph_metrics)): + res.append(self._compare_with_origin_by_metric(modified_metric=modified_graph_metrics[i], + origin_metric=self._origin_metrics[i])) + return res diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/__init__.py b/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/edge_sa_approaches.py b/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/edge_sa_approaches.py new file mode 100644 index 0000000..e79cffd --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/edge_sa_approaches.py @@ -0,0 +1,316 @@ +import random +from abc import ABC +from copy import deepcopy +from os import makedirs +from os.path import exists, join +from typing import List, Optional, Type, Union, Dict, Callable + +from golem.core.log import default_log +from golem.core.dag.graph import Graph +from golem.core.optimisers.objective import Objective +from golem.core.optimisers.timer import OptimisationTimer +from golem.core.paths import default_data_dir +from golem.structural_analysis.base_sa_approaches import BaseAnalyzeApproach +from golem.structural_analysis.graph_sa.entities.edge import Edge +from golem.structural_analysis.graph_sa.results.deletion_sa_approach_result import \ + DeletionSAApproachResult +from golem.structural_analysis.graph_sa.results.object_sa_result import ObjectSAResult +from golem.structural_analysis.graph_sa.results.replace_sa_approach_result import \ + ReplaceSAApproachResult +from golem.structural_analysis.graph_sa.results.utils import EntityTypesEnum +from golem.structural_analysis.graph_sa.sa_requirements import StructuralAnalysisRequirements, \ + ReplacementAnalysisMetaParams + + +class EdgeAnalysis: + """ + Class for successively applying approaches for structural analysis + + :param approaches: methods applied to edges to modify the graph or analyze certain operations.\ + Default: [EdgeDeletionAnalyze, EdgeReplaceOperationAnalyze] + :param path_to_save: path to save results to. Default: ~home/Fedot/structural + """ + + def __init__(self, approaches: Optional[List[Type['EdgeAnalyzeApproach']]] = None, + approaches_requirements: Optional[StructuralAnalysisRequirements] = None, + path_to_save: Optional[str] = None): + + self.approaches = [EdgeDeletionAnalyze, EdgeReplaceOperationAnalyze] \ + if approaches is None else approaches + + self.path_to_save = \ + join(default_data_dir(), 'structural', 'edges_structural') if path_to_save is None else path_to_save + self.log = default_log(self) + + self.approaches_requirements = \ + StructuralAnalysisRequirements() if approaches_requirements is None else approaches_requirements + + def analyze(self, graph: Graph, edge: Edge, + objective: Callable, + timer: Optional[OptimisationTimer] = None) -> ObjectSAResult: + """ + Method runs Edge analysis within defined approaches + + :param graph: graph containing the analyzed Edge + :param edge: Edge object to analyze in Graph + :param objective: objective function for computing metric values + :param timer: timer to check if the time allotted for structural analysis has expired + :return: dict with Edge analysis result per approach + """ + + results = ObjectSAResult(entity_idx= + f'{graph.nodes.index(edge.parent_node)}_{graph.nodes.index(edge.child_node)}', + entity_type=EntityTypesEnum.edge) + + for approach in self.approaches: + if timer is not None and timer.is_time_limit_reached(): + break + + results.add_result(approach(graph=graph, + objective=objective, + requirements=self.approaches_requirements, + path_to_save=self.path_to_save).analyze(edge=edge)) + + return results + + +class EdgeAnalyzeApproach(BaseAnalyzeApproach, ABC): + """ + Base class for edge analysis approach. + + :param graph: Graph containing the analyzing Edge + :param objective: objective function for computing metric values + :param path_to_save: path to save results to. Default: ~home/Fedot/structural + """ + + def __init__(self, graph: Graph, objective: Objective, requirements: StructuralAnalysisRequirements = None, + path_to_save=None): + super().__init__(graph, objective, requirements) + self._origin_metrics = None + + self._path_to_save = \ + join(default_data_dir(), 'structural', 'edges_structural') if path_to_save is None else path_to_save + self.log = default_log(prefix='edge_analysis') + + if not exists(self._path_to_save): + makedirs(self._path_to_save) + + +class EdgeDeletionAnalyze(EdgeAnalyzeApproach): + def __init__(self, graph: Graph, objective: Objective, + requirements: StructuralAnalysisRequirements = None, path_to_save=None): + super().__init__(graph, objective, requirements) + + self._path_to_save = \ + join(default_data_dir(), 'structural', 'edges_structural') if path_to_save is None else path_to_save + if not exists(self._path_to_save): + makedirs(self._path_to_save) + + def analyze(self, edge: Edge, **kwargs) -> DeletionSAApproachResult: + """ + Receives a graph without the specified edge and tries to calculate the loss for it + + :param edge: Edge object to analyze + :return: the ratio of modified graph score to origin score + """ + results = DeletionSAApproachResult() + shortened_graph = self.sample(edge) + if shortened_graph: + losses = self._compare_with_origin_by_metrics(shortened_graph) + self.log.message(f'loss: {losses}') + del shortened_graph + else: + self.log.warning('if remove this edge then get an invalid graph') + losses = [-1.0] * len(self._objective.metrics) + + results.add_results(metrics_values=losses) + return results + + def sample(self, edge: Edge) -> Optional[Graph]: + """ + Checks if it is possible to delete an edge from the graph so that it remains valid, + and if so, deletes + + :param edge: Edge object to delete from Graph object + :return: Graph object without edge + """ + + graph_sample = deepcopy(self._graph) + + parent_node_index_to_delete = self._graph.nodes.index(edge.parent_node) + parent_node_to_delete = graph_sample.nodes[parent_node_index_to_delete] + + child_node_index_to_delete = self._graph.nodes.index(edge.child_node) + child_node_to_delete = graph_sample.nodes[child_node_index_to_delete] + + graph_sample.disconnect_nodes(parent_node_to_delete, child_node_to_delete) + + verifier = self._requirements.graph_verifier + if not verifier.verify(graph_sample): + self.log.warning('Can not delete edge since modified graph can not pass verification') + return None + + return graph_sample + + +class EdgeReplaceOperationAnalyze(EdgeAnalyzeApproach): + """ + Replace edge with operations available for the current task + and evaluate the score difference + """ + + def __init__(self, graph: Graph, objective: Objective, + requirements: StructuralAnalysisRequirements = None, path_to_save=None): + super().__init__(graph, objective, requirements) + + self._path_to_save = \ + join(default_data_dir(), 'structural', 'edges_structural') if path_to_save is None else path_to_save + if not exists(self._path_to_save): + makedirs(self._path_to_save) + + def analyze(self, edge: Edge, **kwargs) -> ReplaceSAApproachResult: + """ + Counts the loss on each changed graph received and returns the biggest loss and + the graph on which it was received + + :param edge: Edge object to analyze + :return: dictionary of the best (the biggest) loss and corresponding to it edge to replace to + """ + result = ReplaceSAApproachResult() + requirements: ReplacementAnalysisMetaParams = self._requirements.replacement_meta + samples_res = self.sample(edge=edge, + edges_idxs_to_replace_to=requirements.edges_to_replace_to, + number_of_random_operations=requirements.number_of_random_operations_edges) + + samples = samples_res['samples'] + edges_nodes_idx_to_replace_to = samples_res['edges_nodes_idx_to_replace_to'] + + loss_values = [] + for i, sample_graph in enumerate(samples): + loss_per_sample = self._compare_with_origin_by_metrics(sample_graph) + self.log.message(f'loss: {loss_per_sample}') + loss_values.append(loss_per_sample) + + child_node_idx = '' + parent_node_idx = '' + part1, part2 = edges_nodes_idx_to_replace_to[i].__str__().split(',') + for char in part1: + if char.isdigit(): + child_node_idx += char + else: + continue + for char in part2: + if char.isdigit(): + parent_node_idx += char + else: + continue + result.add_results(entity_to_replace_to=f'{parent_node_idx}_{child_node_idx}', + metrics_values=loss_per_sample) + return result + + def sample(self, edge: Edge, + edges_idxs_to_replace_to: Optional[List[Edge]], + number_of_random_operations: Optional[int] = 1) \ + -> Dict[str, Union[List[Graph], List[Dict[str, int]]]]: + """ + Tries to replace the given edge with a pool of edges available for replacement (see _edge_generation docstring) + and validates the resulting graphs + + :param edge: Edge object to replace + :param edges_idxs_to_replace_to: edges provided for old_edge replacement + :param number_of_random_operations: number of replacement operations, \ + if edges_to_replace_to not provided + + :return: dictionary of sequence of Graph objects with new edges instead of old one + and indexes of edges to which to change the given edge to get these graphs + """ + + if not edges_idxs_to_replace_to or number_of_random_operations: + edges_idxs_to_replace_to = self._edge_generation(edge=edge, + number_of_operations=number_of_random_operations) + samples = list() + edges_nodes_idx_to_replace_to = list() + for replacing_nodes_idx in edges_idxs_to_replace_to: + sample_graph = deepcopy(self._graph) + + # disconnect nodes + previous_parent_node_index = self._graph.nodes.index(edge.parent_node) + previous_child_node_index = self._graph.nodes.index(edge.child_node) + + previous_parent_node = sample_graph.nodes[previous_parent_node_index] + previous_child_node = sample_graph.nodes[previous_child_node_index] + + # connect nodes + next_parent_node = sample_graph.nodes[replacing_nodes_idx['parent_node_idx']] + next_child_node = sample_graph.nodes[replacing_nodes_idx['child_node_idx']] + + if next_parent_node in sample_graph.nodes and \ + next_child_node in sample_graph.nodes: + sample_graph.connect_nodes(next_parent_node, next_child_node) + + sample_graph.disconnect_nodes(node_parent=previous_parent_node, + node_child=previous_child_node, + clean_up_leftovers=False) + + verifier = self._requirements.graph_verifier + if not verifier.verify(sample_graph): + self.log.warning('Can not connect these nodes') + else: + self.log.message(f'replace edge parent: {next_parent_node}') + self.log.message(f'replace edge child: {next_child_node}') + samples.append(sample_graph) + edges_nodes_idx_to_replace_to.append({'parent_node_id': + replacing_nodes_idx['parent_node_idx'], + 'child_node_id': + replacing_nodes_idx['child_node_idx']}) + + if not edges_nodes_idx_to_replace_to: + res = {'samples': [self._graph], 'edges_nodes_idx_to_replace_to': + [{'parent_node_id': self._graph.nodes.index(edge.parent_node), + 'child_node_id': self._graph.nodes.index(edge.child_node)}]} + return res + + return {'samples': samples, 'edges_nodes_idx_to_replace_to': edges_nodes_idx_to_replace_to} + + def _edge_generation(self, edge: Edge, number_of_operations: int = 1) -> List[Dict[str, int]]: + """ + The method returns possible edges that can replace the given edge. + These edges must not start at the root node, already exist in the graph and must not form cycles + + :param edge: edge to be replaced + :param number_of_operations: limits the number of possible edges to replace to + + :return: edges with which it's possible to replace the passed edge + """ + cur_graph = deepcopy(self._graph) + + parent_node_index = self._graph.nodes.index(edge.parent_node) + child_node_index = self._graph.nodes.index(edge.child_node) + + parent_node = cur_graph.nodes[parent_node_index] + child_node = cur_graph.nodes[child_node_index] + + if child_node is cur_graph.root_node and len(child_node.nodes_from) == 1: + return [] + + cur_graph.disconnect_nodes(node_parent=parent_node, node_child=child_node, + clean_up_leftovers=False) + + edges_in_graph = cur_graph.get_edges() + + available_edges_idx = list() + + for parent_node in cur_graph.nodes[1:]: + for child_node in cur_graph.nodes: + if parent_node == child_node: + continue + if [parent_node, child_node] in edges_in_graph or [child_node, parent_node] in edges_in_graph: + continue + if cur_graph.nodes.index(parent_node) == child_node_index and \ + cur_graph.nodes.index(child_node) == parent_node_index: + continue + available_edges_idx.append({'parent_node_idx': cur_graph.nodes.index(parent_node), + 'child_node_idx': cur_graph.nodes.index(child_node)}) + + edges_for_replacement = random.sample(available_edges_idx, min(number_of_operations, len(available_edges_idx))) + return edges_for_replacement diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/edges_analysis.py b/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/edges_analysis.py new file mode 100644 index 0000000..a265b6c --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/edges_analysis.py @@ -0,0 +1,77 @@ +from os.path import join +from typing import Optional, List, Type + +import multiprocessing + +from golem.core.log import default_log +from golem.core.dag.graph import Graph +from golem.core.optimisers.objective import Objective +from golem.core.optimisers.timer import OptimisationTimer +from golem.core.paths import default_data_dir +from golem.structural_analysis.graph_sa.edge_sa_approaches import EdgeAnalyzeApproach, EdgeAnalysis +from golem.structural_analysis.graph_sa.entities.edge import Edge +from golem.structural_analysis.graph_sa.results.sa_analysis_results import SAAnalysisResults +from golem.structural_analysis.graph_sa.sa_requirements import StructuralAnalysisRequirements + + +class EdgesAnalysis: + """ + This class is for edges structural analysis within an Graph . + It takes edges and approaches to be applied to chosen edges. + To define which edges to analyze pass them to edges_to_analyze filed + or all edges will be analyzed. + + :param objective: list of objective functions for computing metric values + :param approaches: methods applied to edges to modify the graph or analyze certain operations.\ + Default: [EdgeDeletionAnalyze, EdgeReplaceOperationAnalyze] + :param path_to_save: path to save results to. Default: ~home/Fedot/structural + """ + + def __init__(self, objective: Objective, + approaches: Optional[List[Type[EdgeAnalyzeApproach]]] = None, + requirements: Optional[StructuralAnalysisRequirements] = None, + path_to_save: Optional[str] = None): + + self.objective = objective + self.approaches = approaches + self.requirements = \ + StructuralAnalysisRequirements() if requirements is None else requirements + self.log = default_log(self) + self.path_to_save = \ + join(default_data_dir(), 'structural', 'edges_structural') if path_to_save is None else path_to_save + + def analyze(self, graph: Graph, results: Optional[SAAnalysisResults] = None, + edges_to_analyze: Optional[List[Edge]] = None, + n_jobs: int = 1, timer: Optional[OptimisationTimer] = None) -> SAAnalysisResults: + """ + Main method to run the analyze process for every edge. + + :param graph: graph object to analyze + :param results: SA results + :param edges_to_analyze: edges to analyze. Default: all edges + :param n_jobs: n_jobs + :param timer: timer indicating how much time is left for optimization + :return edges_results: dict with analysis result per Edge + """ + + if not results: + results = SAAnalysisResults() + + if n_jobs == -1: + n_jobs = multiprocessing.cpu_count() + + if not edges_to_analyze: + self.log.message('Edges to analyze are not defined. All edges will be analyzed.') + edges_to_analyze = [Edge.from_tuple([edge])[0] for edge in graph.get_edges()] + + edge_analysis = EdgeAnalysis(approaches=self.approaches, + approaches_requirements=self.requirements, + path_to_save=self.path_to_save) + + with multiprocessing.Pool(processes=n_jobs) as pool: + cur_edges_result = pool.starmap(edge_analysis.analyze, + [[graph, edge, self.objective, timer] + for edge in edges_to_analyze]) + results.add_results(cur_edges_result) + + return results diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/entities/__init__.py b/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/entities/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/entities/edge.py b/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/entities/edge.py new file mode 100644 index 0000000..dc2e60f --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/entities/edge.py @@ -0,0 +1,17 @@ +from typing import Tuple, List + +from golem.core.dag.graph_node import GraphNode + + +class Edge: + def __init__(self, parent_node: GraphNode, child_node: GraphNode): + self.parent_node = parent_node + self.child_node = child_node + self.rating = None + + @staticmethod + def from_tuple(edges_in_tuple: List[Tuple[GraphNode, GraphNode]]) -> List['Edge']: + edges = [] + for edge in edges_in_tuple: + edges.append(Edge(child_node=edge[1], parent_node=edge[0])) + return edges diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/graph_structural_analysis.py b/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/graph_structural_analysis.py new file mode 100644 index 0000000..6e1c678 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/graph_structural_analysis.py @@ -0,0 +1,297 @@ +import os +from copy import deepcopy +from pathlib import Path +from typing import List, Optional, Tuple, Dict +import multiprocessing + +from golem.core.log import default_log +from golem.core.dag.graph import Graph, GraphNode +from golem.core.optimisers.objective import Objective +from golem.core.optimisers.opt_node_factory import OptNodeFactory +from golem.core.optimisers.timer import OptimisationTimer +from golem.structural_analysis.graph_sa.edge_sa_approaches import EdgeAnalyzeApproach, EdgeDeletionAnalyze, \ + EdgeReplaceOperationAnalyze +from golem.structural_analysis.graph_sa.edges_analysis import EdgesAnalysis +from golem.structural_analysis.graph_sa.entities.edge import Edge +from golem.structural_analysis.graph_sa.node_sa_approaches import NodeAnalyzeApproach, NodeDeletionAnalyze, \ + NodeReplaceOperationAnalyze, SubtreeDeletionAnalyze +from golem.structural_analysis.graph_sa.nodes_analysis import NodesAnalysis +from golem.structural_analysis.graph_sa.results.object_sa_result import ObjectSAResult +from golem.structural_analysis.graph_sa.results.sa_analysis_results import SAAnalysisResults +from golem.structural_analysis.graph_sa.results.utils import EntityTypesEnum +from golem.structural_analysis.graph_sa.sa_approaches_repository import StructuralAnalysisApproachesRepository +from golem.structural_analysis.graph_sa.sa_requirements import StructuralAnalysisRequirements +from golem.visualisation.graph_viz import NodeColorType + + +class GraphStructuralAnalysis: + """ + This class works as facade and allows to apply all kind of approaches + to whole graph and separate nodes together. + + :param objective: list of objective functions for computing metric values + :param node_factory: node factory to advise changes from available operations and models + :param approaches: methods applied to graph. Default: None + :param requirements: extra requirements to define specific details for different approaches.\ + See StructuralAnalysisRequirements class documentation. + :param path_to_save: path to save results to. Default: ~home/Fedot/structural/ + Default: False + """ + + def __init__(self, objective: Objective, + node_factory: OptNodeFactory, + is_preproc: bool = True, + approaches: List = None, + requirements: StructuralAnalysisRequirements = StructuralAnalysisRequirements(), + path_to_save: str = None, + is_visualize_per_iteration: bool = False): + + self.is_preproc = is_preproc + self._log = default_log(self) + + if approaches: + self.nodes_analyze_approaches = [approach for approach in approaches + if issubclass(approach, NodeAnalyzeApproach)] + self.edges_analyze_approaches = [approach for approach in approaches + if issubclass(approach, EdgeAnalyzeApproach)] + else: + self._log.message('Approaches for analysis are not given, thus will be set to defaults.') + self.nodes_analyze_approaches = [NodeDeletionAnalyze, NodeReplaceOperationAnalyze, + SubtreeDeletionAnalyze] + self.edges_analyze_approaches = [EdgeDeletionAnalyze, EdgeReplaceOperationAnalyze] + + self._nodes_analyze = NodesAnalysis(objective=objective, + node_factory=node_factory, + approaches=self.nodes_analyze_approaches, + requirements=requirements, + path_to_save=path_to_save) + + self._edges_analyze = EdgesAnalysis(objective=objective, + approaches=self.edges_analyze_approaches, + requirements=requirements, + path_to_save=path_to_save) + + self.main_metric_idx = requirements.main_metric_idx + self.path_to_save = path_to_save + self.is_visualize_per_iteration = is_visualize_per_iteration + + def analyze(self, graph: Graph, + result: SAAnalysisResults = None, + nodes_to_analyze: List[GraphNode] = None, edges_to_analyze: List[Edge] = None, + n_jobs: int = 1, timer: OptimisationTimer = None) -> SAAnalysisResults: + """ + Applies defined structural analysis approaches + + :param graph: graph object to analyze + :param result: analysis result + :param nodes_to_analyze: nodes to analyze. Default: all nodes + :param edges_to_analyze: edges to analyze. Default: all edges + :param n_jobs: num of ``n_jobs`` for parallelization (``-1`` for use all cpu's). + Tip: if specified graph isn't huge (as NN, for example) than set n_jobs to default value. + :param timer: timer with timeout left for optimization + """ + + if not result: + result = SAAnalysisResults() + + if n_jobs == -1: + n_jobs = multiprocessing.cpu_count() + + if self.is_preproc: + graph = self.graph_preprocessing(graph=graph) + + if self.nodes_analyze_approaches: + self._nodes_analyze.analyze(graph=graph, + results=result, + nodes_to_analyze=nodes_to_analyze, + n_jobs=n_jobs, timer=timer) + + if self.edges_analyze_approaches: + self._edges_analyze.analyze(graph=graph, + results=result, + edges_to_analyze=edges_to_analyze, + n_jobs=n_jobs, timer=timer) + + return result + + def optimize(self, graph: Graph, + n_jobs: int = 1, timer: OptimisationTimer = None, + max_iter: int = 10) -> Tuple[Graph, SAAnalysisResults]: + """ Optimizes graph by applying 'analyze' method and deleting/replacing parts + of graph iteratively + :param graph: graph object to analyze. + :param n_jobs: num of ``n_jobs`` for parallelization (``-1`` for use all cpu's). + Tip: if specified graph isn't huge (as NN, for example) than set n_jobs to default value. + :param timer: timer with timeout left for optimization. + :param max_iter: max number of iterations of analysis. """ + + approaches_repo = StructuralAnalysisApproachesRepository() + approaches = self._nodes_analyze.approaches + self._edges_analyze.approaches + approaches_names = [approach.__name__ for approach in approaches] + + # what actions were applied on the graph and how many + actions_applied = dict.fromkeys(approaches_names, 0) + + result = SAAnalysisResults() + + analysis_result = self.analyze(graph=graph, result=result, timer=timer, n_jobs=n_jobs) + converged = False + iter = 0 + + if analysis_result.is_empty: + self._log.message(f'{iter} actions were taken during SA') + return graph, analysis_result + + while not converged: + iter += 1 + worst_result = analysis_result.get_info_about_worst_result( + metric_idx_to_optimize_by=self.main_metric_idx) + if self.is_visualize_per_iteration: + self.visualize_on_graph(graph=deepcopy(graph), analysis_result=analysis_result, + metric_idx_to_optimize_by=self.main_metric_idx, + mode='final', + font_size_scale=0.6) + if worst_result['value'] > 1: + # apply the worst approach + postproc_method = approaches_repo.postproc_method_by_name(worst_result['approach_name']) + graph = postproc_method(graph=graph, worst_result=worst_result) + actions_applied[f'{worst_result["approach_name"]}'] += 1 + + if timer is not None and timer.is_time_limit_reached(): + break + + if max_iter and iter >= max_iter: + break + + analysis_result = self.analyze(graph=graph, + result=result, + n_jobs=n_jobs, + timer=timer) + else: + converged = True + + self._log.message(f'{iter} iterations passed during SA') + self._log.message(f'The following actions were applied during SA: {actions_applied}') + + if self.path_to_save: + if not os.path.exists(self.path_to_save): + os.makedirs(self.path_to_save) + analysis_result.save(path=self.path_to_save) + + return graph, analysis_result + + @staticmethod + def apply_results(graph: Graph, analysis_result: SAAnalysisResults, + metric_idx_to_optimize_by: int, iter: int = None) -> Graph: + """ Optimizes graph by applying actions specified in analysis_result. """ + + def optimize_on_iter(graph: Graph, analysis_result: SAAnalysisResults, + metric_idx_to_optimize_by: int, iter: int = None) -> Graph: + """ Get worst result on specified iteration and process graph with it. """ + worst_result = analysis_result.get_info_about_worst_result( + metric_idx_to_optimize_by=metric_idx_to_optimize_by, iter=iter) + approaches_repo = StructuralAnalysisApproachesRepository() + postproc_method = approaches_repo.postproc_method_by_name(worst_result['approach_name']) + graph = postproc_method(graph=graph, worst_result=worst_result) + return graph + + if iter is not None: + return optimize_on_iter(graph=graph, analysis_result=analysis_result, + metric_idx_to_optimize_by=metric_idx_to_optimize_by, iter=iter) + + num_of_iter = len(analysis_result.results_per_iteration) + for i in range(num_of_iter): + graph = optimize_on_iter(graph=graph, analysis_result=analysis_result, + metric_idx_to_optimize_by=metric_idx_to_optimize_by, iter=iter) + return graph + + @staticmethod + def visualize_on_graph(graph: Graph, analysis_result: SAAnalysisResults, + metric_idx_to_optimize_by: int, mode: str = 'final', + save_path: str = None, node_color: Optional[NodeColorType] = None, dpi: Optional[int] = None, + node_size_scale: Optional[float] = None, font_size_scale: Optional[float] = None, + edge_curvature_scale: Optional[float] = None): + """ Visualizes results of Structural Analysis on graph(s). + :param graph: initial graph before SA + :param analysis_result: results of Structural Analysis + :param metric_idx_to_optimize_by: index of optimized metric + :param mode: 'first' -- visualize only first iteration of SA, + 'final' - visualize only the last iteration of SA, + 'by_iteration' -- visualize every iteration of SA. + :param save_path: path to save visualizations + :param node_color: color of nodes to use. + :param node_size_scale: use to make node size bigger or lesser. Supported only for the engine 'matplotlib'. + :param font_size_scale: use to make font size bigger or lesser. Supported only for the engine 'matplotlib'. + :param edge_curvature_scale: use to make edges more or less curved. Supported only for the engine 'matplotlib'. + :param dpi: DPI of the output image. Not supported for the engine 'pyvis'. + """ + + def get_nodes_and_edges_labels(analysis_result: SAAnalysisResults, iter: int) \ + -> Tuple[Dict[int, str], Dict[int, str]]: + """ Get nodes and edges labels in dictionary form. """ + + def get_str_labels(result: ObjectSAResult) -> str: + """ Get string results. """ + approaches = result.result_approaches + cur_label = '' + for approach in approaches: + approach_name = result._get_approach_name(approach=approach) + if 'del' in approach_name.lower(): + short_approach_name = 'D' + else: + short_approach_name = 'R' + cur_label += f'{short_approach_name}: {approach.get_rounded_metrics(idx=2)}\n' + return cur_label + + nodes_labels = {} + for i, node_result in enumerate(analysis_result.results_per_iteration[iter][EntityTypesEnum.node.value]): + nodes_labels[i] = get_str_labels(result=node_result) + + edges_labels = {} + for i, edge_result in enumerate(analysis_result.results_per_iteration[iter][EntityTypesEnum.edge.value]): + edges_labels[i] = get_str_labels(result=edge_result) + + return nodes_labels, edges_labels + + num_of_iter = len(analysis_result.results_per_iteration) + + if mode == 'first': + iters = [0] + else: + iters = range(num_of_iter) + + for i in iters: + nodes_labels, edges_labels = get_nodes_and_edges_labels(analysis_result=analysis_result, iter=i) + if not (mode == 'final' and i != iters[-1]): + if not Path.is_file(Path(save_path)): + j = 0 + while f'pipeline_after_sa_{j}.png' in os.listdir(save_path): + j += 1 + cur_path = os.path.join(save_path, f'pipeline_after_sa_{j}.png') + graph.show(node_color=node_color, dpi=dpi, node_size_scale=node_size_scale, + nodes_labels=nodes_labels, font_size_scale=font_size_scale, + edge_curvature_scale=edge_curvature_scale, + edges_labels=edges_labels, save_path=cur_path) + default_log("SA_visualization").info(f"SA visualization was saved to: {cur_path}") + if mode == 'by_iteration': + graph = GraphStructuralAnalysis.apply_results(graph=graph, analysis_result=analysis_result, + metric_idx_to_optimize_by=metric_idx_to_optimize_by, + iter=i) + + @staticmethod + def graph_preprocessing(graph: Graph) -> Graph: + """ Graph preprocessing, which consists in removing consecutive nodes + with the same models/operations in the graph """ + for node_child in reversed(graph.nodes): + if not node_child.nodes_from or len(node_child.nodes_from) != 1: + continue + nodes_uid_to_delete = [] + for node_parent in node_child.nodes_from: + if node_child.name == node_parent.name: + nodes_uid_to_delete.append(node_parent.uid) + # there is a need to store nodes using uid since after deleting one of the nodes in graph + # other nodes will not remain the same (nodes_from may be changed) + for uid in nodes_uid_to_delete: + node_to_delete = [node for node in graph.nodes if node.uid == uid][0] + graph.delete_node(node_to_delete) + return graph diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/node_sa_approaches.py b/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/node_sa_approaches.py new file mode 100644 index 0000000..a953550 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/node_sa_approaches.py @@ -0,0 +1,326 @@ +import random +from abc import ABC +from copy import deepcopy +from os import makedirs +from os.path import exists, join +from typing import List, Optional, Type, Union, Any + +from golem.core.log import default_log +from golem.core.dag.graph import Graph, GraphNode +from golem.core.optimisers.objective import Objective +from golem.core.optimisers.opt_node_factory import OptNodeFactory +from golem.core.optimisers.timer import OptimisationTimer +from golem.core.paths import default_data_dir +from golem.structural_analysis.base_sa_approaches import BaseAnalyzeApproach +from golem.structural_analysis.graph_sa.results.deletion_sa_approach_result import \ + DeletionSAApproachResult +from golem.structural_analysis.graph_sa.results.object_sa_result import ObjectSAResult +from golem.structural_analysis.graph_sa.results.replace_sa_approach_result import \ + ReplaceSAApproachResult +from golem.structural_analysis.graph_sa.results.utils import EntityTypesEnum +from golem.structural_analysis.graph_sa.sa_requirements import StructuralAnalysisRequirements, \ + ReplacementAnalysisMetaParams + + +class NodeAnalysis: + """ + :param approaches: methods applied to nodes to modify the graph or analyze certain operations.\ + Default: [NodeDeletionAnalyze, NodeTuneAnalyze, NodeReplaceOperationAnalyze] + :param path_to_save: path to save results to. Default: ~home/Fedot/structural + """ + + def __init__(self, node_factory: Any, + approaches: Optional[List[Type['NodeAnalyzeApproach']]] = None, + approaches_requirements: Optional[StructuralAnalysisRequirements] = None, + path_to_save: Optional[str] = None): + + self.node_factory = node_factory + + self.approaches = [NodeDeletionAnalyze, NodeReplaceOperationAnalyze] if approaches is None else approaches + + self.path_to_save = \ + join(default_data_dir(), 'structural', 'nodes_structural') if path_to_save is None else path_to_save + self.log = default_log(self) + + self.approaches_requirements = \ + StructuralAnalysisRequirements() if approaches_requirements is None else approaches_requirements + + def analyze(self, graph: Graph, node: GraphNode, + objective: Objective, + timer: Optional[OptimisationTimer] = None) -> ObjectSAResult: + + """ + Method runs Node analysis within defined approaches + + :param graph: Graph containing the analyzed Node + :param node: Node object to analyze in Graph + :param objective: objective function for computing metric values + :param timer: timer to check if the time allotted for structural analysis has expired + :return: dict with Node analysis result per approach + """ + + results = ObjectSAResult(entity_idx=str(graph.nodes.index(node)), + entity_type=EntityTypesEnum.node) + + for approach in self.approaches: + if timer is not None and timer.is_time_limit_reached(): + break + + results.add_result(approach(graph=graph, + objective=objective, + node_factory=self.node_factory, + requirements=self.approaches_requirements, + path_to_save=self.path_to_save).analyze(node=node)) + return results + + +class NodeAnalyzeApproach(BaseAnalyzeApproach, ABC): + """ + Base class for node analysis approach. + :param graph: Graph containing the analyzed Node + :param objective: objective functions for computing metric values + :param node_factory: node factory to choose nodes to replace to + :param path_to_save: path to save results to. Default: ~home/Fedot/structural + """ + + def __init__(self, graph: Graph, objective: Objective, node_factory: OptNodeFactory, + requirements: StructuralAnalysisRequirements = None, path_to_save=None): + super().__init__(graph, objective, requirements) + self._node_factory = node_factory + + self._origin_metrics = list() + self._path_to_save = \ + join(default_data_dir(), 'structural', 'nodes_structural') if path_to_save is None else path_to_save + self.log = default_log(prefix='node_analysis') + + if not exists(self._path_to_save): + makedirs(self._path_to_save) + + +class NodeDeletionAnalyze(NodeAnalyzeApproach): + def __init__(self, graph: Graph, objective: Objective, + node_factory: OptNodeFactory, + requirements: StructuralAnalysisRequirements = None, path_to_save=None): + super().__init__(graph, objective, node_factory, requirements) + self._path_to_save = \ + join(default_data_dir(), 'structural', 'nodes_structural') if path_to_save is None else path_to_save + if not exists(self._path_to_save): + makedirs(self._path_to_save) + + def analyze(self, node: GraphNode, **kwargs) -> DeletionSAApproachResult: + """ + Receives a graph without the specified node and tries to calculate the loss for it + + :param node: GraphNode object to analyze + :return: the ratio of modified graph score to origin score + """ + results = DeletionSAApproachResult() + if node is self._graph.root_node: + self.log.warning(f'{node} node can not be deleted') + results.add_results(metrics_values=[-1.0] * len(self._objective.metrics)) + return results + else: + shortened_graph = self.sample(node) + if shortened_graph: + losses = self._compare_with_origin_by_metrics(shortened_graph) + self.log.message(f'losses for {node.name}: {losses}') + del shortened_graph + else: + losses = [-1.0] * len(self._objective.metrics) + + results.add_results(metrics_values=losses) + return results + + def sample(self, node: GraphNode): + """ + Checks if it is possible to delete the node from the graph so that it remains valid, + and if so, deletes + + :param node: GraphNode object to delete from Graph object + :return: Graph object without node + """ + graph_sample = deepcopy(self._graph) + node_index_to_delete = self._graph.nodes.index(node) + node_to_delete = graph_sample.nodes[node_index_to_delete] + + if node_to_delete.name == 'class_decompose': + for child in graph_sample.node_children(node_to_delete): + graph_sample.delete_node(child) + + graph_sample.delete_node(node_to_delete) + + verifier = self._requirements.graph_verifier + if not verifier.verify(graph_sample): + self.log.message('Can not delete node since modified graph can not be verified') + return None + + return graph_sample + + +class NodeReplaceOperationAnalyze(NodeAnalyzeApproach): + """ + Replace node with operations available for the current task + and evaluate the score difference + """ + + def __init__(self, graph: Graph, objective: Objective, + node_factory: OptNodeFactory, + requirements: StructuralAnalysisRequirements = None, path_to_save=None): + super().__init__(graph, objective, node_factory, requirements) + + self._path_to_save = \ + join(default_data_dir(), 'structural', 'nodes_structural') if path_to_save is None else path_to_save + if not exists(self._path_to_save): + makedirs(self._path_to_save) + + def analyze(self, node: GraphNode, **kwargs) -> ReplaceSAApproachResult: + """ + Counts the loss on each changed graph received and returns losses + + :param node: GraphNode object to analyze + + :return: the ratio of modified graph score to origin score + """ + result = ReplaceSAApproachResult() + requirements: ReplacementAnalysisMetaParams = self._requirements.replacement_meta + node_id = self._graph.nodes.index(node) + samples = self.sample(node=node, + nodes_to_replace_to=requirements.nodes_to_replace_to, + number_of_random_operations=requirements.number_of_random_operations_nodes) + + for sample_graph in samples: + loss_per_sample = self._compare_with_origin_by_metrics(sample_graph) + self.log.message(f'losses: {loss_per_sample}\n') + + result.add_results(entity_to_replace_to=sample_graph.nodes[node_id].name, metrics_values=loss_per_sample) + + return result + + def sample(self, node: GraphNode, + nodes_to_replace_to: Optional[List[GraphNode]], + number_of_random_operations: int = 1) -> Union[List[Graph], Graph]: + """ + Replaces the given node with a pool of nodes available for replacement (see _node_generation docstring) + + :param node: GraphNode object to replace + :param nodes_to_replace_to: nodes provided for old_node replacement + :param number_of_random_operations: number of replacement operations, \ + if nodes_to_replace_to not provided + :return: Sequence of Graph objects with new operations instead of old one + """ + + if not nodes_to_replace_to: + nodes_to_replace_to = self._node_generation(node=node, + node_factory=self._node_factory, + number_of_operations=number_of_random_operations) + + samples = list() + for replacing_node in nodes_to_replace_to: + sample_graph = deepcopy(self._graph) + replaced_node_index = self._graph.nodes.index(node) + replaced_node = sample_graph.nodes[replaced_node_index] + sample_graph.update_node(old_node=replaced_node, + new_node=replacing_node) + verifier = self._requirements.graph_verifier + if not verifier.verify(sample_graph): + self.log.warning(f'Can not replace {node.name} node with {replacing_node.name} node.') + else: + self.log.message(f'replacing node: {replacing_node.name}') + samples.append(sample_graph) + + if not samples: + samples.append(self._graph) + + return samples + + @staticmethod + def _node_generation(node: GraphNode, + node_factory: OptNodeFactory, + number_of_operations: int = 1) -> List[GraphNode]: + """ + The method returns possible nodes that can replace the given node + + :param node: the node to be replaced + :param number_of_operations: limits the number of possible nodes to replace to + + :return: nodes that can be used to replace + """ + + available_nodes = [] + for i in range(number_of_operations): + available_nodes.append(node_factory.exchange_node(node=node)) + + if number_of_operations: + available_nodes = [i for i in available_nodes if i != node.name] + number_of_operations = min(len(available_nodes), number_of_operations) + random_nodes = random.sample(available_nodes, number_of_operations) + else: + random_nodes = available_nodes + + return random_nodes + + +class SubtreeDeletionAnalyze(NodeAnalyzeApproach): + """ + Approach to delete specified node subtree + """ + + def __init__(self, graph: Graph, objective: Objective, + node_factory: OptNodeFactory, + requirements: StructuralAnalysisRequirements = None, path_to_save=None): + super().__init__(graph, objective, node_factory, requirements) + self._path_to_save = \ + join(default_data_dir(), 'structural', 'nodes_structural')\ + if path_to_save is None else path_to_save + if not exists(self._path_to_save): + makedirs(self._path_to_save) + + def analyze(self, node: GraphNode, **kwargs) -> DeletionSAApproachResult: + """ + Receives a graph without the specified node's subtree and + tries to calculate the loss for it + + :param node: GraphNode object to analyze + :return: the ratio of modified graph score to origin score + """ + results = DeletionSAApproachResult() + if node is self._graph.root_node: + self.log.warning(f'{node} subtree can not be deleted') + results.add_results(metrics_values=[-1.0] * len(self._objective.metrics)) + return results + else: + shortened_graph = self.sample(node) + if shortened_graph: + losses = self._compare_with_origin_by_metrics(shortened_graph) + self.log.message(f'losses for {node.name}: {losses}') + del shortened_graph + else: + losses = [-1.0] * len(self._objective.metrics) + + results.add_results(metrics_values=losses) + return results + + def sample(self, node: GraphNode): + """ + Checks if it is possible to delete the node's subtree from the graph so that it remains valid, + and if so, deletes + + :param node: GraphNode object from which to delete subtree from Graph object + :return: Graph object without subtree + """ + graph_sample = deepcopy(self._graph) + node_index_to_delete = self._graph.nodes.index(node) + node_to_delete = graph_sample.nodes[node_index_to_delete] + + if node_to_delete.name == 'class_decompose': + for child in graph_sample.node_children(node_to_delete): + graph_sample.delete_node(child) + + graph_sample.delete_subtree(node_to_delete) + + verifier = self._requirements.graph_verifier + if not verifier.verify(graph_sample): + self.log.warning('Can not delete subtree since modified graph can not pass verification') + return None + + return graph_sample diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/nodes_analysis.py b/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/nodes_analysis.py new file mode 100644 index 0000000..8bd1426 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/nodes_analysis.py @@ -0,0 +1,81 @@ +from os.path import join +from typing import Optional, List, Type +import multiprocessing + +from golem.core.log import default_log +from golem.core.dag.graph import Graph, GraphNode +from golem.core.optimisers.objective import Objective +from golem.core.optimisers.opt_node_factory import OptNodeFactory +from golem.core.optimisers.timer import OptimisationTimer +from golem.core.paths import default_data_dir +from golem.structural_analysis.graph_sa.node_sa_approaches import NodeAnalyzeApproach, NodeAnalysis +from golem.structural_analysis.graph_sa.results.sa_analysis_results import SAAnalysisResults +from golem.structural_analysis.graph_sa.sa_requirements import StructuralAnalysisRequirements + + +class NodesAnalysis: + """ + This class is for nodes structural analysis within a Graph . + It takes nodes and approaches to be applied to chosen nodes. + To define which nodes to analyze pass them to nodes_to_analyze filed + or all nodes will be analyzed. + + :param objective: objective functions for computing metric values + :param node_factory: node factory to advise changes from available operations and models + :param approaches: methods applied to nodes to modify the graph or analyze certain operations.\ + Default: [NodeDeletionAnalyze, NodeReplaceOperationAnalyze] + :param path_to_save: path to save results to. Default: ~home/Fedot/structural + """ + + def __init__(self, objective: Objective, + node_factory: OptNodeFactory, + approaches: Optional[List[Type[NodeAnalyzeApproach]]] = None, + requirements: Optional[StructuralAnalysisRequirements] = None, + path_to_save: Optional[str] = None): + + self.objective = objective + self.node_factory = node_factory + self.approaches = approaches + self.requirements = \ + StructuralAnalysisRequirements() if requirements is None else requirements + self.log = default_log(self) + self.path_to_save = \ + join(default_data_dir(), 'structural', 'nodes_structural') if path_to_save is None else path_to_save + + def analyze(self, graph: Graph, results: SAAnalysisResults = None, + nodes_to_analyze: Optional[List[GraphNode]] = None, + n_jobs: int = 1, timer: Optional[OptimisationTimer] = None) -> SAAnalysisResults: + """ + Main method to run the analyze process for every node. + + :param graph: graph object to analyze + :param results: SA results + :param nodes_to_analyze: nodes to analyze. Default: all nodes + :param n_jobs: n_jobs + :param timer: timer indicating how much time is left for optimization + :return nodes_results: dict with analysis result per GraphNode + """ + + if not results: + results = SAAnalysisResults() + + if n_jobs == -1: + n_jobs = multiprocessing.cpu_count() + + if not nodes_to_analyze: + self.log.message('Nodes to analyze are not defined. All nodes will be analyzed.') + nodes_to_analyze = graph.nodes + + node_analysis = NodeAnalysis(approaches=self.approaches, + approaches_requirements=self.requirements, + node_factory=self.node_factory, + path_to_save=self.path_to_save) + + with multiprocessing.Pool(processes=n_jobs) as pool: + cur_nodes_results = pool.starmap(node_analysis.analyze, + [[graph, node, self.objective, timer] + for node in nodes_to_analyze]) + + results.add_results(cur_nodes_results) + + return results diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/postproc_methods.py b/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/postproc_methods.py new file mode 100644 index 0000000..a950374 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/postproc_methods.py @@ -0,0 +1,72 @@ +from golem.core.log import default_log +from golem.core.dag.graph import Graph +from golem.structural_analysis.graph_sa.results.utils import get_entity_from_str + + +def nodes_deletion(graph: Graph, worst_result: dict) -> Graph: + """ Extracts the node index from the entity key and removes it from the graph """ + + node_to_delete = graph.nodes[int(worst_result["entity_idx"])] + + graph.delete_node(node_to_delete) + default_log('NodeDeletion').message(f'{node_to_delete.name} was deleted') + + return graph + + +def nodes_replacement(graph: Graph, worst_result: dict) -> Graph: + """ Extracts the node index and the operation to which it needs to be replaced from the entity key + and replaces the node with a new one """ + + # get the node that will be replaced + node_to_replace = get_entity_from_str(graph=graph, entity_str=worst_result["entity_idx"]) + # get node to replace to + new_node = graph.nodes[0].__class__(worst_result["entity_to_replace_to"]) + # new_node = graph.nodes[int(worst_result["entity_to_replace_to"])] + new_node.nodes_from = [] + + graph.update_node(old_node=node_to_replace, new_node=new_node) + + default_log('NodeReplacement').message(f'{node_to_replace.name} was replaced with {new_node.name}') + + return graph + + +def subtree_deletion(graph: Graph, worst_result: dict) -> Graph: + """ Extracts the node index from the entity key and removes its subtree from the graph """ + + node_to_delete = get_entity_from_str(graph=graph, entity_str=worst_result["entity_idx"]) + graph.delete_subtree(node_to_delete) + default_log('SubtreeDeletion').message(f'{node_to_delete.name} subtree was deleted') + + return graph + + +def edges_deletion(graph: Graph, worst_result: dict) -> Graph: + """ Extracts the edge's nodes indices from the entity key and removes edge from the graph """ + parent_node, child_node = get_entity_from_str(graph=graph, entity_str=worst_result["entity_idx"]) + + graph.disconnect_nodes(parent_node, child_node) + default_log('EdgeDeletion').message(f'Edge from {parent_node.name} to {child_node.name} was deleted') + + return graph + + +def edges_replacement(graph: Graph, worst_result: dict) -> Graph: + """ Extracts the edge's nodes indices and the new edge to which it needs to be replaced from the entity key + and replaces the edge with a new one """ + + # get the edge that will be replaced + parent_node, child_node = get_entity_from_str(graph=graph, entity_str=worst_result["entity_idx"]) + + # get an edge to replace + next_parent_node, next_child_node = \ + get_entity_from_str(graph=graph, entity_str=worst_result["entity_to_replace_to"]) + graph.connect_nodes(next_parent_node, next_child_node) + + graph.disconnect_nodes(parent_node, child_node, clean_up_leftovers=False) + + default_log('EdgeReplacement').message(f'Edge from {parent_node.name} to {child_node.name} was replaced with ' + f'edge from {next_parent_node.name} to {next_child_node.name}') + + return graph diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/results/__init__.py b/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/results/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/results/base_sa_approach_result.py b/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/results/base_sa_approach_result.py new file mode 100644 index 0000000..629029f --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/results/base_sa_approach_result.py @@ -0,0 +1,21 @@ + + +class BaseSAApproachResult: + """ Base class for presenting all result classes. + Specifies the main logic of setting and getting calculated metrics. """ + + def get_worst_result(self, metric_idx_to_optimize_by: int) -> float: + """ Returns the worst result among all metrics. """ + raise NotImplementedError() + + def get_worst_result_with_names(self, metric_idx_to_optimize_by: int) -> dict: + """ Returns the worst result with additional info. """ + raise NotImplementedError() + + def add_results(self, **kwargs): + """ Adds newly calculated results. """ + raise NotImplementedError + + def get_dict_results(self) -> dict: + """ Returns dict representation of results. """ + raise NotImplementedError() diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/results/deletion_sa_approach_result.py b/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/results/deletion_sa_approach_result.py new file mode 100644 index 0000000..8ef26c9 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/results/deletion_sa_approach_result.py @@ -0,0 +1,30 @@ +from typing import List + +from golem.structural_analysis.graph_sa.results.base_sa_approach_result import BaseSAApproachResult + + +class DeletionSAApproachResult(BaseSAApproachResult): + """ Class for presenting deletion result approaches. """ + + def __init__(self): + self.metrics = [] + + def add_results(self, metrics_values: List[float]): + self.metrics = metrics_values + + def get_worst_result(self, metric_idx_to_optimize_by: int) -> float: + """ Returns the worst metric among all calculated. """ + return self.metrics[metric_idx_to_optimize_by] + + def get_worst_result_with_names(self, metric_idx_to_optimize_by: int) -> dict: + return {'value': self.get_worst_result(metric_idx_to_optimize_by=metric_idx_to_optimize_by)} + + def get_dict_results(self) -> List[float]: + """ Returns all calculated results. """ + return self.metrics + + def get_rounded_metrics(self, idx: int = 2) -> list: + return [round(metric, idx) for metric in self.metrics] + + def __str__(self): + return 'DeletionSAApproachResult' diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/results/object_sa_result.py b/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/results/object_sa_result.py new file mode 100644 index 0000000..de72d76 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/results/object_sa_result.py @@ -0,0 +1,77 @@ +from typing import List + +from golem.structural_analysis.graph_sa.results.base_sa_approach_result import BaseSAApproachResult +from golem.structural_analysis.graph_sa.results.deletion_sa_approach_result import \ + DeletionSAApproachResult +from golem.structural_analysis.graph_sa.results.replace_sa_approach_result import \ + ReplaceSAApproachResult +from golem.structural_analysis.graph_sa.results.utils import EntityTypesEnum + +NODE_DELETION = 'NodeDeletionAnalyze' +NODE_REPLACEMENT = 'NodeReplaceOperationAnalyze' +SUBTREE_DELETION = 'SubtreeDeletionAnalyze' +EDGE_DELETION = 'EdgeDeletionAnalyze' +EDGE_REPLACEMENT = 'EdgeReplaceOperationAnalyze' + + +class StructuralAnalysisResultsRepository: + approaches_dict = {NODE_DELETION: {'result_class': DeletionSAApproachResult}, + NODE_REPLACEMENT: {'result_class': ReplaceSAApproachResult}, + SUBTREE_DELETION: {'result_class': DeletionSAApproachResult}, + EDGE_DELETION: {'result_class': DeletionSAApproachResult}, + EDGE_REPLACEMENT: {'result_class': ReplaceSAApproachResult}} + + def get_method_by_result_class(self, result_class: BaseSAApproachResult, entity_class: str) -> str: + for method in self.approaches_dict.keys(): + if self.approaches_dict[method]['result_class'] == result_class.__class__ \ + and entity_class in method.lower(): + return method + + def get_class_by_str(self, result_str: str) -> BaseSAApproachResult: + for method in self.approaches_dict.keys(): + if result_str == method: + return self.approaches_dict[method]['result_class'] + + +class ObjectSAResult: + """ Class specifying results of Structural Analysis for one entity(node or edge). """ + + def __init__(self, entity_idx: str, entity_type: EntityTypesEnum): + self.entity_idx = entity_idx + self.entity_type = entity_type + self.result_approaches: List[BaseSAApproachResult] = [] + + def get_worst_result(self, metric_idx_to_optimize_by: int) -> float: + """ Returns the worst result among all result classes. """ + worst_results = [] + for approach in self.result_approaches: + worst_results.append(approach.get_worst_result(metric_idx_to_optimize_by=metric_idx_to_optimize_by)) + if not worst_results: + return 0 + return max(worst_results) + + def get_worst_result_with_names(self, metric_idx_to_optimize_by: int) -> dict: + """ Returns worst result with additional information. """ + worst_result = self.get_worst_result(metric_idx_to_optimize_by=metric_idx_to_optimize_by) + for approach in self.result_approaches: + if approach.get_worst_result(metric_idx_to_optimize_by=metric_idx_to_optimize_by) == worst_result: + sa_approach_name = self._get_approach_name(approach=approach) + result = {'entity_idx': self.entity_idx, 'approach_name': sa_approach_name} + result.update(approach.get_worst_result_with_names(metric_idx_to_optimize_by=metric_idx_to_optimize_by)) + return result + + def add_result(self, result: BaseSAApproachResult): + self.result_approaches.append(result) + + def get_dict_results(self) -> dict: + """ Returns dict representation of results. """ + results = dict() + for approach in self.result_approaches: + sa_approach_name = self._get_approach_name(approach=approach) + results[sa_approach_name] = approach.get_dict_results() + return {self.entity_idx: results} + + def _get_approach_name(self, approach: BaseSAApproachResult) -> str: + sa_approach_name = StructuralAnalysisResultsRepository() \ + .get_method_by_result_class(approach, self.entity_type.value) + return sa_approach_name diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/results/replace_sa_approach_result.py b/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/results/replace_sa_approach_result.py new file mode 100644 index 0000000..50903f9 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/results/replace_sa_approach_result.py @@ -0,0 +1,39 @@ +from typing import List, Dict + +from golem.structural_analysis.graph_sa.results.base_sa_approach_result import BaseSAApproachResult + + +class ReplaceSAApproachResult(BaseSAApproachResult): + """ Class for presenting replacing result approaches. """ + def __init__(self): + """ Main dictionary `self.metrics` contains entities as key and + list with metrics as values""" + self.metrics = dict() + + def add_results(self, entity_to_replace_to: str, metrics_values: List[float]): + """ Sets value for specified metric. """ + self.metrics[entity_to_replace_to] = metrics_values + + def get_worst_result(self, metric_idx_to_optimize_by: int) -> float: + """ Returns value of the worst metric. """ + return max([metrics[metric_idx_to_optimize_by] for metrics in list(self.metrics.values())]) + + def get_worst_result_with_names(self, metric_idx_to_optimize_by: int) -> dict: + """ Returns the worst metric among all calculated with its name and node's to replace to name. """ + worst_value = self.get_worst_result(metric_idx_to_optimize_by=metric_idx_to_optimize_by) + for entity in self.metrics: + if list(self.metrics[entity])[metric_idx_to_optimize_by] == worst_value: + return {'value': worst_value, 'entity_to_replace_to': entity} + + def get_dict_results(self) -> Dict[int, List[float]]: + """ Returns dict representation of results. """ + return self.metrics + + def get_rounded_metrics(self, idx: int = 2) -> dict: + rounded = {} + for metric in self.metrics: + rounded[metric] = [round(metric, idx) for metric in self.metrics[metric]] + return rounded + + def __str__(self): + return 'ReplaceSAApproachResult' diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/results/sa_analysis_results.py b/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/results/sa_analysis_results.py new file mode 100644 index 0000000..75a22a2 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/results/sa_analysis_results.py @@ -0,0 +1,135 @@ +import json +import os.path +from datetime import datetime +from typing import List, Optional, Union + +from golem.core.dag.graph import Graph +from golem.core.log import default_log +from golem.core.paths import project_root +from golem.utilities.serializable import Serializable +from golem.serializers import Serializer +from golem.structural_analysis.graph_sa.results.deletion_sa_approach_result import DeletionSAApproachResult +from golem.structural_analysis.graph_sa.results.object_sa_result import ObjectSAResult, \ + StructuralAnalysisResultsRepository +from golem.structural_analysis.graph_sa.results.utils import EntityTypesEnum + + +class SAAnalysisResults(Serializable): + """ Class presenting results of Structural Analysis for the whole graph. """ + + def __init__(self): + self.results_per_iteration = {} + self._add_empty_iteration_results() + self.log = default_log('sa_results') + + def _add_empty_iteration_results(self): + last_iter_num = int(list(self.results_per_iteration.keys())[-1]) if self.results_per_iteration.keys() else -1 + self.results_per_iteration.update({(last_iter_num + 1): self._init_iteration_result()}) + + @staticmethod + def _init_iteration_result() -> dict: + return {EntityTypesEnum.node.value: [], EntityTypesEnum.edge.value: []} + + @property + def is_empty(self) -> bool: + """ Bool value indicating is there any calculated results. """ + if self.results_per_iteration[0] is None and \ + self.results_per_iteration[0] is None: + return True + return False + + def get_info_about_worst_result(self, metric_idx_to_optimize_by: int, iter: Optional[int] = None) -> dict: + """ Returns info about the worst result. + :param metric_idx_to_optimize_by: metric idx to optimize by + :param iter: iteration on which to search for. """ + worst_value = None + worst_result = None + if iter is None: + iter = list(self.results_per_iteration.keys())[-1] + + nodes_results = self.results_per_iteration[iter][EntityTypesEnum.node.value] + edges_results = self.results_per_iteration[iter][EntityTypesEnum.edge.value] + + for i, res in enumerate(nodes_results + edges_results): + cur_res = res.get_worst_result_with_names( + metric_idx_to_optimize_by=metric_idx_to_optimize_by) + if not worst_value or cur_res['value'] > worst_value: + worst_value = cur_res['value'] + worst_result = cur_res + return worst_result + + def add_results(self, results: List[ObjectSAResult]): + if not results: + return + key = results[0].entity_type.value + iter_num = self._get_last_empty_iter(key=key) + for result in results: + self.results_per_iteration[iter_num][key].append(result) + + def _get_last_empty_iter(self, key: str) -> int: + """ Returns number of last iteration with empty key field. """ + for i, result in enumerate(self.results_per_iteration.values()): + if not result[key]: + return i + self._add_empty_iteration_results() + return list(self.results_per_iteration.keys())[-1] + + def save(self, path: str = None, datetime_in_path: bool = True) -> dict: + """ Saves SA results in json format. """ + dict_results = dict() + for iter in self.results_per_iteration.keys(): + dict_results[iter] = {} + iter_result = self.results_per_iteration[iter] + for entity_type in iter_result.keys(): + if entity_type not in dict_results[iter].keys(): + dict_results[iter][entity_type] = {} + for entity in iter_result[entity_type]: + dict_results[iter][entity_type].update(entity.get_dict_results()) + + json_data = json.dumps(dict_results, cls=Serializer) + + if not path: + path = os.path.join(project_root(), 'sa', 'sa_results.json') + if datetime_in_path: + file_name = os.path.basename(path).split('.')[0] + file_name = f"{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}_{file_name}.json" + path = os.path.join(os.path.dirname(path), 'sa') + if not os.path.exists(path): + os.makedirs(path) + path = os.path.join(path, file_name) + + with open(path, 'w', encoding='utf-8') as f: + f.write(json_data) + self.log.debug(f'SA results saved in the path: {path}.') + + return dict_results + + @staticmethod + def load(source: Union[str, dict], graph: Optional[Graph] = None) -> 'SAAnalysisResults': + """ Loads SA results from json format. """ + if isinstance(source, str): + source = json.load(open(source)) + + sa_result = SAAnalysisResults() + results_repo = StructuralAnalysisResultsRepository() + + for iter in source: + for entity_type in source[iter]: + type_list = [] + for entity_idx in source[iter][entity_type]: + cur_result = ObjectSAResult(entity_idx=entity_idx, + entity_type=EntityTypesEnum(entity_type)) + dict_results = source[iter][entity_type][entity_idx] + for approach in dict_results: + app = results_repo.get_class_by_str(approach)() + if isinstance(app, DeletionSAApproachResult): + app.add_results(metrics_values=dict_results[approach]) + else: + for entity_to_replace_to in dict_results[approach]: + app.add_results(entity_to_replace_to=entity_to_replace_to, + metrics_values=dict_results[approach][entity_to_replace_to]) + cur_result.add_result(app) + type_list.append(cur_result) + sa_result.add_results(type_list) + + return sa_result diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/results/utils.py b/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/results/utils.py new file mode 100644 index 0000000..da2c53a --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/results/utils.py @@ -0,0 +1,21 @@ +from enum import Enum +from typing import Union, Tuple + +from golem.core.dag.graph import Graph +from golem.core.dag.graph_node import GraphNode + + +class EntityTypesEnum(Enum): + node = 'node' + edge = 'edge' + + +def get_entity_from_str(graph: Graph, entity_str: str) -> Union[GraphNode, Tuple[GraphNode, GraphNode]]: + """ Gets entity from entity str using graph. """ + if len(entity_str.split('_')) == 2: + parent_node_idx, child_node_idx = entity_str.split('_') + parent_node = graph.nodes[int(parent_node_idx)] + child_node = graph.nodes[int(child_node_idx)] + return parent_node, child_node + else: + return graph.nodes[int(entity_str)] diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/sa_approaches_repository.py b/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/sa_approaches_repository.py new file mode 100644 index 0000000..408db6d --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/sa_approaches_repository.py @@ -0,0 +1,30 @@ +from golem.structural_analysis.graph_sa.edge_sa_approaches import EdgeDeletionAnalyze, EdgeReplaceOperationAnalyze +from golem.structural_analysis.graph_sa.node_sa_approaches import NodeDeletionAnalyze, NodeReplaceOperationAnalyze, \ + SubtreeDeletionAnalyze +from golem.structural_analysis.graph_sa.postproc_methods import nodes_deletion, nodes_replacement, subtree_deletion, \ + edges_deletion, edges_replacement + +NODE_DELETION = 'NodeDeletionAnalyze' +NODE_REPLACEMENT = 'NodeReplaceOperationAnalyze' +SUBTREE_DELETION = 'SubtreeDeletionAnalyze' +EDGE_DELETION = 'EdgeDeletionAnalyze' +EDGE_REPLACEMENT = 'EdgeReplaceOperationAnalyze' + + +class StructuralAnalysisApproachesRepository: + approaches_dict = {NODE_DELETION: {'approach': NodeDeletionAnalyze, + 'postproc_method': nodes_deletion}, + NODE_REPLACEMENT: {'approach': NodeReplaceOperationAnalyze, + 'postproc_method': nodes_replacement}, + SUBTREE_DELETION: {'approach': SubtreeDeletionAnalyze, + 'postproc_method': subtree_deletion}, + EDGE_DELETION: {'approach': EdgeDeletionAnalyze, + 'postproc_method': edges_deletion}, + EDGE_REPLACEMENT: {'approach': EdgeReplaceOperationAnalyze, + 'postproc_method': edges_replacement}} + + def approach_by_name(self, approach_name: str): + return self.approaches_dict[approach_name]['approach'] + + def postproc_method_by_name(self, approach_name: str): + return self.approaches_dict[approach_name]['postproc_method'] diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/sa_requirements.py b/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/sa_requirements.py new file mode 100644 index 0000000..149b2f8 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/structural_analysis/graph_sa/sa_requirements.py @@ -0,0 +1,54 @@ +import random +from collections import namedtuple +from typing import List, Optional + +from golem.core.dag.graph_verifier import GraphVerifier +from golem.core.dag.verification_rules import DEFAULT_DAG_RULES +from golem.core.dag.graph import GraphNode +from golem.structural_analysis.graph_sa.entities.edge import Edge + +ReplacementAnalysisMetaParams = namedtuple('ReplacementAnalysisMetaParams', ['edges_to_replace_to', + 'number_of_random_operations_edges', + 'nodes_to_replace_to', + 'number_of_random_operations_nodes']) + + +class StructuralAnalysisRequirements: + """ + Use this object to pass all the requirements needed for SA. + + :param graph_verifier: verifier for graph in SA. + :param main_metric_idx: index of metric to optimize by. Other metrics will be calculated and saved if needed. + :param replacement_nodes_to_replace_to: defines nodes which is used in replacement analysis + :param replacement_number_of_random_operations_nodes: if replacement_nodes_to_replace_to is not filled, \ + define the number of randomly chosen operations used in replacement analysis + :param replacement_edges_to_replace_to: defines edges which is used in replacement analysis + :param replacement_number_of_random_operations_edges: if replacement_edges_to_replace_to is not filled, \ + define the number of randomly chosen operations used in replacement analysis + :param is_visualize: defines whether the SA visualization needs to be saved to .png files + :param is_save_results_to_json: defines whether the SA indices needs to be saved to .json file + """ + + def __init__(self, + graph_verifier: GraphVerifier = None, + main_metric_idx: int = 0, + replacement_nodes_to_replace_to: Optional[List[GraphNode]] = None, + replacement_number_of_random_operations_nodes: Optional[int] = 3, + replacement_edges_to_replace_to: Optional[List[Edge]] = None, + replacement_number_of_random_operations_edges: Optional[int] = 3, + is_visualize: bool = False, + is_save_results_to_json: bool = False, + seed: int = random.randint(0, 100)): + + self.graph_verifier = graph_verifier or GraphVerifier(DEFAULT_DAG_RULES) + + self.main_metric_idx = main_metric_idx + + self.replacement_meta = ReplacementAnalysisMetaParams(replacement_edges_to_replace_to, + replacement_number_of_random_operations_edges, + replacement_nodes_to_replace_to, + replacement_number_of_random_operations_nodes) + + self.is_visualize = is_visualize + self.is_save = is_save_results_to_json + self.seed = seed diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/utilities/__init__.py b/paper_experiments/large_bayesian_networks_experiments/golem/utilities/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/utilities/data_structures.py b/paper_experiments/large_bayesian_networks_experiments/golem/utilities/data_structures.py new file mode 100644 index 0000000..c2f1c16 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/utilities/data_structures.py @@ -0,0 +1,317 @@ +import collections.abc +import dataclasses + +from abc import ABC, abstractmethod +from copy import deepcopy +from enum import Enum +from typing import Callable, Container, Generic, Iterable, Iterator, List, Optional, Sequence, Sized, TypeVar, Union, \ + Tuple, Any, Dict + +T = TypeVar('T') + + +class UniqueList(list, Generic[T]): + """ + Simple list that maintains uniqueness of elements. + In comparison to 'set': preserves list interface and element ordering. + + But this class is better to use only for short lists because of linear complexity of uniqueness lookup. + Behaves optimally for <= 4 items, good enough for <= 20 items. + + List addition and multiplication are not overloaded and return a standard list. + But addition with assignment (+=) behaves as UniqueList.extend. + + :param iterable: list of elements to turn into distinct + """ + + def __init__(self, iterable: Optional[Iterable[T]] = None): + iterable = iterable or () + super().__init__(dict.fromkeys(iterable).keys()) # preserves order and uniqueness in the newly created list + + def append(self, value: T): + """ + Adds ``value`` to the end of the list in case it is not present there + + :param value: will be added to the list or ignored if it's already there + """ + if value not in super().__iter__(): + super().append(value) + + def extend(self, iterable: Iterable[T]): + """ + Extends current list with the ``iterable`` elements that aren't present in the list yet + + :param iterable: sequence of elements to be added to the list in case they aren't there + """ + impl = super() + impl.extend(element for element in iterable + if element not in impl.__iter__()) + + def insert(self, index: int, value: T): + """ + Inserts specified ``value`` to the provided ``index`` in the list in case it's unique value + + :param index: position for inserting into the list + :param value: will be inserted at the specified ``index`` if it's unique + """ + if value not in super().__iter__(): + super().insert(index, value) + + def __setitem__(self, key: Union[int, slice], value: Union[T, Iterable[T]]): + """ + Sets specified ``value`` at the specified index or slice if it's unique value + + :param key: for pointing to the elements from the list + :param value: element(s) to be set by the specified ``key`` + """ + if value not in super().__iter__(): + super().__setitem__(key, value) + + def __iadd__(self, other: Iterable[T]) -> 'UniqueList': + """ + Extends current list with the ``iterable`` elements that aren't present in the list yet + + :param other: sequence of elements to be added to the list in case they aren't there + + :return: this class instance + """ + self.extend(other) + return self + + +def remove_items(collection: List[T], items_to_remove: Container[T]): + """ + Removes all specified items from the list. Modifies original collection + + :param collection: list of elements to be filtered + :param items_to_remove: filter list of unwanted elements + + :return: modified ``collection`` parameter + """ + if collection: + collection[:] = [item for item in collection if item not in items_to_remove] + return collection + + +def are_same_length(collections: Iterable[Sized]) -> bool: + """ + Checks if all arguments have the same length + + :param collections: collection of collections + + :return: does collections inside ``collections`` have the same length + """ + it = collections.__iter__() + first = next(it, None) + if first is not None: + first = len(first) + for elem in it: + if len(elem) != first: + return False + return True + + +def ensure_wrapped_in_sequence( + obj: Optional[Union[T, Iterable[T]]], + sequence_factory: Callable[[Iterable[T]], Sequence[T]] = list +) -> Optional[Sequence[T]]: + """ + Makes sure given ``obj`` is of type that acts like sequence and converts ``obj`` to it otherwise + + :param obj: any object to be ensured or wrapped into sequence type + :param sequence_factory: sequence factory for wrapping ``obj`` into in case it is not of any sequence type + + :return: the same object if it's of type sequence (or None) + or ``obj`` wrapped in type provided by ``sequence_factory`` + """ + if obj is None: + return obj + elif isinstance(obj, str) or not isinstance(obj, collections.abc.Iterable): + return sequence_factory([obj]) + elif not isinstance(obj, collections.abc.Sequence) or not isinstance(obj, sequence_factory): + return sequence_factory(obj) + return obj + + +class ComparableEnum(Enum): + """ + The Enum implementation that allows to avoid the multi-module enum comparison problem + (https://stackoverflow.com/questions/26589805/python-enums-across-modules) + """ + + def __eq__(self, other: 'Enum'): + """ + Compares this enum with the ``other_graph`` + + :param other: another enum + + :return: is it equal to ``other`` in terms of the string representation + """ + return str(self) == str(other) + + def __hash__(self): + """ + Gets hashcode of this enum + + :return: hashcode of string representation of this enum + """ + return hash(str(self)) + + +class Copyable: + """Provides default implementations for `copy` & `deepcopy`.""" + + def __copy__(self): + cls = self.__class__ + result = cls.__new__(cls) + result.__dict__.update(self.__dict__) + return result + + def __deepcopy__(self, memo=None): + cls = self.__class__ + result = cls.__new__(cls) + memo[id(self)] = result + for k, v in self.__dict__.items(): + setattr(result, k, deepcopy(v, memo)) + return result + + +class Comparable(ABC): + @abstractmethod + def __eq__(self, other: 'Comparable') -> bool: + """ + Compares this object of :class:`Comparable` interface with the ``other`` comparable + + :param other: other object implementing :class:`Comparable` interface + + :return: is it equal to ``other`` in terms of the comparables + """ + raise NotImplementedError() + + @abstractmethod + def __lt__(self, other: 'Comparable') -> bool: + """ + Compares this object of :class:`Comparable` interface with the ``other`` comparable + + :param other: other object implementing :class:`Comparable` interface + + :return: is less than ``other`` in terms of the comparables + """ + raise NotImplementedError() + + def __ne__(self, other: 'Comparable') -> bool: + """ + Compares this object of :class:`Comparable` interface with the ``other`` comparable + + :param other: other object implementing :class:`Comparable` interface + + :return: is it not equal to ``other`` in terms of the comparables + """ + return not self.__eq__(other) + + def __le__(self, other) -> bool: + """ + Compares this object of :class:`Comparable` interface with the ``other`` comparable + + :param other: other object implementing :class:`Comparable` interface + + :return: is less than or equal to ``other`` in terms of the comparables + """ + return self.__lt__(other) or self.__eq__(other) + + def __gt__(self, other) -> bool: + """ + Compares this object of :class:`Comparable` interface with the ``other`` comparable + + :param other: other object implementing :class:`Comparable` interface + + :return: is greater than ``other`` in terms of the comparables + """ + return not self.__le__(other) + + def __ge__(self, other) -> bool: + """ + Compares this object of :class:`Comparable` interface with the ``other`` comparable + + :param other: other object implementing :class:`Comparable` interface + + :return: is greater than or equal to ``other`` in terms of the comparables + """ + return not self.__lt__(other) + + +class BidirectionalIterator(Iterator[T]): + @abstractmethod + def has_prev(self) -> bool: + """ + Checks if this iterator has implemented previous item getter + + :return: whether this iterator has previous item getter + """ + raise NotImplementedError() + + @abstractmethod + def has_next(self) -> bool: + """ + Checks if this iterator has implemented next item getter + + :return: whether this iterator has next item getter + """ + raise NotImplementedError() + + @abstractmethod + def next(self) -> T: + """ + Gets next item of this iterator + + :return: next item + """ + raise NotImplementedError() + + @abstractmethod + def prev(self) -> T: + """ + Gets previous item of this iterator + + :return: previous item + """ + raise NotImplementedError() + + @abstractmethod + def current(self) -> T: + """ + Get current value pointed to by the iterator + + :return: current item + """ + raise NotImplementedError() + + def __next__(self): + """ + Gets next item of this iterator + + :raises StopIteration: the end of this iterator + + :return: next item + """ + if self.has_next(): + return self.next() + raise StopIteration() + + def __iter__(self): + """ + Returns this iterator + + :return: this iterator + """ + return self + + +def unzip(tuples: Iterable[Tuple]) -> Tuple[Sequence, ...]: + return tuple(zip(*tuples)) + + +def update_dataclass(base_dc: Any, update_dc: Union[Any, Dict]) -> Any: + update_dict = dataclasses.asdict(update_dc) if dataclasses.is_dataclass(update_dc) else update_dc + new_base = dataclasses.replace(base_dc, **update_dict) + return new_base diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/utilities/grouped_condition.py b/paper_experiments/large_bayesian_networks_experiments/golem/utilities/grouped_condition.py new file mode 100644 index 0000000..ff43be2 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/utilities/grouped_condition.py @@ -0,0 +1,42 @@ +from typing import Optional, List, Callable, Tuple, Iterable + +from golem.core.log import default_log + +ConditionType = Callable[[], bool] +ConditionEntryType = Tuple[ConditionType, Optional[str]] + + +class GroupedCondition: + """Represents sequence of ordinary conditions with logging. + All composed conditions are combined with reduce function on booleans. + + By the default 'any' is used, so in this case the grouped condition is True + if any of the composed conditions is True. The message corresponding + to the actual fired condition is logged (if it was provided).""" + + def __init__(self, conditions_reduce: Callable[[Iterable[bool]], bool] = any, results_as_message: bool = False): + self._reduce = conditions_reduce + self._conditions: List[ConditionEntryType] = [] + self._log = default_log(self) + self._results_as_message = results_as_message + + def add_condition(self, condition: ConditionType, log_msg: Optional[str] = None) -> 'GroupedCondition': + """Builder-like method for adding conditions.""" + self._conditions.append((condition, log_msg)) + return self + + def __bool__(self): + return self() + + def __call__(self) -> bool: + return self._reduce(map(self._check_condition, self._conditions)) + + def _check_condition(self, entry: ConditionEntryType) -> bool: + cond, msg = entry + res = cond() + if res and msg: + if self._results_as_message: + self._log.message(msg) + else: + self._log.info(msg) + return res diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/utilities/memory.py b/paper_experiments/large_bayesian_networks_experiments/golem/utilities/memory.py new file mode 100644 index 0000000..7993b77 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/utilities/memory.py @@ -0,0 +1,57 @@ +import logging +import tracemalloc +from typing import Optional + + +class MemoryAnalytics: + is_active = False + active_session_label = 'main' + + @classmethod + def start(cls): + """ + Start memory monitoring session + """ + cls.is_active = True + tracemalloc.start() + + @classmethod + def finish(cls): + """ + Finish memory monitoring session + """ + cls.log(additional_info=f'finish', + logging_level=45) # message logging level + tracemalloc.stop() + cls.is_active = False + + @classmethod + def get_measures(cls): + """ + Estimates Python-related system memory consumption in MiB + :return: current and maximal consumption + """ + current_memory, max_memory = tracemalloc.get_traced_memory() + return current_memory / 1024 / 1024, max_memory / 1024 / 1024 + + @classmethod + def log(cls, logger: Optional[logging.LoggerAdapter] = None, + additional_info: str = 'location', logging_level: int = logging.INFO) -> str: + """ + Print the message about current and maximal memory consumption to the log or console. + :param logger: optional logger that should be used in output. + :param additional_info: label for current location in code. + :param logging_level: level of the message + :return: text of the message. + """ + message = '' + if cls.is_active: + memory_consumption = cls.get_measures() + message = f'Memory consumption for {additional_info} in {cls.active_session_label} session: ' \ + f'current {round(memory_consumption[0], 1)} MiB, ' \ + f'max: {round(memory_consumption[1], 1)} MiB' + if logger is not None: + logger.log(logging_level, message) + else: + print(message) + return message diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/utilities/profiler/__init__.py b/paper_experiments/large_bayesian_networks_experiments/golem/utilities/profiler/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/utilities/profiler/memory_profiler.py b/paper_experiments/large_bayesian_networks_experiments/golem/utilities/profiler/memory_profiler.py new file mode 100644 index 0000000..41d72da --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/utilities/profiler/memory_profiler.py @@ -0,0 +1,97 @@ +import inspect +import os +import time + +import matplotlib.pyplot as plt +import numpy as np +import seaborn as sns + +from golem.core.dag.graph import Graph +from golem.core.optimisers.optimizer import GraphOptimizer +from golem.utilities.requirements_notificator import warn_requirement + +try: + import objgraph + + from memory_profiler import memory_usage +except ImportError: + warn_requirement('objgraph', 'golem[profilers]', should_raise=True) + + +class MemoryProfiler: + """Visual interpretation of memory usage. Create two ``png`` files. + + Args: + function: function to profile. + path: path to save profiling result. + list args: arguments for function in array format. + dict kwargs: arguments for function in dictionary format. + list roots: array with classes used as ROOT nodes in the call-graph. + int max_depth: maximum depth of graph. + """ + + def __init__(self, function, path: str, args=None, kwargs=None, roots=None, max_depth: int = 7, + visualization=False): + if args is None: + args = [] + + if kwargs is None: + kwargs = {} + + self.folder = os.path.abspath(path) + + if not os.path.exists(self.folder): + os.mkdir(self.folder) + + if roots is None: + roots = [Graph, GraphOptimizer] + + if visualization: + # # Create the plot of the memory time dependence. + # self._create_memory_plot(function, args, kwargs) + + # Create call graph. + self._create_memory_graph(roots, max_depth) + + def _create_memory_plot(self, function, args, kwargs, interval: float = 0.1): + start_time = time.time() + mem_res = memory_usage((function, args, kwargs), interval=interval) + total_time = time.time() - start_time + + length = len(mem_res) + division = total_time / length + time_split = np.linspace(division, total_time, length) + + max_index = np.argmax(mem_res) + + plt.figure(figsize=(12, 8)) + + sns.set_style("whitegrid") + + ax = sns.lineplot((time_split, mem_res), linewidth=3, marker="X", markersize=8) + ax.axhline(mem_res[max_index], ls='--', color='red') + ax.axvline(time_split[max_index], ls='--', color='red') + + plt.legend(labels=[f'interval {interval}', 'max memory usage']) + + ax.set_title('Memory time dependence', size=25) + + ax.set_xlabel("time [s]", fontsize=16) + ax.set_ylabel("memory used [KB]", fontsize=16) + + filename = os.path.join(self.folder, 'memory_plot.png') + ax.figure.savefig(filename) + + def _create_memory_graph(self, roots, max_depth): + filename = os.path.join(self.folder, 'memory_graph.png') + + objgraph.show_refs( + roots, + max_depth=max_depth, + filename=filename, + filter=lambda x: not type(x).__name__ in ['module', 'str', 'code', 'IPythonKernel', '_Helper', + 'builtin_function_or_method', 'type', '_Printer', 'Printer', + 'float64', 'float', 'int'], + highlight=lambda x: inspect.isclass(x) or inspect.isfunction(x), + refcounts=True + ) diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/utilities/profiler/time_profiler.py b/paper_experiments/large_bayesian_networks_experiments/golem/utilities/profiler/time_profiler.py new file mode 100644 index 0000000..ec268a3 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/utilities/profiler/time_profiler.py @@ -0,0 +1,64 @@ +import cProfile +import os +import pstats +import subprocess + +from golem.utilities.requirements_notificator import warn_requirement + +try: + import gprof2dot +except ImportError: + warn_requirement('gprof2dot', 'golem[profilers]', should_raise=True) +try: + import snakeviz +except ImportError: + warn_requirement('snakeviz', 'golem[profilers]', should_raise=True) + + +class TimeProfiler: + """Profile code and visual interpret results of it + """ + + def __init__(self): + self.folder = None + self.profiler = cProfile.Profile() + self.profiler.enable() + + def _generate_pstats(self, path: str): + """Aggregate profiler statistics and create :obj:`output.pstats` from ``Profiler`` + + Args: + path: path to save results + """ + + self.path_stats = os.path.join(os.path.abspath(path), 'output.pstats') + stats = pstats.Stats(self.profiler) + stats.dump_stats(self.path_stats) + + def profile(self, path: str, node_percent: float = 0.5, edge_percent: float = 0.1, open_web: bool = False): + """Method to convert the statistics from profiler to visual representation + + Args: + spath: path to save profiling result + node_percent: eliminate nodes below this threshold [default: 0.5] + edge_percent: eliminate edges below this threshold [default: 0.1] + open_web: boolean parametr to open web-interface + """ + + self.profiler.disable() + + self.folder = os.path.abspath(path) + + if not os.path.exists(self.folder): + os.mkdir(self.folder) + + self._generate_pstats(self.folder) + + # Creating the PNG call-graph with the cumulative time. + path = os.path.join(self.folder, 'graph.png') + subprocess.getstatusoutput(f"gprof2dot -n{node_percent} -e{edge_percent} -s -f pstats {self.path_stats} | " + f"dot -Tpng -o {path}") + + # Opening the web interface to view the detailed profiler stats. + if open_web: + subprocess.getstatusoutput(f"snakeviz {self.path_stats}") diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/utilities/random.py b/paper_experiments/large_bayesian_networks_experiments/golem/utilities/random.py new file mode 100644 index 0000000..c55f6dc --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/utilities/random.py @@ -0,0 +1,36 @@ +import random +from typing import Optional + +import numpy as np + +from golem.core.log import default_log + + +class RandomStateHandler: + MODEL_FITTING_SEED = 0 + + @staticmethod + def log_random_state(): + py_state = random.getstate() + np_state = np.random.get_state() + log = default_log(RandomStateHandler.__class__.__name__) + log.debug(f'Random State: random.getstate() follows...\n{py_state}') + log.debug(f'Random State: numpy.random.get_state() follows...\n{np_state}') + + def __init__(self, seed: Optional[int] = None): + if seed is None: + seed = RandomStateHandler.MODEL_FITTING_SEED + self._seed = seed + self._old_seed = None + + def __enter__(self): + self._old_np_state = np.random.get_state() + self._old_state = random.getstate() + + np.random.seed(self._seed) + random.seed(self._seed) + return self._seed + + def __exit__(self, exc_type, exc_value, exc_traceback): + np.random.set_state(self._old_np_state) + random.setstate(self._old_state) diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/utilities/requirements_notificator.py b/paper_experiments/large_bayesian_networks_experiments/golem/utilities/requirements_notificator.py new file mode 100644 index 0000000..fc3be6c --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/utilities/requirements_notificator.py @@ -0,0 +1,14 @@ +from golem.core.log import default_log + + +def warn_requirement(name: str, default_install_path: str, *, should_raise: bool = False): + """ + :param name: module name failed to load + :default_install_path: path to requirements than need to be installed + :param should_raise: bool indicating if ImportError should be raised + """ + msg = f'"{name}" is not installed, use "pip install {default_install_path}" to fulfil requirement' + if should_raise: + raise ImportError(msg) + else: + default_log(prefix='Requirements').debug(f'{msg} or ignore this warning') diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/utilities/sequence_iterator.py b/paper_experiments/large_bayesian_networks_experiments/golem/utilities/sequence_iterator.py new file mode 100644 index 0000000..188b89b --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/utilities/sequence_iterator.py @@ -0,0 +1,91 @@ +from typing import Callable, Optional + +from golem.utilities.data_structures import BidirectionalIterator + + +class SequenceIterator(BidirectionalIterator[int]): + """ + The value of this iterator changes according to specified sequence. Iterator is used in parameter-free evolutionary + scheme for population size control. + :param sequence_func: the function for sequence generation + :param start_value: start value of sequence (if start_value doesn't match any value in the sequence, then iterator + find closest value in sequence) + :param max_sequence_value: maximal value in sequence + :param min_sequence_value: minimal value in sequence + """ + + def __init__(self, sequence_func: Callable, + start_value: int = 0, + max_sequence_value: Optional[int] = None, + min_sequence_value: Optional[int] = None): + self.sequence_func = sequence_func + self.archive = {} + self.index = self.get_sequence_index(start_value) - 1 if start_value is not None else - 1 + self.max_sequence_value = max_sequence_value + self.min_sequence_value = min_sequence_value + + def has_prev(self) -> bool: + if self.index > 0: + if self.min_sequence_value is not None: + has = self.sequence_item_calculation(self.index - 1) >= self.min_sequence_value + else: + has = True + else: + has = False + return has + + def has_next(self) -> bool: + if self.max_sequence_value is not None: + has = self.max_sequence_value >= self.sequence_item_calculation(self.index + 1) + else: + has = True + return has + + def sequence_item_calculation(self, index: int = None): + index = self.index if index is None else index + if index not in list(self.archive): + result = self.sequence_func(index) + self.archive[index] = result + else: + result = self.archive[index] + return result + + def get_sequence_index(self, value: int) -> int: + number = 0 + sequence_value = self.sequence_item_calculation(number) + while sequence_value < value: + number += 1 + sequence_value = self.sequence_item_calculation(number) + return number + + def next(self): + try: + self.index += 1 + if self.min_sequence_value is not None: + if self.sequence_item_calculation(self.index) < self.min_sequence_value: + self.index = self.get_sequence_index(self.min_sequence_value) + result = self.sequence_item_calculation() + except IndexError: + raise StopIteration() + return result + + def prev(self): + self.index -= 1 + if self.index < 0: + raise StopIteration() + return self.sequence_item_calculation() + + def current(self): + """ Get current value pointed to by the iterator """ + return self.sequence_item_calculation(self.index) + + def __iter__(self): + return self + + +def fibonacci_sequence(n: int) -> int: + a = 0 + b = 1 + for __ in range(n): + a, b = b, a + b + return a diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/utilities/serializable.py b/paper_experiments/large_bayesian_networks_experiments/golem/utilities/serializable.py new file mode 100644 index 0000000..d01df70 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/utilities/serializable.py @@ -0,0 +1,38 @@ +from abc import ABC, abstractmethod + +from typing import Tuple, Union, Optional + + +class Serializable(ABC): + + @classmethod + def from_serialized(cls, source: Union[str, dict], internal_state_data: Optional[dict] = None): + """ + Static constructor for convenience. Creates default instance and calls .load() on it. + """ + default_instance = cls() + default_instance.load(source, internal_state_data) + return default_instance + + @abstractmethod + def save(self, path: str = None, datetime_in_path: bool = True) -> Tuple[str, dict]: + """ + Save the graph to a json representation + with pickled custom data (e.g. fitted models). + + :param path: path to json file with operation + :param datetime_in_path: flag for addition of the datetime stamp to saving path + :return: json containing a composite operation description + """ + raise NotImplementedError() + + @abstractmethod + def load(self, source: Union[str, dict], internal_state_data: Optional[dict] = None): + """ + Load the graph from json representation, optionally + with pickled custom internal data (e.g. fitted models). + + :param source: path to json file with operation or json dictionary itself + :param internal_state_data: dictionary of the internal state + """ + raise NotImplementedError() diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/utilities/singleton_meta.py b/paper_experiments/large_bayesian_networks_experiments/golem/utilities/singleton_meta.py new file mode 100644 index 0000000..f49082b --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/utilities/singleton_meta.py @@ -0,0 +1,20 @@ +from threading import RLock + + +class SingletonMeta(type): + """ + This meta class can provide other classes with the Singleton pattern. + It guarantees to create one and only class instance. + Pass it to the metaclass parameter when defining your class as follows: + + class YourClassName(metaclass=SingletonMeta) + """ + _instances = {} + _lock: RLock = RLock() # TODO: seems like it's useless in multiprocessing, but that's even from threading lib?! + + def __call__(cls, *args, **kwargs): + with cls._lock: + if cls not in cls._instances: + instance = super().__call__(*args, **kwargs) + cls._instances[cls] = instance + return cls._instances[cls] diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/utilities/utilities.py b/paper_experiments/large_bayesian_networks_experiments/golem/utilities/utilities.py new file mode 100644 index 0000000..2387bb8 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/utilities/utilities.py @@ -0,0 +1,36 @@ +from typing import Optional + +import numpy as np +from joblib import cpu_count + +import random +from golem.utilities.random import RandomStateHandler + + +def determine_n_jobs(n_jobs=-1, logger=None): + cpu_num = cpu_count() + if n_jobs > cpu_num: + n_jobs = cpu_num + elif n_jobs <= 0: + if n_jobs <= -cpu_num - 1 or n_jobs == 0: + raise ValueError(f"Unproper `n_jobs` = {n_jobs}. " + f"`n_jobs` should be between ({-cpu_num}, {cpu_num}) except 0") + n_jobs = cpu_num + 1 + n_jobs + if logger: + logger.info(f"Number of used CPU's: {n_jobs}") + return n_jobs + + +def urandom_mock(n): + # os.random is the source of random used in the uuid library + # normally, it's „true“ random, but to stabilize tests, + # seeded `random` library is used instead. + return bytes(random.getrandbits(8) for _ in range(n)) + + +def set_random_seed(seed: Optional[int]): + """ Sets random seed for evaluation of models. """ + if seed is not None: + np.random.seed(seed) + random.seed(seed) + RandomStateHandler.MODEL_FITTING_SEED = seed diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/visualisation/__init__.py b/paper_experiments/large_bayesian_networks_experiments/golem/visualisation/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/visualisation/graph_viz.py b/paper_experiments/large_bayesian_networks_experiments/golem/visualisation/graph_viz.py new file mode 100644 index 0000000..09517a8 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/visualisation/graph_viz.py @@ -0,0 +1,516 @@ +from __future__ import annotations + +import datetime +import os +from copy import deepcopy +from pathlib import Path +from textwrap import wrap +from typing import Any, Callable, Dict, Iterable, Literal, Optional, Sequence, TYPE_CHECKING, Tuple, Union +from uuid import uuid4 + +import networkx as nx +import numpy as np +from matplotlib import pyplot as plt +from matplotlib.colors import to_hex +from matplotlib.patches import ArrowStyle +from pyvis.network import Network +from seaborn import color_palette + +from golem.core.dag.convert import graph_structure_as_nx_graph +from golem.core.dag.graph_utils import distance_to_primary_level +from golem.core.log import default_log +from golem.core.paths import default_data_dir + +if TYPE_CHECKING: + from golem.core.dag.graph import Graph + from golem.core.optimisers.graph import OptGraph + from golem.core.dag.graph_node import GraphNode + + GraphType = Union[Graph, OptGraph] + GraphConvertType = Callable[[GraphType], Tuple[nx.DiGraph, Dict[uuid4, GraphNode]]] + +PathType = Union[os.PathLike, str] + +MatplotlibColorType = Union[str, Sequence[float]] +LabelsColorMapType = Dict[str, MatplotlibColorType] +NodeColorFunctionType = Callable[[Iterable[str]], LabelsColorMapType] +NodeColorType = Union[MatplotlibColorType, LabelsColorMapType, NodeColorFunctionType] + + +class GraphVisualizer: + def __init__(self, graph: GraphType, visuals_params: Optional[Dict[str, Any]] = None, + to_nx_convert_func: GraphConvertType = graph_structure_as_nx_graph): + visuals_params = visuals_params or {} + default_visuals_params = dict( + engine='matplotlib', + dpi=100, + node_color=self._get_colors_by_labels, + node_size_scale=1.0, + font_size_scale=1.0, + edge_curvature_scale=1.0, + node_names_placement='auto', + nodes_layout_function=GraphVisualizer._get_hierarchy_pos_by_distance_to_primary_level, + figure_size=(7, 7), + save_path=None, + ) + default_visuals_params.update(visuals_params) + self.visuals_params = default_visuals_params + self.to_nx_convert_func = to_nx_convert_func + self._update_graph(graph) + self.log = default_log(self) + + def _update_graph(self, graph: GraphType): + self.graph = graph + self.nx_graph, self.nodes_dict = self.to_nx_convert_func(self.graph) + + def visualise(self, save_path: Optional[PathType] = None, engine: Optional[str] = None, + node_color: Optional[NodeColorType] = None, dpi: Optional[int] = None, + node_size_scale: Optional[float] = None, font_size_scale: Optional[float] = None, + edge_curvature_scale: Optional[float] = None, figure_size: Optional[Tuple[int, int]] = None, + nodes_labels: Dict[int, str] = None, edges_labels: Dict[int, str] = None, + node_names_placement: Optional[Literal['auto', 'nodes', 'legend', 'none']] = None, + nodes_layout_function: Optional[Callable[[nx.DiGraph], Dict[Any, Tuple[float, float]]]] = None, + title: Optional[str] = None): + engine = engine or self._get_predefined_value('engine') + + if not self.graph.nodes: + raise ValueError('Empty graph can not be visualized.') + + if engine == 'matplotlib': + self._draw_with_networkx(save_path=save_path, node_color=node_color, dpi=dpi, + node_size_scale=node_size_scale, font_size_scale=font_size_scale, + edge_curvature_scale=edge_curvature_scale, figure_size=figure_size, + title=title, nodes_labels=nodes_labels, edges_labels=edges_labels, + nodes_layout_function=nodes_layout_function, + node_names_placement=node_names_placement) + elif engine == 'pyvis': + self._draw_with_pyvis(save_path, node_color) + elif engine == 'graphviz': + self._draw_with_graphviz(save_path, node_color, dpi) + else: + raise NotImplementedError(f'Unexpected visualization engine: {engine}. ' + 'Possible values: matplotlib, pyvis, graphviz.') + + def draw_nx_dag( + self, ax: Optional[plt.Axes] = None, node_color: Optional[NodeColorType] = None, + node_size_scale: Optional[float] = None, font_size_scale: Optional[float] = None, + edge_curvature_scale: Optional[float] = None, nodes_labels: Dict[int, str] = None, + edges_labels: Dict[int, str] = None, + nodes_layout_function: Optional[Callable[[nx.DiGraph], Dict[Any, Tuple[float, float]]]] = None, + node_names_placement: Optional[Literal['auto', 'nodes', 'legend', 'none']] = None): + node_color = node_color or self._get_predefined_value('node_color') + node_size_scale = node_size_scale or self._get_predefined_value('node_size_scale') + font_size_scale = font_size_scale or self._get_predefined_value('font_size_scale') + edge_curvature_scale = (edge_curvature_scale if edge_curvature_scale is not None + else self._get_predefined_value('edge_curvature_scale')) + nodes_layout_function = nodes_layout_function or self._get_predefined_value('nodes_layout_function') + node_names_placement = node_names_placement or self._get_predefined_value('node_names_placement') + + nx_graph, nodes = self.nx_graph, self.nodes_dict + + if ax is None: + ax = plt.gca() + + # Define colors + if callable(node_color): + node_color = node_color([str(node) for node in nodes.values()]) + if isinstance(node_color, dict): + node_color = [node_color.get(str(node), node_color.get(None)) for node in nodes.values()] + else: + node_color = [node_color for _ in nodes] + # Get node positions + if nodes_layout_function == GraphVisualizer._get_hierarchy_pos_by_distance_to_primary_level: + pos = nodes_layout_function(nx_graph, nodes) + else: + pos = nodes_layout_function(nx_graph) + + node_size = self._get_scaled_node_size(len(nodes), node_size_scale) + + with_node_names = node_names_placement != 'none' + + if node_names_placement in ('auto', 'none'): + node_names_placement = GraphVisualizer._define_node_names_placement(node_size) + + if node_names_placement == 'nodes': + self._draw_nx_big_nodes(ax, pos, nodes, node_color, node_size, font_size_scale, with_node_names) + elif node_names_placement == 'legend': + self._draw_nx_small_nodes(ax, pos, nodes, node_color, node_size, font_size_scale, with_node_names) + self._draw_nx_curved_edges(ax, pos, node_size, edge_curvature_scale) + self._draw_nx_labels(ax, pos, font_size_scale, nodes_labels, edges_labels) + + def _get_predefined_value(self, param: str): + if param not in self.visuals_params: + self.log.warning(f'No default param found: {param}.') + return self.visuals_params.get(param) + + def _draw_with_networkx( + self, save_path: Optional[PathType] = None, node_color: Optional[NodeColorType] = None, + dpi: Optional[int] = None, node_size_scale: Optional[float] = None, + font_size_scale: Optional[float] = None, edge_curvature_scale: Optional[float] = None, + figure_size: Optional[Tuple[int, int]] = None, title: Optional[str] = None, + nodes_labels: Dict[int, str] = None, edges_labels: Dict[int, str] = None, + nodes_layout_function: Optional[Callable[[nx.DiGraph], Dict[Any, Tuple[float, float]]]] = None, + node_names_placement: Optional[Literal['auto', 'nodes', 'legend', 'none']] = None): + save_path = save_path or self._get_predefined_value('save_path') + node_color = node_color or self._get_predefined_value('node_color') + dpi = dpi or self._get_predefined_value('dpi') + figure_size = figure_size or self._get_predefined_value('figure_size') + + ax = GraphVisualizer._setup_matplotlib_figure(figure_size, dpi, title) + self.draw_nx_dag(ax, node_color, node_size_scale, font_size_scale, edge_curvature_scale, + nodes_labels, edges_labels, nodes_layout_function, node_names_placement) + GraphVisualizer._rescale_matplotlib_figure(ax) + if not save_path: + plt.show() + else: + plt.savefig(save_path, dpi=dpi) + plt.close() + + def _draw_with_pyvis(self, save_path: Optional[PathType] = None, node_color: Optional[NodeColorType] = None): + save_path = save_path or self._get_predefined_value('save_path') + node_color = node_color or self._get_predefined_value('node_color') + + net = Network('500px', '1000px', directed=True) + nx_graph, nodes = self.nx_graph, self.nodes_dict + node_color = self._define_colors(node_color, nodes) + for n, data in nx_graph.nodes(data=True): + operation = nodes[n] + label = str(operation) + data['n_id'] = str(n) + data['label'] = label.replace('_', ' ') + params = operation.content.get('params') + if isinstance(params, dict): + params = str(params)[1:-1] + data['title'] = params + data['level'] = distance_to_primary_level(operation) + data['color'] = to_hex(node_color.get(label, node_color.get(None))) + data['font'] = '20px' + data['labelHighlightBold'] = True + + for _, data in nx_graph.nodes(data=True): + net.add_node(**data) + for u, v in nx_graph.edges: + net.add_edge(str(u), str(v)) + + if save_path: + net.save_graph(str(save_path)) + return + save_path = Path(default_data_dir(), 'graph_plots', str(uuid4()) + '.html') + save_path.parent.mkdir(exist_ok=True) + net.show(str(save_path)) + remove_old_files_from_dir(save_path.parent) + + def _draw_with_graphviz(self, save_path: Optional[PathType] = None, node_color: Optional[NodeColorType] = None, + dpi: Optional[int] = None): + save_path = save_path or self._get_predefined_value('save_path') + node_color = node_color or self._get_predefined_value('node_color') + dpi = dpi or self._get_predefined_value('dpi') + + nx_graph, nodes = self.nx_graph, self.nodes_dict + node_color = self._define_colors(node_color, nodes) + for n, data in nx_graph.nodes(data=True): + label = str(nodes[n]) + data['label'] = label.replace('_', ' ') + data['color'] = to_hex(node_color.get(label, node_color.get(None))) + + gv_graph = nx.nx_agraph.to_agraph(nx_graph) + kwargs = {'prog': 'dot', 'args': f'-Gnodesep=0.5 -Gdpi={dpi} -Grankdir="LR"'} + + if save_path: + gv_graph.draw(save_path, **kwargs) + else: + save_path = Path(default_data_dir(), 'graph_plots', str(uuid4()) + '.png') + save_path.parent.mkdir(exist_ok=True) + gv_graph.draw(save_path, **kwargs) + + img = plt.imread(str(save_path)) + plt.imshow(img) + plt.gca().axis('off') + plt.gcf().set_dpi(dpi) + plt.tight_layout() + plt.show() + remove_old_files_from_dir(save_path.parent) + + @staticmethod + def _get_scaled_node_size(nodes_amount: int, size_scale: float) -> float: + min_size = 150 + max_size = 12000 + size = max(max_size * (1 - np.log10(nodes_amount)), min_size) + return size * size_scale + + @staticmethod + def _get_scaled_font_size(nodes_amount: int, size_scale: float) -> float: + min_size = 14 + max_size = 30 + size = max(max_size * (1 - np.log10(nodes_amount)), min_size) + return size * size_scale + + @staticmethod + def _get_colors_by_labels(labels: Iterable[str]) -> LabelsColorMapType: + unique_labels = list(set(labels)) + palette = color_palette('tab10', len(unique_labels)) + return {label: palette[unique_labels.index(label)] for label in labels} + + @staticmethod + def _define_colors(node_color, nodes): + if callable(node_color): + colors = node_color([str(node) for node in nodes.values()]) + elif isinstance(node_color, dict): + colors = node_color + else: + colors = {str(node): node_color for node in nodes.values()} + return colors + + @staticmethod + def _setup_matplotlib_figure(figure_size: Tuple[float, float], dpi: int, title: Optional[str] = None) -> plt.Axes: + fig, ax = plt.subplots(figsize=figure_size) + fig.set_dpi(dpi) + plt.title(title) + return ax + + @staticmethod + def _rescale_matplotlib_figure(ax): + """Rescale the figure for all nodes to fit in.""" + + x_1, x_2 = ax.get_xlim() + y_1, y_2 = ax.get_ylim() + offset = 0.2 + x_offset = x_2 * offset + y_offset = y_2 * offset + ax.set_xlim(x_1 - x_offset, x_2 + x_offset) + ax.set_ylim(y_1 - y_offset, y_2 + y_offset) + ax.axis('off') + plt.tight_layout() + + def _draw_nx_big_nodes(self, ax, pos, nodes, node_color, node_size, font_size_scale, with_node_names): + # Draw the graph's nodes. + nx.draw_networkx_nodes(self.nx_graph, pos, node_size=node_size, ax=ax, node_color='w', linewidths=3, + edgecolors=node_color) + if not with_node_names: + return + # Draw the graph's node labels. + node_labels = {node_id: str(node) for node_id, node in nodes.items()} + font_size = GraphVisualizer._get_scaled_font_size(len(nodes), font_size_scale) + for node, (x, y) in pos.items(): + text = '\n'.join(wrap(node_labels[node].replace('_', ' ').replace('-', ' '), 10)) + ax.text(x, y, text, + ha='center', va='center', + fontsize=font_size, + bbox=dict(alpha=0.9, color='w', boxstyle='round')) + + def _draw_nx_small_nodes(self, ax, pos, nodes, node_color, node_size, font_size_scale, with_node_names): + nx_graph = self.nx_graph + markers = 'os^>v len(markers) - 1: + self.log.warning(f'Too much node labels derive the same color: {color}. The markers may repeat.\n' + '\tSpecify the parameter "node_color" to set distinct colors.') + color_count = color_count % len(markers) + marker = markers[color_count] + label_markers[label] = marker + color_counts[color] = color_count + 1 + nx.draw_networkx_nodes(nx_graph, pos, [node_id], ax=ax, node_color=[color], node_size=node_size, + node_shape=marker) + if label in labels_added: + continue + ax.plot([], [], marker=marker, linestyle='None', color=color, label=label) + labels_added.add(label) + if not with_node_names: + return + # @morrisnein took the following code from https://stackoverflow.com/a/27512450 + handles, labels = ax.get_legend_handles_labels() + # Sort both labels and handles by labels + labels, handles = zip(*sorted(zip(labels, handles), key=lambda t: t[0])) + ax.legend(handles, labels, prop={'size': round(20 * font_size_scale)}) + + def _draw_nx_curved_edges(self, ax, pos, node_size, edge_curvature_scale): + nx_graph = self.nx_graph + # The ongoing section defines curvature for all edges. + # This is 'connection style' for an edge that does not intersect any nodes. + connection_style = 'arc3' + # This is 'connection style' template for an edge that is too close to any node and must bend around it. + # The curvature value is defined individually for each edge. + connection_style_curved_template = connection_style + ',rad={}' + default_edge_curvature = 0.3 + # The minimum distance from a node to an edge on which the edge must bend around the node. + node_distance_gap = 0.15 + for u, v, e in nx_graph.edges(data=True): + e['connectionstyle'] = connection_style + p_1, p_2 = np.array(pos[u]), np.array(pos[v]) + p_1_2 = p_2 - p_1 + p_1_2_length = np.linalg.norm(p_1_2) + # Finding the closest node to the edge. + min_distance_found = node_distance_gap * 2 # It just must be bigger than the gap. + closest_node_id = None + for node_id in nx_graph.nodes: + if node_id in (u, v): + continue # The node is adjacent to the edge. + p_3 = np.array(pos[node_id]) + distance_to_node = abs(np.cross(p_1_2, p_3 - p_1)) / p_1_2_length + if (distance_to_node > min(node_distance_gap, min_distance_found) # The node is too far. + or ((p_3 - p_1) @ p_1_2) < 0 # There's no perpendicular from the node to the edge. + or ((p_3 - p_2) @ -p_1_2) < 0): + continue + min_distance_found = distance_to_node + closest_node_id = node_id + + if closest_node_id is None: + continue # There's no node to bend around. + # Finally, define the edge's curvature based on the closest node position. + p_3 = np.array(pos[closest_node_id]) + p_1_3 = p_3 - p_1 + curvature_strength = default_edge_curvature * edge_curvature_scale + # 'alpha' denotes the angle between the abscissa and the edge. + cos_alpha = p_1_2[0] / p_1_2_length + sin_alpha = np.sqrt(1 - cos_alpha ** 2) * (-1) ** (p_1_2[1] < 0) + # The closest node is placed as if the edge matched the abscissa. + # Then, its ordinate shows on which side of the edge it is, "on the left" or "on the right". + rotation_matrix = np.array([[cos_alpha, sin_alpha], [-sin_alpha, cos_alpha]]) + p_1_3_rotated = rotation_matrix @ p_1_3 + curvature_direction = (-1) ** (p_1_3_rotated[1] < 0) # +1 is a "cup" \/, -1 is a "cat" /\. + edge_curvature = curvature_direction * curvature_strength + e['connectionstyle'] = connection_style_curved_template.format(edge_curvature) + # Define edge center position for labels. + edge_center_position = np.mean([p_1, p_2], axis=0) + edge_curvature_shift = np.linalg.inv(rotation_matrix) @ [0, -1 * edge_curvature / 4] + edge_center_position += edge_curvature_shift + e['edge_center_position'] = edge_center_position + # Draw the graph's edges. + arrow_style = ArrowStyle('Simple', head_length=1.5, head_width=0.8) + for u, v, e in nx_graph.edges(data=True): + nx.draw_networkx_edges(nx_graph, pos, edgelist=[(u, v)], node_size=node_size, ax=ax, arrowsize=10, + arrowstyle=arrow_style, connectionstyle=e['connectionstyle']) + self._rescale_matplotlib_figure(ax) + + def _draw_nx_labels(self, ax: plt.Axes, pos: Any, font_size_scale: float, + nodes_labels: Dict[int, str], edges_labels: Dict[int, str]): + """ Set labels with scores to nodes and edges. """ + + def calculate_labels_bias(ax: plt.Axes, y_span: int): + y_1, y_2 = ax.get_ylim() + y_size = y_2 - y_1 + if y_span == 1: + bias_scale = 0.25 # Fits between the central line and the upper bound. + else: + bias_scale = 1 / y_span / 3 * 0.5 # Fits between the narrowest horizontal rows. + bias = y_size * bias_scale + return bias + + def match_labels_with_nx_nodes(nx_graph: nx.DiGraph, labels: Dict[int, str]) -> Dict[str, str]: + """ Matches index of node in GOLEM graph with networkx node name. """ + nx_nodes = list(nx_graph.nodes.keys()) + nx_labels = {} + for index, label in labels.items(): + nx_labels[nx_nodes[index]] = label + return nx_labels + + def match_labels_with_nx_edges(nx_graph: nx.DiGraph, labels: Dict[int, str]) \ + -> Dict[Tuple[str, str], str]: + """ Matches index of edge in GOLEM graph with tuple of networkx nodes names. """ + nx_nodes = list(nx_graph.nodes.keys()) + edges = self.graph.get_edges() + nx_labels = {} + for index, label in labels.items(): + edge = edges[index] + parent_node_nx = nx_nodes[self.graph.nodes.index(edge[0])] + child_node_nx = nx_nodes[self.graph.nodes.index(edge[1])] + nx_labels[(parent_node_nx, child_node_nx)] = label + return nx_labels + + def draw_node_labels(node_labels, ax, bias, font_size, nx_graph, pos): + labels_pos = deepcopy(pos) + for value in labels_pos.values(): + value[1] += bias + bbox = dict(alpha=0.9, color='w') + + nodes_nx_labels = match_labels_with_nx_nodes(nx_graph=nx_graph, labels=node_labels) + nx.draw_networkx_labels( + nx_graph, labels_pos, + ax=ax, + labels=nodes_nx_labels, + font_color='black', + font_size=font_size, + bbox=bbox + ) + + def draw_edge_labels(edge_labels, ax, bias, font_size, nx_graph, pos): + labels_pos_edges = deepcopy(pos) + label_bias_y = 2 / 3 * bias + if len(set([coord[1] for coord in pos.values()])) == 1 and len(list(pos.values())) > 2: + for value in labels_pos_edges.values(): + value[1] += label_bias_y + edges_nx_labels = match_labels_with_nx_edges(nx_graph=nx_graph, labels=edge_labels) + bbox = dict(alpha=0.9, color='w') + # Set labels for edges + for u, v, e in nx_graph.edges(data=True): + if (u, v) not in edges_nx_labels: + continue + current_pos = labels_pos_edges + if 'edge_center_position' in e: + x, y = e['edge_center_position'] + plt.text(x, y, edges_nx_labels[(u, v)], bbox=bbox, fontsize=font_size) + else: + nx.draw_networkx_edge_labels( + nx_graph, current_pos, {(u, v): edges_nx_labels[(u, v)]}, + label_pos=0.5, ax=ax, + font_color='black', + font_size=font_size, + rotate=False, + bbox=bbox + ) + + if not (edges_labels or nodes_labels): + return + + nodes_amount = len(pos) + font_size = GraphVisualizer._get_scaled_font_size(nodes_amount, font_size_scale * 0.75) + _, y_span = GraphVisualizer._get_x_y_span(pos) + bias = calculate_labels_bias(ax, y_span) + + if nodes_labels: + draw_node_labels(nodes_labels, ax, bias, font_size, self.nx_graph, pos) + + if edges_labels: + draw_edge_labels(edges_labels, ax, bias, font_size, self.nx_graph, pos) + + @staticmethod + def _get_hierarchy_pos_by_distance_to_primary_level(nx_graph: nx.DiGraph, nodes: Dict + ) -> Dict[Any, Tuple[float, float]]: + """By default, returns 'networkx.multipartite_layout' positions based on 'hierarchy_level` + from node data - the property must be set beforehand. + :param graph: the graph. + """ + for node_id, node_data in nx_graph.nodes(data=True): + node_data['hierarchy_level'] = distance_to_primary_level(nodes[node_id]) + + return nx.multipartite_layout(nx_graph, subset_key='hierarchy_level') + + @staticmethod + def _get_x_y_span(pos: Dict[Any, Tuple[float, float]]) -> Tuple[int, int]: + pos_x, pos_y = np.split(np.array(tuple(pos.values())), 2, axis=1) + x_span = max(pos_x) - min(pos_x) + y_span = max(pos_y) - min(pos_y) + return x_span, y_span + + @staticmethod + def _define_node_names_placement(node_size): + if node_size >= 1000: + node_names_placement = 'nodes' + else: + node_names_placement = 'legend' + return node_names_placement + + +def remove_old_files_from_dir(dir_: Path, time_interval=datetime.timedelta(minutes=10)): + for path in dir_.iterdir(): + if datetime.datetime.now() - datetime.datetime.fromtimestamp(path.stat().st_ctime) > time_interval: + path.unlink() diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/__init__.py b/paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/arg_constraint_wrapper.py b/paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/arg_constraint_wrapper.py new file mode 100644 index 0000000..553e42b --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/arg_constraint_wrapper.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +import inspect + +from typing import TYPE_CHECKING, Any, Callable, Dict, List + +if TYPE_CHECKING: + from golem.visualisation.opt_history.history_visualization import HistoryVisualization + +ArgConstraintChecker = Callable[..., Dict[str, Any]] + + +def per_time(visualization: HistoryVisualization, **kwargs) -> Dict[str, Any]: + value = kwargs.get('per_time') + if value and visualization.history.generations[0][0].metadata.get('evaluation_time_iso') is None: + visualization.log.warning('Evaluation time not found in optimization history. ' + 'Showing fitness line per generations...') + kwargs['per_time'] = False + return kwargs + + +def best_fraction(visualization: HistoryVisualization, **kwargs) -> Dict[str, Any]: + value = kwargs.get('best_fraction') + if value is not None and not 0 < value <= 1: + raise ValueError('`best_fraction` argument should be in the interval (0, 1].') + return kwargs + + +class ArgConstraintWrapper(type): + """ Metaclass that wraps '.visualize' method into a series of argument constraint checkers. + + The behavior of the program due to invalid keyword argument values is entirely up to the checkers. + Constraint checkers must return `kwargs` that may be modified during the check. + + The `DEFAULT_CONSTRAINTS` attribute is a mapping of argument names to their default constraint functions + that will apply to any visualizations with this argument. + + Note that the checkers have all the `kwargs` of `visualize()` fully available, including the default + ones, as well as the HistoryVisualization instance itself. + """ + DEFAULT_CONSTRAINTS = { + 'best_fraction': best_fraction, + 'per_time': per_time + } + + @staticmethod + def wrap_constraints(constraint_checkers: List[ArgConstraintChecker]): + def decorator(visualize_function): + """Return a wrapped instance method""" + + def outer(visualization, *args, **kwargs): + visualization_parameters = inspect.signature(visualize_function).parameters + # Turn args into kwargs. + parameter_names = list(visualization_parameters.keys())[1:] # Skip `self`. + args = {parameter_names[arg_num]: arg_value for arg_num, arg_value in enumerate(args)} + kwargs.update(args) + # Get default values and add them to the kwargs. + default_kwargs = {p_name: p.default for p_name, p in visualization_parameters.items() + if p.default is not p.empty} + default_kwargs.update(kwargs) + kwargs = default_kwargs + # Filter wrong kwargs with warning. + kwargs_to_ignore = [] + for argument in kwargs.keys(): + if argument not in visualization_parameters: + visualization.log.warning( + f'Argument `{argument}` is not supported for "{visualization.__class__.__name__}". ' + f'It is ignored.') + kwargs_to_ignore.append(argument) + kwargs = {key: value for key, value in kwargs.items() if key not in kwargs_to_ignore} + # Apply constraint_checkers iteratively. + for checker in constraint_checkers: + kwargs = checker(visualization, **kwargs) + # Make a visualization. + visualization.log.info( + 'Visualizing optimization history... It may take some time, depending on the history size.') + return_value = visualize_function(visualization, **kwargs) + return return_value + + return outer + + return decorator + + def __new__(mcs, name, bases, attrs): + """If the class has a 'visualize' method, wrap it""" + constraint_checkers = [] + if 'visualize' in attrs: + parameters = inspect.signature(attrs['visualize']).parameters + for kwarg, constraint_checker in mcs.DEFAULT_CONSTRAINTS.items(): + if kwarg in parameters: + constraint_checkers.append(constraint_checker) + if 'constraint_checkers' in attrs: + # Class-defined checkers + for constraint_checker in attrs['constraint_checkers']: + constraint_checkers.append(constraint_checker) + # Pass the checkers (or empty list) to the wrapper anyway. + attrs['visualize'] = mcs.wrap_constraints(constraint_checkers)(attrs['visualize']) + + return super(ArgConstraintWrapper, mcs).__new__(mcs, name, bases, attrs) diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/diversity.py b/paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/diversity.py new file mode 100644 index 0000000..8e3bdd2 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/diversity.py @@ -0,0 +1,135 @@ +import os +from typing import Optional, Union, TYPE_CHECKING + +import numpy as np +from matplotlib import pyplot as plt +from matplotlib.animation import FuncAnimation + +from golem.core.optimisers.genetic.operators.operator import PopulationT +from golem.visualisation.opt_history.history_visualization import HistoryVisualization +from golem.visualisation.opt_history.utils import show_or_save_figure + +if TYPE_CHECKING: + from golem.core.optimisers.opt_history_objects.opt_history import OptHistory + + +class DiversityLine(HistoryVisualization): + def visualize(self, show: bool = True, save_path: Optional[Union[os.PathLike, str]] = None): + """Plots double graphic to estimate diversity of populations during optimization. + It plots standard deviation of all metrics (y-axis) by generation number (x-axis). + Additional line show ratio of structurally unique individuals.""" + return plot_diversity_dynamic(self.history, show=show, save_path=save_path, dpi=self.visuals_params['dpi']) + + +class DiversityPopulation(HistoryVisualization): + def visualize(self, save_path: Union[os.PathLike, str], fps: int = 4): + """Creates a GIF with violin-plot estimating distribution of each metric in populations. + Each frame shows distribution for a particular generation.""" + return plot_diversity_dynamic_gif(self.history, filename=save_path, fps=fps, dpi=self.visuals_params['dpi']) + + +def compute_fitness_diversity(population: PopulationT) -> np.ndarray: + """Returns numpy array of standard deviations of fitness values.""" + # substitutes None values + fitness_values = np.array([ind.fitness.values for ind in population], dtype=float) + # compute std along each axis while ignoring nan-s + diversity = np.nanstd(fitness_values, axis=0) + return diversity + + +def plot_diversity_dynamic_gif(history: 'OptHistory', + filename: Optional[str] = None, + fig_size: int = 5, + fps: int = 4, + dpi: int = 100, + ) -> FuncAnimation: + metric_names = history.objective.metric_names + # dtype=float removes None, puts np.nan + # indexed by [population, metric, individual] after transpose (.T) + pops = history.generations[1:-1] # ignore initial pop and final choices + fitness_distrib = [np.array([ind.fitness.values for ind in pop], dtype=float).T + for pop in pops] + + # Define bounds on metrics: find min & max on a flattened view of array + q = 0.025 + lims_max = np.max([np.quantile(pop, 1 - q, axis=1) for pop in fitness_distrib], axis=0) + lims_min = np.min([np.quantile(pop, q, axis=1) for pop in fitness_distrib], axis=0) + + # Setup the plot + ncols = max(len(metric_names), 1) + fig, axs = plt.subplots(ncols=ncols) + fig.set_size_inches(fig_size * ncols, fig_size) + axs = np.atleast_1d(np.ravel(axs)) + + # Set update function for updating data on the axes + def update_axes(iframe: int): + for i, (ax, metric_distrib) in enumerate(zip(axs, fitness_distrib[iframe])): + # Clear & Prepare axes + ax: plt.Axes + ax.clear() + ax.set_xlim(0.5, 1.5) + ax.set_ylim(lims_min[i], lims_max[i]) + ax.set_ylabel('Metric value') + ax.grid() + # Plot information + fig.suptitle(f'Population {iframe+1} diversity by metric') + metric_name = metric_names[i] if metric_names else f"metric{i}" + ax.set_title(f'{metric_name}, ' + f'mean={np.mean(metric_distrib).round(3)}, ' + f'std={np.nanstd(metric_distrib).round(3)}') + ax.violinplot(metric_distrib, + quantiles=[0.25, 0.5, 0.75]) + + # Run this function in FuncAnimation + num_frames = len(fitness_distrib) + animate = FuncAnimation( + fig=fig, + func=update_axes, + save_count=num_frames, + interval=200, + ) + # Save the GIF from animation + if filename: + animate.save(filename, fps=fps, dpi=dpi) + return animate + + +def plot_diversity_dynamic(history: 'OptHistory', + show: bool = True, save_path: Optional[str] = None, dpi: int = 100): + labels = history.objective.metric_names + h = history.generations[:-1] # don't consider final choices + xs = np.arange(len(h)) + + # Compute diversity by metrics + np_history = np.array([compute_fitness_diversity(pop) for pop in h]) + ys = {label: np_history[:, i] for i, label in enumerate(labels)} + # Compute number of unique individuals, plot + ratio_unique = [len(set(ind.graph.descriptive_id for ind in pop)) / len(pop) for pop in h] + + fig, ax = plt.subplots() + fig.suptitle('Population diversity') + ax.set_xlabel('Generation') + ax.set_ylabel('Metrics Std') + ax.grid() + line_alpha = 0.8 + + for label, metric_std in ys.items(): + ax.plot(xs, metric_std, label=label, alpha=line_alpha) + + ax2 = ax.twinx() + ax2_color = 'm' # magenta + ax2.set_ylabel('Structural uniqueness', color=ax2_color) + ax2.set_ylim(0.25, 1.05) + ax2.tick_params(axis='y', labelcolor=ax2_color) + ax2.plot(xs, ratio_unique, label='unique ratio', + color=ax2_color, linestyle='dashed', alpha=line_alpha) + + # ask matplotlib for the plotted objects and their labels + # to put them into single legend for both axes + lines, labels = ax.get_legend_handles_labels() + lines2, labels2 = ax2.get_legend_handles_labels() + ax2.legend(lines + lines2, labels + labels2, + loc='upper left', bbox_to_anchor=(0., 1.15)) + + if show or save_path: + show_or_save_figure(fig, save_path, dpi) diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/fitness_box.py b/paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/fitness_box.py new file mode 100644 index 0000000..947935f --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/fitness_box.py @@ -0,0 +1,50 @@ +import os +from typing import Optional, Union + +import seaborn as sns +from matplotlib import pyplot as plt, ticker + +from golem.visualisation.opt_history.utils import get_history_dataframe, show_or_save_figure +from golem.visualisation.opt_history.history_visualization import HistoryVisualization + + +class FitnessBox(HistoryVisualization): + def visualize(self, save_path: Optional[Union[os.PathLike, str]] = None, + dpi: Optional[int] = None, best_fraction: Optional[float] = None): + """ Visualizes fitness values across generations in the form of boxplot. + + :param save_path: path to save the visualization. If set, then the image will be saved, and if not, + it will be displayed. + :param dpi: DPI of the output figure. + :param best_fraction: fraction of the best individuals of each generation that included in the + visualization. Must be in the interval (0, 1]. + """ + save_path = save_path or self.get_predefined_value('save_path') + dpi = dpi or self.get_predefined_value('dpi') + best_fraction = best_fraction or self.get_predefined_value('best_fraction') + + df_history = get_history_dataframe(self.history, best_fraction=best_fraction) + columns_needed = ['generation', 'individual', 'fitness'] + df_history = df_history[columns_needed].drop_duplicates(ignore_index=True) + # Get color palette by mean fitness per generation + fitness = df_history.groupby('generation')['fitness'].mean() + fitness = (fitness - min(fitness)) / (max(fitness) - min(fitness)) + colors = plt.cm.YlOrRd(fitness) + + fig, ax = plt.subplots(figsize=(6.4, 4.8), facecolor='w') + sns.boxplot(data=df_history, x='generation', y='fitness', palette=colors, ax=ax) + fig.set_dpi(dpi) + fig.set_facecolor('w') + + ax.set_title('Fitness by generations') + ax.set_xlabel('Generation') + # Set ticks for every 5 generation if there's more than 10 generations. + if len(self.history.generations) > 10: + ax.xaxis.set_major_locator(ticker.MultipleLocator(5)) + ax.xaxis.set_major_formatter(ticker.ScalarFormatter()) + ax.xaxis.grid(True) + str_fraction_of_graphs = 'all' if best_fraction is None else f'top {best_fraction * 100}% of' + ax.set_ylabel(f'Fitness of {str_fraction_of_graphs} generation graphs') + ax.yaxis.grid(True) + + show_or_save_figure(fig, save_path, dpi) diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/fitness_line.py b/paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/fitness_line.py new file mode 100644 index 0000000..822dddb --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/fitness_line.py @@ -0,0 +1,263 @@ +import functools +import os +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, List, Optional, Union, Sequence, Tuple + +import matplotlib as mpl +import numpy as np +from matplotlib import pyplot as plt +from matplotlib.widgets import Button + +from golem.core.log import default_log +from golem.core.optimisers.fitness import null_fitness +from golem.core.optimisers.opt_history_objects.individual import Individual +from golem.core.paths import default_data_dir +from golem.visualisation.opt_history.history_visualization import HistoryVisualization +from golem.visualisation.opt_history.utils import show_or_save_figure + + +def with_alternate_matplotlib_backend(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + default_mpl_backend = mpl.get_backend() + try: + mpl.use('TKAgg') + return func(*args, **kwargs) + except ImportError as e: + default_log(prefix='Requirements').warning(e) + finally: + mpl.use(default_mpl_backend) + + return wrapper + + +def setup_fitness_plot(axis: plt.Axes, xlabel: str, title: Optional[str] = None, with_legend: bool = False): + if axis is None: + fig, axis = plt.subplots() + + if with_legend: + axis.legend() + axis.set_ylabel('Fitness') + axis.set_xlabel(xlabel) + axis.set_title(title) + axis.grid(axis='y') + + +def plot_fitness_line_per_time(axis: plt.Axes, generations, label: Optional[str] = None, + with_generation_limits: bool = True) \ + -> Dict[int, Individual]: + best_fitness = null_fitness() + gen_start_times = [] + best_individuals = {} + + start_time = datetime.fromisoformat( + min(generations[0], key=lambda ind: ind.metadata['evaluation_time_iso']).metadata[ + 'evaluation_time_iso']) + end_time_seconds = (datetime.fromisoformat( + max(generations[-1], key=lambda ind: ind.metadata['evaluation_time_iso']).metadata[ + 'evaluation_time_iso']) - start_time).seconds + + for gen_num, gen in enumerate(generations): + gen_start_times.append(1e10) + gen_sorted = sorted(gen, key=lambda ind: ind.metadata['evaluation_time_iso']) + for ind in gen_sorted: + if ind.native_generation != gen_num: + continue + evaluation_time = (datetime.fromisoformat(ind.metadata['evaluation_time_iso']) - start_time).seconds + if evaluation_time < gen_start_times[gen_num]: + gen_start_times[gen_num] = evaluation_time + if ind.fitness > best_fitness: + best_individuals[evaluation_time] = ind + best_fitness = ind.fitness + + best_eval_times, best_fitnesses = np.transpose( + [(evaluation_time, abs(individual.fitness.value)) + for evaluation_time, individual in best_individuals.items()]) + + best_eval_times = list(best_eval_times) + best_fitnesses = list(best_fitnesses) + + if best_eval_times[-1] != end_time_seconds: + best_fitnesses.append(abs(best_fitness.value)) + best_eval_times.append(end_time_seconds) + gen_start_times.append(end_time_seconds) + + axis.step(best_eval_times, best_fitnesses, where='post', label=label) + + if with_generation_limits: + axis_gen = axis.twiny() + axis_gen.set_xlim(axis.get_xlim()) + axis_gen.set_xticks(gen_start_times, list(range(len(gen_start_times) - 1)) + ['']) + axis_gen.locator_params(nbins=10) + axis_gen.set_xlabel('Generation') + + gen_ticks = axis_gen.get_xticks() + prev_time = gen_ticks[0] + axis.axvline(prev_time, color='k', linestyle='--', alpha=0.3) + for i, next_time in enumerate(gen_ticks[1:]): + axis.axvline(next_time, color='k', linestyle='--', alpha=0.3) + if i % 2 == 0: + axis.axvspan(prev_time, next_time, color='k', alpha=0.05) + prev_time = next_time + + return best_individuals + + +def plot_fitness_line_per_generations(axis: plt.Axes, generations, label: Optional[str] = None) \ + -> Dict[int, Individual]: + best_fitnesses, best_generations, best_individuals = find_best_running_fitness(generations, metric_id=0) + axis.step(best_generations, best_fitnesses, where='post', label=label) + axis.set_xticks(range(len(generations))) + axis.locator_params(nbins=10) + return best_individuals + + +class FitnessLine(HistoryVisualization): + def visualize(self, save_path: Optional[Union[os.PathLike, str]] = None, dpi: Optional[int] = None, + per_time: Optional[bool] = None): + """ Visualizes the best fitness values during the evolution in the form of line. + :param save_path: path to save the visualization. If set, then the image will be saved, + and if not, it will be displayed. + :param dpi: DPI of the output figure. + :param per_time: defines whether to show time grid if it is available in history. + """ + save_path = save_path or self.get_predefined_value('save_path') + dpi = dpi or self.get_predefined_value('dpi') + per_time = per_time if per_time is not None else self.get_predefined_value('per_time') or False + + fig, ax = plt.subplots(figsize=(6.4, 4.8), facecolor='w') + if per_time: + xlabel = 'Time, s' + plot_fitness_line_per_time(ax, self.history.generations) + else: + xlabel = 'Generation' + plot_fitness_line_per_generations(ax, self.history.generations) + setup_fitness_plot(ax, xlabel) + show_or_save_figure(fig, save_path, dpi) + + +class FitnessLineInteractive(HistoryVisualization): + + @with_alternate_matplotlib_backend + def visualize(self, save_path: Optional[Union[os.PathLike, str]] = None, dpi: Optional[int] = None, + per_time: Optional[bool] = None, graph_show_kwargs: Optional[Dict[str, Any]] = None): + """ Visualizes the best fitness values during the evolution in the form of line. + Additionally, shows the structure of the best individuals and the moment of their discovering. + :param save_path: path to save the visualization. If set, then the image will be saved, and if not, + it will be displayed. + :param dpi: DPI of the output figure. + :param per_time: defines whether to show time grid if it is available in history. + :param graph_show_kwargs: keyword arguments of `graph.show()` function. + """ + + save_path = save_path or self.get_predefined_value('save_path') + dpi = dpi or self.get_predefined_value('dpi') + per_time = per_time if per_time is not None else self.get_predefined_value('per_time') or False + graph_show_kwargs = graph_show_kwargs or self.get_predefined_value('graph_show_params') or {} + + graph_show_kwargs = graph_show_kwargs or self.visuals_params.get('graph_show_params') or {} + + fig, axes = plt.subplots(1, 2, figsize=(15, 10)) + ax_fitness, ax_graph = axes + + if per_time: + x_label = 'Time, s' + x_template = 'time {} s' + plot_func = plot_fitness_line_per_time + else: + x_label = 'Generation' + x_template = 'generation {}' + plot_func = plot_fitness_line_per_generations + + best_individuals = plot_func(ax_fitness, self.history.generations) + setup_fitness_plot(ax_fitness, x_label) + + ax_graph.axis('off') + + class InteractivePlot: + temp_path = Path(default_data_dir(), 'current_graph.png') + + def __init__(self, best_individuals: Dict[int, Individual]): + self.best_x: List[int] = list(best_individuals.keys()) + self.best_individuals: List[Individual] = list(best_individuals.values()) + self.index: int = len(self.best_individuals) - 1 + self.time_line = ax_fitness.axvline(self.best_x[self.index], color='r', alpha=0.7) + self.graph_images: List[np.ndarray] = [] + self.generate_graph_images() + self.update_graph() + + def generate_graph_images(self): + for ind in self.best_individuals: + graph = ind.graph + graph.show(self.temp_path, **graph_show_kwargs) + self.graph_images.append(plt.imread(str(self.temp_path))) + self.temp_path.unlink() + + def update_graph(self): + ax_graph.imshow(self.graph_images[self.index]) + x = self.best_x[self.index] + fitness = self.best_individuals[self.index].fitness + ax_graph.set_title(f'The best graph at {x_template.format(x)}, fitness={fitness}') + + def update_time_line(self): + self.time_line.set_xdata(self.best_x[self.index]) + + def step_index(self, step: int): + self.index = (self.index + step) % len(self.best_individuals) + self.update_graph() + self.update_time_line() + plt.draw() + + def next(self, event): + self.step_index(1) + + def prev(self, event): + self.step_index(-1) + + callback = InteractivePlot(best_individuals) + + if not save_path: # display buttons only for an interactive plot + ax_prev = plt.axes([0.7, 0.05, 0.1, 0.075]) + ax_next = plt.axes([0.81, 0.05, 0.1, 0.075]) + b_next = Button(ax_next, 'Next') + b_next.on_clicked(callback.next) + b_prev = Button(ax_prev, 'Previous') + b_prev.on_clicked(callback.prev) + + show_or_save_figure(fig, save_path, dpi) + + +def find_best_running_fitness(generations: Sequence[Sequence[Individual]], + metric_id: int = 0, + ) -> Tuple[List[float], List[int], Dict[int, Individual]]: + """For each trial history per each generation find the best fitness *seen so far*. + Returns tuple: + - list of best seen metric up to that generation, + - list of indices where current best individual belongs. + - dict mapping of best index to best individuals + """ + best_metric = np.inf # Assuming metric minimization + best_individuals = {} + + # Core logic + for gen_num, gen in enumerate(generations): + for ind in gen: + if ind.native_generation != gen_num: + continue + target_metric = ind.fitness.values[metric_id] + if target_metric <= best_metric: + best_individuals[gen_num] = ind + best_metric = target_metric + + # Additional unwrapping of the data for simpler plotting + best_generations, best_metrics = np.transpose( + [(gen_num, abs(individual.fitness.values[metric_id])) + for gen_num, individual in best_individuals.items()]) + best_generations = list(best_generations) + best_metrics = list(best_metrics) + if best_generations[-1] != len(generations) - 1: + best_metrics.append(abs(best_metric)) + best_generations.append(len(generations) - 1) + + return best_metrics, best_generations, best_individuals diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/genealogical_path.py b/paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/genealogical_path.py new file mode 100644 index 0000000..1e9a422 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/genealogical_path.py @@ -0,0 +1,131 @@ +import math +import os +from functools import partial +from typing import Callable, List, Union, Optional + +from matplotlib import pyplot as plt, animation + +from golem.core.dag.graph import Graph +from golem.core.optimisers.opt_history_objects.individual import Individual +from golem.visualisation.graph_viz import GraphVisualizer +from golem.visualisation.opt_history.history_visualization import HistoryVisualization + + +class GenealogicalPath(HistoryVisualization): + def visualize(self, graph_dist: Callable[[Graph, Graph], float] = None, target_graph: Graph = None, + evolution_time_s: float = 8., hold_result_time_s: float = 2., + save_path: Optional[Union[os.PathLike, str]] = None, show: bool = False): + """ + Takes the best individual from the resultant generation and traces its genealogical path + taking the most similar parent each time (or the first parent if no similarity measure is provided). + That makes the picture more stable (and hence comprehensible) and the evolution process more apparent. + + Saves the result as a GIF with the following layout: + - target graph (if provided) is displayed on the left, + - evolving graphs go as the next subplot, they evolve from the first generation to the last, + - and the fitness plot on the right shows fitness dynamics as the graphs evolve. + + :param graph_dist: a function to measure the distance between two graphs. If not provided, all graphs are + treated as equally distant. + Works on optimization graphs, not domain graphs. If your distance metric works on domain graphs, + adapt it with `adapter.adapt_func(your_metric)`. + :param target_graph: the graph to compare the genealogical path with. Again, optimization graph is expected. + If provided, it will be displayed on the left throughout the animation. + :param save_path: path to save the visualization (won't be saved if it's None). + GIF of video extension is expected. + :param show: whether to show the visualization. + :param evolution_time_s: time in seconds for the part of the animation where the evolution process is shown. + :param hold_result_time_s: time in seconds for the part of the animation where the final result is shown. + """ + # Treating all graphs as equally distant if there's no reasonable way to compare them: + graph_dist = graph_dist or (lambda g1, g2: 1) + + def draw_graph(graph: Graph, ax, title, highlight_title=False): + ax.clear() + ax.set_title(title, fontsize=22, color='green' if highlight_title else 'black') + GraphVisualizer(graph).draw_nx_dag(ax, node_names_placement='legend') + + try: + last_internal_graph = self.history.archive_history[-1][0] + genealogical_path = trace_genealogical_path(last_internal_graph, graph_dist) + except Exception as e: + # At least `Individual.parents_from_prev_generation` my fail + self.log.error(f"Failed to trace genealogical path: {e}") + return + + figure_width = 5 + width_ratios = [1.3, 0.7] + if target_graph is not None: + width_ratios = [1.3] + width_ratios + + fig, axes = plt.subplots( + 1, len(width_ratios), + figsize=(figure_width * sum(width_ratios), figure_width), + gridspec_kw={'width_ratios': width_ratios} + ) + evo_ax, fitness_ax = axes[-2:] + if target_graph is not None: + draw_graph(target_graph, axes[0], "Target graph") # Persists throughout the animation + + fitnesses_along_path = list(map(lambda ind: ind.fitness.value, genealogical_path)) + generations_along_path = list(map(lambda ind: ind.native_generation, genealogical_path)) + + def render_frame(frame_index): + path_index = min(frame_index, len(genealogical_path) - 1) + is_hold_stage = frame_index >= len(genealogical_path) + + draw_graph( + genealogical_path[path_index].graph, evo_ax, + f"Evolution process,\ngeneration {generations_along_path[path_index]}/{generations_along_path[-1]}", + highlight_title=is_hold_stage + ) + # Select only the genealogical path + fitness_ax.clear() + plot_fitness_with_axvline( + generations=generations_along_path, + fitnesses=fitnesses_along_path, + ax=fitness_ax, + axvline_x=generations_along_path[path_index], + current_fitness=fitnesses_along_path[path_index] + ) + return evo_ax, fitness_ax + + frames = len(genealogical_path) + int( + math.ceil(len(genealogical_path) * hold_result_time_s / (hold_result_time_s + evolution_time_s)) + ) + seconds_per_frame = (evolution_time_s + hold_result_time_s) / frames + fps = math.ceil(1 / seconds_per_frame) + + anim = animation.FuncAnimation(fig, render_frame, repeat=False, frames=frames, + interval=1000 * seconds_per_frame) + + try: + if save_path is not None: + anim.save(save_path, fps=fps) + if show: + plt.show() + except Exception as e: + self.log.error(f"Failed to render the genealogical path: {e}") + + +def trace_genealogical_path(individual: Individual, graph_dist: Callable[[Graph, Graph], float]) -> List[Individual]: + # Choose nearest parent each time: + genealogical_path: List[Individual] = [individual] + while genealogical_path[-1].parents_from_prev_generation: + genealogical_path.append(max( + genealogical_path[-1].parents_from_prev_generation, + key=partial(graph_dist, genealogical_path[-1]) + )) + + return list(reversed(genealogical_path)) + + +def plot_fitness_with_axvline(generations: List[int], fitnesses: List[float], ax: plt.Axes, current_fitness: float, + axvline_x: int = None): + ax.plot(generations, fitnesses) + ax.set_title(f'Fitness dynamic,\ncurrent: {current_fitness}', fontsize=22) + ax.set_xlabel('Generation') + ax.set_ylabel('Fitness') + if axvline_x is not None: + ax.axvline(x=axvline_x, color='black') + return ax diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/graphs_interactive.py b/paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/graphs_interactive.py new file mode 100644 index 0000000..8e2b62c --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/graphs_interactive.py @@ -0,0 +1,92 @@ +import os +from pathlib import Path +from typing import Optional, Union, Dict, Any, List + +import numpy as np +from matplotlib import pyplot as plt +from matplotlib.widgets import Button + +from golem.core.optimisers.opt_history_objects.individual import Individual +from golem.core.paths import default_data_dir +from golem.visualisation.opt_history.fitness_line import with_alternate_matplotlib_backend +from golem.visualisation.opt_history.history_visualization import HistoryVisualization +from golem.visualisation.opt_history.utils import show_or_save_figure + + +class GraphsInteractive(HistoryVisualization): + + @with_alternate_matplotlib_backend + def visualize(self, save_path: Optional[Union[os.PathLike, str]] = None, dpi: Optional[int] = None, + per_time: Optional[bool] = None, graph_show_kwargs: Optional[Dict[str, Any]] = None): + """ Shows the structure of the best individuals of all time. + :param save_path: path to save the visualization. If set, then the image will be saved, and if not, + it will be displayed. + :param dpi: DPI of the output figure. + :param per_time: defines whether to show time grid if it is available in history. + :param graph_show_kwargs: keyword arguments of `graph.show()` function. + """ + + save_path = save_path or self.get_predefined_value('save_path') + dpi = dpi or self.get_predefined_value('dpi') + graph_show_kwargs = graph_show_kwargs or self.visuals_params.get('graph_show_params') or {} + + # fig, axes = plt.subplots(1, 2, figsize=(15, 10)) + # ax_fitness, ax_graph = axes + fig, ax_graph = plt.subplots(1, 1, figsize=(10, 10)) + ax_graph.axis('off') + + x_template = 'best individual #{}' + best_individuals = {i: ind + for i, ind in enumerate(self.history.archive_history[-1])} + + class InteractivePlot: + temp_path = Path(default_data_dir(), 'current_graph.png') + + def __init__(self, best_individuals: Dict[int, Individual]): + self.best_x: List[int] = list(best_individuals.keys()) + self.best_individuals: List[Individual] = list(best_individuals.values()) + self.index: int = len(self.best_individuals) - 1 + # self.time_line = ax_fitness.axvline(self.best_x[self.index], color='r', alpha=0.7) + self.graph_images: List[np.ndarray] = [] + self.generate_graph_images() + self.update_graph() + + def generate_graph_images(self): + for ind in self.best_individuals: + graph = ind.graph + graph.show(self.temp_path, **graph_show_kwargs) + self.graph_images.append(plt.imread(str(self.temp_path))) + self.temp_path.unlink() + + def update_graph(self): + ax_graph.imshow(self.graph_images[self.index]) + x = self.best_x[self.index] + fitness = self.best_individuals[self.index].fitness + ax_graph.set_title(f'The best graph at {x_template.format(x)}, fitness={fitness}') + + # def update_time_line(self): + # self.time_line.set_xdata(self.best_x[self.index]) + + def step_index(self, step: int): + self.index = (self.index + step) % len(self.best_individuals) + self.update_graph() + # self.update_time_line() + plt.draw() + + def next(self, event): + self.step_index(1) + + def prev(self, event): + self.step_index(-1) + + callback = InteractivePlot(best_individuals) + + if not save_path: # display buttons only for an interactive plot + ax_prev = plt.axes([0.7, 0.05, 0.1, 0.075]) + ax_next = plt.axes([0.81, 0.05, 0.1, 0.075]) + b_next = Button(ax_next, 'Next') + b_next.on_clicked(callback.next) + b_prev = Button(ax_prev, 'Previous') + b_prev.on_clicked(callback.prev) + + show_or_save_figure(fig, save_path, dpi) diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/history_visualization.py b/paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/history_visualization.py new file mode 100644 index 0000000..37c5158 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/history_visualization.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from abc import abstractmethod +from typing import TYPE_CHECKING, Dict, Any + +from golem.core.log import default_log +from golem.visualisation.opt_history.arg_constraint_wrapper import ArgConstraintWrapper + +if TYPE_CHECKING: + from golem.core.optimisers.opt_history_objects.opt_history import OptHistory + + +class HistoryVisualization(metaclass=ArgConstraintWrapper): + """ Base class for creating visualizations of the optimization history. + The only necessary method is 'visualize' - it must show or save the plot in any form after the call. + + One should refer the OptHistory instance as `self.history` to be able to connect one's visualization + to `OptHistory.show()`. See the examples of connecting visualizations in the module `opt_viz.py`. + + It is good practice to be aware of constraints on your visualizations. You can either implement + default constraints that will catch your kwarg across all the visualizations or define your single + class specific constraints by assigning them to `constraint_checkers` class attribute. + See `golem.core.visualisation.opt_history.arg_constraint_wrapper.py` for examples. + """ + constraint_checkers = [] # Use this for class-specific constraint checkers. + + def __init__(self, history: 'OptHistory', visuals_params: Dict[str, Any] = None): + self.visuals_params = visuals_params or {} + self.history = history + self.log = default_log(self) + + @abstractmethod + def visualize(self, *args, **kwargs): + raise NotImplementedError() + + def get_predefined_value(self, param: str): + return self.visuals_params.get(param) diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/multiple_fitness_line.py b/paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/multiple_fitness_line.py new file mode 100644 index 0000000..38c7267 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/multiple_fitness_line.py @@ -0,0 +1,155 @@ +import os +from pathlib import Path +from statistics import mean, stdev +from typing import Any, Dict, List, Optional, Union, Sequence + +import numpy as np +from matplotlib import pyplot as plt + +from golem.core.log import default_log +from golem.core.optimisers.opt_history_objects.opt_history import OptHistory +from golem.utilities.data_structures import ensure_wrapped_in_sequence +from golem.visualisation.opt_history.arg_constraint_wrapper import ArgConstraintWrapper +from golem.visualisation.opt_history.fitness_line import setup_fitness_plot +from golem.visualisation.opt_history.utils import show_or_save_figure + + +class MultipleFitnessLines(metaclass=ArgConstraintWrapper): + """ Class to compare fitness changes during optimization process. + :param historical_fitnesses: dictionary with labels to display as keys and list of fintess values as dict values.""" + + def __init__(self, + historical_fitnesses: Dict[str, Sequence[Sequence[Union[float, Sequence[float]]]]], + metric_names: List[str], + visuals_params: Dict[str, Any] = None): + self.historical_fitnesses = historical_fitnesses + self.metric_names = metric_names + self.visuals_params = visuals_params or {} + self.log = default_log(self) + + @staticmethod + def from_saved_histories(experiment_folders: List[str], root_path: os.PathLike) -> 'MultipleFitnessLines': + """ Loads histories from specified folders extracting only fitness values + to not store whole histories in memory. + Args: + experiment_folders: names of folders with histories for experiment launches. + root_path: path to the folder with experiments results + """ + root = Path(root_path) + historical_fitnesses = {} + for exp_name in experiment_folders: + trials = [] + for history_filename in os.listdir(root / exp_name): + if history_filename.startswith('history'): + history = OptHistory.load(root / exp_name / history_filename) + trials.append(history.historical_fitness) + print(f"Loaded {history_filename}") + historical_fitnesses[exp_name] = trials + print(f'Loaded {len(trials)} trial histories for experiment: {exp_name}') + metric_names = history.objective.metric_names + + return MultipleFitnessLines(historical_fitnesses, metric_names) + + @staticmethod + def from_histories(histories_to_compare: Dict[str, Sequence['OptHistory']]) -> 'MultipleFitnessLines': + """ + Args: + histories_to_compare: dictionary with labels to display as keys and histories as values.""" + historical_fitnesses = {} + for key, histories in histories_to_compare.items(): + historical_fitnesses.update({key: [history.historical_fitness for history in histories]}) + metric_names = list(histories_to_compare.values())[0][0].objective.metric_names + + return MultipleFitnessLines(historical_fitnesses, metric_names) + + def visualize(self, + save_path: Optional[Union[os.PathLike, str]] = None, + with_confidence: bool = True, + metric_id: int = 0, + dpi: Optional[int] = None): + """ Visualizes the best fitness values during the evolution in the form of line. + :param save_path: path to save the visualization. If set, then the image will be saved, + and if not, it will be displayed. + :param with_confidence: bool param specifying to use confidence interval or not. + :param metric_id: numeric index of the metric to visualize (for multi-objective opt-n). + :param dpi: DPI of the output figure. + """ + save_path = save_path or self.get_predefined_value('save_path') + dpi = dpi or self.get_predefined_value('dpi') + + fig, ax = plt.subplots(figsize=(6.4, 4.8), facecolor='w') + xlabel = 'Generation' + self.plot_multiple_fitness_lines(ax, metric_id, with_confidence) + setup_fitness_plot(ax, xlabel, title=f'Fitness lines for {self.metric_names[metric_id]}') + plt.legend() + show_or_save_figure(fig, save_path, dpi) + + def plot_multiple_fitness_lines(self, ax: plt.axis, metric_id: int = 0, with_confidence: bool = True): + for histories, label in zip(list(self.historical_fitnesses.values()), list(self.historical_fitnesses.keys())): + plot_average_fitness_line_per_generations(ax, histories, label, + with_confidence=with_confidence, + metric_id=metric_id) + + def get_predefined_value(self, param: str): + return self.visuals_params.get(param) + + +def plot_average_fitness_line_per_generations( + axis: plt.Axes, + historical_fitnesses: Sequence[Sequence[Union[float, Sequence[float]]]], + label: Optional[str] = None, + metric_id: int = 0, + with_confidence: bool = True, + z_score: float = 1.96): + """Plots average fitness line per number of histories + with confidence interval for given z-score (default z=1.96 is for 95% confidence).""" + + trial_fitnesses: List[List[float]] = [] + for fitnesses in historical_fitnesses: + best_fitnesses = get_best_fitness_per_generation(fitnesses, metric_id) + trial_fitnesses.append(best_fitnesses) + + # Get average fitness value with confidence values + average_fitness_per_gen = [] + confidence_fitness_per_gen = [] + max_generations = max(len(i) for i in trial_fitnesses) + for i in range(max_generations): + all_fitness_gen = [] + for fitnesses in trial_fitnesses: + if i < len(fitnesses): + all_fitness_gen.append(fitnesses[i]) + else: + # if history is too short - repeat the best obtained fitness + all_fitness_gen.append(fitnesses[-1]) + average_fitness_per_gen.append(mean(all_fitness_gen)) + confidence = stdev(all_fitness_gen) / np.sqrt(len(all_fitness_gen)) \ + if len(all_fitness_gen) >= 2 else 0. + confidence_fitness_per_gen.append(confidence) + + # Compute confidence interval + xs = np.arange(len(average_fitness_per_gen)) + ys = np.array(average_fitness_per_gen) + ci = z_score * np.array(confidence_fitness_per_gen) + + axis.plot(xs, average_fitness_per_gen, label=label) + if with_confidence: + axis.fill_between(xs, (ys - ci), (ys + ci), alpha=.2) + + +def get_best_fitness_per_generation(fitnesses: Sequence[Sequence[Union[float, Sequence[float]]]], + metric_id: int = 0, + ) -> List[float]: + """Per each generation find the best fitness *seen so far*. + Returns tuple: + - list of best seen metric up to that generation + """ + best_metric = np.inf # Assuming metric minimization + best_metrics = [] + + for gen_num, gen_fitnesses in enumerate(fitnesses[metric_id]): + target_metric = min(ensure_wrapped_in_sequence(gen_fitnesses)) + if target_metric <= best_metric: + best_metric = target_metric + best_metrics.append(best_metric) + + return best_metrics diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/operations_animated_bar.py b/paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/operations_animated_bar.py new file mode 100644 index 0000000..ded7e79 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/operations_animated_bar.py @@ -0,0 +1,196 @@ +import os +from pathlib import Path +from typing import List, Optional, Sequence, Union +from numpy.typing import ArrayLike + +import numpy as np +import seaborn as sns +import matplotlib as mpl +from matplotlib import cm, animation, pyplot as plt +from matplotlib.colors import Normalize + +from golem.visualisation.opt_history.history_visualization import HistoryVisualization +from golem.visualisation.opt_history.utils import get_history_dataframe, get_description_of_operations_by_tag, \ + TagOperationsMap, LabelsColorMapType + + +class OperationsAnimatedBar(HistoryVisualization): + def visualize(self, save_path: Optional[Union[os.PathLike, str]] = None, dpi: Optional[int] = None, + best_fraction: Optional[float] = None, show_fitness: Optional[bool] = None, + tags_map: TagOperationsMap = None, palette: Optional[LabelsColorMapType] = None): + """ Visualizes operations used across generations in the form of animated bar plot. + + :param save_path: path to save the visualization. + :param dpi: DPI of the output figure. + :param best_fraction: fraction of the best individuals of each generation that included in the + visualization. Must be in the interval (0, 1]. + :param show_fitness: if False, the bar colors will not correspond to fitness. + :param tags_map: if specified, all operations in the history are grouped based on the provided tags. + If None, operations are not grouped. + :param palette: a map from operation label to its color. If None, colors are picked by fixed colormap + for every history independently. + + """ + + save_path = save_path or self.get_predefined_value('save_path') or 'history_animated_bars.gif' + dpi = dpi or self.get_predefined_value('dpi') + best_fraction = best_fraction or self.get_predefined_value('best_fraction') + show_fitness = show_fitness if show_fitness is not None else self.get_predefined_value('show_fitness') or True + tags_map = tags_map or self.visuals_params.get('tags_map') + palette = palette or self.visuals_params.get('palette') + + def interpolate_points(point_1, point_2, smoothness=18, power=4) -> List[np.array]: + t_interp = np.linspace(0, 1, smoothness) + point_1, point_2 = np.array(point_1), np.array(point_2) + return [point_1 * (1 - t ** power) + point_2 * t ** power for t in t_interp] + + def smoothen_frames_data(data: Sequence[Sequence[ArrayLike]], smoothness=18, power=4) -> List[np.array]: + final_frames = [] + for initial_frame in range(len(data) - 1): + final_frames += interpolate_points(data[initial_frame], data[initial_frame + 1], smoothness, power) + # final frame interpolates into itcls + final_frames += interpolate_points(data[-1], data[-1], smoothness, power) + + return final_frames + + def animate(frame_num): + frame_count = bar_data[frame_num] + frame_color = bar_color[frame_num] if show_fitness else None + frame_title = bar_title[frame_num] + + plt.title(frame_title) + for bar_num in range(len(bars)): + bars[bar_num].set_width(frame_count[bar_num]) + if not show_fitness: + continue + bars[bar_num].set_facecolor(frame_color[bar_num]) + + save_path = Path(save_path) + if save_path.suffix not in ['.gif', '.mp4']: + raise ValueError('A correct file extension (".mp4" or ".gif") should be set to save the animation.') + + animation_frames_per_step = 18 + animation_interval_between_frames_ms = 40 + animation_interpolation_power = 4 + fitness_colormap = mpl.colormaps['YlOrRd'] + + generation_column_name = 'Generation' + fitness_column_name = 'Fitness' + operation_column_name = 'Operation' + column_for_operation = 'tag' if tags_map else 'node' + + df_history = get_history_dataframe(self.history, best_fraction, tags_map) + df_history = df_history.rename({ + 'generation': generation_column_name, + 'fitness': fitness_column_name, + column_for_operation: operation_column_name, + }, axis='columns') + operations_found = df_history[operation_column_name].unique() + if tags_map: + tags_all = list(tags_map.keys()) + operations_found = [tag for tag in tags_all if tag in operations_found] + nodes_per_tag = df_history.groupby(operation_column_name)['node'].unique() + bars_labels = [get_description_of_operations_by_tag(t, nodes_per_tag[t], 22) for t in operations_found] + else: + bars_labels = operations_found + + if palette: + no_fitness_palette = palette + else: + no_fitness_palette = sns.color_palette('tab10', n_colors=len(operations_found)) + no_fitness_palette = {o: no_fitness_palette[i] for i, o in enumerate(operations_found)} + + # Getting normed fraction of individuals per generation that contain operations given. + generation_sizes = df_history.groupby(generation_column_name)['individual'].nunique() + operations_with_individuals_count = df_history.groupby( + [generation_column_name, operation_column_name], + as_index=False + ).aggregate({'individual': 'nunique'}) + operations_with_individuals_count['individual'] = operations_with_individuals_count.apply( + lambda row: row['individual'] / generation_sizes[row[generation_column_name]], + axis='columns') + + if show_fitness: + # Getting fitness per individual with the list of contained operations in the form of + # '.operation_1.operation_2. ... .operation_n.' + individuals_fitness = df_history[[generation_column_name, 'individual', fitness_column_name]] \ + .drop_duplicates() + individuals_fitness['operations'] = individuals_fitness.apply( + lambda row: '.{}.'.format('.'.join( + df_history[ + (df_history[generation_column_name] == row[generation_column_name]) & + (df_history['individual'] == row['individual']) + ][operation_column_name])), + axis='columns') + # Getting mean fitness of individuals with the operations given. + operations_with_individuals_count[fitness_column_name] = operations_with_individuals_count.apply( + lambda row: individuals_fitness[ + (individuals_fitness[generation_column_name] == row[generation_column_name]) & + (individuals_fitness['operations'].str.contains(f'.{row[operation_column_name]}.')) + ][fitness_column_name].mean(), + axis='columns') + del individuals_fitness + # Replacing the initial DataFrame with the processed one + df_history = operations_with_individuals_count.set_index([generation_column_name, operation_column_name]) + del operations_with_individuals_count + + min_fitness = df_history[fitness_column_name].min() if show_fitness else None + max_fitness = df_history[fitness_column_name].max() if show_fitness else None + + generations = generation_sizes.index.unique() + bar_data = [] + bar_color = [] + # Getting data by tags through all generations and filling with zeroes where no such tag + for gen_num in generations: + bar_data.append([df_history.loc[gen_num]['individual'].get(tag, 0) for tag in operations_found]) + if not show_fitness: + continue + fitnesses = [df_history.loc[gen_num][fitness_column_name].get(tag, 0) for tag in operations_found] + # Transfer fitness to color + bar_color.append([ + fitness_colormap((fitness - min_fitness) / (max_fitness - min_fitness)) for fitness in fitnesses]) + + bar_data = smoothen_frames_data(bar_data, animation_frames_per_step, animation_interpolation_power) + title_template = 'Generation {}' + if best_fraction is not None: + title_template += f', top {best_fraction * 100}%' + bar_title = [i for gen_num in generations for i in [title_template.format(gen_num)] * animation_frames_per_step] + + fig, ax = plt.subplots(figsize=(8, 5), facecolor='w') + if show_fitness: + bar_color = smoothen_frames_data(bar_color, animation_frames_per_step, animation_interpolation_power) + sm = cm.ScalarMappable(norm=Normalize(min_fitness, max_fitness), cmap=fitness_colormap) + sm.set_array([]) + fig.colorbar(sm, label=fitness_column_name, ax=ax) + + count = bar_data[0] + color = bar_color[0] if show_fitness else [no_fitness_palette[tag] for tag in operations_found] + title = bar_title[0] + + label_size = 10 + if any(len(label.split('\n')) > 2 for label in bars_labels): + label_size = 8 + + bars = ax.barh(bars_labels, count, color=color) + ax.tick_params(axis='y', which='major', labelsize=label_size) + ax.set_title(title) + ax.set_xlim(0, 1) + ax.set_xlabel('Fraction of graphs containing the operation') + ax.xaxis.grid(True) + ax.set_ylabel(operation_column_name) + ax.invert_yaxis() + plt.tight_layout() + + if not save_path.is_absolute(): + save_path = Path.cwd().joinpath(save_path) + + ani = animation.FuncAnimation( + fig, + animate, + frames=len(bar_data), + interval=animation_interval_between_frames_ms, + repeat=True + ) + ani.save(str(save_path), dpi=dpi) + self.log.info(f'The animation was saved to "{save_path}".') + plt.close(fig=fig) diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/operations_kde.py b/paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/operations_kde.py new file mode 100644 index 0000000..7365488 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/operations_kde.py @@ -0,0 +1,78 @@ +import os +from typing import Optional, Union + +import seaborn as sns +from matplotlib import pyplot as plt + +from golem.visualisation.opt_history.history_visualization import HistoryVisualization +from golem.visualisation.opt_history.utils import get_history_dataframe, get_description_of_operations_by_tag, \ + show_or_save_figure, TagOperationsMap, LabelsColorMapType + + +class OperationsKDE(HistoryVisualization): + def visualize(self, save_path: Optional[Union[os.PathLike, str]] = None, dpi: Optional[int] = None, + best_fraction: Optional[float] = None, tags_map: TagOperationsMap = None, + palette: Optional[LabelsColorMapType] = None): + """ Visualizes operations used across generations in the form of KDE. + + :param save_path: path to save the visualization. If set, then the image will be saved, and if not, + it will be displayed. + :param dpi: DPI of the output figure. + :param best_fraction: fraction of the best individuals of each generation that included in the + visualization. Must be in the interval (0, 1]. + :param tags_map: if specified, all operations in the history are colored and grouped based on the + provided tags. If None, operations are not grouped. + :param palette: a map from operation label to its color. If None, colors are picked by fixed colormap + for every history independently. + """ + + save_path = save_path or self.get_predefined_value('save_path') + dpi = dpi or self.get_predefined_value('dpi') + best_fraction = best_fraction or self.get_predefined_value('best_fraction') + tags_map = tags_map or self.visuals_params.get('tags_map') + palette = palette or self.visuals_params.get('palette') + + generation_column_name = 'Generation' + operation_column_name = 'Operation' + column_for_operation = 'tag' if tags_map else 'node' + + df_history = get_history_dataframe(self.history, best_fraction, tags_map) + df_history = df_history.rename({'generation': generation_column_name, + column_for_operation: operation_column_name}, axis='columns') + operations_found = df_history[operation_column_name].unique() + if tags_map: + tags_all = list(tags_map.keys()) + operations_found = [t for t in tags_all if t in operations_found] # Sort and filter. + + nodes_per_tag = df_history.groupby(operation_column_name)['node'].unique() + legend_per_tag = {tag: get_description_of_operations_by_tag(tag, nodes_per_tag[tag], 22) + for tag in operations_found} + df_history[operation_column_name] = df_history[operation_column_name].map(legend_per_tag) + operations_found = map(legend_per_tag.get, operations_found) + if palette: + palette = {legend_per_tag.get(tag): palette.get(tag) for tag in legend_per_tag} + + if not palette: + palette = sns.color_palette('tab10', n_colors=len(operations_found)) + + plot = sns.displot( + data=df_history, + x=generation_column_name, + hue=operation_column_name, + hue_order=operations_found, + kind='kde', + clip=(0, max(df_history[generation_column_name])), + multiple='fill', + palette=palette + ) + + fig = plot.figure + fig.set_dpi(dpi) + fig.set_facecolor('w') + ax = plt.gca() + ax.set_xticks(range(len(self.history.generations))) + ax.locator_params(nbins=10) + str_fraction_of_graphs = 'all' if best_fraction is None else f'top {best_fraction * 100}% of' + ax.set_ylabel(f'Fraction in {str_fraction_of_graphs} generation graphs') + + show_or_save_figure(fig, save_path, dpi) diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/utils.py b/paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/utils.py new file mode 100644 index 0000000..c7c0d38 --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_history/utils.py @@ -0,0 +1,117 @@ +from __future__ import annotations + +import os +from pathlib import Path +from textwrap import wrap +from typing import TYPE_CHECKING, Dict, List, Optional, Sequence, Union + +import pandas as pd +from matplotlib import pyplot as plt + +from golem.core.log import default_log + +if TYPE_CHECKING: + from golem.core.optimisers.opt_history_objects.opt_history import OptHistory + +MatplotlibColorType = Union[str, Sequence[float]] +LabelsColorMapType = Dict[str, MatplotlibColorType] + +TagOperationsMap = Dict[str, List[str]] + + +def get_history_dataframe(history: OptHistory, best_fraction: Optional[float] = None, + tags_map: Optional[TagOperationsMap] = None): + history_data = { + 'generation': [], + 'individual': [], + 'fitness': [], + 'node': [], + } + if tags_map: + history_data['tag'] = [] + new_map = {} + for tag, operations in tags_map.items(): + new_map.update({operation: tag for operation in operations}) + tags_map = new_map + + uid_counts = {} # Resolving individuals with the same uid + for gen_num, gen in enumerate(history.generations): + for ind in gen: + uid_counts[ind.uid] = uid_counts.get(ind.uid, -1) + 1 + for node in ind.graph.nodes: + history_data['generation'].append(gen_num) + history_data['individual'].append('_'.join([ind.uid, str(uid_counts[ind.uid])])) + fitness = abs(ind.fitness.value) + history_data['fitness'].append(fitness) + history_data['node'].append(str(node)) + if not tags_map: + continue + history_data['tag'].append(tags_map.get(str(node), None)) + + df_history = pd.DataFrame.from_dict(history_data) + + if best_fraction is not None: + generation_sizes = df_history.groupby('generation')['individual'].nunique() + + df_individuals = df_history[['generation', 'individual', 'fitness']] \ + .drop_duplicates(ignore_index=True) + + df_individuals['rank_per_generation'] = df_individuals.sort_values('fitness', ascending=False). \ + groupby('generation').cumcount() + + best_individuals = df_individuals[ + df_individuals.apply( + lambda row: row['rank_per_generation'] < generation_sizes[row['generation']] * best_fraction, + axis='columns' + ) + ]['individual'] + + df_history = df_history[df_history['individual'].isin(best_individuals)] + + return df_history + + +def get_description_of_operations_by_tag(tag: str, operations_by_tag: List[str], max_line_length: int, + format_tag: str = 'it'): + def make_text_fancy(text: str): + return text.replace('_', ' ') + + def format_text(text_to_wrap: str, latex_format_tag: str = 'it') -> str: + formatted_text = f'$\\{latex_format_tag}{{{text_to_wrap}}}$' + formatted_text = formatted_text.replace(' ', '\\;') + return formatted_text + + def format_wrapped_text_from_beginning(wrapped_text: List[str], part_to_format: str, latex_format_tag: str = 'it') \ + -> List[str]: + for line_num, line in enumerate(wrapped_text): + if part_to_format in line: + # The line contains the whole part_to_format. + wrapped_text[line_num] = line.replace(part_to_format, format_text(part_to_format, latex_format_tag)) + break + if part_to_format.startswith(line): + # The whole line should be formatted. + wrapped_text[line_num] = format_text(line, latex_format_tag) + part_to_format = part_to_format[len(line):].strip() + + return wrapped_text + + tag = make_text_fancy(tag) + operations_by_tag = ', '.join(operations_by_tag) + description = f'{tag}: {operations_by_tag}.' + description = make_text_fancy(description) + description = wrap(description, max_line_length) + description = format_wrapped_text_from_beginning(description, tag, format_tag) + description = '\n'.join(description) + return description + + +def show_or_save_figure(figure: plt.Figure, save_path: Optional[Union[os.PathLike, str]], dpi: int = 100): + if not save_path: + plt.show() + else: + save_path = Path(save_path) + if not save_path.is_absolute(): + save_path = Path.cwd().joinpath(save_path) + figure.savefig(save_path, dpi=dpi) + default_log().info(f'The figure was saved to "{save_path}".') + plt.close() diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_viz.py b/paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_viz.py new file mode 100644 index 0000000..bd588ea --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_viz.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +from enum import Enum +from typing import TYPE_CHECKING, Any, Dict, Optional, Union + +from golem.core.log import default_log +from golem.visualisation.opt_history.diversity import DiversityLine, DiversityPopulation +from golem.visualisation.opt_history.fitness_box import FitnessBox +from golem.visualisation.opt_history.fitness_line import FitnessLine, FitnessLineInteractive +from golem.visualisation.opt_history.genealogical_path import GenealogicalPath +from golem.visualisation.opt_history.operations_animated_bar import OperationsAnimatedBar +from golem.visualisation.opt_history.operations_kde import OperationsKDE + +if TYPE_CHECKING: + from golem.core.optimisers.opt_history_objects.opt_history import OptHistory + + +class PlotTypesEnum(Enum): + fitness_box = FitnessBox + fitness_line = FitnessLine + fitness_line_interactive = FitnessLineInteractive + operations_kde = OperationsKDE + operations_animated_bar = OperationsAnimatedBar + diversity_line = DiversityLine + diversity_population = DiversityPopulation + genealogical_path = GenealogicalPath + + @classmethod + def member_names(cls): + return cls._member_names_ + + +class OptHistoryVisualizer: + """ Implements optimization history visualizations available via `OptHistory.show()`. + + `OptHistory.show` points to the initialized instance of this class. + Thus, supported visualizations can be called directly via `OptHistory.show.(...)` or + indirectly via `OptHistory.show(...)`. + + This implies that all supported visualizations are listed in two places: + 1. `PlotTypesEnum` members + + ` = ` + + 2. assigned as the class attributes in `OptHistoryVisualizer.__init__` + + `self. = (self.history).visualize` + """ + + def __init__(self, history: OptHistory, visuals_params: Optional[Dict[str, Any]] = None): + visuals_params = visuals_params or {} + default_visuals_params = dict(dpi=100) + default_visuals_params.update(visuals_params) + + self.visuals_params = default_visuals_params + self.history = history + + self.fitness_box = FitnessBox(history, self.visuals_params).visualize + self.fitness_line = FitnessLine(history, self.visuals_params).visualize + self.fitness_line_interactive = FitnessLineInteractive(history, self.visuals_params).visualize + self.operations_kde = OperationsKDE(history, self.visuals_params).visualize + self.operations_animated_bar = OperationsAnimatedBar(history, self.visuals_params).visualize + self.diversity_line = DiversityLine(history, self.visuals_params).visualize + self.diversity_population = DiversityPopulation(history, self.visuals_params).visualize + self.genealogical_path = GenealogicalPath(history, self.visuals_params).visualize + + self.log = default_log(self) + + def __call__(self, plot_type: Union[PlotTypesEnum, str] = PlotTypesEnum.fitness_line, **kwargs): + """ Visualizes the OptHistory with one of the supported methods. + + :param plot_type: visualization to show. Expected values are listed in + 'golem.core.visualisation.opt_viz.PlotTypesEnum'. + :keyword save_path: path to save the visualization. If set, then the image will be saved, and if not, + it will be displayed. Essential for animations. + :keyword dpi: DPI of the output figure. + :keyword best_fraction: fraction of the best individuals of each generation that included in the + visualization. Must be in the interval (0, 1]. + :keyword show_fitness: if False, visualizations that support this argument will not display fitness. + :keyword per_time: Shows time axis instead of generations axis. Currently, supported for plot types: + 'show_fitness_line', 'show_fitness_line_interactive'. + :keyword use_tags: if True (default), all operations in the history are colored and grouped based on + operation repository tags. If False, operations are not grouped, colors are picked by fixed colormap + for every history independently. + """ + + if isinstance(plot_type, str): + try: + visualize_function = vars(self)[plot_type] + except KeyError: + raise NotImplementedError( + f'Visualization "{plot_type}" is not supported. Expected values: ' + f'{", ".join(PlotTypesEnum.member_names())}.') + else: + visualize_function = vars(self)[plot_type.name] + visualize_function(**kwargs) diff --git a/paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_viz_extra.py b/paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_viz_extra.py new file mode 100644 index 0000000..13eb83a --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem/visualisation/opt_viz_extra.py @@ -0,0 +1,295 @@ +import itertools +import os +from copy import deepcopy +from datetime import datetime +from glob import glob +from os import remove +from typing import Any, List, Sequence, Tuple, Optional + +import numpy as np +import pandas as pd +import seaborn as sns +from PIL import Image +from imageio import get_writer, v2 +from matplotlib import pyplot as plt + +from golem.core.dag.graph import Graph +from golem.core.log import default_log +from golem.core.optimisers.opt_history_objects.individual import Individual +from golem.core.optimisers.opt_history_objects.opt_history import OptHistory +from golem.core.paths import default_data_dir +from golem.visualisation.graph_viz import GraphVisualizer + + +class OptHistoryExtraVisualizer: + """ Implements legacy history visualizations that are not available via `history.show()` + Args: + history: history of optimisation + folder: path to folder to save results of visualization + """ + + def __init__(self, history: OptHistory, folder: Optional[str] = None): + data_dir = folder or default_data_dir() + + self.save_path = os.path.join(data_dir, 'composing_history') + if 'composing_history' not in os.listdir(data_dir): + os.mkdir(self.save_path) + self.history = history + self.log = default_log(self) + self.graphs_imgs = [] + self.convergence_imgs = [] + self.best_graphs_imgs = [] + self.merged_imgs = [] + self.graph_visualizer = GraphVisualizer + + def pareto_gif_create(self, + objectives_numbers: Tuple[int, int] = (0, 1), + objectives_names: Tuple[str] = ('ROC-AUC', 'Complexity')): + files = [] + pareto_fronts = self.history.archive_history + individuals = self.history.generations + array_for_analysis = individuals if individuals else pareto_fronts + all_objectives = extract_objectives(array_for_analysis, objectives_numbers) + min_x, max_x = min(all_objectives[0]) - 0.01, max(all_objectives[0]) + 0.01 + min_y, max_y = min(all_objectives[1]) - 0.01, max(all_objectives[1]) + 0.01 + folder = f'{self.save_path}' + for i, front in enumerate(pareto_fronts): + file_name = f'pareto{i}.png' + visualise_pareto(front, file_name=file_name, save=True, show=False, + folder=folder, generation_num=i, individuals=individuals[i], + minmax_x=[min_x, max_x], minmax_y=[min_y, max_y], + objectives_numbers=objectives_numbers, + objectives_names=objectives_names) + files.append(f'{folder}/{file_name}') + + create_gif_using_images(gif_path=f'{folder}/pareto_history.gif', files=files) + for file in files: + os.remove(file) + + def _visualise_graphs(self, graphs: List[Graph], fitnesses: List[float]): + fitnesses = deepcopy(fitnesses) + last_best_graph = graphs[0] + prev_fit = fitnesses[0] + fig = plt.figure(figsize=(10, 10)) + for ch_id, graph in enumerate(graphs): + self.graph_visualizer(graph).draw_nx_dag() + fig.canvas.draw() + img = figure_to_array(fig) + self.graphs_imgs.append(img) + plt.clf() + if fitnesses[ch_id] > prev_fit: + fitnesses[ch_id] = prev_fit + else: + last_best_graph = graph + prev_fit = fitnesses[ch_id] + plt.clf() + self.graph_visualizer(last_best_graph).draw_nx_dag() + fig.canvas.draw() + img = figure_to_array(fig) + self.best_graphs_imgs.append(img) + plt.clf() + plt.close('all') + + def _visualise_convergence(self, fitness_history): + fitness_history = deepcopy(fitness_history) + prev_fit = fitness_history[0] + for fit_id, fit in enumerate(fitness_history): + if fit > prev_fit: + fitness_history[fit_id] = prev_fit + prev_fit = fitness_history[fit_id] + ts_set = list(range(len(fitness_history))) + df = pd.DataFrame( + {'ts': ts_set, 'fitness': [-f for f in fitness_history]}) + + fig = plt.figure(figsize=(10, 10)) + plt.rcParams['axes.titlesize'] = 20 + plt.rcParams['axes.labelsize'] = 20 + for ts in ts_set: + plt.plot(df['ts'], df['fitness'], label='Optimizer') + plt.xlabel('Evaluation', fontsize=18) + plt.ylabel('Best metric', fontsize=18) + plt.axvline(x=ts, color='black') + plt.legend(loc='upper left') + fig.canvas.draw() + img = figure_to_array(fig) + self.convergence_imgs.append(img) + plt.clf() + plt.close('all') + + def visualise_history(self, metric_index: int = 0): + try: + self._clean(with_gif=True) + all_historical_fitness = self.history.all_historical_quality(metric_index) + historical_graphs = [ind.graph + for ind in list(itertools.chain(*self.history.generations))] + self._visualise_graphs(historical_graphs, all_historical_fitness) + self._visualise_convergence(all_historical_fitness) + self._merge_images() + self._combine_gifs() + self._clean() + except Exception as ex: + self.log.error(f'Visualisation failed with {ex}') + + def _merge_images(self): + for i in range(1, len(self.graphs_imgs)): + im1 = self.graphs_imgs[i] + im2 = self.best_graphs_imgs[i] + im3 = self.convergence_imgs[i] + imgs = (im1, im2, im3) + merged = np.concatenate(imgs, axis=1) + self.merged_imgs.append(Image.fromarray(np.uint8(merged))) + + def _combine_gifs(self): + date_time = datetime.now().strftime('%B-%d-%Y_%H-%M-%S_%p') + save_path = os.path.join(self.save_path, f'history_visualisation_{date_time}.gif') + imgs = self.merged_imgs[1:] + self.merged_imgs[0].save(save_path, save_all=True, append_images=imgs, + optimize=False, duration=0.5, loop=0) + self.log.info(f"Visualizations were saved to {save_path}") + + def _clean(self, with_gif=False): + files = glob(f'{self.save_path}*.png') + if with_gif: + files += glob(f'{self.save_path}*.gif') + for file in files: + remove(file) + + def _create_boxplot(self, individuals: List[Any], generation_num: int = None, + objectives_names: Tuple[str] = ('ROC-AUC', 'Complexity'), file_name: str = 'obj_boxplots.png', + folder: str = None, y_limits: Tuple[float] = None): + folder = f'{self.save_path}/boxplots' if folder is None else folder + fig, ax = plt.subplots() + ax.set_title(f'Generation: {generation_num}', fontsize=15) + objectives = objectives_lists(individuals) + df_objectives = pd.DataFrame({objectives_names[i]: objectives[i] for i in range(len(objectives))}) + sns.boxplot(data=df_objectives, palette="Blues") + if y_limits: + plt.ylim(y_limits[0], y_limits[1]) + if not os.path.isdir('../../tmp'): + os.mkdir('../../tmp') + if not os.path.isdir(f'{folder}'): + os.mkdir(f'{folder}') + path = f'{folder}/{file_name}' + plt.savefig(path, bbox_inches='tight') + + def boxplots_gif_create(self, objectives_names: Tuple[str] = ('ROC-AUC', 'Complexity')): + individuals = self.history.generations + objectives = extract_objectives(individuals) + objectives = list(itertools.chain(*objectives)) + min_y, max_y = min(objectives), max(objectives) + files = [] + folder = f'{self.save_path}' + for generation_num, individuals_in_genaration in enumerate(individuals): + file_name = f'{generation_num}.png' + self._create_boxplot(individuals_in_genaration, generation_num, objectives_names, + file_name=file_name, folder=folder, y_limits=(min_y, max_y)) + files.append(f'{folder}/{file_name}') + create_gif_using_images(gif_path=f'{folder}/boxplots_history.gif', files=files) + for file in files: + os.remove(file) + plt.cla() + plt.clf() + plt.close('all') + + +def visualise_pareto(front: Sequence[Individual], + objectives_numbers: Tuple[int, int] = (0, 1), + objectives_names: Sequence[str] = ('ROC-AUC', 'Complexity'), + file_name: str = 'result_pareto.png', show: bool = False, save: bool = True, + folder: str = '../../tmp/pareto', + generation_num: int = None, + individuals: Sequence[Individual] = None, + minmax_x: List[float] = None, + minmax_y: List[float] = None): + pareto_obj_first, pareto_obj_second = [], [] + for ind in front: + fit_first = ind.fitness.values[objectives_numbers[0]] + pareto_obj_first.append(abs(fit_first)) + fit_second = ind.fitness.values[objectives_numbers[1]] + pareto_obj_second.append(abs(fit_second)) + + fig, ax = plt.subplots() + + if individuals is not None: + obj_first, obj_second = [], [] + for ind in individuals: + fit_first = ind.fitness.values[objectives_numbers[0]] + obj_first.append(abs(fit_first)) + fit_second = ind.fitness.values[objectives_numbers[1]] + obj_second.append(abs(fit_second)) + ax.scatter(obj_first, obj_second, c='green') + + ax.scatter(pareto_obj_first, pareto_obj_second, c='red') + plt.plot(pareto_obj_first, pareto_obj_second, color='r') + + if generation_num is not None: + ax.set_title(f'Pareto frontier, Generation: {generation_num}', fontsize=15) + else: + ax.set_title('Pareto frontier', fontsize=15) + plt.xlabel(objectives_names[0], fontsize=15) + plt.ylabel(objectives_names[1], fontsize=15) + + if minmax_x is not None: + plt.xlim(minmax_x[0], minmax_x[1]) + if minmax_y is not None: + plt.ylim(minmax_y[0], minmax_y[1]) + fig.set_figwidth(8) + fig.set_figheight(8) + if save: + if not os.path.isdir('../../tmp'): + os.mkdir('../../tmp') + if not os.path.isdir(f'{folder}'): + os.mkdir(f'{folder}') + + path = f'{folder}/{file_name}' + plt.savefig(path, bbox_inches='tight') + if show: + plt.show() + + plt.cla() + plt.clf() + plt.close('all') + + +def create_gif_using_images(gif_path: str, files: List[str]): + with get_writer(gif_path, mode='I', duration=0.5) as writer: + for filename in files: + image = v2.imread(filename) + writer.append_data(image) + + +def extract_objectives(individuals: List[List[Any]], objectives_numbers: Tuple[int, ...] = None, + transform_from_minimization=True): + if not objectives_numbers: + objectives_numbers = [i for i in range(len(individuals[0][0].fitness.values))] + all_inds = list(itertools.chain(*individuals)) + all_objectives = [[ind.fitness.values[i] for ind in all_inds] for i in objectives_numbers] + if transform_from_minimization: + transformed_objectives = [] + for obj_values in all_objectives: + are_objectives_positive = all(np.array(obj_values) > 0) + if not are_objectives_positive: + transformed_obj_values = list(np.array(obj_values) * (-1)) + else: + transformed_obj_values = obj_values + transformed_objectives.append(transformed_obj_values) + else: + transformed_objectives = all_objectives + return transformed_objectives + + +def figure_to_array(fig): + img = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8) + img = img.reshape(fig.canvas.get_width_height()[::-1] + (3,)) + return img + + +def objectives_lists(individuals: List[Any], objectives_numbers: Tuple[int] = None): + num_of_objectives = len(objectives_numbers) if objectives_numbers else len(individuals[0].fitness.values) + objectives_numbers = objectives_numbers if objectives_numbers else [i for i in range(num_of_objectives)] + objectives_values_set = [[] for _ in range(num_of_objectives)] + for obj_num in range(num_of_objectives): + for individual in individuals: + value = individual.fitness.values[objectives_numbers[obj_num]] + objectives_values_set[obj_num].append(value if value > 0 else -value) + return objectives_values_set diff --git a/paper_experiments/large_bayesian_networks_experiments/golem_integration.ipynb b/paper_experiments/large_bayesian_networks_experiments/golem_integration.ipynb new file mode 100644 index 0000000..72990af --- /dev/null +++ b/paper_experiments/large_bayesian_networks_experiments/golem_integration.ipynb @@ -0,0 +1,4775 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# imports, data" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from datetime import timedelta\n", + "import os, sys\n", + "import random\n", + "import time\n", + "\n", + "cwd = os.getcwd()\n", + "\n", + "parentdir = os.path.dirname(cwd)\n", + "golemdir = parentdir + '/GOLEM'\n", + "bamtdir = parentdir + '/BAMT'\n", + "sys.path.insert(0, golemdir)\n", + "sys.path.insert(0, bamtdir)\n", + "\n", + "\n", + "import pandas as pd\n", + "\n", + "from pgmpy.estimators import K2Score\n", + "from pgmpy.models import BayesianNetwork\n", + "from golem.core.adapter import DirectAdapter\n", + "from golem.core.dag.convert import graph_structure_as_nx_graph\n", + "from golem.core.dag.graph_utils import ordered_subnodes_hierarchy\n", + "from golem.core.dag.verification_rules import has_no_cycle, has_no_self_cycled_nodes\n", + "from golem.core.optimisers.genetic.gp_optimizer import EvoGraphOptimizer\n", + "from golem.core.optimisers.genetic.gp_params import GPAlgorithmParameters\n", + "from golem.core.optimisers.genetic.operators.crossover import CrossoverTypesEnum\n", + "from golem.core.optimisers.genetic.operators.inheritance import GeneticSchemeTypesEnum\n", + "from golem.core.optimisers.genetic.operators.selection import SelectionTypesEnum\n", + "from golem.core.optimisers.graph import OptGraph, OptNode\n", + "from golem.core.optimisers.objective import Objective, ObjectiveEvaluate\n", + "from golem.core.optimisers.optimization_parameters import GraphRequirements\n", + "from golem.core.optimisers.optimizer import GraphGenerationParams\n", + "\n", + "\n", + "from divided_bn import DividedBN\n", + "import bamt.preprocessors as pp\n", + "from sklearn import preprocessing\n", + "\n", + "import matplotlib\n", + "matplotlib.rcParams['pdf.fonttype'] = 42\n", + "matplotlib.rcParams['ps.fonttype'] = 42\n", + "\n", + "import multiprocessing as mp\n", + "mp.set_start_method('fork')" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "pigs = pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/pigs.csv')\n", + "win95pts = pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/win95pts.csv')\n", + "hailfinder = pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/hailfinder.csv')\n", + "hepar2 = pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/hepar2.csv')\n", + "arth150 = pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/arth150.csv')\n", + "ecoli70 = pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/ecoli70.csv', index_col=0)\n", + "magic_irri = pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/magic-irri.csv', index_col=0)\n", + "magic_niab = pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/magic-niab.csv', index_col=0)\n", + "diabetes = pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/sparsebndata/main/data/diabetes.csv')\n", + "andes = pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/sparsebndata/main/data/andes.csv')\n", + "\n", + "pigs_true = pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/pigs_true.csv')\n", + "win95pts_true = pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/win95pts_true.csv')\n", + "hailfinder_true = pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/hailfinder_true.csv')\n", + "hepar2_true = pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/hepar2_true.csv')\n", + "arth150_true = pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/arth150_true.csv')\n", + "ecoli70_true = pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/ecoli70_true.csv')\n", + "magic_irri_true = pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/magic-irri_true.csv')\n", + "magic_niab_true = pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/magic-niab_true.csv')\n", + "andes_true = pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/andes_true.csv')\n", + "diabetes_true = pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/diabetes_true.csv')" + ], + "outputs": [], + "execution_count": null + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# extras, algorithm" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import math\n", + "\n", + "import numpy as np\n", + "import pandas as pd\n", + "from sklearn.metrics import mutual_info_score\n", + "from sklearn.preprocessing import OrdinalEncoder\n", + "\n", + "\n", + "class BigBraveBN:\n", + " def __init__(self):\n", + " self.possible_edges = []\n", + "\n", + " def set_possible_edges_by_brave(\n", + " self,\n", + " df: pd.DataFrame,\n", + " n_nearest: int = 5,\n", + " threshold: float = 0.3,\n", + " proximity_metric: str = \"MI\",\n", + " ) -> list:\n", + " \"\"\"Returns list of possible edges for structure learning and sets it into attribute\n", + "\n", + " Args:\n", + " df (pd.DataFrame): Data.\n", + " n_nearest (int): Number of nearest neighbors to consider. Default is 5.\n", + " threshold (float): Threshold for selecting edges. Default is 0.3.\n", + " proximity_metric (str): Metric used to calculate proximity. Default is \"MI\".\n", + "\n", + " Returns:\n", + " None: Modifies the object's possible_edges attribute.\n", + " \"\"\"\n", + " df_copy = df.copy(deep=True)\n", + " proximity_matrix = self._get_proximity_matrix(df_copy, proximity_metric)\n", + " brave_matrix = self._get_brave_matrix(df_copy.columns, proximity_matrix, n_nearest)\n", + "\n", + " threshold_value = brave_matrix.max(numeric_only=True).max() * threshold\n", + " filtered_brave_matrix = brave_matrix[brave_matrix > threshold_value].stack()\n", + " self.possible_edges = filtered_brave_matrix.index.tolist()\n", + " return self.possible_edges\n", + "\n", + " @staticmethod\n", + " def _get_n_nearest(\n", + " data: pd.DataFrame, columns: list, corr: bool = False, number_close: int = 5\n", + " ) -> list:\n", + " \"\"\"Returns N nearest neighbors for every column of dataframe.\"\"\"\n", + " groups = []\n", + " for c in columns:\n", + " close_ind = data[c].sort_values(ascending=not corr).index.tolist()\n", + " groups.append(close_ind[: number_close + 1])\n", + " return groups\n", + "\n", + " @staticmethod\n", + " def _get_proximity_matrix(df: pd.DataFrame, proximity_metric: str) -> pd.DataFrame:\n", + " \"\"\"Returns matrix of proximity for the dataframe.\"\"\"\n", + " encoder = OrdinalEncoder()\n", + " df_coded = df.copy()\n", + " columns_to_encode = list(df_coded.select_dtypes(include=[\"category\", \"object\"]))\n", + " df_coded[columns_to_encode] = encoder.fit_transform(df_coded[columns_to_encode])\n", + "\n", + " if proximity_metric == \"MI\":\n", + " df_distance = pd.DataFrame(\n", + " np.zeros((len(df.columns), len(df.columns))),\n", + " columns=df.columns,\n", + " index=df.columns,\n", + " )\n", + " for c1 in df.columns:\n", + " for c2 in df.columns:\n", + " dist = mutual_info_score(df_coded[c1].values, df_coded[c2].values)\n", + " df_distance.loc[c1, c2] = dist\n", + " return df_distance\n", + "\n", + " elif proximity_metric == \"pearson\":\n", + " return df_coded.corr(method=\"pearson\")\n", + "\n", + " def _get_brave_matrix(\n", + " self, df_columns: pd.Index, proximity_matrix: pd.DataFrame, n_nearest: int = 5\n", + " ) -> pd.DataFrame:\n", + " \"\"\"Returns matrix of Brave coefficients for the DataFrame.\"\"\"\n", + " brave_matrix = pd.DataFrame(\n", + " np.zeros((len(df_columns), len(df_columns))),\n", + " columns=df_columns,\n", + " index=df_columns,\n", + " )\n", + " groups = self._get_n_nearest(\n", + " proximity_matrix, df_columns.tolist(), corr=True, number_close=n_nearest\n", + " )\n", + "\n", + " for c1 in df_columns:\n", + " for c2 in df_columns:\n", + " a = b = c = d = 0.0\n", + " if c1 != c2:\n", + " for g in groups:\n", + " a += (c1 in g) & (c2 in g)\n", + " b += (c1 in g) & (c2 not in g)\n", + " c += (c1 not in g) & (c2 in g)\n", + " d += (c1 not in g) & (c2 not in g)\n", + "\n", + " divisor = (math.sqrt((a + c) * (b + d))) * (\n", + " math.sqrt((a + b) * (c + d))\n", + " )\n", + " br = (a * len(groups) + (a + c) * (a + b)) / (\n", + " divisor if divisor != 0 else 0.0000000001\n", + " )\n", + " brave_matrix.loc[c1, c2] = br\n", + "\n", + " return brave_matrix" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "def f1_score(custom_dag_edges, reference_dag_edges):\n", + " # Calculate True Positives (tp), False Positives (fp), and False Negatives (fn)\n", + " tp = fp = fn = 0\n", + " for edge in custom_dag_edges:\n", + " if edge in reference_dag_edges:\n", + " tp += 1\n", + " else:\n", + " fp += 1\n", + "\n", + " for edge in reference_dag_edges:\n", + " if edge not in custom_dag_edges:\n", + " fn += 1\n", + "\n", + " # Calculate Precision and Recall\n", + " if tp + fp == 0:\n", + " precision = 0\n", + " else:\n", + " precision = tp / (tp + fp)\n", + "\n", + " if tp + fn == 0:\n", + " recall = 0\n", + " else:\n", + " recall = tp / (tp + fn)\n", + "\n", + " # Calculate F1 Score\n", + " if precision + recall == 0:\n", + " f1 = 0\n", + " else:\n", + " f1 = 2 * precision * recall / (precision + recall)\n", + "\n", + " return f1\n", + "\n", + "def child_dict(net: list):\n", + " res_dict = dict()\n", + " for e0, e1 in net:\n", + " if e1 in res_dict:\n", + " res_dict[e1].append(e0)\n", + " else:\n", + " res_dict[e1] = [e0]\n", + " return res_dict\n", + "\n", + "def precision_recall(pred_net: list, true_net: list, decimal = 4):\n", + " pred_dict = child_dict(pred_net)\n", + " true_dict = child_dict(true_net)\n", + " corr_undir = 0\n", + " corr_dir = 0\n", + " for e0, e1 in pred_net:\n", + " flag = True\n", + " if e1 in true_dict:\n", + " if e0 in true_dict[e1]:\n", + " corr_undir += 1\n", + " corr_dir += 1\n", + " flag = False\n", + " if (e0 in true_dict) and flag:\n", + " if e1 in true_dict[e0]:\n", + " corr_undir += 1\n", + " pred_len = len(pred_net)\n", + " true_len = len(true_net)\n", + " shd = pred_len + true_len - corr_undir - corr_dir\n", + " return {'SHD': shd}\n", + "\n", + "def f1_score_undirected(custom_dag_edges, reference_dag_edges):\n", + " # Convert edges to sets (ignoring direction) and convert objects inside edges to strings\n", + " custom_dag_edges_set = {frozenset((str(a), str(b))) for a, b in custom_dag_edges}\n", + " reference_dag_edges_set = {frozenset((str(a), str(b))) for a, b in reference_dag_edges}\n", + "\n", + " # Calculate True Positives (tp), False Positives (fp), and False Negatives (fn)\n", + " tp = fp = fn = 0\n", + " for edge in custom_dag_edges_set:\n", + " if edge in reference_dag_edges_set:\n", + " tp += 1\n", + " else:\n", + " fp += 1\n", + "\n", + " for edge in reference_dag_edges_set:\n", + " if edge not in custom_dag_edges_set:\n", + " fn += 1\n", + "\n", + " # Calculate Precision and Recall\n", + " if tp + fp == 0:\n", + " precision = 0\n", + " else:\n", + " precision = tp / (tp + fp)\n", + "\n", + " if tp + fn == 0:\n", + " recall = 0\n", + " else:\n", + " recall = tp / (tp + fn)\n", + "\n", + " # Calculate F1 Score\n", + " if precision + recall == 0:\n", + " f1 = 0\n", + " else:\n", + " f1 = 2 * precision * recall / (precision + recall)\n", + "\n", + " return f1\n", + "\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "class CustomGraphModel(OptGraph):\n", + " def evaluate(self, data: pd.DataFrame):\n", + " nodes = data.columns.to_list()\n", + " _, labels = graph_structure_as_nx_graph(self)\n", + " return len(nodes)\n", + "\n", + "\n", + "class CustomGraphNode(OptNode):\n", + " def __str__(self):\n", + " return f'{self.content[\"name\"]}'\n", + "\n", + "\n", + "def custom_metric(graph: CustomGraphModel, data: pd.DataFrame):\n", + " graph_nx, labels = graph_structure_as_nx_graph(graph)\n", + " struct = []\n", + " for meta_edge in graph_nx.edges():\n", + " l1 = str(labels[meta_edge[0]])\n", + " l2 = str(labels[meta_edge[1]])\n", + " struct.append([l1, l2])\n", + "\n", + " bn_model = BayesianNetwork(struct)\n", + " bn_model.add_nodes_from(data.columns)\n", + "\n", + " score = K2Score(data).score(bn_model)\n", + " return -score\n", + "\n", + "\n", + "def custom_mutation_add(graph: CustomGraphModel, **kwargs):\n", + " num_mut = 100\n", + " try:\n", + " for _ in range(num_mut):\n", + " rid = random.choice(range(len(graph.nodes)))\n", + " random_node = graph.nodes[rid]\n", + " other_random_node = graph.nodes[random.choice(range(len(graph.nodes)))]\n", + " nodes_not_cycling = (random_node.descriptive_id not in\n", + " [n.descriptive_id for n in ordered_subnodes_hierarchy(other_random_node)] and\n", + " other_random_node.descriptive_id not in\n", + " [n.descriptive_id for n in ordered_subnodes_hierarchy(random_node)])\n", + " if nodes_not_cycling:\n", + " random_node.nodes_from.append(other_random_node)\n", + " break\n", + "\n", + " except Exception as ex:\n", + " print(f'Incorrect connection: {ex}')\n", + " return graph\n", + "\n", + "\n", + "def custom_mutation_delete(graph: OptGraph, **kwargs):\n", + " num_mut = 100\n", + " try:\n", + " for _ in range(num_mut):\n", + " rid = random.choice(range(len(graph.nodes)))\n", + " random_node = graph.nodes[rid]\n", + " other_random_node = graph.nodes[random.choice(range(len(graph.nodes)))]\n", + " if random_node.nodes_from is not None and other_random_node in random_node.nodes_from:\n", + " random_node.nodes_from.remove(other_random_node)\n", + " break\n", + " except Exception as ex:\n", + " print(ex)\n", + " return graph\n", + "\n", + "\n", + "def custom_mutation_reverse(graph: OptGraph, **kwargs):\n", + " num_mut = 100\n", + " try:\n", + " for _ in range(num_mut):\n", + " rid = random.choice(range(len(graph.nodes)))\n", + " random_node = graph.nodes[rid]\n", + " other_random_node = graph.nodes[random.choice(range(len(graph.nodes)))]\n", + " if random_node.nodes_from is not None and other_random_node in random_node.nodes_from:\n", + " random_node.nodes_from.remove(other_random_node)\n", + " other_random_node.nodes_from.append(random_node)\n", + " break\n", + " except Exception as ex:\n", + " print(ex)\n", + " return graph\n", + "\n", + "\n", + "def _has_no_duplicates(graph):\n", + " _, labels = graph_structure_as_nx_graph(graph)\n", + " if len(labels.values()) != len(set(labels.values())):\n", + " raise ValueError('Custom graph has duplicates')\n", + " return True\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "def get_edges_by_localstructures(data,\n", + " datatype=\"discrete\",\n", + " max_local_structures=8,\n", + " hidden_nodes_clusters=None,\n", + " time_m=5,\n", + " history_dir=golemdir):\n", + "\n", + " data.dropna(inplace=True)\n", + " data.reset_index(inplace=True, drop=True)\n", + "\n", + " initial_df = data.copy()\n", + "\n", + " # initialize divided_bn\n", + "\n", + " start_time = time.time()\n", + "\n", + " divided_bn = DividedBN(data=data,\n", + " data_type=datatype,\n", + " max_local_structures=max_local_structures,\n", + " hidden_nodes_clusters=hidden_nodes_clusters)\n", + "\n", + " divided_bn.set_local_structures()\n", + "\n", + " local_edges = divided_bn.local_structures_edges\n", + "\n", + " divided_bn.set_hidden_nodes(data=data)\n", + "\n", + " hidden_df = pd.DataFrame.from_dict(divided_bn.hidden_nodes)\n", + "\n", + " hidden_df.columns = hidden_df.columns.astype(str)\n", + "\n", + " root_nodes = divided_bn.root_nodes\n", + " child_nodes = divided_bn.child_nodes\n", + "\n", + " vertices = list(hidden_df.columns)\n", + "\n", + " encoder = preprocessing.LabelEncoder()\n", + " discretizer = preprocessing.KBinsDiscretizer(n_bins=5, encode='ordinal', strategy='quantile')\n", + " p = pp.Preprocessor([('encoder', encoder), ('discretizer', discretizer)])\n", + " discretized_data, _ = p.apply(hidden_df)\n", + "\n", + " objective = Objective({'custom': custom_metric})\n", + " objective_eval = ObjectiveEvaluate(objective, data=discretized_data)\n", + "\n", + " initial = [CustomGraphModel(nodes=[CustomGraphNode(nodes_from=None,\n", + " content={'name': vertex,\n", + " 'type': p.nodes_types[vertex],\n", + " 'parent_model': None}) \n", + " for vertex in vertices])]\n", + " init = initial[0]\n", + "\n", + " requirements = GraphRequirements(\n", + " max_arity=20,\n", + " max_depth=20,\n", + " num_of_generations=75,\n", + " timeout=timedelta(minutes=time_m),\n", + " n_jobs=-1)\n", + "\n", + " optimizer_parameters = GPAlgorithmParameters(\n", + " pop_size=20,\n", + " crossover_prob=0.8,\n", + " mutation_prob=0.9,\n", + " genetic_scheme_type=GeneticSchemeTypesEnum.steady_state,\n", + " mutation_types=[custom_mutation_add, custom_mutation_delete, custom_mutation_reverse],\n", + " crossover_types=[CrossoverTypesEnum.exchange_edges,\n", + " CrossoverTypesEnum.exchange_parents_one,\n", + " CrossoverTypesEnum.exchange_parents_both],\n", + " selection_types=[SelectionTypesEnum.tournament])\n", + "\n", + " adapter = DirectAdapter(base_graph_class=CustomGraphModel, base_node_class=CustomGraphNode)\n", + " constraints = [has_no_self_cycled_nodes, has_no_cycle, _has_no_duplicates]\n", + " graph_generation_params = GraphGenerationParams(\n", + " adapter=adapter,\n", + " rules_for_constraint=constraints)\n", + "\n", + " optimiser = EvoGraphOptimizer(\n", + " objective=objective,\n", + " requirements=requirements,\n", + " initial_graphs=[init],\n", + " graph_generation_params=graph_generation_params,\n", + " graph_optimizer_params=optimizer_parameters\n", + " )\n", + "\n", + "\n", + " optimized_graph = optimiser.optimise(objective_eval)[0]\n", + "\n", + " print(\"--- %s seconds ---\" % (time.time() - start_time))\n", + "\n", + " evolutionary_edges = optimized_graph.operator.get_edges()\n", + "\n", + " print(evolutionary_edges)\n", + "\n", + " local_edges_merged = []\n", + "\n", + " for key in local_edges:\n", + " local_edges_merged += local_edges[key]\n", + "\n", + " # external_edges = divided_bn.connect_structures_simple(evolutionary_edges)\n", + "\n", + " # external_edges = divided_bn.connect_structures_hc(evolutionary_edges)\n", + "\n", + " external_edges = divided_bn.connect_structures_spearman(evolutionary_edges, percentile_threshold=95)\n", + "\n", + " external_edges_merged = []\n", + "\n", + " for key in external_edges:\n", + " external_edges_merged += external_edges[key]\n", + "\n", + " all_edges = local_edges_merged + external_edges_merged\n", + "\n", + " def remove_duplicates(input_list):\n", + " unique_list = []\n", + " for item in input_list:\n", + " if item not in unique_list:\n", + " unique_list.append(item)\n", + " return unique_list\n", + "\n", + "\n", + " def convert_to_strings(nested_list):\n", + " return [[str(item) for item in inner_list] for inner_list in nested_list]\n", + "\n", + " return remove_duplicates(convert_to_strings(all_edges))\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "evo, evo_edges, loc_edges = get_edges_by_localstructures(data=diabetes, datatype=\"discrete\", hidden_nodes_clusters=8, max_local_structures=10)\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "local_structures = {\n", + " 2: [\n", + " [\"ins_sens\", \"activ_ins_4\"],\n", + " [\"ins_sens\", \"ins_abs_5\"],\n", + " [\"ins_sens\", \"ins_indep_3\"],\n", + " [\"ins_sens\", \"ins_abs_1\"],\n", + " [\"ins_sens\", \"ins_indep_util_2\"],\n", + " [\"bg_1\", \"renal_cl_1\"],\n", + " [\"renal_cl_1\", \"meal_0\"],\n", + " [\"renal_cl_1\", \"cho_init\"],\n", + " [\"renal_cl_1\", \"ins_abs_1\"],\n", + " [\"renal_cl_2\", \"endo_bal_2\"],\n", + " [\"renal_cl_2\", \"glu_prod_2\"],\n", + " [\"renal_cl_2\", \"renal_cl_1\"],\n", + " [\"renal_cl_2\", \"bg_1\"],\n", + " [\"renal_cl_2\", \"ins_abs_1\"],\n", + " [\"renal_cl_2\", \"meal_0\"],\n", + " [\"renal_cl_2\", \"ins_dep_util_2\"],\n", + " [\"renal_cl_2\", \"cho_init\"],\n", + " [\"ins_indep_util_2\", \"glu_prod_2\"],\n", + " [\"ins_indep_util_2\", \"renal_cl_2\"],\n", + " [\"ins_indep_util_2\", \"ins_abs_1\"],\n", + " [\"ins_indep_util_2\", \"endo_bal_2\"],\n", + " [\"ins_indep_util_2\", \"bg_1\"],\n", + " [\"ins_indep_util_2\", \"ins_dep_util_2\"],\n", + " [\"ins_dep_util_2\", \"endo_bal_2\"],\n", + " [\"ins_dep_util_2\", \"glu_prod_2\"],\n", + " [\"endo_bal_2\", \"glu_prod_2\"],\n", + " [\"ins_indep_3\", \"ins_dep_3\"],\n", + " [\"ins_indep_3\", \"bg_4\"],\n", + " [\"ins_indep_3\", \"ins_indep_util_2\"],\n", + " [\"ins_indep_3\", \"ins_dep_util_2\"],\n", + " [\"ins_indep_3\", \"renal_cl_2\"],\n", + " [\"endo_bal_3\", \"ins_dep_3\"],\n", + " [\"endo_bal_3\", \"ins_indep_3\"],\n", + " [\"cho_bal_3\", \"cho_4\"],\n", + " [\"cho_bal_3\", \"endo_bal_3\"],\n", + " [\"cho_bal_3\", \"meal_1\"],\n", + " [\"cho_bal_3\", \"basal_bal_3\"],\n", + " [\"basal_bal_3\", \"tot_bal_3\"],\n", + " [\"basal_bal_3\", \"endo_bal_3\"],\n", + " [\"met_irr_3\", \"tot_bal_3\"],\n", + " [\"tot_bal_3\", \"bg_4\"],\n", + " [\"meal_4\", \"cho_4\"],\n", + " [\"meal_4\", \"cho_bal_3\"],\n", + " [\"bg_4\", \"ins_indep_util_5\"],\n", + " [\"bg_4\", \"ins_indep_5\"],\n", + " [\"bg_4\", \"basal_bal_4\"],\n", + " [\"activ_ins_4\", \"basal_bal_4\"],\n", + " [\"activ_ins_4\", \"ins_indep_util_5\"],\n", + " [\"ins_abs_4\", \"activ_ins_4\"],\n", + " [\"basal_bal_4\", \"tot_bal_4\"],\n", + " [\"met_irr_4\", \"tot_bal_4\"],\n", + " [\"tot_bal_4\", \"ins_indep_5\"],\n", + " [\"cho_5\", \"cho_bal_3\"],\n", + " [\"cho_5\", \"basal_bal_5\"],\n", + " [\"cho_5\", \"meal_4\"],\n", + " [\"ins_abs_5\", \"endo_bal_6\"],\n", + " [\"renal_cl_5\", \"basal_bal_5\"],\n", + " [\"renal_cl_5\", \"endo_bal_6\"],\n", + " [\"renal_cl_5\", \"bg_7\"],\n", + " [\"ins_indep_util_5\", \"basal_bal_5\"],\n", + " [\"ins_indep_util_5\", \"bg_7\"],\n", + " [\"ins_dep_util_5\", \"glu_prod_5\"],\n", + " [\"ins_dep_util_5\", \"basal_bal_5\"],\n", + " [\"ins_dep_util_5\", \"ins_abs_5\"],\n", + " [\"ins_dep_util_5\", \"ins_abs_4\"],\n", + " [\"ins_dep_util_5\", \"ins_sens\"],\n", + " [\"glu_prod_5\", \"ins_abs_5\"],\n", + " [\"glu_prod_5\", \"ins_sens\"],\n", + " [\"glu_prod_5\", \"basal_bal_5\"],\n", + " [\"ins_indep_5\", \"renal_cl_5\"],\n", + " [\"ins_indep_5\", \"bg_7\"],\n", + " [\"ins_indep_5\", \"ins_abs_5\"],\n", + " [\"ins_indep_5\", \"ins_indep_util_5\"],\n", + " [\"basal_bal_5\", \"tot_bal_5\"],\n", + " [\"met_irr_5\", \"tot_bal_5\"],\n", + " [\"tot_bal_5\", \"bg_7\"],\n", + " [\"meal_6\", \"cho_5\"],\n", + " [\"cho_6\", \"cho_5\"],\n", + " [\"cho_6\", \"meal_6\"],\n", + " [\"cho_6\", \"tot_bal_6\"],\n", + " [\"endo_bal_6\", \"tot_bal_6\"],\n", + " [\"tot_bal_6\", \"bg_7\"],\n", + " [\"meal_7\", \"cho_7\"],\n", + " [\"meal_7\", \"cho_6\"],\n", + " [\"cho_7\", \"cho_6\"],\n", + " [\"cho_7\", \"cho_9\"],\n", + " [\"cho_7\", \"tot_bal_7\"],\n", + " [\"met_irr_8\", \"tot_bal_8\"],\n", + " [\"cho_9\", \"basal_bal_9\"],\n", + " [\"cho_9\", \"meal_9\"],\n", + " ],\n", + " 1: [\n", + " [\"cho_0\", \"gut_abs_1\"],\n", + " [\"gut_abs_1\", \"basal_bal_1\"],\n", + " [\"gut_abs_1\", \"ins_indep_2\"],\n", + " [\"gut_abs_1\", \"gut_abs_5\"],\n", + " [\"basal_bal_1\", \"ins_indep_2\"],\n", + " [\"ins_indep_2\", \"ins_dep_2\"],\n", + " [\"ins_indep_2\", \"bg_5\"],\n", + " [\"ins_indep_2\", \"renal_cl_18\"],\n", + " [\"gut_abs_5\", \"bg_5\"],\n", + " [\"gut_abs_7\", \"gut_abs_5\"],\n", + " [\"tot_bal_9\", \"met_irr_9\"],\n", + " [\"renal_cl_11\", \"bg_11\"],\n", + " [\"renal_cl_11\", \"tot_bal_9\"],\n", + " [\"renal_cl_11\", \"gut_abs_7\"],\n", + " [\"ins_indep_util_12\", \"renal_cl_11\"],\n", + " [\"ins_indep_util_12\", \"ins_indep_13\"],\n", + " [\"ins_indep_util_12\", \"bg_11\"],\n", + " [\"ins_indep_util_12\", \"renal_cl_12\"],\n", + " [\"endo_bal_14\", \"ins_dep_14\"],\n", + " [\"cho_bal_14\", \"gut_abs_15\"],\n", + " [\"cho_bal_14\", \"gut_abs_12\"],\n", + " [\"tot_bal_17\", \"bg_17\"],\n", + " [\"tot_bal_17\", \"activ_ins_17\"],\n", + " [\"bg_18\", \"tot_bal_17\"],\n", + " [\"bg_18\", \"activ_ins_17\"],\n", + " [\"bg_18\", \"bg_17\"],\n", + " [\"bg_18\", \"bg_19\"],\n", + " [\"bg_18\", \"glu_prod_18\"],\n", + " [\"renal_cl_18\", \"bg_18\"],\n", + " [\"renal_cl_18\", \"activ_ins_11\"],\n", + " [\"renal_cl_18\", \"ins_dep_2\"],\n", + " [\"bg_19\", \"glu_prod_18\"],\n", + " [\"activ_ins_19\", \"tot_bal_20\"],\n", + " [\"meal_21\", \"cho_21\"],\n", + " [\"meal_21\", \"cho_19\"],\n", + " [\"cho_21\", \"cho_19\"],\n", + " [\"cho_21\", \"ins_indep_util_22\"],\n", + " [\"ins_indep_util_22\", \"renal_cl_18\"],\n", + " [\"ins_indep_util_22\", \"tot_bal_20\"],\n", + " [\"ins_indep_util_22\", \"ins_dep_util_22\"],\n", + " [\"ins_indep_util_22\", \"glu_prod_22\"],\n", + " [\"ins_indep_util_22\", \"activ_ins_19\"],\n", + " [\"ins_dep_util_22\", \"glu_prod_22\"],\n", + " [\"glu_prod_22\", \"tot_bal_23\"],\n", + " [\"tot_bal_10\", \"met_irr_10\"],\n", + " [\"tot_bal_10\", \"tot_bal_9\"],\n", + " [\"bg_11\", \"ins_indep_util_11\"],\n", + " [\"bg_11\", \"tot_bal_10\"],\n", + " [\"bg_11\", \"tot_bal_9\"],\n", + " [\"activ_ins_11\", \"renal_cl_12\"],\n", + " [\"activ_ins_11\", \"ins_indep_util_12\"],\n", + " [\"activ_ins_11\", \"activ_ins_5\"],\n", + " [\"ins_abs_11\", \"activ_ins_11\"],\n", + " [\"ins_abs_11\", \"activ_ins_5\"],\n", + " [\"ins_abs_11\", \"renal_cl_11\"],\n", + " [\"gut_abs_11\", \"meal_11\"],\n", + " [\"gut_abs_11\", \"renal_cl_12\"],\n", + " [\"gut_abs_11\", \"ins_indep_util_12\"],\n", + " [\"gut_abs_11\", \"gut_abs_7\"],\n", + " [\"activ_ins_12\", \"activ_ins_19\"],\n", + " [\"gut_abs_12\", \"gut_abs_11\"],\n", + " [\"gut_abs_12\", \"activ_ins_12\"],\n", + " [\"gut_abs_12\", \"endo_bal_13\"],\n", + " [\"gut_abs_12\", \"endo_bal_14\"],\n", + " [\"renal_cl_12\", \"renal_cl_11\"],\n", + " [\"renal_cl_12\", \"ins_indep_13\"],\n", + " [\"renal_cl_12\", \"bg_11\"],\n", + " [\"renal_cl_12\", \"activ_ins_12\"],\n", + " [\"renal_cl_12\", \"met_irr_12\"],\n", + " [\"renal_cl_12\", \"ins_dep_util_22\"],\n", + " [\"renal_cl_12\", \"bg_18\"],\n", + " [\"ins_indep_13\", \"ins_dep_13\"],\n", + " [\"ins_indep_13\", \"activ_ins_12\"],\n", + " [\"ins_indep_13\", \"ins_dep_14\"],\n", + " [\"ins_indep_13\", \"met_irr_12\"],\n", + " [\"ins_dep_13\", \"glu_prod_13\"],\n", + " [\"endo_bal_13\", \"ins_dep_13\"],\n", + " [\"endo_bal_13\", \"ins_indep_13\"],\n", + " [\"gut_abs_15\", \"renal_cl_18\"],\n", + " ],\n", + " 0: [\n", + " [\"activ_ins_0\", \"ins_abs_0\"],\n", + " [\"gut_abs_0\", \"ins_dep_0\"],\n", + " [\"gut_abs_0\", \"ins_indep_1\"],\n", + " [\"renal_cl_0\", \"ins_dep_0\"],\n", + " [\"renal_cl_0\", \"ins_indep_util_0\"],\n", + " [\"renal_cl_0\", \"activ_ins_0\"],\n", + " [\"ins_indep_util_0\", \"activ_ins_0\"],\n", + " [\"ins_dep_0\", \"activ_ins_0\"],\n", + " [\"ins_dep_0\", \"ins_indep_util_0\"],\n", + " [\"ins_indep_util_1\", \"ins_dep_0\"],\n", + " [\"ins_dep_util_1\", \"glu_prod_1\"],\n", + " [\"ins_indep_1\", \"ins_dep_1\"],\n", + " [\"ins_indep_1\", \"ins_dep_0\"],\n", + " [\"ins_indep_1\", \"endo_bal_1\"],\n", + " [\"ins_indep_1\", \"ins_indep_util_1\"],\n", + " [\"ins_indep_1\", \"renal_cl_0\"],\n", + " [\"ins_indep_1\", \"ins_indep_util_0\"],\n", + " [\"ins_dep_1\", \"glu_prod_1\"],\n", + " [\"ins_dep_1\", \"ins_dep_util_1\"],\n", + " [\"endo_bal_1\", \"ins_dep_1\"],\n", + " [\"cho_bal_1\", \"cho_bal_2\"],\n", + " [\"cho_bal_1\", \"gut_abs_0\"],\n", + " [\"cho_bal_2\", \"cho_3\"],\n", + " [\"cho_bal_2\", \"basal_bal_2\"],\n", + " [\"basal_bal_2\", \"tot_bal_2\"],\n", + " [\"met_irr_2\", \"tot_bal_2\"],\n", + " [\"ins_abs_9\", \"activ_ins_9\"],\n", + " [\"gut_abs_9\", \"bg_9\"],\n", + " [\"gut_abs_9\", \"meal_5\"],\n", + " [\"cho_11\", \"gut_abs_9\"],\n", + " [\"cho_11\", \"ins_indep_12\"],\n", + " [\"ins_dep_util_12\", \"ins_dep_12\"],\n", + " [\"ins_dep_util_12\", \"glu_prod_12\"],\n", + " [\"ins_indep_12\", \"basal_bal_12\"],\n", + " [\"ins_indep_12\", \"bg_9\"],\n", + " [\"ins_indep_12\", \"ins_dep_util_12\"],\n", + " [\"ins_dep_12\", \"glu_prod_12\"],\n", + " [\"ins_dep_12\", \"basal_bal_12\"],\n", + " [\"cho_bal_13\", \"cho_11\"],\n", + " [\"cho_bal_13\", \"basal_bal_13\"],\n", + " [\"basal_bal_13\", \"tot_bal_13\"],\n", + " [\"met_irr_13\", \"tot_bal_13\"],\n", + " [\"ins_abs_16\", \"renal_cl_16\"],\n", + " [\"renal_cl_16\", \"activ_ins_20\"],\n", + " [\"renal_cl_16\", \"cho_bal_13\"],\n", + " [\"renal_cl_16\", \"ins_abs_0\"],\n", + " [\"renal_cl_16\", \"meal_14\"],\n", + " [\"renal_cl_16\", \"activ_ins_9\"],\n", + " [\"renal_cl_16\", \"ins_indep_1\"],\n", + " [\"renal_cl_16\", \"ins_indep_12\"],\n", + " [\"renal_cl_16\", \"met_irr_14\"],\n", + " [\"renal_cl_16\", \"endo_bal_1\"],\n", + " [\"renal_cl_16\", \"endo_bal_22\"],\n", + " [\"renal_cl_16\", \"ins_dep_5\"],\n", + " [\"endo_bal_16\", \"ins_abs_16\"],\n", + " [\"endo_bal_16\", \"renal_cl_16\"],\n", + " [\"activ_ins_20\", \"bg_21\"],\n", + " [\"ins_abs_20\", \"activ_ins_20\"],\n", + " [\"ins_abs_20\", \"renal_cl_20\"],\n", + " [\"gut_abs_20\", \"bg_21\"],\n", + " [\"gut_abs_20\", \"activ_ins_21\"],\n", + " [\"renal_cl_20\", \"activ_ins_9\"],\n", + " [\"renal_cl_20\", \"ins_dep_5\"],\n", + " [\"renal_cl_20\", \"ins_dep_util_12\"],\n", + " [\"bg_21\", \"ins_indep_22\"],\n", + " [\"bg_21\", \"renal_cl_20\"],\n", + " [\"cho_22\", \"gut_abs_20\"],\n", + " [\"bg_22\", \"activ_ins_21\"],\n", + " [\"ins_indep_22\", \"bg_22\"],\n", + " [\"ins_indep_22\", \"ins_dep_22\"],\n", + " [\"ins_indep_22\", \"endo_bal_22\"],\n", + " [\"endo_bal_22\", \"ins_dep_22\"],\n", + " ],\n", + " 7: [\n", + " [\"ins_dep_util_0\", \"endo_bal_0\"],\n", + " [\"ins_dep_util_0\", \"glu_prod_0\"],\n", + " [\"ins_dep_util_0\", \"ins_indep_0\"],\n", + " [\"glu_prod_0\", \"tot_bal_1\"],\n", + " [\"glu_prod_0\", \"ins_indep_0\"],\n", + " [\"endo_bal_0\", \"glu_prod_0\"],\n", + " [\"endo_bal_0\", \"ins_indep_0\"],\n", + " [\"cho_bal_0\", \"cho_2\"],\n", + " [\"activ_ins_1\", \"bg_2\"],\n", + " [\"tot_bal_1\", \"met_irr_1\"],\n", + " [\"meal_2\", \"cho_2\"],\n", + " [\"cho_2\", \"gut_abs_2\"],\n", + " [\"cho_2\", \"gut_abs_3\"],\n", + " [\"bg_2\", \"tot_bal_1\"],\n", + " [\"bg_2\", \"renal_cl_3\"],\n", + " [\"bg_2\", \"ins_dep_util_0\"],\n", + " [\"bg_2\", \"ins_abs_2\"],\n", + " [\"bg_2\", \"bg_3\"],\n", + " [\"ins_abs_2\", \"activ_ins_2\"],\n", + " [\"gut_abs_2\", \"bg_2\"],\n", + " [\"gut_abs_2\", \"renal_cl_3\"],\n", + " [\"meal_3\", \"gut_abs_3\"],\n", + " [\"bg_3\", \"ins_indep_util_3\"],\n", + " [\"bg_3\", \"glu_prod_3\"],\n", + " [\"bg_3\", \"ins_abs_2\"],\n", + " [\"activ_ins_3\", \"glu_prod_3\"],\n", + " [\"activ_ins_3\", \"ins_dep_util_3\"],\n", + " [\"ins_abs_3\", \"activ_ins_3\"],\n", + " [\"gut_abs_3\", \"cho_bal_5\"],\n", + " [\"gut_abs_3\", \"endo_bal_5\"],\n", + " [\"renal_cl_3\", \"bg_3\"],\n", + " [\"renal_cl_3\", \"ins_dep_util_3\"],\n", + " [\"renal_cl_3\", \"activ_ins_2\"],\n", + " [\"renal_cl_3\", \"activ_ins_13\"],\n", + " [\"renal_cl_3\", \"activ_ins_3\"],\n", + " [\"ins_indep_util_3\", \"ins_dep_util_3\"],\n", + " [\"cho_bal_5\", \"ins_indep_9\"],\n", + " [\"renal_cl_9\", \"activ_ins_10\"],\n", + " [\"renal_cl_9\", \"bg_13\"],\n", + " [\"renal_cl_9\", \"renal_cl_13\"],\n", + " [\"renal_cl_9\", \"endo_bal_11\"],\n", + " [\"renal_cl_9\", \"bg_10\"],\n", + " [\"renal_cl_9\", \"endo_bal_9\"],\n", + " [\"ins_indep_util_9\", \"bg_10\"],\n", + " [\"ins_dep_util_9\", \"glu_prod_9\"],\n", + " [\"ins_dep_util_9\", \"cho_bal_9\"],\n", + " [\"ins_indep_9\", \"ins_dep_9\"],\n", + " [\"ins_indep_9\", \"renal_cl_9\"],\n", + " [\"ins_indep_9\", \"ins_indep_util_9\"],\n", + " [\"ins_indep_9\", \"bg_10\"],\n", + " [\"ins_indep_9\", \"endo_bal_9\"],\n", + " [\"ins_dep_9\", \"glu_prod_9\"],\n", + " [\"ins_dep_9\", \"ins_dep_util_9\"],\n", + " [\"endo_bal_9\", \"ins_dep_9\"],\n", + " [\"endo_bal_9\", \"bg_10\"],\n", + " [\"bg_10\", \"cho_bal_9\"],\n", + " [\"bg_10\", \"renal_cl_13\"],\n", + " [\"activ_ins_10\", \"activ_ins_1\"],\n", + " [\"ins_abs_10\", \"activ_ins_10\"],\n", + " [\"ins_abs_10\", \"activ_ins_1\"],\n", + " [\"ins_dep_11\", \"ins_abs_10\"],\n", + " [\"endo_bal_11\", \"ins_dep_11\"],\n", + " [\"endo_bal_11\", \"ins_abs_10\"],\n", + " [\"endo_bal_12\", \"ins_abs_12\"],\n", + " [\"cho_bal_12\", \"cho_13\"],\n", + " [\"cho_bal_12\", \"endo_bal_12\"],\n", + " [\"tot_bal_12\", \"endo_bal_12\"],\n", + " [\"meal_13\", \"cho_13\"],\n", + " [\"cho_13\", \"gut_abs_13\"],\n", + " [\"cho_13\", \"cho_14\"],\n", + " [\"bg_13\", \"tot_bal_12\"],\n", + " [\"bg_13\", \"cho_bal_12\"],\n", + " [\"ins_abs_13\", \"activ_ins_13\"],\n", + " [\"renal_cl_13\", \"bg_13\"],\n", + " [\"renal_cl_13\", \"ins_abs_12\"],\n", + " [\"renal_cl_13\", \"activ_ins_10\"],\n", + " [\"renal_cl_13\", \"activ_ins_3\"],\n", + " [\"renal_cl_13\", \"ins_dep_11\"],\n", + " [\"renal_cl_13\", \"activ_ins_13\"],\n", + " [\"renal_cl_13\", \"activ_ins_2\"],\n", + " [\"renal_cl_13\", \"endo_bal_11\"],\n", + " ],\n", + " 8: [\n", + " [\"glu_prod_6\", \"glu_prod_17\"],\n", + " [\"cho_bal_8\", \"cho_bal_10\"],\n", + " [\"basal_bal_8\", \"cho_bal_8\"],\n", + " [\"cho_bal_10\", \"basal_bal_10\"],\n", + " ],\n", + " 4: [\n", + " [\"gut_abs_4\", \"gut_abs_6\"],\n", + " [\"renal_cl_4\", \"gut_abs_4\"],\n", + " [\"renal_cl_4\", \"ins_dep_util_8\"],\n", + " [\"renal_cl_4\", \"ins_abs_6\"],\n", + " [\"renal_cl_4\", \"ins_dep_util_10\"],\n", + " [\"ins_indep_4\", \"renal_cl_4\"],\n", + " [\"ins_indep_4\", \"ins_indep_util_4\"],\n", + " [\"ins_indep_4\", \"glu_prod_4\"],\n", + " [\"activ_ins_6\", \"ins_abs_6\"],\n", + " [\"activ_ins_6\", \"ins_indep_7\"],\n", + " [\"gut_abs_6\", \"ins_indep_7\"],\n", + " [\"gut_abs_6\", \"gut_abs_8\"],\n", + " [\"gut_abs_6\", \"renal_cl_8\"],\n", + " [\"ins_indep_6\", \"activ_ins_6\"],\n", + " [\"ins_indep_6\", \"gut_abs_4\"],\n", + " [\"ins_indep_6\", \"ins_indep_4\"],\n", + " [\"ins_indep_6\", \"ins_dep_6\"],\n", + " [\"ins_indep_6\", \"glu_prod_4\"],\n", + " [\"ins_indep_6\", \"ins_indep_7\"],\n", + " [\"ins_dep_6\", \"activ_ins_6\"],\n", + " [\"activ_ins_7\", \"ins_abs_7\"],\n", + " [\"activ_ins_7\", \"renal_cl_8\"],\n", + " [\"activ_ins_7\", \"ins_indep_util_8\"],\n", + " [\"glu_prod_7\", \"activ_ins_7\"],\n", + " [\"ins_indep_7\", \"activ_ins_7\"],\n", + " [\"ins_indep_7\", \"renal_cl_8\"],\n", + " [\"ins_indep_7\", \"ins_indep_util_8\"],\n", + " [\"ins_indep_7\", \"ins_dep_7\"],\n", + " [\"ins_dep_7\", \"glu_prod_7\"],\n", + " [\"ins_dep_7\", \"activ_ins_7\"],\n", + " [\"gut_abs_8\", \"renal_cl_10\"],\n", + " [\"renal_cl_8\", \"renal_cl_10\"],\n", + " [\"renal_cl_8\", \"glu_prod_8\"],\n", + " [\"renal_cl_8\", \"ins_indep_util_8\"],\n", + " [\"ins_indep_util_8\", \"ins_dep_util_8\"],\n", + " [\"ins_indep_util_8\", \"glu_prod_8\"],\n", + " [\"ins_indep_util_8\", \"renal_cl_10\"],\n", + " [\"ins_dep_util_8\", \"glu_prod_8\"],\n", + " [\"glu_prod_8\", \"renal_cl_10\"],\n", + " [\"gut_abs_10\", \"gut_abs_8\"],\n", + " [\"gut_abs_10\", \"gut_abs_6\"],\n", + " [\"renal_cl_10\", \"ins_dep_util_10\"],\n", + " [\"cho_bal_11\", \"gut_abs_10\"],\n", + " [\"basal_bal_11\", \"tot_bal_11\"],\n", + " [\"basal_bal_11\", \"cho_bal_11\"],\n", + " [\"met_irr_11\", \"tot_bal_11\"],\n", + " [\"bg_15\", \"activ_ins_15\"],\n", + " [\"activ_ins_15\", \"ins_abs_15\"],\n", + " [\"renal_cl_15\", \"ins_abs_15\"],\n", + " [\"renal_cl_15\", \"basal_bal_11\"],\n", + " [\"renal_cl_15\", \"ins_dep_util_10\"],\n", + " [\"renal_cl_15\", \"ins_dep_6\"],\n", + " [\"renal_cl_15\", \"ins_indep_4\"],\n", + " [\"renal_cl_15\", \"bg_15\"],\n", + " [\"ins_indep_util_15\", \"bg_15\"],\n", + " [\"ins_dep_util_15\", \"ins_dep_15\"],\n", + " [\"ins_dep_util_15\", \"activ_ins_15\"],\n", + " [\"ins_dep_util_15\", \"ins_indep_15\"],\n", + " [\"ins_indep_15\", \"bg_15\"],\n", + " [\"ins_indep_15\", \"renal_cl_15\"],\n", + " [\"ins_indep_15\", \"ins_indep_util_15\"],\n", + " [\"ins_indep_15\", \"ins_indep_6\"],\n", + " [\"ins_dep_15\", \"activ_ins_15\"],\n", + " [\"ins_dep_15\", \"ins_indep_15\"],\n", + " [\"cho_17\", \"gut_abs_17\"],\n", + " [\"gut_abs_17\", \"renal_cl_17\"],\n", + " [\"gut_abs_17\", \"ins_dep_util_17\"],\n", + " [\"renal_cl_17\", \"ins_indep_15\"],\n", + " [\"renal_cl_17\", \"ins_abs_6\"],\n", + " [\"renal_cl_17\", \"ins_abs_7\"],\n", + " [\"renal_cl_17\", \"ins_dep_util_15\"],\n", + " [\"renal_cl_17\", \"basal_bal_18\"],\n", + " [\"renal_cl_17\", \"ins_dep_7\"],\n", + " [\"renal_cl_17\", \"ins_indep_6\"],\n", + " [\"renal_cl_17\", \"bg_15\"],\n", + " [\"renal_cl_17\", \"ins_dep_18\"],\n", + " [\"renal_cl_17\", \"glu_prod_4\"],\n", + " [\"renal_cl_17\", \"met_irr_15\"],\n", + " [\"ins_indep_util_17\", \"renal_cl_17\"],\n", + " [\"ins_dep_util_17\", \"ins_dep_17\"],\n", + " [\"ins_dep_util_17\", \"ins_indep_util_17\"],\n", + " [\"ins_dep_util_17\", \"basal_bal_18\"],\n", + " [\"ins_dep_17\", \"ins_indep_util_17\"],\n", + " [\"cho_18\", \"cho_17\"],\n", + " [\"cho_18\", \"endo_bal_18\"],\n", + " [\"cho_18\", \"ins_indep_util_19\"],\n", + " [\"endo_bal_18\", \"ins_dep_18\"],\n", + " [\"cho_bal_18\", \"cho_18\"],\n", + " [\"cho_bal_18\", \"gut_abs_19\"],\n", + " [\"cho_bal_18\", \"endo_bal_18\"],\n", + " [\"basal_bal_18\", \"endo_bal_18\"],\n", + " [\"ins_indep_util_19\", \"ins_dep_18\"],\n", + " [\"ins_indep_util_19\", \"basal_bal_18\"],\n", + " [\"ins_indep_util_19\", \"ins_indep_util_17\"],\n", + " [\"ins_indep_util_19\", \"ins_dep_util_17\"],\n", + " [\"ins_indep_util_19\", \"renal_cl_17\"],\n", + " ],\n", + " 3: [\n", + " [\"cho_1\", \"tot_bal_0\"],\n", + " [\"ins_dep_4\", \"ins_dep_util_4\"],\n", + " [\"endo_bal_4\", \"ins_dep_4\"],\n", + " [\"cho_bal_6\", \"cho_8\"],\n", + " [\"basal_bal_6\", \"bg_6\"],\n", + " [\"met_irr_6\", \"bg_6\"],\n", + " [\"renal_cl_7\", \"bg_6\"],\n", + " [\"renal_cl_7\", \"bg_8\"],\n", + " [\"renal_cl_7\", \"basal_bal_6\"],\n", + " [\"renal_cl_7\", \"cho_bal_6\"],\n", + " [\"renal_cl_7\", \"activ_ins_8\"],\n", + " [\"renal_cl_7\", \"met_irr_6\"],\n", + " [\"renal_cl_7\", \"ins_dep_4\"],\n", + " [\"renal_cl_7\", \"ins_indep_11\"],\n", + " [\"renal_cl_7\", \"ins_dep_util_18\"],\n", + " [\"renal_cl_7\", \"ins_dep_util_11\"],\n", + " [\"renal_cl_7\", \"endo_bal_4\"],\n", + " [\"renal_cl_7\", \"endo_bal_15\"],\n", + " [\"ins_indep_util_7\", \"bg_6\"],\n", + " [\"ins_indep_util_7\", \"renal_cl_7\"],\n", + " [\"ins_indep_util_7\", \"bg_8\"],\n", + " [\"ins_indep_util_7\", \"basal_bal_6\"],\n", + " [\"ins_indep_util_7\", \"cho_bal_6\"],\n", + " [\"ins_indep_util_7\", \"met_irr_6\"],\n", + " [\"ins_dep_util_7\", \"bg_8\"],\n", + " [\"ins_dep_util_7\", \"ins_indep_util_7\"],\n", + " [\"meal_8\", \"cho_8\"],\n", + " [\"cho_8\", \"cho_10\"],\n", + " [\"ins_abs_8\", \"activ_ins_8\"],\n", + " [\"cho_10\", \"ins_indep_11\"],\n", + " [\"cho_10\", \"meal_10\"],\n", + " [\"ins_dep_util_11\", \"glu_prod_11\"],\n", + " [\"ins_indep_11\", \"glu_prod_11\"],\n", + " [\"ins_indep_11\", \"ins_dep_util_11\"],\n", + " [\"basal_bal_14\", \"glu_prod_14\"],\n", + " [\"glu_prod_15\", \"basal_bal_14\"],\n", + " [\"endo_bal_15\", \"glu_prod_15\"],\n", + " [\"endo_bal_15\", \"basal_bal_14\"],\n", + " [\"cho_bal_15\", \"ins_indep_util_18\"],\n", + " [\"basal_bal_17\", \"tot_bal_16\"],\n", + " [\"ins_indep_util_18\", \"activ_ins_18\"],\n", + " [\"ins_indep_util_18\", \"basal_bal_17\"],\n", + " [\"ins_indep_util_18\", \"tot_bal_16\"],\n", + " [\"ins_indep_util_18\", \"ins_dep_util_18\"],\n", + " [\"ins_indep_util_18\", \"met_irr_17\"],\n", + " [\"ins_indep_util_18\", \"ins_indep_util_7\"],\n", + " [\"ins_indep_util_18\", \"glu_prod_15\"],\n", + " [\"ins_indep_util_18\", \"ins_dep_util_7\"],\n", + " [\"ins_indep_util_18\", \"renal_cl_7\"],\n", + " [\"ins_dep_util_18\", \"activ_ins_18\"],\n", + " [\"meal_19\", \"cho_bal_20\"],\n", + " [\"meal_15\", \"cho_bal_15\"],\n", + " [\"cho_bal_20\", \"gut_abs_21\"],\n", + " [\"cho_bal_20\", \"basal_bal_20\"],\n", + " [\"cho_bal_20\", \"cho_bal_22\"],\n", + " [\"gut_abs_21\", \"cho_bal_22\"],\n", + " [\"cho_bal_22\", \"gut_abs_23\"],\n", + " [\"gut_abs_23\", \"cho_24\"],\n", + " [\"gut_abs_23\", \"glu_prod_23\"],\n", + " [\"gut_abs_23\", \"basal_bal_23\"],\n", + " [\"basal_bal_23\", \"glu_prod_23\"],\n", + " [\"basal_bal_23\", \"bg_24\"],\n", + " [\"met_irr_23\", \"bg_24\"],\n", + " [\"meal_24\", \"cho_24\"],\n", + " [\"bg_24\", \"glu_prod_23\"],\n", + " ],\n", + " 9: [\n", + " [\"cho_bal_4\", \"cho_bal_7\"],\n", + " [\"renal_cl_6\", \"cho_bal_4\"],\n", + " [\"renal_cl_6\", \"ins_indep_8\"],\n", + " [\"renal_cl_6\", \"basal_bal_7\"],\n", + " [\"renal_cl_6\", \"ins_indep_18\"],\n", + " [\"renal_cl_6\", \"ins_dep_util_6\"],\n", + " [\"renal_cl_6\", \"ins_indep_util_10\"],\n", + " [\"ins_indep_util_6\", \"renal_cl_6\"],\n", + " [\"ins_indep_util_6\", \"ins_dep_util_6\"],\n", + " [\"ins_indep_util_6\", \"cho_bal_4\"],\n", + " [\"ins_dep_util_6\", \"basal_bal_7\"],\n", + " [\"cho_bal_7\", \"endo_bal_7\"],\n", + " [\"cho_bal_7\", \"ins_indep_8\"],\n", + " [\"basal_bal_7\", \"endo_bal_7\"],\n", + " [\"ins_indep_8\", \"ins_dep_8\"],\n", + " [\"ins_indep_8\", \"endo_bal_8\"],\n", + " [\"ins_indep_8\", \"basal_bal_7\"],\n", + " [\"ins_indep_8\", \"ins_indep_util_10\"],\n", + " [\"ins_indep_8\", \"ins_indep_10\"],\n", + " [\"ins_indep_8\", \"ins_dep_util_6\"],\n", + " [\"endo_bal_8\", \"ins_dep_8\"],\n", + " [\"ins_indep_util_10\", \"ins_indep_10\"],\n", + " [\"ins_indep_util_10\", \"endo_bal_10\"],\n", + " [\"ins_indep_util_10\", \"bg_12\"],\n", + " [\"glu_prod_10\", \"bg_12\"],\n", + " [\"ins_indep_10\", \"ins_dep_10\"],\n", + " [\"ins_indep_10\", \"endo_bal_10\"],\n", + " [\"ins_indep_10\", \"bg_12\"],\n", + " [\"ins_indep_10\", \"endo_bal_8\"],\n", + " [\"ins_dep_10\", \"glu_prod_10\"],\n", + " [\"endo_bal_10\", \"ins_dep_10\"],\n", + " [\"meal_12\", \"cho_12\"],\n", + " [\"cho_12\", \"cho_15\"],\n", + " [\"cho_15\", \"basal_bal_15\"],\n", + " [\"meal_17\", \"cho_bal_19\"],\n", + " [\"meal_18\", \"cho_bal_19\"],\n", + " [\"ins_indep_18\", \"tot_bal_14\"],\n", + " [\"cho_bal_19\", \"cho_bal_21\"],\n", + " [\"cho_bal_19\", \"ins_indep_util_21\"],\n", + " [\"cho_bal_19\", \"ins_indep_21\"],\n", + " [\"cho_bal_21\", \"cho_23\"],\n", + " [\"ins_dep_19\", \"glu_prod_19\"],\n", + " [\"ins_indep_util_21\", \"ins_indep_21\"],\n", + " [\"ins_indep_util_21\", \"bg_23\"],\n", + " [\"ins_indep_util_21\", \"activ_ins_23\"],\n", + " [\"ins_indep_util_21\", \"ins_indep_18\"],\n", + " [\"ins_dep_util_21\", \"bg_23\"],\n", + " [\"ins_dep_util_21\", \"activ_ins_23\"],\n", + " [\"ins_indep_21\", \"ins_dep_21\"],\n", + " [\"ins_indep_21\", \"endo_bal_21\"],\n", + " [\"ins_indep_21\", \"bg_23\"],\n", + " [\"ins_indep_21\", \"ins_indep_18\"],\n", + " [\"ins_indep_21\", \"activ_ins_23\"],\n", + " [\"ins_indep_21\", \"ins_dep_19\"],\n", + " [\"ins_indep_21\", \"renal_cl_6\"],\n", + " [\"ins_dep_21\", \"ins_dep_util_21\"],\n", + " [\"endo_bal_21\", \"ins_dep_21\"],\n", + " [\"met_irr_22\", \"tot_bal_22\"],\n", + " [\"tot_bal_22\", \"bg_23\"],\n", + " [\"meal_23\", \"cho_23\"],\n", + " [\"activ_ins_23\", \"ins_indep_util_6\"],\n", + " ],\n", + " 5: [\n", + " [\"glu_prod_16\", \"activ_ins_16\"],\n", + " [\"tot_bal_19\", \"ins_abs_19\"],\n", + " [\"ins_dep_util_13\", \"ins_indep_util_13\"],\n", + " [\"activ_ins_16\", \"glu_prod_21\"],\n", + " [\"ins_indep_util_16\", \"activ_ins_16\"],\n", + " [\"ins_indep_util_16\", \"gut_abs_16\"],\n", + " [\"ins_indep_16\", \"ins_indep_util_16\"],\n", + " [\"ins_indep_16\", \"activ_ins_16\"],\n", + " [\"ins_indep_16\", \"gut_abs_16\"],\n", + " [\"ins_indep_16\", \"ins_indep_util_13\"],\n", + " [\"ins_indep_16\", \"glu_prod_16\"],\n", + " [\"ins_indep_16\", \"ins_dep_util_13\"],\n", + " [\"meal_20\", \"cho_20\"],\n", + " [\"cho_20\", \"renal_cl_21\"],\n", + " [\"renal_cl_21\", \"ins_indep_16\"],\n", + " [\"renal_cl_21\", \"met_irr_20\"],\n", + " [\"renal_cl_21\", \"ins_abs_19\"],\n", + " [\"renal_cl_21\", \"glu_prod_16\"],\n", + " [\"renal_cl_21\", \"tot_bal_19\"],\n", + " [\"renal_cl_21\", \"glu_prod_21\"],\n", + " [\"renal_cl_21\", \"gut_abs_16\"],\n", + " ],\n", + " 6: [\n", + " [\"ins_indep_17\", \"endo_bal_17\"],\n", + " [\"ins_indep_17\", \"ins_abs_17\"],\n", + " [\"ins_indep_17\", \"ins_indep_19\"],\n", + " [\"ins_indep_17\", \"met_irr_16\"],\n", + " [\"tot_bal_18\", \"ins_indep_19\"],\n", + " [\"tot_bal_18\", \"ins_abs_18\"],\n", + " [\"ins_indep_19\", \"renal_cl_19\"],\n", + " [\"ins_indep_19\", \"endo_bal_19\"],\n", + " [\"ins_indep_19\", \"ins_indep_20\"],\n", + " [\"ins_indep_19\", \"ins_dep_util_19\"],\n", + " [\"endo_bal_19\", \"ins_indep_20\"],\n", + " [\"tot_bal_15\", \"bg_16\"],\n", + " [\"tot_bal_15\", \"ins_indep_14\"],\n", + " [\"meal_16\", \"cho_bal_16\"],\n", + " [\"meal_16\", \"gut_abs_14\"],\n", + " [\"cho_16\", \"gut_abs_14\"],\n", + " [\"cho_16\", \"basal_bal_16\"],\n", + " [\"bg_16\", \"ins_indep_17\"],\n", + " [\"bg_16\", \"ins_indep_14\"],\n", + " [\"bg_16\", \"ins_dep_util_16\"],\n", + " [\"ins_dep_util_16\", \"ins_dep_16\"],\n", + " [\"ins_dep_16\", \"basal_bal_16\"],\n", + " [\"endo_bal_17\", \"ins_abs_17\"],\n", + " [\"cho_bal_17\", \"gut_abs_18\"],\n", + " [\"gut_abs_18\", \"tot_bal_18\"],\n", + " [\"renal_cl_19\", \"endo_bal_17\"],\n", + " [\"renal_cl_19\", \"bg_20\"],\n", + " [\"renal_cl_19\", \"met_irr_19\"],\n", + " [\"renal_cl_19\", \"ins_indep_20\"],\n", + " [\"ins_dep_util_19\", \"endo_bal_19\"],\n", + " [\"bg_20\", \"endo_bal_20\"],\n", + " [\"activ_ins_22\", \"ins_indep_23\"],\n", + " [\"activ_ins_22\", \"renal_cl_22\"],\n", + " [\"ins_abs_22\", \"activ_ins_22\"],\n", + " [\"gut_abs_22\", \"ins_indep_23\"],\n", + " [\"gut_abs_22\", \"meal_22\"],\n", + " [\"gut_abs_22\", \"basal_bal_22\"],\n", + " [\"gut_abs_22\", \"renal_cl_22\"],\n", + " [\"gut_abs_22\", \"ins_abs_22\"],\n", + " [\"renal_cl_22\", \"tot_bal_21\"],\n", + " [\"renal_cl_22\", \"ins_indep_23\"],\n", + " [\"renal_cl_22\", \"endo_bal_20\"],\n", + " [\"renal_cl_22\", \"ins_dep_util_19\"],\n", + " [\"basal_bal_22\", \"ins_abs_22\"],\n", + " [\"basal_bal_22\", \"renal_cl_22\"],\n", + " [\"renal_cl_23\", \"activ_ins_14\"],\n", + " [\"ins_indep_23\", \"ins_dep_23\"],\n", + " [\"ins_indep_23\", \"renal_cl_23\"],\n", + " [\"ins_indep_23\", \"ins_indep_util_23\"],\n", + " [\"ins_indep_23\", \"endo_bal_23\"],\n", + " [\"ins_dep_23\", \"ins_dep_util_23\"],\n", + " [\"endo_bal_23\", \"ins_dep_23\"],\n", + " [\"cho_bal_23\", \"gut_abs_22\"],\n", + " [\"activ_ins_14\", \"ins_dep_util_14\"],\n", + " [\"activ_ins_14\", \"bg_16\"],\n", + " [\"ins_abs_14\", \"activ_ins_14\"],\n", + " [\"ins_abs_14\", \"tot_bal_15\"],\n", + " [\"renal_cl_14\", \"gut_abs_14\"],\n", + " [\"renal_cl_14\", \"bg_14\"],\n", + " [\"renal_cl_14\", \"ins_dep_util_14\"],\n", + " [\"ins_indep_14\", \"bg_14\"],\n", + " [\"ins_indep_14\", \"renal_cl_14\"],\n", + " [\"ins_indep_14\", \"ins_indep_util_14\"],\n", + " [\"ins_indep_14\", \"ins_dep_util_14\"],\n", + " [\"cho_bal_16\", \"cho_16\"],\n", + " [\"cho_bal_16\", \"cho_bal_17\"],\n", + " [\"basal_bal_16\", \"ins_indep_17\"],\n", + " [\"ins_indep_util_20\", \"tot_bal_21\"],\n", + " [\"ins_indep_util_20\", \"bg_20\"],\n", + " [\"ins_dep_util_20\", \"glu_prod_20\"],\n", + " [\"glu_prod_20\", \"tot_bal_21\"],\n", + " [\"ins_indep_20\", \"bg_20\"],\n", + " [\"ins_indep_20\", \"ins_dep_20\"],\n", + " [\"ins_indep_20\", \"ins_indep_util_20\"],\n", + " [\"ins_indep_20\", \"met_irr_19\"],\n", + " [\"ins_dep_20\", \"glu_prod_20\"],\n", + " [\"ins_dep_20\", \"ins_dep_util_20\"],\n", + " [\"endo_bal_20\", \"ins_dep_20\"],\n", + " ],\n", + "}\n", + "\n", + "# Define your inter-structure edges here\n", + "inter_structure_edges = [\n", + " (\"cho_bal_4\", \"cho_4\"),\n", + " (\"cho_bal_7\", \"cho_7\"),\n", + " (\"cho_3\", \"cho_bal_3\"),\n", + " (\"gut_abs_9\", \"cho_9\"),\n", + " (\"gut_abs_19\", \"cho_19\"),\n", + " (\"cho_12\", \"gut_abs_12\"),\n", + " (\"cho_15\", \"gut_abs_15\"),\n", + " (\"ins_indep_18\", \"renal_cl_18\"),\n", + " (\"cho_bal_19\", \"cho_19\"),\n", + " (\"cho_bal_21\", \"cho_21\"),\n", + " (\"cho_0\", \"gut_abs_0\"),\n", + " (\"gut_abs_11\", \"cho_11\"),\n", + " (\"renal_cl_12\", \"ins_indep_12\"),\n", + " (\"cho_1\", \"cho_bal_1\"),\n", + " (\"cho_bal_22\", \"cho_22\"),\n", + " (\"ins_indep_util_0\", \"ins_indep_0\"),\n", + " (\"cho_bal_2\", \"cho_2\"),\n", + " (\"cho_3\", \"gut_abs_3\"),\n", + " (\"cho_bal_13\", \"cho_13\"),\n", + " (\"cho_8\", \"cho_bal_8\"),\n", + " (\"cho_10\", \"cho_bal_10\"),\n", + " (\"gut_abs_1\", \"cho_1\"),\n", + " (\"renal_cl_11\", \"ins_indep_11\"),\n", + " (\"cho_21\", \"gut_abs_21\"),\n", + " (\"ins_indep_6\", \"renal_cl_6\"),\n", + " (\"renal_cl_8\", \"ins_indep_8\"),\n", + " (\"renal_cl_10\", \"ins_indep_10\"),\n", + " (\"cho_23\", \"cho_bal_23\"),\n", + " (\"cho_17\", \"cho_bal_17\"),\n", + " (\"renal_cl_17\", \"ins_indep_17\"),\n", + " (\"cho_18\", \"gut_abs_18\"),\n", + " ]" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "precision_recall(evo, win95pts_true.values.tolist())\n" + ], + "outputs": [], + "execution_count": null + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Experiments" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## F1, SHD" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "def evaluate_bayesian_networks(datasets, hidden_nodes_clusters, time_m=5, num_runs=5):\n", + " results_df = pd.DataFrame(\n", + " columns=[\"dataset\", \"data_type\", \"shd\", \"f1\", \"f1_undir\", \"time\"]\n", + " )\n", + "\n", + " for dataset_name, (\n", + " data,\n", + " reference_dag_edges,\n", + " datatype,\n", + " max_local_structures,\n", + " ) in datasets.items():\n", + " for i in range(num_runs):\n", + " start_time = time.time()\n", + "\n", + " custom_dag_edges = get_edges_by_localstructures(\n", + " data, datatype, max_local_structures, hidden_nodes_clusters, time_m\n", + " )\n", + "\n", + " end_time = time.time()\n", + "\n", + " structure_learning_time = end_time - start_time\n", + "\n", + " f1 = f1_score(custom_dag_edges, reference_dag_edges.values.tolist())\n", + " f1_undir = precision_recall(\n", + " custom_dag_edges, reference_dag_edges.values.tolist()\n", + " )[\"F1_undir\"]\n", + " shd = precision_recall(\n", + " custom_dag_edges, reference_dag_edges.values.tolist()\n", + " )[\"SHD\"]\n", + "\n", + " results_df = results_df.append(\n", + " {\n", + " \"dataset\": dataset_name,\n", + " \"data_type\": datatype,\n", + " \"shd\": float(shd),\n", + " \"f1\": f1,\n", + " \"f1_undir\": f1_undir,\n", + " \"time\": structure_learning_time,\n", + " },\n", + " ignore_index=True,\n", + " )\n", + "\n", + " # Group by 'dataset' and 'data_type', then calculate mean and standard deviation\n", + " agg_funcs = {\n", + " \"shd\": [\"mean\", \"std\"],\n", + " \"f1\": [\"mean\", \"std\"],\n", + " \"f1_undir\": [\"mean\", \"std\"],\n", + " \"time\": [\"mean\", \"std\"],\n", + " }\n", + " results_df = (\n", + " results_df.groupby([\"dataset\", \"data_type\"]).agg(agg_funcs).reset_index()\n", + " )\n", + "\n", + " # Flatten the MultiIndex columns\n", + " results_df.columns = [\n", + " \"_\".join(col).strip() if col[1] else col[0] for col in results_df.columns.values\n", + " ]\n", + "\n", + " return results_df" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "datasets = {\n", + " \"pigs\": (pigs, pigs_true, \"discrete\", 16),\n", + " \"win95pts\": (win95pts, win95pts_true, \"discrete\", 4),\n", + " \"hailfinder\": (hailfinder, hailfinder_true, \"discrete\", 8),\n", + " \"hepar2\": (hepar2, hepar2_true, \"discrete\", 4),\n", + "# \"arth150\": (arth150, arth150_true, \"continuous\", 8),\n", + "# \"ecoli70\": (ecoli70, ecoli70_true, \"continuous\", 4),\n", + "# \"magic_irri\": (magic_irri, magic_irri_true, \"continuous\", 4),\n", + "# \"magic_niab\": (magic_niab, magic_niab_true, \"continuous\", 4),\n", + " \"diabetes\": (diabetes, diabetes_true, \"discrete\", 10),\n", + " \"andes\": (andes, andes_true, \"discrete\", 8)\n", + "}\n", + "\n", + "results_df_lsevo = evaluate_bayesian_networks(datasets,\n", + " hidden_nodes_clusters=8)\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "results_df_lsevo.round(2)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# dataset data_type shd f1 f1_undir time\n", + "# 0 ecoli70 continuous 42.666667 0.559430 0.765200 22.138747\n", + "# 1 hailfinder discrete 61.333333 0.457140 0.491433 28.580819\n", + "# 2 hepar2 discrete 85.000000 0.505263 0.600000 23.675336\n", + "# 3 magic_irri continuous 78.666667 0.343326 0.576600 26.135616\n", + "# 4 magic_niab continuous 56.333333 0.187702 0.718400 21.925407\n", + "# 5 pigs discrete 490.000000 0.510259 0.615500 143.663970\n", + "# 6 win95pts discrete 73.000000 0.608295 0.718900 26.574318\n", + "# 7 andes discrete 297.333333 0.382214 0.542433 89.976317\n", + "# 8 diabetes discrete 594.000000 0.387690 0.570167 150.839022\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "pigs_bidag = pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/pigs_bidag.csv')\n", + "win95pts_bidag = pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/win95pts_bidag.csv')\n", + "hailfinder_bidag = pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/hailfinder_bidag.csv')\n", + "hepar2_bidag = pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/hepar2_bidag.csv')\n", + "arth150_bidag = pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/arth150_bidag.csv')\n", + "ecoli70_bidag = pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/ecoli70_bidag.csv')\n", + "magic_irri_bidag = pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/magic-irri_bidag.csv')\n", + "magic_niab_bidag = pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/magic-niab_bidag.csv')\n", + "andes_bidag = pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/andes_bidag.csv')\n", + "diabetes_bidag = pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/diabetes_bidag.csv')\n", + "\n", + "pigs_sparsebn = pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/pigs_sparsebn.csv')\n", + "win95pts_sparsebn = pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/win95pts_sparsebn.csv')\n", + "hailfinder_sparsebn = pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/hailfinder_sparsebn.csv')\n", + "hepar2_sparsebn = pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/hepar2_sparsebn.csv')\n", + "arth150_sparsebn = pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/arth150_sparsebn.csv')\n", + "ecoli70_sparsebn = pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/ecoli70_sparsebn.csv')\n", + "magic_irri_sparsebn = pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/magic-irri_sparsebn.csv')\n", + "magic_niab_sparsebn = pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/magic-niab_sparsebn.csv')\n", + "andes_sparsebn = pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/andes_sparsebn.csv')\n", + "diabetes_sparsebn = pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/diabetes_sparsebn.csv')" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "def calculate_f1_shd(networks):\n", + " \n", + " results_df = pd.DataFrame(columns=['dataset',\n", + " 'shd_bidag',\n", + " 'f1_bidag',\n", + " 'f1_undir_bidag',\n", + " 'shd_sparsebn',\n", + " 'f1_sparsebn',\n", + " 'f1_undir_sparsebn'])\n", + " \n", + " for dataset_name, (reference_dag_edges, net_bidag, net_sparsebn) in networks.items():\n", + "\n", + " f1_bidag = f1_score(net_bidag.values.tolist(), reference_dag_edges.values.tolist()) \n", + " f1_undir_bidag = f1_score_undirected(net_bidag.values.tolist(), reference_dag_edges.values.tolist())\n", + " shd_bidag = precision_recall(net_bidag.values.tolist(), reference_dag_edges.values.tolist())['SHD']\n", + "\n", + " f1_sparsebn = f1_score(net_sparsebn.values.tolist(), reference_dag_edges.values.tolist()) \n", + " f1_undir_sparsebn = f1_score_undirected(net_sparsebn.values.tolist(), reference_dag_edges.values.tolist())\n", + " shd_sparsebn = precision_recall(net_sparsebn.values.tolist(), reference_dag_edges.values.tolist())['SHD']\n", + " \n", + " results_df = results_df.append({\n", + " 'dataset': f\"{dataset_name}\",\n", + " 'shd_bidag': shd_bidag,\n", + " 'f1_bidag': f1_bidag,\n", + " 'f1_undir_bidag': f1_undir_bidag,\n", + " 'shd_sparsebn': shd_sparsebn,\n", + " 'f1_sparsebn': f1_sparsebn,\n", + " 'f1_undir_sparsebn':f1_undir_sparsebn\n", + " }, ignore_index=True)\n", + " \n", + " return results_df" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "networks_baselines = {\n", + " \"pigs\": (pigs_true, pigs_bidag, pigs_sparsebn),\n", + " \"win95pts\": (win95pts_true, win95pts_bidag, win95pts_sparsebn),\n", + " \"hailfinder\": (hailfinder_true, hailfinder_bidag, hailfinder_sparsebn),\n", + " \"hepar2\": (hepar2_true, hepar2_bidag, hepar2_sparsebn),\n", + "# \"arth150\": (arth150_true, arth150_bidag, arth150_sparsebn),\n", + "# \"ecoli70\": (ecoli70_true, ecoli70_bidag, ecoli70_sparsebn),\n", + "# \"magic_irri\": (magic_irri_true, magic_irri_bidag, magic_irri_sparsebn),\n", + "# \"magic_niab\": (magic_niab_true, magic_niab_bidag, magic_niab_sparsebn),\n", + " \"diabetes\": (diabetes_true, diabetes_bidag, diabetes_sparsebn),\n", + " \"andes\": (andes_true, andes_bidag, andes_sparsebn)\n", + "}\n", + "\n", + "results_df = calculate_f1_shd(networks_baselines)\n", + "print(results_df)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import matplotlib.pyplot as plt\n", + "\n", + "plt.figure()\n", + "plt.rcParams.update({'font.size': 22})\n", + "stat.plot(x=\"dataset\", y=[\"shd_bidag\", \"shd_brave\", \"shd_ga_hidden\"], kind=\"bar\",figsize=(20,10))\n", + "plt.xlabel('Dataset', fontdict={'fontsize': 22})\n", + "plt.ylabel('SHD', fontdict={'fontsize': 22})\n", + "plt.title('Structural hamming distance for different benchmark datasets', fontdict={'fontsize': 20})\n", + "plt.show()" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "plt.figure()\n", + "plt.rcParams.update({'font.size': 22})\n", + "stat.plot(x=\"dataset\", y=[\"time_bidag\", \"time_brave\", \"time_ga_hidden\"], kind=\"bar\",figsize=(20,10))\n", + "plt.xlabel('Dataset', fontdict={'fontsize': 22})\n", + "plt.ylabel('time, s', fontdict={'fontsize': 22})\n", + "plt.title('Time for different benchmark datasets', fontdict={'fontsize': 20})\n", + "plt.show()" + ], + "outputs": [], + "execution_count": null + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## complexity analysis" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "df = pigs.copy()" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import time\n", + "\n", + "\n", + "variable_counts = range(420, 441, 20)\n", + "\n", + "results = []\n", + "\n", + "for count in variable_counts:\n", + " data = df.iloc[:, :count]\n", + " max_local_structures = count // 25\n", + " durations = []\n", + " for i in range(3):\n", + " start_time = time.time()\n", + " get_edges_by_localstructures(data,\n", + " datatype='discrete',\n", + " hidden_nodes_clusters=8,\n", + " max_local_structures=max_local_structures)\n", + " duration = time.time() - start_time\n", + " durations.append(duration)\n", + "\n", + " result = {'variables': count, 'duration': sum(durations) / len(durations)}\n", + " results.append(result)\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "import seaborn as sns\n", + "from sklearn.linear_model import LinearRegression\n", + "from sklearn.preprocessing import PolynomialFeatures\n", + "\n", + "import matplotlib\n", + "matplotlib.rcParams['pdf.fonttype'] = 42\n", + "matplotlib.rcParams['ps.fonttype'] = 42\n", + "\n", + "results_amd_5800x3d = [{'variables': 100, 'duration': 33.55898880958557},\n", + " {'variables': 110, 'duration': 42.41011571884155},\n", + " {'variables': 120, 'duration': 43.851377725601196},\n", + " {'variables': 130, 'duration': 40.22161738077799},\n", + " {'variables': 140, 'duration': 44.97185786565145},\n", + " {'variables': 150, 'duration': 49.20991023381551},\n", + " {'variables': 160, 'duration': 45.58039410909017},\n", + " {'variables': 170, 'duration': 53.09674938519796},\n", + " {'variables': 180, 'duration': 57.14670546849569},\n", + " {'variables': 190, 'duration': 55.86485083897909},\n", + " {'variables': 200, 'duration': 53.71896576881409},\n", + " {'variables': 210, 'duration': 60.018968184789024},\n", + " {'variables': 220, 'duration': 64.62080947558086},\n", + " {'variables': 230, 'duration': 68.80094695091248},\n", + " {'variables': 240, 'duration': 69.57624276479085},\n", + " {'variables': 250, 'duration': 77.65435568491618},\n", + " {'variables': 260, 'duration': 80.35756214459737},\n", + " {'variables': 270, 'duration': 97.4611929257711},\n", + " {'variables': 280, 'duration': 94.7663984298706},\n", + " {'variables': 290, 'duration': 100.50294987360637},\n", + " {'variables': 300, 'duration': 91.80926219622295},\n", + " {'variables': 310, 'duration': 100.70704078674316},\n", + " {'variables': 320, 'duration': 103.72079277038574},\n", + " {'variables': 330, 'duration': 114.90236934026082},\n", + " {'variables': 340, 'duration': 118.3715029557546},\n", + " {'variables': 350, 'duration': 123.63982383410136},\n", + " {'variables': 360, 'duration': 125.58711091677348},\n", + " {'variables': 370, 'duration': 156.0465773741404},\n", + " {'variables': 380, 'duration': 155.64693927764893},\n", + " {'variables': 390, 'duration': 159.9105533758799},\n", + " {'variables': 400, 'duration': 174.69820721944174},\n", + " {'variables': 410, 'duration': 157.47332048416138},\n", + " {'variables': 420, 'duration': 173.25324829419455},\n", + " {'variables': 430, 'duration': 182.21537001927695},\n", + " {'variables': 440, 'duration': 181.12759121259054}]\n", + "\n", + "results_m1_pro_8core = [{'variables': 100, 'duration': 28.07229733467102},\n", + " {'variables': 120, 'duration': 41.0058696269989},\n", + " {'variables': 140, 'duration': 42.688736279805504},\n", + " {'variables': 160, 'duration': 41.67554839452108},\n", + " {'variables': 180, 'duration': 54.11312389373779},\n", + " {'variables': 200, 'duration': 50.42719705899557},\n", + " {'variables': 220, 'duration': 61.47634975115458},\n", + " {'variables': 240, 'duration': 69.97042298316956},\n", + " {'variables': 260, 'duration': 84.46092391014099},\n", + " {'variables': 280, 'duration': 96.36128862698872},\n", + " {'variables': 300, 'duration': 102.87330500284831},\n", + " {'variables': 320, 'duration': 109.13302596410115},\n", + " {'variables': 340, 'duration': 135.1407539844513},\n", + " {'variables': 360, 'duration': 139.88516624768576},\n", + " {'variables': 380, 'duration': 183.65180929501852},\n", + " {'variables': 400, 'duration': 196.62940200169882}\n", + "]\n", + "\n", + "bidag_results_5800x = [{'variables': 100, 'duration': 107.5941},\n", + " {'variables': 120, 'duration': 424.94136},\n", + " {'variables': 140, 'duration': 594.9396},\n", + " {'variables': 160, 'duration': 343.15086},\n", + " {'variables': 180, 'duration': 1069.3152},\n", + " {'variables': 200, 'duration': 1408.1304},\n", + " {'variables': 220, 'duration': 632.2404},\n", + " {'variables': 240, 'duration': 2405.3274},\n", + " {'variables': 260, 'duration': 2354.1102},\n", + " {'variables': 280, 'duration': 1538.4378},\n", + " {'variables': 300, 'duration': 4401.6984},\n", + " {'variables': 320, 'duration': 4924.4436},\n", + " {'variables': 340, 'duration': 3696.9624},\n", + " {'variables': 360, 'duration': 8169.4764},\n", + " {'variables': 380, 'duration': 5181.6456},\n", + " {'variables': 400, 'duration': 6553.476},\n", + " {'variables': 440, 'duration': 9239.5368}\n", + "]\n", + "\n", + "df_amd = pd.DataFrame(results_amd_5800x3d)\n", + "df_m1 = pd.DataFrame(results_m1_pro_8core)\n", + "df_bidag = pd.DataFrame(bidag_results_5800x)\n", + "\n", + "# Fit polynomial regression\n", + "degree = 2\n", + "\n", + "# AMD\n", + "poly_features = PolynomialFeatures(degree)\n", + "X_amd = poly_features.fit_transform(df_amd[['variables']])\n", + "y_amd = df_amd['duration']\n", + "reg_amd = LinearRegression().fit(X_amd, y_amd)\n", + "\n", + "# M1 Pro\n", + "X_m1 = poly_features.fit_transform(df_m1[['variables']])\n", + "y_m1 = df_m1['duration']\n", + "reg_m1 = LinearRegression().fit(X_m1, y_m1)\n", + "\n", + "# Bidag\n", + "X_bidag = poly_features.fit_transform(df_bidag[['variables']])\n", + "y_bidag = df_bidag['duration']\n", + "reg_bidag = LinearRegression().fit(X_bidag, y_bidag),\n", + "\n", + "\n", + "# Plot results\n", + "plt.figure(figsize=(10, 6))\n", + "sns.scatterplot(data=df_amd, x='variables', y='duration', label='AMD 5800x')\n", + "sns.scatterplot(data=df_m1, x='variables', y='duration', label='M1 Pro 8-core')\n", + "sns.scatterplot(data=df_bidag, x='variables', y='duration', label='Bidag 5800x')\n", + "\n", + "# Plot polynomial regression lines\n", + "X_range = np.linspace(100, 440, 100).reshape(-1, 1)\n", + "X_range_poly = poly_features.fit_transform(X_range)\n", + "\n", + "plt.plot(X_range, reg_amd.predict(X_range_poly), label='AMD 5800x Poly Fit')\n", + "plt.plot(X_range, reg_m1.predict(X_range_poly), label='M1 Pro 8-core Poly Fit')\n", + "plt.plot(X_range, reg_bidag.predict(X_range_poly), label='Bidag 5800x Poly Fit')\n", + "\n", + "# Log scale for y-axis\n", + "plt.yscale(\"log\")\n", + "\n", + "# Labels\n", + "plt.xlabel('Variables')\n", + "plt.ylabel('Log(Duration, s)')\n", + "plt.title('Time Complexity Comparison')\n", + "# Get coefficients\n", + "coeff_amd = reg_amd.coef_[1:], reg_amd.intercept_\n", + "coeff_m1 = reg_m1.coef_[1:], reg_m1.intercept_\n", + "coeff_bidag = reg_bidag.coef_[1:], reg_bidag.intercept_\n", + "\n", + "# Create strings with the equations\n", + "equation_amd = f'AMD 5800x3d: y = {coeff_amd[0][0]:.2f}x^2 + {coeff_amd[0][1]:.2f}x + {coeff_amd[1]:.2f}'\n", + "equation_m1 = f'M1 Pro 8-core: y = {coeff_m1[0][0]:.2f}x^2 + {coeff_m1[0][1]:.2f}x + {coeff_m1[1]:.2f}'\n", + "equation_bidag = f'Bidag 5800x: y = {coeff_bidag[0][0]:.2f}x^2 + {coeff_bidag[0][1]:.2f}x + {coeff_bidag[1]:.2f}'\n", + "\n", + "# Plot results (same as before)\n", + "\n", + "# Add equations to the legend\n", + "plt.legend([equation_amd, equation_m1, equation_bidag])\n", + "\n", + "plt.show()" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# extract the x and y values from bidag_data\n", + "bidag_x_values = bidag_data['number of nodes']\n", + "bidag_y_values = bidag_data['exec_time']\n", + "\n", + "# fit a polynomial curve to the data\n", + "poly_degree = 2 # degree of the polynomial regression\n", + "poly_features = PolynomialFeatures(degree=poly_degree)\n", + "x_poly = poly_features.fit_transform(np.array(bidag_x_values).reshape(-1, 1))\n", + "poly_model = LinearRegression()\n", + "poly_model.fit(x_poly, bidag_y_values)\n", + "poly_y_pred = poly_model.predict(x_poly)\n", + "\n", + "# add the bidag_data and polynomial regression to the plot\n", + "plt.scatter(bidag_x_values, bidag_y_values, color='goldenrod', label='bidag average time')\n", + "plt.plot(bidag_x_values, poly_y_pred, color='brown', label='bidag polynomial fit')\n", + "plt.title('Execution Time vs. Number of Variables, Ryzen 5800x')\n", + "plt.xlabel('Number of Variables')\n", + "plt.ylabel('Execution Time (s)')\n", + "plt.legend()\n", + "\n", + "# estimate the time complexity of the algorithm\n", + "coef = poly_model.coef_\n", + "complexity_str = 'O('\n", + "for i in range(len(coef)):\n", + " if i == 0:\n", + " complexity_str += str(round(coef[i], 6))\n", + " else:\n", + " complexity_str += f'+{str(round(coef[i], 6))}*n^{str(i)}'\n", + "complexity_str += ')'\n", + "print(f\"Estimated time complexity: {complexity_str}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "x_values = [result['variables'] for result in results_m1_pro_8core]\n", + "y_values = [result['duration'] for result in results_m1_pro_8core]\n", + "\n", + "# fit a polynomial curve to the data\n", + "poly_degree = 2 # degree of the polynomial regression\n", + "poly_features = PolynomialFeatures(degree=poly_degree)\n", + "x_poly = poly_features.fit_transform(np.array(x_values).reshape(-1, 1))\n", + "poly_model = LinearRegression()\n", + "poly_model.fit(x_poly, y_values)\n", + "poly_y_pred = poly_model.predict(x_poly)\n", + "\n", + "# create the plot\n", + "plt.scatter(x_values, y_values)\n", + "plt.plot(x_values, poly_y_pred, color='red')\n", + "plt.title('Execution Time vs. Number of Variables, m1 pro 8c')\n", + "plt.xlabel('Number of Variables')\n", + "plt.ylabel('Execution Time (s)')\n", + "plt.show()\n", + "\n", + "# estimate the time complexity of the algorithm\n", + "coef = poly_model.coef_\n", + "complexity_str = 'O('\n", + "for i in range(len(coef)):\n", + " if i == 0:\n", + " complexity_str += str(round(coef[i], 6))\n", + " else:\n", + " complexity_str += f'+{str(round(coef[i], 6))}*n^{str(i)}'\n", + "complexity_str += ')'\n", + "print(f\"Estimated time complexity: {complexity_str}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2024-06-19T14:04:40.322394Z", + "start_time": "2024-06-19T14:04:40.311554Z" + } + }, + "cell_type": "code", + "source": [ + "import pandas as pd\n", + "\n", + "# Define the dictionaries\n", + "dict1 = {\n", + " 'dataset': ['ecoli70', 'hailfinder', 'hepar2', 'magic_irri', 'magic_niab', 'pigs', 'win95pts', 'andes', 'diabetes'],\n", + " 'data_type': ['continuous', 'discrete', 'discrete', 'continuous', 'continuous', 'discrete', 'discrete', 'discrete', 'discrete'],\n", + " 'shd_lsevo': [39.66, 63.33, 85.0, 77.333333, 54.666667, 490.0, 74.0, 297.33, 594.0],\n", + " 'f1_lsevo': [0.572215, 0.479836, 0.505263, 0.328704, 0.282016, 0.510259, 0.589862, 0.382214, 0.287690],\n", + " 'time_lsevo': [20.330601, 26.115666, 22.376997, 23.918053, 19.641882, 134.663579, 25.024325, 89.976317, 150.839022],\n", + " 'f1_undir_lsevo': [0.7732, 0.563233, 0.600000, 0.576600, 0.718400, 0.615500, 0.718900, 0.542433, 0.570167],\n", + " 'shd_bidag': [9, 46, 66, 19, 6, 35, 60, 117, 453],\n", + " 'f1_bidag': [0.914286, 0.567164, 0.590476, 0.862559, 0.909091, 0.958298, 0.675439, 0.720737, 0.444240],\n", + " 'time_bidag': [0.94, 1328.2272, 1905.402, 1.25, 0.898, 8630.3592, 993.6678, 171.48, 931.2],\n", + " 'f1_undir_bidag': [0.9571, 0.7463, 0.7810, 0.9573, 1.0000, 0.9821, 0.7982, 0.884903, 0.720737],\n", + " 'shd_sparsebn': [55, 61, 147, 66, 29, 483, 139, 253, 842],\n", + " 'f1_sparsebn': [0.510949, 0.477612, 0.316981, 0.468900, 0.602941, 0.351227, 0.263636, 0.5433, 0.2205],\n", + " 'f1_undir_sparsebn': [0.6861, 0.6119, 0.5736, 0.8995, 0.9706, 0.9080, 0.4727, 0.725434, 0.498859],\n", + " 'time_sparsebn': [5.547351, 350.45382, 16.58553, 21.68742, 6.791835, 11000, 13.75046, 2148, 74520]\n", + "}\n", + "\n", + "dict2 = {\n", + " 'dataset': ['pigs', 'win95pts', 'hailfinder', 'hepar2', 'arth150', 'ecoli70', 'magic_irri', 'magic_niab', 'diabetes', 'andes'],\n", + " 'data_type': ['discrete', 'discrete', 'discrete', 'discrete', 'discrete', 'continuous', 'continuous', 'continuous', 'discrete', 'discrete'],\n", + " 'shd_bbn': [688.0, 112.0, 81.0, 123.0, 247.0, 46.0, 79.0, 51.0, 961.0, 338.0],\n", + " 'f1_bbn': [0.319935691318328, 0.0, 0.2393162393162393, 0.0, 0.0, 0.5354330708661418, 0.3057324840764331, 0.28846153846153844, 0.0, 0.0],\n", + " 'time_bbn': [899.0762053966522, 6.8883123874664305, 6.436980581283569, 6.2710999011993405, 27.879689264297486, 5.929460620880127, 10.764610052108765, 5.364017057418823, 558.3422603130341, 63.081138610839844],\n", + " 'f1_undir_bbn': [0.5739549839228295, 0.0, 0.37606837606837606, 0.0, 0.6882591093117408, 0.7401574803149606, 0.6878980891719745, 0.7307692307692308, 0.0, 0.0],\n", + " 'shd_dagma': [593.0, 112.0, 65.0, 123.0, 217.0, 51.0, 96.0, 64.0, 1052.0, 338.0],\n", + " 'f1_dagma': [0.0, 0.0, 0.17777777777777776, 0.0, 0.0, 0.46280991735537197, 0.09599999999999999, 0.05714285714285715, 0.0, 0.0],\n", + " 'time_dagma': [3549.34708480835, 2212.3995155334474, 4.187403678894043, 989.0881148815155, 2028.950544166565, 3.9232993602752684, 3.999641466140747, 1.8709041118621825, 4177.805352163315, 3685.0702658176424],\n", + " 'f1_undir_dagma': [0.0, 0.0, 0.37777777777777777, 0.0, 0.6175115207373272, 0.6942148760330579, 0.368, 0.1142857142857143, 0.0, 0.0]\n", + "}\n", + "\n", + "# Convert dictionaries to DataFrames\n", + "df1 = pd.DataFrame(dict1)\n", + "df2 = pd.DataFrame(dict2)\n", + "\n", + "# Merge the DataFrames on 'dataset'\n", + "merged_df = pd.merge(df1, df2, on='dataset')\n", + "\n", + "# Remove 'arth150' sample\n", + "merged_df = merged_df[merged_df['dataset'] != 'arth150']\n", + "\n", + "# Convert the merged DataFrame back to a dictionary\n", + "merged_dict = merged_df.to_dict(orient='list')\n", + "\n", + "\n", + "print(merged_dict)\n" + ], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'dataset': ['ecoli70', 'hailfinder', 'hepar2', 'magic_irri', 'magic_niab', 'pigs', 'win95pts', 'andes', 'diabetes'], 'data_type_x': ['continuous', 'discrete', 'discrete', 'continuous', 'continuous', 'discrete', 'discrete', 'discrete', 'discrete'], 'shd_lsevo': [39.66, 63.33, 85.0, 77.333333, 54.666667, 490.0, 74.0, 297.33, 594.0], 'f1_lsevo': [0.572215, 0.479836, 0.505263, 0.328704, 0.282016, 0.510259, 0.589862, 0.382214, 0.28769], 'time_lsevo': [20.330601, 26.115666, 22.376997, 23.918053, 19.641882, 134.663579, 25.024325, 89.976317, 150.839022], 'f1_undir_lsevo': [0.7732, 0.563233, 0.6, 0.5766, 0.7184, 0.6155, 0.7189, 0.542433, 0.570167], 'shd_bidag': [9, 46, 66, 19, 6, 35, 60, 117, 453], 'f1_bidag': [0.914286, 0.567164, 0.590476, 0.862559, 0.909091, 0.958298, 0.675439, 0.720737, 0.44424], 'time_bidag': [0.94, 1328.2272, 1905.402, 1.25, 0.898, 8630.3592, 993.6678, 171.48, 931.2], 'f1_undir_bidag': [0.9571, 0.7463, 0.781, 0.9573, 1.0, 0.9821, 0.7982, 0.884903, 0.720737], 'shd_sparsebn': [55, 61, 147, 66, 29, 483, 139, 253, 842], 'f1_sparsebn': [0.510949, 0.477612, 0.316981, 0.4689, 0.602941, 0.351227, 0.263636, 0.5433, 0.2205], 'f1_undir_sparsebn': [0.6861, 0.6119, 0.5736, 0.8995, 0.9706, 0.908, 0.4727, 0.725434, 0.498859], 'time_sparsebn': [5.547351, 350.45382, 16.58553, 21.68742, 6.791835, 11000.0, 13.75046, 2148.0, 74520.0], 'data_type_y': ['continuous', 'discrete', 'discrete', 'continuous', 'continuous', 'discrete', 'discrete', 'discrete', 'discrete'], 'shd_bbn': [46.0, 81.0, 123.0, 79.0, 51.0, 688.0, 112.0, 338.0, 961.0], 'f1_bbn': [0.5354330708661418, 0.2393162393162393, 0.0, 0.3057324840764331, 0.28846153846153844, 0.319935691318328, 0.0, 0.0, 0.0], 'time_bbn': [5.929460620880127, 6.436980581283569, 6.2710999011993405, 10.764610052108765, 5.364017057418823, 899.0762053966522, 6.8883123874664305, 63.081138610839844, 558.3422603130341], 'f1_undir_bbn': [0.7401574803149606, 0.37606837606837606, 0.0, 0.6878980891719745, 0.7307692307692308, 0.5739549839228295, 0.0, 0.0, 0.0], 'shd_dagma': [51.0, 65.0, 123.0, 96.0, 64.0, 593.0, 112.0, 338.0, 1052.0], 'f1_dagma': [0.46280991735537197, 0.17777777777777776, 0.0, 0.09599999999999999, 0.05714285714285715, 0.0, 0.0, 0.0, 0.0], 'time_dagma': [3.9232993602752684, 4.187403678894043, 989.0881148815155, 3.999641466140747, 1.8709041118621825, 3549.34708480835, 2212.3995155334474, 3685.0702658176424, 4177.805352163315], 'f1_undir_dagma': [0.6942148760330579, 0.37777777777777777, 0.0, 0.368, 0.1142857142857143, 0.0, 0.0, 0.0, 0.0]}\n" + ] + } + ], + "execution_count": 5 + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2024-06-19T14:04:49.002983Z", + "start_time": "2024-06-19T14:04:48.998757Z" + } + }, + "cell_type": "code", + "source": "merged_dict", + "outputs": [ + { + "data": { + "text/plain": [ + "{'dataset': ['ecoli70',\n", + " 'hailfinder',\n", + " 'hepar2',\n", + " 'magic_irri',\n", + " 'magic_niab',\n", + " 'pigs',\n", + " 'win95pts',\n", + " 'andes',\n", + " 'diabetes'],\n", + " 'data_type_x': ['continuous',\n", + " 'discrete',\n", + " 'discrete',\n", + " 'continuous',\n", + " 'continuous',\n", + " 'discrete',\n", + " 'discrete',\n", + " 'discrete',\n", + " 'discrete'],\n", + " 'shd_lsevo': [39.66,\n", + " 63.33,\n", + " 85.0,\n", + " 77.333333,\n", + " 54.666667,\n", + " 490.0,\n", + " 74.0,\n", + " 297.33,\n", + " 594.0],\n", + " 'f1_lsevo': [0.572215,\n", + " 0.479836,\n", + " 0.505263,\n", + " 0.328704,\n", + " 0.282016,\n", + " 0.510259,\n", + " 0.589862,\n", + " 0.382214,\n", + " 0.28769],\n", + " 'time_lsevo': [20.330601,\n", + " 26.115666,\n", + " 22.376997,\n", + " 23.918053,\n", + " 19.641882,\n", + " 134.663579,\n", + " 25.024325,\n", + " 89.976317,\n", + " 150.839022],\n", + " 'f1_undir_lsevo': [0.7732,\n", + " 0.563233,\n", + " 0.6,\n", + " 0.5766,\n", + " 0.7184,\n", + " 0.6155,\n", + " 0.7189,\n", + " 0.542433,\n", + " 0.570167],\n", + " 'shd_bidag': [9, 46, 66, 19, 6, 35, 60, 117, 453],\n", + " 'f1_bidag': [0.914286,\n", + " 0.567164,\n", + " 0.590476,\n", + " 0.862559,\n", + " 0.909091,\n", + " 0.958298,\n", + " 0.675439,\n", + " 0.720737,\n", + " 0.44424],\n", + " 'time_bidag': [0.94,\n", + " 1328.2272,\n", + " 1905.402,\n", + " 1.25,\n", + " 0.898,\n", + " 8630.3592,\n", + " 993.6678,\n", + " 171.48,\n", + " 931.2],\n", + " 'f1_undir_bidag': [0.9571,\n", + " 0.7463,\n", + " 0.781,\n", + " 0.9573,\n", + " 1.0,\n", + " 0.9821,\n", + " 0.7982,\n", + " 0.884903,\n", + " 0.720737],\n", + " 'shd_sparsebn': [55, 61, 147, 66, 29, 483, 139, 253, 842],\n", + " 'f1_sparsebn': [0.510949,\n", + " 0.477612,\n", + " 0.316981,\n", + " 0.4689,\n", + " 0.602941,\n", + " 0.351227,\n", + " 0.263636,\n", + " 0.5433,\n", + " 0.2205],\n", + " 'f1_undir_sparsebn': [0.6861,\n", + " 0.6119,\n", + " 0.5736,\n", + " 0.8995,\n", + " 0.9706,\n", + " 0.908,\n", + " 0.4727,\n", + " 0.725434,\n", + " 0.498859],\n", + " 'time_sparsebn': [5.547351,\n", + " 350.45382,\n", + " 16.58553,\n", + " 21.68742,\n", + " 6.791835,\n", + " 11000.0,\n", + " 13.75046,\n", + " 2148.0,\n", + " 74520.0],\n", + " 'data_type_y': ['continuous',\n", + " 'discrete',\n", + " 'discrete',\n", + " 'continuous',\n", + " 'continuous',\n", + " 'discrete',\n", + " 'discrete',\n", + " 'discrete',\n", + " 'discrete'],\n", + " 'shd_bbn': [46.0, 81.0, 123.0, 79.0, 51.0, 688.0, 112.0, 338.0, 961.0],\n", + " 'f1_bbn': [0.5354330708661418,\n", + " 0.2393162393162393,\n", + " 0.0,\n", + " 0.3057324840764331,\n", + " 0.28846153846153844,\n", + " 0.319935691318328,\n", + " 0.0,\n", + " 0.0,\n", + " 0.0],\n", + " 'time_bbn': [5.929460620880127,\n", + " 6.436980581283569,\n", + " 6.2710999011993405,\n", + " 10.764610052108765,\n", + " 5.364017057418823,\n", + " 899.0762053966522,\n", + " 6.8883123874664305,\n", + " 63.081138610839844,\n", + " 558.3422603130341],\n", + " 'f1_undir_bbn': [0.7401574803149606,\n", + " 0.37606837606837606,\n", + " 0.0,\n", + " 0.6878980891719745,\n", + " 0.7307692307692308,\n", + " 0.5739549839228295,\n", + " 0.0,\n", + " 0.0,\n", + " 0.0],\n", + " 'shd_dagma': [51.0, 65.0, 123.0, 96.0, 64.0, 593.0, 112.0, 338.0, 1052.0],\n", + " 'f1_dagma': [0.46280991735537197,\n", + " 0.17777777777777776,\n", + " 0.0,\n", + " 0.09599999999999999,\n", + " 0.05714285714285715,\n", + " 0.0,\n", + " 0.0,\n", + " 0.0,\n", + " 0.0],\n", + " 'time_dagma': [3.9232993602752684,\n", + " 4.187403678894043,\n", + " 989.0881148815155,\n", + " 3.999641466140747,\n", + " 1.8709041118621825,\n", + " 3549.34708480835,\n", + " 2212.3995155334474,\n", + " 3685.0702658176424,\n", + " 4177.805352163315],\n", + " 'f1_undir_dagma': [0.6942148760330579,\n", + " 0.37777777777777777,\n", + " 0.0,\n", + " 0.368,\n", + " 0.1142857142857143,\n", + " 0.0,\n", + " 0.0,\n", + " 0.0,\n", + " 0.0]}" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 6 + }, + { + "cell_type": "code", + "metadata": { + "ExecuteTime": { + "end_time": "2024-06-24T09:34:48.327539Z", + "start_time": "2024-06-24T09:34:48.319400Z" + } + }, + "source": [ + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "import seaborn as sns\n", + "\n", + "import matplotlib\n", + "matplotlib.rcParams.update({'font.size': 16})\n", + "matplotlib.rcParams['pdf.fonttype'] = 42\n", + "matplotlib.rcParams['ps.fonttype'] = 42\n", + "\n", + "\n", + "# data = {'dataset': ['ecoli70', 'hailfinder', 'hepar2', 'magic_irri', 'magic_niab', 'pigs', 'win95pts', 'andes', 'diabetes'],\n", + "# 'data_type': ['continuous', 'discrete', 'discrete', 'continuous', 'continuous', 'discrete', 'discrete', 'discrete', 'discrete'],\n", + "# 'shd_lsevo': [39.66, 63.33, 85.0, 77.333333, 54.666667, 490.0, 74.0, 297.33, 594.0],\n", + "# 'f1_lsevo': [0.572215, 0.479836, 0.505263, 0.328704, 0.282016, 0.510259, 0.589862, 0.382214, 0.287690],\n", + "# 'time_lsevo': [20.330601, 26.115666, 22.376997, 23.918053, 19.641882, 134.663579, 25.024325, 89.976317, 150.839022],\n", + "# 'f1_undir_lsevo': [0.7732, 0.563233, 0.600000, 0.576600, 0.718400, 0.615500, 0.718900, 0.542433, 0.570167],\n", + "# 'shd_bidag': [9, 46, 66, 19, 6, 35, 60, 117, 453],\n", + "# 'f1_bidag': [0.914286, 0.567164, 0.590476, 0.862559, 0.909091, 0.958298, 0.675439, 0.720737, 0.444240],\n", + "# 'time_bidag':[0.94, 1328.2272, 1905.402, 1.25, 0.898, 8630.3592, 993.6678, 171.48, 931.2],\n", + "# 'f1_undir_bidag': [0.9571, 0.7463, 0.7810, 0.9573, 1.0000, 0.9821, 0.7982, 0.884903, 0.720737],\n", + "# 'shd_sparsebn': [55, 61, 147, 66, 29, 483, 139, 253, 842],\n", + "# 'f1_sparsebn': [0.510949, 0.477612, 0.316981, 0.468900, 0.602941, 0.351227, 0.263636, 0.5433, 0.2205],\n", + "# 'f1_undir_sparsebn': [0.6861, 0.6119, 0.5736, 0.8995, 0.9706, 0.9080, 0.4727, 0.725434, 0.498859],\n", + "# 'time_sparsebn': [5.547351, 350.45382, 16.58553, 21.68742, 6.791835, 11000, 13.75046, 2148, 74520]\n", + "# }\n", + "\n", + "data = {'dataset': ['ecoli70',\n", + " 'hailfinder \\n (56 nodes)',\n", + " 'hepar2 \\n (70 nodes)',\n", + " 'magic_irri',\n", + " 'magic_niab',\n", + " 'pigs \\n (441 nodes)',\n", + " 'win95pts \\n (76 nodes)',\n", + " 'andes \\n (223 nodes)',\n", + " 'diabetes \\n (413 nodes)'],\n", + " 'data_type': ['continuous',\n", + " 'discrete',\n", + " 'discrete',\n", + " 'continuous',\n", + " 'continuous',\n", + " 'discrete',\n", + " 'discrete',\n", + " 'discrete',\n", + " 'discrete'],\n", + " 'shd_lsevo': [39.66,\n", + " 63.33,\n", + " 85.0,\n", + " 77.333333,\n", + " 54.666667,\n", + " 490.0,\n", + " 74.0,\n", + " 297.33,\n", + " 594.0],\n", + " 'f1_lsevo': [0.572215,\n", + " 0.479836,\n", + " 0.505263,\n", + " 0.328704,\n", + " 0.282016,\n", + " 0.510259,\n", + " 0.589862,\n", + " 0.382214,\n", + " 0.28769],\n", + " 'time_lsevo': [20.330601,\n", + " 26.115666,\n", + " 22.376997,\n", + " 23.918053,\n", + " 19.641882,\n", + " 134.663579,\n", + " 25.024325,\n", + " 89.976317,\n", + " 150.839022],\n", + " 'f1_undir_lsevo': [0.7732,\n", + " 0.563233,\n", + " 0.6,\n", + " 0.5766,\n", + " 0.7184,\n", + " 0.6155,\n", + " 0.7189,\n", + " 0.542433,\n", + " 0.570167],\n", + " 'shd_bidag': [9, 46, 66, 19, 6, 35, 60, 117, 453],\n", + " 'f1_bidag': [0.914286,\n", + " 0.567164,\n", + " 0.590476,\n", + " 0.862559,\n", + " 0.909091,\n", + " 0.958298,\n", + " 0.675439,\n", + " 0.720737,\n", + " 0.44424],\n", + " 'time_bidag': [0.94,\n", + " 1328.2272,\n", + " 1905.402,\n", + " 1.25,\n", + " 0.898,\n", + " 8630.3592,\n", + " 993.6678,\n", + " 171.48,\n", + " 931.2],\n", + " 'f1_undir_bidag': [0.9571,\n", + " 0.7463,\n", + " 0.781,\n", + " 0.9573,\n", + " 1.0,\n", + " 0.9821,\n", + " 0.7982,\n", + " 0.884903,\n", + " 0.720737],\n", + " 'shd_sparsebn': [55, 61, 147, 66, 29, 483, 139, 253, 842],\n", + " 'f1_sparsebn': [0.510949,\n", + " 0.477612,\n", + " 0.316981,\n", + " 0.4689,\n", + " 0.602941,\n", + " 0.351227,\n", + " 0.263636,\n", + " 0.5433,\n", + " 0.2205],\n", + " 'f1_undir_sparsebn': [0.6861,\n", + " 0.6119,\n", + " 0.5736,\n", + " 0.8995,\n", + " 0.9706,\n", + " 0.908,\n", + " 0.4727,\n", + " 0.725434,\n", + " 0.498859],\n", + " 'time_sparsebn': [5.547351,\n", + " 350.45382,\n", + " 16.58553,\n", + " 21.68742,\n", + " 6.791835,\n", + " 11000.0,\n", + " 13.75046,\n", + " 2148.0,\n", + " 74520.0],\n", + " 'data_type_y': ['continuous',\n", + " 'discrete',\n", + " 'discrete',\n", + " 'continuous',\n", + " 'continuous',\n", + " 'discrete',\n", + " 'discrete',\n", + " 'discrete',\n", + " 'discrete'],\n", + " 'shd_bbn': [46.0, 81.0, 123.0, 79.0, 51.0, 688.0, 112.0, 338.0, 961.0],\n", + " 'f1_bbn': [0.5354330708661418,\n", + " 0.2393162393162393,\n", + " 0.0,\n", + " 0.3057324840764331,\n", + " 0.28846153846153844,\n", + " 0.319935691318328,\n", + " 0.0,\n", + " 0.0,\n", + " 0.0],\n", + " 'time_bbn': [5.929460620880127,\n", + " 6.436980581283569,\n", + " 6.2710999011993405,\n", + " 10.764610052108765,\n", + " 5.364017057418823,\n", + " 899.0762053966522,\n", + " 6.8883123874664305,\n", + " 63.081138610839844,\n", + " 558.3422603130341],\n", + " 'f1_undir_bbn': [0.7401574803149606,\n", + " 0.37606837606837606,\n", + " 0.0,\n", + " 0.6878980891719745,\n", + " 0.7307692307692308,\n", + " 0.5739549839228295,\n", + " 0.0,\n", + " 0.0,\n", + " 0.0],\n", + " 'shd_dagma': [51.0, 65.0, 123.0, 96.0, 64.0, 593.0, 112.0, 338.0, 1052.0],\n", + " 'f1_dagma': [0.46280991735537197,\n", + " 0.17777777777777776,\n", + " 0.0,\n", + " 0.09599999999999999,\n", + " 0.05714285714285715,\n", + " 0.0,\n", + " 0.0,\n", + " 0.0,\n", + " 0.0],\n", + " 'time_dagma': [3.9232993602752684,\n", + " 4.187403678894043,\n", + " 989.0881148815155,\n", + " 3.999641466140747,\n", + " 1.8709041118621825,\n", + " 3549.34708480835,\n", + " 2212.3995155334474,\n", + " 3685.0702658176424,\n", + " 4177.805352163315],\n", + " 'f1_undir_dagma': [0.6942148760330579,\n", + " 0.37777777777777777,\n", + " 0.0,\n", + " 0.368,\n", + " 0.1142857142857143,\n", + " 0.0,\n", + " 0.0,\n", + " 0.0,\n", + " 0.0]}\n", + "\n", + "# dataset data_type shd f1 f1_undir time\n", + "# 7 andes discrete 297.333333 0.382214 0.542433 89.976317\n", + "# 8 diabetes discrete 594.000000 0.287690 0.570167 150.839022\n", + "\n", + "\n", + "\n", + "df = pd.DataFrame(data)\n", + "\n", + "# dataset shd_bidag f1_bidag f1_undir_bidag shd_sparsebn f1_sparsebn \\\n", + "# 0 pigs 35 0.958298 0.982128 483 0.351227 \n", + "# 1 win95pts 60 0.675439 0.798246 139 0.263636 \n", + "# 2 hailfinder 46 0.567164 0.746269 61 0.477612 \n", + "# 3 hepar2 66 0.590476 0.780952 147 0.316981 \n", + "# 4 ecoli70 9 0.914286 0.957143 55 0.510949 \n", + "# 5 magic_irri 19 0.862559 0.957346 66 0.468900 \n", + "# 6 magic_niab 6 0.909091 1.000000 29 0.602941 \n", + "# 7 diabetes 453 0.444240 0.720737 842 0.220532 \n", + "# 8 andes 117 0.765321 0.884903 253 0.543353 \n", + "\n", + "# f1_undir_sparsebn \n", + "# 0 0.907975 \n", + "# 1 0.472727 \n", + "# 2 0.611940 \n", + "# 3 0.573585 \n", + "# 4 0.686131 \n", + "# 5 0.899522 \n", + "# 6 0.970588 \n", + "# 7 0.498859 \n", + "# 8 0.725434 \n", + "\n", + "# dataset shd_random f1_random f1_undir_random\n", + "# 0 ecoli70 134 0.028571 0.0571\n", + "# 1 magic_irri 200 0.009804 0.0294\n", + "# 2 magic_niab 125 0.015152 0.0909\n", + "# 3 pigs 1179 0.003378 0.0051\n", + "# 4 win95pts 211 0.035714 0.0804\n", + "# 5 hailfinder 129 0.015152 0.0303\n", + "# 6 hepar2 238 0.032520 0.0325\n", + "# 7 andes 669 0.005917 0.0148\n", + "# 8 diabetes 1200 0.001661 0.0050\n" + ], + "outputs": [], + "execution_count": 4 + }, + { + "cell_type": "code", + "metadata": { + "ExecuteTime": { + "end_time": "2024-06-24T09:34:49.193699Z", + "start_time": "2024-06-24T09:34:48.967118Z" + } + }, + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import seaborn as sns\n", + "import matplotlib.pyplot as plt\n", + "\n", + "# Create a DataFrame\n", + "df = pd.DataFrame(data)\n", + "\n", + "df = df[~df['dataset'].isin(['ecoli70', 'magic_niab', 'magic_irri'])]\n", + "\n", + "# Grouped barplot of Log Time by Dataset\n", + "plt.figure(figsize=(15, 9))\n", + "\n", + "# Calculate the logarithm of time for each algorithm\n", + "df['log_time_lsevo'] = np.log(df['time_lsevo'])\n", + "df['log_time_bidag'] = np.log(df['time_bidag'])\n", + "df['log_time_sparsebn'] = np.log(df['time_sparsebn'])\n", + "df['log_time_bbn'] = np.log(df['time_bbn'])\n", + "df['log_time_dagma'] = np.log(df['time_dagma'])\n", + "\n", + "# Create the barplot\n", + "sns.barplot(\n", + " x='dataset',\n", + " y='value',\n", + " hue='algorithm',\n", + " data=df.melt(\n", + " id_vars=['dataset', 'data_type'],\n", + " value_vars=['log_time_lsevo', 'log_time_bbn', 'log_time_bidag', 'log_time_sparsebn', 'log_time_dagma'],\n", + " var_name='algorithm',\n", + " value_name='value'\n", + " )\n", + ")\n", + "\n", + "# Set labels and title\n", + "plt.title('Log Time Comparison by Dataset')\n", + "plt.xlabel('Dataset')\n", + "plt.ylabel('Log Time (seconds)')\n", + "plt.show()\n" + ], + "outputs": [ + { + "data": { + "text/plain": [ + "
    " + ], + "image/png": "iVBORw0KGgoAAAANSUhEUgAABN4AAAMzCAYAAAB5jN/0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOzdd1iV9f/H8ec5gIgTRAXEmXsP3HvkHplK2vBbZlaWKxv6rTS/prazbGilZpojxW2Se2RumW7cCxRQQJnKOb8/+HESOSgieBBfj+vquvD+rPd9nxuMt59hMJvNZkRERERERERERCRbGW0dgIiIiIiIiIiISF6kxJuIiIiIiIiIiEgOUOJNREREREREREQkByjxJiIiIiIiIiIikgOUeBMREREREREREckBSryJiIiIiIiIiIjkACXeREREREREREREcoASbyIiIiIiIiIiIjlAiTcREREREREREZEcYG/rAERERB7UsmXL+O9//wvA3LlzadKkiY0jyh5Vq1Z9oPaffPIJffr0sfST+ufHxdGjR/H19WXXrl1cunSJqKgo8uXLR/HixalRowZt2rShU6dOFCxY0NahPjbGjh3L8uXLady4MfPmzbN1OA9V+/btuXjxIsOGDWP48OG2Dseq7777ju+//z7d9fz581O4cGGcnZ2pWrUqtWvXpmvXrri5udkgShERkUeLZryJiIhInhIVFcVbb71F7969mTFjBoGBgYSHh3Pz5k1iY2M5e/Ysvr6+jB07lnbt2j12CSCR+5WQkEB4eDghISGsWbOGTz75hPbt2zNy5EjCw8NzdOz27dtTtWpVvvvuuxwdx5Yeh3sUEXmcacabiIhILuXn52f1emhoKN27dwfgtdde47XXXrNaz9HRMcdiy60uXLjA4MGDOXPmDAB16tShX79+NGjQAFdXV5KSkrh06RJ///03K1as4NKlS/z6668MHDjQtoGL5DJ//vknHh4eACQnJxMTE8OlS5fw8/Nj+fLlnDlzhr/++os9e/bw008/UbduXRtHLCIikjsp8SYiIpJLZbQEMn/+/JavHRwc7rlU8tixY9kaV26VlJTE8OHDOXPmDAaDgf/+97+8+OKL6eq5u7vToEEDhg4dypw5c/Dx8bFBtI+nTz/9lE8//dTWYUgm5M+fP83PliJFilC6dGkaN27Mq6++yq+//spXX33FtWvXeOONN1i6dCnu7u42jFhERCR30lJTERERyRNmzpzJ4cOHARg+fLjVpNvt8uXLx6uvvsoPP/zwMMITyTOMRiODBw/m7bffBiAiIsLq3nAiIiKiGW8iIiKYzWb+/PNPVq1axaFDh4iOjqZQoUJUrVqVbt260bdvX+zt7/5X5tatW5k3bx4HDx4kISEBDw8POnXqxCuvvEJMTAwdOnQAbHP4w90OVxg4cCB79+7l6aef5tNPP2X37t3MmTOH4OBgbty4QenSpenduzcvvvgi+fLlA+D69ev8/vvvrF27lgsXLmBvb0+9evUYNmzYPZebJSYmsmTJEjZs2MDx48e5fv06hQsXpmbNmvTp04euXbtiMBju+x4TExMte7V5enpmuPzWmsqVK1u9/iDvxZ0b6a9fv54FCxZw9OhREhMTKV++PAMGDOCZZ56x3G94eDhz5sxh06ZNhIaG4uTkRJMmTRg5ciRPPPFEpsZZuXIlS5YsISQkhISEBMqWLUu3bt0YNGhQmpmSt7tx4wb//PMPW7ZsITAwkNDQUG7duoWzszM1a9bkqaeeuuvncuc79Pfff7Nw4UKCg4OJjIzEy8vL8tnc63CFq1evMnfuXLZv387Zs2dJSEigaNGiFCtWjOrVq9OyZUu6d+9u9bnHxcUxf/58Nm7cyOnTp4mLi8PFxYX69evj7e1Nq1atrMZ/4cKFNN+fXl5ezJ8/n5UrV3L69GkAKlWqRL9+/dJ8Xg/qfj6rFStWMGbMGADWrl1LxYoVM+z30qVLdOjQAZPJxPjx43n++eezJV5rXn75ZRYvXsyZM2dYsWIFo0ePplixYmnqXL58mS1btrBt2zaOHj1KREQERqOR4sWL4+XlxfPPP2/150bqu5Lq+++/T5fcu/OgimPHjrFlyxb++ecfTp48SXR0NI6Ojnh6etKiRQtefPFFy/JZa1LfoU2bNnHq1CliY2MpXLgwxYoVo1KlSrRs2ZIePXpQoEABq+337NmDj48PBw4cICIiAnt7e8qUKUP79u156aWXKFq06APfo4iIPHqUeBMRkcfajRs3ePPNN9m9e3ea69euXWP37t3s3r2bRYsW8dNPP1GyZEmrfXz66af8+uuvaa6dPn2an376iTVr1jwyS+t+/vlnvv76a8xms+XaiRMn+PLLL9m3bx8//vgjly9f5pVXXuHUqVNp2m7fvp1du3bxyy+/0KxZM6v9h4SE8Prrr3PhwoU0169evcrff//N33//zapVq5g6dSpOTk73FfvevXu5evUqAH369LlnovResuO9SDVhwgQWLlyY5trhw4cZP348Bw8e5OOPP+bIkSMMGTIkzUb1CQkJ/PXXX/zzzz/Mnz//nqfc3vlLPMDx48c5fvw4f/75J7/99huurq7p2o0ZM4aNGzemux4eHs7WrVvZunUrq1ev5ttvv7UkXzMydepUZsyYcdc6GTlx4gT/+c9/iIyMTHM9MjKSyMhIQkJCWLVqFa1atUqX3Dlx4gRDhgzh0qVLaa5fuXKFdevWsW7dOp566ikmT56Mg4NDhjHEx8fzn//8hwMHDqS5HhQUZPlv8uTJWbq/293vZ9WlSxcmTZrE9evXWbp0Ke+9916GfS9btgyTyYSjoyM9e/Z84FjvxmAw0LdvX7766itu3rzJ3r176dKlS5o6PXr0ICYmJl3bCxcucOHCBVatWsVbb711X8lya44ePcpTTz2V7vqtW7csz3bx4sV89913tGjRIl29K1euMHDgQMv+kKmuXbvGtWvXOHnyJOvWraN69erUrl07TZ2kpCQ+/PBDVq5cmeZ6YmIiR48e5ejRoyxevJgZM2akaysiInmfEm8iIvJYe+uttyzJlaeeeoqBAwdSpkwZLl++jI+PD/PmzePw4cO8/vrr/PHHH+l+aV+yZIkl6VarVi1GjRpFzZo1iY+PZ9OmTUybNo0PPvjgod/X/dq7dy8rVqygU6dOvPzyy5QvX57IyEhmzZrF0qVL2bZtG0uXLmXJkiXExMQwceJEWrVqRf78+dm/fz8TJ04kPDycDz74gPXr16dLfIWFhTFw4ECuXbtmmZHWpEkTXFxciIiIwNfXl59++oktW7YwYcIEPvvss/uKf//+/ZavGzVq9MDP40Hfi1QrV67k/PnzDBgwgAEDBuDh4UFYWBjffPMNW7ZsYfHixbRu3ZpPPvkEJycnvv76axo1aoTRaGTbtm2WZIu15N3tVqxYwYULF+jatSsvv/yyJdZFixaxcOFCQkJCGDFiBL///nu6GVuurq4MHDiQJk2a4OnpSYkSJTCZTISFhbF27VoWLVrE5s2b+e677yxLC63ZuXMnly9fpm3btrzyyitUrFiR2NjYdImMjIwfP57IyEicnJwYMWIEbdq0wdXVleTkZC5evMiBAwdYvXp1unZRUVG8/PLLXL58mXz58vHaa6/RvXt3nJ2dOXnyJNOnT2fHjh2sXLmSwoULM27cuAxjmDRpEpGRkYwePZpOnTpRrFgxTp8+zRdffMH+/fvx8fGha9eutGzZMlP3ZE1WPqv8+fPTo0cPFi5cyKpVqxg9erTV5LLZbLYk9Dp27EiRIkWyHGdmNWjQwPJ1QEBAusRbxYoVadmyJfXq1cPNzQ1XV1fi4uI4c+YMixcvZt26dXz99ddUr16d1q1bW9pNnDiRcePG0aNHDy5dumT1IJk7v++aNm1K+/btqVGjBiVKlMDZ2ZmrV69y6NAh5syZw8GDB3nrrbf4888/KVGiRJq2X3zxBWfOnMHOzo7XXnuNzp07W5Lqly9fxt/fH19fX6szHseMGcPatWuxt7fnueeeo2fPnpQpU4abN29y4MABpk2bxqlTp3j99ddZsWKFZeys3KOIiDyCzCIiIo+4pUuXmqtUqWKuUqWKeffu3Zlut2HDBku7KVOmWK0ze/ZsS5158+alKUtISDA3btzYXKVKFXOvXr3McXFx6drv37/fXK1atSzFl5Hz589b+ps2bdo966fWXbp0abqyF154wVL+4YcfWm0/YMAAc5UqVcw1atQw169f33z69Ol0df7++29LP3///Xe68qFDh5qrVKli7t69uzkqKsrqOFu3brX0ERwcfM/7ut3o0aMtbcPDw++r7Z0e9L0wm83mdu3aWcqnT5+erjwxMdHcvn17y3Nt06aNOTIyMl29RYsWWfo5derUXccZO3as1VinTp1qqbN27dp73X46qZ9LvXr1zNevX09Xfvs7NGrUKLPJZMqwrzFjxpirVKlifuGFF9Jcv379uqWP33777b7imzx5sqXthg0b0pUnJyeb33jjDUudY8eOpSm//fupevXq5r1796brIzY21tyiRQvLPWbFg35WBw8etJRt2rTJavtdu3ZZ6uzcufO+Y5w2bZql/fnz5zPVJjw83NJm9OjR9z3m559/bq5SpYr5+eeft1qe+twy87Pubm7evGn5Wfbtt9+mK0/9WZ7R93xGUn9eVKtWzbxt2zardWJiYsydOnUyV6lSxTxx4sR05dl1jyIikjvpcAUREXlspZ5m6eLiwujRo63Weemll6hUqRIAixcvTlO2efNmoqKiABg9erTV5ZFeXl5069YtG6POGU5OTrz77rtWy7p37w6kLNkaOHAg5cuXT1enRYsWODs7AxAYGJim7Pz582zevBmA999/P90+R6natGlD48aNAazObLqb6Ohoy9cPOsvnQd+L23l4eDBkyJB01/Ply0enTp2AlOf65ptvpls+CSnPPnWGzZ3P9XaOjo6WPcDu9MYbb1hm2CxdujTDPjLSpk0bihUrRlxcHAEBARnWs7OzY+zYsVnaAy05Odny9b2W7t7ZLnWGV9u2bXnyySfT1TEajYwfP94yQ+xun1fXrl2tzpgsUKCAZSZXcHBwpuOzJqufVc2aNalZs6bVslTLli0DoHTp0jRt2vSB4sys27/fbv8+zKynn34aAH9/f+Lj47MtrjvZ29vTo0cPIGV25p1u3boF3N/7B/Dbb78BKd+rt8/Yu13hwoV5/fXXAVizZk2a5fwiIpL3aampiIg8lsxmM35+fgC0a9cOR0dHq/UMBgNdunTh+++/5/jx48TExFh+0Uxtnz9/fqt7BqXq0KEDa9asyeY7yF5169bNMGFVpkwZy9cZbVBvMBgoW7YsUVFRafYpA9i9ezdmsxlHR0fq1KlDbGxshnFUr16dvXv3PnByI6uy4724XfPmzbGzs7Pax+3PNaOli4UKFaJYsWJERkame663a9SokSXxead8+fLRrl07Fi9eTEBAAGazOV1yLCwsjEWLFrF7925Onz7NjRs3LImI250+fTrDWKtVq4abm1uGMd5N0aJFKVWqFJcuXWLq1KkUL16chg0b3rNd6rOHlKRZRtzc3GjQoAF79+5Nt3/b7TJ6vwEqVKgApJzg+SAe5LPq168fhw4dYtu2bURGRqbZB+7GjRusX78eSElmZdchEPdyexIpozGDg4NZsmQJ/v7+XLp0ibi4OEwmU5o6t27d4ty5c/fcy/BeNmzYwJo1azh06BARERFWk3mph2bcrnr16uzbt49Zs2ZRsWJFWrVqleH3bqr4+Hj8/f0BaNas2V1/tqUm6qOiojh//jxly5a9n9sSEZFHmBJvIiLyWLpx44ZldkbqL0QZSS03m81cunTJkmC5ePEiAGXLlr3rZv4ZnUiZm9xtlsftJyxmpl5iYmKa66kHMSQmJuLl5ZWpeFIPSsis22fRxcTEULx48ftqnyo73ovb5eRzvd3dTrm8vfz69etER0enSfxs3LiRd999l7i4uLv2kdo+I7cnErNizJgxjBo1ijNnzvD8889TokQJGjdujJeXFy1atLA60zL1exDu/QwqVarE3r1707S5090Sh6kzWh90VtaDfFY9e/bk888/Jz4+npUrV/Lyyy9bytauXUt8fDxGozHd6cU56fZ3wtps1qlTp/LTTz9lapbX3d6ve4mPj2fYsGHs2LEjS+O8/fbbDBw4kMjISF577TWcnZ1p1KgRXl5eNGvWjGrVqqVrc/78eW7evAmkzOZ9//33MxXr1atXlXgTEXmMKPEmIiKPpdtnJhQsWPCudW8vv71daqKiQIECd21/r/Lc4F4zO1IZjffepeLOX7Cz8st0UlLSfdUvXbq05etTp05lOfGWHe/F7TL7XDNT726Ji3u9Y3fGmprMuXDhAqNHjyYxMRFPT08GDRpE/fr1cXNzw8nJyTKDqVu3boSFhaVZEnqn+z2J9k5dunRhzpw5zJgxgz179hAeHs6ff/7Jn3/+CUD9+vUZM2YM9evXT3Mv1u7RmtTyu81Kysz7/aCy+llBypLFzp07s2LFCpYtW5Ym8Za6/LR58+aUKlUqe4O+i9tnj92ZQF67dq3llNuGDRsyYMAAqlevTrFixciXLx8Gg4GLFy9aTl+92/t1L59++qkl6fb000/TpUsXnnjiCQoXLmw5jXflypX873//szpO/fr1Wbx4MT/88APbtm0jKiqKDRs2sGHDBgAqV67MO++8Q9u2bS1tspoovFsSXURE8h4l3kRE5LF0+y+395rpc3v57e1Sf4G+n/aPo9Tn5OzszJ49e3JkjNuXJe7du9eyV9z9yo73whbuFWtGCaqlS5eSmJhIoUKFWLx4cYYJyxs3bmRPoPfQtGlTmjZtSkxMDP7+/vj7+7N9+3YOHTqEv78/AwcO5Pfff6devXpA1j6vR/WzSuXt7c2KFSsICQkhKCiIOnXqcPLkScv+e3379s3WeO8ldaklpD3hFGD+/PlASlJr3rx5VhOb1pY036/4+HjLXn9DhgzhnXfesVrvXgn9GjVq8MMPPxAXF0dgYCABAQH8888/7N+/n5CQEF577TWmTZtG586dgbSfz/Tp02nfvv0D34uIiOQ9OlxBREQeS4UKFbIsiwoJCblr3dRyg8GQZiZJ6tfnzp2760yN1KWWj6vUJYjR0dGWwyiyW6NGjXBxcQFSNpjP6syZ7HgvbOHkyZOZKr/9/gCOHDkCpCS8Mkq6Xbp06aEl3lIVKVKENm3aMGrUKJYtW8acOXNwdHTk5s2b/PTTT5Z6t890PHHixF37TP28PD09cyboTMrqZ5WqYcOGluXrqQeBpB6q4OzsbPWAiZxiNpstM+3y5cuXbl++1Pera9euGc4mPH78+APHcerUKcssstQDFKw5duxYpvorUKAAzZo1Y+jQofz++++sWLHCMvPwhx9+sNTz9PS03Ne5c+eyGL2IiOR1SryJiMhjyWAwWGZnbN26NcOZEGazmXXr1gFQpUqVNPt4pbZPSEjgn3/+yXCsTZs2ZVfYj6TUgyfMZjO+vr45Mkb+/Pl54YUXgJR9v1KXt2XG7Qm27HgvbGHfvn0ZJjWTkpLYsmULkDLz6PYN8FP3p7pbonLlypXZF2gWNWvWzHJK5+2J7MqVK1ue/V9//ZVh+8uXL1sOzcjsPoM5Jauf1e369esHpCzlvHHjhuUz6tmzp2VZ5cMwa9Yszpw5A0CfPn3SHRqR+v1z50EKt7vX+5W6f+bd3tHbv08zqhcbG5vln8XVqlWznE59+/tXuHBh6tatC6R8FlmVmXsUEZFHlxJvIiLy2PL29gZSNrqeOnWq1Tpz5861JGb69++fpqxDhw6WXzS//vprEhIS0rUPCAjIsWTTo+KJJ56gXbt2AHzzzTf3nEl248YNrly5ct/jDBkyxLIB+vfff8/cuXPvWj8pKYlZs2YxbNiwNNcf9L2whcTERD777DOrZdOnT7eciHrnMsTUGWP+/v5cu3YtXdvjx4/z888/Z3O06V29etXq+KmSk5MthyKkzmyElL3xUg8S2LJlC1u3bk3X1mQyMWnSJMuSxmeeeSYbI79/Wf2sbvf000/j4ODA9evXGT9+fKbaZCeTycTs2bP5+uuvgZS93d5888109VJnu27evNnqHoXLli1j586ddx0r9fO+28+E22c+Wkuumc1mJk2aZDk45U5xcXGEhobeNY7UGW13JhcHDRoEQGBgYJrZmNaYzWarM6Azc48iIvLo0h5vIiKSp5w4cQJHR8e71ilatCgVKlSgQ4cOtG7dmu3btzN79myioqJ4/vnnKV26NFeuXMHHx4d58+YBULNmzXS/sDs6OvLWW2/x0UcfceTIEV544QVGjRpFjRo1SEhIYPPmzXz77bd4eno+9suQPvroI4KDg4mIiOCZZ55h4MCBPPnkk5QuXRqDwcDVq1c5duwY27dvZ/369UyZMoUuXbrc1xiOjo58//33vPzyy5w7d47JkyezZs0a+vXrh5eXF8WKFSMpKYlLly7xzz//sGzZMi5evJhu6eGDvhe2ULp0aZYtW0ZCQgIvv/wyZcqU4fLlyyxatIgFCxYAKUsU73ym3bp1Y9GiRURFRTF48GDefvttqlWrRlxcHJs3b+aHH36gYMGC5MuXL8eWCUPKrMNXX32VJ598knbt2lG9enVcXV1JTEzk1KlTzJkzx7KUtHv37mnaDh06FF9fXy5fvsyIESN4/fXX6d69O0WLFuXkyZPMmDGD7du3AzBw4ECqVKmSY/eRGVn9rG5XrFgx2rdvz7p16yyHT9SsWZPq1atnW5wJCQmW/eZMJhMxMTFcunQJPz8/li1bZpnpVqxYMX788UerJ/N269aN77//nr179/L222/z8ssv4+npyeXLl1m+fDnz5s2jUqVKd10mXKtWLQICAti0aRN79uyhdu3alll9RqMRo9FoOQF37969/PzzzxiNRrp164aLiwsnTpxg5syZbN26NcOxrl69SufOnWndujVPPvkktWrVokSJEiQnJ3P+/HkWLVpkObjhzqWsnTt3pmfPnqxevZqvv/6aAwcO0L9/f2rWrEnBggW5ceMGZ86cYd++faxdu5Zy5cqlS9Bl5h5FROTRpcSbiIjkKRMnTrxnnQ4dOvDjjz8CMHXqVN588012797NsmXLLHsl3a5GjRrMmDEDBweHdGUDBgzg5MmTzJ07l+DgYAYPHpym3NPTk0mTJvGf//wHyPwpl3mNh4cH8+fPZ/jw4Rw/fpyffvrprrNDrD3rzChTpgyLFy/mo48+Yv369QQGBhIYGJhhfRcXF1599dV01x/0vXjYevfuzfnz51m5cqXVJW+VK1fm22+/Tbd0sUmTJjz33HMsWLCAQ4cOpTklE1KS1N988w3vvfdejibeICXRs2bNGtasWZNhnV69evHcc8+luebs7Mzs2bMZMmQIly5d4ttvv+Xbb79N1/app55izJgx2R73/crqZ3Unb29vy3JnyP7ZbncmOO9kb29Pp06d+OCDDzLcH/CVV15h27ZtBAcHpzmhNlXlypWZMmWKZZapNc899xxLliwhKirK8nM01bBhwxg+fDgAEyZM4LnnniMqKorvvvuO7777Lk3dbt260aJFCz744AOr49y6dYvNmzezefPmDGNp3rw5I0aMSHf9k08+oWjRovz+++9s27aNbdu2ZdhHpUqVsnyPIiLyaFLiTUREHmuFChVizpw5rFmzhlWrVnHo0CFiYmIoWLAgVatWpVu3bvTr18+yB481H3zwAc2aNeP333/n4MGDJCQk4OHhwZNPPsmQIUO4fPlymvEeV+XLl2fFihWsXbuWdevWERwczNWrVzGbzbi4uPDEE0/QsGFDOnbsaFkymhUuLi5MmzaNo0eP8ueff7J7924uXbpEdHQ0Dg4OlChRgpo1a9K2bVs6deqEk5NTuj6y47142D7//HOaNm3KkiVLOHnyJImJiZQpU4Zu3brx8ssvkz9/fqvtPvroI+rUqcOiRYs4fvw4JpMJNzc3WrduzaBBgx7KYQT169dnzpw57Nq1iwMHDhAaGkpkZCQmk4kSJUpQt25d+vTpQ6tWray2r1SpEn/++Sfz589n48aNnDp1ivj4eFxcXKhfvz7e3t4ZtrWFrH5Wt2vRogWenp5cvHgRR0dHevbsmWPxOjo6UqhQIVxcXKhWrRp16tShS5cuuLm53bWdk5MT8+bNY+bMmaxdu5YLFy7g6OhImTJl6Ny5My+++CKRkZF37aNixYosXLiQX375BX9/fyIjIy17E95Zb9myZUyfPp3t27dz9epVChcuTJUqVejTpw9PPfWU1QQ6pByUs3DhQnbu3Mn+/fu5ePEiERER3Lx5E1dXV2rUqEHPnj3p2rWr1YSog4MD48aNw9vbm8WLF7Nv3z4uXbpEfHw8BQsWpHTp0tSuXZvWrVvTunXrLN+jiIg8mgxmaxsuiIiISLbZsGGDZR+x3bt3p9mjSuRBtG/fnosXL2pWzGOqa9eunDp1ih49evDVV1/ZOhwRERGxQhsGiIiI5LDUzb49PT2VdBORbBEQEGDZqD/1lFMRERHJfZR4ExEReUB32/tq3759rFq1CkjZY0hEJDvMmTMHSFnC3bRpU9sGIyIiIhnKPRuTiIiIPKJ69OhBly5d6NChA5UqVcLBwYHQ0FDWr1/P7NmzSU5OxtnZmZdeesnWoYrIIy42Npbly5fj6+sLwODBg+95EIOIiIjYjhJvIiIiDyg2NpZ58+Yxb948q+VFixblhx9+yPDkPxGRe9mzZ0+6Ey/r1auX7aeZioiISPZS4k1EROQBffbZZ2zdupXAwEAiIyO5fv06BQoUoGzZsrRu3ZqBAwdSrFgxW4cpInmA0WjEzc2NDh06MGLECOzs7GwdkoiIiNyFTjUVERERERERERHJATpcQUREREREREREJAco8SYiIiIiIiIiIpIDtMfbfYiMvI4W5oqIiIiIiIiIPL4MBnB1LZypukq83QezGSXeREREREREREQkU7TUVEREREREREREJAco8SYiIiIiIiIiIpIDlHgTERERERERERHJAUq8iYiIiIiIiIiI5AAl3kRERERERERERHKAEm8iIiIiIiIiIiI5QIk3ERERERERERGRHGBv6wDyuuTkW5hMJluHISJ5mNFoxM5OP85FRERERERyG/2mlkPi42OJjY3h1q0kW4ciIo8Be/t8FCxYBCengrYORURERERERP6fEm85ID4+lujoCPLlc8LZuQR2dnaAwdZhiUieZCY5OZm4uBtER0cAKPkmIiIiIiKSSyjxlgNiY2PIl88JF5cSGAxKuIlIznJwAEdHJ65dCyc2NkaJNxERERERkVxChytks+TkW9y6lUSBAoWUdBORh8ZgMFCgQEFu3UoiOfmWrcMRERERERERlHjLdqkHKaQsLxUReXhSD1jQgS4iIiIiIiK5gxJvOUaz3UTkYdPPHRERERERkdxEiTcREREREREREZEcoMSbiIiIiIiIiIhIDlDiTUREREREREREJAco8SaPJT+//bRs2ZBhw161dSiZ0rJlQ1q2bHjf7UJDL9GyZUP69euZA1GJiIiIiIiIyN0o8SbyCBs27FVatmyIn99+W4ciIiIiIiIiInewt3UAInJv8+f72DoEEREREREREblPSryJPALKlStv6xBERERERERE5D4p8SZ5wuHDB9m6dRP+/ge4fPkyMTHRFC5chOrVa+LtPYBGjZrcV3+BgQH89tssDh8OJjk5mXLlKtC37zN07drDstfajh3pl3deuXKZ+fN/Y8+eXVy5chkHBweeeKIinTt3p2fP3tjZ2aWpv3btaqZM+R9du/Zg+PC3+PXXmfzzz3bCw69Qs2Ztvv/+Z4B0Y/r57WfEiNct/dz+NcD7739Et25p93Uzm82sWrWclSuXce7cGezs7KhRoxaDB79GrVp10t3L7WOuW7cWH59FnDlzGkdHR7y8GjN06Ajc3d0xm80sW7aY1atXcuHCORwdHWnevBVvvDECF5di9/XcRURERERERPISJd4kT/jppx/x999PhQpPULVqNfLnd+LixQvs3Pk3O3f+zYgRb/PMM89mqq+NG9cxceI4TCYTFStWokKFikREhPPJJxM5c+Z0hu2OHDnE22+PICYmGjc3d1q1asONG7H4+x8gODiI7du38tlnX+Pg4JCubXR0FIMH/4cbN65Tt249qlatbrVeKlfX4nTt2oM9e3Zx9WokjRs3w9XV1VLu6VkmXZspU/7Hhg1/UbdufZo3b0VIyDH27dtDYKA/3333MzVr1rI61owZ37Nw4Tzq1WtAkybNOXLkEJs2rSc4OJA5cxby5ZefsGPHdurX96JUKU+CgwPx9V3D8ePHmDlz7l3vQ0RERERERCQvU+JN8oQBA55n3LiJFC9ePM31gweDePvt4fz447e0a9eBEiVK3rWfiIhwPvtsMiaTiZEj38Hbe4ClLCDAj3ffHWm1XVJSEuPGjSUmJprevfsyatS72NunfHtdvHiBUaPeYO/eXcye/TOvvfZmuvY7d+7Ay6sxU6Z8TsGChe55v+XKleeDDyYwbNirXL0ayQsvvEiDBhmfehoWFoq//wHmzv2DsmXLAZCcnMznn0/mzz9XMWvWDL7++nurbVevXs7MmfOoXLkKAImJCbz11jCCggIYPvxVEhISWLDAB3d3DwCioqJ4/fVBnDwZwpYtG+nUqes970dEREREREQkL9KpppInNGvWIl3SDaBWrTr06fMMt27d4u+/t92znzVrVhIfH0etWnXSJN0A6tVrQO/e/ay227JlI2FhoRQvXoIRI962JN0APD1L8+abKQm7pUsXk5iYmK69vb097733fqaSblk1atS7lqQbgJ2dHa+++gaQklS8deuW1XaDB79uSboBODrmp3//5wE4efIEo0a9Y0m6ATg7O9O7d18A9u/fm+33ISIiIiIiIvKo0Iw3yTOio6PYuXMHp0+f5Pr165ZE0oUL5wA4d+7sPfvw9/cDoGPHLlbLO3XqwsKF86y0OwBAhw6dyJcvX7ryNm3aU7hwEa5fj+HYsSPUqVMvTXnlylXx9Cx9z/iyys7OjqZNm6e77upa3BJXdHQUrq7pk5fNmrVId61MmTKWfhs1apquvHTpsgBEREQ8aOgiIiIiIiIijywl3iRPWLVqOd999zXx8fEZ1omLi71nP+HhlwHw8Chltdzd3fr18PBwAEqVsl5uMBjw8CjF9esxlrq3y2i87OLqWjzNLLzbFSxYkOvXY0hKSrJa7ubmnu6ak1OBu/ZboEBKeVJS+tl9IiIiIiIiIo8LJd7kkXf06BG++GIKRqORoUOH06JFa9zc3MmfPz8Gg4GVK5fxxRdTMJvNme7TYMjoegYFD8jR0TFH+k1lNGZ9Vfnd2j5IvyIiIiIiIiJ5nRJv8sjbsmUjZrOZfv368/zzL6Yrv3DhfKb7KlGiJOfOnSU0NNRqeWjopQzalQDg0qWLGfad2ja1roiIiIiIiIjkbZquIo+8mJgYANzcPNKVJSYmsnXr5kz3VbdufQA2blxntXzDhr+sXq9f3wuATZs2WD08Ydu2LVy/HkOBAgWpWrV6puO5FwcHByDlhFIRERERERGRzDAaDdjbG23yn9GYMyvJcisl3uSRV758eQD++mtNmn3cEhMT+eqrTwkNzXgW2p169HiK/PnzExQUwNKli9OUBQUFsHy5j9V27do9iZubOxER4Xz33dQ0J4ReunSR77//BoC+fZ/J1mWlJUqUBOD06VPZ1qeIiIiIiIjkXUajARdnJ1xcCtrmP2enxyr5pqWm8sjr1q0XS5Ys4vjxY3h796JOnfrY2RkJDAwgMTERb+9nWbJkYab6KlnSjXfffZ/JkycwdernrFq1nAoVniAiIpygoAD693+ehQvnpTtQIF++fEya9Blvvz2CFSt82L37H2rWrEVcXBwHDuwnKSmRxo2b8fLLr2brvbdt24G1a1czffo09u/fi4uLCwaDge7de1G7dt1sHUtEREREREQefUajAaOdHUc+/pi4s2cf6tgFypWj+rhxGI0GTKbM78P+KFPiTR55hQsXZubMecya9RN79+5iz56dFClSlMaNmzBo0KsEBQXcV3+dO3ejZEk35s6dzeHDB7l48Txly5bnvfc+oFGjJixcOI+iRZ3TtatevSa//jqf+fN/Y/funWzfvhUHh3xUqVKVLl260aNH7wxPFs2q5s1bMmbMhyxf7oOf3z4SEhIAqFOnnhJvIiIiIiIikqG4s2e5cTzE1mHkeQbz/Rz1+JiLiLjOvZ7WzZtJREaG4urqgYNDvocTmDw0vr5rmDx5Ai1atOKzz6baOhyRNPTzR0RERERE7sXe3oiLS0EOvPLKQ0+8FapSGa+ZM7l2LZZbt0wPdezsZDBA8eKFM1VXe7yJ3CEsLIzIyIh014OCAvjhh2+BlOWtIiIiIiIiIiJ3o6WmInfw89vHp59+TKVKlXFzc8doNHLx4kVOnDgOQLduPWnTpp2NoxQRERERERGR3E6JN5E71KxZm27dehIY6I+//wHi4+MpXLgwDRs2pnv3XnTs2MXWIYqIiIiIiIjII0CJN5E7lCtXnrFjx9k6DBERERERERF5xGmPNxERERERERERkRygxJuIiIiIiIiIiEgOUOJNREREREREREQkByjxJiIiIiIiIiIikgOUeBMREREREREREckBSryJiIiIiIiIiIjkACXeREREREREREREcoASbyIiIiIiIiIiIjlAiTcREREREREREZEcYG/rAB5XRqMBo9Fg6zAyzWQyYzKZbR2GiIiIiIiIiMgjQ4k3GzAaDTg7F8DO7tGZcJicbCIqKi7bkm/9+vUkLCyUJUtW4eFRKlv6tKXQ0Et4e/fC3d0DH5/Vtg4nW0yePAFf3zW8//5HdOvW09bhiIiIiIiIiDxylHizAaPRgJ2dkQ8X/M3pK9G2DueeKpQsyqTnWmE0Gh7bWW95LVEoIiIiIiIiIjlPiTcbOn0lmqMXr9o6DMkGJUqUZP58H+zt9S0lIiIiIiIiIimUJRDJBvb29pQrV97WYYiIiIiIiIhILqLEm+QqCQkJLFmyiC1bNnD+/HlMpmQ8PDxp3botAwa8QJEiRay2CwwM4LffZnH4cDDJycmUK1eBvn2foWvXHrRs2RCAHTv233c8a9euZsqU/1n+7O3dK035tGkzaNCg4V33eLt9/HXr1uLjs4gzZ07j6OiIl1djhg4dgbu7O2azmWXLFrN69UouXDiHo6MjzZu34o03RuDiUsxqfOfOnWXRot/Zv38vERHhODg4UKlSFXr1eprOnbvd9/1mhslkYvXqFfz11xpOnz5FfHw8hQsXwdW1OPXq1WfAgBfSLce9desWvr5rWLduLSdPniAhIZ7ixUvQpEkzBg4chJubu6XuihVL+fLLT2jcuBlff/2d1Riio6Po3bsrZrOZ5ct9cXFxsZSdPXuG+fN/48CBfVy9Gkn+/E5UqVKVXr360KFDxxx5JiIiIiIiIiLWKPEmuUZMTDQjRw4lJOQ4BQsWxMurIfb29gQE+DF37mw2bFjHtGnT0yV1Nm5cx8SJ4zCZTFSsWIkKFSoSERHOJ59M5MyZ0w8Uk6dnGbp27cHWrZuIj4+nbdv2ODkVsJS7uhbPdF8zZnzPwoXzqFevAU2aNOfIkUNs2rSe4OBA5sxZyJdffsKOHdupX9+LUqU8CQ4OxNd3DcePH2PmzLk4ODik6W/z5o1MmvQRSUmJlCtXnqZNWxAbe4PDhw/y8cfjOXBgH++//9ED3b81n376MWvXriZfPkfq1KmLs7MLMTExXLp0kaVLF+Pl1TjNZxQXF8uYMaPx9z+Ak1MBqlathrOzC6dOnWDFiqVs2bKRqVN/oEqVagA8+WRnvvvua/bv30N4+BVKlCiZLoYNG/7i5s2btGnTLk3SbefOHXz44RiSkhIpW7YcrVu3IyrqGgEBfhw4sI+9e3fx3/+Oz/ZnIiIiIiIiImKNEm+Sa3z11aeEhBynRo1afPHFNxQt6gxAXFwc48ePZffunUyc+CHTp8+2tImICOezzyZjMpkYOfIdvL0HWMoCAvx4992RDxRT3br1qFu3Hv7+B4iPj+fNN0dl+XCF1auXM3PmPCpXrgJAYmICb701jKCgAIYPf5WEhAQWLPDB3d0DgKioKF5/fRAnT4awZctGOnXqaunr5MkTTJo0HjAwefLntGnT3lIWFhbKmDFvsXbtaurX96Jr1x5ZfwB3CAsLY+3a1ZQs6cYvv/yWLvF45sxp8ud3SnPtiy8+wd//AM2bt+K//x2XZvbe4sULmDbta8aPf5/585dgZ2dHoUKFaNOmPevX+/LXX2sZOPCldHGsXZsyq7Bbt39nIF69GsnEiR+SlJTIkCFD+c9/XsZgMABw9Ohh3nprGH/+uYqaNWvTq9fT2fVIRERERERERDJktHUAIpCS0NmyZRMGg4H33vvAknQDKFCgAGPGfEi+fI4EBwcRHBxoKVuzZiXx8XHUqlUnTdINoF69BvTu3e9h3cI9DR78uiXpBuDomJ/+/Z8HUhJpo0a9Y0m6ATg7O9O7d18A9u/fm6avuXNnkZSUxJAhQ9Mk3QDc3T0YO3YcAD4+f2TrPVy7FglAlSpVrc72K1++Au7u/y4bPXPmNBs3rqN48RJMmDAp3ZLZZ555jmbNWnDhwjl2795pud69e0pCzdc37bJdgJCQ4xw/fgxXV1eaNGlmub5q1XJu3LhB1arVefHFwZakG0C1ajX4z39eBmDBgnlZuXURERERERGR+6bEm+QKgYF+mEwmKleuSqVKldOVlyhRkiZNmgLg5/fvXm3+/n4AdOzYxWq/nTpZv24LzZq1SHetTJkyANjZ2dGoUdN05aVLlwUgIiLCcs1kMlmSVBntWVatWg2cnAoQEnKMxMTEB449Vbly5SlQoCC7dv3Db7/N4tKli3etv2vXP5jNZpo2bU6BAgWt1qlf3wuAgweDLNcaNGiIh0cpzp07m+Y6/DvbrXPn7mlOkU19F7p27W51nB49ngLgwoVzRESE3zVuERERERERkeygpaaSK4SHXwGgVKmMl3GWKlX6/+v+mzQJD78MkOHyT3f3rC0LzQm3HyCQKnW/OFfX4mmSSKkKFEgpT0r6N3kWHR1NbGwsAH36WE8y3S4mJtrqPmlZUaBAQd5/fzxTpkzkl1+m88sv03F1LU7NmrVp0qQZHTt2scQMWBJza9asZM2alXftOyrqmuVrg8FA1649mD37Z/78czW1atUBUg5p2LDBF/h3VlyqiIiUd8jDw9Nq/4ULF6ZIkaLExERz5cplihcvcZ93LyIiIiIiInJ/lHiTPOG2VYV3XM+gwAaMxownmN6t7E5ms8nydWb2b3NwyJfpvjOjbdsONGzYhB07thEYGEBwcCDbt29h+/YtzJr1E1On/kDFipXSxFq5chUqVapyt26pUaNWmj9369aTX3/9hS1bNjBq1Ns4Oubnn3+2ExUVRc2atSlXrny23peIiIiIiIhIdlPiTXKF1BlZd1u6mFpWosS/M5VKlCjJuXNnCQ0NtdomNPRSNkaZOxQt6oyjoyOJiYm8+eYonJ2dH3oMhQoVokuX7nTpkjLj7vLlML755gv+/nsbU6d+zvff/wxAyZJuANSuXZfRo8fc1xju7h40aNCIAwf2sm3bFjp16mpZZnrnbDeA4sVLcvbsmQzfoRs3bhATEw2QbTMARURERERERO5Ge7xJrlC3bgOMRiMhIccJCTmerjwiIoI9e3YBKft//duuPgAbN66z2u+GDX9lS3z29g4AJCcnZ0t/DyJlP7gmAGzevMHG0aRwc3Pn5ZdfAyAk5JjletOmzQHYsWN7lvaaS02wrV27mqtXI9m9eyeOjo5W97arX78BAL6+a6z29eefKUtdS5cuq8SbiIiIiIiIPBRKvEmu4O7uTrt2HTCbzXzxxRSio6MsZfHx8Xz++WSSkhKpXbsOtWvXtZT16PEU+fPnJygogKVLF6fpMygogOXLfbIlvpIlUxI1p0+fzJb+HtSgQa/i4ODAjz9+i6/vGkwmU7o6p06dYNu2zdk67vHjR9m0aT2JiQnpyv75ZztAmpNZq1SpRtu27bly5TIffPCu1RmI8fHxrF/vy9WrkenK2rRpR6FChfHz28/cubNJTk6mbdv2FCxYKF3dXr2epmDBghw/fpS5c2djNpvTxP3bb7MBeO65gfd/4yIiIiIiIiJZoKWmkmuMHj2Gs2fPcvjwQfr37039+g2xs7MjIMCPqKhreHh4Mn78pDRtSpZ0491332fy5AlMnfo5q1Ytp0KFJ4iICCcoKID+/Z9n4cJ5Vg8uuB9t2rTHz28/EyeOp3HjJhQuXARISeKULVv+gfrOiqpVqzFu3MdMmTKByZMn8Msv0ylfvgLOzi7ExERz6tRJrly5TIcOHWnTpn22jRsWFsZHH72Po6MjVapUo2RJN5KTkzl16gTnzp3FwcGBoUNHpGnz/vsfcf36DXbv3slzz/WlUqXKeHh4YjabCQu7xIkTIdy8eZP5830oVsw1TVtHR0eefLITK1YsxcfnDwC6d3/KamzFirkyfvwkxo0by88//8i6dWupXLkq165dIyDgAMnJyXTr1pNevZ7OtuchIiIiIiIicjdKvNlQhZJFbR1CpjysOIsWdWbGjNksWbKIzZvXs2/fbkwmM6VKlaJnz948++xAihQpkq5d587dKFnSjblzZ3P48EEuXjxP2bLlee+9D2jUqAkLF86jaFHnB4rt6af7ERcXx/r1a9m1a6fllNFOnbraJPEG0L79k1SvXgMfn0Xs27eH4OBAkpNNFCtWDE/P0vTp4027dk9m65g1a9bi9deHERjoz5kzZwgJOYadnR0lSrjRp483/fr1T/c8ChQoyNSp37Np03rWr/fl2LGjhIQcp2DBgri6Fqdjxy60bNkGT8/SVsfs3r0XK1YsBVJOr61f3yvD+Fq0aMXs2b8zf/5vHDiwj61bN5E/vxN169bnqaf60KFDp2x7FiIiIiIiIiL3YjDfvh5L7ioi4jr3elo3byYRGRmKq6tHhqdJGo0GnJ0LYGf36Kz0TU42ERUVh8n0aL0uvr5rmDx5Ai1atOKzz6baOhyRHJWZnz8iIiIiIvJ4s7c34uJSkAOvvMKN4yEPdexCVSrjNXMm167FcutW+i2THhUGAxQvXjhTdTXjzQZMJjNRUXEYjQZbh5JpJpM51ybdwsLCcHCwx9W1eJrrQUEB/PDDtwB065b+FEwRERERERERkZykxJuN5OZE1qPGz28fn376MZUqVcbNzR2j0cjFixc5cSLldNRu3XrSpk07G0cpIiIiIiIiIo8bJd7kkVezZm26detJYKA//v4HiI+Pp3DhwjRs2Jju3XvRsWMXS92zZ8/w++9zMt33Cy+8RLly5bM/6Ids8uQJma7bqlVbWrdum2OxiIiIiIiIiDwulHiTR165cuUZO3ZcpupGRkbg67sm03137dojTyTe7uee3d09lHgTERERERERyQZKvMljpUGDhuzYsd/WYTx0j+M9i4iIiIiIiNjao3OspoiIiIiIiIiIyCNEiTcREREREREREZEcoMSbiIiIiIiIiIhIDlDiTUREREREREREJAco8SYiIiIiIiIiIpIDlHgTERERERERERHJAUq8iYiIiIiIiIiI5AAl3kRERERERERERHKAEm8iIiIiIiIiIiI5wN7WATyujEYDRqPB1mFkmslkxmQy2zoMEREREREREZFHhhJvNmA0GnBxdsJoZ2frUDLNlJzMtah4Jd9ERERERERERDJJiTcbMBoNGO3siFg2lpsRp2wdzj05FH+C4n0+xWg0ZFvirV+/noSFhbJkySo8PEplS5+2FBp6CW/vXri7e+Djs9rW4WSLYcNeJSDAj2nTZtCgQcNMt5s8eQK+vmt4//2P6NatZw5GKCIiIiIiIpK7KfFmQzcjTnEz7Iitw5BMyGuJQhERERERERHJeUq8iWSDEiVKMn++D/b2+pYSERERERERkRTKEohkA3t7e8qVK2/rMEREREREREQkF1HiTXKVhIQElixZxJYtGzh//jwmUzIeHp60bt2WAQNeoEiRIlbbBQYG8Ntvszh8OJjk5GTKlatA377P0LVrD1q2TNmfbMeO/fcdz9q1q5ky5X+WP3t790pTnrr/2d32eLt9/HXr1uLjs4gzZ07j6OiIl1djhg4dgbu7O2azmWXLFrN69UouXDiHo6MjzZu34o03RuDiUsxqfOfOnWXRot/Zv38vERHhODg4UKlSFXr1eprOnbvd9/1mxN//AHPnzubo0SMkJSXyxBMV6du3P126dL9ru5CQ4/z66y8EBfkTFxdPmTJl6N79Kfr2fQa7Ow4XmTXrJ3799RcGDRpCnz7PMHv2z/zzz3auXo2kWDFXWrVqwyuvDKVw4cLZdl8iIiIiIiIiOUmJN8k1YmKiGTlyKCEhxylYsCBeXg2xt7cnIMCPuXNns2HDOqZNm55uj7WNG9cxceI4TCYTFStWokKFikREhPPJJxM5c+b0A8Xk6VmGrl17sHXrJuLj42nbtj1OTgUs5a6uxTPd14wZ37Nw4Tzq1WtAkybNOXLkEJs2rSc4OJA5cxby5ZefsGPHdurX96JUKU+CgwPx9V3D8ePHmDlzLg4ODmn627x5I5MmfURSUiLlypWnadMWxMbe4PDhg3z88XgOHNjH++9/9ED3D7B9+1aWLVtM2bLlady4KZGREQQFBTBp0keEhBxn+PC3rLY7fPgQX375Ka6urnh5NeL69ev4+x9g2rSvCAoK4OOPP8VgMKRrd+XKZQYPfoFbt25Ru3YdkpKSCA4OZOnSxRw+fJDp02drSa+IiIiIiIg8EvTbq+QaX331KSEhx6lRoxZffPENRYs6AxAXF8f48WPZvXsnEyd+yPTpsy1tIiLC+eyzyZhMJkaOfAdv7wGWsoAAP959d+QDxVS3bj3q1q2Hv/8B4uPjefPNUVk+XGH16uXMnDmPypWrAJCYmMBbbw0jKCiA4cNfJSEhgQULfHB39wAgKiqK118fxMmTIWzZspFOnbpa+jp58gSTJo0HDEye/Dlt2rS3lIWFhTJmzFusXbua+vW96Nq1R9YfAODjs4jXXnuTgQMHWa75+x/gnXdG8Mcf82ncuClNmjRL127FCh+eftqbkSPftiTKTp06yYgRr7N16yZWrlxG795907X7889VdOvWk3fe+S/58uUD4PLlMF5//WWOHDnMli0b6dixywPdk4iIiIiIiMjDYLR1ACIAYWFhbNmyCYPBwHvvfWBJugEUKFCAMWM+JF8+R4KDgwgODrSUrVmzkvj4OGrVqpMm6QZQr14Devfu97Bu4Z4GD37dknQDcHTMT//+zwMpibRRo96xJN0AnJ2dLYmp/fv3pulr7txZJCUlMWTI0DRJNwB3dw/Gjh0HgI/PHw8cd5UqVdMk3QDq1/fi6ae9AVi06Her7VxdizNs2Kg0s9OeeKIigwa98v/t5lttV7KkG6NHj7Ek3QDc3Nzp2/cZIP2zEBEREREREcmtlHiTXCEw0A+TyUTlylWpVKlyuvISJUrSpElTAPz8/t2rzd/fDyDDGVCdOuWemVHNmrVId61MmTIA2NnZ0ahR03TlpUuXBSAiIsJyzWQysXv3TgA6dOhodaxq1Wrg5FSAkJBjJCYmPlDcGe3jljqTLigogOTk5HTl7dt3xNHRMcN2Fy6cIyIiPF25l1cj8ufPn+56+fIVAAgPT99GREREREREJDfSUlPJFcLDrwBQqlTGyzhLlSr9/3X/TbyEh18GyHD5p7t71paF5gQ3N/d011L3i3N1LW5137ICBVLKk5L+TZ5FR0cTGxsLQJ8+dz/cAFL2zitRomSWYoaMn23q9cTERGJiotMdAJFRuwIFClK0aFGio6O5cuUyxYuXSFNu7TmltoO0z0JEREREREQkN1PiTfIEK3v0///1DApswGjMeILp3cruZDabLF9nZv82B4d896zzoMxmcxbbpb+Wmz4zERERERERkQehxJvkCqkzsi5duphhndSyEiX+nSFVokRJzp07S2hoqNU2oaGXsjHK3KFoUWccHR1JTEzkzTdH4ezsnKPjZfQMw8JSrufL50iRIkUz3S4uLpbo6GgASpbM+kw8ERERERERkdxOe7xJrlC3bgOMRiMhIccJCTmerjwiIoI9e3YB0KBBw9va1Qdg48Z1VvvdsOGvbInP3t4BwOpeZg9byn5wTQDYvHlDjo+3bp2v1et//fUnAHXq1LW6THbLlo0kJSVZabcWgNKlyzzQElgRERERERGR3C7XJt5OnTrFvHnzGDt2LD179qRGjRpUrVqVH3/88Z5td+7cyZAhQ2jSpAl16tShS5cuTJ061bIvluQ+7u7utGvXAbPZzBdfTCE6OspSFh8fz+efTyYpKZHatetQu3ZdS1mPHk+RP39+goICWLp0cZo+g4ICWL7cJ1viS52Zdfr0yWzp70ENGvQqDg4O/Pjjt/j6rsFkMqWrc+rUCbZt2/zAYx07doT5839Lcy0wMIBly5YA0L//c1bbRUSE88MP36RJVp45c5o5c2YC8Mwz1tuJiIiIiIiI5BW5dqnpwoULmTt37n23mzNnDp988gkGg4GGDRvi6urKgQMHmDFjBuvWrWPBggUUK1bs3h09BA7Fn7B1CJnysOIcPXoMZ8+e5fDhg/Tv35v69RtiZ2dHQIAfUVHX8PDwZPz4SWnalCzpxrvvvs/kyROYOvVzVq1aToUKTxAREU5QUAD9+z/PwoXzrM7Iuh9t2rTHz28/EyeOp3HjJhQuXASA554bSNmy5R+o76yoWrUa48Z9zJQpE5g8eQK//DKd8uUr4OzsQkxMNKdOneTKlct06NCRNm3aP9BY/foN4KeffuCvv/6kYsXKlmdrMpnw9n6WZs1aWm3Xu3dfVq9eyc6d/1CjRk2uX7+Ov/9+bt68SevW7Xj66X4PFJeIiIiIiIhIbpdrE29VqlTh5ZdfpkaNGtSoUYOffvqJlStX3rXN4cOH+fTTT7Gzs2P69Om0adMGSJkxNXToUHbt2sWECROYNm3aw7iFDJlMZkzJyRTv86lN47gfpuRkTKasbaCfWUWLOjNjxmyWLFnE5s3r2bdvNyaTmVKlStGzZ2+efXYgRYoUSdeuc+dulCzpxty5szl8+CAXL56nbNnyvPfeBzRq1ISFC+dRtKjzA8X29NP9iIuLY/36tezatdNysmanTl1tkngDaN/+SapXr4GPzyL27dtDcHAgyckmihUrhqdnafr08aZduycfeJzWrdvSqlUb5s79ld27/+HmzZtUqVKNvn2fuevhDjVq1KJXr6eZNesn9u/fQ3x8PKVLl6FHj6fo27e/DlEQERERERGRPM9gzupxhA/Z2LFjWb58OSNHjuSNN96wWmfkyJH89ddfeHt7M2lS2plRFy9e5Mknn8RkMrF27VoqVqx43zFERFy3egrj7W7eTCIyMhRXV4+7niZpNBowGh+dxIPJZM7xxFtO8PVdw+TJE2jRohWffTbV1uGI5KjM/vwREREREZHHl729EReXghx45RVuHA95qGMXqlIZr5kzuXYtllu30m+Z9KgwGKB48cKZqptrZ7zdr6SkJLZt2wZAjx7pZ+F4enrSoEED9u/fz8aNG7OUeMtOj2oiKzcKCwvDwcEeV9fiaa4HBQXwww/fAtCtWy9bhCYiIiIiIiIij7E8k3g7c+YM8fHxANSqVctqnVq1arF//34OHz78MEOTHObnt49PP/2YSpUq4+bmjtFo5OLFi5w4kXI6arduPWnTpp2NoxQRERERERGRx02eSbxduHABgCJFilCoUCGrdTw8PNLUvV+Z2ZJK21Y9fDVr1qZbt54EBvrj73+A+Ph4ChcuTMOGjenevRcdO3ax1D179gy//z4n032/8MJLlCtXPvuDfsgmT56Q6bqtWrWldeu2ORaL5DyDQT+LREREREQkd3uUf2e5n9jzTOItNjYWACcnpwzrFChQAIAbN25kaQxX13uv301ISODqVSN2dgbs7Y1ZGkfuT8WKT/Dhhx9lqm5U1FV8fddkuu8ePXpRseKjcfrs3dzPPZcqVYr27R/sJFSxDZPJgNGYsl9D/vz5bR2OiIiIiIiIVS4uBW0dwkOTZxJvD0NkZOYOVzCZTCQnmx/pjQLzqrp1G7Bjx/77apMXPsfH8Z4fR8nJZkwmE9euxeLgcNPW4YiIiIiISC5kZ2e0eeLr2rVYkpMf3d87DYbMTc6CPJR4K1gw5aVJ3efNmri4OIAMl6Lei9nMPRNvj8YZsSKSl2XmZ5WIiIiIiIgtPS6/s+SZtZCenp4AxMTEZLiUNDQ0NE1dERERERERERGRnJJnEm8VKlSw7O928OBBq3VSr9esWfOhxSUiIiIiIiIiIo+nPJN4y5cvH23atAFgzZr0G8lfvHgRf39/AJ588smHGpuIiIiIiIiIiDx+8kziDeDVV1/FYDCwbNkytm/fbrkeHx/PBx98QHJyMp07d6ZixYo2jFJERERERERERB4HufZwhUOHDvG///3P8udz584B8Mcff7B161bL9e+//56SJUsCKUtIx44dyyeffMKrr75Ko0aNcHV1Zf/+/YSHh1OhQgUmTJjwMG9DREREREREREQeU7k28Xbjxg0CAwPTXQ8LCyMsLMzy56SkpDTlL730ElWqVGH27NkEBwcTFxdHqVKl6NOnD6+++mqWTzQVERERERERERG5H7k28dakSROOHTuWpbbNmzenefPm2RyRiIiIiIiIiIhI5uXaxFteZzQaMBoNtg4j00wmMyaT2dZhiIiIiIiIiIg8MpR4swGj0YCzixN2Rjtbh5JpyaZkoq7FK/kmIiIiIiIiIpJJSrzZgNFowM5ox//W/Y8z187YOpx7Ku9Sno86f4TRaMi2xFu/fj0JCwtlyZJVeHiUypY+bSk09BLe3r1wd/fAx2e1rcPJFsOGvUpAgB/Tps2gQYOGmW43a9ZP/PrrLwwaNITBg1/LdDs/v/2MGPE69eo14Pvvf85KyCIiIiIiIiK5ihJvNnTm2hmOhx+3dRiSCXktUSgiIiIiIiIiOU+JN5FsUKJESebP98HeXt9Sffv258knO1O0qLOtQxERERERERGxKWUJRLKBvb095cqVt3UYuYKzszPOzs62DkNERERERETE5pR4k1wlISGBJUsWsWXLBs6fP4/JlIyHhyetW7dlwIAXKFKkiNV2gYEB/PbbLA4fDiY5OZly5SrQt+8zdO3ag5YtU/Yn27Fj/33Hs3btaqZM+Z/lz97evdKUp+5/drc93m4ff926tfj4LOLMmdM4Ojri5dWYoUNH4O7ujtlsZtmyxaxevZILF87h6OhI8+ateOONEbi4FLMa37lzZ1m06Hf2799LREQ4Dg4OVKpUhV69nqZz5273fb8Z8fc/wNy5szl69AhJSYk88URF+vbtT5cu3dPVvdceb76+a1i6dDGnT58kXz5HqlevyYsvvnzX8bdt28yuXf9w6FAw4eHhJCUl4upanPr1vXjhhRcpW7a81Xbx8fH8/vscNm1az+XLYRQpUpQmTZoxZMhQVq1anqW96EREREREREQyS4k3yTViYqIZOXIoISHHKViwIF5eDbG3tycgwI+5c2ezYcM6pk2bnm6PtY0b1zFx4jhMJhMVK1aiQoWKRESE88knEzlz5vQDxeTpWYauXXuwdesm4uPjadu2PU5OBSzlrq7FM93XjBnfs3DhPOrVa0CTJs05cuQQmzatJzg4kDlzFvLll5+wY8d26tf3olQpT4KDA/H1XcPx48eYOXMuDg4OafrbvHkjkyZ9RFJSIuXKladp0xbExt7g8OGDfPzxeA4c2Mf773/0QPcPsH37VpYtW0zZsuVp3LgpkZERBAUFMGnSR4SEHGf48Lcy3dc333yJj88ijEYjderUw9W1OCdPnmD48Nfo27d/hu3Gj/8vDg4OlC//BF5eDUlOTubUqZOsXbuaLVs28vXX31O7dt00beLj4xkx4jWOHDmMk1MBGjVqiqOjI3v27GLXrn9o1qxFlp+JiIiIiIiISGYo8Sa5xldffUpIyHFq1KjFF198Y9kjLC4ujvHjx7J7904mTvyQ6dNnW9pERITz2WeTMZlMjBz5Dt7eAyxlAQF+vPvuyAeKqW7detStWw9//wPEx8fz5pujsny4wurVy5k5cx6VK1cBIDExgbfeGkZQUADDh79KQkICCxb44O7uAUBUVBSvvz6IkydD2LJlI506dbX0dfLkCSZNGg8YmDz5c9q0aW8pCwsLZcyYt1i7djX163vRtWuPrD8AwMdnEa+99iYDBw6yXPP3P8A774zgjz/m07hxU5o0aXbPfnbu3IGPzyKcnJz48stp1K1b31I2b96v/PTTDxm2HT/+Y5o3b4WTk5PlmtlsZvlyH77++jM+/3wyc+f+gcFgsJTPnDmdI0cOU778E3zzzY8UL56SJE1MTOTjj8ezdm3eOH1WREREREREci+jrQMQAQgLC2PLlk0YDAbee++DNBvzFyhQgDFjPiRfPkeCg4MIDg60lK1Zs5L4+Dhq1aqTJukGUK9eA3r37vewbuGeBg9+3ZJ0A3B0zE///s8DKYm0UaPesSTdIGWvtN69+wKwf//eNH3NnTuLpKQkhgwZmibpBuDu7sHYseMA8PH544HjrlKlapqkG0D9+l48/bQ3AIsW/Z6pfhYvXgBAnz7PpEm6AQwcOCjNs7lThw6d0iTdAAwGA336eFOrVh1Onz6VZnZjYmICq1atAGDEiNGWpBuAo6Mjb789lvz582cqbhEREREREZGs0ow3yRUCA/0wmUxUqVKNSpUqpysvUaIkTZo05e+/t+Hnt9+yrNDf3w+Ajh27WO23U6cuLFw4L+cCvw/WljaWKVMGADs7Oxo1apquvHTpsgBERERYrplMJnbv3glAhw4drY5VrVoNnJwKEBJyjMTERBwdHbMct7V93AC6du3BokW/ExQUQHJyMnZ2dhn2cevWLYKCUhKmnTt3tVqnS5fuhIQcz7CPCxfOs2fPTi5cuEBcXCwmkwmAq1cjgZT97ipUeAKAo0ePEh8fh7OzM40bp3+uLi4uNGrUhL//3pbheCIiIiIiIiIPSok3yRXCw68AUKpUxss4S5Uq/f91w29rdxkgw+Wf7u5ZWxaaE9zc3NNdS90vztW1OPb26b8dCxRIKU9KSrRci46OJjY2FoA+fawnxW4XExNNiRIlsxQzZPxsU68nJiYSExOd4QEQqTGk3oOHh2cG/Vm/npyczNSpn7Ny5TLMZnOGY8TFxVq+Tn0v7vb556Z3Q0RERERERPImJd4kT7hta687rmdQYANGY8Yru+9Wdiez2WT5OjP7tzk45Mt031l1t4TYg1qyZCErVizF1dWVYcPeonbturi4FLPM4psw4QM2blxnNYa7ffy56NUQERERERGRPEqJN8kVUmdkXbp0McM6qWUlSpRI0+7cubOEhoZabRMaeikbo8wdihZ1xtHRkcTERN58cxTOzs45Ol5GzzAsLOV6vnyOFClS9K59FClSlHz58pGUlERo6CWeeKJihv3dafPmjQC8++77tGzZJl35hQvn011LfZ8yei/uVSYiIiIiIiKSHXS4guQKdes2wGg0EhJy3Oo+XxEREezZswuABg0a3tYuZZP+jRvXWe13w4a/siU+e3sHIGXZo62l7AfXBIDNmzfk+Hjr1vlavf7XX38CUKdOXavLZG9nb29v2Zcvo8/kr7/WWr0eExMDgJubR7qyU6dOEhJyLN31qlWrkz9/fqKirrFv35505VFRUezfn/66iIiIiIiISHZS4k1yBXd3d9q164DZbOaLL6YQHR1lKYuPj+fzzyeTlJRI7dp1LAkcgB49niJ//vwEBQWwdOniNH0GBQWwfLlPtsRXsmTKDKrTp09mS38PatCgV3FwcODHH7/F13eN5aCB2506dYJt2zY/8FjHjh1h/vzf0lwLDAxg2bIlAPTv/1ym+vH2fhZIOWn19pNpAebP/43jx49abVe+fHkAli1bkuY+IyIimDTpI6vJ0Pz589OjR28Avvvua8sBDABJSUlMnfoZ8fHxmYpbREREREREJKu01NSGyruUt3UImfKw4hw9egxnz57l8OGD9O/fm/r1G2JnZ0dAgB9RUdfw8PBk/PhJadqULOnGu+++z+TJE5g69XNWrVpOhQpPEBERTlBQAP37P8/ChfPuOSPrXtq0aY+f334mThxP48ZNKFy4CADPPTeQsmXLP1DfWVG1ajXGjfuYKVMmMHnyBH75ZTrly1fA2dmFmJhoTp06yZUrl+nQoSNt2rR/oLH69RvATz/9wF9//UnFipUtz9ZkMuHt/SzNmrXMVD8tW7amTx9vli1bwptvDqFu3fq4uhbn5MkQzp49g7f3syxZsjBdu4EDX2bPnl2sXr0cf//9VKlSjdjYWAICDlCqlCetW7dj+/Yt6dq9+uobBAcHcuzYEfr3fxovr4bky+dIUFAAt27dpGvXHvj6rsHBweGBno+IiIiIiIhIRpR4swGTyUyyKZmPOn9k61AyLdmUjMmUcxvoQ8reZTNmzGbJkkVs3ryefft2YzKZKVWqFD179ubZZwdSpEiRdO06d+5GyZJuzJ07m8OHD3Lx4nnKli3Pe+99QKNGTVi4cB5Fizo/UGxPP92PuLg41q9fy65dOy0ndHbq1NUmiTeA9u2fpHr1Gvj4LGLfvj0EBweSnGyiWLFieHqWpk8fb9q1e/KBx2ndui2tWrVh7txf2b37H27evEmVKtXo2/eZTB3ucLvRo8dQtWp1li1bwqFDB8mXz4Fq1Wrw1lvvAVhNvNWsWYuZM+fxyy8/cuTIYXbs2E7Jkm707dufl14azNSpX1gdq0CBAnz33U/Mm/crmzatZ8+eXRQpUoSGDZswZMgb/PrrzwAP/G6IiIiIiIiIZMRgzsnjCPOYiIjr3Otp3byZRGRkKK6uHnc9TdJoNGA0PjrHKppM5hxPvOUEX981TJ48gRYtWvHZZ1NtHY7kErdu3WLgwGc4f/4cs2b9TtWq1WwdUrbI7M8fERERERF5fNnbG3FxKciBV17hxvGQhzp2oSqV8Zo5k2vXYrl1K/2WSY8KgwGKFy+cqbqa8WYjj2oiKzcKCwvDwcEeV9fiaa4HBQXwww/fAtCtWy9bhCY2dvToEapUqYrR+O92lnFxcXz//VTOnz9HxYqV80zSTURERERERHIfJd7kkefnt49PP/2YSpUq4+bmjtFo5OLFi5w4kXI6arduPWnTpp2NoxRb+PDD90hISKBixUo4O7sQFXWNkJDjxMREU6RIUT744NFZ7i0iIiIiIiKPHiXe5JFXs2ZtunXrSWCgP/7+B4iPj6dw4cI0bNiY7t170bFjF0vds2fP8PvvczLd9wsvvES5cuWzP+iHbPLkCZmu26pVW1q3bptjsTxM/fs/z/btWzhz5hTXr1/HYDDg7u5Bp05defbZF3Bzc7d1iCIiIiIiIpKHKfEmj7xy5cozduy4TNWNjIzA13dNpvvu2rVHnki83c89u7t75JnEm7f3ALy9B9g6DBEREREREXlMKfEmj5UGDRqyY8d+W4fx0D2O9ywiIiIiIiJia8Z7VxEREREREREREZH7pcSbiIiIiIiIiIhIDlDiTUREREREREREJAco8SYiIiIiIiIiIpIDlHgTERERERERERHJAUq8iYiIiIiIiIiI5AB7WwcgIiIi8rAYjQaMRoNNxjaZzJhMZpuMLSIiIiK2ocSbiIiIPBaMRgMuzk4Y7exsMr4pOZlrUfFKvomIiIg8RpR4sxFb/ot7Vuhf6UVE5FFnNBow2tlx5OOPiTt79qGOXaBcOaqPG4fRaNDfpyIiIiKPESXebMDW/+KeFfpXehERySvizp7lxvEQW4chIiIiIo8BJd5swJb/4p4VOfGv9P369SQsLJQlS1bh4VEqW/q0pdDQS3h798Ld3QMfn9W2Dkceorz2LouIiIiIiEj2UeLNhvQv7o8OJVdERERERERE5H4p8SaSDUqUKMn8+T7Y2+tbSkRERERERERSKEsgkg3s7e0pV668rcMQERERERERkVxEiTfJVRISEliyZBFbtmzg/PnzmEzJeHh40rp1WwYMeIEiRYpYbRcYGMBvv83i8OFgkpOTKVeuAn37PkPXrj1o2bIhADt27L/veNauXc2UKf+z/Nnbu1ea8mnTZtCgQcO77vF2+/jr1q3Fx2cRZ86cxtHRES+vxgwdOgJ3d3fMZjPLli1m9eqVXLhwDkdHR5o3b8Ubb4zAxaWY1fjOnTvLokW/s3//XiIiwnFwcKBSpSr06vU0nTt3u+/7vd2NGzdYsGAuO3Zs49KliyQnJ1OkSFFKlSqFl1djXnrpFcsMv9vvf9Gi5fzxx3x8ff/k0qWLODnlp0GDRrzyyutWk5OHDx9k69ZN+Psf4PLly8TERFO4cBGqV6+Jt/cAGjVqkq5N6ufStWsPhg9/i19/nck//2wnPPwKNWvW5vvvfwbg6NEjLFgwl+DgQK5du0q+fI4ULepMlSpV6NKlO61atU3X99GjR/jjj/kEBvpz7dpV8ud3onr1Gnh7D6BZs5Z3fWbbtm1h0aLfOXnyBGazmapVq/H88/+x2m7YsFcJCPBj2rQZFC5cmF9/nUlgoB9xcXF4epame/enGDDgeQyGR+f0YxEREREREUlLiTfJNWJiohk5cighIccpWLAgXl4Nsbe3JyDAj7lzZ7NhwzqmTZuebo+1jRvXMXHiOEwmExUrVqJChYpERITzyScTOXPm9APF5OlZhq5de7B16ybi4+Np27Y9Tk4FLOWursUz3deMGd+zcOE86tVrQJMmzTly5BCbNq0nODiQOXMW8uWXn7Bjx3bq1/eiVClPgoMD8fVdw/Hjx5g5cy4ODg5p+tu8eSOTJn1EUlIi5cqVp2nTFsTG3uDw4YN8/PF4DhzYx/vvf5Sl+05ISOCNNwZz6tRJnJ1d8PJqRP78Tly9Gsm5c2cIDp5J//7PU7hw4XRtP/rov/zzz9/Uq9eAihUrceTIIbZs2cju3TuZOvV7atWqk6b+Tz/9iL//fipUeIKqVauRP78TFy9eYOfOv9m5829GjHibZ5551mqc0dFRDB78H27cuE7duvWoWrW65Tnt37+Xd94Zwa1bt6hUqQo1a9bGZDIRHn6FXbv+wWQypUu8LV68kO+/n4rJZKJy5SrUqFGLq1cj8fc/wN69uxk8+DUGDRpiNRYfn0X88ccCqlWrQfPmLbl48QIBAX4EBPgxatQ79Os3wGq7vXt388cf8/H0LE3Dhk2IjIwgODiQH374hitXLjNy5Nv3+rhEREREREQkl1LiTXKNr776lJCQ49SoUYsvvviGokWdAYiLi2P8+LHs3r2TiRM/ZPr02ZY2ERHhfPbZZEwmEyNHvoO397/JjYAAP959d+QDxVS3bj3q1q2Hv/8B4uPjefPNUVk+XGH16uXMnDmPypWrAJCYmMBbbw0jKCiA4cNfJSEhgQULfHB39wAgKiqK118fxMmTIWzZspFOnbpa+jp58gSTJo0HDEye/Dlt2rS3lIWFhTJmzFusXbua+vW96Nq1x33HumXLRk6dOknTps359NOv0+xdZzKZCAz0J3/+/OnahYWFkpAQz8yZ86hUqTIAycnJfPfd1/j4/MGECR+wYMFS8uXLZ2kzYMDzjBs3keLF0yYxDx4M4u23h/Pjj9/Srl0HSpQomW68nTt34OXVmClTPqdgwUJpyubOnc2tW7cYP/7jNM8OUmbz3ZmU3bNnF9999zVFixZl0qTPqVevgaXs5MkTvPvuSGbN+ol69RpQv75XulgWL16YbqxNm9YzYcIHfPfdVBo0aMgTT1RK1+733+fwzjv/pXfvvpZrBw7sY9SoN1i2bDHPPvsCJUu6pWsnIiIiIiIiuZ/R1gGIAISFhbFlyyYMBgPvvfeBJekGUKBAAcaM+ZB8+RwJDg4iODjQUrZmzUri4+OoVatOmqQbQL16Dejdu9/DuoV7Gjz4dUvSDcDRMT/9+z8PpCR2Ro16x5J0A3B2drYkY/bv35umr7lzZ5GUlMSQIUPTJN0A3N09GDt2HAA+Pn9kKdZr164C0KhRk3QHRhiNRurX90o3Ay/Vf/4z2JJ0A7Czs+ONN0ZSokRJwsJC2bp1c5r6zZq1SJd0A6hVqw59+jzDrVu3+PvvbVbHsre357333k+XdAO4ejXlHpo2bZGurFChQtSqVTvNtVmzfsJsNvPOO/9Nk3QDqFixEsOGvQXA0qXWn2nLlm3SJfg6dOhEmzbtSE5OZskS6+3atGmXJukG4OXViMaNm5GcnIyf3/0vkRYREREREZHcQYk3yRUCA/3+f3lf1TRJm1QlSpSkSZOmAGkSEf7+fgB07NjFar+dOlm/bgvNmqVPAJUpUwZISU41atQ0XXnp0mUBiIiIsFwzmUzs3r0TgA4dOlodq1q1Gjg5FSAk5BiJiYn3HWu1ajUAWLBgLr6+a4iJic50W2sz7PLly0f79imx+vsfSFceHR2Fr+8afvzxWz77bBKTJ09g8uQJBASk1D137qzVsSpXroqnZ2mrZTVq1ARg4sQPCQwM4NatWxnGHBUVxZEjh3B0dKRFi9ZW66TOcgsODrJantHMwi5dUq5bu28gw/HKly8PQHh4eIZxi4iIiIiISO6mpaaSK4SHXwGgVKmMl3GWKlX6/+v+m4gID78MkOHyT3f3rC0LzQlubu7prqXuF+fqWjzdzDJIme0HkJT0b/IsOjqa2NhYAPr06X7PcWNioq0u07ybBg0a8vzzL7Jw4TwmT56AwWCgdOky1K5dl1at2tCiRWuMxvR5+0KFClvd9w3+/WxTP7NUq1Yt57vvviY+Pj7DeOLiYq1ev9uy39dee5MTJ0LYvXsnu3fvxNHRkSpVqlG/vhedOnWlfPkKlrqhoRcxm80kJibSrl2zDPsEiIq6dl+xZHTfqay9FwAFChQE0n72IiIiIiIi8mhR4k3yhIwOfsxNJ0JaS1RlpuxOZrPJ8nVm9m9zcMh3zzrWDB06nN69+/LPP9sJCgokODiQtWtXs3btaqpXr8G0aT/h5OR03/2azf9+ffToEb74YgpGo5GhQ4fTokVr3NzcyZ8/PwaDgZUrl/HFF1Mw397oNo6OjhmO4+panFmz5uHvf4D9+/cSHBzI4cMHCQ4OZN68X3nttTd54YWXADCZUvp3cipA27btM+zzQWRwC7nqHRV5WIxGA0bjw3/37ew00V9EREREHi4l3iRXSJ2RdenSxQzrpJaVKFEiTbtz584SGhpqtU1o6KVsjDJ3KFrUGUdHRxITE3nzzVE4Ozvn2FgeHqXo12+A5UTOI0cOMXHiOI4cOcyCBXMZPPi1NPVv3LjO9evXrc56S/2MSpb8d/bdli0bMZvN9OvXn+effzFdmwsXzj9Q/AaDgQYNGtKgQUMAEhMT8fVdzddff87PP/9Iu3ZP4ulZGjc3N0v9//53/H0lQlOFhl5Ks4ffv9fT37fI48xoNODs4oSd0c7WoYiIiIiI5Dgl3iRXqFu3AUajkZCQ44SEHE+XwIiIiGDPnl0AliRKSrv6HDiwj40b19Gnj3e6fjds+Ctb4rO3TzlIIDk5OVv6exAp+8E1YceO7WzevMHqfeeU6tVr8vTT3kyb9hUhIces1lm37k9Loi7VzZs32bx5A0CaE0FjYmIAcHPz4E6JiYnpDmJ4UI6OjvTu3Y+VK5cREnKcEydC8PQsTfHiJahYsTInT4awZ89OmjVred99r1v3J61bt013/a+//gSwehKqyOPIaDRgZ7Tjf+v+x5lrZx7q2E3LNuW15q/du6KIiIiISDbRmgvJFdzd3WnXrgNms5kvvphCdHSUpSw+Pp7PP59MUlIitWvXoXbtupayHj2eIn/+/AQFBbB06eI0fQYFBbB8uU+2xJc6W+n06ZPZ0t+DGjToVRwcHPjxx2/x9V2DyWRKV+fUqRNs25a1xNW2bVsICPBL1++tW7csCdDbT2C93Zw5szh16oTlzyaTienTp3HlymVKlnRLcwpr6gECf/21Js0+bomJiXz11aeEhmY8A/JeFiyYR1hYWLrrZ8+escyku/0ehgwZCsCUKRPZsWN7unZms5lDhw6yd+9uq+Nt376VjRvXpbm2ZctGtm3bjJ2dHX379s/yvYjkRWeuneF4+PGH+l/odeuzo0VEREREcopmvNlQgXLlbB1CpjysOEePHsPZs2c5fPgg/fv3pn79htjZ2REQ4EdU1DU8PDwZP35SmjYlS7rx7rvvM3nyBKZO/ZxVq5ZTocITRESEExQUQP/+z7Nw4TyrBxfcjzZt2uPnt5+JE8fTuHETChcuAsBzzw2kbNnyD9R3VlStWo1x4z5mypSU0z9/+WU65ctXwNnZhZiYaE6dOsmVK5fp0KFjmkRXZgUE+LFkyUKcnZ2pXLkqLi7FiIuL5dChg1y7dpUSJUry3HP/SdfOzc2dqlWr8/LLL1C/vhdFihTl6NHDXLx4AScnJz76aHKafdm6devFkiWLOH78GN7evahTpz52dkYCAwNITEzE2/tZlixZmKVnNHfuLH788VvKlStPuXIVcHR0tLwXycnJdOnSnapVq1nqt2zZmpEj3+H776cyduxoSpcuQ9my5ShYsBBRUdc4cSKEa9eu8vzzL9K4cfoTaL29BzBhwgf88cd8Spcuy8WLFzh8+CAAw4e/ZfW0XhEREREREcnblHizAZPJjCk5merjxtk6lEwzJSdbNqDPKUWLOjNjxmyWLFnE5s3r2bdvNyaTmVKlStGzZ2+efXYgRYoUSdeuc+dulCzpxty5szl8+CAXL56nbNnyvPfeBzRq1ISFC+dRtKjzA8X29NP9iIuLY/36tezatdNy0mSnTl1tkngDaN/+SapXr4GPzyL27dtDcHAgyckmihUrhqdnafr08aZduyez1He3bj1wdHQkKCiAM2dOExDgR8GChXBzc+eZZ56lV6+nrT5Tg8HAxImfsGDBXNatW0tgoD/58zvRtm17Bg9+nQoVnkhTv3DhwsycOY9Zs35i795d7NmzkyJFitK4cRMGDXqVoKCALMUPKYnc/fv3cvToYQIC/EhIiKdYMVcaNWpCr159aNWqTbo23t4D8PJqiI/PH/j5HWD//n0YjQaKFXOlcuWqNG/egrZtO1gdz9v7WWrVqsvixQv+f8acmbp16/Pcc/+hRYtWWb4PEREREREReXQZzBkdFyjpRERcz/BkwlQ3byYRGRmKq6vHXU+TtNWJblllMplzPPGWE3x91zB58gRatGjFZ59NtXU4eVZo6CW8vXvh7u6Bj89qW4fz2Mrszx8RW7K3N+LiUpBBiwZxPPz4Qx27Y5WOTOg8gQOvvMKN4yEPdexCVSrjNXMm167FcutW+u0BRERERB6W1P8f0/8TZZ3BAMWLpz9U0BrNeLORRzWRlRuFhYXh4GCPq2vxNNeDggL44YdvgZQljSIiIiIiIiIiD5MSb/LI8/Pbx6effkylSpVxc3PHaDRy8eJFTpxImUnRrVtP2rRpZ+MoRURERERERORxo8SbPPJq1qxNt249CQz0x9//APHx8RQuXJiGDRvTvXsvOnbsYql79uwZfv99Tqb7fuGFlyhXrnz2B/2QTZ48IdN1W7VqS+vWbXMsFhEREREREZHHhRJv8sgrV648Y8dm7qCKyMgIfH3XZLrvrl175InE2/3cs7u7x30n3jw8SrFjx/77jEpEREREREQkb1PiTR4rDRo0fCwTRI/jPYuIiIiIiIjYmtHWAYiIiIiIiIiIiORFSryJiIiIiIiIiIjkACXeREREREREREREcoASbyIiIiIiIiIiIjlAiTcREREREREREZEcoMSbiIiIiIiIiIhIDlDiTUREREREREREJAco8SYiIiIiIiIiIpIDlHgTERERERERERHJAfa2DuBxZTQaMBoNtg4j00wmMyaT2dZhiIiIiIiIiIg8MpR4swGj0YCLcwGMdo/OhENTsolrUXHZlnzr168nYWGhLFmyCg+PUtnSpy2Fhl7C27sX7u4e+PistnU4D8XkyRPw9V3D++9/RLduPW0djoiIiIiIiEiuo8SbDRiNBox2RtbP9+Pa5Ru2DueeXNwK0en5BhiNhsd21lteSxSKiIiIiIiISM5T4s2Grl2+QfjFaFuHIdmgRImSzJ/vg729vqVEREREREREJIWyBCLZwN7ennLlyts6DBERERERERHJRZR4k1wlISGBJUsWsWXLBs6fP4/JlIyHhyetW7dlwIAXKFKkiNV2gYEB/PbbLA4fDiY5OZly5SrQt+8zdO3ag5YtGwKwY8f++45n7drVTJnyP8ufvb17pSmfNm0GDRo0vOseb7ePv27dWnx8FnHmzGkcHR3x8mrM0KEjcHd3x2w2s2zZYlavXsmFC+dwdHSkefNWvPHGCFxcilmN79y5syxa9Dv79+8lIiIcBwcHKlWqQq9eT9O5c7f7vt87xcRE8+uvM9m+fQtXr0bi4lKMli1b88orr2fY5tq1a2zcuI49e3Zy9uwZIiMjsbe3p0yZsrRr1wFv72dxdHS02vbUqRPMmvUTAQF+JCQk4OlZmh49nqJfvwE888xTVpf73r4M+MyZ08yf/xshIccwGIzUqVOX114bRsWKlQBYv/4vli79g1OnTmJnZ6RBg0a8+eZIPD1Lp4tl27bN7Nr1D4cOBRMeHk5SUiKursWpX9+LF154kbJlyz/YwxUREREREZE8T4k3yTViYqIZOXIoISHHKViwIF5eDbG3tycgwI+5c2ezYcM6pk2bnm6PtY0b1zFx4jhMJhMVK1aiQoWKRESE88knEzlz5vQDxeTpWYauXXuwdesm4uPjadu2PU5OBSzlrq7FM93XjBnfs3DhPOrVa0CTJs05cuQQmzatJzg4kDlzFvLll5+wY8d26tf3olQpT4KDA/H1XcPx48eYOXMuDg4OafrbvHkjkyZ9RFJSIuXKladp0xbExt7g8OGDfPzxeA4c2Mf773+U5Xu/ejWSN94YwoUL5yhcuAjNm7fEZDKzfv1f7NmziwoVnrDabu/eXXz77ZeUKFEST8/S1KhRi6ioKA4fPsiMGd+zY8d2pk2bQb58+dK08/c/wDvvjCAxMRFPz9I0bNiEmJhopk//jkOHgu8Z78qVy5g//zdq1apDkybNCQk5xs6dOwgKCmTWrHmsXLmUP/5Y8P/PvxmHDx9k+/YtHD58kLlz/0iX1B0//r84ODhQvvwTeHk1JDk5mVOnTrJ27Wq2bNnI119/T+3adbP8fEVERERERCTvU+JNco2vvvqUkJDj1KhRiy+++IaiRZ0BiIuLY/z4sezevZOJEz9k+vTZljYREeF89tlkTCYTI0e+g7f3AEtZQIAf77478oFiqlu3HnXr1sPf/wDx8fG8+eaoLB+usHr1cmbOnEflylUASExM4K23hhEUFMDw4a+SkJDAggU+uLt7ABAVFcXrrw/i5MkQtmzZSKdOXS19nTx5gkmTxgMGJk/+nDZt2lvKwsJCGTPmLdauXU39+l507dojS/F+/fXnXLhwjrp16/PZZ1MpVKgQkJIgfeedkezYsd1qu6pVqzNjxq/UqlU7zfWYmBgmTHifvXt34+OziOee+4+lLDExgYkTx5GYmMiAAS/wxhsjMBpTTv09ffoUI0cO5erVyLvGu3jxAqZO/YGGDRsDkJyczIQJH7Bly0b++9+3iYyMSPP8ExISeOutNwgODmL58iW8+OLgNP2NH/8xzZu3wsnJyXLNbDazfLkPX3/9GZ9/Ppm5c//AYDBk5nGKiIiIiIjIY8ho6wBEAMLCwtiyZRMGg4H33vvAknQDKFCgAGPGfEi+fI4EBwcRHBxoKVuzZiXx8XHUqlUnTdINoF69BvTu3e9h3cI9DR78uiXpA+DomJ/+/Z8HUhJpo0a9Y0m6ATg7O9O7d18A9u/fm6avuXNnkZSUxJAhQ9Mk3QDc3T0YO3YcAD4+f2Qp1suXw9i+fQsGg4F33vmvJekGUKRIUd59978Zti1fvkK6pFtKuyKMGvUuAFu2bExTtmXLJsLDr+Du7sHrrw+zJN0AKlR4Il1SzJp+/fpbkm4AdnZ2DBz4EgCnTp1M9/zz58/PgAEvAHDgwL50/XXo0ClN0g3AYDDQp483tWrV4fTpUw88o1JERERERETyNs14k1whMNAPk8lElSrVqFSpcrryEiVK0qRJU/7+ext+fvstS/z8/f0A6Nixi9V+O3XqwsKF83Iu8PvQrFmLdNfKlCkDpCSJGjVqmq68dOmyAERERFiumUwmdu/eCUCHDh2tjlWtWg2cnAoQEnKMxMTEDPdUy0hgoD8mk4mqVatbXVJauXJVKlaszMmTIVbbJycn4+9/gIMHg4iIiCApKRGz2YzZbAZS9qa7XUBAyufYrt2TVk+G7dSpK1Onfn7XmJs2Tf98U58fWH/+/z7fcKt9Xrhwnj17dnLhwgXi4mIxmUwAltl3586dzXDJrYiIiIiIiIgSb5IrhIdfAaBUqYyXcZYqVfr/6/6bJAkPvwyQ4fJPd/esLQvNCW5u7umupe4X5+pa3GrCqUCBlPKkpETLtejoaGJjYwHo06f7PceNiYmmRImS9xXrlSt3f66Q8llZS7ydP3+O999/h9OnT2XYNjX+f8dL+fxvn/F3u8KFC1OoUCFu3LiRYZ/Wnm/q87tXeVJSUprrycnJTJ36OStXLrMkC62Ji4vNsExEREREREREiTfJEzLaZis37b91+/LJ+ym7k9lssnydmf3bHBzy3bNOdvrwwzGcPn2K5s1b8fzz/6F8+QoULFgIe3t7bt68Sbt2zTJse/fP6+6f5b2e4f084yVLFrJixVJcXV0ZNuwtateui4tLMcvMwQkTPmDjxnV3TcqJiIiIiIiIKPEmuULqjKxLly5mWCe1rESJEmnanTt3ltDQUKttQkMvZWOUuUPRos44OjqSmJjIm2+OwtnZOdvHSP08wsKsP1fA6jM/e/YMJ0+G4OJSjClTvkg3i+/8+XMZjFfi/8ez/nnduHGDGzeuZyr27LB5c8oedO+++z4tW7ZJV37hwvmHFouIiIiIiIg8unS4guQKdes2wGg0EhJynJCQ4+nKIyIi2LNnFwANGjS8rV19ADZuXGe13w0b/sqW+OztHYCUJYi2lrIfXBMANm/ekCNj1K3bAIPBwPHjRzl79ky68pCQ41aXmcbERANQvLj1pbPr1/taHa9evQZAyiELt27dSleeXZ9jZsXExADg5pZ+6eupUycJCTn2UOMRERERERGRR5MSb5IruLu7065dB8xmM198MYXo6ChLWXx8PJ9/PpmkpERq165jOVgBoEePp8ifPz9BQQEsXbo4TZ9BQQEsX+6TLfGVLJkyA+z06ZPZ0t+DGjToVRwcHPjxx2/x9V1j2fT/dqdOnWDbts1Z6t/d3Z3WrdtiMpn48stPiI39d2+1mJgYvv76U6vLLMuUKYednR2nTp3Ez29/mrIdO7azePECq+O1a/ckrq7FCQ29xM8//5jmfs6ePcOcOb9k6T6yqnz58gAsW7YkTSwRERFMmvRRrkjAioiIiIiISO6npaY25OJWyNYhZMrDinP06DGcPXuWw4cP0r9/b+rXb4idnR0BAX5ERV3Dw8OT8eMnpWlTsqQb7777PpMnT2Dq1M9ZtWo5FSo8QUREOEFBAfTv/zwLF86zOvvqfrRp0x4/v/1MnDiexo2bULhwEQCee24gZcuWf6C+s6Jq1WqMG/cxU6ZMYPLkCfzyy3TKl6+As7MLMTHRnDp1kitXLtOhQ0fatGmfpTFGjx7DiRMh+PsfwNv7KerXb4DZDH5++ylatCgtW7Zmx47tado4OzvTp88zLFmykFGj3qBOnXoUL16Cc+fOcvz4UV58cTC//TYr3Vj58+dn/PiPeffdUSxYMJft27dQtWp1rl+Pwd//AC1btuHw4YNcvhyGg4NDlu7nfgwc+DJ79uxi9erl+Pvvp0qVasTGxhIQcIBSpTxp3bod27dvyfE4RERERERE5NGmxJsNmExmTMkmOj3fwNahZJop2YTJlLMbyRct6syMGbNZsmQRmzevZ9++3ZhMZkqVKkXPnr159tmBFClSJF27zp27UbKkG3Pnzubw4YNcvHiesmXL8957H9CoURMWLpxH0aLODxTb00/3Iy4ujvXr17Jr107LKaOdOnW1SeINoH37J6levQY+PovYt28PwcGBJCebKFasGJ6epenTx5t27Z7Mcv+ursX5+ec5/PrrL2zfvpWdO3fg4lKMJ5/sxCuvDOWHH76x2m7EiNFUrFiJ5ct9OHbsKCdOHOeJJyrxv/9NoUOHTlYTbwBeXo34+ec5zJ79MwEBfvz99zZKlfJkyJA38PYeQKdOrTEajZakZ06qWbMWM2fO45dffuTIkcPs2LGdkiXd6Nu3Py+9NJipU7/I8RhERERERETk0Wcw61i+TIuIuM69ntbNm0lERobi6upx19MkjUYDRmPuOXHzXkwmc44n3nKCr+8aJk+eQIsWrfjss6m2DkeyKCDAj2HDXqVixUr89tsiW4eTa2X254+ILdnbG3FxKcigRYM4Hp5+T8+c1LFKRyZ0nsCBV17hxvH0+1TmpEJVKuM1cybXrsVy61b67QFEREREHpbU/x/T/xNlncEAxYsXzlRdzXizkUc1kZUbhYWF4eBgj6tr8TTXg4IC+OGHbwHo1q2XLUKT+3Dt2jXi4+MoVcozzfVTp07w2WcpS4y7detpi9BEREREREREskSJN3nk+fnt49NPP6ZSpcq4ubljNBq5ePEiJ06kzKTo1q0nbdq0s3GUci+nT59kxIjXKV/+CUqV8sTR0ZHQ0EscP34Uk8lEo0ZN6Nu3v63DFBEREREREck0Jd7kkVezZm26detJYKA//v4HiI+Pp3DhwjRs2Jju3XvRsWMXS92zZ8/w++9zMt33Cy+8RLly5bM/6Ids8uQJma7bqlVbWrdum2OxZKRs2XL06eNNQIAfwcGBxMXFUqBAQWrVqkPHjl3o2bP3Ax+SISIiIiIiIvIw6bdYeeSVK1eesWPHZapuZGQEvr5rMt1316498kTi7X7u2d3dwyaJt+LFSzB69JiHPq6IiIiIiIhITlHiTR4rDRo0ZMeO/bYO46F7HO9ZREREREREcic7O6NNxrXFfvtKvImIiIiIiIiISI5zKFYMk8lMkSJONhnflGziWlTcQ02+KfEmIiIiIiIiIiI5zr5QIYxGA+vn+3Ht8o2HOraLWyE6Pd8Ao9GgxJuIiIiIiIiIiORN1y7fIPxitK3DeChss6j2sfBw1wyLiOjnjoiIiIiISO6ixFs2MxpTHmlycrKNIxGRx01y8i3g359DIiIiIiIiYlv67Syb2dnZY2+fj7i4G5jNmn0iIg+H2WwmLi4We/t82NlpFwEREREREZHcQL+d5YCCBYsQHR3BtWvhFChQ8P9/CTbYOiwRyZPMJCffIi4ulqSkeIoWLW7rgEREREQkA0ajAaPRNr8bmkzmh7qhvIikUOItBzg5FQQgNjaGqKgIG0cjIo8De/t8FC1a3PLzR0RERERyF6PRgItzAYx2tll4Zko2cS0qTsk3kYdMibcc4uRUECengiQn38JkMtk6HBHJw4xGo5aXioiIiORyRqMBo52R9fP9uHb5xkMd28WtEJ2eb4DRaFDiTeQh029qOczOzh47O1tHISIiIrmBna1mOWh5kYhIrnHt8g3CL0bbOgwReUiUeBMRERHJYQ7FimEymSlSxMkm42t5kYiIiIhtKPEmIiIiksPsCxXCaDRoeZGIiIjIY0aJNxEREZGHRMuLRERERB4veTLxdunSJWbOnMk///xDaGgoZrOZEiVK0KhRIwYNGkS1atVsHaKIiIiIiIiIiORxttnhNwcFBgbSo0cP5s+fT3x8PC1atKBNmzYYDAZWrFhB37598fX1tXWYIiIiIiIiIiKSx+W5GW/jxo0jNjaW/v37M27cOBwcHAAwmUxMmzaN6dOnM378eNq3b4+jo6ONoxURERERERERkbwqT814u3btGseOHQNg1KhRlqQbgNFoZPjw4eTPn5+YmBhOnjxpqzBFREREREREROQxkKcSb/ny5ct0XRcXlxyMREREREREREREHnd5KvFWsGBBGjZsCMA333zDzZs3LWUmk4nvvvuOhIQEWrdujYeHh63CFBERERERERGRx0Ce2+Pt448/5tVXX+WPP/5g69at1KpVCzs7Ow4fPszly5d56qmnGD9+fJb6NhiyOVgRERGRh0j/LyMiIvq7QOTBvw/up32eS7w98cQT/PHHH7z33nvs2LGDy5cvW8oqVapE48aNKVSoUJb6dnUtnF1hioiIiDxULi4FbR2CiIjYmP4uEHn43wd5LvF24MABhg8fjp2dHV999RVNmzbFwcEBPz8/Pv30Uz744AP8/PyYMmXKffcdGXkdszkHghYREXlM2NkZ9T/9NnLtWizJySZbhyEi8tjKDX8H6u8CgdzxLtpSdnwfGAyZn5yVpxJvMTExDBs2jGvXrvHHH39Qt25dS1m7du2oVKkSPXv2ZOnSpfTq1YumTZveV/9mM0q8iYiIyCNL/x8jIiL6u0Dk4X4f5KnDFbZu3crVq1cpU6ZMmqRbqjJlylCnTh0Adu3a9bDDExERERERERGRx0ieSryFhoYC3HUPt8KFU6YCRkVFPYyQRERERERERETkMZWnEm9ubm4AnDp1iuvXr6crv3nzJocPHwagdOnSDzU2ERERERERERF5vOSpxFvr1q0pUKAACQkJfPjhh8TGxlrKkpKS+OSTT7h06RIODg506dLFhpGKiIiIiIiIiEhel6cOVyhWrBgTJkzg/fff56+//mLv3r3Url0be3t7Dh48yOXLlzEajXzwwQeUKVPG1uGKiIiIiIiIiEgelqcSbwBPPfUUVatW5bfffmPfvn3s2rULs9lMyZIl6dmzJ//5z38sByyIiIiIiIiIiIjklDyXeAOoVq0an3zyia3DEBERERERERGRx1ie2uNNREREREREREQkt1DiTUREREREREREJAco8SYiIiIiIiIiIpIDlHgTERERERERERHJAUq8iYiIiIiIiIiI5AAl3kRERERERERERHKAva0DEBERERGRvM9oNGA0GmwytslkxmQy22RsERF5vCnxJiIiIiIiOcpoNODiXACjnW0W3JiSTVyLilPyTUREHjol3kREREREJEcZjQaMdkbWz/fj2uUbD3VsF7dCdHq+AUajQYk3AWw3+9LORolnEbGtLCXeYmJi2LFjB7t27eLQoUNERkYSExNDkSJFcHV1pVatWjRt2pSWLVtSpEiR7I5ZREREREQeQdcu3yD8YrStw5DHWMrsSyeMdna2DkVEHhP3lXg7duwYc+fO5c8//yQxMRGzOe2/GMXHx3P58mUOHz7MkiVLcHR0pEePHrzwwgtUq1YtWwMXERERERERuR8psy/tOPLxx8SdPftQxy7WpAkVhgx5qGOKiO1lKvEWGRnJV199xYoVKzCZTLi4uNCmTRvq169P5cqVcXZ2plChQly/fp2oqChCQkLw9/dn3759+Pj4sGzZMp5++mlGjx6Nq6trTt+TiIiIiIiISIbizp7lxvGQhzqmU9myD3U8EckdMpV469SpE7GxsbRt25Z+/frRtm1b7O0zbtqyZUsGDRrErVu32LJlC0uXLmXp0qWsX7+effv2ZVvwIiIiIiIiIiIiuVWmEm916tTh7bffplatWvfXub09HTt2pGPHjgQFBTF16tQsBSkiIiIiIv/H3n2HR1Hu7x+/d5NAElpCldCLGwgQqjQp0gQ9eKQpKIIgIKigHo9UEeSrICJiAQQVRYqidBu9SJFeAykgIB5KSCghQAqkzO8PfrtmSQLJkk3b9+u6vC6Z9nx2M8/O7L0zzwAAgLwmQ8Hb3Llz77uhwMDALNkOAAAAAAAAkBc49FRTAIBjcurx9ZKUnGwoOdm494IAAAAAgCzhlODt+vXrKly4sEymnPlyCQC5kdlsko+vl9zMOfP4+qTkJF2NiiN8AwAAAIBs4lDwdvz4ce3atUstW7ZUlSpVbNN37dqlMWPGKDw8XMWKFdOIESPUrVu3LCsWAPIys9kkN7ObJqydoNNRp7O17cq+lTW+43iZzSaCNwAAAADIJg4FbwsWLNCyZcvUoUMH27SoqCi98soriomJkSRdvXpVY8eOVY0aNRQQEJA11QJAPnA66rSOXzye02UAAAAAAJzM7MhKBw4cUPXq1VW2bFnbtJ9++kkxMTHq2bOn9u3bpw8++EDJyclasGBBlhULAAAAAAAA5BUOBW+XLl2Sn5+f3bQdO3bIzc1Nr7/+ugoXLqwnn3xSAQEBOnToUFbUCQAAAAAAAOQpDgVvMTExKly4sN20w4cPq0aNGvL19bVNq1SpkiIiIu6vQgAAAAAAACAPcih4K1SokF2gdvLkSUVHR6t+/fqpluXJpgAAAAAAAHBFDgVvNWvW1MGDB/X3339LkpYuXSqTyaTGjRvbLXf27FmVKlXq/qsEAAAAAAAA8hiHnmras2dP7dq1S926dVOFChV07NgxlShRQo888ohtmRs3big0NFRt27bNqloBAAAAAACAPMOhK94ee+wxDR06VElJSQoLC5Ofn58++eQTFShQwLbM6tWrlZiYqIceeijLigUAAAAAAADyCoeueJOkoUOH6sUXX9SNGzdUvHjxVPMffvhhrVy5UhUqVLivAgEAAAAAAIC8yOHgTZIKFCiQZugmSX5+fvLz87ufzQMAAAAAAAB5lkO3mgIAAAAAAAC4uwxd8TZ69GiHGzCZTJo0aZLD6wMAAAAAAAB5UYaCtxUrVqQ53WQySZIMw0h3OsEbAAAAAAAAXFGGgrf3338/1bQjR47o+++/V8mSJfXYY4+pfPnykqRz585pzZo1ioyM1LPPPqs6depkbcUAAAAAAABAHpCh4K1r1652/z5+/LjeeecdPfvssxo1apQKFChgN//NN9/UBx98oGXLlqlnz55ZVy0AAAAAAACQRzj0cIUZM2aoVKlSGjt2bKrQTbr9tNO33npLJUuW1IwZM+67SAAAAAAAACCvcSh427t3r+rWrSuzOf3VzWaz6tatq3379jlcHAAAAAAAAJBXORS8xcTEKDo6+p7LRUdHKzY21pEmAAAAAAAAgDwtQ2O83alSpUras2eP/vrrL1WpUiXNZU6dOqXdu3ercuXK91Mf8hGz2SSz2ZQjbScnG0pONu69IAAAAAAAQBZxKHjr3r27Jk+erD59+ujVV1/VE088IS8vL0lSXFycfv31V02fPl2JiYnq3r17lhaMvMlsNsnH10tuZrccaT8pOUlXo+II3wAAAAAAQLZxKHjr06eP9u7dq40bN2r8+PEaP368fH19JUlRUVGSJMMw1LZtW/Xt2zfrqkWeZTab5GZ204S1E3Q66nS2tl3Zt7LGdxwvs9lE8AYAAAAAALKNQ8Gbm5ubZsyYoe+//17z5s3T//73P125csU2v0KFCnr++efVu3dvmUw5c2shcqfTUad1/OLxnC4DAAAAAADA6RwK3iTJZDKpd+/e6t27tyIiIhQRESFJKlOmjMqUKZNlBQIAAAAAAAB5kcPBW0qEbQAAAAAAAIA9c04XAAAAAAAAAORH93XF2+HDh7Vjxw5FRETo5s2baS5jMpk0adKk+2kGAAAAAAAAyHMcCt5u3bql//73v9qwYYOk208wTQ/BGwAAAAAAAFyRQ8Hb559/rvXr18vLy0tPPvmkqlWrpsKFC2d1bQAAAAAAAECe5VDw9ttvv8nLy0tLlixR9erVs7omAAAAAAAAIM9z6OEKFy5cUIMGDQjdAAAAAAAAgHQ4FLwVK1ZMxYoVy+paAAAAAAAAgHzDoeCtWbNmOnz48F0fqgAAAAAAAAC4MoeCt9dee03R0dGaPn16VtcDAAAAAAAA5AsOPVxh37596tatm2bNmqVt27apdevW8vPzk9mcdo7XpUuX+6kRAAAAAAAAyHMcCt5GjRolk8kkwzB05MgRHT169K7LE7wBAAAAAADA1TgUvHXp0kUmkymrawEAAAAAAADyDYeCt8mTJ2d1HQCAfMxsNslszpkfbJKTDSUn8zAgALkHn4kAALgOh4I3AAAyymw2ydfHW2Y3h57nc9+Sk5IVdTWWL5oAcgWz2SQfXy+5md1ypP2k5CRdjYrjMxEAgGySJcGbYRiKioqSJPn4+KT7kAUAgOsxm00yu5m17rsDioq4ka1t+5YprEd7N5DZbOJLJoBcwWw2yc3spglrJ+h01Olsbbuyb2WN7ziez0QAALLRfQVvO3fu1Jw5c7R//37dvHlTklSwYEE1atRIAwYMULNmzbKkSABA3hcVcUMXz0XndBkAkCucjjqt4xeP53QZAIAclFNDD7jl0J0orsrh4G3GjBmaOXOmDMP+17L4+Hht375df/zxh4YNG6aXX375vosEAAAAAADIL3J66AFkH4eCtx07dmjGjBny8PBQz5491aNHD1WoUEGSdObMGS1dulSLFy/W9OnTVb9+fa58AwAgl+EXVgAAgJyTk0MPNK3YVIObD87WNl2ZQ8Hb/PnzZTKZ9Pnnn6tly5Z282rUqKGxY8fqkUce0aBBgzR//nyCNwAAcpHbD7zwktmNX1gBAAByUk4MPVDJt1K2tufqHAregoKCVL9+/VShW0otWrRQ/fr1dejQIUdrAwAATnD7gRduurR8lBIuncrWtj2rt5Bv21eztU0AAAAgpzgUvF27dk1+fn73XM7Pz09BQUGONAEAAJws4dIpJVwIzdY23UtUydb2AAAAgJzk0EArvr6+OnXq3r+Qnzp1Sr6+vo40AQAAAAAAAORpDgVvDRo0UGhoqH755Zd0l/n5558VEhKihg0bOlwcAAAAAAAAkFc5dKvpgAEDtH79eo0cOVIbNmxQ165dVb58eUm3n2q6YsUKbdiwQW5ubnrhhReytGAAAAAAAAAgL3AoeAsMDNQ777yj//u//9PatWu1bt06u/mGYcjd3V3jxo1TYGBglhQKAAAAAAAA5CUOBW+S9PTTT6tevXqaN2+e9u7dq4iICElSmTJl1LhxY/Xt21cWiyXLCgUAAAAAAADyEoeDN0myWCyaOHFiVtUCAAAAAAAA5BsOPVwBAAAAAAAAwN05FLyFh4dr5cqVOnXqVLrLnDx5UitXrtSFCxccLg4AAAAAAADIqxwK3hYsWKDRo0fLMIy7Ljdq1Ch9//33DhUGAAAAAAAA5GUOBW9//PGHqlWrpmrVqqW7TLVq1VS9enVt27bN4eIAAAAAAACAvMqh4O3ChQuqWLHiPZerWLGiwsPDHWkCAAAAAAAAyNMcCt7i4uLk6el5z+U8PT0VExPjSBMAAAAAAABAnuZQ8FaqVCmFhobec7mwsDCVKFHCkSYAAAAAAACAPM2h4K1Ro0Y6ffq01q5dm+4y69at06lTp9SoUSOHiwMAAAAAAADyKoeCt759+8pkMmnkyJGaN2+ebty4YZt348YNzZs3TyNHjpTZbFbfvn2zrFgAAAAAAAAgr3B3ZKVatWrpjTfe0NSpUzV58mRNmTJFpUqVkiRdvHhRycnJMgxDb7zxhgIDA7O0YAAAAAAAACAvcCh4k6SBAweqSpUqmj59usLCwnThwgXbvBo1amjo0KFq3759lhQJAAAAAAAA5DUOB2+S1K5dO7Vr106XLl3S+fPnJUl+fn4qWbJklhQHAAAAAAAA5FX3FbxZlSxZkrANAAAAAAAASOG+g7fr16/ryJEjunLlivz8/NSgQYOsqAsAAAAAAADI0xx6qql0++mlb731lpo1a6YBAwZo+PDhWrJkiW3+kiVL1KJFCx0+fDhLCgUAAAAAAADyEoeCt/j4ePXt21fLli1TsWLF1KpVKxmGYbfMI488osuXL2vDhg1ZUigAAAAAAACQlzgUvM2dO1chISH617/+pfXr1+uLL75ItUypUqVUrVo17d69+76LBAAAAAAAAPIah4K3VatWqWTJkpo0aZK8vb3TXa5y5cq6cOGCw8UBAAAAAAAAeZVDD1c4c+aMmjdvroIFC951OU9PT0VFRTlUGAAAAAAAgDOZzSaZzaZsb9fNzeEh95HHOBS8mc1mJSYm3nO5iIiIu14RBwAAAAAAkBPMZpN8fbxkdnPL6VKQjzkUvFWsWFFhYWFKTEyUu3vam4iJidGxY8dUrVq1+yoQAAAAAAAgq5nNJpnd3HRp+SglXDqVrW17Vm8h37avZmubyBkOBW9t27bVrFmzNGvWLA0bNizNZWbNmqXr16+rQ4cO91UgAAAAAACAsyRcOqWEC6HZ2qZ7iSrZ2h5yjkPBW79+/bR8+XJ9/vnnCg0N1WOPPSZJunz5statW6fVq1drzZo1KleunHr16pWlBQMAAAAAAAB5gUPBW9GiRTVnzhy99NJL2rRpkzZv3iyTyaRt27Zp27ZtMgxDfn5+mj17NmO8AQAAAAAAwCU5FLxJUvXq1fXrr79q+fLl2rJli86ePavk5GSVLVtWLVu2VM+ePeXl5ZWVtQIAAAAAAAB5hsPBmyQVLFhQzzzzjJ555pmsqgcAAAAAAADIF8w5XQAAAAAAAACQHzl0xVtSUpLi4uLk6ekpd/d/NhEfH685c+YoNDRU5cqV04ABA1SmTJksKxYAAAAAAADIKxwK3mbOnKlZs2ZpwYIFatSokSTJMAz16dNHR48elWEYMplMWr9+vVauXKlixYpladEAAAAAAABAbufQraY7d+5UyZIlbaGbJG3atElHjhxRpUqVNGbMGD388MO6cOGCFi9enGXFAgAAAAAAAHmFQ8Hb2bNnVbVqVbtpGzdulMlk0tSpU9W3b1/Nnj1bxYsX19q1a7OkUAAAAAAAACAvcehW06tXr6pkyZJ20w4cOKAyZcqodu3atzfs7q66devq8OHD91+lA27duqUffvhBq1ev1smTJxUXFydfX19ZLBZ169ZNjz/+eI7UBQAAAAAAANfgUPDm7u6uuLg427+jo6P1999/67HHHrNbrlChQrp+/fr9VeiACxcuaMCAATpx4oR8fX3VoEEDeXl5KTw8XPv27ZO3tzfBGwAAAAAAAJzKoeCtfPnyOnz4sJKTk2U2m7V582YZhqGGDRvaLXflyhUVL148SwrNqPj4ePXv31+nTp3SsGHDNHjwYHl4eNjmx8XF6fTp09laEwAAAAAAAFyPQ2O8tW3bVpcvX9bLL7+sefPmaerUqXJzc1ObNm1syxiGoZCQEJUvXz7Lis2IL774QqdOnVLPnj01dOhQu9BNkry8vFSzZs1srQkAAAAAAACux6Er3gYNGqRNmzbp999/1++//y5JevHFF+Xn52dbZv/+/YqKikp1FZwzJSQkaNGiRZKkAQMGZFu7AAAAAAAAwJ0cCt4KFy6sJUuWaM2aNbp8+bLq1Kmjxo0b2y1z9epV9e3bN9W4b84UEhKiqKgolS5dWpUqVdKxY8e0fv16RUZGqmjRomrUqJFatWols9mhC/0AAAAAAACADHMoeJMkT09PdenSJd357du3V/v27R3dvEOOHTsmSXrggQc0depUzZkzR4Zh2OZ/9dVXCggI0MyZM+2uzgMAAAAAAACymsPBW2509epVSVJoaKiCgoLUu3dv9enTR6VKlVJQUJAmTJigkJAQDR48WMuXL081/tu9mExOKBrZir8h4Lr9wFVfN5AS/QBWrrovuOrrBlKiHwD33w8ys36Ggrfg4GDVqlXL0XqyfDvpsV7dlpCQoM6dO2vcuHG2ec2bN9fcuXPVqVMnHT9+XL/99ttdr9hLS4kSRbKyXGQzX99COV0CkONctR+46usGUqIfwMpV9wVXfd1ASvQDIPv7QYaCtx49eqhTp04aOnSoqlWrlulG/vzzT82YMUPr1q1TaGhoptfPqEKF/nnzevbsmWq+n5+fHnnkEa1du1Y7d+7MdPB2+fJ1pbhzFZng5mbO8Q/5qKgYJSUl52gNcG2u2g9c9XXnZrnhb4LsRz/IHXJD/+NYAFeWG/bFnEQ/sOfq+4Oryop+YDJl/OKsDAVvffr00ffff681a9aoXr166tatm5o2baoKFSqku86ZM2f0xx9/aMWKFQoKCpKbm5v69u2bsVfgoJT1pFdb+fLlJUkXL17M9PYNQwRveRx/P8B1+4Grvm4gJfoBrFx1X3DV1w2kRD8AsrcfZCh4GzNmjHr27KkpU6Zo69atOnTokCSpePHiqlq1qnx9fVWoUCHFxMQoKipKp06d0pUrV2zrt27dWsOHD3foarnMCAgIkMlkkmEYioqKUtmyZVMtExUVJUny9vZ2ai0AAAAAAABwbRl+uEK1atX0xRdf6PTp01q4cKE2bdqk8+fP6/Lly2ku7+fnp3bt2ql3796qXLlyVtV7V6VKlVLDhg21b98+7dixQwEBAXbzExIStHfvXklSYGBgttQEAAAAAAAA15Tpp5pWrlxZY8eO1dixY3XmzBmFhITo0qVLunHjhooUKaISJUooICDgrrehOtPQoUPVr18/ffnll2rUqJHq1asnSUpMTNQHH3ygM2fOqFChQurWrVuO1AcAAAAAAADXkOngLaUKFSrkWMCWnmbNmum1117Tp59+qt69e6tOnToqVaqUgoODde7cOXl6emratGkqWbJkTpcKAAAAAACAfOy+grfc6uWXX1ZgYKDmzZunoKAgHT16VCVLllS3bt00cOBAp481h9zJzc2cI+0mJxtKTmYEUwAAAAAAXE2+DN4kqUWLFmrRokVOl4FcoLh3cRlJSSpa1CtH2k9OSlLU1TjCNwAAAAAAXEy+Dd4AqyIFi8jk5qbQd99V7N9/Z2vb3pUqqebbb8tsNhG8AQAAAADgYgje4DJi//5bN47/mdNlAAAAAAAAF5Ezg14BAAAAAAAA+RzBGwAAAAAAAOAEBG8AAAAAAACAExC8AQAAAAAAAE5w3w9XuH79uo4cOaIrV67Iz89PDRo0yIq6AAAAAAAAgDzN4Svebty4obfeekvNmjXTgAEDNHz4cC1ZssQ2f8mSJWrRooUOHz6cJYUCAAAAAAAAeYlDwVt8fLz69u2rZcuWqVixYmrVqpUMw7Bb5pFHHtHly5e1YcOGLCkUAAAAAAAAyEscCt7mzp2rkJAQ/etf/9L69ev1xRdfpFqmVKlSqlatmnbv3n3fRQIAAAAAAAB5jUPB26pVq1SyZElNmjRJ3t7e6S5XuXJlXbhwweHiAAAAAAAAgLzKoeDtzJkzCgwMVMGCBe+6nKenp6KiohwqDAAAAAAAAMjLHArezGazEhMT77lcRETEXa+IAwAAAAAAAPIrh4K3ihUrKiws7K7hW0xMjI4dO6aqVas6XBwAAAAAAACQVzkUvLVt21YXL17UrFmz0l1m1qxZun79ujp06OBwcQAAAAAAAEBe5e7ISv369dPy5cv1+eefKzQ0VI899pgk6fLly1q3bp1Wr16tNWvWqFy5curVq1eWFgwAAAAAAADkBQ4Fb0WLFtWcOXP00ksvadOmTdq8ebNMJpO2bdumbdu2yTAM+fn5afbs2YzxBgAAAAAAAJfkUPAmSdWrV9evv/6q5cuXa8uWLTp79qySk5NVtmxZtWzZUj179pSXl1dW1goAAAAAAADkGQ4Hb5JUsGBBPfPMM3rmmWeyqh4AAAAAAAAgX3Do4QoAAAAAAAAA7o7gDQAAAAAAAHACh281PXPmjL766ivt3LlTkZGRunXrVprLmUwmhYSEOFwgAAAAAAAAkBc5FLyFhISoT58+io2NlWEYd132XvMBAAAAAACA/Mih4G3q1KmKiYnRo48+qiFDhqhSpUoqVKhQVtcGAAAAAAAA5FkOBW8HDx5UlSpV9Omnn8pkMmV1TQAAAAAAAECe59DDFTw8PFSzZk1CNwAAAAAAACAdDgVvAQEBunDhQlbXAgAAAAAAAOQbDgVvL7zwgg4dOqTdu3dndT0AAAAAAABAvuDQGG+tWrXSW2+9pZdfflnPPvusWrVqpbJly8psTjvH8/Pzu68iAQAAAAAAgLzGoeBNun27aenSpTVnzhzNmTMn3eVMJpNCQkIcbQYAAAAAAADIkxwK3vbt26cBAwbo5s2bkiQfHx95e3tnaWEAAAAAAABAXuZQ8Pbpp5/q5s2b6t+/v4YMGaJixYpldV0AAAAAAABAnuZQ8BYcHKyAgACNHDkyq+sBAAAAAAAA8gWHnmrq4eGhKlWqZHUtAAAAAAAAQL7hUPBWt25d/fXXX1ldCwAAAAAAAJBvOBS8vfzyyzp+/Lh+/fXXrK4HAAAAAAAAyBccGuMtISFBffv21YgRI7Rp0ya1atVKZcuWldmcdo730EMP3VeRAAAAAAAAQF7jUPDWp08fmUwmGYah1atXa/Xq1ekuazKZFBIS4nCBAAAAAAAAQF7kUPDGFWwAAAAAAADA3TkUvC1YsCCr6wAAAAAAAADyFYeCNwAAAAAAHGU2m2Q2m7K9XTc3h54vCAAOI3gDAAAAAGQbs9kkH18vuZndcroUAHC6DAVv58+flySVKVNGbm5utn9nlJ+fX+YrAwAAAADkO2azSW5mN01YO0Gno05na9tNKzbV4OaDs7VNAK4tQ8Fb27ZtZTab9dtvv6lKlSpq27atTKaMXRbMU00BAAAAAHc6HXVaxy8ez9Y2K/lWytb2ACBDwZv1ijV3d3e7fwMAAAAAAABIW4aCt02bNt313wAAAAAAAADsZeiRLqNHj9ayZcucXQsAAAAAAACQb2ToircVK1ZIkrp37+7UYgAAAAAAgGsxm00ymzM2jnxWcnPL0LVIwH3JUPAGAAAAAACQ1cxmk3x8vAnBkG8RvAEAAAAAgBxhNpvk5mbW2O+36a/I6Gxtu7m/n155rEG2tgnXQ/AGAAAAAABy1F+R0Qo7dyVb26xcqmi2tgfXxLWcAAAAAAAAgBNk+Iq3tWvXas+ePZluwGQyacOGDZleDwAAAAAAAMjLMhy8xcbGKjY2NtMNmEzZ/2QSAAAAAAAAIKdlOHhr2LChevTo4cxaAABwKWazSWZz9v9AxVPDAAAAgOyR4eCtYsWK6tq1qzNrAQDAZZjNJvn4eBOCAQAAAPkYTzUFACAHmM0mubmZNfb7bforMjpb227u76dXHmuQrW0CAAAArojgDQCAHPRXZLTCzl3J1jYrlyqare0BAAAAror7WwAAAAAAAAAnIHgDAAAAAAAAnCBDt5pu3LhR3t7ezq4FAAAAAAAAyDcyFLyVK1fO2XUAAAAAAAAA+Qq3mgIAAAAAAABOQPAGAAAAAAAAOAHBGwAAAAAAAOAEBG8AAAAAAACAExC8AQAAAAAAAE5A8AYAAAAAAAA4AcEbAAAAAAAA4ATujqzUt2/fDC3n4eEhX19f1a5dW507d1bJkiUdaQ4AAAAAAADIcxwK3vbs2SNJMplMkiTDMFItYzKZbNN/++03ffLJJ3rnnXfUpUsXB0sFAAAAAAAA8g6Hgrf58+dr8+bNmjt3rurUqaPOnTurXLlyMplMOnfunH799VcFBQWpf//+qlGjhnbt2qWVK1dq7Nixqlq1qgIDA7P6dQAAAAAAAAC5ikPBm4eHhxYsWKBRo0apX79+qeb37dtX8+bN04cffqj58+frySefVP369TVu3DjNmzdPH3300f3WDQAAAAAAAORqDj1c4fPPP1eVKlXSDN2snn/+eVWpUkWzZs2SJD311FMqV66cDhw44FChAAAAAAAAQF7iUPAWFBQki8Vyz+UsFouCgoIk3R7zrXr16rp8+bIjTQIAAAAAAAB5ikPB282bN3Xx4sV7Lnfx4kXdvHnT9m8vLy+5ubk50iQAAAAAAACQpzgUvFWtWlX79+/X4cOH013m8OHD2r9/v6pVq2abFhERIV9fX0eaBAAAAAAAAPIUh4K3Z599VklJSXrhhRf0ySef6OTJk4qPj1d8fLxOnjypTz/9VAMGDFBycrKeeeYZSVJcXJxCQ0NVu3btLH0BAAAAAAAAQG7k0FNNe/TooaNHj+qHH37QF198oS+++CLVMoZhqGfPnurRo4ck6dy5c3rsscf0+OOP31/FAAAAAJBPmM0mmc2mHGk7OdlQcrKRI20DgKtwKHiTpHfeeUctW7bU/PnzdejQIdtYbgUKFFC9evXUt29ftW/f3rZ89erV9f77799/xQAAAACQD5jNJvn4eMvNzaEbke5bUlKyrl6NJXwDACdyOHiTpHbt2qldu3ZKSkpSVFSUJMnHx0fu7ve1WQAAAADI98xmk9zczBr7/Tb9FRmdrW1XKV1M7z3bUmazieANAJwoSxIyNzc3lSxZMis2BQAAAAAu5a/IaIWdu5LTZQAAnOC+g7dbt24pODhYERERkqQyZcqoVq1aKlCgwH0XBwAAAAAAAORVDgdviYmJmjFjhhYuXKiYmBi7eYUKFVKfPn30yiuvcNspAAAAAAAAXJJDqVhycrJeeuklbd++XYZhqFixYipXrpyk208vjY6O1uzZsxUcHKzZs2fLbM6ZwUIBAAAAAACAnOJQ8LZkyRJt27ZN5cqV08iRI/Xoo4/azV+/fr0mT56sbdu2aenSpXr66aezpFgAAAAAAAAgr3DoUrSVK1fK09NT8+bNSxW6SVKHDh307bffqkCBAlqxYsV9FwkAAAAAAADkNQ4Fb3/++acaN26s8uXLp7tMhQoV1LRpU/35558OFwcAAAAAAADkVQ4Fb7du3VKRIkXuuVyhQoV069YtR5oAAAAAAAAA8jSHgreyZcvq4MGDSkpKSneZpKQkHTp0SA888IDDxQEAAAAAAAB5lUPBW4sWLRQeHq6JEycqISEh1fxbt27pvffeU3h4uFq1anXfRQIAAAAAAAB5jUNPNX3xxRf166+/atGiRdq4caMef/xx23hvZ8+e1apVqxQZGalixYpp0KBBWVowAAAAAAAAkBc4FLyVKVNGX331lV5//XWdP39e3377rd18wzDk5+enTz/9VGXKlMmKOgEAAAAAAIA8xaHgTZICAwO1Zs0arVmzRnv27FFERISk26Fc48aN1alTJxUoUCDLCgUAAAAAAADyEoeDN0kqUKCA/v3vf+vf//53mvNDQ0N148YNPfTQQ/fTDAAAAAAAAJDn3Ffwdi/vvPOOjhw5opCQEGc2AwAAAAAAAOQ6Dj3VNDMMw3B2EwAAAAAAAECu4/TgDQAAAAAAAHBFBG8AAAAAAACAEzh1jDcAAAAAQO7l5pb912LkRJsAkFMI3gAAAADAxZQo4ikjOUlFi3rldCkAkK+5RPA2ZcoUff3115Kk1157TS+//HIOVwQAAAAAOaeIZwGZzG66tHyUEi6dyta2Pau3kG/bV7O1TQDIKRkK3lauXOnQxq9cueLQelnpwIEDmjt3rkwmE09YBQAAAIAUEi6dUsKF0Gxt071ElWxtDwByUoaCt1GjRslkMmV644ZhOLReVomLi9Po0aNVqlQp1alTRxs2bMixWgAAAAAAAOBaMhS8+fn5ObsOp/joo490+vRpffnll1q9enVOlwMAAAAAAAAXkqHgbdOmTc6uI8vt3r1bCxcuVJcuXdS6dWuCNwAAAAAAAGSrfPkc55iYGI0ZM0YlS5bUmDFjcrocAAAAAAAAuKB8+VTTDz74QGfPntXMmTNVrFixLNtuDg5Xh3yA/Qe5havui676uoGU6AewctV9wVVfN5AS/QC4/36QmfXzXfC2fft2/fjjj/rXv/6l9u3bZ+m2S5QokqXbg+vw9S2U0yUAklx3X3TV1w2kRD+AlavuC676uoGU6AdA9veDfBW8Xb9+XW+99ZaKFy+usWPHZvn2L1++LsPI8s26BDc3s0t/yEdFxSgpKTmny0AOyw39ICf2RVd93feSG94XuJbc2A9cUW7o+xwLco/c8L7AtdAPgKzpByZTxi/OylfB26RJk3ThwgV9/PHHKl68eJZv3zBE8AaHse8gt3DVfdFVXzeQEv0AVq66L7jq6wZSoh8A2dsP8lXwtn79erm7u2vRokVatGiR3bxTp05JkpYuXaqdO3eqZMmS+vjjj3OiTAAAAAAAALiAfBW8SVJiYqL27NmT7vxz587p3LlzKleuXDZWBQAAAAAAAFeTr4K3ffv2pTtv1KhRWrFihV577TW9/PLL2VgVAAAAAAAAXJE5pwsAAAAAAAAA8iOCNwAAAAAAAMAJCN4AAAAAAAAAJ8hXY7zdzeTJkzV58uScLgMAAAAAAAAugiveAAAAAAAAACcgeAMAAAAAAACcgOANAAAAAAAAcAKCNwAAAAAAAMAJCN4AAAAAAAAAJyB4AwAAAAAAAJyA4A0AAAAAAABwAoI3AAAAAAAAwAkI3gAAAAAAAAAnIHgDAAAAAAAAnIDgDQAAAAAAAHACgjcAAAAAAADACQjeAAAAAAAAACcgeAMAAAAAAACcgOANAAAAAAAAcAKCNwAAAAAAAMAJCN4AAAAAAAAAJyB4AwAAAAAAAJyA4A0AAAAAAABwAoI3AAAAAAAAwAkI3gAAAAAAAAAnIHgDAAAAAAAAnIDgDQAAAAAAAHACgjcAAAAAAADACQjeAAAAAAAAACcgeAMAAAAAAACcgOANAAAAAAAAcAKCNwAAAAAAAMAJCN4AAAAAAAAAJyB4AwAAAAAAAJyA4A0AAAAAAABwAoI3AAAAAAAAwAkI3gAAAAAAAAAncM/pAlyR2WyS2WzKkbaTkw0lJxs50jYAAAAAAIArIXjLZmazST4+3nJzy5mLDZOSknX1aizhGwAAAAAAgJMRvGUzs9kkNzezxn6/TX9FRmdr21VKF9N7z7aU2WwieAMAADmOuwAAAEB+R/CWQ/6KjFbYuSs5XQYAAECO4C4AAADgCgjeAAAAkO24CwAAALgCgjcAAADkGO4CAAAA+VnOXNsPAAAAAAAA5HMEbwAAAAAAAIATELwBAAAAAAAATsAYbwByhNlsktlsypG2k5MNBtMGAAAAADgdwRuAbGc2m+Tj4y03t5y56DYpKVlXr8YSvgEAAAAAnIrgDUC2M5tNcnMza+z32/RXZHS2tl2ldDG992xLmc0mgjcAAAAAgFMRvAHIMX9FRivs3JWcLgMAAAAAAKfg4QoAAAAAAACAExC8AQAAAAAAAE5A8AYAAAAAAAA4AcEbAAAAAAAA4AQEbwAAAAAAAIATELwBAAAAAAAATkDwBgAAAAAAADgBwRsAAAAAAADgBARvAAAAAAAAgBMQvAEAAAAAAABOQPAGAAAAAAAAOAHBGwAAAAAAAOAEBG8AAAAAAACAE7jndAEAgOzj5pb9v7fkRJsAAAAAkBsQvAGACyjuXVxGUpKKFvXK6VIAAAAAwGUQvAGACyhSsIhMbm4Kffddxf79d7a2XbxJE1UZNChb2wQAAACA3IDgDQBcSOzff+vG8T+ztU2vihWztT0AAAAAyC0YeAcAAAAAAABwAoI3AAAAAAAAwAkI3gAAAAAAAAAnIHgDAAAAAAAAnIDgDQAAAAAAAHACgjcAAAAAAADACQjeAAAAAAAAACdwz+kCAAAAAGQfN7fs/+09J9oEACA3IHgDAAAAXEBx7+IykpJUtKhXTpcCAIDLIHgDAAAAXECRgkVkcnNT6LvvKvbvv7O17eJNmqjKoEHZ2iYAALkBwRsAAADgQmL//ls3jv+ZrW16VayYre0BAJBbMNgCAAAAAAAA4AQEbwAAAAAAAIATELwBAAAAAAAATkDwBgAAAAAAADgBwRsAAAAAAADgBARvAAAAAAAAgBMQvAEAAAAAAABOQPAGAAAAAAAAOAHBGwAAAAAAAOAEBG8AAAAAAACAExC8AQAAAAAAAE5A8AYAAAAAAAA4AcEbAAAAAAAA4AQEbwAAAAAAAIATELwBAAAAAAAATkDwBgAAAAAAADgBwRsAAAAAAADgBARvAAAAAAAAgBMQvAEAAAAAAABOQPAGAAAAAAAAOAHBGwAAAAAAAOAEBG8AAAAAAACAExC8AQAAAAAAAE5A8AYAAAAAAAA4AcEbAAAAAAAA4AQEbwAAAAAAAIATELwBAAAAAAAATuCe0wVkpYSEBO3bt09bt27Vnj179PfffysuLk4+Pj6qU6eOevXqpUceeSSnywQAAAAAAIALyFfB2969e9W/f39JUqlSpdSwYUN5eXnp5MmT2rx5szZv3qyePXtqwoQJMplMOVwtAAAAAAAA8rN8FbyZTCZ17NhRffv2VaNGjezmrVq1Sm+++aZ+/PFHNWjQQF26dMmZIgEAAAAAAOAS8tUYb82aNdNnn32WKnSTpMcff1xdu3aVJK1cuTKbKwMAAAAAAICryVfB270EBARIksLDw3O4EgAAAAAAAOR3LhW8nT59WpJUunTpnC0EAAAAAAAA+V6+GuPtbi5evKgVK1ZIkh599FGHtpGfnseQn15LXsF7nvvwN3Et/L0B+kFaeE9cC39vgH4ASPffDzKzvksEb4mJiRo+fLiuX78ui8Winj17OrSdEiWKZHFlOcPXt1BOl+ByeM9zH/4mroW/N0A/SAvviWvh7w3QDwAp+/uBSwRv48eP186dO+Xj46PPPvtMBQoUcGg7ly9fl2HcXy1ubuYc/7CLiopRUlJytraZG153TsqJ9zw3yw37A/3AteTGPsj+gOyW2/pBbugDHAtcS27rAxL7A7If/QDImn5gMmX84qx8H7y99957Wrp0qYoVK6a5c+eqSpUqDm/LMHTfwVtukV9eR17Ce5778DdxLfy9AfpBWnhPXAt/b4B+AEjZ2w/ydfA2efJkLViwQEWLFtXXX39te6opAAAA4OaW/c8Zy4k2AQBAzsm3wduUKVM0d+5cFSlSRF9//bXq1KmT0yUBAAAgFyhRxFNGcpKKFvXK6VIAAEA+ly+Dt6lTp+rrr79WkSJF9M033ygwMDCnSwIAAEAuUcSzgExmN11aPkoJl05la9ue1VvIt+2r2domAADIOfkuePv444/11Vdf2W4vJXQDAABAWhIunVLChdBsbdO9hOPjDQMAgLwnXwVvGzdu1OzZsyVJFStW1Pfff6/vv/8+1XK+vr4aOXJkdpeXazCeCQAAAAAAgPPlq+AtOjra9v9Hjx7V0aNH01yuXLlyLhm8MZ4JAAAAAABA9slXwVu3bt3UrVu3nC4j12I8E+AfXPkJAAAAAHC2fBW8IWMYzwSujCs/AQAAAADZheANgEvhyk8AAAAAQHYheAPgkrjyEwAAAADgbAw4BAAAAAAAADgBwRsAAAAAAADgBARvAAAAAAAAgBMQvAEAAAAAAABOQPAGAAAAAAAAOAHBGwAAAAAAAOAEBG8AAAAAAACAExC8AQAAAAAAAE5A8AYAAAAAAAA4AcEbAAAAAAAA4AQEbwAAAAAAAIATELwBAAAAAAAATkDwBgAAAAAAADgBwRsAAAAAAADgBARvAAAAAAAAgBMQvAEAAAAAAABOQPAGAAAAAAAAOAHBGwAAAAAAAOAEBG8AAAAAAACAE7jndAGAK3Bzy5mMOznZUHKykSNtAwAAAADg6gjeACfyKF5cycmGihb1ypH2k5OSFXU1lvANAAAAAIAcQPAGOJF74cIym01a990BRUXcyNa2fcsU1qO9G8hsNhG8AQAAAACQAwjegGwQFXFDF89F53QZAAAAAAAgG/FwBQAAAAAAAMAJCN4AAAAAAAAAJyB4AwAAAAAAAJyA4A0AAAAAAABwAoI3AAAAAAAAwAkI3gAAAAAAAAAnIHgDAAAAAAAAnIDgDQAAAAAAAHACgjcAAAAAAADACQjeAAAAAAAAACcgeAMAAAAAAACcgOANAAAAAAAAcAKCNwAAAAAAAMAJCN4AAAAAAAAAJyB4AwAAAAAAAJyA4A0AAAAAAABwAoI3AAAAAAAAwAkI3gAAAAAAAAAnIHgDAAAAAAAAnIDgDQAAAAAAAHACgjcAAAAAAADACQjeAAAAAAAAACcgeAMAAAAAAACcgOANAAAAAAAAcAKCNwAAAAAAAMAJCN4AAAAAAAAAJyB4AwAAAAAAAJyA4A0AAAAAAABwAoI3AAAAAAAAwAkI3gAAAAAAAAAnIHgDAAAAAAAAnIDgDQAAAAAAAHACgjcAAAAAAADACQjeAAAAAAAAACcgeAMAAAAAAACcgOANAAAAAAAAcAKCNwAAAAAAAMAJCN4AAAAAAAAAJyB4AwAAAAAAAJyA4A0AAAAAAABwAoI3AAAAAAAAwAkI3gAAAAAAAAAnIHgDAAAAAAAAnIDgDQAAAAAAAHACgjcAAAAAAADACQjeAAAAAAAAACcgeAMAAAAAAACcgOANAAAAAAAAcAKCNwAAAAAAAMAJCN4AAAAAAAAAJyB4AwAAAAAAAJyA4A0AAAAAAABwAoI3AAAAAAAAwAkI3gAAAAAAAAAnIHgDAAAAAAAAnIDgDQAAAAAAAHACgjcAAAAAAADACQjeAAAAAAAAACcgeAMAAAAAAACcgOANAAAAAAAAcAKCNwAAAAAAAMAJCN4AAAAAAAAAJyB4AwAAAAAAAJyA4A0AAAAAAABwAoI3AAAAAAAAwAkI3gAAAAAAAAAnIHgDAAAAAAAAnIDgDQAAAAAAAHACgjcAAAAAAADACQjeAAAAAAAAACcgeAMAAAAAAACcgOANAAAAAAAAcAKCNwAAAAAAAMAJCN4AAAAAAAAAJ3DP6QKcZfXq1fr+++8VFhamhIQEVaxYUU888YT69esnDw+PnC4PAAAAAAAA+Vy+DN4mTpyo+fPny93dXU2bNpW3t7d27dqlqVOnavPmzfrmm2/k6emZ02UCAAAAAAAgH8t3wduGDRs0f/58eXt7a+HChapVq5Yk6cqVK3r++ee1f/9+ffrppxo5cmQOVwoAAAAAAID8LN+N8TZ79mxJ0osvvmgL3SSpePHiGj9+vCRp4cKFun79eo7UBwAAAAAAANeQr4K3iIgIHTlyRJLUuXPnVPMbNWqksmXL6tatW9qyZUt2lwcAAAAAAAAXkq+Ct5CQEEmSj4+PKlSokOYytWvXtlsWAAAAAAAAcIZ8FbydPXtWklS2bNl0l3nggQfslgUAAAAAAACcIV89XCEmJkaS5OXlle4yhQoVsls2M8xmyTAcq+1ONfyKy6tA9r79lUoXlSQVeKCmTB7pv0fO4F6yiiTJUtIiT/fsfaJsRZ+KkqTCDz4oczY/zda74u22S5YrKvcCbtnatk+pQrb/N+fSiJ1+kH3oB/SDlOgH9IPchD6QfegDubMPSPSD7EQ/oB+kRD+gHzjKZMrEsoaRVVFSzps9e7Y+/vhjNWjQQIsWLUpzmY8//lizZ89WixYt9PXXX2dzhQAAAAAAAHAVuTTrdoz1ara4uLh0l7Fe6WZdFgAAAAAAAHCGfBW8lStXTpIUHh6e7jIXLlywWxYAAAAAAABwhnwVvAUEBEiSrl69qjNnzqS5zNGjRyVJtWrVyra6AAAAAAAA4HryVfD2wAMPqE6dOpKkX3/9NdX8ffv2KTw8XAUKFFDr1q2zuzwAAAAAAAC4kHwVvEnSkCFDJElffvmlgoODbdOjoqI0YcIESdJzzz2nIkWK5Eh9AAAAAAAAcA356qmmVu+9954WLFggDw8PNW3aVN7e3tq5c6euXbumBg0aaO7cufLM5kfmAgAAAAAAwLXky+BNklatWqXvv/9eoaGhSkxMVMWKFfXEE0+oX79+KlCgQE6XBwAAAAAAgHwu3wZvAAAAAAAAQE7Kd2O8AQAAAAAAALkBwVs+4+/vL39//2xp6+zZs/L391fbtm1TzWvbtq38/f119uzZVPNOnDihl19+Wc2aNVPNmjXl7++v6dOnS5L69Okjf39/7d692+n1S9Ly5cvl7++vUaNGZUt7yL2ys+/kRtevX9fq1as1ZswYPf7446pbt67q1Kmjdu3aafTo0Tp27FhOl4h8LLs/+wFHjBo1Sv7+/lq+fHmWbnfbtm0aNGiQmjRpotq1a6tt27YaN26cLly4kO461vOs9P57+umns7RGILe52/cQICuk9302q89ZXP07iKtwz+kC4FpiY2P14osv6ty5c6pdu7ZatGghNzc31axZM6dLA1zanDlzNHv2bElS5cqV1apVKyUlJSk4OFjLly/XL7/8onfffVddu3bN4UoBIP/45JNPNGvWLElSrVq1VL58eR07dkw//vijVq9erXnz5ikgICDd9Tt27Chvb+9U0ytUqJCldfbp00d79uzR/Pnz1aRJkyzdNgAge5w9e1bt2rVTuXLltGnTppwux6UQvMFhZcqU0apVq+Th4ZHhdY4cOaJz586pfv36+uGHH1LN/+CDDxQXFyc/P7+sLBXAPXh7e6t///7q1auXKleubJuekJCgqVOn6ttvv9Xbb7+tBg0aqFKlSjlXKPIlPvuRF7zxxhsaNGiQSpcunSXb27Jli2bNmiWz2ayPP/5YnTp1kiQZhqGZM2dq+vTpGjZsmFavXp3ug8FGjBih8uXLZ0k9AIB745wFjiB4g8M8PDxUrVq1TK0THh4uSXZf7FPiAwzIGYMHD05zuoeHh0aOHKnff/9dp0+f1m+//aaXX345m6tDfsdnP/KC0qVLZ1noJknz58+XJD355JO20E2STCaTXnnlFW3atEnBwcH66aef9NRTT2VZuwAAx3HOAkcwxls+tnbtWj3zzDNq0KCB6tWrp169emnLli1pLnvixAl99tln6tWrl1q2bKnatWurSZMm6tevn1atWpXmOpkZW2H37t3y9/fXyJEjJUkrVqywG4vEKr175lOOq3LmzBkNHz5cDz/8sGrXrq327dvr448/1q1bt9JsOzExUd9++62eeOIJ1alTR02bNtWwYcMyNGbVX3/9pXHjxql9+/aqU6eOGjZsqN69e+unn35Kc/mU9e/bt09DhgxR06ZNVaNGjSwfEwbOk5m+I93ex5YsWaI+ffqocePGtjF6xo8fbwubU7L2hz59+iguLk7Tpk1Thw4dVKdOHbVo0UJjxoxRREREmm3t2LFD7777rp588knbeECtWrXS66+/rqCgoDTXmT59um0sxfPnz2vMmDFq3bq1atWqlaHxDc1ms62f3m3MISDlZ/rixYvVrVs31atXT40aNdKgQYN06NChNNe723gpsbGx+uSTT/Too4/ahigYPXq0IiIi7PbtlJKTk/Xjjz+qV69eatSokWrVqqVmzZrp3//+t9599900xx9F/mMYhpo0aaIaNWooKirKbl5QUJBtf/3uu+9SrduuXTv5+/vrzJkztmnpjfGWcj+8cuWKJkyYoNatW6t27dpq3bq13n33XV27di1VG0eOHJEkNWvWLNU8k8mkpk2bSrp9TMoqme2j1uPVnj17JEl9+/a1O39L+V4cPXpUr7/+ulq1aqXatWurQYMGateunYYNG6YNGzZk2WtAzgkKCtKUKVPUo0cP23l48+bNNWTIEO3YsSPNdVKOpxwbG6uPPvpIHTp0UO3atfXwww9r5MiR6Z7zSNLmzZv13HPPqX79+mrYsKGeffbZDO1P0dHR+uyzz/Tkk0+qfv36qlu3rp544gl9/vnniouLS7U8xw3XdOLECb366qtq0qSJAgMD1blzZ3399ddKSkpKd530zlmuXLmi+fPna9CgQWrbtq0CAwPVoEEDdevWTV9++aVu3rx5z3oyc+4kZe47yKhRo9SuXTtJ0rlz51KNDXqno0eP6r///a8eeeQR1a5dW40bN9aAAQPS/U4UGRmp9957Tx07dlSdOnVUt25dtW7dWs8//7y+/vrre772/I4r3vKpzz77TJ9//rnq16+v1q1b69SpUzp48KAGDx6s6dOnq0OHDnbLz507V0uXLlXVqlVlsVhUtGhRhYeHa/fu3dq5c6cOHz6s0aNHO1xPyZIl1bVrV/399986cOCAKlasqIYNG2Z6O6GhoZo4caKKFSumhx56SNHR0Tpw4IBmz56tEydOaObMmXbLJycn67XXXtOGDRvk4eGhJk2aqGjRojp8+LCeeuopde/ePd22Vq9erZEjR+rmzZuqWrWqWrdurevXrysoKEgjRozQrl279P7776e57po1a/TDDz+oatWqat68uaKjo9O9TQS5S2b7zo0bN/TSSy9pz5498vb2Vu3ateXr66vjx4/rhx9+0Jo1azR37tw0x+hJSEhQv379dOzYMTVu3FgBAQHav3+/li1bpq1bt2rhwoWprg61HkgffPBBNWjQQO7u7jp16pRWr16t9evXa9q0aerYsWOar+306dPq2rWrPDw81KBBAxmGIV9f3wy9L3///bckqVSpUhlaHq7t/fff17x582xfvI8fP66tW7dqx44d+uSTT1L1o/TExsaqb9++OnLkiLy9vdWiRQsVLFhQ27Zt05YtW9S6des013vrrbe0fPlyFSxYUA0bNlTx4sV19epVnT17VgsXLlSzZs24Pc8FWMOrNWvWaOfOnXr88cdt81KGBDt37lTv3r1t/z5z5ozOnj2r8uXLZ2qstPDwcHXt2lWJiYlq0KCBbt68qQMHDmjhwoU6fPiwFi1aZDc8R2xsrCTJx8cnze1ZP5+Dg4PTbXP58uWKjo5WYmKiSpcurcaNG+uhhx66Z60Z7aPW87dt27bp0qVLatGihd1xoGLFipJuv4eDBg1SQkKCatSooXr16ik5OVkRERH6/ffflZSUpPbt29+zLuRu06ZN0+7du1W9enXVqlVLXl5eOnPmjDZv3qzNmzdrzJgxev7559Nc9/r16+rVq5fCw8PVsGFDPfjggzp06JBWrlypvXv36qefflKRIkXs1vn2229t59qBgYGqWLGiTp8+rVdeeUX9+/dPt84TJ05o4MCBCg8PV6lSpdSwYUO5u7vryJEj+vTTT7Vu3TotWLDArj2OG65n3759GjRokGJjY1WhQgU9/PDDioqK0scff6zDhw9nenvbtm3TxIkTVaZMGVWqVEn16tXTlStXdPjwYX300UfatGmT5s+fn+53wsyeO2X2O0jDhg0VGxurtWvXytvbO93vC5I0b948TZ48WcnJyapZs6YCAwN16dIl7d69W9u3b9ewYcM0dOhQ2/IXL15U9+7dFRkZKT8/P7Vs2VIFCxZUZGSkwsLCFBwcrAEDBmT6Pc1XDOQrFovFsFgsRqNGjYxDhw7Zzfvss88Mi8ViPProo6nW2717t/G///0v1fSTJ08arVq1MiwWi3H48GG7eWfOnDEsFovRpk2bVOu1adPGsFgsxpkzZ+ymL1u2zLBYLMbIkSPTrP+5554zLBaLsWvXLrvpI0eOtL22adOmGYmJibZ5x44dM+rVq2dYLBbjwIEDdustXLjQsFgsRvPmzY0TJ07YpickJBjjx4+3bfPOesLCwozatWsbderUMdauXWs37+zZs0bnzp0Ni8VirFixIs36LRaLsXDhwjRfI3InR/vOG2+8YVgsFmPw4MHGpUuX7ObNnTvXtl7KfXbXrl229jp06GCcO3fONi8+Pt4YNmyYYbFYjKeffjpVe+vXrzeuXr2a5vSAgACjcePGRlxcXJr1WywW48033zRu3ryZsTfl/9uyZYthsVgMf39/IzQ0NFPrwrVY97PAwEBjx44ddvO++uorw2KxGA0bNkzVV9L77J80aZJhsViMxx9/3IiIiLBNT9lPLBaL8dlnn9nmnTt3zrBYLEarVq2MyMjIVDWeOHHCrs8hf/vhhx8Mi8VijB071m56nz59jFq1ahmdOnUyGjVqZPcZnd461nORZcuW2U1P+Rk7atQou8/Y8+fPGy1btjQsFovxyy+/2K1nnZ7e+cLbb79t225MTIzdPOt5Vlr/de/e3Th9+nSa28zqPmrVp08fw2KxGD/99FOqedeuXTMOHjyY5nrIW37//Xe7z2KrAwcOGA0aNDBq1aplXLhwwW6e9dzfYrEYL7zwgnH9+nXbvKtXrxpPPvmkYbFYjNmzZ9utFxoaatSsWdOoUaOGsXr1art5P/30k+Hv75/m95C4uDijffv2hsViMT7++GO7/hgbG2s7bxs1apRtOscN1xMfH2+0bt3asFgsxsSJE+2OAaGhoUaTJk1s++2d32fT+zw8ceJEmp91V69eNV544QXDYrEYX331Var5jn4uO/Id5G7f3622bt1q+Pv7G02aNDH27NljNy8sLMyWDezevds2ffr06YbFYjHefvttIzk52W6dW7dupXpdrohbTfOpV199VXXr1rWbNnjwYBUpUkSnT59Odelp48aN0/xVt2rVqrbxnNasWeO8gjOoVq1aev311+Xm5mabZrFY9O9//1uSUl3mPm/ePEnS0KFD7cajc3d31+jRo9O9emf27Nm6deuWXn/9dT366KN288qVK6eJEydK+md8ljs1bdrU7tdz5B2Z6TsnT57Ub7/9ptKlS2vq1KkqUaKE3Xr9+vVT69atdfr0aW3dujXN9kaMGGE3VkTBggU1fvx4eXl56dChQzpw4IDd8u3bt1exYsVSbad9+/bq1KmTrl69mu7jzX18fDRu3LhMXX0ZERGht956S5L09NNPq0aNGhleF66rZ8+eqW6fGzhwoGrXrq3r169ryZIl99xGfHy8Fi9eLEkaPXq03dhaBQsW1DvvvCMvL69U6126dEmSFBAQkOZnfLVq1RifxYU0b95ckv35QXx8vA4ePKj69eurTZs2unbtmo4ePWqbb102rVtA7+aBBx5I9RlbtmxZPffcc6lqkGS7lXTp0qUyDMNuXnR0tN15140bN+zmt27dWh999JHWr1+voKAgbdy4UR988IH8/Px05MgR9enTR5cvX0631qzooylZ20rrKtQiRYqoXr16mdoecqfWrVunOc5h/fr11bt3byUkJKR7G6i3t7fef/99FS5c2DatWLFievHFFyWl7h8LFy5UUlKSOnXqZDcGoiT9+9//TneomxUrVuh///uf2rRpo9dff92uP3p5een//u//VKJECf3888+Kjo6WxHHDFa1du1bh4eEqW7ashg8fbvfdskaNGhoyZEimt1mtWrU0P+uKFSumsWPHSrr79+nMfC5nxXeQ9EyfPl2GYWjChAmprqC23jYu3e6jVtZjQMuWLWUymezW8fDwyPTxND8ieMun2rRpk2pagQIFbOFaWmMpxMTEaPXq1Zo2bZrefvttjRo1SqNGjdK6desk3R7vLKe1adMmVWeWZAvVUr6uiIgI2+1x1mAupYIFC6Y6kEu3b0+1fkClvC0lpTp16sjb21uhoaFp3q9/t0t3kbtlpu9s2bJFhmGoVatWdieSKTVu3FiSdPDgwVTzihYtahtrIaUSJUqoZcuWkmQbVyeliIgILV68WJMnT9Zbb71l66t//vmnpPT7arNmzVLdxnE3N27c0JAhQxQZGanAwEBbAAfcS9euXdOc3qVLF0lp79d3Onr0qGJjY+Xr66sWLVqkml+8eHFbqJJS1apVVahQIW3dulWzZs2yG6MLrqdChQoqX768zp49q//973+Sbt9edOvWLTVv3jxVMGcYhnbt2iWTyZTpLwrNmjVLMwxO6xxFkgYNGqSCBQsqJCREQ4cO1fHjxxUTE6ODBw+qf//+tltRpdtjbaY0fvx4de7cWRUrVlTBggVVvnx5denSRStWrFC5cuUUERGh2bNnp1trVvTRlAIDAyVJb775pvbt26fExMRMrY+8IyoqSitXrtSUKVM0duxY2zmIdZ9J7xykdu3aaYZ2VatWlZS6f1i3l9Y5vJT+Pmwdf+qxxx5Lc36hQoVUu3ZtJSYm2sZZ5Ljheqz712OPPWY3BIBVevvXvSQlJWnnzp2aOXOm3nnnHY0ePVqjRo2yfR7f7ft0Zj6X7/c7SHquXLmioKAgeXp6pvmdSJKaNGkiSXYXB1iPAVOnTtW6desUExOT4TZdBWO85VPp/Spj7Zh3hkWbNm3S6NGjdfXq1XS3eeevrTmhbNmyaU63vq6UD1iwDgLv6+urQoUKpbleWmM1XL161fZa0xs/6M7ly5QpYzetXLly91wPuVNm+o71xGzp0qVaunTpXbd75cqVVNPKlSuXZpAs/bNv3vkwgxkzZmj27NlKSEhIt630+mpm9suYmBgNHDhQISEhCggI0Jw5c1SwYMEMrw/Xlt44OOnt12mxfgm7236b1rzChQvr/fff1+jRo/XJJ5/ok08+UalSpVSvXj21bNlSnTt3TveYgPypefPmWrx4sXbs2KGKFSvaQraHH35YFotFBQoU0I4dO/TSSy8pJCREV69eVUBAQIbHwLTKzDmKJD344IOaPn263nzzTW3YsMHuSiEfHx+NGjVK7777rkwmk4oWLZqhGnx8fPT8889r0qRJ2rx5c7o/mGRFH03pjTfe0LFjx7R161Zt3bpVnp6eCggIUOPGjfXvf//b7q4D5F2LFy/W+++/bxcK3ym9L9yZ7R/WffBe++qdrOdmI0aM0IgRI9KtU/rn3Izjhuu51/5VrFgxFSlSRNevX8/wNk+fPq2hQ4fafghPy92+T2fmc/l+v4Ok5+zZszIMQ/Hx8apTp85dl0350KInn3xSf/zxh3755RcNGzZMbm5uqlatmho2bKiOHTtyxZsI3vKtO38ZvZuIiAj95z//UXx8vAYOHKgnnnhC5cuXl7e3t8xms7Zv355rBkPMzOtyVHJysu3/M/JrR1q/knh6emZpTcg+mdnHrPtKzZo173kL5p23r2ZUytuP1q1bp+nTp8vb21tvv/22mjZtqtKlS8vT01Mmk0nTpk3TF198keqWJauM7pexsbEaPHiwDh48KH9/f33zzTdp3t4KOCq9fTQt6YXTd5vXsWNHNW/eXBs3btT+/ft14MABrV+/XuvXr9dnn32mb775Js0neCF/atasmS1469Wrl3bu3KlixYqpdu3aMpvNql+/vg4cOKC4uDiHbzOVHDtHad26tTZu3Ki1a9fq2LFjSkxMVPXq1fX4448rJCREklS5cuVMDRFgDbnu5ynUmemj0u0H7yxbtkx79uzRjh07dODAAQUFBenAgQP64osv9MYbb9huKUTedPToUY0bN05ubm5688031bZtW5UtW1ZeXl4ymUz68ccfNW7cuHT3new4h5f+OTdr2bKlSpYseddlU/7YynED9+vVV1/Vn3/+qTZt2mjgwIGqVq2aChcuLA8PD926deueQda9pOxbzvoOYm3jXg9fuJPZbNbUqVM1ZMgQ/f777zpw4IAOHDigRYsWadGiRWrTpo1mzpxpd0uvqyF4gzZt2qT4+Hh16NBBw4cPTzXfertmXmO9Ci0qKkoxMTFp/lJ17ty5VNN8fX3l6emp+Ph4jRgxQsWLF3d6rcibrL/eNmjQQOPGjcv0+mntf3fOe+CBB2zTVq9eLUn6z3/+o549e6Za5/Tp05mu4U5xcXEaPHiw9u7dK39/f3377beZvuoDOHv2rGrWrJlqelr7dXqsn+EZ6SdpKVKkiLp06WK7RSM8PFzvvvuuNm7cqHfffddubBLkb82aNZPJZNLu3bt1+fJlhYaGqkOHDrYgoHnz5tq9e7f27t2rnTt32qZll6JFi+qpp55KNX3fvn2Sbl+ZlxnWuxfudoVOVvTRO5lMJjVp0sR2G9LNmze1fPly/d///Z8+/vhjderUyfYUVOQ9a9askWEYeu655zRo0KBU87PiHCSlMmXK6H//+5/OnTunBx98MNX89D7/y5Ytq1OnTqlHjx5pDilzNxw3XIf1HOPs2bNpzr927VqmrnY7efKkjh07phIlSmjGjBlyd7ePWTLyfTozn8v3+x0kPdY2TCaTJk2alOnAvHr16qpevbqkf4Zu+O9//6vNmzdr5cqV6t69e5bVmtcwxhtsA4umdYudYRj65ZdfsrukLPHAAw/YxuX69ddfU82/detWmgNcurm52U64rUEHkJZWrVpJuh1epzXW371cu3ZNmzZtSjX9ypUr2rZtm6R/xmeQ7t5XL1++nGpg4syKj4/X4MGDtWfPHlvoRvAMR/z00093nZ5yv05PrVq15OXlpStXrqS5b6c3PT1ly5bVq6++KkkKDQ3N8HrI+3x9fVWzZk1dvXpVc+bMkWEYdsGa9f9///137d+/XwUKFFCjRo1yqlxJ0vXr17V06VK5ubnpmWeeydS6v/32m6R/xtxJS2b7qPXq/qSkpAzXUbBgQT3zzDPy9/dXcnKyjh07luF1kfvc7Rzk5s2btjGhs4p1UPf0voesXLkyzenWc7OsOIfnuJF/WfevNWvWpDl8S3r7V3qs/aN06dKpQjdJ+vnnn++5jcx8Ljv6HcT6WZ7eOJxlypSRv7+/YmJibN9FHGUdK7Vz586S6EMEb7DdkrB27VpFRkbapiclJenTTz/N1ICMuc3zzz8v6fbTWU6ePGmbnpSUpA8++MDu9aY0dOhQeXh46MMPP9SKFSvsbj+1On78eJafZCBvCQgIUMeOHRUeHq6hQ4em+atZbGysfv75Z9sTs+70wQcf2N0OdOvWLU2YMEGxsbEKDAxUw4YNbfOsAxAvXrzYbiyU69eva+TIkZn6Ze5ON2/e1EsvvaTdu3cTuuG+LVq0KNXTdb/99lsFBQWpUKFC6tGjxz234eXlZVvu/ffft+tDt27d0rvvvpvmOEMhISFatWqV4uPjU82zBt08nc71WG8d/e677yTZX0VWu3ZtFS1aVEuXLlV8fLzq16+fbUNGBAUFpbo178KFC3rppZd08eJFDRw40Hb1gNWGDRvsnsJqdePGDU2cONG2n/fv3z/ddjPbR61Xh6Q3dtHXX3+t8+fPp5p+8uRJ25Ue9Lu8zfp9YeXKlXbjVN28eVPvvPNOulcOOapPnz5yc3PT6tWrtX79ert5v/32W7pPT3366adVrlw5rVmzRh9++GGaY2pdvHjR9tRsieOGK+rUqZPKlCmj8+fPa9q0aXbf9Y4fP65Zs2ZlanuVK1eWm5ubjh8/nuqzddOmTfr222/vuY3MfC47+h2kePHi8vDw0KVLl9Id2/3111+XdPuJ8mldIGAYhg4fPqzt27fbpq1cuTLd45L1oRCuPgY6t5pCbdq0Ua1atRQcHKyOHTuqcePG8vLyUlBQkCIjIzVo0CB99dVXOV2mQ3r37q0//vhDmzdv1pNPPqkmTZqoWLFiOnz4sC5evKhnnnlGixYtSrVerVq19OGHH9qeRPPJJ5+oevXq8vX1VXR0tI4fP64LFy7o8ccf16OPPpoDrwy5xaRJk3Tt2jVt3bpVnTp1Uo0aNVS+fHkZhqFz584pLCxMCQkJWrVqVaqxRurXr6/k5GR16tRJTZs2laenp/bv36/IyEiVKFFCH3zwgd3yzz//vH766Sdt2bJF7du3V7169ZSQkKC9e/fK09NT3bt317Jlyxx6HdOmTbNdPeTn56cpU6akuVzDhg3TvCUKSKlnz556/vnn1ahRI5UpU0bHjx/X8ePH5ebmpkmTJqlUqVIZ2s5//vMfHThwQMHBwerQoYOaNm2qggULav/+/UpISFDXrl21YsUKu7E2z58/r//85z+2wd3Lli2rxMREHT9+XH/99Zc8PDzSHFYB+Vvz5s319ddf6+bNmypfvrzdLY9ms1lNmjSxfbnPzttMX3jhBXl5ecliscjHx0eRkZE6ePCgEhIS1LNnT9sXoJR2796t+fPny8/PTxaLRUWKFFFkZKTCwsIUHR0td3d3jRgx4q6vI7N9tGPHjlq+fLk+/PBD7dy5U8WLF5fJZFL37t3VoEEDzZo1S1OmTFHVqlVVrVo1FSxYUJGRkTpw4IASExPVpUsX1apVK6vfPmSjbt26af78+QoJCVG7du3UqFEjubm5ad++fYqPj1ffvn01f/78LGuvZs2aeuONN/Thhx9q6NChqlu3ripUqKC///5bR44cUb9+/dIMM7y9vfXFF19o8ODBmjNnjhYvXix/f3+VKVNG8fHxOn36tE6ePKkSJUro6aeflsRxwxV5enpq6tSpevHFF/XNN99ow4YNqlOnjq5evao9e/aoTZs2Cg4OvuuQFikVL15cvXv31vz589WvXz81atRIpUuX1l9//aXg4GC99NJL9wzzMvu57Mh3EA8PD7Vt21Zr165Vly5d1LBhQ9sPTRMnTpQktW3bVm+99ZY++OADvfTSS6pUqZKqVKmiwoULKyoqSmFhYbp8+bIGDRpke+r8unXrNHLkSJUuXVo1a9ZU0aJFde3aNR04cEDXr1+XxWJx+e8PBG+Qu7u7FixYoC+//FJr167Vzp07VbhwYdWvX1+fffaZYmJi8mzwZjabNWPGDC1YsEBLly7Vnj175O3trYYNG2rmzJkKCQlJM3iTbj9euk6dOlqwYIFtoOCkpCSVLFlSFStWVO/evTM9dgTyn8KFC+ubb77RqlWr9PPPPys4OFhhYWEqVKiQSpcurSeeeELt2rVLc1wbDw8PffHFF5oxY4bWrl2riIgIFStWTN26ddOrr76a6glgFSpU0IoVK/TJJ59o//792rx5s0qVKqV//etfGjZsWLr7ckZYL5GXpM2bN991WVc/cOLexowZoypVqujHH3/UkSNH5O7urpYtW+rll19WgwYNMrydQoUK2Y5Pv/32m7Zt2yYfHx81b95cr7/+umbMmCFJduMQ1q1bV//973+1b98+nTx5UqGhoXJzc9MDDzyg3r1767nnnrNdPQrX0ahRIxUoUEC3bt1KM5Bq1qxZjgRvffv21R9//KHg4GDduHFDPj4+euSRR9SrVy/bF5o7tW/fXrGxsQoJCdHRo0cVHR0tDw8PlS1bVo899pieffbZew4Cn9k++sgjj+i9997TokWLtGvXLsXFxUm6/WOMdYyhnTt36ujRo9q7d69iY2NVqlQpNW/eXD179lS7du3u/81CjrJeFTp9+nRt375dW7dulY+Pjx5++GENHTpU+/fvz/I2Bw4cqCpVqujrr79WaGio/vzzT/n7++uzzz5TrVq10r2K6MEHH9TPP/+sH374QRs2bNCxY8d06NAh+fj46IEHHtALL7ygDh062JbnuOGaGjdurMWLF2v69Onas2eP1q9frwoVKujVV1/VCy+8kOmLK8aMGSN/f399//33Onr0qNzc3GSxWPTxxx/r8ccfv2fwltnPZUe/g/zf//2ffHx8tG3bNq1du9Z2q601eJNuH5uaNm2qhQsXavfu3dq5c6fMZrNKliypmjVr6pFHHrF7f1544QWVL19eBw8etD0d3MfHR9WrV1fnzp3VrVs3eXt7Z+r9zG9MRmYfWwQAuC+7d+9W37591bhxYy1YsCCnywGyjPXLfnaM5ZSQkKDOnTvr9OnTWr58OVfTABmQnX0UAADcxhhvAAAg1zp69GiqcTZjYmL07rvv6vTp0/L39yd0AwAAQK7FraYAACDXevXVVxUXFyeLxaISJUro8uXLCgsLs93GMHny5JwuEQAAAEgXwRsAAMi1+vXrp/Xr1+vkyZM6cOCAzGaz/Pz89MQTT2jAgAGpxkIEAAAAchPGeAMAAAAAAACcgDHeAAAAAAAAACcgeAMAAAAAAACcgDHe8qmIiAh16tRJTZo00ezZs+3mtW3bVufOnUt33bp162rx4sXpzr9165Z++OEHrV69WidPnlRcXJx8fX1lsVjUrVs3Pf7441n2OrKb9b3ZuHGjypcvn61tv/XWW1qxYoVWrFghf3//bG07v0qvHyxfvlyjR4++5/omk0lhYWGppicnJ2vx4sVatmyZTpw4IUmqXr26evTooaefflomkynrXkQOoB/kPXf7zL9TUlKSnn32WR06dEiS9N1336lRo0YZaue1117TmjVrJElTpkzRk08+mWqZU6dO6Y8//lBwcLCCg4N18uRJJSUl6bXXXtPLL7+cuReWS9FH8hZnHQsk6caNG1qwYIHWr1+vv//+WwkJCSpRooQCAgLUu3dvNW/ePMteR3az7l/Hjh3L9rb79eunoKAgrV27VqVKlcr29vOq9Pb169eva/v27dq2bZsOHTqkc+fOKTk5WaVLl1bjxo3Vr1+/ND9P4uLitGvXLm3btk379u3TmTNnbPt4gwYN9Nxzz6lhw4Zp1vLzzz9r+/btCgsL08WLF3Xt2jV5enqqSpUq6tChg5577jkVKlTIae9FduBYkDs545woODhYu3btsp3b/P333zIMI91zISv6gXPlpX5A8JZPTZkyRfHx8frPf/6T7jIdO3aUt7d3qukVKlRId50LFy5owIABOnHihHx9fdWgQQN5eXkpPDxc+/btk7e3d54O3nLSsGHD9Msvv+i9997TggULcrqcfCG9flCxYkV17do13fV27dql8PBwNWnSJNW8pKQkvf7661q3bp28vLzUtGlTSdLOnTs1btw47dixQx9//LHMZi4odgT9wDEZ+cy3+vrrr3Xo0CGZTCZlZpjXVatWac2aNfdcb9GiRZo/f36Gt4vMoY9knjOOBZJ0/PhxDRw4UBEREXrggQfUtGlTubm5KTw8XFu2bFHFihXzdPCWk/773/+qR48emjZtmt5///2cLifPSG9fnzNnji2AqFy5slq1aqWkpCQFBwdr+fLl+uWXX/Tuu++m6g+//vqrxo4dK0kqV66cmjVrJnd3d4WFhWnVqlVavXq1XnvtNb300kupalm0aJEOHjyoatWqKSAgQD4+Prp06ZIOHTqkI0eOaNmyZVqwYIHKlCnjpHcjf+NYkD5nnBPNnDlTGzduzHQt9APnykv9gOAtHwoKCtKvv/6qTp063TX5HTFiRKZS6fj4ePXv31+nTp3SsGHDNHjwYHl4eNjmx8XF6fTp0/dTukt74IEH9NRTT2nhwoXauHGj2rVrl9Ml5Wl36weNGjVK9wqfmzdvqmXLlpKkHj16pJq/YMECrVu3TmXKlNF3331nC6rPnDmjZ599VmvWrNFDDz2k5557LotfkWugH2ReRj/zJenPP//U9OnT1aZNGx0/fvyuVz+ndOnSJU2YMEEBAQHy9PTUgQMH0l3WYrHohRdeUEBAgAICAvTFF1/op59+ytRrQvroI5njrGPBpUuX1K9fP0VHR+udd95Rr1697K52vnbtmiIjI7PwlbiWOnXqqE2bNlqxYoWef/551ahRI6dLyvXutq97e3urf//+6tWrlypXrmybnpCQoKlTp+rbb7/V22+/rQYNGqhSpUq2+e7u7urevbuee+45BQQE2KYbhqFvv/1WkydP1ieffKKGDRuqcePGdm2OGjVKlSpVko+Pj930qKgovfLKK9q/f78++OADTZs2LeveBBfCsSBtzjonqlevnh588EHbuc2YMWO0Z8+ee9ZDP3CuvNQPuCQjH5o3b56ktE8U78cXX3yhU6dOqWfPnho6dKhd6CZJXl5eqlmzZpa26WqsfzPr3xCOc7QfrF+/XtHR0SpatKgeffRRu3nJycmaM2eOJOnNN9+0uzq0QoUKevPNNyXd7ivJycn3U75Lox9kTkb39cTERI0cOVKenp6aMGFCptp4++23FRMTo/fff1/u7nf/ze6pp57SyJEj9cQTT6hatWpc/ekE9JGMc8axQLp9RcXly5f16quv6plnnkk1xEDRokVVvXp1xwuHevToIcMw2M8z6G77+uDBgzVq1Ci70E2SPDw8NHLkSFWuXFkJCQn67bff7OZ37dpVkyZNsgvdpNu3X/fv31/NmjWTpDR/XKlbt26qsEGSfH199cYbb0iS/vjjjwy/PqTGsSA1Z50Tvfjii/rPf/6jjh073vXusDvRD5wvr/QDrnjLZy5duqS1a9eqdOnSevjhh7NsuwkJCVq0aJEkacCAAVm23T59+mjPnj2aP3++ihYtqpkzZ2rv3r2KiYlRxYoV1aNHD/Xv3z/NMbMSExO1ZMkS/fTTT/rzzz9169YtlS1bVq1atdKgQYPSvWT3xIkT+uyzz7R7927FxcXZbjXp16/fXWtNTEzUihUr9PPPP+vYsWOKjY1V6dKl1bJlSw0ZMkRly5ZNtc6OHTs0f/58BQUFKTo6Wt7e3vL19VVgYKB69uyphx56yG75mjVrqkaNGtq9e7dOnjypatWqZfzNhM399INly5ZJkp544gkVLFjQbt7Bgwd18eJFFShQQB07dky1bseOHfXWW28pMjJShw8fVv369TPUJv2AfuCozOzrs2fPVnBwsCZNmpSpWxpWrlypTZs26ZVXXsmxq07oI/QRRzjrWHD58mWtWrVKnp6e6t27d5bVm3KcnLNnz+rLL7/UkSNHdPPmTVWrVk3PP/+8unTpkua6cXFxWrBggVavXq3Tp08rOTlZ5cuXV/v27fXCCy+oWLFiaa538OBBzZw5U4cOHVJSUpKqVKmiZ5999p5fWuPj4/X9999rzZo1OnXqlG7evCk/Pz+1a9dOgwYNkq+vb6p1Vq9erR9//FGhoaG6ceOGChcubDdW2J2fL61bt5avr69+++03jRw5Ms0vr7jtfvZ1s9ksf39/nT59WhcuXMjUujVr1tTOnTszvZ6bm5skpfoR/144FnAsuJvsOCfKSvQD1+oHBG/5zJYtW5SQkKCmTZve8yqD5cuXKzo6WomJibbBVe/cka1CQkIUFRWl0qVLq1KlSjp27JjWr1+vyMhIFS1aVI0aNVKrVq0cvrJh+/btmjt3ripWrKiHH35YFy9etF16Gx4errfeestu+Vu3bmnw4MHasWOHChYsqCZNmqhw4cI6ePCgFixYoF9//VVff/21atWqZbfevn37NGjQIMXGxqpChQp6+OGHFRUVpY8//liHDx9Ot74bN27opZde0p49e+Tt7a3atWvL19dXx48f1w8//KA1a9Zo7ty5dr8IrlixwjZoc2BgoJo0aaL4+HhFRERo1apV8vX1TfP9bt68ucLCwrRhw4Zc+8GR22WmH6R0/vx57dq1S1Lav5SFhoZKkh588MFUX8QkydPTUw8++KBCQkIUEhKS4eDNin7wD/pBxmR0Xw8NDdXs2bPVokULde/ePcPbj4iI0MSJE2WxWDRkyJCsKPm+0Ef+QR+5N2cdC3bv3q2EhATVrl1bhQsX1oEDB7RlyxZFRUWpePHiat68earb7jJj2bJlmjVrlgICAtSyZUudO3dOhw4d0siRI3X16tVUX3Ss00JDQ1W4cGE1bdpUHh4e2rNnj2bPnq1ff/1V8+bNSzW8yOrVq/Xf//5XSUlJslgsslgsCg8P19ixY20PDkpLRESEBg4cqOPHj8vHx0d16tRRoUKFFBISoq+//lpr1qzRggULVK5cOds6M2bM0PTp0+Xu7q769eurTJkyun79usLDw7V06VJVr149VfDm4eGhxo0ba+3atdq+fbs6d+7s8Hua3zm6r1v9/fffkpTpB1k4st6NGzc0Y8YMSbfDZkdwLPgHx4J/OPucKCvRD1ywHxjIV958803DYrEYCxcuTHeZNm3aGBaLJc3/unfvbpw+fTrVOj/++KNhsViMHj16GB9++KHh7++fat0uXboY586dy1S9zz33nG39RYsW2c3bsWOH4e/vb9SsWdMIDw+3m/fhhx8aFovFaN++vXHmzBnb9Fu3bhljxowxLBaL0bZtW+PmzZu2efHx8Ubr1q0Ni8ViTJw40UhMTLTNCw0NNZo0aWKrJeU2DcMw3njjDcNisRiDBw82Ll26ZDdv7ty5hsViMR599FG7bbZt29awWCzG3r17U73uS5cuGcHBwWm+J+vWrTMsFovx/PPPp/Ou4V4y0g/SMn36dNu+nJb333/fsFgsxssvv5zuNoYMGWJYLBZj8uTJGW6XfpAa/SBjMrKv37x503jiiSeM+vXr231GW48Faf1trAYMGGDUrFnTCAoKsk2z7q8rV67MUI0jR440LBaLMXPmzAwtnxb6SGr0kXtz1rFg2rRphsViMYYOHWpr487/+vXrZ1y9ejVT7Vr7ZK1atYxNmzbZzVu2bJlhsViMhg0bGnFxcXbzXn/9dcNisRhPPfWUceXKFdv0GzduGAMHDjQsFovRs2dPu3UiIyON+vXrGxaLxZg7d67dvB07dhh16tSxvZaUkpOTjV69ehkWi8UYM2aMcf36ddu8hIQEY/LkyYbFYjH69Oljm37z5k0jMDDQqFevnnHy5MlUr/vs2bPGiRMn0nxPrP1mzJgxac7HbY7u64ZhGFu2bDEsFovh7+9vhIaGZni9sLAwIyAgwLBYLMbGjRvTXW7btm3GyJEjjeHDhxsvvPCCbb8bMGCAce3atUzVyrEgNY4F/3D2OVFKmT0Xoh/QDxh4JZ+xXpFzt6S3devW+uijj7R+/XoFBQVp48aN+uCDD+Tn56cjR46oT58+unz5st06V69etW3/q6++sg0iv3//fs2dO1eVK1dWSEiIBg8erISEhEzX/eijj6pXr15205o1a6YWLVooKSnJ9suzdHvA4++++06SNHr0aLtfcD08PDR27FiVLFlSZ8+e1dq1a23z1q5dq/DwcJUtW1bDhw+3Xd4rSTVq1Ej3ao6TJ0/qt99+U+nSpTV16lSVKFHCbn6/fv3UunVrnT59Wlu3brVNv3z5sooUKZLmwM0lSpRINV6GlXVMmJCQkDTn494y0g/uZBiGli9fLin9cSFiYmIk3R7PMD3WJwVbl80M+sE/6AcZk5F9febMmTp27JhGjBghPz+/DG978eLF2rZtmwYMGKA6dercd61ZgT7yD/rIvTnrWBAVFSVJ2rx5s3777TcNGzZMGzdu1J49ezRjxgyVKlVKO3bssI3fk1nPPfec2rRpYzetW7duqlq1qq5fv66jR4/app8/f972tOH/+7//s7vFs1ChQnrvvfdUsGBBHTx40O6hKEuXLlVMTIzq1auX6gq6Zs2aqWfPnmnWtm3bNh04cEA1a9bUhAkTVLhwYds8d3d3DR8+XBaLRbt379bx48cl3b7qIT4+XhUqVFDVqlVTbbNcuXLp/o3YzzPGkX1dun31ovWqmKeffjrDwwnExMTozTffVGJiolq0aHHXK3ZOnDihFStW6KefftL27dsVExOjzp07a/LkySpSpEim6rXiWPAP+sg/nHlOdL/oB/QDgrd85tKlS5J013Ewxo8fr86dO6tixYoqWLCgypcvry5dumjFihUqV66cIiIibI8ctzL+/+OVExIS1LlzZ40bN05VqlRR4cKF1bx5c82dO1cFCxbU8ePHUw3MmhF3nmBaWT84Uz4Z7MiRI4qNjZWPj0+aB3ovLy89/vjjkm7fDmJlffLMY489lua99Hc+Qt1qy5YtMgxDrVq1sjvBTMl6S8nBgwdt0+rUqaPr169rxIgROnr0aIYH27f+7aKjo3Xr1q0MrQN7GekHd9q5c6fOnTunggUL5tjtLPSDf9APMuZe+3pQUJC++uorNW3aNN0v0mk5d+6cJk+erGrVqmnYsGFZUWqWoI/8gz5yb84+FiQkJGjgwIEaOnSoypcvr2LFiqlDhw6aOXOmTCaTtm/frn379mW67nvt5xEREbZpe/fuVXJysgICAtIMTcqUKaMWLVpISns/f+KJJ9Js6277uXT7C19aD1oxm822L0zW/bx48eIqV66cjh07psmTJ9/1NtY7Wf921r8l0ubIvn7jxg0NGTJEkZGRCgwMTHVbWnoSEhL02muv6fjx46pQoYI+/PDDuy7fr18/HTt2TEePHtX69es1atQobdu2Tf/617+0d+/eDNebEseCf3As+IezzomyAv0gNVfrB4zxls/cuHFDktLdwe/Gx8dHzz//vCZNmqTNmzfbHYALFSpk+/+0Pqj8/Pz0yCOPaO3atdq5c2e6g/+mJ62BFaV/XsfNmzdt06wfIinHDrlTxYoVJdmfnFoHfr1zjBOrYsWKqUiRIrp+/brd9DNnzki6/evw0qVL7/o6rly5Yvv/d955R4MHD9ZPP/2kn376SYUKFVKdOnXUtGlTPfnkk+n+ypLyb3f9+vVUvxDg3hzpB9aBtDt06JDuINTWfhAXF5fudmJjY+2WzQz6wT/oBxlzt3395s2bGjVqlAoWLKj33nsvzUF202IYhsaMGaO4uDhNmjRJBQoUyNKa7wd95B/0kXtz9rFASvucqG7dugoICFBwcLB27NiR5i/3d3Ovv3nK/dy676a3v0qO7efpTbfu559++qk+/fTTdNuU7PfzKVOm6NVXX9XcuXM1d+5c+fj4KDAwUA8//LD+/e9/q3jx4mluw/qar127dte2XF1m9/WYmBgNHDhQISEhCggI0Jw5c9Icu/ZOiYmJeuONN7Rt2zaVK1dO8+bNS/dvdycPDw9VrFhR/fv3V4MGDdSzZ08NHz5ca9askaenZ4a2YcWx4B8cC/7hjHOirEY/SM1V+gHBWz5TpEgRXblyxfbBk1nWhPzOpxOlfGxyeo9QtnbIixcvZrpdRx/KkB2sSbv1iSl3U7duXdv/V6tWTWvWrNEff/yhXbt26eDBg9q/f7927dqlmTNnauLEiXryySdTbSPlB1fRokWz6FW4lsz2g2vXrmn9+vWS7v74cevBKjw8PN1l7nWAuhv6wT/oBxlzt3391KlTOnnypHx9fTVmzJhU862f1e+9956KFCmili1b6sUXX9T169e1a9cueXt766OPPkq1nvVWjtmzZ2vp0qWqUaNGhq+UuF/0kX/QR+7NWccC6+e7u7t7ul98KlSooODgYIfOiXLqC2FGWPfzhg0b2r6kpefBBx+0/X+jRo20adMm/f7779q7d68OHjyo7du3a+vWrfrss880c+ZMNWvWLNU2rPs5+/jdZWZfj42N1eDBg3Xw4EH5+/vrm2++STdkTikpKUlvvvmm1q1bp7Jly2revHl3/RJ/N3Xr1lX16tX1559/6ujRo5kOpzkW/INjwT+ccU7kTPSD21ylHxC85TMlSpTQlStXbGOyZZZ1vTuv1gkICJDJZJJhGIqKikrzRNM65ol1jCtnKV26tKTbt0Klx5qyp3wksvX/z549m+Y6165dS5XWS//8mtCgQQONGzcuU7W6u7urdevWat26taTbv8TMnTtXM2bM0Pjx49WhQ4dU75f1b1CsWLFMP14at2W2H/zyyy+6efOmypcvr6ZNm6a7nHVcgT///FM3b95M9etwfHy8/vzzT7tlnYV+AClj+3pUVJTtVoG0WIO0O79AxcbG3nW9U6dO6dSpU5krOBvRR+CsY0Ht2rUl3b7658aNG2me5GfXOZF1f7Xuy2lJbz8/depUuv0jvenW/bxdu3YaMGBApmr19PRUp06d1KlTJ0m3r3L45JNP9OOPP2rMmDHavHlzqnWsf7uSJUtmqi1Xk9F9PS4uToMHD9bevXvl7++vb7/91m5cwPQkJSVp+PDhWr16tcqWLav58+en+0N8RlnHy71zXOmsxrHAdTjznMhZ6Aep5dd+kHtjUjjE+mX/5MmTDq1vHZ8tMDDQbnqpUqXUsGFDSdKOHTtSrZeQkGC7P/3OdbNanTp15O3tratXr2rjxo2p5sfHx2vVqlWSpCZNmtimWx89vGbNmjQfALFy5co022vVqpUkadOmTXaX7DqicOHCGjZsmIoWLaq4uDidPn061TLW4ObORzkj4zLbD6y3FnXr1u2uVxrUr19fpUqV0q1bt+wGHrVau3atEhISVLp0abtfb5yBfgDp7vt6zZo1dezYsXT/s55Ufvfdd7axl6TbvxTebT3rmBxTpkzRsWPHtGDBgmx6tZlDH4GzjgWBgYG2213++OOPVPOvXr2q4OBg27LO9NBDD8lsNis0NFRhYWGp5kdGRmrbtm2S0t7Pf/nllzS3e6/9fM2aNbbxfx1VvHhxDR8+XNLth0RER0enWob9PGMysq/Hx8dr8ODB2rNnjy10y8htosnJyRoxYoR+++03W+h2r6sd7+XKlSu2/bVy5cr3ta174VjgOpxxTuRM9IOMyS/9gOAtn7F2lJSDFKa0YcMGu6dhWd24cUMTJ07Upk2bJEn9+/dPtczQoUMlSV9++aUOHTpkm56YmKgPPvhAZ86cUaFChdStW7f7fRl3VbBgQfXu3VuS9MEHH9gl9wkJCZo4caIuXryo8uXLq2PHjrZ5nTp1UpkyZXT+/HlNmzbNbrDG48ePa9asWWm2FxAQoI4dOyo8PFxDhw5NM/GPjY3Vzz//bBvUMy4uTnPnzrW7Z91q3759unbtmtzc3PTAAw+kmm/9293t13bc3b36QUphYWEKDg6W2Wy+575rNps1cOBASdLUqVPtrjA4c+aM7ba8wYMHO/3yb/oBpMzt666GPgJnHQtMJpNeeeUVSdKHH35od+VnXFycxo0bpxs3bsjPz0/t27e/j1dwb35+furUqZMMw9C4ceNsV9pJt/e3cePG6ebNm6pfv74aNGhgm9ejRw95e3vr4MGDmj9/vt02d+/erR9++CHN9tq1a6c6deooKChIo0ePTnPfjY6O1qJFi5SYmCjp9hUWS5YsSfP2L+t5Z7FixdIcl4n9PGPuta/fvHlTL730knbv3p3p0G306NH69ddfMxW6nThxQj///HOaX7j/+usvvfbaa7p165bq1asnf3//e27vfnAscB257ZyIfkA/SIlbTfOZ1q1by8PDQ7t27VJSUpLdI3+l2ydT8+fPl5+fnywWi4oUKaLIyEiFhYUpOjpa7u7uGjFihJo3b55q282aNdNrr72mTz/9VL1791adOnVUqlQpBQcH69y5c/L09NS0adOy5XaAV199VUePHtXOnTv1+OOPq0mTJipUqJAOHTqk8+fPy8fHR59++qndoOCenp6aOnWqXnzxRX3zzTfasGGD6tSpo6tXr2rPnj1q06aN7bXcadKkSbp27Zq2bt2qTp06qUaNGipfvrwMw9C5c+cUFhamhIQErVq1SiVLllRCQoImT56sKVOmyGKxqFKlSvLw8NC5c+dsoeWQIUPSPOmxXlHYrl0757x5LuBe/SAl64CfDz/8cLpj9aTUp08f7du3T+vXr9cTTzxhG5Nm586diouLU8eOHfXss89mzQu5B/oBMrOvZ5fg4GBNmDDB9u///e9/kqQff/xRv//+u236jBkzbLc+OAt9xLU581jQo0cPHTp0SEuWLFGXLl1Ut25dFSlSREFBQbp48aJt38rIgPX3a9y4cTp16pQOHz6sDh06qEmTJnJzc9PevXt15coVlS9fXlOnTrVbp0yZMnrvvfc0fPhwTZw4UUuWLJHFYlFERIT27dun559/Xt9++22qtsxms2bOnKnBgwdrxYoVWrt2rfz9/eXn56eEhASdOXNGx48fV1JSkrp16yZ3d3ddu3ZNY8eO1YQJE2x9QpL+/vtvhYSEyGQyafjw4an+Pta7KQoWLGh7MivSdq99fdq0abbPDD8/P02ZMiXN7TRs2FBPPfWU7d8LFy60XdFSoUIFff7552muV7VqVbvxsC5fvqzhw4dr/Pjxqlmzph544AElJCTo/PnzCgkJUXJysqpVq6aPP/74fl52hnEscA3OPCf6/fff7fZ/69OZZ8yYoe+++842ffHixbb/px/QD1IieMtnSpYsqY4dO+rXX3/V9u3bbfdGW7Vv316xsbEKCQnR0aNHFR0dLQ8PD5UtW1aPPfaYnn322bsm7i+//LICAwM1b948BQUF6ejRoypZsqS6deumgQMH2h7O4GwFChTQnDlztHjxYv3000/at2+fbt26pbJly6pPnz4aNGiQ3f3pVo0bN9bixYs1ffp07dmzR+vXr1eFChX06quv6oUXXtCjjz6aZnuFCxfWN998o1WrVunnn39WcHCwwsLCVKhQIZUuXVpPPPGE2rVrZ/sV0NvbWxMmTNDevXsVEhKiHTt22G5BfPTRR/XMM8+kOYhwSEiIjh07piZNmqh69epZ+6a5kHv1A6tbt27ZbrPp3r17hrbt5uamzz77TIsXL9aSJUu0a9cuSVL16tXVo0cP9ezZM9sGxqYfIKP7ena6ceOGDh8+nGr6hQsX7B7ckx2Pe6ePuDZnHguk24NwN2vWTD/88INCQ0MVHx+vsmXL6rnnntOgQYPS/FXeGXx9ffXDDz9owYIFWrVqlf744w8lJyerfPnyevrpp/XCCy+kOXj+v/71L5UpU0azZs3SoUOHdObMGVWpUkUTJkxQz5490wzepNuh3eLFi7V8+XKtWrVKx44d05EjR1SsWDGVLl1avXr1Utu2bW2hY4UKFTRmzBjt3btXf/75p7Zs2SLp9phDXbp0UZ8+fWzj5qX0+++/KyoqSt26dZOPj0+WvV/50b329ZS38aY1ll5KKYO3lOvdbVysxo0b2wVvDz74oP7zn/9o3759OnXqlEJDQ5WQkCAfHx81a9ZMHTp0UPfu3bPtqdkcC1yDM8+Jrly5kua5zf/+9z/bD4x3oh/QD1IyGfc7QANynaCgID311FN69NFHNX369JwuB5nw7rvvauHChfr8889zdWKfF9AP8i76Qeawr7se+kjG0T/yriFDhuj333/XihUrVLNmzZwuJ9djX3c9HAtSox+4nrzSDxjjLR8KDAxU586dtX79+jQH2kXuFB4eriVLlqhx48a5+kMjr6Af5E30g8xjX3ct9JHMoX/kTUFBQdq8ebO6du1K6JZB7OuuhWNB2ugHriUv9QOCt3xqxIgR8vLyyrZ7xnH/ZsyYocTERL311ls5XUq+QT/Ie+gHjmFfdx30kcyjf+Q906ZNU6FChfTGG2/kdCl5Cvu66+BYkD76gevIS/2AW00BAAAAAAAAJ+CKNwAAAAAAAMAJCN4AAAAAAAAAJyB4AwAAAAAAAJyA4A0AAAAAAABwAoI3AAAAAAAAwAkI3gAAAAAAAAAncM/pAgAAACC1bdtW586ds/3bZDLJy8tLRYoUUaVKlVS7dm099thjCgwMzMEqAQAAkBlc8QYAAJCLNGjQQF27dlWXLl3UunVrValSRceOHdM333yjp556Sn369NGZM2eypK2zZ8/K399fbdu2zZLtZbfp06fL399f06dPz+lSAAAA0sQVbwAAALnIU089pW7dutlNMwxDW7du1aRJk7Rnzx716tVLP/zwgypUqJBDVQIAACAjuOINAAAglzOZTGrdurWWLFmiypUr69KlSxo7dmxOlwUAAIB7MBmGYeR0EQAAAK7OOsbb+++/n+qKt5S2bNmiF198UZK0bNky1a5dW5J04sQJrVq1Sjt27NC5c+cUFRWlQoUKqWbNmnr66af1+OOP221n1KhRWrFiRbrtHDt2TJJ048YNrVq1Slu3btXx48cVGRkpSapQoYLatm2rAQMGqGjRoqnWj4yM1Jdffqlt27bp/PnzMpvN8vHxUeXKldWqVSsNGDAg1ToRERH65ptvtHXrVts6VatWVdeuXdWrVy+5u/9zs4a/v3+6tXft2lWTJ09Odz4AAEB24VZTAACAPKRVq1by8fHR1atXtWPHDlvwNnfuXC1dulRVq1aVxWJR0aJFFR4ert27d2vnzp06fPiwRo8ebdtOw4YNFRsbq7Vr18rb21sdO3ZMs72wsDC9/fbbKl68uKpUqaJatWrp2rVrOnr0qGbPnq3Vq1frxx9/lK+vr22dixcvqnv37oqMjJSfn59atmypggULKjIyUmFhYQoODk4VvO3du1evvPKKoqOjVa5cOTVv3ly3bt3SkSNH9O6772rz5s2aPXu2PDw8JN0O10JDQxUWFqYaNWqoZs2adq8NAAAgNyB4AwAAyENMJpMCAgK0Y8cO/fnnn7bpTz75pIYMGZJq3LdTp06pf//++vbbb/Wvf/3L9lTUp556Ss2aNdPatWvl6+ub7hVi5cuX17fffqsmTZrIbP5nlJK4uDi98847WrlypT777DONHz/eNu/HH39UZGSkevbsqQkTJshkMtnmJSQkaN++fXZtXLx4UUOHDtW1a9c0fvx49erVy9ZWVFSUXn/9dW3fvl1ffPGFhg4dKkmaPHmypk+frrCwMLVv317Dhg1z5O0EAABwKsZ4AwAAyGOsV5ddvXrVNq1x48ZpPmyhatWqevnllyVJa9asyXRbDzzwgJo1a2YXukmSl5eX3nnnHbm7u6fa7uXLlyVJLVu2tAvdJMnDw0PNmjWzmzZv3jxdvXpVvXv31rPPPmvXlq+vr6ZMmSIPDw999913YpQUAACQl3DFGwAAQB6TnJwsSalCrZiYGG3dulWhoaGKiopSQkKCpNtXlEnSX3/95XCbBw4c0L59+xQeHq74+HhbAObh4aErV64oOjpaxYoVkyQFBgbq+++/19SpU2UYhh5++GEVKlQo3W1v2fL/2rubkCq3PY7j37YEJZudZFaWoqVZQiUqCGGTChPBijKCCKKBQQUGEQ2iYfQyCKGBE4sQCoRGloMwsrIXiPIlIotAYZdo4EuabkVM8wzi7Hu9es7txN23gu9nuNbzX2s97NmP9ex/EwAlJSVzzi9btoy0tDQ6OjoIh8OsWrXqh99DkiTp/8ngTZIk6TczODgIEA26AO7fv8/p06dn3IL7T5FI5B/vNTAwQEVFBS0tLX/7XCQSiZ5n165dPH36lPr6eioqKoiLiyMjI4P8/HyKi4tn3Xjr6uoC4MCBA//1PJ8+fTJ4kyRJvw2DN0mSpN/I9PQ0b9++BSArKwv41g30xIkTjI+PU15ezo4dO0hJSSE+Pp5AIMCTJ0/m7CL6Pc6cOUNLSwu5ublUVFSwbt06QqFQtMnB5s2b6evrm/EJaCAQ4NKlSxw5coSHDx/S2tpKa2srtbW11NbWsmXLFqqqqoiLiwP+dYOvuLiY+Pj4vz1PQkLCD72HJEnSz2DwJkmS9Btpamri8+fPwLfQC77ddhsfH6eoqIhTp07Nqnn//v0P7TU2NsajR48IBAJUV1cTCoVmzff39/9lfWZmJpmZmcC3wPDZs2ecPHmSBw8eUFdXR1lZGQDJycmEw2EOHz7Mhg0bfuiskiRJvyKbK0iSJP0mRkZGuHDhAgCFhYVkZ2cDRIO4FStWzKqZnp6mvr5+zvX+vLU2OTn5l/tNTU0RDAZnhW4At2/f/u5mB/PmzWPTpk2UlpYCRG/twbcmDAB37tz5rrW+9/ySJEk/m8GbJEnSL256epqmpib27t1LOBwmKSmJs2fPRuczMjIAaGhooLe3Nzo+NTXF5cuXaWtrm3PdxYsXM3/+fPr7++f8b7glS5awaNEihoeHqaurmzH38uVLKisr51y3rq6O169fzxqPRCI8f/4cgJUrV0bHy8vLCYVC1NTUcO3aNSYmJmbVdnV1cevWrRljy5cvB6Cjo2POc0iSJP1s86btyS5JkvTTbd26le7ubvLy8khLSwNgYmKCwcFB3rx5Ew3GCgoKOH/+PKmpqdHayclJ9u3bR3t7O/Hx8RQUFLBw4UJevXpFb28vhw4d4sqVKxQUFHD9+vUZ+x4/fpyGhgaSk5PJz89nwYIFAJw7dw6Ampqa6C27nJwcUlNT6enpoa2tjZ07d9Lc3Ex3dzeNjY2kpKQAcOzYMRobG1m6dCnZ2dmEQiGGh4dpbW1lZGSErKwsamtrCQaD0XO8ePGCiooKBgcHSUxMZM2aNSQlJRGJROjs7OTDhw/k5ORw8+bNaE1/fz9FRUWMjY2Rl5dHeno6gUCAvLy86GeskiRJP5PBmyRJ0i/gz+Dt38XHxxMMBklPT2f9+vWUlJSwcePGOetHR0eprq6moaGBnp4egsEgubm5HD16lNHRUQ4ePDhn8DY0NERlZSWPHz+mr6+PL1++APDu3bvoM/fu3ePq1at0dnYyOTnJ6tWrKSsrY//+/Wzbtm1W8Nbc3Mzdu3dpa2vj48ePDA0NkZCQQEpKCqWlpezZs2fOJgoDAwPcuHGDpqYmwuEwExMTnkI+WwAAAL9JREFUJCYmkpycTGFhIdu3b2ft2rUzapqbm6mqqqK9vZ2RkRG+fv3K7t27uXjx4j//ESRJkv7HDN4kSZIkSZKkGPA/3iRJkiRJkqQYMHiTJEmSJEmSYsDgTZIkSZIkSYoBgzdJkiRJkiQpBgzeJEmSJEmSpBgweJMkSZIkSZJiwOBNkiRJkiRJigGDN0mSJEmSJCkGDN4kSZIkSZKkGDB4kyRJkiRJkmLA4E2SJEmSJEmKAYM3SZIkSZIkKQb+AIQhcpxZeprNAAAAAElFTkSuQmCC" + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "execution_count": 5 + }, + { + "cell_type": "code", + "metadata": { + "ExecuteTime": { + "end_time": "2024-06-24T09:34:50.697162Z", + "start_time": "2024-06-24T09:34:49.997808Z" + } + }, + "source": [ + "# Grouped barplot of SHD by Dataset\n", + "plt.figure(figsize=(15, 9))\n", + "sns.barplot(x='dataset', y='shd', hue='algorithm', data=df.melt(id_vars=['dataset', 'data_type'], value_vars=['shd_bbn', 'shd_lsevo', 'shd_bidag', 'shd_sparsebn', 'shd_dagma'], var_name='algorithm', value_name='shd'))\n", + "plt.title('SHD Comparison by Dataset')\n", + "plt.xlabel('Dataset')\n", + "plt.ylabel('SHD')\n", + "plt.show()\n", + "\n", + "# Grouped barplot of F1 Score by Dataset\n", + "plt.figure(figsize=(15, 9))\n", + "sns.barplot(x='dataset', y='f1', hue='algorithm', data=df.melt(id_vars=['dataset', 'data_type'], value_vars=['f1_bbn', 'f1_lsevo', 'f1_bidag', 'f1_sparsebn', 'f1_dagma'], var_name='algorithm', value_name='f1'))\n", + "plt.title('F1 Score Comparison by Dataset')\n", + "plt.xlabel('Dataset')\n", + "plt.ylabel('F1 Score')\n", + "plt.show()\n", + "\n", + "plt.figure(figsize=(15, 9))\n", + "sns.barplot(x='dataset', y='f1_undirected', hue='algorithm', data=df.melt(id_vars=['dataset', 'data_type'], value_vars=['f1_undir_bbn', 'f1_undir_lsevo', 'f1_undir_bidag', 'f1_undir_sparsebn', 'f1_undir_dagma'], var_name='algorithm', value_name='f1_undirected'))\n", + "plt.title('F1 Undirected Score Comparison by Dataset')\n", + "plt.xlabel('Dataset')\n", + "plt.ylabel('F1 Undirected Score')\n", + "plt.show()" + ], + "outputs": [ + { + "data": { + "text/plain": [ + "
    " + ], + "image/png": "iVBORw0KGgoAAAANSUhEUgAABPoAAAMzCAYAAAA/OnxeAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAADyM0lEQVR4nOzdd1iV5ePH8c85oAiIioiCuDfundvMTVpmmmVZOXKkld922dCGLTOzcmRmaa4cqLkzt6CmgnvgwIEooCCCCMI5vz/4cfJ4DgqKoof367q4Lnju5x7P83D8xud7P/dtMJvNZgEAAAAAAAB4oBlzewAAAAAAAAAA7hxBHwAAAAAAAOAACPoAAAAAAAAAB0DQBwAAAAAAADgAgj4AAAAAAADAARD0AQAAAAAAAA6AoA8AAAAAAABwAAR9AAAAAAAAgAMg6AMAAAAAAAAcgHNuDwAAANy+U6dOae7cudq2bZtOnjypK1euyMPDQ15eXvLx8VGtWrXUrFkzNWjQQE5OTlZ1f/jhB/3444+SpH/++UelSpW6aV8LFy7Ue++9J0maPn26HnrooUzbu16BAgXk4eGhIkWKqGrVqqpVq5Y6d+6sEiVK3Mml23X69GktWbJEwcHBOn36tC5evChnZ2d5enrK399fzZo1U0BAgDw9PXO8b9iX8Xvh5+entWvX5vZw7qk+ffpo+/bteuKJJ/Tll1/m9nDsuv5zfb38+fPLw8NDhQoVUpUqVVSzZk116NBB5cqVu/eDBAAAWUbQBwDAA2r69On6+uuvde3aNavjsbGxio2N1dGjR7V582ZNnDhR8+fPV61atXJlnFevXtXVq1cVHR2tsLAwLV26VN98843atWunDz74QN7e3nfcR3Jysr788kvNmzfP5n6kpKToypUrioiI0Jo1a/T111/rhRde0Ouvv37H/QKOKiUlRRcuXNCFCxd04sQJrVq1SmPHjlWzZs300Ucf3dXA70EISO9UXrhGAEDuIOgDAOAB9Ndff+nzzz+XJPn6+ur555/XQw89JB8fH5lMJp05c0YhISFas2aNdu7cec/Ht2zZMvn6+kqS0tLSFB8fr7Nnz2rXrl0KDAxUeHi4Vq5cqW3btmny5MmqU6fObfd16dIlvfTSS9q9e7ckqUKFCnrmmWfUuHFjeXt7Ky0tTefPn1dwcLAWLVqkY8eOafLkyQR9wA1+/vlnNWzYUJJkNpsVHx+v8+fPa/fu3Vq8eLEOHDigLVu26IknntCYMWPUtm3bXB4xAAC4EUEfAAAPoO+++06S5Ofnp8DAQBUuXNiq3NvbW/Xq1VO/fv0UFhamokWL3tPxFShQQO7u7pafCxUqpFKlSqlx48YaOHCgpk2bpm+//VaxsbF6+eWXtWDBAvn4+GS7H7PZrLffftsS8vXv319vvPGGzWvKxYsXV61atTRgwAAtWLBAo0ePvrMLRJa98soreuWVV3J7GMiCGz+3BQsWVMmSJVWvXj29+OKLWrx4sT7++GNduXJFb775pmbPnq1q1arl4ogBAMCN2IwDAIAHTHh4uCIiIiRJTz31lE3Id6PKlSvLy8vrXgwtS4xGoyWQk6SYmBi7a/tlxZIlS7R+/XpJ0pNPPqm3337bJuS7se+ePXvqzz//vK3+gLzs8ccf1xdffCFJunLlir766qtcHhEAALgRM/oAAHjAXLx40fL99bNvHjT9+vXTn3/+qfDwcC1atEivv/56tmce/vLLL5IkNzc3vfvuu1muV7ly5UzLNmzYoAULFigkJESxsbFyc3NThQoV1L59e/Xu3Vuurq5269245tbWrVv122+/ae/evUpISFCpUqXUrVs3vfDCC8qfP78k6fLly/rjjz+0fPlynTlzRs7Ozqpbt66GDRuW6evMN/azfv16zZgxQwcPHlRCQoJ8fX3Vtm1bDRo0KNMQODk5WVu3btXatWu1a9cuRUREKDk5WYUKFVLVqlUVEBCgbt26WcZ5o3fffVeBgYFq3LixZsyYod27d2v69OnauXOnYmJiVLx4ccvGG7fajOPKlSuaOXOm/vnnHx0/flyJiYny8PBQ0aJFValSJbVo0UJdunSRm5ubTd1r165p/vz5WrFihY4cOaKEhAQVLlxYNWvW1OOPP67OnTvLYDDYvYaqVatKkr744gt1795dgYGB+vPPP3X06FGlpKSobNmy6tq1q9XzulPZeVb//vuvnnvuOUnSlClT1KpVq0zbTUpKUvPmzZWYmKj+/fvr7bffzpHx2tO5c2fNmzdPW7ZsUVBQkA4dOmQzqy82NlYbNmzQ+vXrtX//fkVFRclkMsnLy0u1a9fWU089pRYtWti0feOGPoGBgQoMDLQ658Y17U6dOqW1a9dq06ZNOnLkiGJjY5UvXz6VKFFCjRs31gsvvKCKFStmej0Zv0PLly9XWFiYLl++LHd3d3l6eqp8+fJq1qyZunTpkum/TQcPHtSsWbO0fft2RUVFyWw2q2TJkmrZsqX69etns+nQ7VwjAADZQdAHAMAD5vpAICgoSH369MnF0dw+g8GgJ598Ut9++62uXbum7du3q1OnTlmuf/z4cR05ckSS1KlTJxUqVOiOxpOSkqL33ntPS5cutTp+6dIlhYSEKCQkRDNnztSUKVNuGhxI6WudjR07Vmaz2XLs6NGjGjNmjP79919NmDBB58+f14ABA3T8+HGruhs3blRwcLCmTJmipk2b3rSf8ePH66effrI6Fh4erqlTp2rp0qX6/fffVb58eZt63377rX7//Xeb4xcvXlRwcLCCg4O1YMECTZky5Zb3dc6cOfrkk0+UlpZ20/PsiYqKUp8+fRQeHm51PGNDmWPHjmnVqlXy9/e32Uzm/Pnzeumll3T48GGr4zExMVq/fr3Wr1+vBQsW6Pvvv1fBggUzHUNaWppeffVVrVq1yur44cOHdfjwYW3dulVTpkyR0XhnL8Jk91k1atRI5cqVU3h4uBYsWHDToG/VqlVKTEyUlD6z9W7r0aOHtmzZIknasmWLTdDXt29fHTx40KZeZGSkIiMjtWrVKj399NMaNWrUHY3j8uXLat++vc3xa9eu6cSJEzpx4oQWLlyozz77TN26dbM5LzExUf369VNoaKjV8UuXLunSpUsKDw/XunXrVLx4cZt/m8xms8aMGaOpU6dafc4l6dixYzp27Jj+/PNPjRs3Tq1bt76j6wQAIDsI+gAAeMBUqFBBJUqU0Pnz57V27Vp99NFHevHFF1WhQoXcHlq21a9f3/J9aGhotoK+6zcZadSo0R2P5fPPP7eEfK1atdKgQYNUsWJFxcXFadmyZZo0aZIiIiLUv39/LVmyJNMAbPv27Vq0aJE6dOigfv36qVy5crpw4YKmTp2qBQsWWGYMzps3T/Hx8frkk0/UsmVLFShQQDt27NAnn3yi6OhojRgxQqtXr5azs/3/XNu+fbsiIiLUtGlTDR06VJUqVVJsbKwWL16sX375RefPn9fgwYO1ePFiFShQwKquh4eHevTooebNm6t06dLy9vZWvnz5dO7cOf3zzz/6448/FBoaqpEjR2rs2LGZ3rPjx4/r008/Ve3atfXyyy+rRo0aSklJsRvy2PPNN98oPDxcTk5OGjRokDp27KjixYtLSg/yQkJCtGLFCptZeSkpKRo4cKAOHz4so9GoPn36qEePHipevLhOnTql6dOn66+//tLmzZv15ptvatKkSZmOYfLkyTp79qxeeuklPfbYYypRooQiIyP1448/6u+//9bmzZs1b9489erVK0vXZM/tPqsePXpozJgxWrt2reLi4lSkSBG77S9YsECSVK9evVuG0Dnhxs/tjUqWLKkWLVqoYcOG8vHxUfHixZWcnKwzZ85o0aJFWrhwoebMmSN/f389/fTTlnqDBg1Sv3799NJLL2nnzp3q2rWrTRiYL18+q59r166tDh06qFatWvL29lbRokUVHx+vsLAw/fHHHwoODtYHH3wgf39/yyzODFOmTLGM/9lnn1W3bt3k6+urfPnyKTo6Wnv37tXq1avtLgfw7bff6pdffpHBYFC3bt3Uo0cPy73fu3evfvrpJ+3evVuvvfaa5s2bZ5lFfDvXCABAdhD0AQDwgDEYDHr77bcta9zNnTtXc+fOlY+Pj2rVqqUaNWqoYcOGqlu3bpb/YLx69aplRlBmUlJS7njsNypXrpzl++jo6GzVPXPmjOX7Ow03Dh06pDlz5kiSOnbsqO+//94SLnl6emrYsGGqUqWKXnnlFUVGRmrixIl655137LYVERGhp556Sp9++qnlWJEiRTR69GidOHFCu3bt0ieffCIXFxctXLjQ6h506NBBbm5u6t+/vyIiIrR161a7rzhm9NO8eXP9/PPPljDQ09NT//vf/1SqVCl98MEHCg8P1x9//KEBAwZY1c1scwwvLy/VqFFDHTt21BNPPKHly5frf//7n0qXLm33/JiYGDVo0EC//fab1eutGTsu38rGjRslpb+O/Nprr1mVFS1aVP7+/urdu7dNvdmzZ+vQoUOSpPfee0/PP/+8paxIkSIaM2aMihQpohkzZmjdunVat26d2rRpY3cMp0+f1jfffKPHHnvMcqxw4cIaP368unfvroMHD2rBggV3FPTd7rN64okn9P333yslJUV//fWX3dm7p0+f1r///ivp3szmkyQfHx+5uLgoOTnZ7ud2woQJduv5+vqqUaNGqlGjhkaNGqWff/5ZvXr1snzW8ufPr/z581uCNWdn55suT+Dh4aF58+bZHPf09FTZsmXVrl07/e9//9Py5cv166+/2qwpmPH71759e3300UdWZUWKFFHlypXVvXt3m/b3799vWTbgk08+0VNPPWVV3rp1azVr1kwvvPCCdu7cqW+//dYSNmf3GgEAyC424wAA4AHUpUsXjR8/3mr9p3Pnzunvv//WuHHj9Nxzz6lly5YaN26crly5csv2Hn30UdWvX/+mXx9//HGOX8f1s+IuXbqUrbpxcXGW7z08PO5oHBlhgbOzsz744AO767p16NBBLVu2lJQ+g8pkMtlty9XVVW+99ZbdskcffVSSlJqaqj59+liFfBmaN29umbmVsZtwZt5//327M/569uypGjVqWMaaXVWrVlX16tVlNpsVFBR003Pfeeed217DLjU1VZIss/iyav78+ZKkKlWqZPrq+ptvvmm5jzfbfKVevXpWIV8Go9Foed3zwIEDlrHertt5VsWKFbMElJk9x4ULF8psNsvNzU0BAQF3NMbsyPjsZvdzK6UHmFJ6AHrja9s5LeMZZrxqfL3b/f2bMWOGzGaz6tevbxPyZciXL5+GDx8uKX3dz/j4+Gz1AQDA7WJGHwAAD6iOHTuqTZs2Wr9+vdatW6ddu3bp5MmTlvWiYmNjNXHiRK1Zs0bTp0/P9kYX98L1a1tltmnCvbBjxw5JUoMGDW76R39AQIA2bdqkS5cu6ciRIzZrk0lSnTp1Mn2t9/qZcRmh4Y0MBoPKlCmjuLi4m85yLF++vCpVqpRpeYcOHbR//34dP35csbGx8vT0tCqPi4vTn3/+qU2bNunYsWOKj4/XtWvXbNo5ceJEpn0UKVIk001DssLf31///vuvpk6dqooVK6ply5Y33TVZSg+WwsLCJKV/BjL7vSlQoIDatGmjwMBA7dq1K9P2MnsOkixr5l27dk3x8fG3/Rm6k2fVo0cPrV69WgcPHtSBAwdUvXp1S5nJZNKiRYskpa9TeS9nhmV8djO7/8ePH9fcuXP177//6vTp00pMTLS7juOJEyfsriOZHcHBwVq0aJF2796t8+fPKykpyWbdvOjoaCUkJFit1+jv76/Dhw9r4cKFqlu3rjp16pSl0Doj/M7YACUzGc/cZDJp//79t1xzEwCAnEDQBwDAAyx//vzq0KGDOnToIElKSEhQSEiIVq5cqcWLF+vatWsKCwvThx9+aLMRwPX++ecflSpV6qZ9LVy4UO+9916Ojv/y5cuW7zPbITYz169Xdn07t+Ps2bOSdNMw5sbyiIgIu0HfzYLC69dfy8p5ycnJmZ5zq9eVrx/r2bNnrcKj0NBQDRkyxGoH58zc7N5m9kpvVr3xxhvq06ePLly4oEGDBqlIkSJq1KiRGjRooKZNm9q9v5GRkZYQJ6vPKy4uzibkyZDV55WUlJSla7LnTp5Vy5Yt5evrq8jISC1YsMAq6AsODrb87t6r13YzZPxe2Pvczpo1S6NHj7YbHGfWzu1IS0vTiBEjbHatzcyNvwPDhg3TmjVrlJCQoLfeeksjR45UgwYN1KBBAzVp0kR16tSxCTITExN1/vx5Sek76P7www9Z6jsrnzUAAHICQR8AAA6kYMGCatmypVq2bKlevXrp2WefVUpKitasWaPIyMgsr512r1w/Wyy7r89dH0weP378jmaWZczKudWMqOvLM5vJc6sZaRmysovrjbOSrufm5nbTuteXXz/WhIQEDR06VBcvXlTRokXVt29fNW7cWL6+vnJzc7OMa8CAAdq1a9dNd9N1dXW95TXcTL169fTnn3/qp59+0oYNGxQXF6e///5bf//9tySpcuXKevPNN/Xwww9bjT9Ddp+XvaAvq8/rZs/iVm73WUnpvyfdu3fXTz/9pKVLl1q9Kp3xOm/58uXVsGHD2x5fdp09e9YSQt/4uQ0NDdUnn3wis9msqlWrqk+fPpaNMlxcXGQwGGQ2m9WgQQNJuq3dmjNMnTrVEvK1adNG3bt3V6VKleTp6Wm5Rzt27NDAgQMlyeb169KlSyswMFA//vijVq9ercTERG3cuNGydp+fn59eeeUVy6vGkvXvX3bcLLQHACAnEfQBAOCgateurZ49e2rmzJmS0heQv9+CvpCQEMv31+/kmRUZQYGUvqvp9X+MZ5e7u7vi4+NvuZ7h9eW5vYD+7Y515cqViomJkdFo1PTp0y27gd7oVpuz5JTq1avrp59+0pUrV7R7926FhoZqy5Yt2rFjh8LCwjRo0CCNHz9eHTt2lCSrsO5BeV53Os4nn3xSEydOVFxcnNasWaOAgADFx8drzZo1kmR3w4i76frPbb169azKZs2aJbPZrFKlSmnu3Ll2w+DbWdfPnlmzZklKf6X+u+++s3vOrTYRKlOmjL7++mt99tln2rt3r0JDQxUcHKzg4GBFRETo3XffVVxcnPr27SvJOpT94IMPMl0jEgCA3MJmHAAAOLDrXwm8evVqLo7EltlstsxIyp8/f7ZnJFWoUMESUq1cufKOXgH08/OTJMvab5m5vjyjTm45duzYTcuPHj1q+b5kyZKW7zN2q61atWqmIV9KSspd3yThRm5ubmratKmGDBmiP/74Q4sWLbK8nn39a+e+vr6W1ymz+ryKFClidzbfvXK7zyqDn5+fmjVrJum/WXxLly5VcnKynJ2d7yjkvh0Zm6FI6evUXe/gwYOSpLZt22Y64/PIkSN3PIa4uDhFRkZK+m+TG3sOHz6cpfby58+vBg0aqH///vrll1/0999/q2zZspKkiRMnWmYeenh4WH4vT506dQdXAADA3UHQBwCAAzt37pzl++y+Gnu3TZ061RImde/e3WrNvawaMGCApPQZUV988UWW690YEGWEjLt27VJMTEym9VauXCkpfV2yKlWqZHe4OerEiRNWAdGNVq9eLSk9EL1+zbeMGU43e2Vy5cqVuf6qYbVq1Sy7yB4/ftxyvHDhwpaAMuMa7bl69arWrVsnKfuzRXPa7T6r6/Xs2VNS+kYQ586dswR+LVu2lLe3dw6POHMrVqywbEbRsmVLm7A44/crs12pJWnx4sU37SNjd+Kb/Y5eP1Mvs77S0tK0dOnSm/aVmZIlS1p21L106ZIuXLhgKWvRooUk6e+//77ljMHMZOUaAQC4HQR9AAA8YE6dOqWxY8cqNjb2puedPXtWf/75p6T01x3r1q17D0Z3ayaTSb/++qvGjh0rKT2AHDp06G219fjjj1t2TV2wYIG++eabm/7hbDKZtHDhQssf8Bl69OghKX131c8++8zuemxr1qzRhg0bJKW/SpmVNfbuttGjR9u93vnz52v//v2SbDdpyFjb8Pjx43Z31D1//rzGjBlzF0Zr7cqVK5YZWZnJmDF1YwicEXodPnzY8mr6jcaOHau4uDhJUq9eve5ssDngdp7V9dq2bauiRYvKZDLp66+/1r59+yT997t7LyxZssSyIY+bm5vefvttm3MyNmjZtGmT3RAsODhYCxcuvGk/GWFnVFRUpud4eXlZXqP9559/7J4zYcKEm+4afauZlhm/f05OTvLw8LAcf/HFFyWlbwzz+eef3zTUzKyfrFwjAAC3gzX6AAB4wFy9elWTJ0/WtGnT1LZtW7Vu3Vo1a9aUl5eXjEajIiMjtWnTJv3666+WMHDYsGGWxenv1Rgz1ngzmUyKj4/X2bNntWvXLi1cuNAyk69o0aKaMGHCbc82NBgM+uabbzRgwADt27dPv/zyi9atW6enn35ajRs3VvHixZWWlqbz589r69atWrx4sd3XBqtVq6ann35ac+bM0YoVK5SUlKSBAweqYsWKiouL07JlyzRp0iRJ6a+ODhky5PZuTA7y8/PTli1b1L9/fw0bNswy1sWLF2vKlCmSpLJly+q5556zqtexY0eNGzdOqampGjRokN566y3VqVNHJpNJQUFB+v777xUfHy8/Pz9FRETctfFfvHhRHTt2VKtWrdSuXTvVrFlT3t7eSktL0+nTpzVnzhxt3rxZktSlSxeruk8//bQWLFigQ4cO6bPPPtPp06f15JNPytvbW2fOnNH06dMts8batGljtZlHbrjdZ3W9fPnyqVu3bvr111+1bNkySVKxYsVy9Nqu/9yazWZdvnxZ58+fV2hoqJYsWWIJJN3c3DRu3Di7s1oDAgK0adMmhYeHa9CgQRo2bJjKly+vuLg4LV++XFOmTFH58uVvOsOxZs2aWrZsmXbu3Kk1a9aoSZMmlh2QjUajjEajnJyc1LFjRwUGBmrRokXy8PDQU089JW9vb50+fVqzZs1SYGCgKlWqlGlfjz76qJo0aaL27durTp06ltfCIyMjtWTJEsv/UXLja8i1atXS4MGDNWnSJM2ZM0eHDx/W888/r9q1a6tQoUJKSkrS6dOntWvXLq1cuVIpKSk2Mwuzco0AANwOgj4AAB4w+fPnV758+ZSSkqIVK1ZoxYoVmZ7r7OyswYMHWxaSv1dutmaWlD6uDh06aMSIESpWrNgd9eXp6ak//vhDX3zxhRYsWKBjx47p888/z/R8Nzc39e/f3+b4iBEjlJCQoKVLl2r9+vVav369zTl+fn6aMmWKChUqdEdjzgmNGzeWj4+PJk6cqODgYJvyEiVKaPLkyZbwIEPZsmX1+uuv65tvvtHJkyc1bNgwq3IXFxd98803+uOPP+5q0Cel74K6du1arV27NtNzmjVrpldffdXqWP78+fXzzz/rpZde0uHDhzVt2jRNmzbNpm7z5s3vyezEW7ndZ3WjHj166Ndff7X8/Nhjj1leAc0JGbvTZsZgMKh58+b66KOPLOvX3ahbt25avXq11q1bp6CgIMtrvhl8fHz0ww8/qHPnzpn2061bN02ZMkUXL160me37xBNP6Msvv5Qkvfnmm/r333915swZzZgxQzNmzLA6t1GjRnrppZcyvS6z2WzZeCMz/v7++vjjj22ODx8+XAUKFNAPP/ygkJAQqw1KblS9evXbvkYAALKLoA8AgAdMuXLltHXrVm3atEn//vuvDhw4oFOnTik+Pl6SVKhQIZUrV06NGjXSE088oXLlyuXqeF1cXFSwYEF5enqqWrVqql27tjp16qQSJUrkWB+urq765JNPNGDAAC1ZskTBwcE6deqUYmNj5ezsrKJFi8rf318tW7ZUQECA3aAuf/78+vbbb/XYY49p/vz5Cg0NVWxsrFxdXVWxYkW1a9dOzz77bKYbDOSG4cOHq1atWpo5c6YOHDigxMRE+fr6ql27dho0aJAKFy5st96AAQNUsWJF/fbbb9q3b59SUlLk7e2tJk2a6MUXX1SVKlX0xx9/3NWxlyxZUrNnz1ZQUJB27NihiIgIxcTE6Nq1a/Ly8lL16tXVtWtXde7c2bL5xvVKlCihBQsWaP78+Vq+fLmOHDmixMREFS5cWDVq1NDjjz+ugIAAu3Vzw+0+q+tVrFhRDRo00M6dOyXd3dd28+XLJw8PD8t6lDVr1lTHjh0zDfgyGI1G/fTTT5oxY4YWLVqk48ePy8nJSSVLllTbtm3Vt2/fTNchzFC0aFHNnTtXEydO1Pbt2xUVFWX3NeBixYpp/vz5mjRpktasWaPz58/L3d1d5cuXV5cuXfTMM89ox44dmfazcOFCBQcHa9u2bTp16pRiYmKUnJyswoULq1q1aurUqZO6deumfPny2dQ1GAwaMmSIunTpotmzZ2vr1q06ffq0EhMT5erqqpIlS6p69epq0aKF2rZte9vXCABAdhnM9hahAQAAwH2pT58+2r59O7N+8qgBAwZo06ZNqlevnubMmZPbwwEAAPcZFn8AAAAAHgDnz5+3vAp7s407AABA3kXQBwAAADwApk+frrS0NHl4eNxyHUwAAJA3sUYfAAAAcB+7evWq1q5dq+nTp0uSnn32Wbm5ueXyqAAAwP2IoA8AAAC4D505c8ZmI4fSpUvfcndcAACQd/HqLgAAAHCf8/b21uOPP64//vhD7u7uuT0cAABwn2LXXQAAAAAAAMABMKMPAAAAAAAAcAAEfQAAAAAAAIADYDOO+9iFC5fFi9UAAAAAAAB5m8EgeXl53PI8gr77mNksgj4AAAAAAABkCa/uAgAAAAAAAA6AoA8AAAAAAABwAAR9AAAAAAAAgAMg6AMAAAAAAAAcAEEfAAAAAAAA4AAI+gAAAAAAAAAHQNAHAAAAAAAAOADn3B4AcpbZbFZaWqrMZnNuDwXI04xGo5yc+CcWAAAAAHDv8Feog0hNvabLl+OUknJVZrMpt4cDQJKzc365uxeSq6t7bg8FAAAAAJAHEPQ5gJSUZMXGRsloNMrd3UP58rnIaDRKMuT20IA8yqy0tDRduZKgS5diJImwDwAAAABw1xH0OYCEhDg5OTmraNES/x/wAcht+fJJLi6uio2NVmJiPEEfAAAAAOCuIxV6wKWlpSkl5arc3T0I+YD7jMFgkJubu1JTU5SWlprbwwEAAAAAODiSoQecyZQmSXJ2zpfLIwFgT8aGHCYTa2cCAAAAAO4ugj6HwXp8wP2JzyYAAAAA4N4g6AMAAAAAAAAcAEEfAAAAAAAA4AAI+gAAAAAAAAAHQNCHPGnXrh1q0aKhhg0bmNtDyZIWLRqqRYuG2a4XGXlWLVo0VI8eXe/CqAAAAAAAwP2EoA94gA0bNlAtWjTUrl07cnsoAAAAAAAglznn9gAA3NrMmfNzewgAAAAAAOA+R9AHPADKli2X20MAAAAAAAD3OYI+OIQDB/Zp/fp/FBKyU+fPn1d8/CV5eBSSv38N9ez5tBo1eihb7e3eHarff5+qAwf2Ki0tTWXLlteTTz6lzp27WNbK27zZ9nXZqKjzmjnzd23bFqyoqPPKly+fKlSoqI4dH1XXrt3k5ORkdf7y5X9p9OhR6ty5i1555X+aNu0XbdmyUdHRUapRo5Z+/PFnSbLpc9euHXr11cGWdq7/XpLef/9jBQRYr8tnNpu1ZEmgFi9eqFOnwuXk5KTq1Wuqf/9Bqlmzts21XN/nqlXLNX/+HIWHn5CLi4saNGisIUNelY+Pj8xmsxYu/FN//bVYZ86ckouLi5o1a6mXX35Vnp5Fs3XfAQAAAADA7SPog0OYPHmCQkJ2qHz5CqpatZoKFHBVRMQZBQVtUlDQJr366ht66qlnstTWmjWr9MknH8pkMqlixUoqX76iYmKi9cUXnyg8/ESm9Q4e3K833nhV8fGXVKKEj1q2bK2EhESFhOzU3r17tHHjen311Vjly5fPpu6lS3Hq3/95JSRcVp06dVW1qr/d8zJ4eRVT585dtG1bsC5evKDGjZvKy8vLUu7nV9qmzujRo/T33ytVp049NWvWUmFhh/Xvv9u0e3eIfvjhZ9WoUdNuX5Mm/ajZs2eobt36euihZjp4cL/++We19u7drd9+m60xY77Q5s0bVa9eA5Us6ae9e3drxYqlOnLksH75ZfpNrwMAAAAAAOQcgj44hKefflYffviJihUrZnV83749euONVzRhwvdq06atvL2L37SdmJhoffXV5zKZTHrttTfVs+fTlrLQ0F16663X7NZLSUnRhx++q/j4S+rW7UkNH/6WnJ3TP14REWc0fPjL2r49WL/++rMGDRpqUz8oaLMaNGis0aO/lrt7wVteb9my5TRixEgNGzZQFy9e0HPPvaD69TPflffcuUiFhOzU9OlzVaZMWUlSWlqavv76cy1btkRTp07S2LE/2q3711+B+uWXGapcuYokKTn5qv73v2HasydUr7wyUFevXtWsWfPl4+MrSYqLi9PgwX117FiY1q1bow4dOt/yegAAAAAAwJ1j1104hKZNm9uEfJJUs2Ztde/+lFJTU7Vp04ZbtrN06WIlJV1RzZq1rUI+Sapbt766detht966dWt07lykihXz1quvvmEJ+STJz6+Uhg5NDwgXLPhTycnJNvWdnZ319tvvZynku13Dh79lCfkkycnJSQMHviwpPcRMTU21W69//8GWkE+SXFwKqFevZyVJx44d1fDhb1pCPkkqUqSIunV7UpK0Y8f2HL8OAAAAAABgHzP64DAuXYpTUNBmnThxTJcvX7YEV2fOnJIknTp18pZthITskiS1b9/JbnmHDp00e/YMO/V2SpLatu2g/Pnz25S3bv2IPDwK6fLleB0+fFC1a9e1Kq9cuar8/Erdcny3y8nJSU2aNLM57uVVzDKuS5fi5OVlG5Y2bdrc5ljp0qUt7TZq1MSmvFSpMpKkmJiYOx06AAAAAADIIoI+OIQlSwL1ww9jlZSUlOk5V64k3rKd6OjzkiRf35J2y3187B+Pjo6WJJUsab/cYDDI17ekLl+Ot5x7vcz6yyleXsWsZhlez93dXZcvxyslJcVueYkSPjbHXF3dbtqum1t6eUqK7exFAAAAAABwdxD04YF36NBBffPNaBmNRg0Z8oqaN2+lEiV8VKBAARkMBi1evFDffDNaZrM5y20aDJkdz6TgDrm4uNyVdjMYjbf/lv7N6t5JuwAAAAAAIGcR9OGBt27dGpnNZvXo0UvPPvuCTfmZM6ez3Ja3d3GdOnVSkZGRdssjI89mUs9bknT2bESmbWfUzTgXAAAAAAAgJzEdBw+8+Ph4SVKJEr42ZcnJyVq/fm2W26pTp54kac2aVXbL//57pd3j9eo1kCT988/fdjfb2LBhnS5fjpebm7uqVvXP8nhuJV++fJLSd9AFAAAAAADWjEaDnJ2NufJlNN6dtwJver33vEcgh5UrV06StHLlUqt1+JKTk/Xtt18qMjLzWXY36tLlcRUoUEB79oRqwYI/rcr27AlVYOB8u/XatGmnEiV8FBMTrR9++M5qB9uzZyP044/jJElPPvlUjr6m6+1dXJJ04sTxHGsTAAAAAABHYDQa5FnETZ6e7rnzVcTtnod9vLqLB15AwGOaN2+Ojhw5rJ49H1Pt2vXk5GTU7t2hSk5OVs+ez2jevNlZaqt48RJ666339fnnI/Xdd19ryZJAlS9fQTEx0dqzJ1S9ej2r2bNn2GxAkT9/fn322Vd6441XtWjRfG3dukU1atTUlStXtHPnDqWkJKtx46bq129gjl77ww+31fLlf2nixPHasWO7PD09ZTAY9Oijj6lWrTo52hcAAAAAAA8So9Ego5NRq2fuUuz5hHvat2eJgurwbH0ZjQaZTFnfM+BOEfThgefh4aFffpmhqVMna/v2YG3bFqRChQqrceOH1LfvQO3ZE5qt9jp2DFDx4iU0ffqvOnBgnyIiTqtMmXJ6++0RatToIc2ePUOFCxexqefvX0PTps3UzJm/a+vWIG3cuF758uVXlSpV1alTgLp06Zbpzre3q1mzFnrnnQ8UGDhfu3b9q6tXr0qSateuS9AHAAAAAICk2PMJio64lNvDuCcM5uxsRYp7Kibmsm71dK5dS9GFC5Hy8vJVvnz5783A8rAVK5bq889Hqnnzlvrqq+9yezh4APAZBQAAAIDc4exslKenu+aO3XjPgz5vv8Lq9XorxcYmKjXVdMftGQxSsWIetzyPNfqAG5w7d04XLsTYHN+zJ1Q//fS9pPTXhQEAAAAAAO4nvLoL3GDXrn/15ZefqlKlyipRwkdGo1ERERE6evSIJCkgoKtat26Ty6MEAAAAAACwRtAH3KBGjVoKCOiq3btDFBKyU0lJSfLw8FDDho316KOPqX37Trk9RAAAAAAAABsEfcANypYtp3ff/TC3hwEAAAAAAJAtrNEHAAAAAAAAOACCPgAAAAAAAMABEPQBAAAAAAAADoCgDwAAAAAAAHAABH0AAAAAAACAAyDoAwAAAAAAABwAQR8AAAAAAADgAAj6AAAAAAAAAAdA0AcAAAAAAAA4AOfcHgDuDaPRIKPRkNvDyBaTySyTyZzbwwAAAAAAAHggEPTlAUajQUWKuMnJ6cGawJmWZlJc3JV7HvYtX/6XRo8epc6du2jEiJE51m5k5Fn17PmYfHx8NX/+X/e8jRYtGkqSNm/ecVt9AwAAAACA+xtBXx5gNBrk5GTUB7M26UTUpdweTpaUL15Yn/VuKaPRwKw+AAAAAACALCDoy0NORF3SoYiLuT0MAAAAAAAA3AUP1rucAAAAAAAAAOxiRh/yjNOnT2nGjGkKCdmpmJhoOTs7q1ChwqpQoaIefritHn30MZs6SUlJ+v33qVq3bo2ios7Lw6OQGjduokGDhsrbu7jdfrZs2aTZs2fo8OFDMhoNqlixsp555jlVqlQlR68nNTVVc+fO1IoVy3T2bIRcXQuofv1GGjBgsMqWLXfTukuWBGrRogU6dSpczs7Oqlmztl588SXVrFnL5twePbrq3LlIzZu3RJGRZzVjxjQdPHhAKSkpKleunHr2fEadO3fJ0WsDAAAAAADZx4w+5AnHjx/VgAF9tHz5X8qXL5+aNWuhJk2ay9u7uEJDQzRv3hybOgkJCRo8uJ8WLVqgcuXKq0mTZjKbzVq5cpmGDOmvhIQEmzpz587UO+/8T6Ghu1SuXHk1bdpCKSkpeu+9NzV//twcvaaPP35PU6ZMVLFixdSyZWu5uxfUunVrNGDA89q3b0+m9X74Yay++Wa0ChQooBYtWqt48RLaujVIQ4cO0IYN6zKtt2zZEr322hDFx8froYeaqnLlKjpy5LA+/3yk/vxzVo5eGwAAAAAAyD5m9CFPmDNnphITE/XSS0P0wgv9rcqSk6/q4MEDNnU2bVqvxo2basKEKXJ3LyhJio+P12uvDVZY2BEFBs5Tnz59LecfPRqmCRPGy2g0atSo0WrTpp2lbPXqFfr0049y7HrOnYvU1atJ+uWXGapUqbIkKS0tTT/8MFbz58/VyJEjNGvWAuXPn9+m7qJFCzRu3AQ1aNDIcmzWrOmaMGG8vvhilGrXriNPz6I29f744zd9+eVYNW/e0nIsY4fiX3/9WY8/3l0uLgVy7BoBAAAAAED23Lcz+o4fP64ZM2bo3XffVdeuXVW9enVVrVpVEyZMuGXdoKAgvfTSS3rooYdUu3ZtderUSd99950SExNvWu/kyZN699131apVK9WsWVOtWrXSu+++q9OnT9+0XkJCgsaOHauOHTuqdu3aeuihhzRw4EAFBwdn65px98TGpm9C0rRpc5syF5cCqlu3vs1xV1dXvf/+x5aQT5IKFSqk5557UZK0Y8d2q/MXLJirtLQ0tWnT1irkk6QOHTqrRYtWd3oZVp5/vr8l5JMkJycnvfzya/L2Lq5z5yK1fv1au/Uef7y7VcgnSb17P69q1aorISFBf/21yG69J5/sZRXySVJAQFeVLVtOCQkJOnTo4J1dEAAAAAAAuCP3bdA3e/ZsffbZZwoMDNSRI0eUlpaWpXq//fab+vbtq02bNqly5cpq06aNEhISNGnSJD355JO6eNH+rrM7d+7U448/rsDAQBUqVEjt27dXoUKFFBgYqMcee0yhoaF26124cEFPPvmkJk+erMTERLVp00aVK1fWxo0b1bdvX82YMeN2bwFykL9/DUnSmDFfatu2YCUnJ9+yTtWq/ipWrJjN8bJly0uSoqOjrI6HhOyUJHXoEGC3vU6dcnYdO3vr4uXPn1+PPNLeajxZqSdJnToF3LTejSFfhszuBwAAAAAAuLfu21d3q1Spon79+ql69eqqXr26Jk+erMWLF9+0zoEDB/Tll1/KyclJEydOVOvWrSWlb6gwZMgQBQcHa+TIkRo/frxVvaSkJA0fPlxJSUkaNGiQXn/9dUvZ2LFjNXnyZA0fPlwrV65UgQLWryZ++OGHCg8PV9OmTTVx4kS5urpKkjZs2KAhQ4Zo9OjRatSokapVq5YTtwW3qXfv57VnT6h27NiuN954Rc7OzqpUqYrq1Kmndu06WILA65Uo4WO3LXd3d0lSSkqK1fGoqPSgy9e3pN16JUvaP347Chb0kIeHx037iY4+b7fc19fvpsczC+yyez8AAAAAAMC9dd/O6OvZs6feeecdde3aVRUrVpTReOuhTp48WWazWd27d7eEfFL6K5iff/65jEajVq1apWPHjlnVW7hwoaKiolSuXDkNHz7cqmz48OEqV66cIiMjtWjRIquyo0eP6p9//pGTk5M+//xzS8gnSa1bt9YTTzwhk8mkn3/+Ofs3ADmqQIECGjdugqZM+V0DBgxWgwaNdfr0Sc2dO1MvvfSCvv32K5s6Wfmdu5+Zzbdbz37FB/1+AAAAAADg6BzmL/eUlBRt2LBBktSli+2riX5+fqpfP30dtjVr1liVZfz86KOP2oQZRqNRAQHprzT+/fffVmUZP9evX19+frazpDLGsW7dOl27di3b14Sc5+9fQy++OEDffjtey5b9o08//VIuLi4KDJynXbt23FHb3t7ektI3yrAnMtL+8duRkHBZly9fvmk/xYsXz6Q8wu7xc+fOSpK8ve3XAwAAAAAA9zeHCfrCw8OVlJQkSapZs6bdczKOHzhgvcNqxs/ZrXfw4MGb1qtVq5Yk6cqVKzp58uQtrwH3lrOzs9q0aafGjZtKksLCDt9RexkbeqxevcJu+cqVy+6o/RutWmXb3rVr17R2bXoAXa9eg0zGsfymxzOrBwAAAAAA7m/37Rp92XXmzBlJ6buiFixY0O45vr6+VudK6TvmxsXFScp8DbWMehcvXtSVK1fk5uZm1U5G+Y0KFiyoggULKiEhQWfOnFGlSpWydU0GQ86cA2nhwnlq2LCRypQpZ3X8woUYHT6cHtj6+Nh/jlnVo0cvrVy5TOvWrVGbNu3UunUbS9maNau0adP6O2r/Rr/9NlX16zdUhQrpv1cmk0kTJ45XVNR5FS9eQq1bP2K33qJF89W8eUvVr9/Qcmzu3Jk6eHC/3Nzc1aXL4zk6TqQzGPi8AgAAAEBelBN/C2a1DYcJ+hITEyXJap28G2UEdAkJCTb1blY3o15G3YyfM+peX26vbkJCglWfWeXlZX+zhetdvXpVFy8a5eRkkLOz/QmaTk4P7sTNnBr7X38FauzYr1SypJ8qVKgod3d3xcXFKjQ0VMnJV9WwYSO1bv2wnJ2NMhrTPz0Gg/17ev2Yri/39/fXkCHD9OOP32vEiLdUo0ZN+fmV1pkzp3TgwH49/fSzmjNnpk297Mjo28fHR1Wr+qtfv+dUv35DFS5cWAcP7teZM2fk6uqqTz4ZLXd3+7/PTzzxpF57bYjq1q0nb+/iOnbsqI4dOyonJyd98MHHKlHC/qu7Tk5Gu+M2/P+/NkZj5r+DeZnJZJDRaJSnp7vNZj4AAAAAAMfm6el+T/tzmKDPEV24cPmWGypcu5Yik8mktDSzUlNNNz23fPHCOTi6uytjrGlpplteV1a89NLLCgrarAMH9mrfvr1KTEyQp2dRVa9eQwEBXdW+fSdJRqWmmmQypd90s9n+PU1L++/YjeVPP91HpUqV0axZMxQWdljHjx9XpUqV9NlnX6lqVX9L0He71/Rf3waNGvWFZs2arlWrlis0dJcKFHDVww8/ov79B6t8+QqZ9jFs2OsqVaqMFi9eqAMH9svZ2VkPPdRML77YX7Vq1cm0XmbPImPzDpPp1r+DeVFamlkmk0mxsYnKl4+1OgEAAADgXnFyMt7zoO1GsbGJVjnC7TIYsjYhzGGCPnf39AeXsU6fPVeuXJEkq1d7M+rdrG5GvczqXl+elT6zymy+9c6pWdlZ1WQyKy3NpM96t8z2GHJTWtp/odudatashZo1a5GlcwMCuiogoGum5b6+JbV5c+Ybd7Ro0VotWrS2W3azellxY9/PP99Pzz/fL0t1r6/XrVsPdevWI0v15s//66blI0aM1IgRI7PUVl6Wlc8zAAAAAMDx3Mu/BR0m6MvY9TY+Pl4JCQl2g7WM3Uiv3yG3YMGCKlKkiOLi4nT27FlVq1Yt03qenp5Wr+n6+flp//79me6mev0ru/Z25b1XTCaz4uKuWF5JfVCYTOYcC/oAAAAAAAAcncMsqFW+fHnLGnv79u2ze07G8Ro1algdr169+l2pt3fvXknp6/SVK1fuVpdwV2W8VvkgfRHyAQAAAAAAZJ3DzOjLnz+/WrdurZUrV2rp0qVq0qSJVXlERIRCQkIkSe3atbMqa9eunYKCgrRs2TINGzZMRuN/+afJZNLy5cslSe3bt7epN27cOO3atUtnz5612bV36dKlkqQ2bdooX758OXOhcBi7d4dq6dJFWT5/6NDhKlKkyF0bDwAAAAAAeLA5TNAnSQMHDtSqVau0cOFCdejQQa1atZKUvvbeiBEjlJaWpo4dO6pixYpW9bp3765JkyYpPDxc33//vf73v/9Zyr7//nuFh4fLx8dH3bp1s6pXuXJltW3bVv/8849GjBihiRMnWnbV3LBhgwIDA2U0GjVw4MC7e+F4IEVEnNaKFUuzfH6/fgMJ+gAAAAAAQKYMZvP9uTz8/v37NWrUKMvPp06dUmxsrHx8fFSiRAnL8R9//FHFixe3/Pzbb7/piy++kMFgUKNGjeTl5aUdO3YoOjpa5cuX16xZs1S0aFGb/nbu3Kn+/fsrKSlJVapUUeXKlRUWFqYjR47Izc1N06ZNU926dW3qXbhwQb1791Z4eLi8vb3VsGFDXbhwQf/++6/MZrNGjBih559//rbuQUxM1nbdvXAhUl5evsqXL/9t9QPg7uEzCgAAAAC5w9k5fdfduWM3Kjri0j3t29uvsHq93kqxsYlKTc2ZXXeLFXuAd91NSEjQ7t27bY6fO3dO586ds/yckpJiVf7iiy+qSpUq+vXXX7V3715duXJFJUuWVPfu3TVw4MBMd79t0KCBFi9erAkTJigoKEirV6+Wp6enunXrpqFDh6pMmTJ263l5eWnBggWaPHmyVq9erX/++Udubm5q0aKF+vfvr6ZNm97BXQAAAAAAAACy5r6d0Qdm9AGOgM8oAAAAAOSOvDijz2F23QUAAAAAAADyMoI+AAAAAAAAwAEQ9AEAAAAAAAAOgKAPAAAAAAAAcAAEfQAAAAAAAIADIOgDAAAAAAAAHABBHwAAAAAAAOAAnHN7ALg3jEaDjEZDbg8jW0wms0wmc24PAwAAAAAA4IFA0JcHGI0GeRZxldHJKbeHki2mtDTFxiUR9gEAAAAAAGQBQV8eYDQaZHRyUszCd3Ut5nhuDydL8hWroGLdv5TRaLjnQd/y5X9p9OhR6ty5i0aMGJlj7UZGnlXPno/Jx8dX8+f/ddvttGjRUJK0efOOnBoaAAAAAABwAAR9eci1mOO6du5gbg8DAAAAAAAAdwGbcQAAAAAAAAAOgKAPAAAAAAAAcAC8uos84/TpU5oxY5pCQnYqJiZazs7OKlSosCpUqKiHH26rRx99zKZOUlKSfv99qtatW6OoqPPy8Cikxo2baNCgofL2Lm63ny1bNmn27Bk6fPiQjEaDKlasrGeeeU6VKlW5q9cXExOjP/74TVu3Bikq6pwMBoMKFy6i0qXL6KGHmql37z526kRr9uwZ2ro1SOfORcpodFLZsuXUufOjevzxJ+Xs/N8/EYMG9dX+/Xs1cuTnateuo90xLFgwV999941atnxYX3wxxqpszZpV+uuvRTpy5LCuXk1S0aJeatCgkZ577kWVKVM2Z28GAAAAAAB5EEEf8oTjx49qyJD+SkxMVJkyZdWsWQsZjU6Kjo5SaGiIoqOjbYK+hIQEDR7cT+fPn1OdOnVVoUJF7du3VytXLlNo6C799ttsFSxY0KrO3Lkz9cMP30mS/P1ryM+vlM6cOa333ntTvXo9e9eu78KFGA0Y0EcxMdEqUcJHDz3UVPnz51dMTIzCwo7o8OGDNkFfaOguvffem7p8OV6+viXVqNFDSkm5poMH9+u7777Rli2b9PXX4yxhX0BAV+3fv1fLly/NNOhbtix9k5Hr76XZbNbnn4/UypXL5OTkpLp166tIEU8dOXJIy5f/pbVr/9Znn32tJk2a3aW7AwAAAAD/MRoNMhoNudK3yWS+5xtOIm8h6EOeMGfOTCUmJuqll4bohRf6W5UlJ1/VwYMHbOps2rRejRs31YQJU+Tunh7oxcfH67XXBiss7IgCA+epT5++lvOPHg3ThAnjZTQaNWrUaLVp085Stnr1Cn366Ud35dokacmSQMXEROuxx57QW2+9L4Phv//RSk1NVWjoLqvzL1yI0YgRbykh4bLeeONdPf54dxmN6W/yX7oUp48+ek/bt2/VjBnT1LfvS5Kkdu066IcfxmrHjm2Kjo6ymdF49GiYjhw5pKJFvaxCu8WLF2jlymUqUqSIvvvuJ1WuXFVSegD4668/a9q0KRo5coRmz14oT0/Pu3J/AAAAAEBKD/mKFHGTk1PurGSWlmZSXNwVwj7cNQR9yBNiYy9Kkpo2bW5T5uJSQHXr1rc57urqqvff/9gS8klSoUKF9NxzL+rjj9/Xjh3brYK+BQvmKi0tTW3btrcK+SSpQ4fOWrdujTZt2pBTl2Tl4sX063vooWZWIZ8kOTs7q2HDxlbH/vxzti5duqQnn3xKTzzRw6qscOEi+uCDUerZ8zEtWPCnXnxxgAwGg9zdC6p160e0atVyrVy5zOraJWn58iWSpI4dA6xe+Z09+w9J0osvDrCEfJJkMBjUr99Abdy4XseOhemvvwL1/PP97vBOAAAAAEDmjEaDnJyM+mDWJp2IunRP+y5fvLA+691SRqOBoA93DUEf8gR//xoKDt6iMWO+VP/+g1S3bn25uLjctE7Vqv4qVqyYzfGyZctLkqKjo6yOh4TslCR16BBgt71OnbrctaCvevUaCgycp0mTfpBkVqNGTeTm5pbp+cHBmyVJjzzSwW65t3dxlSpVRuHhx3X69CnLGnqPPvqYVq1arhUrlloFfampqVq9eqXlnAxRUecVEXFGktS5cxebfgwGgx59tKvGjx+rXbt2EPQBAAAAuCdORF3SoYiLuT0MIMcR9CFP6N37ee3ZE6odO7brjTdekbOzsypVqqI6deqpXbsO8vevYVOnRAkfu225u7tLklJSUqyOR0WlB3++viXt1itZ0v7xnNCxY4D+/XebVq9eoREj3paTk5PKlSuvWrXqqk2btmrQoJHV+WfPRkiShg4dcMu24+JiLUFfvXoNVLKkn06dOqm9e3erVq06ktI3IImLi1X16jVVrlx5S93o6GhJUuHCha1mRl6vZMlSktI3BgEAAAAAALePoA95QoECBTRu3AQdPLhf27YFa+/ePdq3b7cOHTqguXNn6okneuqNN96xqpOxZt2DwGg06qOPPlWfPn0VHLxZe/fu1t69u7Vo0XwtWjRfzZu31OjRY+Tk5CRJlmniDz/cVq6urjdtu3DhIpbvDQaDAgK66pdfJmn58qWWoC/jtd2AgK534eoAAAAAAEBWEPQhT/H3r2GZvZeamqpNm9brs88+VmDgPLVp01b16ze87ba9vb0VEXFG585FqkKFijblkZGRt912VpUvX0Hly1eQlL7Zxc6d/2rUqA+0ZcsmrVy5zPJabfHiJXTmzCk999wLqlaterb66NSpi3799WetXbtaw4e/ocTERG3dGiQXFxeb3Xi9vb0lSZcuXVJiYoLdWX0ZswuLFfPO9vUCAAAAAID/PDhTloAc5uzsrDZt2qlx46aSpLCww3fUXsaGHqtXr7BbvnLlsjtqP7sMBoMaNmys9u3Tw7ewsCOWsoxdcdeu/Tvb7fr4+KhBg0ZKTEzUhg3rtGrVCqWlpal160dUsKB1kFe8eAn5+aW/mrt8+VKbtsxms1as+EuS7ihkBQAAAAAABH3IIxYunKdTp8Jtjl+4EKPDhw9Kknx8fO+ojx49esnJyUnr1q3Rhg3rrMrWrFmlTZvW31H7N7NixVIdOnTQ5viVK4mWTUJ8fP5bc7B37z4qWNBDc+fO0uzZf+jatWs2dc+ejdCqVcvt9pcxM3DZsr9u+druM888J0n6/fdfrMJGs9ms33+fqrCwIypY0ENduz6RlUsFAAAAAACZ4NXdPCRfsQq5PYQsy+mxLlkSqLFjv5Kvr58qVKggd/eCiouL1e7dIUpOTlaDBo3UvHmrO+qjcuWqGjRoqCZMGK8RI95S9eo15edXSmfOnNLBgwfUq1dvzZ07K4euyNrGjev0+ecjVayYtypXriIPj0K6fDlee/fuVkJCgipUqKjHHvsvSCtevIS+/PJbffDB2/rpp3GaNWu6KlSoKC+vYkpISNDJkycUEXFG1avXVMeOtrsIt2z5sDw8Cmnnzu2S0jcguXHDjwyPP/6k9u7do1WrlmvAgD6qW7eBPD09deTIIZ06dVIuLi76+OPP5OnpeVfuDQAAAAAAeQVBXx5gMpllSktTse5f5vZQssWUlmbZNOJODRz4soKCNuvAgb3av3+fEhMT5OlZVNWr11RAQFe1b99Jzs53/nHo3ft5lSlTVrNmzVBY2GGdOHFclSpV0meffaWqVf3vWtD39NPPyde3pPbu3aMjRw4pPj5ehQoVUrlyFdS+fUcFBDxms+lG3br1NWPGn1qw4E8FBW3WwYMHdO1aijw9i6pEiRLq0KGzHn64rd3+MtbjCwycJ0nq1OlRGQwGu+caDAZ9+OEnatKkmZYsCdThwwd19WqSihb1UkBAVz333AsqU6Zcjt4PAAAAAADyIoPZbM6ZJAU5Libmsm71dK5dS9GFC5Hy8vJVvnz5Mz3PaDTIaLQfxNyvTCZzjgV9QG7J6mcUAAAAwN3n7GyUp6e7nh23VIciLt7Tvqv5FdXM4V0UG5uo1FTTPe07r8p43nPHblR0xKV72re3X2H1er1Vjj1vg0EqVszjlucxoy+PIDQDAAAAAABwbGzGAQAAAAAAADgAZvQBuWT37lAtXbooy+cPHTpcRYoUuWvjAQAAAAAADzaCPiCXRESc1ooVS7N8fr9+Awn6AAAAAABApgj6gFwSENBVAQFdc3sYAAAAAADAQbBGHwAAAAAAAOAACPoAAAAAAAAAB0DQBwAAAAAAADgAgj4AAAAAAADAARD0AQAAAAAAAA6AoA8AAAAAAABwAAR9AAAAAAAAgAMg6AMAAAAAAAAcgHNuDwD3htFokNFoyO1hZIvJZJbJZM7tYQAAAAAAADwQCPryAKPRoCKernIyOuX2ULIlzZSmuNgkwj4AAAAAAIAsIOjLA4xGg5yMThq1apTCY8NzezhZUs6znD7u+LGMRsM9D/qWL/9Lo0ePUufOXTRixMgcazcy8qx69nxMPj6+mj//r3veRo8eXXXuXKTmzVsiX9+SWa73+ecjtWLFUr3//scKCOia3SEDAAAAAIB7hKAvDwmPDdeR6CO5PQwAAAAAAADcBQR9QB7x/fcTlZqaKm/v4rk9FAAAAAAAcBcQ9AF5hJ9fqdweAgAAAAAAuIsI+pBnnD59SjNmTFNIyE7FxETL2dlZhQoVVoUKFfXww2316KOP2dRJSkrS779P1bp1axQVdV4eHoXUuHETDRo0NNOZcVu2bNLs2TN0+PAhGY0GVaxYWc8885wqVaqSo9eTmpqquXNnasWKZTp7NkKurgVUv34jDRgwWGXLlrM5/2Zr9MXHX9K0ab9o48Z1unjxgjw9i6pFi1YaMGBwpv3HxsZqzZpV2rYtSCdPhuvChQtydnZW6dJl1KZNW/Xs+YxcXFzs1j1+/KimTp2s0NBdunr1qvz8SqlLl8fVo8fTeuqpx29rLUEAAAAAAPI6gj7kCcePH9WQIf2VmJioMmXKqlmzFjIanRQdHaXQ0BBFR0fbBH0JCQkaPLifzp8/pzp16qpChYrat2+vVq5cptDQXfrtt9kqWLCgVZ25c2fqhx++kyT5+9eQn18pnTlzWu+996Z69Xo2R6/p44/f05Ytm1S3bn1VrFhJBw/u17p1a7R1a5C+++5H1axZO0vtXLx4QS+//JLOnDklD49CatashUwms1avXqlt24JVvnwFu/W2bw/W99+Pkbd3cfn5lVL16jUVFxenAwf2adKkH7V580aNHz9J+fPnt6oXErJTb775qpKTk+XnV0oNGz6k+PhLmjjxB+3fv/eO7wsAAAAAAHkVQR/yhDlzZioxMVEvvTREL7zQ36osOfmqDh48YFNn06b1aty4qSZMmCJ39/RALz4+Xq+9NlhhYUcUGDhPffr0tZx/9GiYJkwYL6PRqFGjRqtNm3aWstWrV+jTTz/Kses5dy5SV68m6ZdfZqhSpcqSpLS0NP3ww1jNnz9XI0eO0KxZC2xCNnvGjv1aZ86cUp069fTVV99Zwsv4+Et6883XtHnzRrv1qlb116RJ01SzZi2r4/Hx8Ro58n1t375V8+fPUe/ez1vKkpOv6pNPPlRycrKefvo5vfzyqzIajZKkEyeO67XXhujixQu3dU8AAAAAAMjrjLk9AOBeiI29KElq2rS5TZmLSwHVrVvf5rirq6vef/9jS8gnSYUKFdJzz70oSdqxY7vV+QsWzFVaWpratGlrFfJJUocOndWiRas7vQwrzz/f3xLySZKTk5Nefvk1eXsX17lzkVq/fu0t2zh//pw2blwng8GgN998z2qGYqFChfXWW+9lWrdcufI2IV96vUIaPvwtSdK6dWusytat+0fR0VHy8fHV4MHDLCGfJJUvX8EmhAUAAAAAAFlH0Ic8wd+/hiRpzJgvtW1bsJKTk29Zp2pVfxUrVszmeNmy5SVJ0dFRVsdDQnZKkjp0CLDbXqdOXbI15lvp3Nm2vfz58+uRR9pbjedmdu8OkclkUpUq1ey+olu5clVVrFjZTs10aWlp2rFju3777ReNGfOlRo8epc8/H6np03+VJJ06ddLq/NDQXZKkNm3aydnZdkJxhw6dbzlmAAAAAABgH6/uIk/o3ft57dkTqh07tuuNN16Rs7OzKlWqojp16qlduw6WIPB6JUr42G3L3d1dkpSSkmJ1PCoqPfjLbAOJkiVzbmOJggU95OHhcdN+oqPP37KdqKj0c2626UXJkiV17FiYzfHTp0/p/fff1IkTxzOtm5iYeEN/6ffIx8fX7vkeHh4qWLCgEhISbjl2AAAAAABgjaAPeUKBAgU0btwEHTy4X9u2BWvv3j3at2+3Dh06oLlzZ+qJJ3rqjTfesapz/WulDyKz+e62/8EH7+jEieNq1qylnn32eZUrV17u7gXl7Oysa9euqU2bppnWNRgMN2n5ZmUAAAAAACAzBH3IU/z9a1hm76WmpmrTpvX67LOPFRg4T23atFX9+g1vu21vb29FRJzRuXORqlChok15ZGTkbbd9o4SEy7p8+bLdWX0Z/RQvXvyW7Xh7p59z7lzmY7M37pMnw3XsWJg8PYtq9OhvbF7DPX36VCb9ef9/f2ftlickJCgh4fItxw0AAAAAAGw92FOWgDvg7OysNm3aqXHj9JlnYWGH76i9jA09Vq9eYbd85cpld9T+jVatsm3v2rVrWrv2b0lSvXoNbtlGnTr1ZTAYdOTIIZ08GW5THhZ2xO5ru/HxlyRJxYoVs7vWXmb3IOMerVv3j1JTU23K//575S3HDAAAAAAA7CPoQ56wcOE8nToVbnP8woUYHT58UFLm68ZlVY8eveTk5KR169Zow4Z1VmVr1qzSpk3r76j9G/3221QdP37U8rPJZNLEieMVFXVexYuXUOvWj9yyDR8fH7Vq9bBMJpPGjPlCiYn/rY0XHx+vsWO/lNnOO8ClS5eVk5OTjh8/pl27dliVbd68UX/+Octuf23atJOXVzFFRp7Vzz9PkMlkspSdPBmu336bcssxAwAAAAAA+3h1Nw8p51kut4eQZTk91iVLAjV27Ffy9fVThQoV5O5eUHFxsdq9O0TJyclq0KCRmjdvdUd9VK5cVYMGDdWECeM1YsRbql69pvz8SunMmVM6ePCAevXqrblz7Qdg2VWihI+qVvVXv37PqV69BipUqLAOHTqgiIgzcnV11ccffy4XF5cstfX66+/o6NEwhYTsVM+ej6tevfoym6Vdu3aocOHCatGilTZv3mhVp0iRIure/SnNmzdbw4e/rNq166pYMW+dOnVSR44c0gsv9Nfvv0+16atAgQL66KNP9dZbwzVr1nRt3LhOVav66/LleIWE7FSLFq114MA+nT9/Tvny5cuRewUAAAAAQF5B0JcHmExmpZnS9HHHj3N7KNmSZkqTyZQzO0oMHPiygoI268CBvdq/f58SExPk6VlU1avXVEBAV7Vv38nuK6jZ1bv38ypTpqxmzZqhsLDDOnHiuCpVqqTPPvtKVav651jQZzAY9MknX2jWrOlatWq5du8OUYECrnr44UfUv/9glS9fIctteXkV088//6Zp06Zo48b1CgraLE/PomrXroMGDBiin34aZ7feq6++rooVKykwcL4OHz6ko0ePqEKFSho1arTatu1gN+iTpAYNGunnn3/Tr7/+rNDQXdq0aYNKlvTTSy+9rJ49n1aHDq1kNBrl4VHodm4NAAAAAAB5lsFs77083BdiYi7fcufUa9dSdOFCpLy8fJUvX/5MzzMaDTIaH6zdTE0mc44FfXgwhIbu0rBhA1WxYiX9/vuc3B5OjsjqZxQAAADA3efsbJSnp7ueHbdUhyIu3tO+q/kV1czhXRQbm6jUVNOtK+COZTzvuWM3Kjri0j3t29uvsHq93irHnrfBIBUrZrsh542Y0ZdHEJrhfhEbG6ukpCsqWdLP6vjx40f11VefSZICArrmxtAAAAAAAHigEfQBuKdOnDimV18drHLlKqhkST+5uLgoMvKsjhw5JJPJpEaNHtKTT/bK7WECAAAAAPDAIegDcsnu3aFaunRRls8fOnS4ihQpctfGc6+UKVNW3bv3VGjoLu3du1tXriTKzc1dNWvWVvv2ndS1a7ccWS8RAAAAAIC8hr+mgVwSEXFaK1YszfL5/foNdIigr1gxb73++ju5PQwAAAAAABwOQR+QSwICurIWHQAAAAAAyDHG3B4AAAAAAAAAgDtH0AcAAAAAAAA4AII+AAAAAAAAwAEQ9AEAAAAAAAAOgKAPAAAAAAAAcAAEfQAAAAAAAIADIOgDAAAAAAAAHABBHwAAAAAAAOAAnHN7ALg3jEaDjEZDbg8jW0wms0wmc24PAwAAAAAA4IFA0JcHGI0GeRZxldHJKbeHki2mtDTFxiUR9gEAAAAAAGQBQV8eYDQaZHRy0sFPP9WVkydzezhZ4la2rPw//FBGo+GeB33Ll/+l0aNHqXPnLhoxYmSOtRsZeVY9ez4mHx9fzZ//V461C1u7du3Qq68OVt269fXjjz/n9nAAAAAAALgnCPrykCsnTyrhSFhuDwMAAAAAAAB3AZtxAAAAAAAAAA6AoA8AAAAAAABwALy6izzj9OlTmjFjmkJCdiomJlrOzs4qVKiwKlSoqIcfbqtHH33Mpk5SUpJ+/32q1q1bo6io8/LwKKTGjZto0KCh8vYubrefLVs2afbsGTp8+JCMRoMqVqysZ555TpUqVcmR6zh06KBmzZquvXt3Kzb2ovLnd1HhwkVUpUoVder0qFq2fNhy7tSpkzVt2hT17fuSHn30MU2ZMlE7dmzT5cuXVby4j9q376jnnntBLi4FrPpITU3VP/+s1tatQTp8+KBiYmKUmpqqEiVK6KGHmuq5515UsWLeNmMbNmygQkN3afz4SXJyctLMmdN14MBeXbp0Se+995ECArrKZDLpr78WaeXKpTpx4riSkpLk4VFIXl7FVLduPT399HPy9S1pM54VK5Zq1arlOnbsqK5eTVKxYt566KGm6tOnr0qU8Mn0fl29elW//z5Va9f+rejoKHl4FFKTJs00YMBgm2d4/TqK8+Yt0ZIlgVq8eKFOnQqXk5OTqlevqf79B6lmzdq38eQAAAAAALi7CPqQJxw/flRDhvRXYmKiypQpq2bNWshodFJ0dJRCQ0MUHR1tE/QlJCRo8OB+On/+nOrUqasKFSpq3769WrlymUJDd+m332arYMGCVnXmzp2pH374TpLk719Dfn6ldObMab333pvq1evZO76OHTu26803X1VqaqoqVaqiGjVqyWQyKTo6SsHBW2QymayCvgyRkWfVv/9zcnJyVp069ZScnKyQkB2aNm2KduzYrnHjJsjFxcVy/sWLF/Tppx+pYMGCKlu2vCpWrKyrV5MUFnZE8+fP1Zo1qzVp0q8qVaq03XGuW/ePFi9eoDJlyqlBg8a6fDle+fLlkyR9+eWnWr78L+XP76LateuoSBFPxcfH6+zZCC1Y8KcaNGhsFfRduZKod955XSEhO+Xq6qaqVaupSBFPHT9+VIsWLdC6dWv03Xc/qUqVajbjSE1N1WuvDdGxY2GqV6+BqlSppj17QrVs2RJt3bpFP/44RaVLl7F7DaNHj9Lff69UnTr11KxZS4WFHda//27T7t0h+uGHn1WjRs3sPDoAAAAAAO46gj7kCXPmzFRiYqJeemmIXnihv1VZcvJVHTx4wKbOpk3r1bhxU02YMEXu7umBXnx8vF57bbDCwo4oMHCe+vTpazn/6NEwTZgwXkajUaNGjVabNu0sZatXr9Cnn350x9cxffqvSk1N1UcffaoOHTpblSUkJCg8/ITdeitXLlPLlq01cuTnltl7UVHn9dprQ7R3725NmzZFgwcPs5xfsGBBffnlt3rooWaWgE5KD86mTp2sGTOm6fvvx+ibb763219g4Dy9/vo76t69p9Xxc+fOafnyv1S8eAlNmfK7vLyKWZWHh59QgQKuVse++eYLhYTsVLNmLfXeex/K07OopezPP2dp/Pix+uij9zVz5jw5OTlZ1d23b49KlSqtP/6YLx+f9Fl/ycnJ+vTTD7V+/Vp99tnHmjx5ms34z52LVEjITk2fPldlypSVJKWlpenrrz/XsmVLNHXqJI0d+6PdawcAAAAAILewRh/yhNjYi5Kkpk2b25S5uBRQ3br1bY67urrq/fc/toR8klSoUCE999yLktJn111vwYK5SktLU5s2ba1CPknq0KGzWrRodaeXoYsX06+jSRPb6yhYsKBq1qxlt56Li4vefPM9q1d0ixcvoWHD/icpPZhLTk62lLm5uatFi9ZWIZ8kOTs7a9CgoSpWzFvbtgXrypVEu/01aNDIJuSTpNjYC5KkKlWq2oR8klSuXHlLICelB39r1qxSsWLeGjnyM6uQT5Keeqq3mjZtrjNnTmnr1iC7Yxk69DWrNl1cXPTGG++qQIEC2r9/r/bu3W233vDhb1lCPklycnLSwIEvS5JCQ3cpNTXVbj0AAAAAAHILQR/yBH//GpKkMWO+1LZtwVahVmaqVvVXsWK2YVTZsuUlSdHRUVbHQ0J2SpI6dAiw216nTl2yNWZ7qldPv45PPvlAu3eHZjlsaty4id1grXnzlipcuLASExN15Mghm/KwsCOaM+cPfffd1xo9epQ+/3ykPv98pNLS0mQymXTmzGm7/T38cFu7x8uWLSc3N3cFB2/R779P1dmzETcdd3DwFpnNZjVp0kxubu52z6lXr4Gk9Nl7NypY0EMtWrS2Oe7pWVQPPdRU0n/P7XpOTk5q0qSZzXEvr2Ly8CiklJQUXboUd9OxAwAAAABwr/HqLvKE3r2f1549odqxY7veeOMVOTs7q1KlKqpTp57atetgCQKvl9kGD+7u6YFTSkqK1fGoqPTg78aNJDKULGn/eHYMGjRUR4+GaevWIG3dGiQXFxdVqVJN9eo1UIcOnVWuXHm79TIbkyT5+JTUpUuXLOOX0jch+fTTj7Rx47qbjicx0f6MPh8fX7vH3dzc9f77H2n06E80ZcpETZkyUV5exVSjRi099FBTtW/fSW5ubpbzM4LApUsXa+nSxTcdS1xcrM0xX19fGQwGu+f7+vpJktV1Z/DyKiZnZ/v/PLq7u+vy5Xib5w8AAAAAQG4j6EOeUKBAAY0bN0EHD+7Xtm3B2rt3j/bt261Dhw5o7tyZeuKJnnrjjXes6hiN99+EVy+vYpo6dYZCQnZqx47t2rt3tw4c2Ke9e3drxoxpGjRoqOXV4uwzW76bPPlHbdy4TmXLltPgwcPk719DhQsXsbzKO3hwP+3bt0dms9luS9dv7HGjhx9uq4YNH9LmzRu0e3eo9u7drY0b12njxnWaOnWyvvvuJ1WsWCl9RGaTJKly5Sq33LW4evXb3RzD9hrux2cPAAAAAMCtEPQhT/H3r2GZvZeamqpNm9brs88+VmDgPLVp01b16ze87ba9vb0VEXFG585FqkKFijblkZGRt9329QwGg+rXb2gZa3Jyslas+Etjx36tn3+eoDZt2snPr9QNfZ/NtL1z587+//iLW46tXbtGkjRq1BeqVKmyTZ0zZ07d0TUULFhQnTo9qk6dHpUknT9/TuPGfaNNmzbou+++1o8//iwpfR1BSapVq45ef/2dTNvLzM3uub3rBgAAAADgQca0FeRZzs7OatOmnRo3Tl+rLSzs8B21l7Ghx+rVK+yWr1y57I7az4yLi4u6deuhihUryWQy6ejRMJtztm/fatmQ5HrBwZt16dIlubm5q2pVf8vx+PhLkuy/grttW7Di4uJy7gKU/pp0v36DJFk/h4x18jZv3pildRVvlJBwWZs3b7Q5Hhsbq23bgiX9t8YfAAAAAAAPOoI+5AkLF87TqVPhNscvXIjR4cMHJWW+rlxW9ejRS05OTlq3bo02bLBe227NmlXatGn9HbUvSbNmzdC5c+dsjp88GW7ZGMPedSQnJ2vMmC+UnHzVciwmJlo//jhOktStW3er120zNhyZP3+OVTunToVrzJgvbnv8R44c0j//rLYaR4YtWzbajL9KlWp6+OFHFBV1XiNGvGV3ZmJSUpJWr16hixcv2O3zxx/HKSrqvOXnlJQUjR37lZKSkuTvX0O1a9e97esBAAAAAOB+wqu7eYhb2bK5PYQsy+mxLlkSqLFjv5Kvr58qVKggd/eCiouL1e7dIUpOTlaDBo3UvHmrO+qjcuWqGjRoqCZMGK8RI95S9eo15edXSmfOnNLBgwfUq1dvzZ076476mD59qiZM+F5ly5ZT2bLl5eLiopiYaO3ZE6q0tDR16vSoqlatZlOvU6dHFRS0WU899bhq166nlJRk7dq1Q0lJSapZs7b69x9kdX6/fi/pgw/e0S+/TNK6dWtUrlwFy/2qU6eeihUrpr17bXe5vZVz587p44/ft2wiUrx4CaWlpen48aM6deqk8uXLpyFDXrWq8/77H+vy5QRt3Rqk3r2fVKVKleXr6yez2axz587q6NEwXbt2TTNnzlfRol5WdWvWrC2TyaTevZ9U/fqNVKBAAe3ZE6qYmGh5ehbVhx+OyvY1AAAAAABwvyLoywNMJrNMaWny//DD3B5KtpjS0mQy2d/sIbsGDnxZQUGbdeDAXu3fv0+JiQny9Cyq6tVrKiCgq9q375TpLqvZ0bv38ypTpqxmzZqhsLDDOnHiuCpVqqTPPvtKVav633HQ9/rr72jHju06dOiAQkN36erVJBUt6qVGjR7SY491V8uWre3W8/UtqV9+ma6ff56gXbt26PLleJUo4aP27Tvp2WdfkItLAavzW7d+RD/++LN+/XWKjh07ooiIMypZ0k/9+g3UM8/00f/+N/S2xl+jRk0NHjxMu3eHKDw8XGFhh+Xk5CRv7xLq3r2nevTopTJlylnVcXNz13ff/ah//lmt1atX6PDhQwoLOyJ3d3d5eRVT+/ad1KJFa5t1CaX017O/+eZ7TZv2s9atW6uYmCh5eBRSQEBX9e8/KNOdlQEAAAAAOcdoNMhoNNzzfp2c8t6LrAZzZttmItfFxFzWrZ7OtWspunAhUl5evsqXL3+m5+XWh+pOmEzmHAv68qqpUydr2rQp6tv3JZtZe7g3svoZBQAAAHD3OTsb5enprmfHLdWhCNt1zO+man5FNXN4F8XGJio11XRP+85NRqNBnkVcZXRyyrUxzB27UdERl+5pn95+hdXr9VY59rwNBqlYMY9bnseMvjyC0AwAAAAAANxrRqNBRicnHfz0U105efKe9l30oYdU/qWX7mmfuY2gDwAAAAAAAHfVlZMnlXAk7J726VqmzD3t735A0Afkkt27Q7V06aIsnz906HAVKVLkro0HAAAAAAA82Aj6gFwSEXFaK1YszfL5/foNzHbQ17//INbmAwAAAAAgjyDoA3JJQEBXBQR0ze1hAAAAAAAAB5H39hkGAAAAAAAAHBBBHwAAAAAAAOAACPoAAAAAAAAAB0DQBwAAAAAAADgAgj4AAAAAAADAARD0AQAAAAAAAA6AoA8AAAAAAABwAAR9AAAAAAAAgAMg6AMAAAAAAAAcgHNuDwD3htFokNFoyO1hZIvJZJbJZM7tYQAAAAAAADwQCPryAKPRIM8ibjI6PVgTOE1pJsXGXbnnYd/y5X9p9OhR6ty5i0aMGJlj7UZGnlXPno/Jx8dX8+f/lWPt3qhFi4aSpM2bd9y1PgAAAAAAwP2HoC8PMBoNMjoZtXrmLsWeT8jt4WSJZ4mC6vBsfRmNBmb1AQAAAAAAZAFBXx4Sez5B0RGXcnsYAAAAAAAAuAscMug7e/asfvnlF23ZskWRkZEym83y9vZWo0aN1LdvX1WrVs1uvaCgIE2bNk179uxRUlKSSpYsqY4dO2rgwIFyd3fPtL+TJ09q4sSJCgoK0sWLF1W0aFE1a9ZMQ4cOVenSpe/WZQIAAAAAAAAWDhf07d69W3379lViYqJKlCih5s2by8nJSQcPHtSiRYu0dOlSjRkzRp07d7aq99tvv+mLL76QwWBQw4YN5eXlpZ07d2rSpElatWqVZs2apaJFi9r0t3PnTvXv319JSUmqXLmyGjRooLCwMAUGBmrVqlWaNm2a6tate4+uHjdz+vQpzZgxTSEhOxUTEy1nZ2cVKlRYFSpU1MMPt9Wjjz5mUycpKUm//z5V69atUVTUeXl4FFLjxk00aNBQeXsXt9vPli2bNHv2DB0+fEhGo0EVK1bWM888p0qVquTYtezbt0fTpv2i/fv3KC0tTWXKlNMTT/RQly6PZ1rnwIF9Wr/+H4WE7NT58+cVH39JHh6F5O9fQz17Pq1GjR6yW89sNmvZsiUKDJyv8PDjcnEpIH//Gnrxxf66du2aXn11sOrWra8ff/zZUmfXrh2W42PH/qg//vhNq1evVFTUORUp4ql27Tqqf/9BcnFxUUJCgn777Rdt2LBOFy5Eq2hRL3Xu3EUvvNBfzs7W/0TFxsZqzZpV2rYtSCdPhuvChQtydnZW6dJl1KZNW/Xs+YxcXFxy5iYDAAAAAPCAcbig78MPP1RiYqJ69eqlDz/8UPny5ZMkmUwmjR8/XhMnTtRHH32kRx55xBIIHDhwQF9++aWcnJw0ceJEtW7dWlJ6yDNkyBAFBwdr5MiRGj9+vFVfSUlJGj58uJKSkjRo0CC9/vrrlrKxY8dq8uTJGj58uFauXKkCBQrcozsAe44fP6ohQ/orMTFRZcqUVbNmLWQ0Oik6OkqhoSGKjo62CfoSEhI0eHA/nT9/TnXq1FWFChW1b99erVy5TKGhu/Tbb7NVsGBBqzpz587UDz98J0ny968hP79SOnPmtN5770316vVsjlzL2rVrNGrUCKWlpalChYqqUKGSoqLO66uvPtOJE8czrTd58gSFhOxQ+fIVVLVqNRUo4KqIiDMKCtqkoKBNevXVN/TUU8/Y1Pv226+0aNF8GY1G1a5dV15exXT8+FENGzZQPXvann+91NRUvf76MIWFHVa9eg1UpkxZ7dkTolmzpis8/IQ++GCkBg/up/j4eNWtW09XrpRWaGiIpk2botjYi3rzzfes2tu+PVjffz9G3t7F5edXStWr11RcXJwOHNinSZN+1ObNGzV+/CTlz5//9m4uAAAAAAAPMIcK+mJjY3X48GFJ0vDhwy0hnyQZjUa98sormjZtmuLj43Xs2DFVr15dkjR58mSZzWZ1797dEvJJkqurqz7//HO1a9dOq1at0rFjx1SxYkVL+cKFCxUVFaVy5cpp+PDhVmMZPny4Vq1apfDwcC1atEhPP/30Xbxy3MqcOTOVmJiol14aohde6G9Vlpx8VQcPHrCps2nTejVu3FQTJkyRu3t6oBcfH6/XXhussLAjCgycpz59+lrOP3o0TBMmjJfRaNSoUaPVpk07S9nq1Sv06acf3fF1XLgQoy+//FRpaWl65ZX/WYWHO3Zs19tv/y/Tuk8//aw+/PATFStWzOr4vn179MYbr2jChO/Vpk1bq5mKmzdv0KJF8+Xq6qaxY39QrVp1LGVz5vyhH38cd9Px7tu3R/7+NfTnn4tVuHARSdK5c5Hq2/dZBQVt0iuvDFLp0mU0atQXljD80KEDGjSor5YsCdRzz/WVj4+Ppb2qVf01adI01axZy6qf+Ph4jRz5vrZv36r58+eod+/nbzouAAAAAAAckTG3B5CTsjOLx9PTU5KUkpKiDRs2SJK6dOlic56fn5/q168vSVqzZo1VWcbPjz76qIxG61tpNBoVEBAgSfr777+zPC7cHbGxFyVJTZs2tylzcSmgunXr2xx3dXXV++9/bAn5JKlQoUJ67rkXJaUHa9dbsGCu0tLS1KZNW6uQT5I6dOisFi1a3ellaOnSxbpyJVE1atSymSHYsGFjPf5490zrNm3a3Cbkk6SaNWure/enlJqaqk2bNliVzZs3R5LUo0cvq5BPkp5++jn5+1e/6XgNBoPee+9DS8gnST4+vurYMf2zcfbsWb377odWM16rVauuJk2ayWQyKSRkh1V75cqVtwn5pPTnMnz4W5KkdevW2JQDAAAAAJAXONSMPnd3dzVs2FA7duzQuHHjbF7d/eGHH3T16lW1atVKvr6+kqTw8HAlJSVJkmrWrGm33Zo1a2rHjh06cMB61lfGzzerd/15yD3+/jUUHLxFY8Z8qf79B6lu3fq3XMutalV/u8FY2bLlJUnR0VFWx0NCdkqSOnQIsNtep05dbIK07Pqvj052yzt3flTz5s3OtP6lS3EKCtqsEyeO6fLly0pNTZUknTlzSpJ06tRJy7mpqanau3ePJKl9e/v9tW/fye5syAwlSvioQoVKNsczNqmpWrWaPD1t174sVaqMJCkmJsamLC0tTSEhO7Vv3x7FxMQoJSVZZrNZZrPZ5hoAAAAAAMhLHCrok6RPP/1UAwcO1Ny5c7V+/XrVrFlTTk5OOnDggM6fP6/HH39cH3303yuUZ86ckZQ+I+jG9dYyZISCGedK6eu3xcXFSZJKlix503oXL17UlStX5ObmdsfXh9vTu/fz2rMnVDt2bNcbb7wiZ2dnVapURXXq1FO7dh3k71/Dpk6JEj52WpJlB+aUlBSr41FR6cGfr6/934fMfk+yIyNc9PX1s1ue2XFJWrIkUD/8MNYSbNtz5Uqi5ftLl+KUkpL8/+3aH7uPz82vKbN76OrqdtPyjM9KRv8ZTp8+pffff/OmaxEmJiZmWgYAAAAAgCNzuKCvQoUKmjt3rt5++21t3rxZ58+ft5RVqlRJjRs3tgr0MkIBV1fXTNvMCB0SEhJs6t2s7vXBXkJCQraDPoMhZ86BVKBAAY0bN0EHD+7Xtm3B2rt3j/bt261Dhw5o7tyZeuKJnnrjjXes6tz4OvaD7NChg/rmm9EyGo0aMuQVNW/eSiVK+KhAgQIyGAxavHihvvlmtGVWXFbd6vfPcIsTsnuPP/jgHZ04cVzNmrXUs88+r3LlysvdvaCcnZ117do1tWnTNFvt3UsGA59XAAAAAOn42yBvyYnnndU2HC7o27lzp1555RU5OTnp22+/VZMmTZQvXz7t2rVLX375pUaMGKFdu3Zp9OjRuT3UW/Ly8rjlOVevXtXFi0Y5ORnk7Gw/NHFyenADq5wee61atVSrVvoab6mpqdq4cb1GjfpIgYHz1K5dOzVo0EhGY/qnx2Cwf0+vH9P15cWLe+vMmTOKjj6nKlUq29SLijpnt152eHsX18mT4YqKirTbRnS0/T42bPhHZrNZPXs+rRde6GtT7+zZ05Ksr9nLy1P58+dXSkqKYmLOq3z5Cple0433KuMeZXYPb3WPM8qNxv/Kw8NP6NixMHl6FtXXX38rZ2frf75Onfpvxu3t3t+7wWQyyGg0ytPTnd23AQAAAMjT0z23h4B76F4/b4cK+uLj4zVs2DDFxsZq7ty5qlPnv80D2rRpo0qVKqlr165asGCBHnvsMTVp0sTyGubNX2e8IklWMwEz6t2sbka9G+tm1YULl3WrCVbXrqXIZDIpLc2s1FRTtvu436Wlme7idRnVqtUjatx4uTZtWq9Dhw6pTp0GMpnSb7rZbP+epqX9d+z68jp16uvMmTNasWK5HnrIdtOPZcuW2q2XHXXr1teOHdu1cuUKdevW004ff9ntIy7ukiSpeHEfm76Tk5O1du1aSTdes5Nq1KilkJCdWrFiuQYOfNmmv1WrVtqp9989yuwe3uoeZ5SbTP+Vx8bGSdL/r5totKm3fPkyu9ee29LSzDKZTIqNTVS+fNdyezgAAABAnubkZMz1oC02NtHq70pHdz/c89yUU8/bYMjahLD7Z9pLDli/fr0uXryo0qVLW4V8GUqXLq3atWtLkoKDgyWl76orpYeE17+ae73IyEirc6X04K5IkSKS0ncOvVk9T0/P21qfz2zO2hdubeHCeTp1Ktzm+IULMTp8+KCk9N1g70SPHr3k5OSkdevWaMOGdVZla9as0qZN6++ofUnq0uVxubq6ad++PZYdcTPs2rVDixYtsFuvXLlykqSVK5darcOXnJysb7/9UpGREXbr9ejxtCRp/vy52rdvr1XZn3/O1oED+273UrKtdOmycnJy0vHjx7Rrl/VuvJs3b9Sff866Z2O5HVn9PPPFF1988cUXX3zxxRdfd+/rfpHb9yEv3vPcdC/vpUPN6MsI1m42e87DIz39zNhIo3z58nJ1dVVSUpL27dunJk2a2NTZty89zKhRw3rDhurVqysoKEj79u3TI488kuV6ucWzRPZnFeaWnB7rkiWBGjv2K/n6+qlChQpydy+ouLhY7d4douTkZDVo0EjNm7e6oz4qV66qQYOGasKE8Rox4i1Vr15Tfn6ldObMKR08eEC9evXW3Ll3FkYVK+atd94ZoU8//Ujffz9GS5cuUvnyFRUTE63du0P01FPP2O0jIOAxzZs3R0eOHFbPno+pdu16cnIyavfuUCUnJ6tnz2fs7tbbunUbPfbYE1qyJFBDhw5Q7dp15eVVTMePH9XJk+GWa8rY3fpuKlKkiLp3f0rz5s3W8OEvq3btuipWzFunTp3UkSOH9MIL/fX771Pv+jgAAAAAALhfOVTQV6JECUnS8ePHdfnyZUuol+HatWs6cOCAJKlUqVKSpPz586t169ZauXKlli5dahP0RUREKCQkRJLUrl07q7J27dopKChIy5Yt07Bhw6w2FjCZTFq+fLkkqX379jl4ldlnMpllSjOpw7P1c3Uc2WVKM1le4bxTAwe+rKCgzTpwYK/279+nxMQEeXoWVfXqNRUQ0FXt23eyWfPtdvTu/bzKlCmrWbNmKCzssE6cOK5KlSrps8++UtWq/ncc9ElSu3Yd5e1dQr//PlX79+9RRMQZlSlTVm+++Z4ef7y73T48PDz0yy8zNHXqZG3fHqxt24JUqFBhNW78kPr2Hag9e0Iz7e+tt96Xv38NBQbO1/79+5Q/f35Vr15Db7zxriIj02ezFi5c5I6vKyteffV1VaxYSYGB83X48CEdPXpEFSpU0qhRo9W2bQeCPgAAAABAnmYwZ3ebzfvYxYsX1bZtW125ckWdOnXS6NGjLWvppaSk6Msvv9TMmTOVL18+rVixQqVLl5Yk7d+/X08++aSMRqMmTZqkVq3SZ3YlJSVpyJAhCg4OVseOHTV+/Hir/pKSktShQwdFRUVp8ODB+t///mcp++677zRp0iT5+Pho1apVt7UIf0xM1tbou3AhUl5evsqXL3+m5xmNBssGBw8Kk8mcY0Ef7o7Ro0dp+fK/NGzYcD399HO5PZz7UlY/owAAAADuPmfn9PXinh23VIciLt7Tvqv5FdXM4V0UG5t4X60rfrdl3POdAwYo4UjYPe3bu11bVf/oI80du1HREZfubd9+hdXr9VY59rwNBqlYsVuv0edQM/qKFi2qkSNH6v3339fKlSu1fft21apVS87Oztq3b5/Onz8vo9GoESNGWEI+Kf3V2nfffVdffPGFBg4cqEaNGsnLy0s7duxQdHS0ypcvr5EjR9r05+rqqnHjxql///6aNGmS1q5dq8qVKyssLExHjhyRm5ubvv/++/tip01CM9yu48ePyde3pFxdXS3HTCaTli5drBUrlip/fhe1a9cpF0cIAAAAAAAkBwv6JOnxxx9X1apV9fvvv+vff/9VcHCwzGazihcvrq5du+r555+3bMhxvRdffFFVqlTRr7/+qr179+rKlSsqWbKkunfvroEDB2a67l+DBg20ePFiTZgwQUFBQVq9erU8PT3VrVs3DR06VGXKlLnblwzcVbNnz9DatX+rSpWqKlasuK5eTVJ4+AlFRp6Vk5OT3njjnf/fCRcAAAAAAOQmhwv6JKlatWr64osvsl2vWbNmatasWbbrlS1bVl999VW26yFv2707VEuXLsry+UOHDrfs9HwvPfJIeyUmJurw4YMKCzuitLQ0eXoWVdu27dWzZ2/VrFnrno8JAAAAAADYcsigD3gQRESc1ooVS7N8fr9+A3Ml6GvatLmaNm1+z/sFAAAAAADZQ9AH5JKAgK4KCOia28MAAAAAAAAOwpjbAwAAAAAAAABw5wj6AAAAAAAAAAdA0AcAAAAAAAA4AII+h2HO7QEAsIvPJgAAAADg3iDoe8AZjU6SpNTUa7k8EgD2pKWlSpKMRv65BQAAAADcXfzl+YBzcnJS/vwFlJh4WSaTKbeHA+A6ZrNZV64kytk5v5yc2OQcAAAAAHB38ZenAyhYsIhiY6N04UKkChRwV/78Lv8/e8iQ20MD8iiz0tJSdeVKolJSklS4cLHcHhAAAAAAIA8g6HMA+fO7yMvLRwkJcbpy5bISEy/l9pAASHJ2zq/ChYvJ1dU9t4cCAAAAAMgDCPochLNzPhUp4i2zOX0mkdnMBgBAbjIajbyuCwAAAAC4p/gr1MEYDAY5O+fL7WEAAAAAAADgHmMzDgAAAAAAAMABEPQBAAAAAAAADoCgDwAAAAAAAHAABH0AAAAAAACAAyDoAwAAAAAAABwAQR8AAAAAAADgAAj6AAAAAAAAAAdA0AcAAAAAAAA4AII+AAAAAAAAwAEQ9AEAAAAAAAAOgKAPAAAAAAAAcAAEfQAAAAAAAIADIOgDAAAAAAAAHABBHwAAAAAAAOAACPoAAAAAAAAAB0DQBwAAAAAAADgAgj4AAAAAAADAARD0AQAAAAAAAA6AoA8AAAAAAABwAAR9AAAAAAAAgAMg6AMAAAAAAAAcAEEfAAAAAAAA4AAI+gAAAAAAAAAHQNAHAAAAAAAAOACCPgAAAAAAAMABEPQBAAAAAAAADoCgDwAAAAAAAHAABH0AAAAAAACAAyDoAwAAAAAAABwAQR8AAAAAAADgAAj6AAAAAAAAAAdA0AcAAAAAAAA4AII+AAAAAAAAwAEQ9AEAAAAAAAAOgKAPAAAAAAAAcAAEfQAAAAAAAIADIOgDAAAAAAAAHABBHwAAAAAAAOAACPoAAAAAAAAAB0DQBwAAAAAAADgAgj4AAAAAAADAARD0AQAAAAAAAA6AoA8AAAAAAABwAAR9AAAAAAAAgAMg6AMAAAAAAAAcAEEfAAAAAAAA4AAI+gAAAAAAAAAHQNAHAAAAAAAAOADn3B4AAAC4N4xGg4xGQ670bTKZZTKZc6VvAAAAIK8g6AMAIA8wGg0qUsRNTk65M5k/Lc2kuLgrhH0AAADAXUTQBwBAHmA0GuTkZNQHszbpRNSle9p3+eKF9VnvljIaDQR9AAAAwF1E0AcAQB5yIuqSDkVczO1hAAAAALgL2IwDAAAAAAAAcAAEfQAAAAAAAIADIOgDAAAAAAAAHABBHwAAAAAAAOAACPoAAAAAAAAAB0DQBwAAAAAAADgAgj4AAAAAAADAARD0AQAAAAAAAA6AoA8AAAAAAABwAAR9AAAAAAAAgAMg6AMAAAAAAAAcAEEfAAAAAAAA4AAI+gAAAAAAAAAHQNAHAAAAAAAAOACCPgAAAAAAAMABEPQBAAAAAAAADoCgDwAAAAAAAHAABH0AAAAAAACAAyDoAwAAAAAAABwAQR8AAAAAAADgAAj6AAAAAAAAAAdA0AcAAAAAAAA4AII+AAAAAAAAwAEQ9AEAAAAAAAAOgKAPAAAAAAAAcAAEfQAAAAAAAIADIOgDAAAAAAAAHABBHwAAAAAAAOAACPoAAAAAAAAAB0DQBwAAAAAAADgAgj4AAAAAAADAARD0AQAAAAAAAA6AoA8AAAAAAABwAAR9AAAAAAAAgAMg6AMAAAAAAAAcAEEfAAAAAAAA4AAI+gAAAAAAAAAHQNAHAAAAAAAAOACCPgAAAAAAAMABEPQBAAAAAAAADoCgDwAAAAAAAHAABH0AAAAAAACAAyDoAwAAAAAAABwAQR8AAAAAAADgAAj6AAAAAAAAAAfgnNsDuFtSUlI0Z84crVixQseOHVNSUpI8PT1VpUoVde/eXQEBATZ1goKCNG3aNO3Zs0dJSUkqWbKkOnbsqIEDB8rd3T3Tvk6ePKmJEycqKChIFy9eVNGiRdWsWTMNHTpUpUuXvpuXCQAAAAAAAEhy0KDv3Llz6t+/v44ePSpPT0/Vr19frq6uioyM1I4dO+Tm5mYT9P3222/64osvZDAY1LBhQ3l5eWnnzp2aNGmSVq1apVmzZqlo0aI2fe3cuVP9+/dXUlKSKleurAYNGigsLEyBgYFatWqVpk2bprp1696jKwcAAAAAAEBe5XBB39WrV9W3b18dP35cr7zyigYNGqR8+fJZypOSkhQeHm5V58CBA/ryyy/l5OSkiRMnqnXr1pZzhwwZouDgYI0cOVLjx4+3qpeUlKThw4crKSlJgwYN0uuvv24pGzt2rCZPnqzhw4dr5cqVKlCgwN27aAAAAAAAAOR5DrdG3+TJk3X8+HH16tVLw4YNswr5JMnV1VX+/v42dcxms7p3724J+TLO/fzzz2U0GrVq1SodO3bMqt7ChQsVFRWlcuXKafjw4VZlw4cPV7ly5RQZGalFixbl6DUCAAAAAAAAN3KooO/atWuaPXu2JKl///5ZqpOSkqINGzZIkrp06WJT7ufnp/r160uS1qxZY1WW8fOjjz4qo9H6VhqNRsvrwX///Xc2rgIAAAAAAADIPod6dffAgQOKjY1V8eLFVbZsWR0+fFh///23oqKiVKhQITVs2FCtWrWyCuXCw8OVlJQkSapZs6bddmvWrKkdO3bowIEDNv3dqt715wEAAAAAAAB3i0MFfYcPH5Yk+fj4aMyYMfrll19kNpst5VOmTFH16tX1008/qWTJkpKkM2fOSJIKFSqkggUL2m3X19fX6lxJSkhIUFxcnCRZ2sqs3sWLF3XlyhW5ubll63oMhmydDgDAfY//bQMAAOC/ifKanHjeWW3DoYK+jODt4MGD2rNnj5599ln16dNH3t7e2rNnj0aNGqUDBw5o0KBBWrhwofLly6fExERJ6evxZSYjoEtISLAcy6h3s7rXB3sJCQnZDvq8vDyydT4AAPczT0/33B4CAABAruO/ifKWe/28HSroy5i9d+3aNXXp0kUfffSRpaxZs2aaNm2aOnXqpCNHjmjZsmXq1q1bLo00ay5cuKzrJiQCAHDbnJyMuf4flbGxiUpLM+XqGAAAQN7GfxPde/fDPc9NOfW8DYasTQhzqKDP3f2/X5xevXrZlJcsWVIPP/ywVq1apeDgYHXr1s1SJ2OdPnuuXLkiSVav9l7fV2Z1M+rdWDerzGYR9AEAHAr/uwYAAMB/E+U19/J5O1TQV7p0abvfX69UqVKSpOjoaEnpu+pKUnx8vBISEuwGcpGRkVbnSunBXZEiRRQXF6ezZ8+qWrVqmdbz9PTM9mu7AAAAd8JoNMhozJ0FgEwms0wm/oIBAMAeJyfjrU+6C/jf57zBoYK+6tWry2AwyGw2KzY21rIZxvViY2Ml/bd+Xvny5eXq6qqkpCTt27dPTZo0samzb98+SVKNGjVs+gsKCtK+ff/X3n1HR1E97h9/dkMgCRASeu9sIPSANCkfOiIoTUEBBQWCCooFKSrKF6kiCIhiAwQUQaQo0osUQwtFWiAUgxCQUBIgJEDK/v7gtytLNiELKTB5v87xHJmZO/fOZu7M7LMzdw6qWbNmqS4HAACQnsxmk3x9vGTOrC8SCYmKjIrhywQAAHfIl9tD1sQEeXsn/46A9JSYkKDIqFjOzwZnqKCvQIECqlWrloKDgxUUFCR/f3+H+XFxcdq1a5ckqVq1apKk7Nmzq0mTJlq1apWWL1+eJOgLDw/X3r17JUktWrRwmNeiRQsFBQXp999/14ABA2Q2/3cxnZiYqBUrVkiSWrZsmbYbCgAAkAKz2SSzm1lrftijyPPR9y6QhnwL5VKr7gEym018kQAA4A65PbLLZHbTxcVDFXfxZIbW7Z6/rPJ3Gsf5OQswVNAnSQMGDFCvXr309ddfq3bt2qpRo4YkKT4+XuPHj9fp06eVM2dOderUyV6mX79+Wr16tRYvXqxWrVqpcePGkm6Pvffee+8pISFBrVu3Vrly5Rzq6tSpk2bMmKGwsDBNmTJFb775pn3elClTFBYWpsKFCz/0L/0AAADGFHk+WhfCr2R2MwAAwB3iLp5U3L8hmd0MGJThgr769evrjTfe0JQpU9S9e3dVrVpVBQoU0KFDhxQeHi4PDw9NmjRJ+fPnt5epXLmyhg4dqrFjx6pfv3567LHHlC9fPgUHB+vChQsqU6aMPvrooyR1eXp66rPPPtPLL7+sGTNmaMOGDapQoYKOHTum0NBQeXl5acqUKfLw8MjATwAAAAAAAABZkeGCPkl69dVXVa1aNX3//ffav3+/Dh48qPz586tTp07q06dPkjvzJKlXr16yWCyaOXOmDhw4oJiYGBUtWlSdOnVSv379kn1rbq1atbRs2TJ98cUXCgoK0po1a+Tr66sOHTrotddeU8mSJdN7cwEAAAAAAABjBn2S1LBhQzVs2NClMg0aNFCDBg1crqtUqVIaP368y+UAAAAAAACAtJI5r2IDAAAAAAAAkKYe6I6+xMREHTx4UMeOHVNUVJRMJpPy5Mkji8WiKlWqyGQypVU7AQAAAAAAAKTgvoK+uLg4ffvtt5o9e7auXr3qdBkfHx/17t1bL730krJlM+wTwgAAAAAAAMBDweUELiYmRoGBgQoODpbVapUkubu7K0+ePLJarbp69ari4uIUGRmpyZMnKygoSDNmzODNswAAAAAAAEA6cjnoGz9+vHbt2qUcOXKoR48eeuqpp2SxWOyP6SYmJio0NFTLli3Tjz/+qB07dmjChAkaMWJEmjceAAAAAAAAwG0uBX1nz57VwoULlStXLs2ZM0f+/v5JljGbzapYsaIqVqyodu3a6YUXXtCCBQvUr18/FS5cOM0aDgAAAAAAAOA/Lr1197fffpMkvf32205DvrtVrlxZb7/9thISEvTrr7/eXwsBAAAAAAAA3JNLQd/evXuVI0cOde7cOdVlOnfurBw5cmjv3r0uNw4AAAAAAABA6rgU9B0/flyVKlVS9uzZU10mR44c8vf317Fjx1xuHAAAAAAAAIDUcSnou3LligoUKOByJQULFtSVK1dcLgcAAAAAAAAgdVwK+q5fv66cOXO6XImXl5diYmJcLgcAAAAAAAAgdVwK+hITE++7ogcpCwAAAAAAACBl2VwtEBMTo7Nnz7pU5vr1665WAwAAAAAAAMAFLgd9a9as0Zo1a9KjLQAAAAAAAADuk8tBn9Vqva+KTCbTfZUDAAAAAAAAcG8uBX3r169Pr3YAAAAAAAAAeAAuBX3FihVLr3YAAAAAAAAAeAAuvXUXAAAAAAAAwMOJoA8AAAAAAAAwAJce3R02bNh9V2QymTRmzJj7Lg8AAAAAAAAgeS4FfUuWLJHJZHL65l3bW3XvnmdbnqAPAAAAAAAASD8uBX0dOnSwB3p3W7JkiUqVKqWAgIA0aRgAAAAAAACA1HMp6Bs3blyy85YsWaKAgACNHTv2gRsFAAAAAAAAwDW8jAMAAAAAAAAwAII+AAAAAAAAwAAI+gAAAAAAAAADIOgDAAAAAAAADICgDwAAAAAAADAAgj4AAAAAAADAALK5svDnn3+e4vwjR46kuMyAAQNcqQ4AAAAAAABAKrkc9JlMpmTnHzlyREeOHEl2PkEfAAAAAAAAkD5cCvoee+yx9GoHAAAAAAAAgAfgUtA3d+7c9GoHAAAAAAAAgAfAyzgAAAAAAAAAA3Dpjr57iY+P15w5c7Ru3TpFRkaqcOHCevLJJ9WlS5e0rAYAAAAAAADAXVwK+tasWaMPP/xQzz77rN58802HeYmJiQoMDFRQUJCsVqsk6e+//9b27dsVHByscePGpV2rAQAAAAAAADhw6dHdHTt2KCoqSq1bt04yb+HChfrzzz9ltVrVrFkzffDBB+rTp488PDy0bNkybd26Nc0aDQAAAAAAAMCRS3f0/fXXXypQoID8/f2TzFuwYIFMJpPatm2rTz/91D69WrVqev3117Vs2TI1bNjwwVsMAAAAAAAAIAmX7ui7cOGCKlWqlGT65cuXFRISIknq06ePw7xWrVqpWLFi2r9//wM0EwAAAAAAAEBKXAr6IiMj5e3tnWT6gQMHJEl58+Z1GgSWL19eERER99lEAAAAAAAAAPfiUtDn5uamy5cvJ5l++PBhSXL6SK8k5c6dW/Hx8ffRPAAAAAAAAACp4VLQV7RoUR0+fFi3bt1ymL5t2zaZTCZVr17dabnIyEjlz5///lsJAAAAAAAAIEUuBX1169ZVVFSUpkyZYp+2fft27dq1S5LUpEkTp+VCQkJUsGDBB2gmAAAAAAAAgJS49NbdF198UYsWLdLMmTO1fPly5c2bV8eOHZMkVa9eXVWrVk1SZu/evbp8+bKefPLJtGkxAAAAAAAAgCRcuqOvVKlSmjhxojw9PXX+/HmFhIQoPj5eBQsW1Lhx45yWWbBggSSpfv36D95aAAAAAAAAAE65dEefJLVq1Uq1atXSxo0bdenSJRUpUkQtWrSQl5eX0+WrVq2qSpUqqV69eg/cWAAAAAAAAADOuRz0SVK+fPnUpUuXVC3bvXv3+6kCAAAAAAAAgAtcenQXAAAAAAAAwMOJoA8AAAAAAAAwAII+AAAAAAAAwAAI+gAAAAAAAAADIOgDAAAAAAAADICgDwAAAAAAADAAgj4AAAAAAADAAAj6AAAAAAAAAAMg6AMAAAAAAAAMgKAPAAAAAAAAMACCPgAAAAAAAMAAsmV2AwAAQNbg5pY5vy8mJlqVmGjNlLoBAACAjETQBwAA0lW+3B6yJibI29szU+pPTEhQZFQsYR8AAAAMj6APAACkq9we2WUyu+ni4qGKu3gyQ+t2z19W+TuNk9lsIugDAACA4RH0AQCADBF38aTi/g3J7GYAAAAAhkXQBwAAkE7MZpPMZlOG15tZ4yECAAAgcxH0AQAApAOz2SRfH0+Z3dwyuykAAADIIgj6AAAA0oHZbJLZzU0ho0Yp5tSpDK07b926KtO3b4bWCQAAgMxH0AcAAJCOYk6dUnTosQyt07NkyQytDwAAAA8HBnABAAAAAAAADICgDwAAAAAAADAAgj4AAAAAAADAAAj6AAAAAAAAAAMg6AMAAAAAAAAMgKAPAAAAAAAAMACCPgAAAAAAAMAACPoAAAAAAAAAAyDoAwAAAAAAAAyAoA8AAAAAAAAwAII+AAAAAAAAwAAI+gAAAAAAAAADIOgDAAAAAAAADICgDwAAAAAAADAAgj4AAAAAAADAALJldgMAAAAAAACQ/tzcMv5+r8yoMysj6AMAAAAAADAwc858SkhMkLe3Z2Y3BemMoA8AAAAAAMDAzB7ecjO7aeTqkQqLDMvQuuuVrKfABoEZWmdWRtAHAAAAAACQBYRFhin0QmiG1lnKt1SG1pfV8aA0AAAAAAAAYAAEfQAAAAAAAIABEPQBAAAAAAAABkDQBwAAAAAAABgAQR8AAAAAAABgAAR9AAAAAAAAgAEQ9AEAAAAAAAAGQNAHAAAAAAAAGABBHwAAAAAAAGAABH0AAAAAAACAARD0AQAAAAAAAAZA0AcAAAAAAAAYAEEfAAAAAAAAYAAEfQAAAAAAAIABEPQBAAAAAAAABkDQBwAAAAAAABgAQR8AAAAAAABgAAR9AAAAAAAAgAEQ9AEAAAAAAAAGQNAHAAAAAAAAGABBHwAAAAAAAGAABH0AAAAAAACAARD0AQAAAAAAAAaQJYK+CRMmyM/PT35+fvriiy+SXS4oKEh9+/ZV3bp1Va1aNbVp00aTJ0/W9evXU1z/qVOnNHToUDVu3FhVqlRR48aNNXToUJ0+fTqtNwUAAAAAAABwyvBB3549ezRr1iyZTKYUl5s9e7Z69+6tLVu2qEKFCmratKmio6M1Y8YMde7cWZcvX3Zabvfu3Xr66ae1ZMkSeXt7q2XLlvL29taSJUv01FNPad++femwVQAAAAAAAIAjQwd9sbGxGjZsmAoUKKDmzZsnu9zhw4c1btw4ubm56auvvtK8efM0ZcoUrV27VvXr19fff/+tjz76yOn6Bw0apNjYWAUGBmr58uWaPHmyli9frsDAQMXExGjQoEG6ceNGOm4lAAAAAAAAYPCg79NPP1VYWJhGjRql3LlzJ7vcV199JavVqk6dOqlJkyb26Z6enho9erTMZrNWr16tEydOOJRbvHixIiIiVLp0aQ0aNMhh3qBBg1S6dGmdO3dOS5cuTcvNAgAAAAAAAJIwbNC3Y8cOzZs3Tx06dHAI7+5269Ytbdq0SZLUrl27JPOLFSumgIAASdK6desc5tn+/eSTT8psdvwozWaz2rZtK0lau3bt/W8IAAAAAAAAkAqGDPquX7+u4cOHK3/+/Bo+fHiKy4aFhSk2NlaSVKVKFafL2KYfPnzYYbrt366WAwAAAAAAANJatsxuQHoYP368zpw5o+nTpytPnjwpLnvmzBlJkre3t3LlyuV0mSJFijgsK0nR0dGKioqSJBUtWjTFcpcvX1ZMTIy8vLxc2g4AAAAAAAAgtQwX9G3dulULFizQk08+qRYtWtxz+evXr0u6PR5fcmwBXXR0dJJyKZW9M9iLjo52Oei7x4uCAQCACzivZjw+cwAAgLS5JkrtOgwV9F27dk3vvfee8ubNq/fffz+zm/PA8uVL/gUiAAAg9Xx9c2Z2E7IcPnMAAICMvyYyVNA3ZswY/fvvv5o8ebLy5s2bqjI5c97+wG3j9DkTExMjSQ6P9trKpVTWVu7usql16dI1Wa0uFwMAIAk3N3OWDl4iI68rISExQ+vkM8/4zxwAgHvJ6udnZLy0uiYymVJ3Q5ihgr61a9cqW7Zsmj9/vubPn+8w7+TJk5KkRYsWadu2bcqfP78mT56sYsWKSZKuXr2q6Ohop4HcuXPnJMm+rHQ7uPPx8VFUVJTOnj2rihUrJlvO19f3vsbns1pF0AcAQBrhnJrx+MwBAAAy9prIUEGfJMXHx2vnzp3Jzg8PD1d4eLg9tCtTpow8PT0VGxurgwcPql69eknKHDx4UJJUuXJlh+n+/v4KCgrSwYMH1axZs1SXAwAAAAAAANKaObMbkJaCg4N19OhRp/917NhRkvTGG2/o6NGj2rBhgyQpe/bsatKkiSRp+fLlSdYZHh6uvXv3SlKSl3vY/v37778rMdHxNszExEStWLFCktSyZcs03EoAAAAAAAAgKUMFfferX79+MplMWrx4sTZv3myfHhsbq/fee08JCQlq3bq1ypUr51CuU6dOKliwoMLCwjRlyhSHeVOmTFFYWJgKFy6sDh06ZMRmAAAAAAAAIAsz3KO796Ny5coaOnSoxo4dq379+umxxx5Tvnz5FBwcrAsXLqhMmTL66KOPkpTz9PTUZ599ppdfflkzZszQhg0bVKFCBR07dkyhoaHy8vLSlClT5OHhkfEbBQAAAAAAgCyFO/r+v169emnWrFlq2LChQkNDtX79euXMmVOBgYFatGhRsm/xrVWrlpYtW6YOHTooKipKa9asUVRUlDp06KBly5apRo0aGbshAAAAAAAAyJKyzB1948aN07hx41JcpkGDBmrQoIHL6y5VqpTGjx9/v00DAAAAAAAAHhh39AEAAAAAAAAGQNAHAAAAAAAAGABBHwAAAAAAAGAABH0AAAAAAACAARD0AQAAAAAAAAZA0AcAAAAAAAAYAEEfAAAAAAAAYAAEfQAAAAAAAIABEPQBAAAAAAAABkDQBwAAAAAAABgAQR8AAAAAAABgAAR9AAAAAAAAgAEQ9AEAAAAAAAAGQNAHAAAAAAAAGABBHwAAAAAAAGAABH0AAAAAAACAARD0AQAAAAAAAAZA0AcAAAAAAAAYAEEfAAAAAAAAYAAEfQAAAAAAAIABEPQBAAAAAAAABkDQBwAAAAAAABgAQR8AAAAAAABgAAR9AAAAAAAAgAEQ9AEAAAAAAAAGQNAHAAAAAAAAGABBHwAAAAAAAGAABH0AAAAAAACAAWTL7AYAAAAAADKO2WyS2WzKlLoTE61KTLRmSt1ZFX9vIGsh6AMAAACALMJsNsnHx0tubpnzcFdCQqKiomIIfzKI2WySr4+XzJn0905MSFQkf28gQxH0AQAAAEAWYTab5OZm1vs/btHfEVcytO4yBfPo4+cbyWw2EfxkELPZJLObWWt+2KPI89EZWrdvoVxq1T2AvzeQwQj6AAAAACCL+Tviio6EX87sZiCDRJ6P1oXwjA12AWQOXsYBAAAAAAAAGABBHwAAAAAAAGAABH0AAAAAAACAARD0AQAAAAAAAAZA0AcAAAAAAAAYAEEfAAAAAAAAYAAEfQAAAAAAAIABEPQBAAAAAAAABkDQBwAAAAAAABgAQR8AAAAAAABgAAR9AAAAAAAAgAEQ9AEAAAAAAAAGQNAHAAAAAAAAGABBHwAAAAAAAGAABH0AAAAAAACAARD0AQAAAAAAAAZA0AcAAAAAAAAYAEEfAAAAAAAAYAAEfQAAAAAAAIABEPQBAAAAAAAABkDQBwAAAAAAABgAQR8AAAAAAABgANkyuwEAAAAAgKzDzS1z7jdJTLQqMdGaKXUDQEYh6AMAAAAApLt8uT1kTUyQt7dnptSfmJCgyKhYwj4AhkbQBwAAAABId7k9sstkdtPFxUMVd/Fkhtbtnr+s8ncaJ7PZRNAHwNAI+gAAAAAAGSbu4knF/RuS2c0AAEPiZRwAAAAAAACAARD0AQAAAAAAAAZA0AcAAAAAAAAYAEEfAAAAAAAAYAAEfQAAAAAAAIABEPQBAAAAAAAABkDQBwAAAAAAABgAQR8AAAAAAABgAAR9AAAAAAAAgAEQ9AEAAAAAAAAGQNAHAAAAAAAAGABBHwAAAAAAAGAABH0AAAAAAACAARD0AQAAAAAAAAZA0AcAAAAAAAAYAEEfAAAAAAAAYAAEfQAAAAAAAIABEPQBAAAAAAAABkDQBwAAAAAAABgAQR8AAAAAAABgAAR9AAAAAAAAgAEQ9AEAAAAAAAAGQNAHAAAAAAAAGABBHwAAAAAAAGAA2TK7AQAAAAAAGJnZbJLZbMrwet3cuLcHyGoI+gAAAAAASCdms0m+Pp4yu7lldlMAZAEEfQAAAAAApBOz2SSzm5tCRo1SzKlTGVp33rp1VaZv3wytE0DmIugDAAAAACCdxZw6pejQYxlap2fJkhlaH4DMxwP7AAAAAAAAgAEQ9AEAAAAAAAAGQNAHAAAAAAAAGABBHwAAAAAAAGAABH0AAAAAAACAARD0AQAAAAAAAAZA0AcAAAAAAAAYAEEfAAAAAAAAYAAEfQAAAAAAAIABEPQBAAAAAAAABkDQBwAAAAAAABgAQR8AAAAAAABgAAR9AAAAAAAAgAEQ9AEAAAAAAAAGQNAHAAAAAAAAGABBHwAAAAAAAGAABH0AAAAAAACAARD0AQAAAAAAAAZA0AcAAAAAAAAYAEEfAAAAAAAAYAAEfQAAAAAAAIABEPQBAAAAAAAABkDQBwAAAAAAABhAtsxuQFqKi4tTcHCwNm/erJ07d+rUqVOKjY2Vj4+Pqlatqm7duul///tfsuWDgoI0a9Ys7d+/X7GxsSpatKhat26tfv36KWfOnMmWO3XqlL788ksFBQXp8uXLyps3rxo0aKDXXntNJUqUSIctBQAAAAAAABwZ6o6+Xbt2qVevXpo5c6bOnz+vWrVqqWXLlsqbN682btyowMBAjRgxQlarNUnZ2bNnq3fv3tqyZYsqVKigpk2bKjo6WjNmzFDnzp11+fJlp3Xu3r1bTz/9tJYsWSJvb2+1bNlS3t7eWrJkiZ566int27cvnbcaAAAAAAAAMNgdfSaTSa1bt9YLL7yg2rVrO8xbsWKF3nnnHS1YsEABAQHq0KGDfd7hw4c1btw4ubm56csvv1STJk0kSbGxsXrllVe0bds2ffTRR5o6darDOmNjYzVo0CDFxsYqMDBQb731ln3epEmT9NVXX2nQoEFatWqVPDw80m/DAQAAAAAAkOUZ6o6++vXra+rUqUlCPklq27atOnbsKElaunSpw7yvvvpKVqtVnTp1sod8kuTp6anRo0fLbDZr9erVOnHihEO5xYsXKyIiQqVLl9agQYMc5g0aNEilS5fWuXPnktQHAAAAAAAApDVDBX334u/vL0k6d+6cfdqtW7e0adMmSVK7du2SlClWrJgCAgIkSevWrXOYZ/v3k08+KbPZ8aM0m81q27atJGnt2rVptAUAAAAAAACAc1kq6AsLC5MkFSxY0GFabGysJKlKlSpOy9mmHz582GG67d+ulgMAAAAAAADSmqHG6EvJhQsXtGTJEklSq1at7NPPnDkjSfL29lauXLmcli1SpIjDspIUHR2tqKgoSVLRokVTLHf58mXFxMTIy8vLpTabTC4tDgAAUsB5NePxmQN4GHFsynh85sjq0qIPpHYdWSLoi4+P1+DBg3Xt2jVZLBZ17drVPu/69euSbo/HlxxbQBcdHZ2kXEpl7wz2oqOjXQ768uXL7dLyAADAOV/fnJndhCyHzxzAw4hjU8bjM0dWl9F9IEsEfR9++KG2bdsmHx8fTZ06VdmzZ8/sJqXKpUvXZLVmdisAAEbg5mbO0hfakZHXlZCQmKF18pln/GcO4N44NnE+yGgP2/kgq/89kPHSqg+YTKm7IczwQd/HH3+sRYsWKU+ePJo1a5bKlCnjMD9nztsd3DZOnzMxMTGS5PBor61cSmVt5e4um1pWqwj6AABII5xTMx6fOYCHEcemjMdnjqwuI/uAoV/GMW7cOM2dO1fe3t767rvv7G/dvVOxYsUkSVevXnV4NPdOtrf02paVbgd3Pj4+kqSzZ8+mWM7X19flx3YBAAAAAAAAVxg26JswYYJmzZql3Llz67vvvlPVqlWdLlemTBn7GHsHDx50uoxteuXKlR2m24JDV8sBAAAAAAAAac2QQd/EiRP13XffKXfu3Jo5c6aqVauW7LLZs2dXkyZNJEnLly9PMj88PFx79+6VJLVo0cJhnu3fv//+uxITHZ+3TkxM1IoVKyRJLVu2vP+NAQAAAAAAAFLBcEHf5MmT9c0338jb2/ueIZ9Nv379ZDKZtHjxYm3evNk+PTY2Vu+9954SEhLUunVrlStXzqFcp06dVLBgQYWFhWnKlCkO86ZMmaKwsDAVLlxYHTp0SJNtAwAAAAAAAJJjqJdxrF+/XjNmzJAklSxZUj/++KN+/PHHJMv5+vpqyJAh9n9XrlxZQ4cO1dixY9WvXz899thjypcvn4KDg3XhwgWVKVNGH330UZL1eHp66rPPPtPLL7+sGTNmaMOGDapQoYKOHTum0NBQeXl5acqUKfLw8Ei3bQYAAAAAAAAkgwV9V65csf//wYMHkx07r1ixYg5BnyT16tVLFotFM2fO1IEDBxQTE6OiRYuqU6dO6tevX7Jvza1Vq5aWLVumL774QkFBQVqzZo18fX3VoUMHvfbaaypZsmTabSAAAAAAAACQDEMFfZ06dVKnTp3uu3yDBg3UoEEDl8uVKlVK48ePv+96AQAAAAAAgAdluDH6AAAAAAAAgKyIoA8AAAAAAAAwAII+AAAAAAAAwAAI+gAAAAAAAAADIOgDAAAAAAAADICgDwAAAAAAADAAgj4AAAAAAADAAAj6AAAAAAAAAAMg6AMAAAAAAAAMgKAPAAAAAAAAMACCPgAAAAAAAMAACPoAAAAAAAAAAyDoAwAAAAAAAAyAoA8AAAAAAAAwAII+AAAAAAAAwAAI+gAAAAAAAAADIOgDAAAAAAAADICgDwAAAAAAADAAgj4AAAAAAADAAAj6AAAAAAAAAAMg6AMAAAAAAAAMgKAPAAAAAAAAMIBsmd0AAIAxmc0mmc2mTKk7MdGqxERrptQNAAAAAJmFoA8AkObMZpN8fTxldnPLlPoTExIUGRVL2AcAAAAgSyHoAwCkObPZJLObm0JGjVLMqVMZWrdXqVKq9MEHMptNBH0AAAAAshSCPgBAuok5dUrRoccyuxkAAAAAkCXwMg4AAAAAAADAALijDwAAAEC64eVMAABkHII+AAAAAOmClzMBAJCxCPoAAAAApAtezgQAQMYi6AMAAACQrng5EwAAGYOXcQAAAAAAAAAGQNAHAAAAAAAAGABBHwAAAAAAAGAABH0AAAAAAACAARD0AQAAAAAAAAZA0AcAAAAAAAAYQLbMbgAAZASz2SSz2ZQpdScmWpWYaM2UurMyN7fM+S2LvzcAAACAzELQB8DwzGaTfHy8Mi34SUhIVFRUDOFPBnHPm1eJiVZ5e3tmSv2JCYmK5O8NAAAAIBMQ9AEwPLPZJDc3s97/cYv+jriSoXWXKZhHHz/fSGazieAng2TLlUtms0lrftijyPPRGVq3b6FcatU9gL83AAAAgExB0Acgy/g74oqOhF/O7GYgg0Sej9aF8IwNdgEAAAAgMxH0AQAAAACyhMwYyiWzho8BkDUR9AEAAAAADM2cM58SEhMybQxfAMgoBH0AAAAAAEMze3jLzeymkatHKiwyLEPrrleyngIbBGZonQCyLoI+AAAAZBlms0lmsylT6k5MtPKinkyQWY9N8vd+OIVFhin0QmiG1lnKt1SG1gcgayPoAwAAQJZgNpvk4+OVacFPQkKioqJiCH8yiHvevEpMtGbao5qJCYmK5O8NAMhgBH0AAADIEsxmk9zczHr/xy36OyJj38pdpmAeffx8I5nNJoKfDJItVy6ZzSat+WGPIs9HZ2jdvoVyqVX3AP7eAIAMR9AHAACALOXviCs6En45s5uBDBJ5PloXwjM22AUAILPwnm8AAAAAAADAAAj6AAAAAAAAAAMg6AMAAAAAAAAMgKAPAAAAAAAAMACCPgAAAAAAAMAACPoAAAAAAAAAAyDoAwAAAAAAAAyAoA8AAAAAAAAwAII+AAAAAAAAwAAI+gAAAAAAAAADIOgDAAAAAAAADICgDwAAAAAAADAAgj4AAAAAAADAALJldgMAICtwc8uc31USE61KTLRmSt0AAAAAgIxF0AcA6Shfbg9ZExPk7e2ZKfUnJiQoMiqWsA8AAAAAsgCCPgBIR7k9sstkdtPFxUMVd/Fkhtbtnr+s8ncaJ7PZRNAHAAAAAFkAQV8WYTabZDabMqVuHh0EpLiLJxX3b0hmNwMAAAAAYGAEfVmA2WySj49Xpo0RlpCQqKioGMI+AAAAAACAdETQlwWYzSa5uZn1/o9b9HfElQytu0zBPPr4+UY8OggAAAAAAJDOCPqykL8jruhI+OXMbgYAAAAAAADSAUEfAAAAkEEyaygVxkwGACBrIOgDAAAA0lm+3B6yJibI29szU+pPTEhQZFQsYR8AAAZH0AcAAACks9we2WUyu+ni4qGKu3gyQ+t2z19W+TuNY8xkAACyAII+AAAAIIPEXTypuH9DMrsZAADAoDJnkBAAAAAAAAAAaYo7+gDA4DJj4PfMGmweAAAAALIygj7AgMxmk8xmU6bUzVv9Hh7mnPmUkIkDvwMAAAAAMhZBHzJEZt3dkxVDJ7PZJF8fL5kz6zNPSFRkVEyW+9wfRmYPb7mZ3TRy9UiFRYZlaN31StZTYIPADK0TAAAAALI6gj6kq3y5PWTNxDuKEhMSFBkVmymhU2bdVefmZpbZzaw1P+xR5PnoDK3bt1AuteoewFv9HjJhkWEKvRCaoXWW8i2VofUBAAAAAAj6kM5ye2SXyeymi4uHKu7iyQyt2z1/WeXvNC5TQqfbd9V5yuzmlqH13inyfLQuhF/JtPoBAMDDhTFbAQAwPoI+ZIi4iycV929IZjcjw5jNJpnd3BQyapRiTp3K0Lrz1q2rMn37ZmidAADg4cWYrQAAZB0EfUA6ijl1StGhxzK0Ts+SJTO0PgAA8HBjzFYAALIOgj4AAAAgC2DMVgAAjI9BMwAAAAAAAAADIOgDAAAAAAAADICgDwAAAAAAADAAgj4AAAAAAADAAAj6AAAAAAAAAAPgrbswPDe3jM+zM6NOAAAAAACQtRH0wbDMOfMpITFB3t6emd0UAAAAAACAdEfQB8Mye3jLzeymkatHKiwyLEPrrleyngIbBGZonQAAAAAAIGsj6IPhhUWGKfRCaIbWWcq3VIbWBwAAAAAAwEBiAAAAAAAAgAEQ9AEAAAAAAAAGQNAHAAAAAAAAGABj9AEAAMNzc8v43zYzo04AAABkbQR9AADAsMw58ykhMUHe3p6Z3RQAAAAg3RH0AQAAwzJ7eMvN7KaRq0cqLDIsQ+uuV7KeAhsEZmidAAAAyNoI+gAAgOGFRYYp9EJohtZZyrdUhtYHAAAAMHgMAAAAAAAAYAAEfQAAAAAAAIABEPQBAAAAAAAABkDQBwAAAAAAABgAQR8AAAAAAABgAAR9AAAAAAAAgAEQ9AEAAAAAAAAGQNAHAAAAAAAAGABBHwAAAAAAAGAABH0AAAAAAACAARD0AQAAAAAAAAaQLbMbYCQrV67Ujz/+qCNHjiguLk4lS5ZU+/bt1atXL7m7u2d28wAAAAAAAGBgBH1pZPTo0ZozZ46yZcumevXqycvLS9u3b9fEiRO1ceNGzZw5Ux4eHpndTAAAAAAAABgUQV8aWLdunebMmSMvLy/NmzdPlStXliRdvnxZL774onbv3q0pU6ZoyJAhmdxSAAAAAAAAGBVj9KWBGTNmSJL69etnD/kkKW/evPrwww8lSfPmzdO1a9cypX0AAAAAAAAwPoK+B3T+/HkdOHBAktSuXbsk82vXrq0iRYro1q1b2rRpU0Y3DwAAAAAAAFkEQd8DOnz4sCTJx8dHJUqUcLpMlSpVHJYFAAAAAAAA0hpB3wM6c+aMJKlIkSLJLlO4cGGHZQEAAAAAAIC0xss4HtD169clSZ6enskukzNnTodlU8tslqzW+2/b3SoWzSvP7Bn7Jy9V0FuSlL1wJZnck/+M0kO2/GUkSZb8Fnlky9g3Hpf0KSlJylWhgswZ/LZlr5K3685fzFvZsrtlaN0+BXLa/9/8EP6MQB/IOPSBh7MPSPSDjEQ/oB/ciX5AP3jY0A8yDv3g4ewH9IGMQx9Imz5gMqVyOas1LaOkrGfGjBmaPHmyAgICNH/+fKfLTJ48WTNmzFDDhg313XffZXALAQAAAAAAkBU8hLn6o8V2t15sbGyyy9ju5LMtCwAAAAAAAKQ1gr4HVKxYMUnSuXPnkl3m33//dVgWAAAAAAAASGsEfQ/I399fkhQVFaXTp087XebgwYOSpMqVK2dYuwAAAAAAAJC1EPQ9oMKFC6tq1aqSpOXLlyeZHxwcrHPnzil79uxq0qRJRjcPAAAAAAAAWQRBXxro37+/JOnrr7/WoUOH7NMjIyM1cuRISVKPHj2UO3fuTGkfAAAAAAAAjI+37qaRjz/+WHPnzpW7u7vq1asnLy8vbdu2TVevXlVAQIBmzZoljwx+jTQAAAAAAACyDoK+NLRixQr9+OOPCgkJUXx8vEqWLKn27durV69eyp49e2Y3DwAAAAAAAAZG0AcAAAAAAAAYAGP0AQAAAAAAAAZA0IcH5ufnJz8/vwyp68yZM/Lz81OzZs2SzGvWrJn8/Px05syZJPOOHz+uV199VfXr11elSpXk5+enadOmSZJ69uwpPz8/7dixI93bL0mLFy+Wn5+fhg4dmiH14eGVkX3nYXTt2jWtXLlSw4cPV9u2bVW9enVVrVpVzZs317Bhw3T06NHMbiIMLKOP/cD9GDp0qPz8/LR48eI0Xe+WLVvUt29f1a1bV1WqVFGzZs00YsQI/fvvv8mWsV1nJfffs88+m6ZtBB42KX0PAR5Uct9l0/p6Jat//8gqsmV2A4D0FhMTo379+ik8PFxVqlRRw4YN5ebmpkqVKmV204As7dtvv9WMGTMkSaVLl1bjxo2VkJCgQ4cOafHixfrtt980atQodezYMZNbCgDG8dlnn+nLL7+UJFWuXFnFixfX0aNHtWDBAq1cuVLff/+9/P39ky3funVreXl5JZleokSJNG1nz549tXPnTs2ZM0d169ZN03UDANLfmTNn1Lx5cxUrVkwbNmzI7OZkKQR9eKQUKlRIK1askLu7e6rLHDhwQOHh4apZs6Z++umnJPPHjx+v2NhYFS1aNC2bCuAevLy81Lt3b3Xr1k2lS5e2T4+Li9PEiRM1e/ZsffDBBwoICFCpUqUyr6EwJI79eBS89dZb6tu3rwoWLJgm69u0aZO+/PJLmc1mTZ48WW3atJEkWa1WTZ8+XdOmTdPAgQO1cuXKZF8k9+6776p48eJp0h4AQMq4XsH9IOjDI8Xd3V3lypVzqcy5c+ckySFIuBMHTSBzBAYGOp3u7u6uIUOG6I8//lBYWJh+//13vfrqqxncOhgdx348CgoWLJhmIZ8kzZkzR5L09NNP20M+STKZTHrttde0YcMGHTp0SMuWLdMzzzyTZvUCAO4P1yu4H4zRhzS1evVqPffccwoICFCNGjXUrVs3bdq0yemyx48f19SpU9WtWzc1atRIVapUUd26ddWrVy+tWLHCaRlXxsbYsWOH/Pz8NGTIEEnSkiVLHMaSsUlu3IM7x8U5ffq0Bg8erMcff1xVqlRRixYtNHnyZN26dctp3fHx8Zo9e7bat2+vqlWrql69eho4cGCqxhz7+++/NWLECLVo0UJVq1ZVrVq11L17dy1btszp8ne2Pzg4WP3791e9evVUsWLFNB/TB+nHlb4j3d7Hfv75Z/Xs2VN16tSxj7H04Ycf2sPtO9n6Q8+ePRUbG6tJkyapZcuWqlq1qho2bKjhw4fr/PnzTusKCgrSqFGj9PTTT9vHc2rcuLEGDRqk/fv3Oy0zbdo0+1iYZ8+e1fDhw9WkSRNVrlw5VeNTms1mez9Nacwo4M5j+sKFC9WpUyfVqFFDtWvXVt++fbVv3z6n5VIa8yYmJkafffaZWrVqZR/yYdiwYTp//rzDvn2nxMRELViwQN26dVPt2rVVuXJl1a9fX0899ZRGjRrldPxYGI/ValXdunVVsWJFRUZGOszbv3+/fX/94YcfkpRt3ry5/Pz8dPr0afu05Mbou3M/vHz5skaOHKkmTZqoSpUqatKkiUaNGqWrV68mqePAgQOSpPr16yeZZzKZVK9ePUm3z0lpxdU+ajtf7dy5U5L0wgsvOFy/3flZHDx4UIMGDVLjxo1VpUoVBQQEqHnz5ho4cKDWrVuXZtuAzLN//35NmDBBXbp0sV+HN2jQQP3791dQUJDTMneOhx0TE6NPP/1ULVu2VJUqVfT4449ryJAhyV7zSNLGjRvVo0cP1axZU7Vq1dLzzz+fqv3pypUrmjp1qp5++mnVrFlT1atXV/v27fXFF18oNjY2yfKcN7Ke48eP6/XXX1fdunVVrVo1tWvXTt99950SEhKSLZPc9crly5c1Z84c9e3bV82aNVO1atUUEBCgTp066euvv9bNmzfv2R5Xrpsk175/DB06VM2bN5ckhYeHJxnX9W4HDx7U22+/rf/973+qUqWK6tSpo5dffjnZ70MRERH6+OOP1bp1a1WtWlXVq1dXkyZN9OKLL+q7776757YbHXf0Ic1MnTpVX3zxhWrWrKkmTZro5MmT2rt3rwIDAzVt2jS1bNnSYflZs2Zp0aJFKlu2rCwWi7y9vXXu3Dnt2LFD27Zt019//aVhw4bdd3vy58+vjh076tSpU9qzZ49KliypWrVqubyekJAQjR49Wnny5NFjjz2mK1euaM+ePZoxY4aOHz+u6dOnOyyfmJioN954Q+vWrZO7u7vq1q0rb29v/fXXX3rmmWfUuXPnZOtauXKlhgwZops3b6ps2bJq0qSJrl27pv379+vdd9/V9u3bNXbsWKdlV61apZ9++klly5ZVgwYNdOXKlWQfu8HDxdW+Ex0drVdeeUU7d+6Ul5eXqlSpIl9fX4WGhuqnn37SqlWrNGvWLKdjLMXFxalXr146evSo6tSpI39/f+3evVu//PKLNm/erHnz5iW5+9V28q5QoYICAgKULVs2nTx5UitXrtTatWs1adIktW7d2um2hYWFqWPHjnJ3d1dAQICsVqt8fX1T9bmcOnVKklSgQIFULY+sbezYsfr+++/tX/RDQ0O1efNmBQUF6bPPPkvSj5ITExOjF154QQcOHJCXl5caNmyoHDlyaMuWLdq0aZOaNGnitNx7772nxYsXK0eOHKpVq5by5s2rqKgonTlzRvPmzVP9+vV53DELsIVlq1at0rZt29S2bVv7vDtDiW3btql79+72f58+fVpnzpxR8eLFXRrr7ty5c+rYsaPi4+MVEBCgmzdvas+ePZo3b57++usvzZ8/32G4k5iYGEmSj4+P0/XZjs+HDh1Kts7FixfrypUrio+PV8GCBVWnTh099thj92xravuo7fpty5Ytunjxoho2bOhwHihZsqSk259h3759FRcXp4oVK6pGjRpKTEzU+fPn9ccffyghIUEtWrS4Z7vwcJs0aZJ27Nih8uXLq3LlyvL09NTp06e1ceNGbdy4UcOHD9eLL77otOy1a9fUrVs3nTt3TrVq1VKFChW0b98+LV26VLt27dKyZcuUO3duhzKzZ8+2X2tXq1ZNJUuWVFhYmF577TX17t072XYeP35cffr00blz51SgQAHVqlVL2bJl04EDBzRlyhStWbNGc+fOdaiP80bWEhwcrL59+yomJkYlSpTQ448/rsjISE2ePFl//fWXy+vbsmWLRo8erUKFCqlUqVKqUaOGLl++rL/++kuffvqpNmzYoDlz5iT7fdDV6yZXv3/UqlVLMTExWr16tby8vJL9riBJ33//vcaNG6fExERVqlRJ1apV08WLF7Vjxw5t3bpVAwcO1IABA+zLX7hwQZ07d1ZERISKFi2qRo0aKUeOHIqIiNCRI0d06NAhvfzyyy5/poZiBR6QxWKxWiwWa+3ata379u1zmDd16lSrxWKxtmrVKkm5HTt2WP/5558k00+cOGFt3Lix1WKxWP/66y+HeadPn7ZaLBZr06ZNk5Rr2rSp1WKxWE+fPu0w/ZdffrFaLBbrkCFDnLa/R48eVovFYt2+fbvD9CFDhti3bdKkSdb4+Hj7vKNHj1pr1KhhtVgs1j179jiUmzdvntVisVgbNGhgPX78uH16XFyc9cMPP7Sv8+72HDlyxFqlShVr1apVratXr3aYd+bMGWu7du2sFovFumTJEqftt1gs1nnz5jndRjyc7rfvvPXWW1aLxWINDAy0Xrx40WHerFmz7OXu3Ge3b99ur69ly5bW8PBw+7wbN25YBw4caLVYLNZnn302SX1r1661RkVFOZ3u7+9vrVOnjjU2NtZp+y0Wi/Wdd96x3rx5M3Ufyv+3adMmq8Visfr5+VlDQkJcKousxbafVatWzRoUFOQw75tvvrFaLBZrrVq1kvSV5I79Y8aMsVosFmvbtm2t58+ft0+/s59YLBbr1KlT7fPCw8OtFovF2rhxY2tERESSNh4/ftyhz8HYfvrpJ6vFYrG+//77DtN79uxprVy5srVNmzbW2rVrOxyjkytjuxb55ZdfHKbfeYwdOnSowzH27Nmz1kaNGlktFov1t99+cyhnm57c9cIHH3xgX+/169cd5tmus5z917lzZ2tYWJjTdaZ1H7Xp2bOn1WKxWJctW5Zk3tWrV6179+51Wg6Plj/++MPhWGyzZ88ea0BAgLVy5crWf//912Ge7drfYrFYX3rpJeu1a9fs86KioqxPP/201WKxWGfMmOFQLiQkxFqpUiVrxYoVrStXrnSYt2zZMqufn5/T7yGxsbHWFi1aWC0Wi3Xy5MkO/TEmJsZ+3TZ06FD7dM4bWcuNGzesTZo0sVosFuvo0aMdjv8hISHWunXr2vfZu7/LJncsPH78uNPjXFRUlPWll16yWiwW6zfffJNk/v0ek+/n+0dK391tNm/ebPXz87PWrVvXunPnTod5R44csecCO3bssE+fNm2a1WKxWD/44ANrYmKiQ5lbt24l2a6siEd3kWZef/11Va9e3WFaYGCgcufOrbCwsCS389apU8fpr9Zly5a1j8e1atWq9GtwKlWuXFmDBg2Sm5ubfZrFYtFTTz0lSUkeG/j+++8lSQMGDHAYTzBbtmwaNmxYsncnzZgxQ7du3dKgQYPUqlUrh3nFihXT6NGjJf03vs7d6tWr53B3AB4drvSdEydO6Pfff1fBggU1ceJE5cuXz6Fcr1691KRJE4WFhWnz5s1O63v33XcdxvvIkSOHPvzwQ3l6emrfvn3as2ePw/ItWrRQnjx5kqynRYsWatOmjaKiopw+/ijdvmtkxIgRLt1dev78eb333nuSpGeffVYVK1ZMdVlkXV27dk3yOGKfPn1UpUoVXbt2TT///PM913Hjxg0tXLhQkjRs2DCHsdFy5Mihjz76SJ6enknKXbx4UZLk7+/v9Bhfrlw5xtjJQho0aCDJ8frgxo0b2rt3r2rWrKmmTZvq6tWrOnjwoH2+bVlnj9SmpHDhwkmOsUWKFFGPHj2StEGS/dHcRYsWyWq1Osy7cuWKw3VXdHS0w/wmTZro008/1dq1a7V//36tX79e48ePV9GiRXXgwAH17NlTly5dSratadFH72Sry9ldtrlz51aNGjVcWh8eTk2aNHE6TmXNmjXVvXt3xcXFJftYrZeXl8aOHatcuXLZp+XJk0f9+vWTlLR/zJs3TwkJCWrTpo3DGJaS9NRTTyU7dNCSJUv0zz//qGnTpho0aJBDf/T09NT//d//KV++fPr111915coVSZw3sprVq1fr3LlzKlKkiAYPHuzwvbJixYrq37+/y+ssV66c0+Ncnjx59P7770tK+bu0K8fktPj+kZxp06bJarVq5MiRSe4Otz2CL93unza243+jRo1kMpkcyri7u7t8LjUigj6kmaZNmyaZlj17dnuY52wsjOvXr2vlypWaNGmSPvjgAw0dOlRDhw7VmjVrJN0ery6zNW3aNMkBRJI9xLtzu86fP29/3NAWBN4pR44cSS4cpNuP+9oOinc+5nOnqlWrysvLSyEhIU7HXEjpdmg83FzpO5s2bZLValXjxo0dLlzvVKdOHUnS3r17k8zz9va2j5dxp3z58qlRo0aSZB8X6U7nz5/XwoULNW7cOL333nv2vnrs2DFJyffV+vXrJ3ksJiXR0dHq37+/IiIiVK1aNXvgB9xLx44dnU7v0KGDJOf79d0OHjyomJgY+fr6qmHDhknm582b1x7i3Kls2bLKmTOnNm/erC+//NJhjDVkPSVKlFDx4sV15swZ/fPPP5JuP7J169YtNWjQIEkQaLVatX37dplMJpe/nNSvX99p+OzsGkWS+vbtqxw5cujw4cMaMGCAQkNDdf36de3du1e9e/e2P9or3R4r9U4ffvih2rVrp5IlSypHjhwqXry4OnTooCVLlqhYsWI6f/68ZsyYkWxb06KP3qlatWqSpHfeeUfBwcGKj493qTweHZGRkVq6dKkmTJig999/334NYttnkrsGqVKlitOQsGzZspKS9g/b+pxdw0vJ78O2McSeeOIJp/Nz5sypKlWqKD4+3j5OJueNrMW2bz3xxBMOwynYJLdv3UtCQoK2bdum6dOn66OPPtKwYcM0dOhQ+7E4pe/SrhyTH/T7R3IuX76s/fv3y8PDw+n3IUmqW7euJDnciGA7/k+cOFFr1qzR9evXU11nVsEYfUgzyf3qZDsY3B1ObdiwQcOGDVNUVFSy67z71+TMUKRIEafTbdt15ws5bC8N8PX1Vc6cOZ2WczbWRlRUlH1bkxv/6e7lCxUq5DCtWLFi9yyHh5Mrfcd2Ibho0SItWrQoxfVevnw5ybRixYo5Da6l//bNu19+8fnnn2vGjBmKi4tLtq7k+qor++X169fVp08fHT58WP7+/vr222+VI0eOVJdH1pbcOEbJ7dfO2L70pbTfOpuXK1cujR07VsOGDdNnn32mzz77TAUKFFCNGjXUqFEjtWvXLtlzAoypQYMGWrhwoYKCglSyZEl7qPf444/LYrEoe/bsCgoK0iuvvKLDhw8rKipK/v7+qR7D1MaVaxRJqlChgqZNm6Z33nlH69atc7gTysfHR0OHDtWoUaNkMpnk7e2dqjb4+PjoxRdf1JgxY7Rx48Zkf6BJiz56p7feektHjx7V5s2btXnzZnl4eMjf31916tTRU0895fBUBR5dCxcu1NixYx1C6Lsl9yXf1f5h2wfvta/ezXZt9u677+rdd99Ntp3Sf9dmnDeylnvtW3ny5FHu3Ll17dq1VK8zLCxMAwYMsP/o7kxK36VdOSY/6PeP5Jw5c0ZWq1U3btxQ1apVU1z2zhdcPf300/rzzz/122+/aeDAgXJzc1O5cuVUq1YttW7dmjv6RNCHNHT3L78pOX/+vN58803duHFDffr0Ufv27VW8eHF5eXnJbDZr69atD80Amq5s1/1KTEy0/39qftFx9kuQh4dHmrYJGceVfcy2r1SqVOmej7Te/Thwat35ONeaNWs0bdo0eXl56YMPPlC9evVUsGBBeXh4yGQyadKkSfrqq6+SPAJmk9r9MiYmRoGBgdq7d6/8/Pw0c+ZMp48LA/cruX3UmeTC8JTmtW7dWg0aNND69eu1e/du7dmzR2vXrtXatWs1depUzZw50+lb5mBM9evXtwd93bp107Zt25QnTx5VqVJFZrNZNWvW1J49exQbG3vfj+1K93eN0qRJE61fv16rV6/W0aNHFR8fr/Lly6tt27Y6fPiwJKl06dIuDblgC9Ue5C3prvRR6faLmn755Rft3LlTQUFB2rNnj/bv3689e/boq6++0ltvvWV/RBOPpoMHD2rEiBFyc3PTO++8o2bNmqlIkSLy9PSUyWTSggULNGLEiGT3nYy4hpf+uzZr1KiR8ufPn+Kyd/64y3kDD+L111/XsWPH1LRpU/Xp00flypVTrly55O7urlu3bt0zOLuXO/tVen3/sNVxr5d13M1sNmvixInq37+//vjjD+3Zs0d79uzR/PnzNX/+fDVt2lTTp093eEQ6qyHoQ6bYsGGDbty4oZYtW2rw4MFJ5tsef33U2O6yi4yM1PXr153+EhceHp5kmq+vrzw8PHTjxg29++67yps3b7q3FY8m26/TAQEBGjFihMvlne1/d88rXLiwfdrKlSslSW+++aa6du2apExYWJjLbbhbbGysAgMDtWvXLvn5+Wn27Nku39UCnDlzRpUqVUoy3dl+nRzbMTw1/cSZ3Llzq0OHDvbHXs6dO6dRo0Zp/fr1GjVqlMP4MjC2+vXry2QyaceOHbp06ZJCQkLUsmVLe/DQoEED7dixQ7t27dK2bdvs0zKKt7e3nnnmmSTTg4ODJd2+89AVtqczUroDKS366N1MJpPq1q1rf7Tr5s2bWrx4sf7v//5PkydPVps2bexv6cWjZ9WqVbJarerRo4f69u2bZH5aXIPcqVChQvrnn38UHh6uChUqJJmf3PG/SJEiOnnypLp06eJ0iJ6UcN7IGmzXF2fOnHE6/+rVqy7dzXfixAkdPXpU+fLl0+eff65s2RxjndR8l3blmPyg3z+SY6vDZDJpzJgxLofz5cuXV/ny5SX9NwzG22+/rY0bN2rp0qXq3LlzmrX1UcMYfcgUtoFonT2yaLVa9dtvv2V0k9JE4cKF7eOqLV++PMn8W7duOR0U1c3NzX6BbwtWAGcaN24s6XZY7mysxnu5evWqNmzYkGT65cuXtWXLFkn/jbEhpdxXL126lGQga1fduHFDgYGB2rlzpz3kI+jG/Vi2bFmK0+/cr5NTuXJleXp66vLly0737eSmJ6dIkSJ6/fXXJUkhISGpLodHn6+vrypVqqSoqCh9++23slqtDkGe7f//+OMP7d69W9mzZ1ft2rUzq7mSpGvXrmnRokVyc3PTc88951LZ33//XdJ/4yY542oftT29kJCQkOp25MiRQ88995z8/PyUmJioo0ePprosHj4pXYPcvHnTPqZ3WrG9CCC57yFLly51Ot12bZYW1/CcN4zJtm+tWrXK6VA4ye1bybH1jYIFCyYJ+STp119/vec6XDkm3+/3D9txPLkxVAsVKiQ/Pz9dv37d/j3kftnGuW3Xrp0k+g9BHzKF7RGP1atXKyIiwj49ISFBU6ZMcWkQz4fNiy++KOn2G4ROnDhhn56QkKDx48c7bO+dBgwYIHd3d33yySdasmSJw+O8NqGhoWl+UYNHi7+/v1q3bq1z585pwIABTn8ZjImJ0a+//mp/o9vdxo8f7/B41a1btzRy5EjFxMSoWrVqqlWrln2ebcDqhQsXOoxlc+3aNQ0ZMsSlXx/vdvPmTb3yyivasWMHIR8e2Pz585O8/Xn27Nnav3+/cubMqS5dutxzHZ6envblxo4d69CHbt26pVGjRjkdJ+rw4cNasWKFbty4kWSeLVjn7YlZj+1R3B9++EGS411yVapUkbe3txYtWqQbN26oZs2aGTYEx/79+5M86vjvv//qlVde0YULF9SnTx/7HRI269atc3hLsE10dLRGjx5t38979+6dbL2u9lHbHTDJjT/13Xff6ezZs0mmnzhxwn43C/3u0Wb7vrB06VKHscZu3rypjz76KNm7o+5Xz5495ebmppUrV2rt2rUO837//fdk3+777LPPqlixYlq1apU++eQTp+OiXbhwwf5Wd4nzRlbTpk0bFSpUSGfPntWkSZMcvueFhobqyy+/dGl9pUuXlpubm0JDQ5McVzds2KDZs2ffcx2uHJPv9/tH3rx55e7urosXLyY7Lv+gQYMkScOGDXN6M4LVatVff/2lrVu32qctXbo02XOS7SUiWX38eh7dRaZo2rSpKleurEOHDql169aqU6eOPD09tX//fkVERKhv37765ptvMruZ96V79+76888/tXHjRj399NOqW7eu8uTJo7/++ksXLlzQc889p/nz5ycpV7lyZX3yySf2tyV99tlnKl++vHx9fXXlyhWFhobq33//Vdu2bdWqVatM2DI8LMaMGaOrV69q8+bNatOmjSpWrKjixYvLarUqPDxcR44cUVxcnFasWJFkrJiaNWsqMTFRbdq0Ub169eTh4aHdu3crIiJC+fLl0/jx4x2Wf/HFF7Vs2TJt2rRJLVq0UI0aNRQXF6ddu3bJw8NDnTt31i+//HJf2zFp0iT73VFFixbVhAkTnC5Xq1Ytp4+YAXfq2rWrXnzxRdWuXVuFChVSaGioQkND5ebmpjFjxqhAgQKpWs+bb76pPXv26NChQ2rZsqXq1aunHDlyaPfu3YqLi1PHjh21ZMkSh7FSz549qzfffNP+MoAiRYooPj5eoaGh+vvvv+Xu7u50mAoYW4MGDfTdd9/p5s2bKl68uMMjpGazWXXr1rWHCRn52O5LL70kT09PWSwW+fj4KCIiQnv37lVcXJy6du1q/9J1px07dmjOnDkqWrSoLBaLcufOrYiICB05ckRXrlxRtmzZ9O6776a4Ha720datW2vx4sX65JNPtG3bNuXNm1cmk0mdO3dWQECAvvzyS02YMEFly5ZVuXLllCNHDkVERGjPnj2Kj49Xhw4dVLly5bT++JCBOnXqpDlz5ujw4cNq3ry5ateuLTc3NwUHB+vGjRt64YUXNGfOnDSrr1KlSnrrrbf0ySefaMCAAapevbpKlCihU6dO6cCBA+rVq5fTAMXLy0tfffWVAgMD9e2332rhwoXy8/NToUKFdOPGDYWFhenEiRPKly+fnn32WUmcN7IaDw8PTZw4Uf369dPMmTO1bt06Va1aVVFRUdq5c6eaNm2qQ4cOpTg8yJ3y5s2r7t27a86cOerVq5dq166tggUL6u+//9ahQ4f0yiuv3DM8dPWYfD/fP9zd3dWsWTOtXr1aHTp0UK1atew/ao0ePVqS1KxZM7333nsaP368XnnlFZUqVUplypRRrly5FBkZqSNHjujSpUvq27evGjZsKOn2GOJDhgxRwYIFValSJXl7e+vq1avas2ePrl27JovFkuW/OxD0IVNky5ZNc+fO1ddff63Vq1dr27ZtypUrl2rWrKmpU6fq+vXrj2zQZzab9fnnn2vu3LlatGiRdu7cKS8vL9WqVUvTp0/X4cOHnQZ90u1XrletWlVz5861DyydkJCg/Pnzq2TJkurevbvLY3/AeHLlyqWZM2dqxYoV+vXXX3Xo0CEdOXJEOXPmVMGCBdW+fXs1b97c6bhE7u7u+uqrr/T5559r9erVOn/+vPLkyaNOnTrp9ddfT/KGuhIlSmjJkiX67LPPtHv3bm3cuFEFChTQk08+qYEDBya7L6eG7bEDSdq4cWOKy2b1kzXubfjw4SpTpowWLFigAwcOKFu2bGrUqJFeffVVBQQEpHo9OXPmtJ+ffv/9d23ZskU+Pj5q0KCBBg0apM8//1ySHMaRrF69ut5++20FBwfrxIkTCgkJkZubmwoXLqzu3burR48e9rtjkXXUrl1b2bNn161bt5wGYPXr18+UoO+FF17Qn3/+qUOHDik6Olo+Pj763//+p27dutm/RN2tRYsWiomJ0eHDh3Xw4EFduXJF7u7uKlKkiJ544gk9//zz93xpgKt99H//+58+/vhjzZ8/X9u3b1dsbKyk2z/+2MaJ2rZtmw4ePKhdu3YpJiZGBQoUUIMGDdS1a1c1b978wT8sZCrbXa/Tpk3T1q1btXnzZvn4+Ojxxx/XgAEDtHv37jSvs0+fPipTpoy+++47hYSE6NixY/Lz89PUqVNVuXLlZO+UqlChgn799Vf99NNPWrdunY4ePap9+/bJx8dHhQsX1ksvvaSWLVval+e8kfXUqVNHCxcu1LRp07Rz506tXbtWJUqU0Ouvv66XXnrJ5Rs5hg8fLj8/P/344486ePCg3NzcZLFYNHnyZLVt2/aeQZ+rx+T7/f7xf//3f/Lx8dGWLVu0evVq+6PLtqBPun1eqlevnubNm6cdO3Zo27ZtMpvNyp8/vypVqqT//e9/Dp/PSy+9pOLFi2vv3r32N9f7+PiofPnyateunTp16iQvLy+XPk+jMVldfcUVAOCRs2PHDr3wwguqU6eO5s6dm9nNAdKMLVzIiLG44uLi1K5dO4WFhWnx4sXcLQSkQkb2UQAAwBh9AAAADg4ePJhknNTr169r1KhRCgsLk5+fHyEfAAAAHko8ugsAAHCH119/XbGxsbJYLMqXL58uXbqkI0eO2B8NGTduXGY3EQAAAHCKoA8AAOAOvXr10tq1a3XixAnt2bNHZrNZRYsWVfv27fXyyy8nGcsSAAAAeFgwRh8AAAAAAABgAIzRBwAAAAAAABgAQR8AAAAAAABgAIzRhzRz/vx5tWnTRnXr1tWMGTMc5jVr1kzh4eHJlq1evboWLlyY7Pxbt27pp59+0sqVK3XixAnFxsbK19dXFotFnTp1Utu2bdNsOzKa7bNZv369ihcvnqF1v/fee1qyZImWLFkiPz+/DK3biJLrA4sXL9awYcPuWd5kMunIkSNJpicmJmrhwoX65ZdfdPz4cUlS+fLl1aVLFz377LMymUxptxGZgD7w6EnpeH+3hIQEPf/889q3b58k6YcfflDt2rVTVc8bb7yhVatWSZImTJigp59+OskyJ0+e1J9//qlDhw7p0KFDOnHihBISEvTGG2/o1VdfdW3DHlL0kUdPep0PJCk6Olpz587V2rVrderUKcXFxSlfvnzy9/dX9+7d1aBBgzTbjoxm28eOHj2a4XX36tVL+/fv1+rVq1WgQIEMr/9Rldy+fu3aNW3dulVbtmzRvn37FB4ersTERBUsWFB16tRRr169nB5TYmNjtX37dm3ZskXBwcE6ffq0fR8PCAhQjx49VKtWLadt+fXXX7V161YdOXJEFy5c0NWrV+Xh4aEyZcqoZcuW6tGjh3LmzJlun0VG4Hzw8EmPa6JDhw5p+/bt9mubU6dOyWq1JnstZEMfSF+PUh8g6EOamTBhgm7cuKE333wz2WVat24tLy+vJNNLlCiRbJl///1XL7/8so4fPy5fX18FBATI09NT586dU3BwsLy8vB7poC8zDRw4UL/99ps+/vhjzZ07N7Ob88hLrg+ULFlSHTt2TLbc9u3bde7cOdWtWzfJvISEBA0aNEhr1qyRp6en6tWrJ0natm2bRowYoaCgIE2ePFlmMzdo3w/6wP1JzfHe5rvvvtO+fftkMpnkyrDAK1as0KpVq+5Zbv78+ZozZ06q1wvX0EfuT3qcDyQpNDRUffr00fnz51W4cGHVq1dPbm5uOnfunDZt2qSSJUs+0kFfZnr77bfVpUsXTZo0SWPHjs3s5jwyktvXv/32W3voUbp0aTVu3FgJCQk6dOiQFi9erN9++02jRo1K0h+WL1+u999/X5JUrFgx1a9fX9myZdORI0e0YsUKrVy5Um+88YZeeeWVJG2ZP3++9u7dq3Llysnf318+Pj66ePGi9u3bpwMHDuiXX37R3LlzVahQoXT6NIyN84Fz6XFNNH36dK1fv97lttAH0tej1AcI+pAm9u/fr+XLl6tNmzYpptvvvvuuS8n7jRs31Lt3b508eVIDBw5UYGCg3N3d7fNjY2MVFhb2IE3P0goXLqxnnnlG8+bN0/r169W8efPMbtIjK6U+ULt27WTvYLp586YaNWokSerSpUuS+XPnztWaNWtUqFAh/fDDD/ZQ/PTp03r++ee1atUqPfbYY+rRo0cab1HWQB9wXWqP95J07NgxTZs2TU2bNlVoaGiKd3bf6eLFixo5cqT8/f3l4eGhPXv2JLusxWLRSy+9JH9/f/n7++urr77SsmXLXNomJI8+4rr0Oh9cvHhRvXr10pUrV/TRRx+pW7duDnd0X716VREREWm4JVlL1apV1bRpUy1ZskQvvviiKlasmNlNeuiltK97eXmpd+/e6tatm0qXLm2fHhcXp4kTJ2r27Nn64IMPFBAQoFKlStnnZ8uWTZ07d1aPHj3k7+9vn261WjV79myNGzdOn332mWrVqqU6deo41Dl06FCVKlVKPj4+DtMjIyP12muvaffu3Ro/frwmTZqUdh9CFsL5IKn0uiaqUaOGKlSoYL+2GT58uHbu3HnP9tAH0tej1Ae4BQRp4vvvv5fk/ML0QXz11Vc6efKkunbtqgEDBjiEfJLk6empSpUqpWmdWY3tb2b7G+L+3G8fWLt2ra5cuSJvb2+1atXKYV5iYqK+/fZbSdI777zjcOdriRIl9M4770i63U8SExMfpPlZGn3ANand1+Pj4zVkyBB5eHho5MiRLtXxwQcf6Pr16xo7dqyyZUv5N8lnnnlGQ4YMUfv27VWuXDnubk0H9BHXpMf5QLp918ilS5f0+uuv67nnnksybIO3t7fKly9//w2HunTpIqvVyr6eSint64GBgRo6dKhDyCdJ7u7uGjJkiEqXLq24uDj9/vvvDvM7duyoMWPGOIR80u3H2Xv37q369etLktMfdKpXr54k4JAkX19fvfXWW5KkP//8M9Xbh6Q4HzhKr2uifv366c0331Tr1q1TfPLtbvSB9Peo9AHu6MMDu3jxolavXq2CBQvq8ccfT7P1xsXFaf78+ZKkl19+Oc3W27NnT+3cuVNz5syRt7e3pk+frl27dun69esqWbKkunTpot69ezsd9yw+Pl4///yzli1bpmPHjunWrVsqUqSIGjdurL59+yZ7G/Tx48c1depU7dixQ7GxsfZHd3r16pViW+Pj47VkyRL9+uuvOnr0qGJiYlSwYEE1atRI/fv3V5EiRZKUCQoK0pw5c7R//35duXJFXl5e8vX1VbVq1dS1a1c99thjDstXqlRJFStW1I4dO3TixAmVK1cu9R8mJD1YH/jll18kSe3bt1eOHDkc5u3du1cXLlxQ9uzZ1bp16yRlW7durffee08RERH666+/VLNmzVTVSR+gD9wvV/b1GTNm6NChQxozZoxLj4gsXbpUGzZs0GuvvZZpd9TQR+gj9yu9zgeXLl3SihUr5OHhoe7du6dZe+8c6+jMmTP6+uuvdeDAAd28eVPlypXTiy++qA4dOjgtGxsbq7lz52rlypUKCwtTYmKiihcvrhYtWuill15Snjx5nJbbu3evpk+frn379ikhIUFlypTR888/f88vyjdu3NCPP/6oVatW6eTJk7p586aKFi2q5s2bq2/fvvL19U1SZuXKlVqwYIFCQkIUHR2tXLlyOYz1dvcxpkmTJvL19dXvv/+uIUOGOP3CjNseZF83m83y8/NTWFiY/v33X5fKVqpUSdu2bXO5nJubmyQluWngXjgfcD5ITkZcE6Ul+kDW6gMEfXhgmzZtUlxcnOrVq3fPOykWL16sK1euKD4+3j4Y792dx+bw4cOKjIxUwYIFVapUKR09elRr165VRESEvL29Vbt2bTVu3Pi+797YunWrZs2apZIlS+rxxx/XhQsX7Lcznzt3Tu+9957D8rdu3VJgYKCCgoKUI0cO1a1bV7ly5dLevXs1d+5cLV++XN99950qV67sUC44OFh9+/ZVTEyMSpQooccff1yRkZGaPHmy/vrrr2TbFx0drVdeeUU7d+6Ul5eXqlSpIl9fX4WGhuqnn37SqlWrNGvWLIdfPJcsWWIf5LtatWqqW7eubty4ofPnz2vFihXy9fV1+nk3aNBAR44c0bp16x7ag9XDzJU+cKezZ89q+/btkpz/EhgSEiJJqlChQpIvfZLk4eGhChUq6PDhwzp8+HCqgz4b+sB/6AOpk9p9PSQkRDNmzFDDhg3VuXPnVK///PnzGj16tCwWi/r3758WTX4g9JH/0EdSJ73OBzt27FBcXJyqVKmiXLlyac+ePdq0aZMiIyOVN29eNWjQIMljjK745Zdf9OWXX8rf31+NGjVSeHi49u3bpyFDhigqKirJlyvbtJCQEOXKlUv16tWTu7u7du7cqRkzZmj58uX6/vvvkwzXsnLlSr399ttKSEiQxWKRxWLRuXPn9P7779tfNuXM+fPn1adPH4WGhsrHx0dVq1ZVzpw5dfjwYX333XdatWqV5s6dq2LFitnLfP7555o2bZqyZcummjVrqlChQrp27ZrOnTunRYsWqXz58kmCPnd3d9WpU0erV6/W1q1b1a5du/v+TI3ufvd1m1OnTkmSyy8+uZ9y0dHR+vzzzyXdDrfvB+eD/3A+uC29r4nSEn0gC/YBK/CA3nnnHavFYrHOmzcv2WWaNm1qtVgsTv/r3LmzNSwsLEmZBQsWWC0Wi7VLly7WTz75xOrn55ekbIcOHazh4eEutbdHjx728vPnz3eYFxQUZPXz87NWqlTJeu7cOYd5n3zyidVisVhbtGhhPX36tH36rVu3rMOHD7daLBZrs2bNrDdv3rTPu3HjhrVJkyZWi8ViHT16tDU+Pt4+LyQkxFq3bl17W+5cp9Vqtb711ltWi8ViDQwMtF68eNFh3qxZs6wWi8XaqlUrh3U2a9bMarFYrLt27Uqy3RcvXrQeOnTI6WeyZs0aq8Visb744ovJfGpISWr6gDPTpk2z78fOjB071mqxWKyvvvpqsuvo37+/1WKxWMeNG5fqeukDSdEHUic1+/rNmzet7du3t9asWdPh+Gw7Dzj729i8/PLL1kqVKln3799vn2bbX5cuXZqqNg4ZMsRqsVis06dPT9XyztBHkqKPpE56nQ8mTZpktVgs1gEDBtjruPu/Xr16WaOiolyq19YvK1eubN2wYYPDvF9++cVqsVistWrVssbGxjrMGzRokNVisVifeeYZ6+XLl+3To6OjrX369LFaLBZr165dHcpERERYa9asabVYLNZZs2Y5zAsKCrJWrVrVvi13SkxMtHbr1s1qsVisw4cPt167ds0+Ly4uzjpu3DirxWKx9uzZ0z795s2b1mrVqllr1KhhPXHiRJLtPnPmjPX48eNOPxNb3xk+fLjT+bjtfvd1q9Vq3bRpk9VisVj9/PysISEhqS535MgRq7+/v9VisVjXr1+f7HJbtmyxDhkyxDp48GDrSy+9ZN/vXn75ZevVq1ddaivng6Q4H9yW3tdEd3L1Wog+QB9gIBs8MNtdRyml2U2aNNGnn36qtWvXav/+/Vq/fr3Gjx+vokWL6sCBA+rZs6cuXbrkUCYqKsq+/m+++cb+4oHdu3dr1qxZKl26tA4fPqzAwEDFxcW53O5WrVqpW7duDtPq16+vhg0bKiEhwf7LunR7gOwffvhBkjRs2DCHX6jd3d31/vvvK3/+/Dpz5oxWr15tn7d69WqdO3dORYoU0eDBg+23TEtSxYoVk71j5cSJE/r9999VsGBBTZw4Ufny5XOY36tXLzVp0kRhYWHavHmzffqlS5eUO3dupwN958uXL8l4Jza2MX0OHz7sdD5Slpo+cDer1arFixdLSn5cj+vXr0u6PRZlcmxvsbYt6wr6wH/oA6mTmn19+vTpOnr0qN59910VLVo01eteuHChtmzZopdffllVq1Z94LamBfrIf+gjqZNe54PIyEhJ0saNG/X7779r4MCBWr9+vXbu3KnPP/9cBQoUUFBQkH0MJlf16NFDTZs2dZjWqVMnlS1bVteuXdPBgwft08+ePWt/I/b//d//OTwymzNnTn388cfKkSOH9u7d6/AinUWLFun69euqUaNGkjsE69evr65duzpt25YtW7Rnzx5VqlRJI0eOVK5cuezzsmXLpsGDB8tisWjHjh0KDQ2VdPvOjhs3bqhEiRIqW7ZsknUWK1Ys2b8R+3rq3M++Lt2+O9N258+zzz6b6iEarl+/rnfeeUfx8fFq2LBhinclHT9+XEuWLNGyZcu0detWXb9+Xe3atdO4ceOUO3dul9prw/ngP/SR29LzmuhB0QfoAwR9eGAXL16UpBTHMfnwww/Vrl07lSxZUjly5FDx4sXVoUMHLVmyRMWKFdP58+c1Y8YMhzLW///K8bi4OLVr104jRoxQmTJllCtXLjVo0ECzZs1Sjhw5FBoammQg39S4+4LWxnawvvPNdQcOHFBMTIx8fHycXlh4enqqbdu2km4/XmNjezvSE0884XQ8hI4dOzptw6ZNm2S1WtW4cWOHC9o72R7R2bt3r31a1apVde3aNb377rs6ePBgql/QYPvbXblyRbdu3UpVGfwnNX3gbtu2bVN4eLhy5MiRaY8G0Qf+Qx9InXvt6/v379c333yjevXqJful3Znw8HCNGzdO5cqV08CBA9OiqWmCPvIf+kjqpPf5IC4uTn369NGAAQNUvHhx5cmTRy1bttT06dNlMpm0detWBQcHu9zue+3r58+ft0/btWuXEhMT5e/v7zSkKVSokBo2bCjJ+b7evn17p3WltK9Lt79kOns5j9lstn9Js+3refPmVbFixXT06FGNGzcuxceC72b729n+lnDufvb16Oho9e/fXxEREapWrVqSR/2SExcXpzfeeEOhoaEqUaKEPvnkkxSX79Wrl44ePaqDBw9q7dq1Gjp0qLZs2aInn3xSu3btSnV778T54D+cD25Lr2uitEAfSCqr9QHG6MMDi46OlqRkO1VKfHx89OKLL2rMmDHauHGjwwk/Z86c9v93dnAsWrSo/ve//2n16tXatm1bsoNFJ8fZYJzSf9tx8+ZN+zTbgevOsV/uVrJkSUmOF8O2gYLvHqPGJk+ePMqdO7euXbvmMP306dOSbv/6vWjRohS34/Lly/b//+ijjxQYGKhly5Zp2bJlypkzp6pWrap69erp6aefTvaXpDv/dteuXUvyKwhSdj99wDboesuWLZMdsNzWB2JjY5NdT0xMjMOyrqAP/Ic+kDop7es3b97U0KFDlSNHDn388cdOB2V2xmq1avjw4YqNjdWYMWOUPXv2NG3zg6CP/Ic+kjrpfT6QnF8TVa9eXf7+/jp06JCCgoKc3p2Qknv93e/c1237b3L7rHR/+3py0237+pQpUzRlypRk65Qc9/UJEybo9ddf16xZszRr1iz5+PioWrVqevzxx/XUU08pb968Ttdh2+arV6+mWFdW5+q+fv36dfXp00eHDx+Wv7+/vv32W6fjD98tPj5eb731lrZs2aJixYrp+++/T/Zvdzd3d3eVLFlSvXv3VkBAgLp27arBgwdr1apV8vDwSNU6bDgf/IfzwW3pcU2U1ugDSWWVPkDQhweWO3duXb582X6wc5XtV4C7355156vEk3utuO0gcOHCBZfrvd+XeGQE268Jtrf6pKR69er2/y9XrpxWrVqlP//8U9u3b9fevXu1e/dubd++XdOnT9fo0aP19NNPJ1nHnQdLb2/vNNqKrMPVPnD16lWtXbtWUvKPaUn/nRzPnTuX7DL3OiGmhD7wH/pA6qS0r588eVInTpyQr6+vhg8fnmS+7Tj98ccfK3fu3GrUqJH69euna9euafv27fLy8tKnn36apJzt0ZgZM2Zo0aJFqlixYqrvAnlQ9JH/0EdSJ73OB7ZjfLZs2ZL9slWiRAkdOnTovq6JMutLaGrY9vVatWrZvxgmp0KFCvb/r127tjZs2KA//vhDu3bt0t69e7V161Zt3rxZU6dO1fTp01W/fv0k67Dt6+znKXNlX4+JiVFgYKD27t0rPz8/zZw5M9lQ+04JCQl65513tGbNGhUpUkTff/99isFBSqpXr67y5cvr2LFjOnjwoMthOOeD/3A+uC09ronSE33gtqzSBwj68MDy5cuny5cv28fUc5Wt3N13JPn7+8tkMslqtSoyMtLpha1tzBrbOGXppWDBgpJuP16WHNsvCXe+Jtz2/2fOnHFa5urVq0l+kZD++8UkICBAI0aMcKmt2bJlU5MmTdSkSRNJt39tmjVrlj7//HN9+OGHatmyZZLPy/Y3yJMnj8uvXIfrfeC3337TzZs3Vbx4cdWrVy/Z5WzjQhw7dkw3b95M8sv3jRs3dOzYMYdl0wt9AFLq9vXIyEj7oxfO2IK7u7+sxcTEpFju5MmTOnnypGsNzkD0EUjpdz6oUqWKpNt3N0VHRzv9YpFR10S2fda2PzuT3L5+8uTJZPtIctNt+3rz5s318ssvu9RWDw8PtWnTRm3atJF0+06Ozz77TAsWLNDw4cO1cePGJGVsf7v8+fO7VFdWk9p9PTY2VoGBgdq1a5f8/Pw0e/Zsh3Edk5OQkKDBgwdr5cqVKlKkiObMmZPsD/+pZRvz+O5xwdMa54OsIT2vidILfSApo/aBhzeWxSPDFjCcOHHivsrbxterVq2aw/QCBQqoVq1akqSgoKAk5eLi4uxjDNxdNq1VrVpVXl5eioqK0vr165PMv3HjhlasWCFJqlu3rn267XXcq1atcvrCkKVLlzqtr3HjxpKkDRs2ONwGfT9y5cqlgQMHytvbW7GxsQoLC0uyjC0suvv15kgdV/uA7TGtTp06pXgXRc2aNVWgQAHdunXLYaBam9WrVysuLk4FCxZ0+HUqPdAHIKW8r1eqVElHjx5N9j/bRewPP/xgHzdLuv1LaErlbGOqTJgwQUePHtXcuXMzaGtdQx+BlH7ng2rVqtkfIfrzzz+TzI+KitKhQ4fsy6anxx57TGazWSEhITpy5EiS+REREdqyZYsk5/v6b7/95nS999rXV61aZR+/+X7lzZtXgwcPlnT7pSJXrlxJsgz7euqkZl+/ceOGAgMDtXPnTnvIl5rHbhMTE/Xuu+/q999/t4d897qb814uX75s319Lly79QOu6F84HWUN6XBOlJ/pA6hilDxD04YHZOuedA1vead26dQ5va7OJjo7W6NGjtWHDBklS7969kywzYMAASdLXX3+tffv22afHx8dr/PjxOn36tHLmzKlOnTo96GakKEeOHOrevbskafz48Q6/TsTFxWn06NG6cOGCihcvrtatW9vntWnTRoUKFdLZs2c1adIkhwE+Q0ND9eWXXzqtz9/fX61bt9a5c+c0YMAAp79qxMTE6Ndff7UPBBsbG6tZs2Y5jDtgExwcrKtXr8rNzU2FCxdOMt/2t0vpbgIk71594E5HjhzRoUOHZDab77nfms1m9enTR5I0ceJEh7snTp8+bX/MMTAwMN1vp6cPQHJtX89q6COQ0u98YDKZ9Nprr0mSPvnkE4e7W2NjYzVixAhFR0eraNGiatGixQNswb0VLVpUbdq0kdVq1YgRI+x3Ekq397kRI0bo5s2bqlmzpgICAuzzunTpIi8vL+3du1dz5sxxWOeOHTv0008/Oa2vefPmqlq1qvbv369hw4Y53X+vXLmi+fPnKz4+XtLtu0h+/vlnp4/U2a478+TJ43RsLfb11LnXvn7z5k298sor2rFjh8sh37Bhw7R8+XKXQr7jx4/r119/dfol/++//9Ybb7yhW7duqUaNGvLz87vn+h4E54Os4WG7JqIP0AfuxKO7eGBNmjSRu7u7tm/froSEBIfXYEu3L97mzJmjokWLymKxKHfu3IqIiNCRI0d05coVZcuWTe+++64aNGiQZN3169fXG2+8oSlTpqh79+6qWrWqChQooEOHDik8PFweHh6aNGlShjxe8frrr+vgwYPatm2b2rZtq7p16ypnzpzat2+fzp49Kx8fH02ZMsVhIHkPDw9NnDhR/fr108yZM7Vu3TpVrVpVUVFR2rlzp5o2bWrflruNGTNGV69e1ebNm9WmTRtVrFhRxYsXl9VqVXh4uI4cOaK4uDitWLFC+fPnV1xcnMaNG6cJEybIYrGoVKlScnd3V3h4uD0k7d+/v9OLLNsdk82bN0+fD8/g7tUH7mQbIPbxxx9PdpylO/Xs2VPBwcFau3at2rdvbx9PaNu2bYqNjVXr1q31/PPPp82G3AN9AK7s6xnl0KFDGjlypP3f//zzjyRpwYIF+uOPP+zTP//8c/ujJOmFPoL0PB906dJF+/bt088//6wOHTqoevXqyp07t/bv368LFy7Y96/UvODgQY0YMUInT57UX3/9pZYtW6pu3bpyc3PTrl27dPnyZRUvXlwTJ050KFOoUCF9/PHHGjx4sEaPHq2ff/5ZFotF58+fV3BwsF588UXNnj07SV1ms1nTp09XYGCglixZotWrV8vPz09FixZVXFycTp8+rdDQUCUkJKhTp07Kli2brl69qvfff18jR4609wtJOnXqlA4fPiyTyaTBgwcn+fvYnhbJkSOH/c3BcO5e+/qkSZPsx42iRYtqwoQJTtdTq1YtPfPMM/Z/z5s3z37XTokSJfTFF184LVe2bFmHMc0uXbqkwYMH68MPP1SlSpVUuHBhxcXF6ezZszp8+LASExNVrlw5TZ48+UE2O9U4Hxhfel4T/fHHHw77vu3N4Z9//rl++OEH+/SFCxfa/58+QB+4E0EfHlj+/PnVunVrLV++XFu3brU/327TokULxcTE6PDhwzp48KCuXLkid3d3FSlSRE888YSef/75FH9VePXVV1WtWjV9//332r9/vw4ePKj8+fOrU6dO6tOnj/1lHukte/bs+vbbb7Vw4UItW7ZMwcHBunXrlooUKaKePXuqb9++DmMM2NSpU0cLFy7UtGnTtHPnTq1du1YlSpTQ66+/rpdeekmtWrVyWl+uXLk0c+ZMrVixQr/++qsOHTqkI0eOKGfOnCpYsKDat2+v5s2b23/l9PLy0siRI7Vr1y4dPnxYQUFB9sc6W7Vqpeeee87poNOHDx/W0aNHVbduXZUvXz5tP7Qs4l59wObWrVv2R5Y6d+6cqnW7ublp6tSpWrhwoX7++Wdt375dklS+fHl16dJFXbt2zbBB1OkDSO2+npGio6P1119/JZn+77//Orzk6datW+neFvoI0vN8IN0euL1+/fr66aefFBISohs3bqhIkSLq0aOH+vbt6/TOg/Tg6+urn376SXPnztWKFSv0559/KjExUcWLF9ezzz6rl156yenLFp588kkVKlRIX375pfbt26fTp0+rTJkyGjlypLp27eo06JNuh4QLFy7U4sWLtWLFCh09elQHDhxQnjx5VLBgQXXr1k3NmjWzh5wlSpTQ8OHDtWvXLh07dkybNm2SdHvcqA4dOqhnz572cQ/v9McffygyMlKdOnWSj49Pmn1eRnSvff3Ox6KdjYV4pzuDvjvLpTS2WZ06dRyCvgoVKujNN99UcHCwTp48qZCQEMXFxcnHx0f169dXy5Yt1blz5wx7szvnA+NLz2uiy5cvO722+eeff+w/aN6NPkAfuJPJ+qCDXQCS9u/fr2eeeUatWrXStGnTMrs5cMGoUaM0b948ffHFFw/1rxIPO/rAo4s+4Br29ayHPuIa+sijq3///vrjjz+0ZMkSVapUKbOb89BjX896OB84og9kPY9KH2CMPqSJatWqqV27dlq7dq3TgZnxcDp37px+/vln1alT56E+UD0K6AOPJvqA69jXsxb6iOvoI4+m/fv3a+PGjerYsSMhXyqxr2ctnA+Sog9kLY9SHyDoQ5p599135enpmWHP/ePBff7554qPj9d7772X2U0xBPrAo4c+cH/Y17MO+sj9oY88eiZNmqScOXPqrbfeyuymPFLY17MOzgfO0QeyjkepD/DoLgAAAAAAAGAA3NEHAAAAAAAAGABBHwAAAAAAAGAABH0AAAAAAACAARD0AQAAAAAAAAZA0AcAAAAAAAAYAEEfAAAAAAAAYADZMrsBAAAAyBzNmjVTeHi4/d8mk0menp7KnTu3SpUqpSpVquiJJ55QtWrVMrGVAAAASC3u6AMAAMjiAgIC1LFjR3Xo0EFNmjRRmTJldPToUc2cOVPPPPOMevbsqdOnT6dJXWfOnJGfn5+aNWuWJuvLaNOmTZOfn5+mTZuW2U0BAABIgjv6AAAAsrhnnnlGnTp1cphmtVq1efNmjRkzRjt37lS3bt30008/qUSJEpnUSgAAANwLd/QBAAAgCZPJpCZNmujnn39W6dKldfHiRb3//vuZ3SwAAACkwGS1Wq2Z3QgAAABkPNsYfWPHjk1yR9+dNm3apH79+kmSfvnlF1WpUkWSdPz4ca1YsUJBQUEKDw9XZGSkcubMqUqVKunZZ59V27ZtHdYzdOhQLVmyJNl6jh49KkmKjo7WihUrtHnzZoWGhioiIkKSVKJECTVr1kwvv/yyvL29k5SPiIjQ119/rS1btujs2bMym83y8fFR6dKl1bhxY7388stJypw/f14zZ87U5s2b7WXKli2rjh07qlu3bsqW7b8HYPz8/JJte8eOHTVu3Lhk5wMAAGQEHt0FAABAiho3biwfHx9FRUUpKCjIHvTNmjVLixYtUtmyZWWxWOTt7a1z585px44d2rZtm/766y8NGzbMvp5atWopJiZGq1evlpeXl1q3bu20viNHjuiDDz5Q3rx5VaZMGVWuXFlXr17VwYMHNWPGDK1cuVILFiyQr6+vvcyFCxfUuXNnRUREqGjRomrUqJFy5MihiIgIHTlyRIcOHUoS9O3atUuvvfaarly5omLFiqlBgwa6deuWDhw4oFGjRmnjxo2aMWOG3N3dJd0O80JCQnTkyBFVrFhRlSpVctg2AACAzEbQBwAAgBSZTCb5+/srKChIx44ds09/+umn1b9//yTj9p08eVK9e/fW7Nmz9eSTT9rf2vvMM8+ofv36Wr16tXx9fZO9A6548eKaPXu26tatK7P5v5FmYmNj9dFHH2np0qWaOnWqPvzwQ/u8BQsWKCIiQl27dtXIkSNlMpns8+Li4hQcHOxQx4ULFzRgwABdvXpVH374obp162avKzIyUoMGDdLWrVv11VdfacCAAZKkcePGadq0aTpy5IhatGihgQMH3s/HCQAAkG4Yow8AAAD3ZLt7Lioqyj6tTp06Tl/OUbZsWb366quSpFWrVrlcV+HChVW/fn2HkE+SPD099dFHHylbtmxJ1nvp0iVJUqNGjRxCPklyd3dX/fr1HaZ9//33ioqKUvfu3fX888871OXr66sJEybI3d1dP/zwgxjpBgAAPCq4ow8AAAD3lJiYKElJQrTr169r8+bNCgkJUWRkpOLi4iTdvmNOkv7+++/7rnPPnj0KDg7WuXPndOPGDXvg5u7ursuXL+vKlSvKkyePJKlatWr68ccfNXHiRFmtVj3++OPKmTNnsuvetGmTJOmJJ55wOr9QoUIqVaqUjh8/rrCwMJUpU+a+twMAACCjEPQBAADgniIjIyXJHqxJ0oYNGzRs2DCHu/zuFh0d7XJdly5d0sCBA7V79+4Ul4uOjra35+mnn9aff/6p3377TQMHDpSbm5vKlSunWrVqqXXr1knu6Dt9+rQkqXv37vdsz+XLlwn6AADAI4GgDwAAACmyWq0KCQmRJFksFkm331b75ptv6saNG+rTp4/at2+v4sWLy8vLS2azWVu3bnX6ltvUeO+997R7927VrFlTAwcOVMWKFeXt7W1/KUbDhg114cIFh0dqzWazJk6cqP79++uPP/7Qnj17tGfPHs2fP1/z589X06ZNNX36dLm5uUn67w7F1q1by8vLK8X2+Pj43Nd2AAAAZDSCPgAAAKRo06ZNunLliqTbIZt0+26+GzduqGXLlho8eHCSMqdOnbqvumJiYrR582aZzWZ9/fXX8vb2TjL/4sWLyZYvX768ypcvL+l2QLl9+3a9/fbb2rhxo5YuXarOnTtLkooUKaKwsDD17dtXVatWva+2AgAAPGx4GQcAAACSde3aNY0dO1aS9Pjjj6tSpUqSZA/+ihYtmqSM1WrVb7/95nR9trvy4uPjk60vISFBuXLlShLySdKvv/6a6pdjmEwm1a9fX+3atZMk+12J0u2XdkjSypUrU7Wu1LYfAAAgMxH0AQAAIAmr1apNmzapS5cuCgsLU4ECBTRq1Cj7/HLlykmSVq9erYiICPv0hIQETZkyRXv37nW63rx588rd3V0XL150OrZf/vz5lSdPHl29elVLly51mLdv3z5NmjTJ6XqXLl2qgwcPJpkeHR2tnTt3SpKKFStmn96nTx95e3tr9uzZmjlzpm7dupWk7OnTp7Vs2TKHaYULF5YkHT9+3Gk7AAAAMpPJmtqfRAEAAGAozZo1U3h4uAICAlSqVClJ0q1btxQZGanDhw/bg7g6depozJgxKlGihL1sfHy8nn32WR06dEheXl6qU6eOPD09tX//fkVERKhXr1765ptvVKdOHc2dO9eh3tdff12rV69WkSJFVKtWLXl4eEiSRo8eLUmaPXu2/S7C6tWrq0SJEjp79qz27t2rp556SsHBwQoPD9f69etVvHhxSdKrr76q9evXq2DBgqpUqZK8vb119epV7dmzR9euXZPFYtH8+fOVK1cuezt27dqlgQMHKjIyUvny5VOFChVUoEABRUdH68SJE/rnn39UvXp1LVy40F7m4sWLatmypWJiYhQQEKDSpUvLbDYrICDA/lgwAABAZiHoAwAAyKJsQd+dvLy8lCtXLpUuXVpVqlTRE088oWrVqjktf/36dX399ddavXq1zp49q1y5cqlmzZp65ZVXdP36db3wwgtOg76oqChNmjRJW7Zs0YULFxQXFydJOnr0qH2ZdevW6dtvv9WJEycUHx+vsmXLqnPnznruuefUvHnzJEFfcHCw1qxZo7179+rcuXOKioqSj4+Pihcvrnbt2qlTp05OX7px6dIlzZs3T5s2bVJYWJhu3bqlfPnyqUiRInr88cfVqlUr+fn5OZQJDg7W9OnTdejQIV27dk2JiYnq2LGjxo0b5/ofAQAAIA0R9AEAAAAAAAAGwBh9AAAAAAAAgAEQ9AEAAAAAAAAGQNAHAAAAAAAAGABBHwAAAAAAAGAABH0AAAAAAACAARD0AQAAAAAAAAZA0AcAAAAAAAAYAEEfAAAAAAAAYAAEfQAAAAAAAIABEPQBAAAAAAAABkDQBwAAAAAAABgAQR8AAAAAAABgAP8PjzsYre2wWRwAAAAASUVORK5CYII=" + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "
    " + ], + "image/png": "iVBORw0KGgoAAAANSUhEUgAABOUAAAMzCAYAAADpn6Y7AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAADhm0lEQVR4nOzdd3yNd//H8fc5J0ISIYuITe0tZlFK1dahtG5FjaJVdO8q2hp3d9EWtWrVLmqW2ruIvUeMLEkkRhKJ5JzfH345t8gJScQ5kbyej0cfj+S6vuNzncHtfX+v62uwWCwWAQAAAAAAALAbo6MLAAAAAAAAAHIbQjkAAAAAAADAzgjlAAAAAAAAADsjlAMAAAAAAADsjFAOAAAAAAAAsDNCOQAAAAAAAMDOCOUAAAAAAAAAOyOUAwAAAAAAAOyMUA4AAAAAAACwMydHFwAAgL0sXrxYH3/8cbra1q9fXzNnzkxxLCwsTAcOHNDBgwd18OBBHTlyRDdu3JAkjR49Wp06dcqyWjdu3Khly5bp0KFDCg8PV1JSkry8vOTt7a0yZcqodu3aatq0qUqWLJllc+Z0x48f16pVq7Rjxw4FBwcrOjpazs7O8vHxUZUqVdSsWTO1atVKbm5uji411/joo4/0559/2vy+5XQtWrRQUFCQBg0apMGDBzu6HJvGjRun8ePHpzqeL18+ubu7y8PDQxUrVlT16tXVtm1b+fr6OqBKAAAeXYRyAACk03/+8x8FBQU91Dlu3LihN998U1u3bk11LjQ0VKGhoTpy5IiWL1+u8uXLa/ny5Q+1npwgOjpaI0aM0KpVq2SxWFKcu3XrlmJiYnT+/HmtWrVKo0eP1uDBg9WjRw8HVQtkfzdv3tTNmzcVHh6uU6dOafny5frmm2/UsmVLffbZZypUqNBDm/tRCDMfVG64RgDAbYRyAIBcadKkSapbt26a500mU5rnChQooGrVqqlgwYJatWpVltZ1ZyDXqFEjdevWTWXLlpWPj4+uX7+ukydPaufOnVqzZk2WzptTXbp0SX379lVgYKAkqUaNGurcubP8/f3l7e2thIQEBQcHa8uWLVqyZImCg4M1bdo0QjngLitWrJCfn58kKSkpSdeuXVNwcLD27dunP//8U4GBgVq9erV27dqliRMnqmbNmg6uGACA7I9QDgCQK+XLly/Dtyl+9tlnKl26tMqUKSODwaBdu3ZlaSi3a9cuayDXo0cPffbZZynOFyxYUMWLF1eLFi304YcfKiAgIMvmzokSEhI0ePBgBQYGymAw6OOPP9Yrr7ySql2RIkXk7++v119/XdOnT9fChQsdUG3uNGbMGI0ZM8bRZSAd7v4zs0CBAipevLjq16+v/v37a9q0afruu+8UFRWlgQMHatGiRSpSpIgDKwYAIPtjowcAANKpRYsWKlu2rAwGw0MZf/v27daf+/Xrd8+2JpPpniv9IE2ePFlHjx6VJA0ePNhmIHcnZ2dn9e/fXz///LM9ygNyDKPRqL59++rdd9+VJEVERNh8Fh0AAEiJlXIAAGQTV65csf6cFZsNXLp0SbNnz9aOHTsUFBSkmzdvqnDhwipdurSeeuoptWnTRl5eXqn6xcbGavbs2Vq3bp3OnTun2NhYeXp6qnbt2urSpYueeOKJNOd76qmnJEkzZsxQrVq1NGPGDK1evVoXLlzQtWvXUm2IER8frwULFmjt2rU6efKkrl+/Lnd3d1WtWlWdOnVS27ZtMxWCxsfHWzcOKFasmAYMGJDuvuXLl7d53GKxaMWKFVq2bJmOHDmiq1evKn/+/KpYsaLatWunF154QU5Otv+n1d3PiPr77781Z84cHT9+XPHx8SpdurS6du2qF1980Xq94eHhmj59uv755x+FhITIxcVFDRo00JtvvqmyZcuma56lS5dqwYIFOnXqlG7evKmSJUuqXbt26t27t/Lly2dzjBs3bmjbtm3asGGDDhw4oJCQECUmJsrDw0NVq1bVs88+e8/3pUePHtq9e7eef/55jRkzRlu2bNEff/yhQ4cOKTIyUnXq1LG+N/fb6OHKlSuaMWOGNm/erPPnz+vmzZsqWLCgvLy8VLlyZTVp0kTt27e3+bpn1ee4Tp06mj17tpYuXapz585JksqVK6fOnTuneL8eVEbeqyVLlujDDz+UJK1cuVKPPfZYmuMGBwfrqaeektls1ueff66XX345S+q1pU+fPpo/f74CAwO1ZMkSvfPOO6n+jAkLC9OGDRu0adMmHT9+XBERETIajfLx8VGdOnX08ssv27z1Nfmzkmz8+PGpgr+7n8F24sQJbdiwQdu2bdOZM2d09epV5c2bV8WKFVPjxo31yiuvWG/JtSX5M/TPP//o7NmziomJkbu7u7y8vFSuXDk1adJEHTp0kKurq83+u3bt0sKFC7V3715FRETIyclJJUqUUIsWLdSrVy8VLFjwga8RAPBoI5QDACCbKFCggPXnHTt26Omnn870WL///ru++eYb3bp1K8XxS5cu6dKlS9q6dasiIyNT/ePu9OnT6tevn4KDg1Mcv3z5stasWaM1a9bo2Wef1ciRI5UnT54057969aq6dOmiEydOpNnm1KlTeu2113Tp0qUUx69cuaItW7Zoy5YtWrZsmX744Qe5uLik99IlSbt377aGnJ06dUozLEuvGzdu6I033tDOnTtTHI+KitLOnTu1c+dOzZ07VxMnTlThwoXvOdbw4cP1xx9/pDh29OhRff755zp8+LC+/PJLHTt2TP369VN4eLi1zc2bN7V69Wpt27ZNs2fPVsWKFe85z93/wJekkydP6uTJk1qxYoV+//13eXt7p+r34Ycfat26damOh4eHa+PGjdq4caP++usv/fTTT3J2dr5nDT/88IMmTJhwzzZpOX36tHr27KnIyMgUxyMjIxUZGalTp05p2bJleuKJJ1IFP1n1OY6Li1PPnj21d+/eFMeTd2A+ePCgRo4cmanru1NG36s2bdroq6++0vXr17Vo0SJ98MEHaY69ePFimc1m5c2bVx07dnzgWu/FYDDohRde0Hfffadbt25p9+7datOmTYo2HTp00LVr11L1Tf6zadmyZXr77bczFKTbcvz4cT377LOpjicmJlpf2/nz52vcuHFq3LhxqnaXL19Wjx49rM+jTBYVFaWoqCidOXNGa9asUeXKlVW9evUUbRISEvTZZ59p6dKlKY7Hx8fr+PHjOn78uObPn68JEyak6gsAyF0I5QAAyCYef/xxTZ48WdLt59dFRUWpbdu2cnd3z9A4M2fO1KhRoyRJpUuXVv/+/VW/fn0VKFBA0dHROnjwoFavXi2jMeVTLKKjo9WnTx+FhYXJ2dlZAwYMUPv27eXh4aEzZ87o119/1datW7V06VK5u7tr6NChadYwcuRIRUVFafDgwWrbtq28vLx08eJF64qS0NBQ9ejRQ1FRUdaVbA0aNJCnp6ciIiK0atUqTZw4URs2bNDw4cP13//+N0OvwZ49e6w/16tXL0N9bXn77betgdyzzz6rHj16qESJEgoLC9PChQs1c+ZMHT16VK+99prmzZuXZtCzdOlSXbx4UV27dlXXrl3l5+en0NBQ/fjjj9qwYYPmz5+vpk2bavTo0XJxcdH333+vevXqyWg0atOmTdYgxlawd6clS5bo0qVLatu2rfr06WOtde7cufrjjz906tQpDRkyRLNmzUq10svb21s9evRQgwYNVKxYMRUqVEhms1mhoaFauXKl5s6dq/Xr12vcuHHW2xVt2b59u8LCwvTkk0/q1Vdf1WOPPaaYmJhUIUdaPv/8c0VGRsrFxUVDhgxRs2bN5O3traSkJAUFBWnv3r3666+/UvXLys/xV199pcjISL3zzjtq1aqVvLy8dO7cOX3zzTfas2ePFi5cqLZt26pJkybpuiZbMvNe5cuXTx06dNAff/yhZcuW6Z133rEZPFssFmvY9/TTT6cI/h8Wf39/68/79+9PFco99thjatKkiWrVqiVfX195e3srNjZWgYGBmj9/vtasWaPvv/9elStXVtOmTa39vvjiCw0dOlQdOnRQcHCwBgwYkCq4u/t717BhQ7Vo0UJVqlRRoUKF5OHhoStXrujIkSOaPn26Dh8+rLffflsrVqxItWPsN998o8DAQJlMJg0YMECtW7e2Bu5hYWEKCAjQqlWrbK6U/PDDD7Vy5Uo5OTmpW7du6tixo0qUKKFbt25p7969Gjt2rM6ePavXXntNS5Yssc6dmWsEADziLAAA5BKLFi2yVKhQwVKhQgXLxo0bLTdu3LD5X0xMTLrG27lzp3W8RYsWZUmNffv2tY5ZoUIFS5UqVSzPPfec5dNPP7XMmzfPcu7cuXv2DwkJsVStWtVSoUIFS5cuXSzXr19Ps+2tW7dS/D5y5EjrvGvXrk3VPikpyTJw4EBrmxMnTqQ4f/HixRS1b9y4Mc25X3/9dUuFChUs7du3t0RHR9tss3HjRutYhw4dutdlp/LOO+9Y+4aHh2eo793Wrl1rHWvUqFE220ydOtXaZubMmanON2/e3Hr+119/TXU+Pj7e0qJFC+t73qxZM0tkZGSqdnPnzrWOc/bs2XvO89FHH9ms9YcffrC2Wbly5f0uP5Xk96VWrVo2P1/du3e3jv/WW29ZzGZzmmN9+OGHlgoVKli6d++e4vj169etY/z+++8Zqi8rP8eVK1e27N69O9UYMTExlsaNG1uvMTMe9L06fPiw9dw///xjs/+OHTusbbZv357hGseOHWvtf/HixXT1CQ8Pt/Z55513Mjzn119/balQoYLl5Zdftnk++XUbO3Zshse+061btyxdu3a1VKhQwfLTTz+lOl+/fv17fufTkvznRaVKlSybNm2y2ebatWuWVq1aWSpUqGD54osvUp3PqmsEAGR/bPQAAMiV+vfvL39/f5v/3bk6w97Gjx+v7t27W1e9JCYm6ujRo1qwYIGGDh2q1q1bq1OnTlq/fr3N/nPmzNGtW7dkMBg0ZswY5c+fP8257lxZk5SUZF1R8+STT6ply5ap2huNRn3++efWfvPnz09z7CeeeELNmjWzee7ixYvW+j/55JNUz1VK1qxZM9WvX1+SbK6IuperV69af37Q1UHJu7F6enrqnXfesdmmV69eKleunKR7vy5+fn42N/FwdnZWq1atJN1+z9944w2bz/tr3769dWXOgQMH0pwnb9681meO3W3gwIHWlTmLFi1Kc4y0NGvWTF5eXoqNjdX+/fvTbGcymfTRRx9l6plrSUlJ1p/vdzvw3f2y8nPctm1bmystXV1drSvADh06lO76bMnse1W1alVVrVrV5rlkixcvliQVL15cDRs2fKA60+vO79ud38P0ev755yVJAQEBiouLy7K67ubk5KQOHTpISrnJTrLExERJGfv8SbcfHSDd/q6m9XeJu7u7XnvtNUnS8uXLZbFYMjQHACDn4PZVAACykXz58mno0KHq37+/Vq5cqR07dujAgQOKjo62tjly5Ihef/11vfrqq3r//fdT9E/+x2WtWrXS3AzAlpMnT1qf89S2bds02/n6+srf31+7d+9O9ZytOz355JNpntu5c6csFovy5s2rGjVqKCYmJs22lStX1u7dux84+Mgsi8Wiffv2SZKaN2+uvHnz2mxnMBjUpk0bjR8/3vpa2goDGzVqJJPJZHOMEiVKWH9O63bI/Pnzy8vLS5GRkSmeN3e3evXqycPDw+Y5Z2dnNW/eXPPnz9f+/ftlsVhSBWehoaGaO3eudu7cqXPnzunGjRvWkOJO586dS7PWSpUqydfXN80a76VgwYIqWrSogoOD9cMPP8jHxydduw1n9ec4rc0gJKlMmTKSbu80+iAe5L3q3Lmzjhw5ok2bNikyMjLFc+du3Lihv//+W9LtoOth7Rp9tzsDprTmPHTokBYsWKCAgAAFBwcrNjZWZrM5RZvExERduHDhvs9OvJ+1a9dq+fLlOnLkiCIiImwGfckbeNypcuXK+vfffzVlyhQ99thjeuKJJ9L87iaLi4tTQECApNuPI7jXn23JIX50dLQuXryokiVLZuSyAAA5BKEcACBXmjFjhho0aODoMtLk6+ur3r17q3fv3pJury7buXOnFi5caF2dNHnyZNWuXTvFaqCLFy9Kuv0PyowICgqy/nyvnRyl2/+Y3L17d4o+d7szYLrb2bNnJd1+6HmdOnXSVd+dO9Omx52r765duyYfH58M9U9248YN62qf5H9EpyX5vMViUXBwsM1Q7l6rbu7cYTM97eLj49Nsc7/3MPn89evXdfXq1RSh0Lp16/T+++8rNjb2nmMk90/LvT4D6fHhhx/qrbfeUmBgoF5++WUVKlRI9evXV506ddS4cWOVLl06VZ+s/hzfK1RM3nzkQVdzPch71bFjR3399deKi4vT0qVL1adPH+u5lStXKi4uTkajMcWOxw/bnZ8JW6tgf/jhB02cODFdq8Pu9fm6n7i4OA0aNEhbt27N1DzvvvuuevToocjISA0YMEAeHh6qV6+e6tSpo8cff1yVKlVK1efixYvWzXU++eQTffLJJ+mq9cqVK4RyAJBLcfsqAACPgBIlSqhLly6aN2+eNaiTpFmzZqVod+PGDUmSm5tbhsa/c0XH/fomn7/XKpA7A6a7ZeYf2gkJCRlqX7x4cevPySFgZmTmdbm7353ut9ImI+3uFWokb6iRlrRqvXTpkt555x3FxsaqWLFi+uyzz7Ro0SJt3bpVe/fu1b59+7Rv3z4VKVJEUsrbTO+W0R1z79amTRtNnz5djz/+uIxGo8LDw7VixQp98cUXat26tbp27WpdlWTrWrLic3z3ZigPQ2bfK+n2bZCtW7eW9L9bVZMl39LaqFEjFS1aNCtKTZc7V53dHS6vXLlSEyZMkMViUd26dfXtt99qxYoV2rFjh/Xzdeet6vf6fN3PmDFjrIHc888/r4kTJ2rt2rXauXOn9XM8bNiwNOepXbu25s+fr5YtWypPnjyKjo7W2rVrNWbMGD377LPq0KGDNm7cmKJPZkPEewXsAICcjZVyAAA8Yt59910tWrRI165d05EjR1Kcy58/v6Kjo+8ZNNhy5z/877dCKvl8RoO/ZMkhhIeHh3bt2pWpMe7nzlsdd+/ebX02XUZl5nW5u58j3K/WtMKrRYsWKT4+Xvnz59f8+fPTXGGYHP4+bA0bNlTDhg117do1BQQEKCAgQJs3b9aRI0cUEBCgHj16aNasWapVq5Yk+36Os0pm36tkXbp00ZIlS3Tq1CkdPHhQNWrU0JkzZ6wral944YUsrfd+7gxK79yJVZJmz54t6XbgNXPmTJuhp63bpDMqLi7O+mzBfv366b333rPZ7n5hf5UqVfTzzz8rNjZWBw4c0P79+7Vt2zbt2bNHp06d0oABAzR27FhrMHrn+/Prr7+qRYsWD3wtAICcjZVyAAA8YvLkyWO9de/uW+eSb4E6duxYhsa8c2XZ6dOn79n21KlTkqRixYplaI5kybc1Xr16NcWz8rJSvXr15OnpKen2CqLMrrjJnz+/9Ra85OtOS/J5g8Fg15VJtpw5cyZd5++8Pul/n5uGDRumGcgFBwfbLZRLVqBAATVr1kxvvfWWFi9erOnTpytv3ry6deuWJk6caG1nz89xVsnse5Wsbt261udHJm9KkrxqzsPDw+ZmFw+LxWKxrtBzdnZO9RzA5M9X27Zt01yFePLkyQeu4+zZs9bVZ8mbOdhy4sSJdI3n6uqqxx9/XK+//rpmzZqlJUuWWG8j/vnnn63tihUrZr2uCxcuZLJ6AEBuQigHAMAjxmKxKCwsTFLq28MaNWok6fbOnIGBgekes3z58tZnoK1evTrNdmFhYdaND9L7PLi7NW7cWNLt61i1alWmxriffPnyqXv37pJuP2dswoQJ6e57Z/hmMBisq302btyY5soai8WiNWvWSJIqVKjwwDu+Pqh///03zcAzISFBGzZskHR7xdKdD+NPfh7WvULMpUuXZl2hmfT4449bdxO98/Zke36Os0pm36s7de7cWdLt20Nv3LhhfY86duwoZ2fnrC86DVOmTLH+udOpU6dUG1gkf3/u3tThTvf7fCXvmnuvz+id39O02sXExOiff/6551xpqVSpktq1aycp5efP3d1dNWvWlHT7vcis9FwjACBnIJQDACCbmD9/vlauXHnPf7BK0pw5c6yh3N07X3br1k158uSR2WzWRx99dM/bWO+8TcxkMlkfBr9hw4ZUz0qSbv9D+quvvrL2e/HFF9N1XXcrW7asmjdvLkn68ccf77sC7caNG7p8+XKG5+nXr5/1Yezjx4/XjBkz7tk+ISFBU6ZM0aBBg1Ic79Kli6TbD2P/4YcfbPadMWOG9TpeeumlDNea1eLj4/Xf//7X5rlff/3VunPr3bc2Jq80CwgIUFRUVKq+J0+e1KRJk7K42tSuXLlic/5kSUlJ1g0akldESvb9HGeVzL5Xd3r++eeVJ08eXb9+XZ9//nm6+mQls9msqVOn6vvvv5d0+/8seOONN1K1S14lu379epvPRFy8eLF1B+m0JL/f9/oz4c4Vk7aCN4vFoq+++sq6icvdYmNjFRIScs86klfC3R08Jj/z88CBAylWcdpisVhsPvMyPdcIAMgZeKYcAADpFBoaqtDQUOvvd94ed+HCBesznCTJy8srw7vpnTt3TlOnTtXXX3+t9u3bq379+nrsscdUoEABxcXF6fTp01q6dKmWLVsm6fbziwYMGJBiDF9fX3300Uf68ssvFRAQoBdeeEH9+/dX/fr1VaBAAV29elWHDh3S6tWrValSJQ0cONDa9/XXX9eqVasUFhamIUOG6LXXXlP79u1VsGBBnTlzRhMmTNDmzZslST169FCFChUydH13GjZsmA4dOqSIiAi9+OKL6tGjh1q2bKnixYvLYDDoypUrOnHihDZv3qy///5bo0aNUps2bTI0R968eTV+/Hj16dNHFy5c0MiRI7V8+XJ17txZderUkZeXlxISEhQcHKxt27Zp8eLFCgoKSnU741NPPaWmTZtq8+bNmjp1qqKjo/Xyyy+rePHiunz5shYuXKiZM2dKkqpWrerwkEe6HUosXrxYN2/eVJ8+fVSiRAmFhYVp7ty5mjNnjqTbtz3e/Zq2a9dOc+fOVXR0tPr27at3331XlSpVUmxsrNavX6+ff/5Zbm5ucnZ2fmi3Hku3Vyv2799fLVu2VPPmzVW5cmV5e3srPj5eZ8+e1fTp063fv/bt26foa8/PcVbI7Ht1Jy8vL7Vo0UJr1qzRihUrJN3+LGZ0F+Z7uXnzpjXkN5vNunbtmoKDg7Vv3z4tXrzYukLOy8tLv/zyi80dhNu1a6fx48dr9+7devfdd9WnTx8VK1ZMYWFh+vPPPzVz5kyVK1funrceV6tWTfv379c///yjXbt2qXr16tbVgEajUUaj0bpT7+7duzVp0iQZjUa1a9dOnp6eOn36tCZPnqyNGzemOdeVK1fUunVrNW3aVC1btlS1atVUqFAhJSUl6eLFi5o7d651E4m7b49t3bq1OnbsqL/++kvff/+99u7dq5deeklVq1aVm5ubbty4ocDAQP37779auXKlSpUqlSq8S881AgByBkI5AADSacGCBRo/frzNc7/++qt+/fVX6+/PP/+8xowZk6Hxkx8SHhISosmTJ2vy5MlptvXz89MPP/xg83lY3bt3161bt/Ttt9/q3Llz+vjjj22OUb58+RS/e3h4aOrUqerXr5+Cg4P1008/6aeffkrV79lnn9WHH36YkUuzWf/s2bM1ePBgnTx5UhMnTrznqpI8efJkap4SJUpo/vz5GjZsmP7++28dOHBABw4cSLO9p6en+vfvn+r4Dz/8oDfeeEM7d+7U4sWLU+10Kd1+KPyECRMyXWtWeu6553Tx4kUtXbrU5m105cuX108//ZTqdsgGDRqoW7dumjNnjo4cOaI+ffqkOF+wYEH9+OOP+uCDDx5qKCfdDoGWL1+u5cuXp9nmmWeeUbdu3VIcs+fnOCtk9r26W5cuXay3UEtZv0ru7vDzbk5OTmrVqpU+/fTTNJ9H+Oqrr2rTpk06dOiQVqxYYQ0Qk5UvX16jRo2yrk61pVu3blqwYIGio6PVs2fPFOcGDRqkwYMHS5KGDx+ubt26KTo6WuPGjdO4ceNStG3Xrp0aN26sTz/91OY8iYmJWr9+vdavX59mLY0aNdKQIUNSHR89erQKFiyoWbNmadOmTdq0aVOaY5QrVy7T1wgAePQRygEAkE0MGjRIHTp00ObNm7Vv3z6dPn1aISEhiouLU968eeXl5aWKFSuqefPm6tChg1xcXNIcq3fv3mrRooVmzpypHTt2KDg4WGazWYUKFVKpUqX09NNPW3cMvFO5cuW0YsUKzZ49W+vWrdPZs2cVFxcnT09P1a5dW126dNETTzyRJddbunRpLVmyRCtXrtSaNWt06NAhXblyRRaLRZ6enipbtqzq1q2rp59+2nobamZ4enpq7NixOn78uFasWKGdO3cqODhYV69eVZ48eVSoUCFVrVpVTz75pFq1amXzdc2fP7+mT5+u5cuXa9myZTpy5IiuXbsmNzc3VaxYUe3atVPnzp2tz4LKDr7++ms1bNhQCxYs0JkzZxQfH68SJUqoXbt26tOnj/Lly2ez37Bhw1SjRg3NnTtXJ0+elNlslq+vr5o2barevXvbZWOE2rVra/r06dqxY4f27t2rkJAQRUZGWj/DNWvWVKdOndL8LNrzc5wVMvte3alx48YqVqyYgoKClDdvXnXs2PGh1Zs3b17lz59fnp6eqlSpkmrUqKE2bdrI19f3nv1cXFw0c+ZMTZ48WStXrtSlS5eUN29elShRQq1bt9Yrr7yiyMjIe47x2GOP6Y8//tBvv/2mgIAARUZGWp+FeHe7xYsX69dff9XmzZt15coVubu7q0KFCurUqZOeffZZm+G6JBUtWlR//PGHtm/frj179igoKEgRERG6deuWvL29VaVKFXXs2FFt27a1GZbmyZNHQ4cOVZcuXTR//nz9+++/Cg4OVlxcnNzc3FS8eHFVr15dTZs2VdOmTTN9jQCAR5/BYuuBDgAAAHjktGjRQkFBQaymyaXatm2rs2fPqkOHDvruu+8cXQ4AALgPHkgAAAAAPOL2799v3TQgeTdWAACQvRHKAQAAAI+46dOnS7p9W3jDhg0dWwwAAEiX7PPgEwAAAAAZEhMToz///FOrVq2SJPXt2/e+m0IAAIDsgVAOAAAAeMTs2rUr1c6ctWrVyvJdVwEAwMOTbUO5s2fPatu2bTpy5IiOHDmiM2fOKCkpSW+++aYGDhyY6XG3b9+uadOm6eDBg4qLi1PRokXVunVr9e/fX25ubll4BQAAAMDDZTQa5evrq6eeekpDhgyRyWRydEkAACCdsu3uqyNHjtSMGTNSHX+QUG769OkaPXq0DAaD6tatK29vb+3du1fh4eEqU6aM5syZIy8vrwctHQAAAAAAALinbLtSrkKFCurTp4+qVKmiKlWqaOLEiVq6dGmmxzt69KjGjBkjk8mkX3/9Vc2aNZMkxcXF6fXXX9eOHTs0fPhwjR07NqsuAQAAAAAAALAp24ZyXbp0SfG70fhgG8VOnDhRFotFnTp1sgZykuTi4qKRI0eqZcuWWrNmjc6cOaPHHnvsgeYCAAAAAAAA7iXbhnJZKSEhQZs2bZIkdejQIdX5YsWKyd/fX3v27NG6desyFcpFRl5X9rwRGAAAAAAAAPZgMEje3u7papsrQrnAwEDFxcVJkqpVq2azTbVq1bRnzx4dPXo0U3NYLCKUAwAAAAAAQLrkilDu0qVLkqQCBQoof/78Ntv4+fmlaJtRBkPmagMAAAAAAEDOkJF8KFeEcjExMZJuPz8uLa6urpKkGzduZGqO9C5NBAAAAAAAAHJFKGcPPFMOAAAAAAAgd+OZcndxc3OTJOtz5WyJjY2VpDRvb70fnikHAAAAAACA9DI6ugB7KFasmCTp2rVrad6eGhISkqItAAAAAAAA8LDkilCuTJky1ufJHT582Gab5ONVq1a1W10AAAAAAADInXJFKOfs7KxmzZpJkpYvX57qfFBQkAICAiRJLVu2tGttAAAAAAAAyH1y1DPlZs2apVmzZqlGjRr6+uuvU5zr37+/1qxZo8WLF6tVq1Zq2rSppNvPmfv000+VlJSk1q1b67HHHnNE6QAAAAAAZKmkpESZzWZHlwE8soxGo0ymhxedZdtQ7siRIxoxYoT19wsXLkiS5s2bp40bN1qPjx8/XoULF5YkRUVF6dy5cypUqFCq8apWraqPPvpIo0ePVv/+/VWvXj15e3trz549Cg8PV5kyZTR8+PCHek0AAAAAADxscXExiom5psTEBEeXAjzynJyc5eZWQC4ublk/dpaPmEVu3LihAwcOpDoeGhqq0NBQ6+8JCen/Q6ZXr16qUKGCpk6dqkOHDik2NlZFixZVp06d1L9//0zvvAoAAAAAQHYQFxejq1cj5OzsIg+PQjKZTJIMji4LeARZlJSUpNjYG7p6NUKSsjyYM1gsFkuWjphLRURcF68kAAAAAMCRIiJCZDSa5OlZSAYDYRzwoCwWi6KiwmU2J8nHx+++7Q0GycfHPV1j54qNHgAAAAAAyOmSkhKVmJggV9f8BHJAFjEYDHJ1dVNiYoKSkhKzdGxCOQAAAAAAcoDkTR1u37IKIKskb/aQ1RunEMoBAAAAAJCjsEoOyFoP5ztFKAcAAAAAAADYGaEcAAAAAAAAYGeEcgAAAAAAAICdEcoBAAAAAIAcad++PWrSpK4GDerv6FLSpUmTumrSpG6G+4WEBKtJk7rq3LnjQ6gKDwuhHAAAAAAAQDY2aFB/NWlSV/v27XF0KchCTo4uAAAAAAAAANLs2QsdXQLsiFAOAAAAAAAgGyhVqrSjS4AdEcoBAAAAAIBHwtGjh7Vx4z8KCNirsLAwXbt2Ve7uBVS5clV16dJV9eo1yNB4Bw7s1++/T9HRo4eUlJSkUqXK6IUXXlTbth2sz3bbujX1LaOXL4dp9uzftWvXDl2+HKY8efKobNnH1Lp1e3Xs+JxMJlOK9itX/qVRo0aobdsOGjz4bU2bNlnbtm1WePhlVa1aXePHT5KkVHPu27dHQ4a8Zh3nzp8l6ZNPhqldu5TPkbNYLFq27E8tXbpYFy4EymQyqUqVaurbd4CqVauR6lrunHPNmpVauHCuAgPPKW/evKpTp75ef32IihQpIovFosWL5+uvv5bq0qULyps3rxo1ekIDBw6Rp6dXhl533EYoBwAAAAAAHgkTJ/6igIA9KlOmrCpWrKR8+VwUFHRJ27dv0fbtWzRkyLt68cX/pGusdevW6IsvhspsNuuxx8qpTJnHFBERrtGjv1Bg4Lk0+x07dkTvvjtE165dla9vET3xRDPduBGjgIC9OnTooDZv3qj//vd75cmTJ1Xfq1ej1bdvT924cV01a9ZSxYqVbbZL5u3to7ZtO2jXrh26ciVS9es/Lm9vb+v5YsVKpOozatQIrV27WjVr1lajRk/o1KkT+vffXTpwIEDjxk1S1arVbM41YcJ4/fHHTNWq5a8GDRrp2LEj+uefv3Xo0AFNn/6Hvv12tLZu3azateuoaNFiOnTogFatWq6TJ09o8uQZ97wO2EYoBwAAAAAAHgldu76soUO/kI+PT4rjhw8f1LvvDtYvv/yk5s2fUqFChe85TkREuP7735Eym81688331KVLV+u5/fv36f3337TZLyEhQUOHfqRr167quede0FtvvS8np9vRSlDQJb311kDt3r1DU6dO0oABb6Tqv337VtWpU1+jRn0tN7f8973eUqVK69NPh2vQoP66ciVS3bu/In//tHdnDQ0NUUDAXs2YMU8lS5aSJCUlJenrr0dqxYplmjJlgr7/frzNvn/99acmT56p8uUrSJLi42/q7bcH6eDB/Ro8uL9u3rypOXMWqkgRP0lSdHS0Xnutt86cOaUNG9apVau2970epMTuqwAAAAAA4JHw+OONUwVyklStWg116vSiEhMTtWXLpvuOs3z5UsXFxapatRopAjlJqlXLX88919lmvw0b1ik0NEQ+PoU0ZMi71kBOkooVK6433rgd5i1aNF/x8fGp+js5OemDDz5JVyCXWW+99b41kJMkk8mk/v0HSrodOCYmJtrs17fva9ZATpLy5s2nl156WZJ05sxpvfXWe9ZATpI8PDz03HMvSJL27Nmd5deRG7BSDgAAAAAAPDKuXo3W9u1bde7cGV2/ft0aMl26dEGSdOHC+fuOERCwT5L09NNtbJ5v1aqN/vhjpo1+eyVJTz3VSs7OzqnON2vWQu7uBXT9+jWdOHFMNWrUSnG+fPmKKlas+H3ryyyTyaSGDRulOu7t7WOt6+rVaHl7pw42H3+8capjJUqUsI5br17DVOeLFy8pSYqIiHjQ0nMlQjkAAAAAAPBIWLbsT40b973i4uLSbBMbG3PfccLDwyRJfn5FbZ4vUsT28fDwcElS0aK2zxsMBvn5FdX169esbe+U1nxZxdvbJ8XqvTu5ubnp+vVrSkhIsHne17dIqmMuLq73HNfV9fb5hITUqwJxf4RyAAAAAAAg2zt+/Ji++WaUjEajXn99sBo3bipf3yLKly+fDAaDli5drG++GSWLxZLuMQ2GtI6nceIB5c2b96GMm8xozPxTyu7V90HGRdoI5QAAAAAAQLa3YcM6WSwWde78kl5++ZVU5y9dupjusQoVKqwLF84rJCTE5vmQkOA0+hWSJAUHB6U5dnLf5LZAWog6AQBAtmE0GuTkZHTIf0bjw/l/xAEAQNa4du2aJMnX1y/Vufj4eG3cuD7dY9WsWVuStG7dGpvn165dbfN47dp1JEn//LPW5kYOmzZt0PXr1+Tq6qaKFSunu577yZMnj6TbO6ki5yCUAwAA2YLRaJCHp4s8Pd0c8p+HpwvBHAAA2Vjp0qUlSatXL0/x3Lj4+Hh9990YhYSkvXrtbh06PKt8+fLp4MH9WrRofopzBw/u159/LrTZr3nzlvL1LaKIiHCNG/dDip1Mg4ODNH78j5KkF154MUtvVS1UqLAk6dy5s1k2JhyP21cBAEC2YDQaZDKaNGLNCAVGBdp17tKepTWs9TAZjQaZzel/Dg0AALCfdu2e0YIFc3Xy5Al16fKMatSoLZPJqAMH9is+Pl5duvxHCxb8ka6xChf21fvvf6KRI4frhx++1rJlf6pMmbKKiAjXwYP79dJLL+uPP2am2tzA2dlZX331X7377hAtWbJQO3duU9Wq1RQbG6u9e/coISFe9es/rj59+mfptT/55FNaufIv/frrWO3Zs1uenp4yGAxq3/4ZVa9eM0vngv0QygEAgGwlMCpQJ8NPOroMAACQzbi7u2vy5JmaMmWidu/eoV27tqtAgYKqX7+Bevfur4MH92dovNat26lwYV/NmDFVR48eVlDQRZUsWVoffPCp6tVroD/+mKmCBT1S9atcuaqmTZut2bN/186d27V580blyeOsChUqqk2bdurQ4bk0d0DNrEaNmujDDz/Tn38u1L59/+rmzZuSpBo1ahHKPcIMloxsS4I0RURcF68kAACZ5+RklKenm3rP7W33UK5CoQqa1nWaoqJilJhotuvcAABklVu3EhQZGSJvbz/lyePs6HIeaatWLdfIkcPVuPET+u9/f3B0OXCwjHy3DAbJx8c9XePyTDkAAAAAAJDrhIaGKjIyItXxgwf36+eff5J0+5ZZ4GHh9lUAAAAAAJDr7Nv3r8aM+VLlypWXr28RGY1GBQUF6fTp2yv227XrqGbNmju4SuRkhHIAAAAAACDXqVq1utq166gDBwIUELBXcXFxcnd3V9269dW+/TN6+uk2ji4RORyhHAAAAAAAyHVKlSqtjz4a6ugykIvxTDkAAAAAAADAzgjlAAAAAAAAADsjlAMAAAAAAADsjFAOAAAAAAAAsDNCOQAAAAAAAMDOCOUAAAAAAAAAOyOUAwAAAAAAAOyMUA4AAAAAAACwM0I5AAAAAAAAwM6cHF0AAAAAAACwH6PRIKPR4Ogy0s1stshstji6DCDLEcoBAAAAAJBLGI0GeXi4ymR6dG6cS0oyKzo6NsuDua1bN2vOnBk6ffqUYmNjJEljx06Qm1t+7d37r06cOKYTJ44rKOiiLBaLhg79Qq1bt8uSuZs0qfv/NezJUL/OnTsqNDRECxYsk59f0SypBY5DKAcAAAAAQC5hNBpkMhn12ZwtOnf5qqPLua8yhQvqq25PyGg0ZGkod+rUCX322QeyWCzy968rb28fGQwGeXv7aMKEcdqyZVOWzQWkhVAOAAAAAIBc5tzlqzoedMXRZTjM5s0blZiYqB49emvAgDdSnKtSpbpKly6rihUrqXz5iho9+gvt37/PQZUiJyOUAwAAAAAAuUpYWKgkqUSJkqnO9ejRy87VILcilAMAAAAAALnClCkTNW3ab9bfR40aoVGjRkiSatXy1/jxk+xe07Jlf2rJkkW6cCFQTk5Oqlathnr16qdq1arfs9+mTRs0d+4snTlzWhaLRRUrVtLLL/fU4483SdV20KD+2r9/n8aOnSB3d3dNmzZZBw7sU2xsrIoVK6727Z9V164vy2B4dDYAyQkenSc7AgAAAAAAPIDy5SuqbdsOKlasuCSpevWaatu2g9q27aAGDRrZvZ5x477XN9+MUr58+dSkSTMVLuyrnTu36403XtWmTRvS7Ldw4Vx9+un7unXrlho1aqLSpcto//59ev/9t7Rw4dw0++3evVP9+/fShQuBqlu3gapWra6LFy/o559/1Nix3z+MS8Q9sFIOAAAAAADkCk2bPqmmTZ/UyJHDFRR0SR07Pqd27To6rJ4lSxbpxx9/UZ069azH5syZoV9+GavRo0eoRo2a8vT0StVv/vw/9PnnX6pVq7bWY//887eGD/9U48b9IH//uipbtlyqfrNmTdd7732s5557wXps795/9dZbA7V48Xz95z/dVbiwbxZfJdLCSjkAAAAAAAAHePbZTikCOUnq1q2nKlWqohs3buivv5bY7NekSbMUgZwkPfVUKzVr1lxJSUlasGCezX7NmjVPEchJUp069VS//uNKSkrSvn17Mn8xyDBCOQAAAAAAAAdo27aDzeNt2rSTJAUE7M1gvw737Ne4cVObx0uXLi1JCg8PT7NWZD1COQAAAAAAAAfw8yt2z+Ph4ZfTOF/U5vGiRYv+f78wm+d9fYvYPO7q6iZJSkiIT7tYZDlCOQAAAAAAgGzIYrFksp/t4+yumr0QygEAAAAAADhASEiQzeOhocGSpEKFCqfRLziN4yGSpMKFbfdD9kIoBwAAAAAA4ACrV6+85/HatevYPL9mzYo0+q24Zz9kL4RyAAAAAAAADrBkycJUO57Omzdbx44dkaurmzp0eNZmv82bN2rdujUpjm3YsE6bNq2XyWTSCy+89NBqRtZxcnQBAAAAAAAA2cX27Vs1ffpk6++BgeckSVOnTtKiRfOtxydNmv7Acz37bCe9+ebrqlmztnx8CuncuTM6c+a0TCaTPv54qLy9fWz269Klq4YP/1Tz5s1W8eIlFRR0SUePHpYkDR78tsqVK//AteHhI5QDAAAAACCXKVO4oKNLSBdH1BkdHWUNuO4UFHRJQUGXsnSuIUPeVcmSpbR06WIdO3ZETk5OatCgkXr16qvq1Wum2a9Ll/+oWrWamj9/jrZu3SzJopo1a6tbt55q3PiJLK0RD4/BktmtPJBCRMT1NHc3AQAA9+fkZJSnp5t6z+2tk+En7Tp3hUIVNK3rNEVFxSgx0WzXuQEAyCq3biUoMjJE3t5+ypPH2WYbo9EgDw9XmUyPztOskpLMio6OldnMP7rhGOn5biUzGCQfH/d0jctKOQAAAAAAcgmz2aLo6FgZjQZHl5JuZrOFQA45EqEcAAAAAAC5CCEXkD0QygEAAAAAAGTC+fOBmjVrerrbd+/eS6VKlX5o9eDRQigHAAAAAACQCZGREVq1anm627dt24FQDlaEcgAAAAAAAJng719XW7fucXQZeEQ9OtutAAAAAAAAADkEoRwAAAAAAABgZ4RyAAAAAAAAgJ0RygEAAAAAAAB2RigHAAAAAAAA2BmhHAAAAAAAAGBnhHIAAAAAAACAnRHKAQAAAAAAAHZGKAcAAAAAAADYmZOjCwAAAAAAAPZjNBpkNBocXUa6mc0Wmc0WR5cBZDlCOQAAAAAAcgmj0SBPDxcZTSZHl5Ju5qQkRUXHEcwhxyGUAwAAAAAglzAaDTKaTIpY/JFuRZx1dDn3lcenrHw6jZHRaMjyUG7r1s2aM2eGTp8+pdjYGEnS2LET5OaWX3v3/qsTJ47pxInjCgq6KIvFoqFDv1Dr1u2yZO7OnTsqNDRECxYsk59f0SwZE48eQjkAAAAAAHKZWxFndSv0mKPLcJhTp07os88+kMVikb9/XXl7+8hgMMjb20cTJozTli2bHF0icgFCOQAAAAAAkKts3rxRiYmJ6tGjtwYMeCPFuSpVqqt06bKqWLGSypevqNGjv9D+/fscVClyMkI5AAAAAACQq4SFhUqSSpQomepcjx697FwNcitCOQAAAAAAkCtMmTJR06b9Zv191KgRGjVqhCSpVi1/jR8/yVGlSZJu3LihOXNmaOvWTQoODlJSUpIKFCiookWLqk6d+urV61U5OaWMcq5du6YFC/7Qli2bFBR0SWZzkooVK64WLZ5W167dlS9fPmvbYcM+0T///K0BAwalGT5u27ZFH374tsqXr6Bp0+akOLdr1w4tWjRPR48e0Y0b11WwoIdq1qytbt16qFKlKln+euR0hHIAAAAAACBXKF++otq27aCDB/crKOiSqlevqeLFS0iSSpYs7dDabt68qYED++rs2TPy8PBUnTr1lC+fi65cidSFC4E6dGiyXnrpZbm7u1v7nDt3Vu++O1iXL4fJ29tHNWrUkpOTSceOHdXkyRO0adN6jRs3Sfnz55cktW//jP7552+tXr08zVBu5cpl1rZ3+u23X/X771NkMBhUrVoN+foW0fnz57R+/Vpt2rRe77//iTp0ePbhvDg5FKEcAAAAAADIFZo2fVJNmz6pkSOHKyjokjp2fE7t2nV0dFmSpA0b1uns2TNq2LCRxoz5PsWKOLPZrAMHAlKseouPv6mPPnpHly+H6ZVX+qpXr1eVJ08eSbcDvjFjvtS6dWs0dux3+uSTYZKkunXr/3+YFqjDhw+pWrXqKWqIjo7Wtm1blCdPHj39dBvr8Z07t+v336fI2Tmv/vvf71SvXkPrueXLl2jMmK/07bejVaVKNZUt+9hDeX1yIqOjCwAAAAAAAMjtoqKuSJLq1WuQ6hZVo9Go2rXrWEM3SVq1armCgi6pUaMn1K/f6ynO5cuXTx988Kk8Pb20Zs1KXbt2zTpO27YdJP1vRdyd/v57lRITE9W4cVMVLOhhPf7HH7MkSc8/3zlFICdJHTo8p0aNnlBiYqIWLJj7AK9A7kMoBwAAAAAA4GDJz2SbM2eGVq1armvXrt6z/fbt2yRJTz31tM3zrq6uqlSpspKSknT8+FHr8bZtO8hgMGj9+rWKj7+Zos/KlX9JSnnramJiog4dOiBJaa4qTL5tNSBgzz1rRkrcvgoAAAAAAOBg/v519fLLr+iPP2Zq5MjhMhgMKl68hKpXr6knnmimxo2bymj839qq4OAgSdKXX36uL7/8/J5jR0dHWX8uVqy4atXyV0DAXm3atFGtWt2+TfXkyeM6ffqkfHwKqX79/62Gu3btqhIS4iVJfn5FbY5frFhxSVJ4+OVMXHnuRSgHAAAAAACQDbz++mA999wL2rZtsw4ePKBDhw5o5cq/tHLlX6pcuYrGjp0oFxcXSZLFYpYkNWjQSF5eXvcc19fXL8Xv7ds/o4CAvVq16i9rKJe8Sq5Nm/YymUxZfWmwgVAOAAAAAAAgm/DzK6rOnbuqc+eukqRjx47oiy+G6tixo5ozZ4b69h0gSSpc2FfnzweqQ4dn1Lx5ywzN8eSTT+mHH77W3r3/KiwsVF5e3lq7drWk1LeoFihQUM7OzkpISFBwcJDKlSufarzg4EuSpEKFCmf4enMznikHAAAAAACQTVWuXFXPP99FknTq1Anr8YYNG0mS1q9fl+Ex8+XLpxYtWslsNmv16hXatm2zrl69qurVa6pkyVIp2jo5Oal69VqSpFWr/rI53ooVtzeNqF27boZryc0I5QAAAAAAABxs06YN2r9/n8xmc4rjiYmJ2rVrhySpSJH/3Yb6zDOdVKSInzZsWKdffhmr2NiYVGNGRkZo2bI/bc6XvJnDqlXLraFa+/a2N3Lo2vVlSdKffy7Snj27U5xbufIvbd26WU5OTurSpWt6LhX/j9tXAQAAAADIZfL4lHV0CeniiDq3b9+q6dMnW38PDDwnSZo6dZIWLZpvPT5p0vQsnXf//n1asOAPeXh4qHz5ivL09FJsbIyOHDmsqKgrKlSosLp162lt7+Lioq+//lEffvi25syZoWXL/lS5cuVVqFBh3bx5UxcvXtD58+fk6emlZ555PtV81apVV+nSZRQYeE6XLl2Ui4uLWrRoZbO2xx9vrFde6avff5+it99+Q9Wr15SvbxGdPx+okyePy2Qy6b33PlbZso9l6WuS0xHKAQAAAACQS5jNFpmTkuTTaYyjS0k3c1KSzGaL3eaLjo7S0aOHUx0PCrqkoKBLD23edu06KG/evDp4cL8CA89p//59cnPLL1/fInrxxf/omWeeV8GCHin6lC37mH7//Q8tWbJImzdv1OnTp3T48EEVLOihwoUL6z//6a6mTZvfY86O+uWXsZJuP2fO1dU1zbb9+r2u6tVratGieTp69LCOHDkkDw8PNW/eUv/5T3dVqVItS16H3MRgsVjs98nOwSIirotXEgCAzHNyMsrT00295/bWyfCTdp27QqEKmtZ1mqKiYpSYaL5/BwAAsqFbtxIUGRkib28/5cnjnGY7o9Ego9Fgx8oejNlssWsoB9wtvd8tSTIYJB8f93SNy0o5AAAAAAByEUIuIHtgowcAAAAAAADAzlgpBwAAAAAAkAnnzwdq1qzp6W7fvXsvlSpV+qHVg0cLoRwAAAAAAEAmREZGaNWq5elu37ZtB0I5WBHKAQAAAAAAZIK/f11t3brH0WXgEcUz5QAAAAAAAAA7I5QDAAAAAAAA7IxQDgAAAAAAALAzQjkAAAAAAADAzgjlAAAAAAAAADsjlAMAAAAAAADsjFAOAAAAAAAAsDNCOQAAAAAAAMDOnBxdAAAAAAAAsB+j0SCj0eDoMtLNbLbIbLY4ugwgyxHKAQAAAACQSxiNBnl4ushkNDm6lHRLMicpOiqOYA45DqEcAAAAAAC5hNFokMlo0og1IxQYFejocu6rtGdpDWs9TEajIctDua1bN2vOnBk6ffqUYmNjJEljx06Qm1t+7d37r06cOKYTJ44rKOiiLBaLhg79Qq1bt8uSuZs0qfv/NezJUL9Bg/pr//59Gjt2gvz966a735QpEzVt2m/q3buf+vYdkKE58fAQygEAAAAAkMsERgXqZPhJR5fhMKdOndBnn30gi8Uif/+68vb2kcFgkLe3jyZMGKctWzY5ukTkAoRyAAAAAAAgV9m8eaMSExPVo0dvDRjwRopzVapUV+nSZVWxYiWVL19Ro0d/of379zmo0pQ+++wLxcfflK9vEUeXgixAKAcAAAAAAHKVsLBQSVKJEiVTnevRo5edq0m/IkUI43ISQjkAAAAAAJArJD9bLdmoUSM0atQISVKtWv4aP36S3WtatuxPLVmySBcuBMrJyUnVqtVQr179VK1a9VRt7/VMufj4m5o163etXbtaYWGhKlCggOrVa6hXX309zbljY2O0bt3f2rlzu86ePa2IiHBJUtGixdS4cVN169ZT7u7uNvuGhoZoypSJ2rVrh27cuK7ChX319NNt1L17L7377uBMPfsutyGUAwAAAAAAuUL58hXVtm0HHTy4X0FBl1S9ek0VL15CklSyZGm71zNu3PeaP/8PVa9eU02aNNPZs6e1c+d2/fvvLn3xxRg1a9Y8XePcvHlTb775uo4cOSQXFxfVq9dQefPm1e7dO7Vjx1Y9/ngTm/1OnTqlr78eKQ8PT5UsWUoVK1bS9evXdeLEMc2cOU0bNqzTxInTVLCgR4p+586d1eDB/RUdHS0fn0Jq0qSZbt6M09y5s7R377+yWMwP+tLkCoRyAAAAAAAgV2ja9Ek1bfqkRo4crqCgS+rY8Tm1a9fRYfUsWbJIP/74i+rUqWc9NmfODP3yy1iNHj1CNWrUlKen133HmTJloo4cOaRSpUrrp59+lY9PIUm3w7oRIz7V6tUrbPbz8/PTjz/+In//ujIajdbjN2/e1Lffjtbq1Ss0efJEvfvuhyn6ffnl54qOjtZTT7XSp58Ol7OzsyQpPPyy3nzzdV24cD7Dr0VuZLx/EwAAAAAAAGS1Z5/tlCKQk6Ru3XqqUqUqunHjhv76a8l9x4iPv6mlSxdLkgYPfscayElSvnz59N57H8vZOa/NvoUL+6pu3fopArk7+5lMJm3YsC7FuQMHAnTy5HG5uLjq3Xc/tAZyklSoUGENGvT2fWvGbayUAwAAAAAAcIC2bTvYPN6mTTsdP35UAQF71bNnn3uOceLEccXGxsjDw0MNGzZKdd7b20f16zfQ1q2b0xzj0KEDOnAgQGFhYYqPvymLxSJJypMnj6Kjo3Tt2jUVKFBAkhQQsFeS1KDB4ypQoGCqsRo1aqL8+d1148b1e9YNQjkAAAAAAACH8PMrds/j4eGX7ztGcpsiRYpmeJ6oqCv69NMPdPDg/nvOERsbYw3lkufz80t7viJF/HT6NKHc/RDKAQAAAAAAZEPJK9YeljFjvtTBg/tVrVoN9e3bX+XKVZC7ewE5Od2Oi559to0iIyNs1mEwpD3uvc7hfwjlAAAAAAAAHCAkJEjly1dMdTw0NFjS7We03Y+PT+EUfWyxdS4uLk47d26X0WjUN9/8JHd391Tnr1yJTNUvuaaQkJB7zBd637rBRg8AAAAAAAAOsXr1ynser127zn3HqFSpklxcXBUdHa3du3emOn/lSqTN4zExN5SUlCRXV7dUgZwkrVmz0uYKuZo1a0uSdu3aoWvXrqU6v2PHNl2/nvo4UiOUAwAAAAAAcIAlSxZq3749KY7Nmzdbx44dkaurmzp0ePa+Y+TNm0/PPPO8JGns2O8VERFhPRcff1PffjtG8fHxqfp5enrJ3b2Abty4rtWrV6Q4d/jwIU2c+LPN+WrV8le5chUUGxujH3/8Rrdu3bKei4gI188//3jfmnEbt68CAAAAAJDLlPYs7egS0sURdW7fvlXTp0+2/h4YeE6SNHXqJC1aNN96fNKk6Q8817PPdtKbb76umjVry8enkM6dO6MzZ07LZDLp44+HytvbJ13jvPrqazp4cL+OHTui//ynk/z968jZOa8OHgxQYmKi2rRpnyp4M5lM6t37VY0d+72++mqYFi9eoKJFiyksLFSHDx9Uq1ZtdeBAgEJDU96majAY9PnnX2jQoAH6++9VCgjYq+rVa+rmzZsKCNijcuUqqFq1Gjp8+KDy5MnzwK9RTkYoBwAAAABALmE2W5RkTtKw1sMcXUq6JZmTZDY/3A0P7hQdHaWjRw+nOh4UdElBQZeydK4hQ95VyZKltHTpYh07dkROTk5q0KCRevXqq+rVa6Z7HBcXF40bN1GzZk3X2rWrtXv3Trm7F1DduvXVr9/rWrnyL5v9Xnyxm/z8imrOnBk6d+6czp07q1KlSuuddz7Uc8+9oC5dnrHZr2zZcpoyZaYmT56g3bt3asuWjSpc2FedO3dVr1591aPHS5KkggU9MviK5C4Gy8PeyiOXiIi4Ll5JAAAyz8nJKE9PN/We21snw0/ade4KhSpoWtdpioqKUWKi2a5zAwCQVW7dSlBkZIi8vf2UJ49zmu2MRoOMxkdne0yz2WLXUA4PJjg4SF27Pi9XV1etXLleRuOj/+S09H63pNs7z/r4pH5Gny2slAMAAAAAIBch5MKDiouLU0hIsMqWfSzF8dDQEH3xxVCZzWa1adMhRwRyD1O2D+VWrVqlOXPm6Pjx47p165ZKliypjh07qlevXhm+Nzk2NlYzZ87UmjVrFBgYqPj4eHl4eKhatWp68cUX9dRTTz2kqwAAAAAAAMgZoqOj1LPnSypWrLhKlCgpNzc3hYWF6eTJ40pISFC5chXUr99rji4z28vWodzIkSM1Y8YMOTk5qWHDhnJ1ddXOnTv17bffasOGDZo6dary5cuXrrGioqLUvXt3nT59Wq6urvL395e7u7suXLigjRs3auPGjerRo4c+++yzh3xVAAAAAAAgJzh/PlCzZk1Pd/vu3XupVKnSD60eeylY0EP/+U8P7d37r44fP6rr168rX758euyxcmrWrIU6d+6a7rwmN8u2ody6des0Y8YMubq6atasWapataok6cqVK3rllVe0d+9e/fTTT/rwww/TNd7PP/+s06dPq2rVqpo6dao8PDys5zZt2qSBAwdq5syZ6tChg2rVqvUQrggAAAAAAOQkkZERWrVqebrbt23bIUeEcq6urnrjjTcdXcYjL9uGchMmTJAk9e/f3xrISZKXl5eGDRuml19+WbNmzdLAgQPl7n7/B+jt2rVLktSvX78UgZwkNWvWTA0aNNC2bdu0f/9+QjkAAAAAAHBf/v51tXXrHkeXgUdUtnziXlhYmA4dOiRJ6tChQ6rzdevWlZ+fnxISErRp06Z0jensfO/dMZLdHdgBAAAAAAAAWS1bhnJHjx6VdDsgK1GihM021apVS9H2fpo2bSpJ+u233xQdHZ3i3KZNm7Rr1y4VKlSIzR4AAAAAAADw0GXL21cvXbokSfLz80uzTZEiRVK0vZ9+/frp4MGD2rp1q5o3by5/f38VKFBA58+f15EjR+Tv76+RI0em61ZYAAAAAAAA4EFky1AuJiZGkuTi4pJmGzc3txRt78fV1VUTJkzQ999/r2nTpmnr1q3Wcx4eHmrUqJF8fX0zXbPBkOmuAAAgG+HvdADAo4q/w4CHy2C4//csI9/DbBnKPQyXL1/WwIEDdeLECb311ltq3769vL29dfr0af30008aP3681q1bp9mzZyt//vwZHt/bmxV2AAA86jw93RxdAgAAmXbz5k1duWKUyWSQk1O2fFoV8Egymw0yGo3y9HRTvnz5smzcbBnKJa+Ci4uLS7NN8gq55Lb389FHH+nQoUN6//339eqrr1qP16hRQxMmTFCnTp10/PhxTZ06VUOGDMlwzZGR12WxZLgbAAD4fyaT0eGhWFRUjJKSzA6tAQCAzLp1K0Fms1lJSRYlJvL3GZBVkpIsMpvNioqKUZ48t+7Z1mBI/8KtbBnKFStWTJIUEhKSZpvQ0NAUbe8lLCxM27Ztk2R7N9c8efKodevWOnnypLZv356pUM5iEaEcAAA5AH+fAwAeVfwdBjxcWZ39ZMv1rFWqVJEkRUdH6+LFizbbHD58WJJUtWrV+44XHBxs/TmtW1OTN3i4evVqhmoFAAAAAAAAMipbhnJFihRR9erVJUnLly9PdX7Pnj0KCQmRs7OzmjVrdt/x7tzA4cCBAzbbJB8vXrx4ZkoGAAAAAOCRYDTefubco/Kf0cgOFsiZsuXtq5L02muv6Y033tCkSZPUtGlT64q4qKgojRgxQpLUvXt36wo3SVq7dq2+++47+fr66vfff7ceL1q0qKpXr65Dhw5p5MiRmjRpUorwbenSpVq5cqUk27e3AgAAAACQExiNBnl6uMhoMjm6lHQzJyUpKjpOZjP35yJnybahXMuWLdWjRw/NnDlTL730kho2bChXV1ft2LFD165dk7+/v958880Ufa5fv65z584pISEh1XijRo1Sz549debMGbVr1041a9aUp6enzp49q1OnTkmSnnnmGT3zzDN2uT4AAAAAAOzNaDTIaDLp2JdfKvb8eUeXc1+upUqp8tChMhoNWR7Kbd26WXPmzNDp06cUG3t7M8mxYyfIzS2/9u79VydOHNOJE8cVFHRRFotFQ4d+odat22VpDUhbSEiwunR5RkWK+Gnhwr8cXc5DkW1DOUn67LPP5O/vrzlz5iggIECJiYkqWbKk+vXrp169esnZ2TndY1WoUEHLly/X9OnTtXnzZh0+fFgJCQkqUKCAmjRpohdeeEHt2vHlAgAAAADkfLHnz+vGyVOOLsNhTp06oc8++0AWi0X+/nXl7e0jg8Egb28fTZgwTlu2bHJ0icgFsnUoJ0nt2rVLd1jWqVMnderUKc3zPj4+eu+99/Tee+9lVXkAAAAAAOARs3nzRiUmJqpHj94aMOCNFOeqVKmu0qXLqmLFSipfvqJGj/5C+/fvc1ClyMmyfSgHAAAAAACQlcLCQiVJJUqUTHWuR49edq4GuRWhHAAAAAAAyBWmTJmoadN+s/4+atQIjRp1ezPJWrX8NX78JLvUcfHiBc2cOU0BAXsVEREuJycnFShQUGXLPqYnn3xK7dv/73n3K1f+pVGjRqht2w4aNOgtTZ48Udu2bVZU1BV5eXmrWbPm6tWrnwoUKJBqnk2b1mvHjm06cuSQwsPDlZAQL29vH9WuXUfdu7+ikiVLp+ozcuRwrVq1XJ98MkyVKlXW9OlTdODAPkVFRemVV/qqb98BkqT169dp6dLFOnXqhGJibsjNLb+8vLxUvXpNvfDCSypXrnyqsTdsWKe//lqqkyeP6caNG/Lw8JS/f1316NFbZcqUTfP1SkxM1Lx5s7Vq1QoFBwfJxSWf/P3r6dVXX1OpUqmvoUmTupKkrVv3aOPGfzRv3hydOXNaZnOSypevoJ49++jxx5vc72166AjlAAAAAABArlC+fEW1bdtBBw/uV1DQJVWvXlPFi5eQJJsB1cNw9uxpvf56X8XExKhkyVJq1KiJjEaTwsMva//+AIWHh6cI5ZJdv35N/fv30tWrV1W7dh0ZDAYFBOzV/Pl/aOfO7fr558ny9PRM0efzzz9Wnjx5VLp0WdWpU1dJSUk6e/aMVq78Sxs2rNP3349X9eo1bdZ56NBBffvtaHl7+6hmTX/Fx9+Uq6ubJGnatN80ZcpEmUwmVa9eUz4+hRQTc0NhYaFavnypypQpmyKUS0xM1BdfDNX69Wvl7OysihUrycensC5evKC//16lTZvWa+TIb9SwYSObtQwb9rG2bduiWrX89dhj5XTs2BFt2LBOO3du1w8/jFe1ajVs9psyZaKmT5+satVq6PHHG+n8+fM6dOigPvjgbX311ddq1qx5ut6zh4VQDgAAAAAA5ApNmz6ppk2f1MiRwxUUdEkdOz6ndu062rWGuXNnKyYmRv36va5XXumb4lx8/E0dO3bUZr+tWzeratXq+u2331WgQEFJ0vXr1/XBB2/q0KGD+vHHbzRixKgUfT7//Es1avSEXFxcrMcsFov+/HOhvv/+v/r665GaMWOeDAZDqvn++utPde/eS/37D5TRaLQeT0hI0KxZ0+Xi4qopU2akCjNDQ0MUHx+f4tiUKRO1fv1aValSTcOHj1TRosWs5zZsWKfhwz/ViBGfaf78pXJ3d0813s2bcZo8eaY16EtKStK4cd9r4cJ5Gj78U82Zs8jmZqALFszVhAnTVLVqtRS1TJv2myZMGOfwUM54/yYAAAAAAADIClFRVyRJjz/eONW5vHnzqVYt/zT7vvfeR9ZATpLc3d313nufyGAwaMOGdbp8OSxF+6eeapUikJMkg8GgTp26qFq1Gjp37qwCA8/ZnKtEiZLq1+/1FIGcJMXExCg+Pl5FixazubqwSBG/FLeUXrt2VfPnz5Gzc16NHPl1ikBOkpo3b6lnnumk69ev6e+/V9qspWfPvilW3plMJg0c+KYKFSqs0NAQbdy43ma/V18dkCKQk6QePXorf/78unjxgvXZgo5CKAcAAAAAAGAnlStXlSR9++0Y7dq1I9WqsrSUK1dB5ctXTHX8scfKqXz5ijKbzdq/PyDV+UuXLmrRonn66afvNHr0Fxo5crhGjhyuK1ciJUkXLpy3Od8TTzwpk8mU6rinp6f8/IrqzJlTGjfuB507d/aede/bt0fx8fGqXr2mChUqbLNN7dp1JN2+ZdaWtm07pDrm7OysFi2eliQFBOy12a9x46Y2+yUHg+Hh4fes/WHj9lUAAAAAAAA76datpw4e3K89e3br3XcHy8nJSeXKVVDNmrXVsmUra2h3Nz+/ommOWbRoUZ08eVzh4f9bKZeUlKQffvhaS5culsViSbNvbGxMhuf77LMR+uyzDzVv3mzNmzdbBQoUVJUqVVWvXgO1bt1eHh4e1rbBwUGSpL17d1s3YEhLdHRUqmP587unuqU1WdGit2u887rv5OtbxObx5GfjJSSkLxB9WAjlAAAAAAAA7CRfvnz68cdfdOzYEe3atUOHDh3U4cMHdPz4Uc2bN1vPP99F7777YabGvjN8W7DgDy1Zskje3t4aNOhtVa9eU56eXsqbN68kafjwT7Vu3Zo0A7vkdrbUrFlbCxcu0/btW7V//z4dOnRQu3fv1M6d2zVlyiSNGvWN6tatL0kym82SpOLFS6S5qUSyzG62kVbmePett9kNoRwAAAAAAICdVa5c1boqLjExUVu2bNRXXw3Tn38uUPPmT8nfP+WqspCQ4DTHCgkJkSQVLuxrPbZ+/TpJ0vvvf6ImTZql6nPp0sUHqj9v3nxq3rylmjdvKUmKiorSb7/9omXL/tTo0V9o0aLl/1/T7dVqJUuW0qefDs/wPDduXNf169dtrpb733Xbvi02u8vekSEAAAAAAEAO5+TkpObNW6p+/cclSadOnUjV5syZUzp9+lSq42fPntHJk8dlNBpVs2Zt6/Fr165Jknx9/Wz2sTXHg/D09NTAgW9KksLCQq3z161bT3ny5FFAwF7rJhcZtWbNilTHbt26pfXr10r63zPpHjWEcgAAAAAAAHayePECXbgQmOp4ZGSETpw4Jun2DqZ3s1gs+u670dawS5Ju3Lih774bI4vFombNWqR4hlrp0qWt8yXfQipJERER+uqrYUpKSspU/aGhIfrrryWKibmR6ty2bZslSe7uBeTmdvu5bV5e3nrhhZcUFxenDz54W2fOnE7VLyEhQVu3btL584E255w+fYrOnv1fP7PZrF9/HavLl8NUuLCvmjVrkalrcTRuXwUAAAAAIJdxLVXK0SWkiyPq3L59q6ZPn2z9PTDwnCRp6tRJWrRovvX4pEnTMzX+smV/6vvv/ys/v2IqW7as3NzyKzo6SgcOBCg+Pl516tSzuWtokyZNdfbsGb344rPy968rg0EKCNina9euqnjxknrnnQ9StO/Ro4927dqhv/76UwEBe1ShQiXFxMRo//69Klq0mJo2ba7NmzdkuP7r16/pv//9St99N0bly1eQn9/tnUwvXbqgkydPyGAw6I03hqTYufW11wYpMjJCa9euVu/e3VSuXHkVLVpMJpNJly9f1unTJxUXF6dvvx2rUqVKp5jP17eIKlasrD59uqt27ToqUKCgjh8/qqCgS3JxcdGwYSPv+fy77IxQDgAAAACAXMJstsiclKTKQ4c6upR0MyclyWxOe/fQrBYdHaWjRw+nOh4UdElBQZceePz+/Qdq+/atOnr0kI4cOayYmBvy9PRSlSrV1K5dRz39dBs5OaWOa9zdC2jixOmaPPlX7dixTVFRV+Tp6aVWrdqqT59+KlCgYIr2VatW0+TJM/Xbb7/o2LGj2rp1swoX9tULL7ykXr366ocfvslU/cWKFdeQIe9q//59Onv2jAIDt0myyMenkNq0aa/OnbuqUqXKKfo4OTlp2LCv1KpVWy1fvkRHjx7R2bNnlC+fi3x8fNSo0RNq0qSpatXyTzWfwWDQF1+M1pw5M7RmzUodOBCgfPlc9OSTLdS372sqU6Zspq4jOzBY7rUvLtItIuJ6mrt9AACA+3NyMsrT00295/bWyfCTdp27QqEKmtZ1mqKiYpSYaL5/BwAAsqFbtxIUGRkib28/5cnjnGY7o9Ego9Fgx8oejNlssWsol92sXPmXRo0aobZtO2RqowQ8uPR+tyTJYJB8fFJvSmELK+UAAAAAAMhFcnvIBWQXbPQAAAAAAAAA2Bkr5QAAAAAAADLh/PlAzZo1Pd3tu3fvlWojA+RehHIAAAAAAACZEBkZoVWrlqe7fdu2HTIcyrVr11Ht2nXMYGV4FBDKAQAAAAAAZIK/f11t3brH0WXgEcUz5QAAAAAAAAA7I5QDAAAAAAAA7IxQDgAAAAAAALAzQjkAAAAAAADAzgjlAAAAAAAAADsjlAMAAAAAAADsjFAOAAAAAAAAsDNCOQAAAAAAAMDOCOUAAAAAAAAAO3NydAEAAAAAAMB+jEaDjEaDo8tIN7PZIrPZ4ugygCxHKAcAAAAAQC5hNBrk6eEqo+nRuXHOnGRWVHRslgdzW7du1pw5M3T69CnFxsZIksaOnSA3t/zau/dfnThxTCdOHFdQ0EVZLBYNHfqFWrdul6U1pKVz544KDQ3RggXL5OdX1C5zwv4I5QAAAAAAyCWMRoOMJqP+nr1PUWE3HF3OfXn65lerl/1lNBqyNJQ7deqEPvvsA1ksFvn715W3t48MBoO8vX00YcI4bdmyKcvmAtJCKAcAAAAAQC4TFXZD4UFXHV2Gw2zevFGJiYnq0aO3Bgx4I8W5KlWqq3TpsqpYsZLKl6+o0aO/0P79+xxUKXIyQjkAAAAAAJCrhIWFSpJKlCiZ6lyPHr3sXA1yK0I5AAAAAACQK0yZMlHTpv1m/X3UqBEaNWqEJKlWLX+NHz/JbrWcO3dWU6ZMVEDAHsXF3VSxYsXUtm1HvfRSt3v2Wb9+rfbs2aWQkBBFR0fJ1dVN5ctX1DPPPK+nnno6zb5btmzUH3/M0qlTJ2U0GlS+fEV169ZTZcqUVZcuz6hIET8tXPiXtX1ISLD1+Pz5S7Vo0XwtX75Ely5dVP78+dWkSTMNGPCGChQoqISEBM2ZM0N//71KoaGhcnd3V/PmLTVgwBtycXFJUUdsbIzWrftbO3du19mzpxURES5JKlq0mBo3bqpu3XrK3d39gV7bRwWhHAAAAAAAyBXKl6+otm076ODB/QoKuqTq1WuqePESkqSSJUvbrY4DB/brvfcGKy4uTkWLFlPdug109Wq0Jk36WUePHkqz37x5s7V8+VKVKlVaZcuWk7t7foWFhSkgYI/27t2to0cPafDgd1L1mz37d/366zhJUpUq1VS0aDEFBV3UBx+8pW7det633i++GKotWzaqVq06Klq0uA4fPqilSxfr2LEj+vnnyXr33cE6c+aUateuo+LFS+rgwQAtXDhXly5d0Lffjk0x1qlTp/T11yPl4eGpkiVLqWLFSrp+/bpOnDimmTOnacOGdZo4cZoKFvTIyEv6SCKUAwAAAAAAuULTpk+qadMnNXLkcAUFXVLHjs+pXbuOdq0hPj5eI0Z8qri4OL344n/0xhtvyWQySZJOnz6lt956XdHR0Tb7tm7dTj169FaxYsVTHL9wIVBvvfWG5s2bo6eeaqUqVapZz508eVyTJv0ik8mkL74Yo2bNmlvPrV+/TsOHf3LPekNDQ2QymTR79kIVKeInSbp6NVoDBvTRyZMn9NprveXsnFfz5y+1BmnBwUHq27eHdu7croMH96tGjVrW8fz8/PTjj7/I37+ujMb/7QJ88+ZNffvtaK1evUKTJ0/Uu+9+eN/X8lH36OyBDAAAAAAA8IjbtGm9Ll8OU+HCvho48E1rICdJ5cqVV8+efdLsW7t2nVSBnHR7ld8rr/SVJG3c+E+Kc4sWzVdSUpKaN2+ZIpCTpBYtWqpp05THbHnrrfesgZwkFSzooeeff0GSdPbsGX388dAUK9uKFi2m1q3bSpL27v03xViFC/uqbt36KQI5ScqXL5/ee+9jmUwmbdiw7r415QSslAMAAAAAALCTgIC9kqQWLZ6Wk1PqWKZNmw4aO/b7NPvHxsZq587tOnXqhKKjo5WYeEuSFBkZIUm6cOF8ivbJO8e2atXW5nitWrVNFeTdyWQyqV69hqmOFy9+e5MMX98iKlu2nI3zt28LTn5m3N0OHTqgAwcCFBYWpvj4m7JYLJKkPHnyKDo6SteuXVOBAgXSrCsnIJQDAAAAAACwk8uXL0uS/PyK2jxfoEAB5c+fXzdu3Eh1buvWzRo9eoSuXr2a5vgxMTEpfg8Pvz3fnSvd7uTnZ/t4Mm9vH5vhYfIGDr6+RWz2c3V1kyQlJCSkOB4VdUWffvqBDh7cf895Y2NjCOUAAAAAAADgWOHhlzVs2MeKj49Xt2491apVW/n5+cnFxVVGo1G7d+/UO+8Msq44u5vBYEhj5LSO33b3babpH9e2MWO+1MGD+1WtWg317dtf5cpVkLt7AWvw9+yzbRQZGZHmdeQkhHIAAAAAAAB2UqhQIUm3N1Cw5fr16zZXyW3btlnx8fFq2rS5Bg4ckur8xYsXbI7n41NIwcFBCg0NVpkyZVOdDw0Nzkj5DyQuLk47d26X0WjUN9/8JHd391Tnr1yJtFs9jsZGDwAAAAAAAHZSq5a/JGn9+rVKTExMdX716hU2+127dk2SVKRI6ttFLRaL1q1bfc/51q5dY/N8WscfhpiYG0pKSpKrq1uqQE6S1qxZmStWyCUjlAMAAAAAALCT5s2fUqFChRUWFqoJE8bLbDZbz509e1q//z7FZr9SpcpIkjZuXK+IiAjr8aSkJE2ePEGHDh202a9TpxdlNBr1zz9/a8uWjSnObdq0Xps2rX+g68kIT08vubsX0I0b11OFj4cPH9LEiT/brZbsgNtXAQAAAADIZTx98zu6hHRxRJ3bt2/V9OmTrb8HBp6TJE2dOkmLFs23Hp80aXqmxs+bN58+//xLvf/+m5o7d5a2bNmoSpWq6Nq1qwoI2KvGjZ/QiRPHU93e2rjxE6pYsbJOnDim//ynk2rX9le+fC46evSwIiLC9fLLr2j27N9TzVepUmX16/e6Jk78WR9//J6qVq2uokWL6dKlizp27Ii6du2uuXNnyckpT6auJyNMJpN6935VY8d+r6++GqbFixeoaNFiCgsL1eHDB9WqVVsdOBCQ5q29OQ2hHAAAAAAg2zAaDTIaM/bg+KxiNltkNufsW+fMZovMSWa1etnf0aWkmznJbNf3JTo6SkePHk51PCjokoKCLmXJHLVr19GkSdM1ZcpEBQTs05YtG1W0aDH17fua/vOf7ura9flUfZycnDRu3ETNnDlNmzat1549/8rNzU3VqtXQV199rdjYGJuhnCT16NFbJUuW1ty5s3T69EmdO3dW5cqV1+jR36pAgYKaO3eWPDw8suTa7ufFF7vJz6+o5syZoXPnzuncubMqVaq03nnnQz333Avq0uUZu9SRHRgsuelm3YcoIuK6eCUBAMg8JyejPD3d1Htub50MP2nXuSsUqqBpXacpKipGiYnm+3cAADwURqNBHp4uMhlNDpk/yZyk6Ki4RzaYu3UrQZGRIfL29lOePM5ptnNk8JkZuSEsdaRp037TlCkT1bnzS3rrrfcdXU62lN7vliQZDJKPT+rn5dnCSjkAAAAAQLZgNBpkMpo0Ys0IBUYF2nXu0p6lNaz1MBmNhhwfABFy5T4XL15QwYIeKlCgQIrjW7du0syZ02UwGNSmTQcHVZd7EcoBAAAAALKVwKhAu6+aBnKyv/9epZkzp6l8+Yry9fVVYmKiLlw4rwsXzkuS+vTpr0qVKju4ytyHUA4AAAAAACATzp8P1KxZ09Pdvnv3XipVqvRDqyctDRo00qVLF3XkyCGdPx+ohIR4FSxYUI0bP6Hnn++ihg0b2b0mEMoBAAAAAABkSmRkhFatWp7u9m3bdnBIKFetWnVVq1bd7vPi3gjlAAAAAAAAMsHfv662bt3j6DLwiDI6ugAAAAAAAAAgtyGUAwAAAAAAAOyMUA4AAAAAAACwM0I5AAAAAAByFIujCwBymIfznSKUAwAAAAAgBzAab/8TPykpycGVADlLUlKipP99x7IKoRwAAAAAADmAyeQkJydnxcbekMXCajkgK1gsFsXGxsjJyVkmk1OWjp21owEAAAAAAIdxcyugq1cjFBUVLldXt/8PEQyOLgt4BFmUlJSo2NgYJSTEqWBBnyyfgVAOAAAAAIAcwsXFTZIUE3NN0dERDq4GePQ5OTmrYEEf63crS8fO8hEBAAAAAIDDuLi4ycXFTUlJiTKbzY4uB3hkGY3GLL9l9U6EcgAAAAAA5EAmk5NMJkdXASAtbPQAAAAAAAAA2BmhHAAAAAAAAGBnhHIAAAAAAACAnRHKAQAAAAAAAHZGKAcAAAAAAADYGaEcAAAAAAAAYGeEcgAAAAAAAICdEcoBAAAAAAAAdkYoBwAAAAAAANgZoRwAAAAAAABgZ4RyAAAAAAAAgJ0RygEAAAAAAAB2RigHAAAAAAAA2BmhHAAAAAAAAGBnTo4uAAAAAMBtRqNBRqPBIXObzRaZzRaHzA0AQG5EKAcAAABkA0ajQR6eLjIZTQ6ZP8mcpOioOII5AADshFAOAAAAyAaMRoNMRpNGrBmhwKhAu85d2rO0hrUeJqPRQCgHAICdEMoBAAAA2UhgVKBOhp90dBkAAOAhY6MHAAAAAAAAwM4I5QAAAAAAAAA7I5QDAAAAAAAA7IxQDgAAAAAAALAzQjkAAAAAAADAzgjlAAAAAAAAADsjlAMAAAAAAADsjFAOAAAAAAAAsDNCOQAAAAAAAMDOCOUAAAAAAAAAOyOUAwAAAAAAAOyMUA4AAAAAAACwM0I5AAAAAAAAwM4I5QAAAAAAAAA7I5QDAAAAAAAA7IxQDgAAAAAAALAzQjkAAAAAAADAzgjlAAAAAAAAADsjlAMAAAAAAADsjFAOAAAAAAAAsDNCOQAAAAAAAMDOCOUAAAAAAAAAOyOUAwAAAAAAAOyMUA4AAAAAAACwM0I5AAAAAAAAwM4I5QAAAAAAAAA7I5QDAAAAAAAA7IxQDgAAAAAAALAzQjkAAAAAAADAzgjlAAAAAAAAADsjlAMAAAAAAADsjFAOAAAAAAAAsDNCOQAAAAAAAMDOCOUAAAAAAAAAOyOUAwAAAAAAAOyMUA4AAAAAAACwM0I5AAAAAAAAwM4I5QAAAAAAAAA7I5QDAAAAAAAA7IxQDgAAAAAAALAzQjkAAAAAAADAzgjlAAAAAAAAADtzcnQB97Nq1SrNmTNHx48f161bt1SyZEl17NhRvXr1Up48eTI15rp167Rw4UIdOnRIV69elbu7u0qVKqUmTZpo0KBBWXwFAAAAAAAAQErZOpQbOXKkZsyYIScnJzVs2FCurq7auXOnvv32W23YsEFTp05Vvnz50j1eQkKC3n//fa1evVr58uVTrVq15OPjo/DwcJ0+fVozZ84klAMAAAAAAMBDl21DuXXr1mnGjBlydXXVrFmzVLVqVUnSlStX9Morr2jv3r366aef9OGHH6Z7zKFDh2r16tVq2bKlvvzyS3l5eVnPmc1mHTx4MMuvAwAAAAAAALhbtn2m3IQJEyRJ/fv3twZykuTl5aVhw4ZJkmbNmqXr16+na7wdO3ZoyZIlqlChgn788ccUgZwkGY1G1apVK2uKBwAAAAAAAO4hW4ZyYWFhOnTokCSpQ4cOqc7XrVtXfn5+SkhI0KZNm9I15syZMyVJPXv2zPSz6AAAAAAAAICskC1vXz169KgkycPDQyVKlLDZplq1agoJCdHRo0dtBnd3SkpK0o4dOyRJ9erVU3h4uFasWKFz587J2dlZVapUUatWreTm5pa1FwIAAAAAAADYkC1DuUuXLkmS/Pz80mxTpEiRFG3v5eLFi4qNjZUk7d+/XyNGjLD+nuzrr7/W999/r8cffzxTNRsMmeoGAACyGf5OR27HdwDgewAg8zLy50e2DOViYmIkSS4uLmm2SV7Vltz2XqKjo60/f/bZZ6pdu7Y++OADlS1bVhcvXtT333+vTZs2aeDAgfrzzz9VunTpDNfs7e2e4T4AACB78fRk1TxyN74DAN8DAPaTLUO5rGaxWKw/Fy5cWFOmTJGzs7MkqVKlSvr111/13HPP6eTJk5o0aZJGjRqV4TkiI6/rjmkAAEAGmUxGh/9DKCoqRklJZofWgNyL7wDA9wDAo89gSP/CrWwZyiWvgouLi0uzTfIKufQ8B+7ONp06dbIGcslMJpNeeuklffnll9Znz2WUxSJCOQAAcgD+Pkdux3cA4HsAwD6y5e6rxYoVkySFhISk2SY0NDRF2/uNZ/j/m3qLFy9us03yhhLh4eEZqhUAAAAAAADIqGwZylWpUkXS7WfBXbx40Wabw4cPS5KqVq163/Hc3NxUpkwZ65i2REVFSZJcXV0zWi4AAAAAAACQIdkylCtSpIiqV68uSVq+fHmq83v27FFISIicnZ3VrFmzdI3Zpk0bSdL27dttnt+2bZskWecFAAAAAAAAHpZsGcpJ0muvvSZJmjRpko4cOWI9HhUVpREjRkiSunfvLnf3/z08b+3atWrTpo1eeeWVVOP16NFDBQsW1KZNmzR37twU51asWKG//vpLktSzZ88svxYASA+j0SAnJ6ND/jMaM7BvNwDkcI7689hkyrb/0xwAADwE2XKjB0lq2bKlevTooZkzZ+qll15Sw4YN5erqqh07dujatWvy9/fXm2++maLP9evXde7cOSUkJKQaz8vLSz/88INef/11DRs2TLNmzVLZsmV18eJFHT16VJI0cODAdK+8A4CsZDQa5OHpIpPR5JD5k8xJio6Kk9nMU40B5G5Go0GeHi4ymhzz5zEAAMg9sm0oJ0mfffaZ/P39NWfOHAUEBCgxMVElS5ZUv3791KtXr1S7qN5P48aNtXTpUk2cOFHbt2/X+vXr5ebmpmbNmqlnz55q0qTJQ7oSALg3o9Egk9GkEWtGKDAq0K5zl/YsrWGth8loNBDKAcj1jEaDjCaTIhZ/pFsRZ+06d75yTeTZYohd5wQAAI6TrUM5SWrXrp3atWuXrradOnVSp06d7tmmTJkyGjNmTFaUBgBZLjAqUCfDTzq6DADI9W5FnNWt0GN2ndPJu4xd5wMAAI7FgysAAAAAAAAAOyOUAwAAAAAAAOyMUA4AAAAAAACwM0I5AAAAAAAAwM4I5QAAAAAAAAA7I5QDAAAAAAAA7IxQDgAAAAAAALAzQjkAAAAAAADAzgjlAAAAAAAAADsjlAMAAAAAAADsjFAOAAAAAAAAsDOnrBjk/PnzunLlijw8PFSmTJmsGBIAAAAAAADIsTK9Ui4pKUm//PKLGjdurDZt2qhbt26aNGmS9fyyZcvUtWtXnTp1KksKRc5mNBrk5GR0yH9Go8HRlw8AAAAAAHKZTK2US0pK0oABA7Rt2zaZTCY99thjOn36dIo2/v7++uCDD/T333+rfPnyWVIsciaj0SAPTxeZjCaHzJ9kTlJ0VJzMZotD5gcAAAAAALlPpkK5uXPnauvWrWrYsKH++9//ytfXV5UqVUrRpnjx4ipZsqS2bdumN954I0uKRc5kNBpkMpo0Ys0IBUYF2nXu0p6lNaz1MBmNBkI5AAAAAABgN5kK5f78808VLFhQP/30kwoWLJhmu7Jly+rEiROZLg65S2BUoE6Gn3R0GQAAAAAAAA9dpp4pd/bsWdWoUeOegZwkubu7KzIyMlOFAQAAAAAAADlVpkI5s9ksZ2fn+7YLDw9PVzsAAAAAAAAgN8lUKFe0aNH73pZ669YtnTp1SqVKlcpUYQAAAAAAAEBOlalQ7oknnlBQUJDmzZuXZptZs2bpypUrevLJJzNbGwAAAAAAAJAjZWqjh759++rPP//UiBEjdPr0abVt21aSFBcXpyNHjmjVqlWaPn26PD099fLLL2dpwQAAAAAAAMCjLlOhXOHChfXzzz9r0KBBmjlzpmbNmiWDwaA1a9ZozZo1slgsKlCggMaOHSsvL6+srhkAAAAAAAB4pGUqlJOkevXqacWKFZo+fbo2bdqkS5cuyWw2q0iRImratKleffVV+fr6ZmWtAAAAAAAAQI6QqVAuODhYBoNBfn5+eu+99/Tee+9ldV0AAAAAAABAjpWpjR5atGiht99+O6trAQAAAAAAAHKFTIVy+fPnV/HixbO6FgAAAAAAACBXyFQoV65cOYWGhmZ1LQAAAAAAAECukKlQrkuXLtq3b58OHjyY1fUAAAAAAAAAOV6mNnp44YUXdOzYMfXt21d9+/ZVq1atVLx4cTk7O2d1fQAAAAAA2I3JlKm1Kw/MbLbIbLY4ZG4AjpGpUK5y5crWn3/66Sf99NNPabY1GAw6evRoZqYBAAAAAMAuvFy9ZElKUoECLg6Z35yUpKjoOII5IBfJVChnsaT/D4mMtAUAAAAAwBHc87rLYDLp2JdfKvb8ebvO7VqqlCoPHSqj0UAoB+QimQrljh8/ntV1AAAAAADgcLHnz+vGyVOOLgNALuCYm+UBAAAAAACAXIxQDgAAAAAAALCzTN2+miwxMVFr1qzRrl27FBYWJkny9fVVgwYN1Lp1azk5PdDwAAAAAAAAQI6U6dTs2LFjGjJkiC5dupRqM4cFCxZYd2W9c6dWAAAAAAAAAJkM5cLCwtSnTx9FRUXJx8dH7dq1U8mSJSVJFy9e1IoVK3ThwgX17dtXS5YsUeHChbO0aAAAAAAAAOBRlqlQ7rffflNUVJS6dOmiTz/9VPny5Utx/p133tFXX32lBQsWaPLkyfrkk0+ypFgAAAAAAAAgJ8jURg9btmxR0aJFNXz48FSBnCTlzZtXw4YNU9GiRbVp06YHLhIAAAAAAADISTIVyoWEhKh27doymUxptnFyclKtWrUUEhKS6eIAAAAAAACAnChToZyzs7Nu3Lhx33YxMTFydnbOzBQAAAAAAABAjpWpUK5cuXLatWvXPVfBBQcHa9euXSpXrlymiwMAAAAAAAByokyFcs8++6xu3rypXr162Xxm3IYNG9S7d2/Fx8frueeee9AaAQAAAAAAgBwlU7uvvvjii/r777+1Y8cOvfbaaypYsKCKFy8uSbp06ZKuXr0qi8WiRo0a6cUXX8zSggEAAAAAAIBHXaZCOZPJpIkTJ2rs2LGaM2eOoqOjFR0dbT3v6uqql19+WYMHD5bRmKnFeAAAAAAAAECOlalQTrq92cN7772nIUOG6NChQwoLC5Mk+fr6qnr16mzwAAAAAAAAAKQh06FcMmdnZ9WpUycragEAAAAAAAByBe4tBQAAAAAAAOwsU6HcrFmzVLlyZa1fvz7NNuvXr1flypU1d+7cTBcHAAAAAAAA5ESZCuX++ecfeXl56cknn0yzTbNmzeTp6am1a9dmtjYAAAAAAAAgR8pUKHf27FmVL1/+njurmkwmVahQQWfPns10cQAAAAAAAEBOlKmNHq5cuaJ69erdt52Pj4/27duXmSkAAICDGI0GGY0Gu89rMvGoWwAAAOQemQrl3NzcdPny5fu2u3z5slxcXDIzBQAAcACj0SBPDxcZTSZHlwIAAADkaJkK5SpVqqS9e/cqJCREfn5+NtuEhIQoICBAtWrVepD6AACAHRmNBhlNJkUs/ki3Iuz7CIp85ZrIs8UQu84JAAAAOEqmQrkOHTpo586dGjRokCZMmKBChQqlOB8eHq7BgwcrMTFRHTp0yJJCAQCA/dyKOKtbocfsOqeTdxm7zgcAAAA4UqZCueeff16LFy/Wvn379PTTT6tZs2YqW7aspNubQGzevFlxcXGqVauWXnjhhSwtGAAeJp6lBQAAAACwh0yFciaTSZMmTdLHH3+stWvXas2aNTIYbv8j1mKxSJKeeuopjR49Wk5OmZoCAOyOZ2kBAAAAAOwl04lZ/vz5NW7cOB0/flxbtmxRcHCwJMnPz09NmzZVpUqVsqxIALAHnqUFAAAAALCXB17GVqlSJQI4ADkKz9ICAAAAADxsPMQIAAAAAAAAsLMseeBbYmKiZsyYoXXr1ikqKkpFihRR+/bt1blz56wYHgAAAAAAAMhR0hXK/f333xo2bJhefPFFvf322ynOmc1mDRgwQNu3b7du8nDu3Dnt3LlTe/bs0ZgxY7K+agAAAAAAAOARlq7bV3ft2qXo6Gi1bt061bn58+dr27ZtslgsatGihYYOHapXX31V+fLl09KlS7V169YsLxoAAAAAAAB4lKVrpdyBAwdUqFAhValSJdW5efPmyWAwqF27dvruu++sx2vUqKEhQ4Zo6dKlatKkSdZVDAAAAAAAADzi0rVSLjw8XJUrV051/MqVKzp27PYOha+++mqKc61atVKxYsV08ODBLCgTAAAAAAAAyDnSFcpFRUWpQIECqY4fOnRIkuTl5WUztCtXrpwuX778gCUCAAAAAAAAOUu6QjmTyaQrV66kOn706FFJsnlbqyS5u7srMTHxAcoDAAAAAAAAcp50hXJFixbV0aNHlZCQkOL4jh07ZDAYVLNmTZv9oqKi5OPj8+BVAgAAAAAAADlIukK5Bg0aKDo6Wj/99JP12M6dO/Xvv/9Kkpo1a2az37Fjx1S4cOEsKBMAAAAAAADIOdK1++orr7yihQsXaurUqVq+fLm8vLx06tQpSVLNmjVVvXr1VH0CAgJ05coVtW/fPmsrBgAAAAAAAB5x6VopV6pUKX377bdycXFRWFiYjh07psTERBUuXFhjxoyx2WfevHmSpMcffzzrqgUAAAAAAABygHStlJOkVq1aqU6dOtqwYYMiIyPl5+enli1bytXV1Wb76tWrq3LlymrYsGGWFQsAAAAAAADkBOkO5STJ29tbnTt3Tlfbl19+OVMFwXGMRoOMRoPd5zWZ0rVgEwAAAAAAIMfIUCiHnMtoNMjTw0VGk8nRpQAAAAAAAOR4hHKQ9P+r5EwmRSz+SLciztp17nzlmsizxRC7znk3R63WM5stMpstDpkbAAAAAAA4DqEcUrgVcVa3Qo/ZdU4n7zJ2ne9OXq5esiQlqUABF4fMb05KUlR0HMEcAAAAAAC5DKEccjX3vO4ymEw69uWXij1/3q5zu5YqpcpDh8poNBDKAQAAAACQyxDKAZJiz5/XjZOnHF0GAAAAAADIJdj2EgAAAAAAALAzQjkAAAAAAADAzgjlAAAAAAAAADsjlAMAAAAAAADs7KGGcoMGDVLLli0f5hQAAAAAAADAI+ehhnLh4eEKCgp6mFMAAAAAAAAAjxxuXwUAAAAAAADszCk9jfbt25epwWNiYjLVDwAAAAAAAMjJ0hXKdevWTQaDIcODWyyWTPUDAAAAAAAAcrJ0hXLJ/Pz8MjR4eHi4EhMTM9QHAAAAAAAAyOnSFcoVLVpUISEhmjt3rgoXLpzuwV966SUdPHgw08UBAAAAAAAAOVG6NnqoVq2aJOnYsWMPtRgAAAAAAAAgN0hXKFe1alVZLBYdPnz4YdcDAAAAAAAA5Hjpun21Tp06qlSpkmJjYzM0eOfOnfXEE09kqjAAAAAAAAAgp0pXKFe3bl0tWbIkw4N36dIlw30AAAAAAACAnC5dt68CAAAAAAAAyDqEcgAAAAAAAICdpSuUmzFjhrZv3/6wawEAAAAAAAByhXSFcqNGjdJff/1l81zPnj3122+/ZWlRAAAAAAAAQE6Wro0e7mX37t0qVqxYVtQCAAAAAACQ6xmNBhmNBofMbTZbZDZbHDJ3bvPAoRwAAAAAAACyhtFokIeni0xGk0PmTzInKToqjmDODgjlAAAAAAAAsgmj0SCT0aQRa0YoMCrQrnOX9iytYa2HyWg0EMrZAaEcAAAAAABANhMYFaiT4ScdXQb+r737Do+i3Ns4fqdRQgud0KVsIBBK4NDLoQlyQKUoKCCoIKiAlSqCiBSVg0o5YgUBRVEpFiDSpEgNCS2hI0gJhJIGSUib9w/eXVh2ExJIJoF8P9fldeHUZzfPb2f23plnslC6HvQAAAAAAAAAIPOk+0q5f/75R8uXL8/wPEl6/PHHM9gsAAAAAAAA4MGV7lAuKChIQUFBDtNdXFxSnWedTygHAAAAAAAA3JSuUK5s2bJZ3Q4AAAAAAAAg10hXKLd+/fqsbgcAAAAAAACQa/CgBwAAAAAAAMBkhHIAAAAAAACAyQjlAAAAAAAAAJMRygEAAAAAAAAmI5QDAAAAAAAATEYoBwAAAAAAAJiMUA4AAAAAAAAwGaEcAAAAAAAAYDJCOQAAAAAAAMBkhHIAAAAAAACAyQjlAAAAAAAAAJMRygEAAAAAAAAmI5QDAAAAAAAATEYoBwAAAAAAAJiMUA4AAAAAAAAwWY4P5VatWqV+/frpX//6l+rVq6dHH31UX3zxhRITE+952xs3bpSPj498fHw0YMCAe28sAAAAAAAAkA7u2d2AtEyePFkLFiyQu7u7mjRpIk9PT23fvl3Tp0/Xhg0b9PXXXytfvnx3te2oqCiNGzdOLi4uMgwjk1sOAAAAAAAApC7HXim3du1aLViwQJ6enlqyZIm++uorzZo1SwEBAbJYLNq9e7c++eSTu97+pEmTdPnyZfXu3TsTWw0AAAAAAADcWY4N5ebOnStJeuGFF1SrVi3b9GLFimnChAmSpEWLFikmJibD216zZo1+/fVXDRgwQHXq1MmcBgMAAAAAAADplCNDuQsXLmj//v2SpC5dujjMb9iwoby9vZWQkKCNGzdmaNtXrlzRhAkT9NBDD+mVV17JlPYCAAAAAAAAGZEjQ7nQ0FBJkpeXlypUqOB0mdq1a9stm17vvPOOIiIiNHnyZOXNm/feGgoAAAAAAADchRz5oIczZ85Ikry9vVNdpkyZMnbLpsfvv/+ugIAAPfPMM2rQoMG9NfI2Li6ZujnkMvQf5BT0RYA6AKgBIHtRg8gp6It3JyPvW44M5a5duyZJyp8/f6rLFChQwG7ZO7l48aLeffddVaxYUa+//vq9N/I2xYsXyvRtIncoWrRAdjcBkERfBCTqAKAGgOxFDSKnoC+aI0eGclnh7bffVlRUlGbOnJlm2He3Ll+OkWFk+mZN4+bmStFlk4iIa0pOTsnuZkDUAX0REnVAHYAaoAaQvahBahA5ow7oi3fPxSX9F27lyFDOehVcXFxcqstYr5CzLpuWZcuWacOGDXrqqafUuHHjzGnkbQxD93Uoh+xF30FOQV8EqAOAGgCyFzWInIK+mPVyZChXrlw5SVJYWFiqy5w/f95u2bSsWbNGkrR//37169fPbt7FixclSSEhIbZ5M2bMUMmSJTPecAAAAAAAACAdcmQo5+vrK0mKjIzU6dOnnT6B9cCBA5KkWrVqpXu71nWciY6O1s6dOyVJ169fz0hzAQAAAAAAgAzJkaFcmTJl5Ofnp/379+u3337Tiy++aDc/MDBQYWFhypMnj1q3bn3H7f3vf/9Ldd7SpUs1ZswYNW3aVPPnz7/XpgMAAAAAAAB35JrdDUjNkCFDJEmff/65QkJCbNMjIiI0ceJESVLfvn1VqNDNwfPWrFmjTp06qX///uY2FgAAAAAAAMiAHHmlnCS1b99e/fr108KFC9WrVy81adJEnp6e2rZtm6Kjo+Xv769XXnnFbp2YmBj9/fffSkhIyKZWAwAAAAAAAHeWY0M5SRo3bpz8/f313XffKTg4WElJSapYsaIGDRqkAQMGKE+ePNndRAAAAAAAACDDcnQoJ0mdO3dW586d07Vs9+7d1b179wxt/27WAQAAAAAAAO5Fjh1TDgAAAAAAAHhQ5fgr5QAAAAAA5nJ1dZGrq4vp+3Vz47oRALkHoRwAAAAAwMbV1UVFvfLL1c0tu5sCAA80QjkAAAAAgI2rq4tc3dx0aeloJV46Yeq+81VroaJth5u6TwDILoRyAAAAAAAHiZdOKPH8QVP36V78IVP3BwDZiRv2AQAAAAAAAJMRygEAAAAAAAAmI5QDAAAAAAAATEYoBwAAAAAAAJiMUA4AAAAAAAAwGaEcAAAAAAAAYDJCOQAAAAAAAMBkhHIAAAAAAACAyQjlAAAAAAAAAJMRygEAAAAAAAAmI5QDAAAAAAAATEYoBwAAAAAAAJiMUA4AAAAAAAAwGaEcAAAAAAAAYDJCOQAAAAAAAMBkhHIAAAAAAACAyQjlAAAAAAAAAJMRygEAAAAAAAAmc8/uBgAAAOQUbm7Z83tlSoqhlBQjW/YNAACA7EEoBwAAcr1insVkJCercOH82bL/lORkRUTGEcwBAADkIoRyAAAg1yuUt5Bc3Nx0cNIkxZ46Zeq+PStVUs2335arqwuhHAAAQC5CKAcAAPD/Yk+d0tUjR7O7GQAAAMgFeNADAAAAAAAAYDJCOQAAAAAAAMBkhHIAAAAAAACAyQjlAAAAAAAAAJMRygEAAAAAAAAmI5QDAAAAAAAATEYoBwAAAAAAAJiMUA4AAAAAAAAwGaEcAAAAAAAAYDJCOQAAAAAAAMBkhHIAAAAAAACAyQjlAAAAAAAAAJMRygEAAAAAAAAmI5QDAAAAAAAATEYoBwAAAAAAAJiMUA4AAAAAAAAwGaEcAAAAAAAAYDL37G4AAABw5OrqIldXF9P36+bG73UAAACAGQjlAADIYVxdXeTl5UlABgAAADzACOUAAJKy7wqplBRDKSlGtuw7p3J1dZGbm6vGfbdZf4dHmbrvZj5l9fIj/qbuEwAAAMiNCOUAIJcr5llMRnKyChfOny37T0lOVkRkHMGcE3+HR+nQ2Sum7rNyycKm7g8AAADIrQjlACCXK5S3kFzc3HRw0iTFnjpl6r49K1VSzbfflqurC6EcAAAAgFyFUA4AIEmKPXVKV48cze5mAAAAAECuwAjSAAAAAAAAgMkI5QAAAAAAAACTEcoBAAAAAAAAJiOUAwAAAAAAAExGKAcAAAAAAACYjFAOAAAAAAAAMBmhHAAAAAAAAGAyQjkAAAAAAADAZIRyAAAAAAAAgMkI5QAAAAAAAACTEcoBAAAAAAAAJiOUAwAAAAAAAExGKAcAAAAAAACYjFAOAAAAAAAAMBmhHAAAAAAAAGAyQjkAAAAAAADAZO7Z3QAAAAAAAICcxtXVRa6uLqbv182N66dyC0I5AAAAAACAW7i6uqioV365urlld1PwACOUAwAAAAAAuIWrq4tc3dx0aeloJV46Yeq+81VroaJth5u6T2QPQjkAAAAAAAAnEi+dUOL5g6bu0734Q6buD9mHG5UBAAAAAAAAkxHKAQAAAAAAACYjlAMAAAAAAABMRigHAAAAAAAAmIxQDgAAAAAAADAZoRwAAAAAAABgMkI5AAAAAAAAwGSEcgAAAAAAAIDJCOUAAAAAAAAAkxHKAQAAAAAAACYjlAMAAAAAAABMRigHAAAAAAAAmIxQDgAAAAAAADCZe3Y3AAAAAEDO4OaWPb/Zp6QYSkkxsmXfAABkF0I5AAAAIJcr5llMRnKyChfOny37T0lOVkRkHMEcACBXIZQDAAAAcrlCeQvJxc1NBydNUuypU6bu27NSJdV8+225uroQygEAchVCOQAAAACSpNhTp3T1yNHsbgYAALkCD3oAAAAAAAAATEYoBwAAAAAAAJiMUA4AAAAAAAAwGaEcAAAAAAAAYDJCOQAAAAAAAMBkhHIAAAAAAACAyQjlAAAAAAAAAJMRygEAAAAAAAAmI5QDAAAAAAAATEYoBwAAAAAAAJiMUA4AAAAAAAAwGaEcAAAAAAAAYDJCOQAAAAAAAMBk7tndANhzdXWRq6uL6ft1cyOfBQAAAAAAMAuhXA7i6uoiLy9PAjIAAAAAAIAHHKFcDuLq6iI3N1eN+26z/g6PMnXfzXzK6uVH/E3dJwAAAAAAQG5FKJcD/R0epUNnr5i6z8olC5u6PwAAAAAAgNyM+yQBAAAAAAAAkxHKAQAAAAAAACYjlAMAAAAAAABMRigHAAAAAAAAmIxQDgAAAAAAADAZoRwAAAAAAABgMkI5AAAAAAAAwGSEcgAAAAAAAIDJCOUAAAAAAAAAk7lndwMAAAAAAACQc7i5Zc81XCkphlJSjGzZd3YglAMAAAAAAICKeRaTkZyswoXzZ8v+U5KTFREZl2uCOUI5AAAAAAAAqFDeQnJxc9PBSZMUe+qUqfv2rFRJNd9+W66uLoRyAAAAAAAAyH1iT53S1SNHs7sZDzwe9AAAAAAAAACYjFAOAAAAAAAAMBmhHAAAAAAAAGAyQjkAAAAAAADAZDn+QQ+rVq3Sd999p0OHDikxMVEVK1ZU165dNWDAAHl4eKR7O6Ghodq8ebO2bt2qo0ePKioqSp6enqpevbr+85//6Mknn8zQ9gAAAAAAAIC7laNDucmTJ2vBggVyd3dXkyZN5Onpqe3bt2v69OnasGGDvv76a+XLl++O20lKSlK3bt0kSZ6envLz81OJEiV0/vx57dmzR7t379by5cv11VdfqXDhwln9sgAAAAAAAJDL5dhQbu3atVqwYIE8PT21aNEi1apVS5J05coV9e/fX7t379Ynn3yiUaNGpWt7tWrV0qBBg9SuXTvlyZPHNv3w4cN6/vnntW/fPk2dOlVTp07NktcDAAAAAAAAWOXYMeXmzp0rSXrhhRdsgZwkFStWTBMmTJAkLVq0SDExMXfclru7u5YuXapHHnnELpCTJB8fH40YMUKStHLlSiUmJmbWSwAAAAAAAACcypGh3IULF7R//35JUpcuXRzmN2zYUN7e3kpISNDGjRvveX++vr6SpPj4eEVERNzz9gAAAAAAAIC05MhQLjQ0VJLk5eWlChUqOF2mdu3adsvei1OnTkmSPDw85OXldc/bAwAAAAAAANKSI8eUO3PmjCTJ29s71WXKlCljt+zdMgxDX375pSSpTZs2Dre3ppeLyz01A7kc/QegDgCJOgCoAYA6AKT7uw4y0vYcGcpdu3ZNkpQ/f/5UlylQoIDdsndr9uzZCg4Olqenp95444273k7x4oXuqR3IvYoWLZDdTQCyHXUAUAcANQBQB4CUu+ogR4ZyZlm+fLnmzJkjV1dXTZkyRZUrV77rbV2+HCPDuLf2uLm55qrOhxsiIq4pOTklu5sBUYPZiTqwR1/MnaiDnIH6yz7UQM5BHWQf6iDnoA6yz/1eBy4u6b9wK0eGctar4OLi4lJdxnqFnHXZjFq1apXGjh0rSZo0aZIeeeSRu9qOlWHonkM55F70HYA6ACTqAKAGAOoAkHJPHeTIBz2UK1dOkhQWFpbqMufPn7dbNiP++OMPvfnmm0pJSdG7776rnj173l1DAQAAAAAAgLuQI0M5X19fSVJkZKROnz7tdJkDBw5IkmrVqpWhba9du1avv/66kpOT9c477+jJJ5+8t8YCAAAAAAAAGZQjQ7kyZcrIz89PkvTbb785zA8MDFRYWJjy5Mmj1q1bp3u769ev16uvvqqkpCS988476t27d6a1GQAAAAAAAEivHBnKSdKQIUMkSZ9//rlCQkJs0yMiIjRx4kRJUt++fVWo0M3B89asWaNOnTqpf//+DtvbuHGjhg8frqSkJE2cOJFADgAAAAAAANkmRz7oQZLat2+vfv36aeHCherVq5eaNGkiT09Pbdu2TdHR0fL399crr7xit05MTIz+/vtvJSQk2E2/fPmyhg4dqsTERJUpU0bBwcEKDg52ut+RI0eqWLFiWfa6AAAAAAAAgBwbyknSuHHj5O/vr++++07BwcFKSkpSxYoVNWjQIA0YMEB58uRJ13bi4uJsQd358+e1bNmyVJcdOnQooRwAAAAAAACyVI4O5SSpc+fO6ty5c7qW7d69u7p37+4wvXz58jp8+HBmNw0AAAAAAAC4Kzl2TDkAAAAAAADgQUUoBwAAAAAAAJiMUA4AAAAAAAAwGaEcAAAAAAAAYLIc/6AH4EHn5pY92XhKiqGUFCNb9g0AAAAAQG5HKAdkE49ixZSSYqhw4fzZsv+U5BRFRMYSzAEAAAAAkA0I5YBs4l6woFxdXfTHt0GKuHDV1H0XLV1QD/fxl6urC6EcAAAAAADZgFAOyGYRF67q4tmo7G4GAAAAAAAwEQ96AAAAAAAAAExGKAcAAAAAAACYjFAOAAAAAAAAMBmhHAAAAAAAAGAyQjkAAAAAAADAZIRyAAAAAAAAgMkI5QAAAAAAAACTEcoBAAAAAAAAJiOUAwAAAAAAAExGKAcAAAAAAACYjFAOAAAAAAAAMBmhHAAAAAAAAGAyQjkAAAAAAADAZIRyAAAAAAAAgMkI5QAAAAAAAACTEcoBAAAAAAAAJiOUAwAAAAAAAExGKAcAAAAAAACYjFAOAAAAAAAAMBmhHAAAAAAAAGAyQjkAAAAAAADAZIRyAAAAAAAAgMkI5QAAAAAAAACTEcoBAAAAAAAAJiOUAwAAAAAAAExGKAcAAAAAAACYjFAOAAAAAAAAMBmhHAAAAAAAAGAyQjkAAAAAAADAZIRyAAAAAAAAgMkI5QAAAAAAAACTEcoBAAAAAAAAJiOUAwAAAAAAAExGKAcAAAAAAACYjFAOAAAAAAAAMBmhHAAAAAAAAGAyQjkAAAAAAADAZIRyAAAAAAAAgMkI5QAAAAAAAACTEcoBAAAAAAAAJiOUAwAAAAAAAExGKAcAAAAAAACYjFAOAAAAAAAAMBmhHAAAAAAAAGAyQjkAAAAAAADAZIRyAAAAAAAAgMkI5QAAAAAAAACTEcoBAAAAAAAAJiOUAwAAAAAAAExGKAcAAAAAAACYjFAOAAAAAAAAMBmhHAAAAAAAAGAyQjkAAAAAAADAZIRyAAAAAAAAgMkI5QAAAAAAAACTEcoBAAAAAAAAJiOUAwAAAAAAAExGKAcAAAAAAACYjFAOAAAAAAAAMBmhHAAAAAAAAGAyQjkAAAAAAADAZIRyAAAAAAAAgMkI5QAAAAAAAACTEcoBAAAAAAAAJiOUAwAAAAAAAExGKAcAAAAAAACYjFAOAAAAAAAAMBmhHAAAAAAAAGAyQjkAAAAAAADAZIRyAAAAAAAAgMkI5QAAAAAAAACTEcoBAAAAAAAAJiOUAwAAAAAAAExGKAcAAAAAAACYjFAOAAAAAAAAMBmhHAAAAAAAAGAyQjkAAAAAAADAZIRyAAAAAAAAgMkI5QAAAAAAAACTEcoBAAAAAAAAJiOUAwAAAAAAAExGKAcAAAAAAACYjFAOAAAAAAAAMBmhHAAAAAAAAGAyQjkAAAAAAADAZIRyAAAAAAAAgMkI5QAAAAAAAACTEcoBAAAAAAAAJiOUAwAAAAAAAExGKAcAAAAAAACYjFAOAAAAAAAAMBmhHAAAAAAAAGAyQjkAAAAAAADAZIRyAAAAAAAAgMkI5QAAAAAAAACTEcoBAAAAAAAAJiOUAwAAAAAAAExGKAcAAAAAAACYjFAOAAAAAAAAMBmhHAAAAAAAAGAyQjkAAAAAAADAZIRyAAAAAAAAgMkI5QAAAAAAAACTEcoBAAAAAAAAJiOUAwAAAAAAAExGKAcAAAAAAACYjFAOAAAAAAAAMBmhHAAAAAAAAGAyQjkAAAAAAADAZIRyAAAAAAAAgMkI5QAAAAAAAACTEcoBAAAAAAAAJnPP7gbcyapVq/Tdd9/p0KFDSkxMVMWKFdW1a1cNGDBAHh4eGd7egQMH9PnnnyswMFAxMTEqWbKk2rRpo5deeknFixfPglcAAAAAAAAA2MvRV8pNnjxZr776qoKCglSnTh21bNlSYWFhmj59uvr376/4+PgMbW/16tXq1auXAgICVLZsWbVr106urq5atGiRHn30UZ06dSqLXgkAAAAAAABwU469Um7t2rVasGCBPD09tWjRItWqVUuSdOXKFfXv31+7d+/WJ598olGjRqVrexcuXNDo0aOVlJSkd999V7169ZIkJScna/To0frll1/0xhtv6Mcff5SLi0uWvS4AAAAAAAAgx14pN3fuXEnSCy+8YAvkJKlYsWKaMGGCJGnRokWKiYlJ1/a++eYbxcXFqVmzZrZATpLc3Nz0zjvvqFChQtq/f7+2bNmSia8CAAAAAAAAcJQjQ7kLFy5o//79kqQuXbo4zG/YsKG8vb2VkJCgjRs3pmuba9euTXV7BQoUUNu2bSVJa9asudtmAwAAAAAAAOmSI0O50NBQSZKXl5cqVKjgdJnatWvbLZuWq1ev2saLs653L9sDAAAAAAAA7kWODOXOnDkjSfL29k51mTJlytgtm5azZ8/a/l22bFmny1j3lZ7tAQAAAAAAAPciRz7o4dq1a5Kk/Pnzp7pMgQIF7JZNz/bS2qanp6ekG1fV3Q1XV8kw7mpVBzXKFlP+POb+aSqVKixJylOmplw8Un/fs4J7iYckSZYSFuVzz2fqvit6VZQkFaxeXa75zN23Z8Ub+y5RrrDc87iZum+vkgVs/3bNgdG8i4tLtjxwxc3txptBHZjH8/+vhra+92YzDENGZn14ZwGOB+ahDnJeHXAsoAbMklNrQKIOqAPzUAeOqAPq4G5lpLu6GDmw8ubOnauPPvpI/v7+Wrx4sdNlPvroI82dO1ctWrTQV199leb2goKC9NRTT0mSQkJC5O7u+AXnr7/+0nPPPScPDw8dOHDg3l8EAAAAAAAAkIoceI3Mzavg4uLiUl3GevWbddn0bC+tbcbGxkqSChYsmO52AgAAAAAAAHcjR4Zy5cqVkySFhYWlusz58+ftlk3P9iTp3LlzTpex7is92wMAAAAAAADuRY4M5Xx9fSVJkZGROn36tNNlrLeY1qpV647bK1iwoCpVqmS33r1sDwAAAAAAALgXOTKUK1OmjPz8/CRJv/32m8P8wMBAhYWFKU+ePGrdunW6ttm+fftUt3ft2jVt2LBBktShQ4e7bTYAAAAAAACQLjkylJOkIUOGSJI+//xzhYSE2KZHRERo4sSJkqS+ffuqUKFCtnlr1qxRp06d1L9/f4ft9e/fX/nz59fWrVu1ZMkS2/Tk5GRNnDhR0dHR8vPzU4sWLbLqJQEAAAAAAACScujTV63ee+89LVy4UB4eHmrSpIk8PT21bds2RUdHy9/fX/PmzVO+Wx7Ru3TpUo0ZM0blypXT+vXrHba3atUqvfHGG0pOTlbdunVVrlw57d+/X6dPn1aJEiX03Xff2W5zBQAAAAAAALKKe3Y3IC3jxo2Tv7+/vvvuOwUHByspKUkVK1bUoEGDNGDAAOXJkydD23vkkUdUoUIFffbZZwoMDFRoaKhKlSqlPn366KWXXlKJEiWy6JUAAAAAAAAAN+XoK+UAAAAAAACAB1GOHVMOAAAAAAAAeFARyuUiPj4+8vHxMWVfZ86ckY+Pj9q2beswr23btvLx8dGZM2cc5h07dkwvvfSSmjZtqpo1a8rHx0ezZs2SJPXr108+Pj7asWNHlrdfujFGoY+Pj0aPHm3K/pBzmVk7OVFMTIxWrVqlsWPHqnPnzqpbt678/PzUrl07jRkzRocPH87uJuIBZ/bnP3A3Ro8eLR8fHy1dujRTt7t582YNGjRIjRs3Vu3atdW2bVuNHz9e58+fT3Ud67lWav89+eSTmdpGIKdJ67sIcK9S+z6b2ecruf07SG6Ro8eUQ+4SGxurF154QWfPnlXt2rXVokULubm5qWbNmtndNCBX+/LLLzV37lxJUuXKldWqVSslJycrJCRES5cu1a+//qpJkyapW7du2dxSAHiwfPzxx/r0008lSbVq1VL58uV1+PBh/fDDD1q1apW++eYb+fr6prp+x44d5enp6TC9QoUKmdrOfv36aefOnVqwYIEaN26cqdsGAGS9M2fOqF27dqk+NBNZh1AOWaJ06dJauXKlPDw80r3O/v37dfbsWdWvX1/ff/+9w/z3339fcXFxKlu2bGY2FcAdeHp66tlnn1Xv3r1VuXJl2/TExERNnz5d8+fP19tvvy1/f3+eYI0swec/7gevv/66Bg0apFKlSmXK9jZu3KhPP/1Urq6u+uijj9SpUydJkmEYmjNnjmbNmqVhw4Zp1apVqT78bOTIkSpfvnymtAcAkDbOV3A3COWQJTw8PFS1atUMrRMWFiZJdl/6b8WHG5A9Bg8e7HS6h4eHRo0apT///FMnT57U77//rpdeesnk1iE34PMf94NSpUplWiAnSQsWLJAkPfbYY7ZATpJcXFz08ssva/369QoJCdGKFSv0xBNPZNp+AQB3h/MV3A3GlMulAgIC9NRTT8nf31/16tVT7969tXHjRqfLHjt2TDNnzlTv3r3VsmVL1a5dW40bN9aAAQO0cuVKp+tkZByHHTt2yMfHR6NGjZIkLVu2zG7cE6vU7tG/dQyX06dPa8SIEWrevLlq166t9u3b66OPPlJCQoLTfSclJWn+/Pnq2rWr/Pz81KRJEw0bNixdY2T9/fffGj9+vNq3by8/Pz81aNBAffr00YoVK5wuf2v7AwMDNWTIEDVp0kQ1atTI9PFnkHUyUjvSjT72448/ql+/fmrUqJFtPKAJEybYguhbWeuhX79+iouL04wZM9ShQwf5+fmpRYsWGjt2rC5cuOB0X1u3btWkSZP02GOP2cYeatWqlV599VXt27fP6TqzZs2yjd147tw5jR07Vq1bt1atWrXSNZ6iq6urrU7TGt8IkOzHRlmyZIm6d++uevXqqWHDhho0aJD27NnjdL20xmiJjY3Vxx9/rIcfftg29MGYMWN04cIFu/59q5SUFP3www/q3bu3GjZsqFq1aqlp06Z69NFHNWnSJKdjnuLBYxiGGjdurBo1aigiIsJu3r59+2z99dtvv3VYt127dvLx8dHp06dt01IbU+7WfnjlyhVNnDhRrVu3Vu3atdW6dWtNmjRJ0dHRDvvYv3+/JKlp06YO81xcXNSkSRNJN45LmSWjNWo9Zu3cuVOS9Mwzz9idw936Xhw4cECvvvqqWrVqpdq1a8vf31/t2rXTsGHDtHbt2kx7Dcg++/bt0wcffKCePXvazsWbNWumIUOGaOvWrU7XuXUM59jYWP33v/9Vhw4dVLt2bTVv3lyjRo1K9bxHkjZs2KC+ffuqfv36atCggZ5++ul09aeoqCjNnDlTjz32mOrXr6+6deuqa9eu+t///qe4uDiH5Tlu5D7Hjh3T8OHD1bhxY9WpU0ddunTRV199peTk5FTXSe185cqVK1qwYIEGDRqktm3bqk6dOvL391f37t31+eef6/r163dsT0bOm6SMfQcZPXq02rVrJ0k6e/aswziktztw4IDeeOMN/fvf/1bt2rXVqFEjPf/886l+JwoPD9d7772njh07ys/PT3Xr1lXr1q3Vv39/ffXVV3d87Q86rpTLhWbOnKn//e9/ql+/vlq3bq0TJ04oODhYgwcP1qxZs9ShQwe75efNm6effvpJVapUkcViUeHChRUWFqYdO3Zo27Zt2rt3r8aMGXPX7SlRooS6deumU6dOKSgoSBUrVlSDBg0yvJ2DBw9q8uTJKlKkiP71r38pKipKQUFBmjt3ro4dO6Y5c+bYLZ+SkqJXXnlFa9eulYeHhxo3bqzChQtr7969euKJJ9SjR49U97Vq1SqNGjVK169fV5UqVdS6dWvFxMRo3759GjlypLZv366pU6c6XXf16tX6/vvvVaVKFTVr1kxRUVGp3naCnCWjtXP16lW9+OKL2rlzpzw9PVW7dm0VLVpUR44c0ffff6/Vq1dr3rx5TscDSkxM1IABA3T48GE1atRIvr6+2r17t37++Wdt2rRJixYtcriq1HqQrV69uvz9/eXu7q4TJ05o1apVWrNmjWbMmKGOHTs6fW0nT55Ut27d5OHhIX9/fxmGoaJFi6brfTl16pQkqWTJkulaHpg6daq++eYb25fyI0eOaNOmTdq6das+/vhjh1pKTWxsrJ555hnt379fnp6eatGihfLmzavNmzdr48aNat26tdP13nrrLS1dulR58+ZVgwYNVKxYMUVGRurMmTNatGiRmjZtyi1/uYA12Fq9erW2bdumzp072+bdGiBs27ZNffr0sf3/6dOndebMGZUvXz5DY7OFhYWpW7duSkpKkr+/v65fv66goCAtWrRIe/fu1eLFi+2G/YiNjZUkeXl5Od2e9TM6JCQk1X0uXbpUUVFRSkpKUqlSpdSoUSP961//umNb01uj1nO4zZs369KlS2rRooXdsaBixYqSbryHgwYNUmJiomrUqKF69eopJSVFFy5c0J9//qnk5GS1b9/+ju1CzjZjxgzt2LFD1apVU61atZQ/f36dPn1aGzZs0IYNGzR27Fj179/f6boxMTHq3bu3wsLC1KBBA1WvXl179uzR8uXLtWvXLq1YsUKFChWyW2f+/Pm28+06deqoYsWKOnnypF5++WU9++yzqbbz2LFjGjhwoMLCwlSyZEk1aNBA7u7u2r9/vz755BP98ccfWrhwod3+OG7kLoGBgRo0aJBiY2NVoUIFNW/eXBEREfroo4+0d+/eDG9v8+bNmjx5skqXLq1KlSqpXr16unLlivbu3av//ve/Wr9+vRYsWJDqd8KMnjdl9DtIgwYNFBsbq4CAAHl6eqb6fUGSvvnmG02bNk0pKSmqWbOm6tSpo0uXLmnHjh3asmWLhg0bpqFDh9qWv3jxonr06KHw8HCVLVtWLVu2VN68eRUeHq5Dhw4pJCREzz//fIbf0weKgVzDYrEYFovFaNiwobFnzx67eTNnzjQsFovx8MMPO6y3Y8cO459//nGYfvz4caNVq1aGxWIx9u7dazfv9OnThsViMdq0aeOwXps2bQyLxWKcPn3abvrPP/9sWCwWY9SoUU7b37dvX8NisRjbt2+3mz5q1Cjba5sxY4aRlJRkm3f48GGjXr16hsViMYKCguzWW7RokWGxWIxmzZoZx44ds01PTEw0JkyYYNvm7e05dOiQUbt2bcPPz88ICAiwm3fmzBmjS5cuhsViMZYtW+a0/RaLxVi0aJHT14ic6W5r5/XXXzcsFosxePBg49KlS3bz5s2bZ1vv1j67fft22/46dOhgnD171jYvPj7eGDZsmGGxWIwnn3zSYX9r1qwxIiMjnU739fU1GjVqZMTFxTltv8ViMd58803j+vXr6XtT/t/GjRsNi8Vi+Pj4GAcPHszQush9rH2tTp06xtatW+3mffHFF4bFYjEaNGjgUC+pff5PmTLFsFgsRufOnY0LFy7Ypt9aKxaLxZg5c6Zt3tmzZw2LxWK0atXKCA8Pd2jjsWPH7OoOD7bvv//esFgsxrhx4+ym9+vXz6hVq5bRqVMno2HDhnaf06mtYz0f+fnnn+2m3/o5O3r0aLvP2XPnzhktW7Y0LBaL8euvv9qtZ52e2jnD22+/bdvutWvX7OZZz7Wc/dejRw/j5MmTTreZ2TVq1a9fP8NisRgrVqxwmBcdHW0EBwc7XQ/3lz///NPus9gqKCjI8Pf3N2rVqmWcP3/ebp71/N9isRjPPfecERMTY5sXGRlpPPbYY4bFYjHmzp1rt97BgweNmjVrGjVq1DBWrVplN2/FihWGj4+P0+8icXFxRvv27Q2LxWJ89NFHdvUYGxtrO3cbPXq0bTrHjdwlPj7eaN26tWGxWIzJkyfbff4fPHjQaNy4sa3P3v59NrXPwmPHjjn9nIuMjDSee+45w2KxGF988YXD/Lv9TL6b7yBpfX+32rRpk+Hj42M0btzY2Llzp928Q4cO2bKBHTt22KbPmjXLsFgsxttvv22kpKTYrZOQkODwunIjbl/NhYYPH666devaTRs8eLAKFSqkkydPOlzO2qhRI6e/BFepUsU2ftTq1auzrsHpVKtWLb366qtyc3OzTbNYLHr00UclyeGy+W+++UaSNHToULvx79zd3TVmzJhUr/qZO3euEhIS9Oqrr+rhhx+2m1euXDlNnjxZ0s2xYG7XpEkTu1/ccf/ISO0cP35cv//+u0qVKqXp06erePHidusNGDBArVu31smTJ7Vp0yan+xs5cqTd2BR58+bVhAkTlD9/fu3Zs0dBQUF2y7dv315FihRx2E779u3VqVMnRUZGpvqIdi8vL40fPz5DV21euHBBb731liTpySefVI0aNdK9LnK3Xr16OdySN3DgQNWuXVsxMTH68ccf77iN+Ph4LVmyRJI0ZswYu7G88ubNq3feeUf58+d3WO/SpUuSJF9fX6ef81WrVmVMmFykWbNmkuzPEeLj4xUcHKz69eurTZs2io6O1oEDB2zzrcs6u600LWXKlHH4nPX29lbfvn0d2iDJdnvqTz/9JMMw7OZFRUXZnXtdvXrVbn7r1q313//+V2vWrNG+ffu0bt06vf/++ypbtqz279+vfv366fLly6m2NTNq9FbWfTm7erVQoUKqV69ehraHnKl169ZOx1WsX7+++vTpo8TExFRvLfX09NTUqVNVsGBB27QiRYrohRdekORYH4sWLVJycrI6depkN+aiJD366KOpDqGzbNky/fPPP2rTpo1effVVu3rMnz+/3n33XRUvXly//PKLoqKiJHHcyG0CAgIUFhYmb29vjRgxwu67ZY0aNTRkyJAMb7Nq1apOP+eKFCmicePGSUr7+3RGPpMz4ztIambNmiXDMDRx4kSHq66tt6FLN+rTyvr537JlS7m4uNit4+HhkeFj6YOIUC4XatOmjcO0PHny2II3Z+M2XLt2TatWrdKMGTP09ttva/To0Ro9erT++OMPSTfGV8tubdq0cSh0SbbA7dbXdeHCBdstd9bQ7lZ58+Z1OMBLN255tX543Xqby638/Pzk6empgwcPOh0fIK3LgZGzZaR2Nm7cKMMw1KpVK7sTzFs1atRIkhQcHOwwr3DhwraxHW5VvHhxtWzZUpJsY/jc6sKFC1qyZImmTZumt956y1arR48elZR6rTZt2tThtpC0XL16VUOGDFF4eLjq1KljC+eA9OjWrZvT6Y8//rgk5337dgcOHFBsbKyKFi2qFi1aOMwvVqyYLXC5VZUqVVSgQAFt2rRJn376qd2YYMh9KlSooPLly+vMmTP6559/JN24bSkhIUHNmjVzCO0Mw9D27dvl4uKS4S8STZs2dRoUOztPkaRBgwYpb968Cg0N1dChQ3XkyBFdu3ZNwcHBevbZZ223t0o3xve81YQJE9SlSxdVrFhRefPmVfny5fX4449r2bJlKleunC5cuKC5c+em2tbMqNFb1alTR5L05ptvKjAwUElJSRlaH/ePiIgILV++XB988IHGjRtnOw+x9pnUzkNq167tNNCrUqWKJMf6sG7P2Xm8lHofto559cgjjzidX6BAAdWuXVtJSUm2cR05buQu1r71yCOP2A0pYJVa37qT5ORkbdu2TXPmzNE777yjMWPGaPTo0bbP4rS+T2fkM/lev4Ok5sqVK9q3b5/y5cvn9DuRJDVu3FiS7C4csH7+T58+XX/88YeuXbuW7n3mFowplwul9kuOtWhvD5LWr1+vMWPGKDIyMtVt3v4LbXbw9vZ2Ot36um592IN1QPqiRYuqQIECTtdzNi5EZGSk7bWmNlbR7cuXLl3ablq5cuXuuB5ypozUjvWE7aefftJPP/2U5navXLniMK1cuXJOQ2bpZt+8/cEKs2fP1ty5c5WYmJjqvlKr1Yz0y2vXrmngwIEKDQ2Vr6+vvvzyS+XNmzfd6wOpjbuTWt92xvoFLa2+62xewYIFNXXqVI0ZM0Yff/yxPv74Y5UsWVL16tVTy5Yt1aVLl1SPC3gwNWvWTEuWLNHWrVtVsWJFWwDXvHlzWSwW5cmTR1u3btWLL76o0NBQRUZGytfXN93jblpl5DxFkqpXr65Zs2bpzTff1Nq1a+2uMPLy8tLo0aM1adIkubi4qHDhwulqg5eXl/r3768pU6Zow4YNqf6gkhk1eqvXX39dhw8f1qZNm7Rp0ybly5dPvr6+atSokR599FG7OxZw/1qyZImmTp1qFxjfLrUv5BmtD2sfvFNfvZ31/GzkyJEaOXJkqu2Ubp6fcdzIXe7Ut4oUKaJChQopJiYm3ds8efKkhg4davuR3Jm0vk9n5DP5Xr+DpObMmTMyDEPx8fHy8/NLc9lbH5702GOP6a+//tKvv/6qYcOGyc3NTVWrVlWDBg3UsWNHrpQToVyudPuvqWm5cOGCXnvtNcXHx2vgwIHq2rWrypcvL09PT7m6umrLli05ZmDGjLyuu5WSkmL7d3p+JXH260q+fPkytU0wT0b6mLWv1KxZ8463dd5+S2x63Xo70x9//KFZs2bJ09NTb7/9tpo0aaJSpUopX758cnFx0YwZM/TZZ5853AJlld5+GRsbq8GDBys4OFg+Pj76+uuvnd4yC9yL1PqpM6mF12nN69ixo5o1a6Z169Zp9+7dCgoK0po1a7RmzRrNnDlTX3/9tdOnjeHB1LRpU1so17t3b23btk1FihRR7dq15erqqvr16ysoKEhxcXF3feuqdHfnKa1bt9a6desUEBCgw4cPKykpSdWqVVPnzp0VGhoqSapcuXKGhh6wBmD38sTsjNSodONBQD///LN27typrVu3KigoSPv27VNQUJA+++wzvf7667bbFHF/OnDggMaPHy83Nze9+eabatu2rby9vZU/f365uLjohx9+0Pjx41PtO2acx0s3z89atmypEiVKpLnsrT/GctzAvRg+fLiOHj2qNm3aaODAgapataoKFiwoDw8PJSQk3DHkupNb6yqrvoNY93GnB0HcztXVVdOnT9eQIUP0559/KigoSEFBQVq8eLEWL16sNm3aaM6cOXa3Cec2hHJI0/r16xUfH68OHTpoxIgRDvOtt4Deb6xXr0VEROjatWtOf906e/asw7SiRYsqX758io+P18iRI1WsWLEsbyvuT9ZffP39/TV+/PgMr++s/90+r0yZMrZpq1atkiS99tpr6tWrl8M6J0+ezHAbbhcXF6fBgwdr165d8vHx0fz58zN8pQgg3fi1tWbNmg7TnfXt1Fg/x9NTK84UKlRIjz/+uO3Wj7CwME2aNEnr1q3TpEmT7MZDwYOtadOmcnFx0Y4dO3T58mUdPHhQHTp0sIUEzZo1044dO7Rr1y5t27bNNs0shQsX1hNPPOEwPTAwUNKNK/oywnrnQ1pX9mRGjd7OxcVFjRs3tt3edP36dS1dulTvvvuuPvroI3Xq1Mn2tFbcf1avXi3DMNS3b18NGjTIYX5mnIfcqnTp0vrnn3909uxZVa9e3WF+ap//3t7eOnHihHr27Ol0qJq0cNzIHaznF2fOnHE6Pzo6OkNXyR0/flyHDx9W8eLFNXv2bLm720cw6fk+nZHP5Hv9DpIa6z5cXFw0ZcqUDAfp1apVU7Vq1STdHArijTfe0IYNG7R8+XL16NEj09p6v2FMOaTJOsCps9v2DMPQr7/+anaTMkWZMmVs44D99ttvDvMTEhKcDrbp5uZmOxG3hiCAM61atZJ0I9h2NrbgnURHR2v9+vUO069cuaLNmzdLujkehJR2rV6+fNlhgOSMio+P1+DBg7Vz505bIEcojbu1YsWKNKff2rdTU6tWLeXPn19Xrlxx2r9Tm54ab29vDR8+XJJ08ODBdK+H+1/RokVVs2ZNRUZG6ssvv5RhGHahm/Xff/75p3bv3q08efKoYcOG2dVcSVJMTIx++uknubm56amnnsrQur///rukm+P8OJPRGrXeGZCcnJzuduTNm1dPPfWUfHx8lJKSosOHD6d7XeQ8aZ2HXL9+3TYOdWaxDjKf2neR5cuXO51uPT/LjPN4jhsPJmvfWr16tdMhYVLrW6mx1kapUqUcAjlJ+uWXX+64jYx8Jt/tdxDr53hqY36WLl1aPj4+unbtmu27yN2yjsvapUsXSdQPoRzSZL3FISAgQOHh4bbpycnJ+uSTTzI0OGRO079/f0k3niJz/Phx2/Tk5GS9//77dq/3VkOHDpWHh4c+/PBDLVu2zO6WVqsjR45k+skH7i++vr7q2LGjwsLCNHToUKe/tsXGxuqXX36xPdXrdu+//77d7UUJCQmaOHGiYmNjVadOHTVo0MA2zzoQ8pIlS+zGXYmJidGoUaMy9Ive7a5fv64XX3xRO3bsIJBDpli8eLHDk4Dnz5+vffv2qUCBAurZs+cdt5E/f37bclOnTrWro4SEBE2aNMnpuEahoaFauXKl4uPjHeZZg3Ceopf7WG9H/fbbbyXZX31Wu3ZtFS5cWD/99JPi4+NVv35904ai2Ldvn8PtfufPn9eLL76oixcvauDAgbYrD6zWrl1r97RYq6tXr2ry5Mm2fv7ss8+mut+M1qj1ypLUxkv66quvdO7cOYfpx48ft10lQt3d36zfGZYvX243Ntb169f1zjvvpHrV0d3q16+f3NzctGrVKq1Zs8Zu3u+//57qU16ffPJJlStXTqtXr9aHH37odByvixcv2p7uLXHcyG06deqk0qVL69y5c5oxY4bdd70jR47o008/zdD2KleuLDc3Nx05csThc3X9+vWaP3/+HbeRkc/ku/0OUqxYMXl4eOjSpUupjiX/6quvSrrx1HtnFw8YhqG9e/dqy5YttmnLly9P9ZhkfUBFbh9zndtXkaY2bdqoVq1aCgkJUceOHdWoUSPlz59f+/btU3h4uAYNGqQvvvgiu5t5V/r06aO//vpLGzZs0GOPPabGjRurSJEi2rt3ry5evKinnnpKixcvdlivVq1a+vDDD21PzPn4449VrVo1FS1aVFFRUTpy5IjOnz+vzp076+GHH86GV4acYsqUKYqOjtamTZvUqVMn1ahRQ+XLl5dhGDp79qwOHTqkxMRErVy50mFck/r16yslJUWdOnVSkyZNlC9fPu3evVvh4eEqXry43n//fbvl+/fvrxUrVmjjxo1q37696tWrp8TERO3atUv58uVTjx499PPPP9/V65gxY4btiqOyZcvqgw8+cLpcgwYNnN5iBdyuV69e6t+/vxo2bKjSpUvryJEjOnLkiNzc3DRlyhSVLFkyXdt57bXXFBQUpJCQEHXo0EFNmjRR3rx5tXv3biUmJqpbt25atmyZ3fie586d02uvvWYbaN7b21tJSUk6cuSI/v77b3l4eDgdrgEPtmbNmumrr77S9evXVb58ebvbKF1dXdW4cWPbF38zb1197rnnlD9/flksFnl5eSk8PFzBwcFKTExUr169bF+QbrVjxw4tWLBAZcuWlcViUaFChRQeHq5Dhw4pKipK7u7uGjlyZJqvI6M12rFjRy1dulQffvihtm3bpmLFisnFxUU9evSQv7+/Pv30U33wwQeqUqWKqlatqrx58yo8PFxBQUFKSkrS448/rlq1amX22wcTde/eXQsWLFBoaKjatWunhg0bys3NTYGBgYqPj9czzzyjBQsWZNr+atasqddff10ffvihhg4dqrp166pChQo6deqU9u/frwEDBjgNOzw9PfXZZ59p8ODB+vLLL7VkyRL5+PiodOnSio+P18mTJ3X8+HEVL15cTz75pCSOG7lNvnz5NH36dL3wwgv6+uuvtXbtWvn5+SkyMlI7d+5UmzZtFBISkuYQGbcqVqyY+vTpowULFmjAgAFq2LChSpUqpb///lshISF68cUX7xj0ZfQz+W6+g3h4eKht27YKCAjQ448/rgYNGth+gJo8ebIkqW3btnrrrbf0/vvv68UXX1SlSpX00EMPqWDBgoqIiNChQ4d0+fJlDRo0SC1atJB0Y9zrUaNGqVSpUqpZs6YKFy6s6OhoBQUFKSYmRhaLJdd/fyCUQ5rc3d21cOFCff755woICNC2bdtUsGBB1a9fXzNnztS1a9fu21DO1dVVs2fP1sKFC/XTTz9p586d8vT0VIMGDTRnzhyFhoY6DeWkG4/I9vPz08KFC20DFicnJ6tEiRKqWLGi+vTpk+FxKvDgKViwoL7++mutXLlSv/zyi0JCQnTo0CEVKFBApUqVUteuXdWuXTunY+h4eHjos88+0+zZsxUQEKALFy6oSJEi6t69u4YPH+7wlLIKFSpo2bJl+vjjj7V7925t2LBBJUuW1H/+8x8NGzYs1b6cHtbL7iVpw4YNaS6b2w+qSJ+xY8fqoYce0g8//KD9+/fL3d1dLVu21EsvvSR/f/90b6dAgQK2Y9Tvv/+uzZs3y8vLS82aNdOrr76q2bNnS5Ld2Id169bVG2+8ocDAQB0/flwHDx6Um5ubypQpoz59+qhv3762K0+RezRs2FB58uRRQkKC07CqadOm2RLKPfPMM/rrr78UEhKiq1evysvLS//+97/Vu3dv2xee27Vv316xsbEKDQ3VgQMHFBUVJQ8PD3l7e+uRRx7R008/fccB6TNao//+97/13nvvafHixdq+fbvi4uIk3fixxjqu0bZt23TgwAHt2rVLsbGxKlmypJo1a6ZevXqpXbt29/5mIVtZryadNWuWtmzZok2bNsnLy0vNmzfX0KFDtXv37kzf58CBA/XQQw/pq6++0sGDB3X06FH5+Pho5syZqlWrVqpXIFWvXl2//PKLvv/+e61du1aHDx/Wnj175OXlpTJlyui5555Thw4dbMtz3Mh9GjVqpCVLlmjWrFnauXOn1qxZowoVKmj48OF67rnnMnzhxdixY+Xj46PvvvtOBw4ckJubmywWiz766CN17tz5jqFcRj+T7/Y7yLvvvisvLy9t3rxZAQEBttt3raGcdOO41KRJEy1atEg7duzQtm3b5OrqqhIlSqhmzZr697//bff+PPfccypfvryCg4NtTzD38vJStWrV1KVLF3Xv3l2enp4Zej8fNC5GRh+fBADIMjt27NAzzzyjRo0aaeHChdndHCBTWYMAM8aOSkxMVJcuXXTy5EktXbqUq3CAdDCzRgEAAGPKAQCA+9iBAwccxva8du2aJk2apJMnT8rHx4dADgAAADkSt68CAID71vDhwxUXFyeLxaLixYvr8uXLOnTokO32iGnTpmV3EwEAAACnCOUAAMB9a8CAAVqzZo2OHz+uoKAgubq6qmzZsuratauef/55h/EXAQAAgJyCMeUAAAAAAAAAkzGmHAAAAAAAAGAyQjkAAAAAAADAZIwplwtduHBBnTp1UuPGjTV37ly7eW3bttXZs2dTXbdu3bpasmRJqvMTEhL0/fffa9WqVTp+/Lji4uJUtGhRWSwWde/eXZ07d86012E263uzbt06lS9f3tR9v/XWW1q2bJmWLVsmHx8fU/f9oEqtDpYuXaoxY8bccX0XFxcdOnTIYXpKSoqWLFmin3/+WceOHZMkVatWTT179tSTTz4pFxeXzHsR2YA6uP+k9Zl/u+TkZD399NPas2ePJOnbb79Vw4YN07WfV155RatXr5YkffDBB3rsscccljlx4oT++usvhYSEKCQkRMePH1dycrJeeeUVvfTSSxl7YTkUNXJ/yapjgSRdvXpVCxcu1Jo1a3Tq1CklJiaqePHi8vX1VZ8+fdSsWbNMex1ms/avw4cPm77vAQMGaN++fQoICFDJkiVN3//9KrW+HhMToy1btmjz5s3as2ePzp49q5SUFJUqVUqNGjXSgAEDnH6exMXFafv27dq8ebMCAwN1+vRpWx/39/dX37591aBBA6dt+eWXX7RlyxYdOnRIFy9eVHR0tPLly6eHHnpIHTp0UN++fVWgQIEsey/MwLEgZ8qKc6KQkBBt377ddm5z6tQpGYaR6rmQFXWQte6nOiCUy4U++OADxcfH67XXXkt1mY4dO8rT09NheoUKFVJd5/z583r++ed17NgxFS1aVP7+/sqfP7/CwsIUGBgoT0/P+zqUy07Dhg3Tr7/+qvfee08LFy7M7uY8EFKrg4oVK6pbt26prrd9+3aFhYWpcePGDvOSk5P16quv6o8//lD+/PnVpEkTSdK2bds0fvx4bd26VR999JFcXblI+W5QB3cnPZ/5Vl999ZX27NkjFxcXZWTI2ZUrV2r16tV3XG/x4sVasGBBureLjKFGMi4rjgWSdOTIEQ0cOFAXLlxQmTJl1KRJE7m5uSksLEwbN25UxYoV7+tQLju98cYb6tmzp2bMmKGpU6dmd3PuG6n19S+//NIWTlSuXFmtWrVScnKyQkJCtHTpUv3666+aNGmSQz389ttvGjdunCSpXLlyatq0qdzd3XXo0CGtXLlSq1at0iuvvKIXX3zRoS2LFy9WcHCwqlatKl9fX3l5eenSpUvas2eP9u/fr59//lkLFy5U6dKls+jdeLBxLEhdVpwTzZkzR+vWrctwW6iDrHU/1QGhXC6zb98+/fbbb+rUqVOaifHIkSMzlGbHx8fr2Wef1YkTJzRs2DANHjxYHh4etvlxcXE6efLkvTQ9VytTpoyeeOIJLVq0SOvWrVO7du2yu0n3tbTqoGHDhqleGXT9+nW1bNlSktSzZ0+H+QsXLtQff/yh0qVL69tvv7WF2KdPn9bTTz+t1atX61//+pf69u2bya8od6AOMi69n/mSdPToUc2aNUtt2rTRkSNH0rxq+laXLl3SxIkT5evrq3z58ikoKCjVZS0Wi5577jn5+vrK19dXn332mVasWJGh14TUUSMZk1XHgkuXLmnAgAGKiorSO++8o969e9tdJR0dHa3w8PBMfCW5i5+fn9q0aaNly5apf//+qlGjRnY3KcdLq697enrq2WefVe/evVW5cmXb9MTERE2fPl3z58/X22+/LX9/f1WqVMk2393dXT169FDfvn3l6+trm24YhubPn69p06bp448/VoMGDdSoUSO7fY4ePVqVKlWSl5eX3fSIiAi9/PLL2r17t95//33NmDEj896EXIRjgXNZdU5Ur149Va9e3XZuM3bsWO3cufOO7aEOstb9VAdcrpHLfPPNN5Kcn0Tei88++0wnTpxQr169NHToULtATpLy58+vmjVrZuo+cxvr38z6N8Tdu9s6WLNmjaKiolS4cGE9/PDDdvNSUlL05ZdfSpLefPNNu6tKK1SooDfffFPSjVpJSUm5l+bnatRBxqS3ryclJWnUqFHKly+fJk6cmKF9vP3227p27ZqmTp0qd/e0f+t74oknNGrUKHXt2lVVq1blqtEsQI2kX1YcC6QbV2JcvnxZw4cP11NPPeUwbEHhwoVVrVq1u2841LNnTxmGQT9Pp7T6+uDBgzV69Gi7QE6SPDw8NGrUKFWuXFmJiYn6/fff7eZ369ZNU6ZMsQvkpBu3dD/77LNq2rSpJDn94aVu3boOQYQkFS1aVK+//rok6a+//kr364MjjgWOsuqc6IUXXtBrr72mjh07pnlX2e2og6x3v9QBV8rlIpcuXVJAQIBKlSql5s2bZ9p2ExMTtXjxYknS888/n2nb7devn3bu3KkFCxaocOHCmjNnjnbt2qVr166pYsWK6tmzp5599lmnY3QlJSXpxx9/1IoVK3T06FElJCTI29tbrVq10qBBg1K9DPjYsWOaOXOmduzYobi4ONvtKwMGDEizrUlJSVq2bJl++eUXHT58WLGxsSpVqpRatmypIUOGyNvb22GdrVu3asGCBdq3b5+ioqLk6empokWLqk6dOurVq5f+9a9/2S1fs2ZN1ahRQzt27NDx48dVtWrV9L+ZsLmXOvj5558lSV27dlXevHnt5gUHB+vixYvKkyePOnbs6LBux44d9dZbbyk8PFx79+5V/fr107VP6oA6uFsZ6etz585VSEiIpkyZkqHbJJYvX67169fr5ZdfzrarVagRauRuZNWx4PLly1q5cqXy5cunPn36ZFp7bx2X58yZM/r888+1f/9+Xb9+XVWrVlX//v31+OOPO103Li5OCxcu1KpVq3Ty5EmlpKSofPnyat++vZ577jkVKVLE6XrBwcGaM2eO9uzZo+TkZD300EN6+umn7/iFNj4+Xt99951Wr16tEydO6Pr16ypbtqzatWunQYMGqWjRog7rrFq1Sj/88IMOHjyoq1evqmDBgnZjk93++dK6dWsVLVpUv//+u0aNGuX0iy1uuJe+7urqKh8fH508eVLnz5/P0Lo1a9bUtm3bMryem5ubJDn8wH8nHAs4FqTFjHOizEQd5K46IJTLRTZu3KjExEQ1adLkjlcnLF26VFFRUUpKSrIN9Hp7J7cKDQ1VRESESpUqpUqVKunw4cNas2aNwsPDVbhwYTVs2FCtWrW66ysitmzZonnz5qlixYpq3ry5Ll68aLucNywsTG+99Zbd8gkJCRo8eLC2bt2qvHnzqnHjxipYsKCCg4O1cOFC/fbbb/rqq69Uq1Ytu/UCAwM1aNAgxcbGqkKFCmrevLkiIiL00Ucfae/evam27+rVq3rxxRe1c+dOeXp6qnbt2ipatKiOHDmi77//XqtXr9a8efPsfklctmyZbQDpOnXqqHHjxoqPj9eFCxe0cuVKFS1a1On73axZMx06dEhr167NsR8qOV1G6uBW586d0/bt2yU5/4Xt4MGDkqTq1as7fEmTpHz58ql69eoKDQ1VaGhoukM5K+rgJuogfdLb1w8ePKi5c+eqRYsW6tGjR7q3f+HCBU2ePFkWi0VDhgzJjCbfE2rkJmrkzrLqWLBjxw4lJiaqdu3aKliwoIKCgrRx40ZFRESoWLFiatasmcOtfBnx888/69NPP5Wvr69atmyps2fPas+ePRo1apQiIyMdvgRZpx08eFAFCxZUkyZN5OHhoZ07d2ru3Ln67bff9M033zgMWbJq1Sq98cYbSk5OlsVikcViUVhYmMaNG2d7iJEzFy5c0MCBA3XkyBF5eXnJz89PBQoUUGhoqL766iutXr1aCxcuVLly5WzrzJ49W7NmzZK7u7vq16+v0qVLKyYmRmFhYfrpp59UrVo1h1DOw8NDjRo1UkBAgLZs2aIuXbrc9Xv6oLvbvm516tQpScrwQzXuZr2rV69q9uzZkm4E0XeDY8FNHAtuyupzosxEHeTCOjCQa7z55puGxWIxFi1alOoybdq0MSwWi9P/evToYZw8edJhnR9++MGwWCxGz549jQ8//NDw8fFxWPfxxx83zp49m6H29u3b17b+4sWL7eZt3brV8PHxMWrWrGmEhYXZzfvwww8Ni8VitG/f3jh9+rRtekJCgjF27FjDYrEYbdu2Na5fv26bFx8fb7Ru3dqwWCzG5MmTjaSkJNu8gwcPGo0bN7a15dZtGoZhvP7664bFYjEGDx5sXLp0yW7evHnzDIvFYjz88MN222zbtq1hsViMXbt2ObzuS5cuGSEhIU7fkz/++MOwWCxG//79U3nXcCfpqQNnZs2aZevLzkydOtWwWCzGSy+9lOo2hgwZYlgsFmPatGnp3i914Ig6SJ/09PXr168bXbt2NerXr2/3GW09Fjj721g9//zzRs2aNY19+/bZpln76/Lly9PVxlGjRhkWi8WYM2dOupZ3hhpxRI3cWVYdC2bMmGFYLBZj6NChtn3c/t+AAQOMyMjIDO3XWpO1atUy1q9fbzfv559/NiwWi9GgQQMjLi7Obt6rr75qWCwW44knnjCuXLlim3716lVj4MCBhsViMXr16mW3Tnh4uFG/fn3DYrEY8+bNs5u3detWw8/Pz/ZabpWSkmL07t3bsFgsxtixY42YmBjbvMTERGPatGmGxWIx+vXrZ5t+/fp1o06dOka9evWM48ePO7zuM2fOGMeOHXP6nljrZuzYsU7n44a77euGYRgbN240LBaL4ePjYxw8eDDd6x06dMjw9fU1LBaLsW7dulSX27x5szFq1ChjxIgRxnPPPWfrd88//7wRHR2dobZyLHDEseCmrD4nulVGz4WoA+qAwVxyEeuVPGklxK1bt9Z///tfrVmzRvv27dO6dev0/vvvq2zZstq/f7/69euny5cv260TGRlp2/4XX3xhG9B+9+7dmjdvnipXrqzQ0FANHjxYiYmJGW73ww8/rN69e9tNa9q0qVq0aKHk5GTbL9bSjcGXv/32W0nSmDFj7H759fDw0Lhx41SiRAmdOXNGAQEBtnkBAQEKCwuTt7e3RowYYbtkWJJq1KiR6lUgx48f1++//65SpUpp+vTpKl68uN38AQMGqHXr1jp58qQ2bdpkm3758mUVKlTI6SDSxYsXdxifw8o6Bk1oaKjT+biz9NTB7QzD0NKlSyWlPg7FtWvXJN0YPzE11icaW5fNCOrgJuogfdLT1+fMmaPDhw9r5MiRKlu2bLq3vWTJEm3evFnPP/+8/Pz87rmtmYEauYkaubOsOhZERERIkjZs2KDff/9dw4YN07p167Rz507Nnj1bJUuW1NatW23jBWVU37591aZNG7tp3bt3V5UqVRQTE6MDBw7Ypp87d872VOR3333X7rbRAgUK6L333lPevHkVHBxs94CWn376SdeuXVO9evUcrrxr2rSpevXq5bRtmzdvVlBQkGrWrKmJEyeqYMGCtnnu7u4aMWKELBaLduzYoSNHjki6cbVEfHy8KlSooCpVqjhss1y5cqn+jejn6XM3fV26cdWj9WqaJ598Mt1DFFy7dk1vvvmmkpKS1KJFizSv9Dl27JiWLVumFStWaMuWLbp27Zq6dOmiadOmqVChQhlqrxXHgpuokZuy8pzoXlEH1AGhXC5y6dIlSUpz3I0JEyaoS5cuqlixovLmzavy5cvr8ccf17Jly1SuXDlduHDB9th0K+P/HxGdmJioLl26aPz48XrooYdUsGBBNWvWTPPmzVPevHl15MgRh0Fi0+P2k08r64fqrU8w279/v2JjY+Xl5eX0JCB//vzq3LmzpBu3mFhZn5DzyCOPOL13//bHwFtt3LhRhmGoVatWdieft7LephIcHGyb5ufnp5iYGI0cOVIHDhxI98D/1r9dVFSUEhIS0rUO7KWnDm63bds2nT17Vnnz5s22W2Sog5uog/S5U1/ft2+fvvjiCzVp0iTVL9nOnD17VtOmTVPVqlU1bNiwzGhqpqBGbqJG7iyrjwWJiYkaOHCghg4dqvLly6tIkSLq0KGD5syZIxcXF23ZskWBgYEZbved+vmFCxds03bt2qWUlBT5+vo6DVRKly6tFi1aSHLez7t27ep0X2n1c+nGl0FnD31xdXW1fZmy9vNixYqpXLlyOnz4sKZNm5bmrbG3s/7trH9LOHc3ff3q1asaMmSIwsPDVadOHYdb3VKTmJioV155RUeOHFGFChX04Ycfprn8gAEDdPjwYR04cEBr1qzR6NGjtXnzZv3nP//Rrl270t3eW3EsuIljwU1ZdU6UGagDR7mtDhhTLhe5evWqJKXa+dPi5eWl/v37a8qUKdqwYYPdwblAgQK2fzv7ECtbtqz+/e9/KyAgQNu2bUt1IOLUOBvkUbr5Oq5fv26bZv2AuXWskttVrFhRkv2Jq3UQ2tvHVLEqUqSIChUqpJiYGLvpp0+flnTjV+Wffvopzddx5coV27/feecdDR48WCtWrNCKFStUoEAB+fn5qUmTJnrsscdS/XXm1r9dTEyMwy8LuLO7qQProN4dOnRIdUBsax3ExcWlup3Y2Fi7ZTOCOriJOkiftPr69evXNXr0aOXNm1fvvfee0wF/nTEMQ2PHjlVcXJymTJmiPHnyZGqb7wU1chM1cmdZfSyQnJ8T1a1bV76+vgoJCdHWrVud/uKfljv9zW/t59a+m1p/le6un6c23drPP/nkE33yySep7lOy7+cffPCBhg8frnnz5mnevHny8vJSnTp11Lx5cz366KMqVqyY021YX3N0dHSa+8rtMtrXr127poEDByo0NFS+vr768ssvnY6Ve7ukpCS9/vrr2rx5s8qVK6dvvvkm1b/d7Tw8PFSxYkU9++yz8vf3V69evTRixAitXr1a+fLlS9c2rDgW3MSx4KasOCfKbNSBo9xSB4RyuUihQoV05coV24dSRlmT9dufonTro59Tewy0tVgvXryY4f3e7QMizGBN6K1PdklL3bp1bf+uWrWqVq9erb/++kvbt29XcHCwdu/ere3bt2vOnDmaPHmyHnvsMYdt3PqhVrhw4Ux6FblLRusgOjpaa9askZT2I9StB7KwsLBUl7nTwSst1MFN1EH6pNXXT5w4oePHj6to0aIaO3asw3zrZ/V7772nQoUKqWXLlnrhhRcUExOj7du3y9PTU//9738d1rPeHjJ37lz99NNPqlGjRrqvsLhX1MhN1MidZdWxwPr57u7unuqXogoVKigkJOSuzomy68tielj7eYMGDWxf4FJTvXp1278bNmyo9evX688//9SuXbsUHBysLVu2aNOmTZo5c6bmzJmjpk2bOmzD2s/p42nLSF+PjY3V4MGDFRwcLB8fH3399depBtC3Sk5O1ptvvqk//vhD3t7e+uabb9L8gp+WunXrqlq1ajp69KgOHDiQ4eCaY8FNHAtuyopzoqxEHdyQW+qAUC4XKV68uK5cuWIbAy6jrOvdfpWPr6+vXFxcZBiGIiIinJ6EWsdYsY6plVVKlSol6cbtVamxpvO3PtbZ+u8zZ844XSc6Otoh5Zdu/grh7++v8ePHZ6it7u7uat26tVq3bi3pxi848+bN0+zZszVhwgR16NDB4f2y/g2KFCmS4Udk44aM1sGvv/6q69evq3z58mrSpEmqy1nHMTh69KiuX7/u8KtyfHy8jh49ardsVqEOIKWvr0dERNhuP3DGGrLd/uUqNjY2zfVOnDihEydOZKzBJqJGkFXHgtq1a0u6cdXQ1atXnX4BMOucyNpfrX3ZmdT6+YkTJ1Ktj9SmW/t5u3bt9Pzzz2eorfny5VOnTp3UqVMnSTeujvj444/1ww8/aOzYsdqwYYPDOta/XYkSJTK0r9wmvX09Li5OgwcP1q5du+Tj46P58+fbjUOYmuTkZI0YMUKrVq2St7e3FixYkOqP9OllHZ/39nGsMxvHgtwjK8+Jsgp14OhBrYOcG6Ei01mDgOPHj9/V+tbx4OrUqWM3vWTJkmrQoIEkaevWrQ7rJSYm2u6Hv33dzObn5ydPT09FRkZq3bp1DvPj4+O1cuVKSVLjxo1t062PT169erXTh1EsX77c6f5atWolSVq/fr3dZcB3o2DBgho2bJgKFy6suLg4nTx50mEZa6hz++OokX4ZrQPr7Urdu3dP8wqF+vXrq2TJkkpISLAbBNUqICBAiYmJKlWqlN2vPlmBOoCUdl+vWbOmDh8+nOp/1hPOb7/91jbWk3TjF8a01rOOAfLBBx/o8OHDWrhwoUmvNmOoEWTVsaBOnTq2W2j++usvh/mRkZEKCQmxLZuV/vWvf8nV1VUHDx7UoUOHHOaHh4dr8+bNkpz3819//dXpdu/Uz1evXm0bb/huFStWTCNGjJB044EVUVFRDsvQz9MnPX09Pj5egwcP1s6dO22BXHpuPU1JSdHIkSP1+++/2wK5O10leSdXrlyx9dfKlSvf07buhGNB7pEV50RZiTpInwelDgjlchFrEd06YOKt1q5da/fULqurV69q8uTJWr9+vSTp2WefdVhm6NChkqTPP/9ce/bssU1PSkrS+++/r9OnT6tAgQLq3r37vb6MNOXNm1d9+vSRJL3//vt2iX9iYqImT56sixcvqnz58urYsaNtXqdOnVS6dGmdO3dOM2bMsBs48siRI/r000+d7s/X11cdO3ZUWFiYhg4d6vSXgtjYWP3yyy+2AUbj4uI0b948u3vkrQIDAxUdHS03NzeVKVPGYb71b5fWr/RI253q4FaHDh1SSEiIXF1d79h3XV1dNXDgQEnS9OnT7a5MOH36tO1Wv8GDB2f5JeXUAaSM9fXchhpBVh0LXFxc9PLLL0uSPvzwQ7srRuPi4jR+/HhdvXpVZcuWVfv27e/hFdxZ2bJl1alTJxmGofHjx9uu0JNu9Lfx48fr+vXrql+/vvz9/W3zevbsKU9PTwUHB2vBggV229yxY4e+//57p/tr166d/Pz8tG/fPo0ZM8Zp342KitLixYuVlJQk6caVGT/++KPTW8qs551FihRxOg4U/Tx97tTXr1+/rhdffFE7duzIcCA3ZswY/fbbbxkK5I4dO6ZffvnF6Zfxv//+W6+88ooSEhJUr149+fj43HF794JjQe6R086JqAPq4FbcvpqLtG7dWh4eHtq+fbuSk5PtHlss3TjRWrBggcqWLSuLxaJChQopPDxchw4dUlRUlNzd3TVy5Eg1a9bMYdtNmzbVK6+8ok8++UR9+vSRn5+fSpYsqZCQEJ09e1b58uXTjBkzTLnFYPjw4Tpw4IC2bdumzp07q3HjxipQoID27Nmjc+fOycvLS5988ondAOX58uXT9OnT9cILL+jrr7/W2rVr5efnp8jISO3cuVNt2rSxvZbbTZkyRdHR0dq0aZM6deqkGjVqqHz58jIMQ2fPntWhQ4eUmJiolStXqkSJEkpMTNS0adP0wQcfyGKxqFKlSvLw8NDZs2dtgeaQIUOcnhBZr0Rs165d1rx5ucCd6uBW1sFHmzdvnurYQLfq16+fAgMDtWbNGnXt2tU2Bs62bdsUFxenjh076umnn86cF3IH1AEy0tfNEhISookTJ9r+/59//pEk/fDDD/rzzz9t02fPnm27nSKrUCO5W1YeC3r27Kk9e/boxx9/1OOPP666deuqUKFC2rdvny5evGjrW+kZPP9ejR8/XidOnNDevXvVoUMHNW7cWG5ubtq1a5euXLmi8uXLa/r06XbrlC5dWu+9955GjBihyZMn68cff5TFYtGFCxcUGBio/v37a/78+Q77cnV11Zw5czR48GAtW7ZMAQEB8vHxUdmyZZWYmKjTp0/ryJEjSk5OVvfu3eXu7q7o6GiNGzdOEydOtNWEJJ06dUqhoaFycXHRiBEjHP4+1rsw8ubNa3uCLJy7U1+fMWOG7TOjbNmy+uCDD5xup0GDBnriiSds/79o0SLblTAVKlTQ//73P6frValSxW78rcuXL2vEiBGaMGGCatasqTJlyigxMVHnzp1TaGioUlJSVLVqVX300Uf38rLTjWNB7pCV50R//vmnXf+3PkV69uzZ+vbbb23TlyxZYvs3dUAd3IpQLhcpUaKEOnbsqN9++01btmyx3Ytt1b59e8XGxio0NFQHDhxQVFSUPDw85O3trUceeURPP/10mkn9Sy+9pDp16uibb77Rvn37dODAAZUoUULdu3fXwIEDbQ+KyGp58uTRl19+qSVLlmjFihUKDAxUQkKCvL291a9fPw0aNMjufnirRo0aacmSJZo1a5Z27typNWvWqEKFCho+fLiee+45Pfzww073V7BgQX399ddauXKlfvnlF4WEhOjQoUMqUKCASpUqpa5du6pdu3a2Xw89PT01ceJE7dq1S6Ghodq6davttsaHH35YTz31lNMBjUNDQ3X48GE1btxY1apVy9w3LRe5Ux1YJSQk2G7d6dGjR7q27ebmppkzZ2rJkiX68ccftX37dklStWrV1LNnT/Xq1cu0QbqpA6S3r5vp6tWr2rt3r8P08+fP2z1EyIxH1lMjuVtWHgukGwOCN23aVN9//70OHjyo+Ph4eXt7q2/fvho0aJDTX/OzQtGiRfX9999r4cKFWrlypf766y+lpKSofPnyevLJJ/Xcc885Hcj/P//5j0qXLq1PP/1Ue/bs0enTp/XQQw9p4sSJ6tWrl9NQTroR6C1ZskRLly7VypUrdfjwYe3fv19FihRRqVKl1Lt3b7Vt29YWSFaoUEFjx47Vrl27dPToUW3cuFHSjTGOHn/8cfXr1882Tt+t/vzzT0VERKh79+7y8vLKtPfrQXSnvn7rrcHOxu671a2h3K3rpTUOV6NGjexCuerVq+u1115TYGCgTpw4oYMHDyoxMVFeXl5q2rSpOnTooB49epj2dG+OBblDVp4TXblyxem5zT///GP78fF21AF1cCsX414HfcB9Zd++fXriiSf08MMPa9asWdndHGTApEmTtGjRIv3vf//L0Un//YA6uH9RBxlDX899qJH0oz7uX0OGDNGff/6pZcuWqWbNmtndnByPvp77cCxwRB3kPvdLHTCmXC5Tp04ddenSRWvWrHE66C9yprCwMP34449q1KhRjv5AuV9QB/cn6iDj6Ou5CzWSMdTH/Wnfvn3asGGDunXrRiCXTvT13IVjgXPUQe5yP9UBoVwuNHLkSOXPn9+0e9Rx72bPnq2kpCS99dZb2d2UBwZ1cP+hDu4OfT33oEYyjvq4/8yYMUMFChTQ66+/nt1Nua/Q13MPjgWpow5yj/upDrh9FQAAAAAAADAZV8oBAAAAAAAAJiOUAwAAAAAAAExGKAcAAAAAAACYjFAOAAAAAAAAMBmhHAAAAAAAAGAyQjkAAAAAAADAZO7Z3QAAAADcWdu2bXX27Fnb/7u4uCh//vwqVKiQKlWqpNq1a+uRRx5RnTp1srGVAAAASC+ulAMAALiP+Pv7q1u3bnr88cfVunVrPfTQQzp8+LC+/vprPfHEE+rXr59Onz6dKfs6c+aMfHx81LZt20zZntlmzZolHx8fzZo1K7ubAgAA4IAr5QAAAO4jTzzxhLp37243zTAMbdq0SVOmTNHOnTvVu3dvff/996pQoUI2tRIAAAB3wpVyAAAA9zkXFxe1bt1aP/74oypXrqxLly5p3Lhx2d0sAAAApMHFMAwjuxsBAACAtFnHlJs6darDlXK32rhxo1544QVJ0s8//6zatWtLko4dO6aVK1dq69atOnv2rCIiIlSgQAHVrFlTTz75pDp37my3ndGjR2vZsmWp7ufw4cOSpKtXr2rlypXatGmTjhw5ovDwcElShQoV1LZtWz3//PMqXLiww/rh4eH6/PPPtXnzZp07d06urq7y8vJS5cqV1apVKz3//PMO61y4cEFff/21Nm3aZFunSpUq6tatm3r37i1395s3gfj4+KTa9m7dumnatGmpzgcAADADt68CAAA8QFq1aiUvLy9FRkZq69attlBu3rx5+umnn1SlShVZLBYVLlxYYWFh2rFjh7Zt26a9e/dqzJgxtu00aNBAsbGxCggIkKenpzp27Oh0f4cOHdLbb7+tYsWK6aGHHlKtWrUUHR2tAwcOaO7cuVq1apV++OEHFS1a1LbOxYsX1aNHD4WHh6ts2bJq2bKl8ubNq/DwcB06dEghISEOodyuXbv08ssvKyoqSuXKlVOzZs2UkJCg/fv3a9KkSdqwYYPmzp0rDw8PSTeCt4MHD+rQoUOqUaOGatasaffaAAAAshuhHAAAwAPExcVFvr6+2rp1q44ePWqb/thjj2nIkCEO48ydOHFCzz77rObPn6///Oc/tqe3PvHEE2ratKkCAgJUtGjRVK8sK1++vObPn6/GjRvL1fXmyChxcXF65513tHz5cs2cOVMTJkywzfvhhx8UHh6uXr16aeLEiXJxcbHNS0xMVGBgoN0+Ll68qKFDhyo6OloTJkxQ7969bfuKiIjQq6++qi1btuizzz7T0KFDJUnTpk3TrFmzdOjQIbVv317Dhg27m7cTAAAgyzCmHAAAwAPGelVaZGSkbVqjRo2cPvihSpUqeumllyRJq1evzvC+ypQpo6ZNm9oFcpKUP39+vfPOO3J3d3fY7uXLlyVJLVu2tAvkJMnDw0NNmza1m/bNN98oMjJSffr00dNPP223r6JFi+qDDz6Qh4eHvv32WzEyCwAAuF9wpRwAAMADJiUlRZIcAq9r165p06ZNOnjwoCIiIpSYmCjpxpVokvT333/f9T6DgoIUGBiosLAwxcfH28IxDw8PXblyRVFRUSpSpIgkqU6dOvruu+80ffp0GYah5s2bq0CBAqlue+PGjZKkRx55xOn80qVLq1KlSjp27JhOnjyphx566K5fBwAAgFkI5QAAAB4wERERkmQLwSRp/fr1GjNmjN3Vc7e7evVqhvd1+fJlDRs2TLt3705zuatXr9ra89hjj+mvv/7Sr7/+qmHDhsnNzU1Vq1ZVgwYN1LFjR4cr5U6fPi1J6tOnzx3bc+XKFUI5AABwXyCUAwAAeIAYhqGDBw9KkiwWi6QbTy197bXXFB8fr4EDB6pr164qX768PD095erqqi1btjh92ml6vPXWW9q9e7fq16+vYcOGqUaNGipcuLDtgQstWrTQxYsX7W4rdXV11fTp0zVkyBD9+eefCgoKUlBQkBYvXqzFixerTZs2mjNnjtzc3CTdvPKvY8eO8vT0TLM9Xl5ed/U6AAAAzEYoBwAA8ADZuHGjoqKiJN0IxKQbV8nFx8erQ4cOGjFihMM6p06duqt9xcbGatOmTXJ1ddXnn3+uwoULO8y/dOlSqutXq1ZN1apVk3QjTNy+fbveeOMNbdiwQcuXL1ePHj0kSd7e3jp58qQGDRokPz+/u2orAABATsODHgAAAB4QMTExmjp1qiSpefPmqlmzpiTZQrqyZcs6rGMYhn799Ven27Ne7ZaUlJTq/pKTk1WwYEGHQE6Sfvnll3Q/eMHFxUVNmzZVly5dJMl2tZ9044EQkrRq1ap0bSu97QcAAMhOhHIAAAD3OcMwtHHjRvXs2VMnT55UyZIlNWnSJNv8qlWrSpICAgIUHh5um56cnKxPPvlEwcHBTrdbrFgxeXh46NKlS07HoitRooSKFCmi6OhoLV++3G7enj17NGPGDKfbXb58uQ4cOOAw/erVq9q5c6ckqVy5crbpAwcOVOHChTV//nx9/fXXSkhIcFj39OnTWrFihd20MmXKSJKOHTvmtB0AAADZycXgufEAAAA5Xtu2bXX27Fn5+/urUqVKkqSEhARFREQoNDTUFpo1atRIU6ZMUYUKFWzrJiUl6cknn1RISIg8PT3VqFEj5c+fX/v27VN4eLgGDBigL774Qo0aNdLChQvt9jt8+HAFBATI29tbDRo0UL58+SRJkydPliTNnz/fdnVe3bp1VaFCBZ07d07BwcF69NFHFRgYqLNnz2rdunUqX768JOmll17SunXrVKpUKdWsWVOFCxdWdHS0goKCFBMTI4vFosWLF6tgwYK2duzatUvDhg1TRESEihcvrurVq6tkyZK6evWqjh8/rn/++Ud169bVkiVLbOtcunRJHTp0UGxsrPz9/VW5cmW5urrK39/fdmssAABAdiGUAwAAuA9YQ7lbeXp6qmDBgqpcubJq166tRx55RHXq1HG6/rVr1/T5558rICBA586dU8GCBVW/fn29+OKLunbtmp555hmnoVxkZKRmzJihzZs36+LFi0pMTJQkHT582LbM2rVr9eWXX+r48eNKSkpSlSpV1KNHDz311FNq166dQygXGBioP/74Q8HBwQoLC1NkZKS8vLxUvnx5denSRd27d3f6QIfLly9r0aJF2rhxo06ePKmEhAQVL15c3t7eat68uR5++GH5+PjYrRMYGKg5c+YoJCREMTExSklJUbdu3TRt2rSM/xEAAAAyEaEcAAAAAAAAYDLGlAMAAAAAAABMRigHAAAAAAAAmIxQDgAAAAAAADAZoRwAAAAAAABgMkI5AAAAAAAAwGSEcgAAAAAAAIDJCOUAAAAAAAAAkxHKAQAAAAAAACYjlAMAAAAAAABMRigHAAAAAAAAmIxQDgAAAAAAADAZoRwAAAAAAABgsv8DoCL0r0RBEp0AAAAASUVORK5CYII=" + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "
    " + ], + "image/png": "iVBORw0KGgoAAAANSUhEUgAABOUAAAMzCAYAAADpn6Y7AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOzdd1hT1/8H8HcS9hAQFMQBaoUqKorWVWfrxFkVa111a9U6qrbWatX2K1rrV+to3eLee2utKCKOioAIIjhA2XsjI8nvD37J15iEJSSI79fz+Dx6zzn3fO69STAfzhBIpVIpiIiIiIiIiIiISGOE2g6AiIiIiIiIiIjoQ8OkHBERERERERERkYYxKUdERERERERERKRhTMoRERERERERERFpGJNyREREREREREREGsakHBERERERERERkYYxKUdERERERERERKRhTMoRERERERERERFpGJNyREREREREREREGqaj7QCIiKhsTpw4gR9//LFEddu0aYO9e/cqHIuLi0NAQAAePnyIhw8fIigoCJmZmQCAFStWYPDgwe8co6OjIwDgiy++wMqVK4utP3r0aNy7dw+1a9fGtWvX3rn/svrss88QFRWFGTNm4Ntvv1UoW7BgAU6ePKnynlLZvPlafvLkSZnPExISgiNHjuD+/fuIiopCbm4uzMzMUL16ddSuXRvNmzdHx44d0axZMwgEgvIKv0pLTEzE6dOn4ePjg+fPnyM5ORkAYG5uDgcHB7Rt2xb9+/eHtbW1liP9cJTX++V99D58/t69exdjxoxROq6rqwsTExNUq1YNDRo0QNOmTfHZZ5+hSZMmWoiSiIgqCybliIg+UF999RWioqK0HQZpQVFJx/fV6tWrsWPHDkgkEoXjiYmJSExMRGhoKDw9PbFu3Trcvn0b1atX11Kk7weJRII///wTO3bsQE5OjlJ5bGwsYmNj4eXlhbVr12Lo0KH48ccfYWBgoIVoiSq//Px8pKSkICUlBREREfD09MSGDRvg7OyMRYsWoXnz5hXW9/uQzHxXH8I1ElHVxKQcEVEVsHXrVrRu3VptuUgkUltWrVo1NG3aFGZmZrh48WJFhEdUobZs2YJt27YBABo0aICxY8fC2dkZNWvWRH5+PiIiInD//n1cuXIFjx8/1nK0lV9eXh5mzpwJT09PAICNjQ1GjhyJdu3awcbGBjo6OoiPj8e9e/dw9uxZPHz4EIcOHcKkSZNQp04dLUdPVHksW7YM/fv3BwBIpVJkZmYiISEBDx8+xKVLl3Dv3j0EBATgq6++wuLFizF8+HAtR0xERJrGpBwRURVgYGAAY2PjUrVZtGgR7O3tUb9+fQgEAty9e5dJuRJYuXJliabikmZkZ2dj06ZNAABnZ2fs3bsX+vr6CnWsra3Rpk0bTJs2Df7+/jAyMtJGqO8Nd3d3eUKuX79+WL58udIIuOrVq+Pjjz/GmDFjcPXqVfz888/aCPWDNHjw4HJZXoAqnp6ensLPZhMTE9jY2KBZs2YYOXIkvL29MW/ePKSkpOCXX35B7dq10alTJy1GTEREmsaNHoiIPlCfffYZGjRowLW16L3m5+cnn145evRopYTc21q0aMEplkW4f/8+Dh48CABo3749Vq9eXez96t69O06cOAFzc3MNREhUdXTs2BGbNm2Cjo4OxGIxli9frjQFn4iIqjaOlCMiovfC24ubJycnY9u2bfjnn38QExMDIyMjNGvWDBMnTkS7du2KPNezZ8+wZcsW3L59GykpKbC0tES7du0wYcIEODg4FNm2qHVr3o4xNjYWO3fuxI0bNxAXF4ecnBz8888/ClP8EhISsG/fPty8eROvXr1CTk4OrKys0Lp1a4wZM6bYdYYkEgnOnz+PS5cuITAwECkpKTAyMoKNjQ1atWqFPn364JNPPlGIXWbjxo3YuHGjwvnUrTN39epVnDlzBgEBAUhOToaBgQHq16+Pnj17YuTIkTA0NFQbo1gsxqFDh3DixAk8f/4cIpEI9vb2GDhwIEaMGFHk9RVHtvEAgFKPFlUlMzMThw4dwo0bN/D06VNkZGTAwsICtWvXRseOHeHq6ooGDRootZNKpTh//jzOnDmDoKAgpKWlwcTEBI6OjnB1dcWQIUOgo6P6v11vr/F3/vx5HD9+XP46HzhwoNLozHd5HkXZvn07AEAgEGDp0qUlTtrb2NioLfPz88OBAwfg6+uLhIQE6Ovro27duujWrRvGjBmjNpn39nstKCgIO3fuxL///ovU1FTY2Nigd+/emDRpEkxNTQEUTr09dOgQTp06hZcvX0IsFsPJyQmTJk1Cly5dStSPr68vdu3aBT8/P6SlpaFGjRro3Lkzpk6dqvY6CwoK4OvrC09PT9y7dw+vXr1CdnY2TExM0KBBA/To0QPDhw9XO0pzw4YN2Lhxo3yTm2fPnmHXrl3w8fFBfHw88vLy5Js6FLfRQ35+Po4dO4YLFy4gLCwMGRkZMDY2hoWFBerXr48OHTqgX79+KtdVLM/X8T///IP9+/cjODgYWVlZsLW1lT8vExMTlecordI8q8jISHTv3h1SqRRLly7FV199VeS5e/bsiYiICPTu3Rvr1q0rl3hVadmyJQYPHowjR47gxYsX8PT0xOeff65QJzMzE7du3YKnpycCAgIQExODgoICmJubw8nJCQMHDkSfPn2U3q9vbwh17949+QZMMm//LIuLi4Onpydu3LiBkJAQJCYmQigUwsrKCq1atcLIkSPh7Oys9npkr6HTp0/j8ePHSE1Nhb6+PqpXr446deqgQ4cO6Nu3L2xtbVW2f/nyJfbu3Yvbt28jOjoaBQUFsLa2Rrt27TBu3Dilz9+yXCMRUWXCpBwREb13nj59ivHjxyMuLk5+LC8vDzdv3oS3tzdWrlyJQYMGqWx79epVzJkzB3l5efJjsbGxOHXqFC5dulRuX74ePnyISZMmITU1VW2dixcvYuHChcjOzlY4HhMTg7Nnz+Ls2bOYOXMmpk+frrJ9TEwMpk+fjqCgIIXjeXl5SE1NRUhICPbv3/9OOzRmZGRg9uzZ8Pb2VuojICAAAQEBOHr0KLZv3466desqtc/JycGUKVNw9+5dheOBgYEIDAzEP//8A1dX1zLHV61aNfnffXx88Nlnn5X5XHfv3sXs2bMVEn0AEB8fj/j4ePj5+eHu3btKX+4yMzMxffp03LlzR+F4SkoK7ty5gzt37uDQoUPYsmULatasqbZ/qVSK77//HqdPn1Zb512fR1Gys7Nx8+ZNAIVfYu3t7UvV/m1SqRSrVq3Czp07lWINDg5GcHAw9u/fj02bNsHFxaXIc50+fRo//fQT8vPz5cciIiKwZcsWeHt7Y8+ePSgoKMA333yDBw8eKLT9999/cf/+/SI/F2SOHj2KJUuWQCwWy49FRUXh4MGDOHv2LLZt26Yy1v3798Pd3V3peGpqKh48eIAHDx7gyJEj2LFjB2rXrl1kDNeuXcOcOXPw+vXrIuupkpWVhfHjx8Pf31/heFpaGtLS0hAeHg5PT0/UrFkTvXv3VqhTXq9joHAK9O7duxWOhYeHY/Pmzbh+/ToOHDjwzkn00j4rWULo1q1bOH78eJFJuX///RcREREAgCFDhrxTnCUxdOhQHDlyBABw69YtpaTcDz/8gKtXryq1S0hIwPXr13H9+nWcPXsW69atg56e3jvF0q9fP6Snpysdj4yMRGRkJM6cOYM5c+ZgypQpSnXEYrE8Ifum/Px8ZGZm4uXLl/Dx8YFQKMSECROU2u/ZswerVq1SeJ8DhYm6ly9f4sSJE/jll1808kyIiDSFSTkiInrvTJ06FTo6Ovjtt9/Qrl076Onp4cGDB/jPf/6DmJgYLFu2DF26dIGFhYVCu2fPnskTchYWFvjuu+/QuXNn6Ojo4N69e/jvf/+L77//HlKp9J1j/Pbbb2FgYAB3d3d06NABenp6CAoKgpmZGQDgxo0bmDNnDqRSKVq1aoVx48ahWbNmMDAwkI8UOHPmDNavXw8bGxulLyHp6ekYM2YMXr58CaFQCDc3NwwaNAj29vaQSqUIDw+Hj48PTpw4IW/zyy+/YPHixejXrx+io6MxZcoUpS9Wurq68r8XFBRg8uTJePDgAYyMjDB+/Hh0794dtWrVQk5ODnx8fPDHH38gPDwcU6ZMwfHjx5VGaC1dulSekOvVqxcmTpyIunXrIi4uDocOHcLBgwfx6tWrMt/nli1bwsDAAK9fv8b+/fthZGSE4cOHqx2FoU5gYCAmTpyIvLw8mJqaYsKECfj8889Rs2ZNvH79Gk+ePIGnpydiY2OV2s6ZM0eeyBg4cCBGjx4tv8Zjx45h7969CA4OxtSpU3H48GGFe/ym48ePIzY2Fl988QVGjBiBunXrIiUlBYmJiQDK53kUxd/fHwUFBQAgH135LrZt2yZPyDk7O2PmzJlo0qQJsrOz8c8//2D9+vVITU3FpEmTcOrUKbVJxPDwcCxatAguLi6YNm0aHBwckJWVhaNHj2LLli0ICgrCjh07EBoaipCQEHz//ffo0aMHqlWrhuDgYCxbtgzh4eH49ddf0bVrV7Uj8yIiIrBs2TI4Ojpizpw5aNq0KbKysvD3339jw4YNyMzMxDfffIMLFy7A0tJSoa2BgQH69euHTp06oX79+rCysoKhoSHi4+Ph4+ODXbt24cWLF/juu+9w+PBhtfcsLS0N8+bNQ+3atTFz5kx5UsnPz6/E91yWkBs5ciQGDRqEWrVqQVdXFwkJCQgMDMSVK1dUbv5TXq/j06dP49WrVxg2bBiGDRuGunXrIikpCXv27MGhQ4cQEhKCLVu24LvvvivRNalS1mc1dOhQ3Lp1C4GBgQgLC0OjRo1Unv/48eMAgFq1aqFjx45ljrOkmjRpAn19feTm5iolVAHA0tISo0ePRtu2bVG7dm3UqFEDEokEsbGxuHDhAg4dOoRr165hw4YNmDt3rrzdgAED0KtXLyxZsgRnz55Fq1at5JviyLz9WmjYsCE6duyIFi1awNraGpaWlsjOzkZ4eDiOHDmCy5cvY82aNWjcuDE6d+6s0PbEiRPyhFzfvn3ln2MGBgZISkrC48ePce3aNZWJw4MHD2L58uUAgG7dumHkyJFwdHSErq4uwsLCsG3bNnh5eWHRokWwtbVF+/bty3yNRESVipSIiN5Lx48flzo4OEgdHByk169fl2ZmZqr8k5WVVaLz3blzR36+48ePl0uMsvP98MMPJao/atQoqYODg7Rbt25KZW9eb6dOnaSJiYlKdR49eiSvc+DAAaXyKVOmSB0cHKTNmjWThoaGKpXHxsZKO3ToID/H+vXrler88MMPUgcHB+moUaOKjLFNmzbS6Oholdf5+vVreT9z586VSiQSlfVWrVoldXBwkLZv3176+vVrhbIlS5bI+zp//rzK9lKpVJqfn690rFu3bmqv700eHh5SBwcHaYsWLaTBwcEq60RHR0vbtm0rdXBwkO7YsUOhLDAwUB7j/PnzVbZfu3atvI6Dg0OR8aizZcsWhXM4ODhIu3fvLp0zZ450x44d0oCAAKlYLFbbXiKRSPv27St/bk+fPlVb9+37+ffff8v7dHd3V9lm586d8jp79+5VKpc9DwcHB+mqVavU9v2uz6M4hw8fLtFrqiQSExOlTZs2lTo4OEi//PJLpdevVCqV+vv7S52cnKQODg7Sb7/9Vqlc9l5zcHCQTpw4UVpQUKBUZ+7cuVIHBwdpkyZNpE2aNJHev39fqc7Tp0+ljo6OUgcHB+mhQ4eK7Kdfv37SzMxMpTq3bt2Sn2Pp0qUlvQ1ysbGx0tatW0sdHBykt2/fVipfv369PIaePXtK09PT1Z7rzc+Zt33xxRdSBwcH6fTp00sVX3m/jv/66y+V55B9Bn/66aelik/mXZ9Vbm6u/P2xYsUKlX1kZmZKW7RoIXVwcJCuXbu21DGW9Wdp7969y3xvrl+/Lv9syMjIUCov6udWacl+Lo0cOVKpbMaMGVIHBwfpF198UapzxsfHS5s1ayZ1cHCQrlmzRmUdiUQinT17tvzZv608r5GISJO40QMRURUwefJkuLi4qPzz9m+yq4Lp06crjVQBACcnJ/laMoGBgQplSUlJuHHjBgDgq6++UjlCwtraGlOnTi2XGCdMmIBatWqpLDt//jwSExNhaGiIJUuWqF23a8aMGTAyMkJSUpLCdMXMzEz5SI5+/foVOf1T3fpPJbFnzx4AwJgxY9C4cWOVdWrVqoVRo0YBAM6ePatQJhulp6uriwULFqhsP23aNNSoUaPMMQKFr/+ff/5ZYQTUy5cvcf78efz2229wc3PDZ599Bg8PD6VpUUDhdLGwsDAAwNy5c9GwYUO1fb19P48dOwYA8pGXqowdOxYfffQRAMinqKliZmaGmTNnqi1/1+dRnDenWsvWaCur06dPy6eI//TTTyo34HB2dpaPAL169arStOE3LVy4UOVol759+wIoHEXYp08ftGrVSqlOw4YN0aRJEwBAQEBAkXHPmzdP5bTKDh06oEePHvJrk40oLClra2t06NABQOE066LMmjWrzPdfFldx00vfVp6v41q1amHy5Mkqy2TPOyEhATExMaWK8W1leVZ6enryKcxnzpxR+Xlw8eJFZGdnQyAQaHSapGwqvqqpo8Xp0qULqlevjuzsbJUj7crTF198AUBxkx0Z2b0u7Wf6oUOHkJubi9q1a6tc0xQoXOdy3rx5ACAfFUtEVBUwKUdERO+dohKN9evXBwD5lD8ZPz8/+a52si9sqvTs2bMcIgS6du2qtkz2pbxFixYQCoXIyspS+Ucikciv580ko6+vrzzhMXjw4HKJ920RERGIiooCALRr105tjFlZWfLNMUJCQhTW6vP19QUAtG7dWuWi8kDhl+Ru3bq9c7wjR47EtWvX8Ntvv6Ffv35K63bFxMRg5cqVGD9+vNJaXbLnoauri/79+5e4T6lUKl+/rFu3bmp3fhUIBPL1u0JDQ9V+6W7Xrp3ac5TH89Ck+/fvAyhcx6tZs2Zq68kSymKxWO0Uzbp168rfB2+rV6+e/O9FTTOU1UtISFBbx8jIqMhzyD4bsrKyVK7TmJOTg/3792PChAno2LEjmjVrBkdHR/mfS5cuAQBevHihtg+BQPBOv0iRJWtPnDiBM2fOlOj5l/fruEOHDmqnC775HIt6FsV5l2fl5uYGoPAXNdevX1dqK/uFR5s2bUq9LuO7kP7/sgnqfkkTGxuLP/74A8OHD0fbtm3lv4SS/ZEltYt6fZVUYGAgfv75Z/Tv3x+tWrVC48aN5f28mQh/+fKlQjvZ68/Lywt79uxRWi9VHdlncNu2bZGbm6v2s83c3Fy+LMXbv3gjInpfcU05IqIqYM+ePWjbtq22w9CYokaByNbQevs3+JGRkfK/FzUSytraGqampsjIyHinGIv6Mvf8+XMAwO3bt4td4F7mzVFEb34RUjdi6l3JYgQKR8iUhEQike+ACECeRCrqfpekvKSMjY0xaNAg+UiYlJQU+Pr64uzZs7hy5QokEgnu3buHNWvWYOHChfJ2sjXt7OzsSrUGW2ZmJtLS0gBAPoJIHVm5VCpFdHS0wgYVMm/uyvu28ngexXlzpOG7vv6jo6MBFH9f3hyxKnu9vK2o9/ubCSRra2u19QwMDACgyM0T7Ozsilx76s1riYqKgpOTk/zf4eHhmDBhgsLnjDpF3VsLC4t32pl0xowZuHr1KjIzMzF//nwsXboUrVq1QqtWrdCuXTs4OzsrJX3K+3Vc1POSPQeg6GdRnHd5Vg0bNkTLli3h5+eH48ePK/ySJjw8XJ6g1PRmArLXhWzd0TddvXoV8+fPL1GS613fu2vXrsWWLVtKtLbq232NHTsWJ06cQExMDJYvX47Vq1ejRYsWaN26Ndq0aYPWrVurHL0t+3w7ceKEwjqoRUlJSSlRPSKiyo5JOSIiqjA6OjooKChQOUVIFVm94qZclmXR5je/zBS365+RkdE7f7EpKrmTmZlZ6vO9OeLlzfbvuoOhOmW9/tzcXPnfZffcyMioyDYVdQ0WFhbo3r07unfvjqtXr2LGjBmQSqU4cuQI5s2bJ19sXHY/SxtHVlaW/O/FtX2z/M12byrqNVMez6M4byYF30wCloXsGsvjvpT0/S4UvtsEkOJep2+WvxmrbMfJyMhIGBkZYcyYMejYsSPq1KkDY2Njefw///wzzp07p7Bb6NtKkxRWpW7dujh58iQ2btyIK1euICsrC15eXvDy8gIA+fRA2RTEt69Fk8+rJEkfdcr6rGTc3Nzg5+eHmzdvIiEhQZ64lo2SMzU1Ra9evcocX2nl5+fLE7pvJzUjIyPx3Xffyad3jhs3Di1btoS1tTUMDQ3lSVZXV1fExsYW+foqzoULF7B582YAhSOchw8fjsaNG6N69erQ09ODQCBAVFSUfETx231Vq1YNx44dw19//YWzZ88iPT0dd+/exd27d/Hnn3/C0tISEydOxNixYxXer2X5mViazzYiosqMSTkiIqowpqamSElJKfEaObLRGqpGX7yrt7+kqdr9TaakU27eNZaePXtiw4YNpW7/5kiarKwstdPN3sWb9+vMmTPytfpKe46MjIxi76e6L/flqXv37ujSpQuuX7+OnJwcPH/+HB9//DGA/93P0sbxZoKiuGssTVJYlfJ4HsVxdnaWJ9Lv3bv3TueSXWNpnn1FJWdLqqzP8N69ewgNDQUArFu3Tu3004r+XJGpV68eVq1ahf/85z8IDAyEv78/bt++jdu3byMqKgoLFixAamoqxo0bB0Czr+Py8q5x9unTB8uXL0dWVhZOnTqFSZMmQSwW49SpUwAK1yp8c1RfRQsKCpL/4qVly5YKZcePH0dubi5MTExw5MgRWFlZqTxHWRJbb9u/f788hr1796pMdBe3nqKVlRV+/vln/PTTTwgODkZAQADu3LmDmzdvIikpCb/99huio6OxaNEieRsjIyOkp6dj3LhxatcfJSKqqrimHBERVRjZOk7Pnj0rtm5eXp58GuGb60SVlzdHARUVT1xc3DuPkiuObGrr2+vxlNSb9+fx48flEtPb3px+K3supSVb162451+S10d5eHNK25vTm2X3MyIiQmnac1FMTEzkU81kG0WoIysXCASwtbUtcR8y5fE8imNsbIxPP/0UQGGiKSIiosznkj37kt6XN9toS0RERJGjjJ4+fSr/+5uxyhacNzMzK3I9OFniTlP09PTQqlUrTJgwAdu3b8fff/8NOzs7AMCmTZvk16rJ13F5KeuzkjEyMkK/fv0A/G9DGm9vb8THxwMAhg4dWp7hFku20QagvDai7DO+Xbt2ahNy0dHR5ZKUk/XVp08ftSNPS/o6FolEaNasGUaNGoWNGzfi+vXr8oTj/v37FZZkeNefiURE7zMm5YiIqMLIdkKMiooqdqe0GzduyKevtm7dutxjadmypfxLxt9//6223pUrV8q977fJvnQ9efJE4ctjSbVq1Uo+Ou7kyZOlbi+bHlzUl1oHBwf5NKrz58+Xug/gf8///v37anfWzMvLg6enZ5nOX1qxsbHyv785RUyWiMrPz8e5c+dKfD6BQCBfE/D69etqF9WXSqW4fPkygML7WpaRoOXxPEpi0qRJAApjXrp0aYmnGMbFxSkkBWTv4cjISAQFBaltJ9v8QCQSKY0Q0rTs7GyFXY7fJvtsMDY2lm+mAfxvanlR7ydfX98SrTdXkWxtbTFs2DAAhaOSk5KSAGj2dVxeyvqs3iTb8OH58+fy9eUAwNHRscjNScqbn5+f/HP8o48+Ukrsyn4uFvX6On36dJF9lOQzH/jfa1m2KVJZ+lLHwsICX3/9tfz8byb9ZT8Tb9++Xea14kp6jURElQ2TckREVGHc3Nzk69389ttvateWy8jIwNq1awEUfokqzQ6YJWVpaYkuXboAAA4dOqRyREh8fLx8PZ2KNGDAAFhZWUEqlWLBggXFjsx79eqVwhdlExMT+UiOc+fOyb8oq6JqqpFs9zrZqBBVBAKBfHrbxYsXcebMmSJjFIvFSiOrZDvD5ufnY+XKlSrbbdq06Z12YfT19cW2bduKnc4WFBQk/6Jub2+vMHqmffv28i/uq1evRnh4uNrzvH0/ZV/sk5OT5a/ht+3Zs0f+evvyyy+LviA1yuN5lMQnn3wij9HHxwfz588vdu0mT09PfPHFF0hNTZUfGzBggHyK+PLly1Umeh4+fIijR48CKJxerG6HXk1avXq1yteSj4+PPJk/cOBA6Orqystko3wyMzNx584dpbaZmZlYtmxZBUWsqLhRp7KRSCKRCKampvLjmnodl6eyPKs3NWvWTD6FfceOHbh27RqAitvRWpVbt27hm2++QUFBAXR0dLBo0SKljThko7z9/PxUJqxCQ0OxdevWIvspyWc+8L/X8rVr11Qm5E+cOCHfKVWV4l5/b47yfXNjmZEjR0JfXx/Z2dn48ccfi901WFU/Jb1GIqLKhkk5IqIPVGxsLPz9/eV/3hyx9fLlS4Wysk4padCgASZMmACg8IvSV199hYsXLyI6Ohrp6el4+fIlTpw4gaFDh8r/k71gwQKFL4vlaf78+dDT08Pr168xZswYHD16FHFxcUhMTMTFixcxYsQI5OXlVfgIEENDQ6xYsQIikQiBgYEYOHAg9u7di6dPn8pHsAQFBeHw4cOYPHkyevXqpTQ1ac6cOahXrx6kUinmzJmDpUuXws/PD8nJyUhOToa/vz/++usv9OzZU6n/pk2bAgD++ecf3L17F9nZ2SgoKEBBQYHCCInRo0ejTZs2kEqlmD9/PubNm4dbt24hPj4e6enpiIyMxI0bN/Dbb7/h888/x+7du5X6ke2Eevr0acyaNQsPHz5Eamoqnjx5gmXLluGvv/4qctfR4qSnp2P16tXo2LEjfvzxR5w/fx7Pnz9HamoqkpOT8fDhQ6xduxajRo2Sf9GbO3euwjkEAgFWrFgBPT09pKamYtiwYdiyZQvCwsKQlpaGuLg4eHt749dff8WMGTMU2n7++efykS07d+7Ejz/+iEePHiE1NRWhoaFwd3eXJySdnJzkI5XK4l2fR0n99NNP8ms6e/YsevXqha1btyIwMBAJCQlITk5GSEgI9u/fj+HDh2Pq1KnyUVcylpaW+PbbbwEUJk7HjBmDW7duITk5GVFRUdi7dy/Gjx+P/Px8mJiY4Pvvvy/zfSkv1tbWePHiBUaNGgUvLy95rB4eHpg+fTqkUinMzc2VXgOdOnWSf2bNnTsXp06dQnR0NBISEnDp0iUMGzYMYWFhqF+/foVfQ9++fTF27Fjs378fjx49QlJSEpKTkxEUFIQVK1bgyJEjAApft29uKqHJ13F5KOuzepssGfn3338jPz8furq6GDBgQLnFmZeXh6ysLGRlZSEzMxNxcXEICgrCgQMHMHr0aIwfPx4pKSnQ1dXF0qVL0b59e6VzuLq6AgBSU1MxYcIE3Lp1C0lJSXj16hV2796NUaNGwdjYWCHJ9TbZZ/6rV69w+PBhpKWlyT/z3xxZJuvr3r17mDt3Lh49eoSUlBSEhIRgxYoVWLRoUZE79E6aNAnDhg3Dzp074efnh4SEBKSkpCA0NBR//vkn1q9fD6DwNfTm+8Ha2lq+xpynpycGDx6MY8eOISIiAunp6UhISEBAQAD27NmDUaNGqZxeXNJrJCKqbLjRAxHRB+ro0aPYuHGjyrJNmzZh06ZN8n9/8cUXakc6Fee7776DWCzGrl27EBgYiNmzZ6usp6Ojg/nz51fol72GDRti7dq1mDNnDpKTkxUWmgYAfX19rFu3Dr/++muJN6coq86dO+Ovv/7CDz/8gKioKPznP/9RW1ckEintaGhqaordu3dj2rRpePz4MQ4ePIiDBw+WqO8RI0bg6NGjSE1NxZgxYxTKZsyYIU+m6OrqYtOmTVi4cCEuX76Ms2fP4uzZs2rPq2pEytKlSxETE4O7d+/i0qVL8umKMu3atYOrqyt+/vnnEsX+NkNDQ4hEImRlZeHEiRPy9aHU1f3xxx/VJiq3bduGWbNmITU1FWvWrMGaNWuU6rVp00bp2Nq1azF9+nTcuXNHbQxNmjTB5s2b1Y7aKYnyeB4loa+vj82bN2P9+vXYtWsXYmJi8N///rfIfr766iv5DpYykyZNQkpKivwL+vjx45Xampubv3NitrzY2dlhxowZWLJkiXwa75tMTEywadMmWFpaKhw3NTXFsmXLMH/+fCQmJuKHH35QKBcKhVi4cCGCgoLw4sWLCr0GqVQq39RBncaNG2PJkiVKxzX1Oi4PZX1WbxswYAB+//13vH79GgDw2WefleuIzSVLlqi8129q0aIFFi1apHbKbNu2bTFixAgcOHAAQUFBSu8jMzMz/PHHH/j+++8VRqu+qVu3bqhfvz5evHiBn3/+WeHztk2bNti7dy8AYOLEibhx4wYCAwNx/vx5panyjRo1gru7uzyZqUpAQAACAgLUlteuXVvl58mwYcMgFArx66+/IiwsDD/99JPac8jWQCzLNRIRVTZMyhERUYUSiURYsGABvvjiCxw+fBj//vsvYmJikJOTAxMTE9SpUwft2rXDV199pZEv5t27d8epU6ewZcsW+Pj4IDU1FdWrV0ebNm0wceJEfPzxx/j1118rPA4A6Nq1K65evYojR47gxo0bCAsLQ3p6OnR1dVGjRg00atQI7du3R69evVR+CbG1tcXx48dx5swZXLhwAcHBwUhLS4OpqSmsra3RunVr9OnTR6ldw4YNcfDgQWzbtg1+fn5ISkpSO7XYxMQE69evx/3793HixAn4+voiPj4eeXl5MDExgZ2dHZo3b45u3bqpHOVhaGgIDw8PHDp0CCdOnMDz588hEAhgb2+PAQMGYNSoUcVOxSxKu3bt4O3tDS8vL9y/fx+PHz9GZGQkMjMzIRKJUK1aNTRs2BDt27fHF198AWtr6yLP9ffff+PAgQPw9PTEixcvkJ2dDUtLS9SqVQudO3dG3759Vd6jXbt24dy5czhz5gyCgoKQnp4OY2NjODo6wtXVFUOHDpWvefQu3vV5lJRIJMKcOXMwatQonD59Grdu3ZKPQJRKpbCwsICDgwM6dOiAfv36KSXkgMIRiD/88AN69uyJ/fv3w9fXF4mJidDT00O9evXQtWtXfP3110WO8NG0YcOGoUGDBvDw8EBAQABSU1NRs2ZNdOrUCd988w1sbGxUtuvbty9sbGywbds2PHjwQP66admyJUaPHo1WrVppZFfJEydO4Pbt27h79y5evnyJxMRE5ObmwszMDB9//DF69+6NQYMGqUyqafJ1XB7K+qzeVK1aNfTs2VP+GVSRGzzo6urC2NhY/pnUtGlTdO/eXT6FtihLlixB8+bNcejQIYSGhkIikcDa2hqdO3fGuHHjit0kRU9PD/v27cPmzZvh7e2N6OholdPSDQ0NsXfvXmzfvh0XLlxAZGQk9PX1UbduXfTq1Qtff/210qjYN23fvh23bt3CnTt3EB4ejoSEBGRlZcHU1BQfffQRPv/8c3z55ZcKu0m/aejQoejatSsOHToEb29vvHjxAhkZGdDX14e1tTUaN26MDh06oEePHmW+RiKiykYgLekKvkREREREVcyCBQtw8uRJjqb5QC1ZsgSHDh2CjY0NPD091e46SkREVBH4U4eIiIiIiD44ubm5uHDhAoDCZRqYkCMiIk3jTx4iIiIiIvrgHD9+HOnp6RAKhVrfvIKIiD5MlWNRCCIiIiIiIg3Iy8vD/fv3sW7dOgCFu47a2tpqOSoiIvoQMSlHREREREQfBEdHR4V/m5ubY/78+VqKhoiIPnScvkpERERERB8UCwsLfP755zhw4ECJdmolIiKqCNx9lYiIiIiIiIiISMM4Uo6IiIiIiIiIiEjDmJQjIiIiIiIiIiLSMG70UE6SkjLAicBERERERERERB8ugQCwtDQtUV0m5cqJVAom5YiIiIiIiIiIqEQ4fZWIiIiIiIiIiEjDmJQjIiIiIiIiIiLSMCbliIiIiIiIiIiINIxJOSIiIiIiIiIiIg1jUo6IiIiIiIiIiEjDmJQjIiIiIiIiIiLSMCbliIiIiIiIiIiINExH2wF8yMTiAkgkEm2HQURVhEAggEikA4FAoO1QiIiIiIiIqBhMymlBTk4WsrLSUVCQp+1QiKiKEQiE0NMzgKmpOXR0dLUdDhEREREREanBpJyG5eRkIS0tEXp6hjA3rwGRSASAo1qI6F1JIZFIkJ+fi5ycLCQlxcLCoib09PS1HRgRERERERGpUGmTcs+fP8etW7cQFBSEoKAgPHv2DGKxGLNmzcK0adPKfF4fHx94eHjg4cOHyMnJga2tLXr16oXJkyfD2Ni4HK9AtaysdOjpGcLCoganmBFRudPXN4SRUTUkJ8chMzMV1atbazskIiIiIiIiUqHSJuUOHjyIPXv2lOs5d+3ahRUrVkAgEKB169awtLSEr68vNm/ejMuXL+PAgQOoXr16ufb5JrG4AAUFeTA3Z0KOiCqOUCiEsbEp0tKSIBaL/39ELhEREREREVUmlTYp5+DggPHjx6NJkyZo0qQJtmzZgtOnT5f5fMHBwVi5ciVEIhE2bdqELl26AABycnLwzTff4Pbt21i6dCnWr19fXpegRLapA78gE1FFE4kK15OTSJiUIyIiIiIiqowqbVLOzc1N4d9CofCdzrdlyxZIpVIMHjxYnpADAENDQyxfvhzdu3fH5cuX8ezZMzRs2PCd+ioeR8kRUcXiaFwiIiIiIqLK7d0yXe+JvLw83LhxAwDQr18/pfLatWvDxcUFAHD16lWNxkZERERERERERB+eDyIpFx4ejpycHABA06ZNVdaRHQ8ODtZYXERERERERERE9GGqtNNXy1NkZCQAoFq1ajAxMVFZp1atWgp1S6skM8U4m4yINE0g4GcPERERERGRppTm+9cHkZTLysoCULh+nDpGRkYAgMzMzDL1YWlpWmyd169fIzlZCJFIAB2dD2KQ4nvD1/c+pk+fjJYtW2HTpm3aDqdY7doVTre+c+dBqdpFR0dj8OB+sLGphVOnzldEaFRJSCQCCIVCWFgYw8DAQNvhEBERERER0Vs+iKScJiQlZUAqLbpOfn4eJBIJxGIpCgokmgmMSkQsLnweUun79WzejnXGjMnw93+A9es3w8WltVJ92XWqaktVi1gshUQiQUpKFnR187UdDhERERER0QdBICjZwC3gA0nKGRsbA4B8XTlVsrOzAUDt9NbiSKUoNilXXDlRSe3ff0zbIdB7oiSfTURERERERKR5H0RSrnbt2gCA9PR0ZGZmqky8xcTEKNQlqszs7Oy1HQIRERERERERvYMPIilXv359GBoaIicnB48ePUK7du2U6jx69AgA4OTkpOnwqAIEBz/C9ev/wM/PF3FxcUhPT4OpaTU0buwEN7fh+OSTtqU6X0CAP3bv3oHg4ECIxWLY2dXHkCHD0KdPP3TsWDhN1Nv7vlK7+Pg47N+/G3fv3kZ8fBx0dXXRoEFD9OrVF/37D4JIJFKof+HCWbi7L0OfPv3w7bdz4OGxHbdueSEhIR5OTs2wceNWAFDq88GD+5g5c6r8PG/+HQAWLlwCV9f+CsekUinOnDmJ06dP4OXLcIhEIjRp0hQTJkxB06bNla7lzT4vX76AY8cOITz8BfT19dGqVRt8881M2NjYQCqV4sSJIzh79jQiI19CX18fHTp0wrRpM2FhUb1U952IiIiIiIioqvogknJ6enro0qULLl26hHPnzikl5aKiouDn5wcA6N69uzZCpHK2Zctf8PO7j/r1G8DR8WMYGBgiKioSPj434eNzEzNnzsWwYV+V6FxXr17GL78shkQiQcOGH6F+/YZITEzAihW/IDz8hdp2jx8HYe7cmUhPT4O1tQ06deqCzMws+Pn5IjDwIby8ruO339ZAV1dXqW1aWiomTBiDzMwMODu3gKNjY5X1ZCwtrdCnTz/cvXsbyclJaNOmPSwtLeXltWvXVWrj7r4Mf/99Cc7OLdGhQyeEhT3Bv//eRUCAHzZs2Aonp6Yq+9q8eSMOHtyLFi1c0LZtBzx+HIR//rmCwMAA7Np1EKtXr4C3txdatmwFW9vaCAwMwMWL5xAa+gTbt+8p8jqIiIiIiIiIPhRVKim3b98+7Nu3D82bN8eqVasUyiZPnozLly/jxIkT6NmzJzp37gygcJ25n376CWKxGL169ULDhg21ETqVs+HDR2Lx4l9gZWWlcPzRo4eYO/db/PXXOnTr9jlq1KhZ5HkSExPw22/LIZFIMGvWPLi5DZeX+fs/wPz5s1S2y8vLw+LFC5CenoZBg4Zg9uz50NEpfLtFRUVi9uxpuHfvNnbu3IopU6Yrtffx8UarVm3g7r4KxsbFr3NoZ2ePn35aihkzJiM5OQmjRn2tcqMHmdjYGPj5+WLPnsOoV88OACAWi7Fq1XKcP38GO3Zsxpo1G1W2PXv2JLZv34tGjRwAALm5rzFnzgw8fOiPb7+djNevX+PAgWOwsakFAEhNTcXUqePw7FkYPD2vomfPPsVeDxEREREREVFVJ9R2AOoEBQVh2LBh8j/Xr18HABw+fFjheHx8vLxNSkoKXrx4IV8f7k1OTk5YsGABxGIxJk+ejNGjR2P27Nno0aMHbt++jfr162Pp0qUaujqqaO3bf6qUkAOApk2bY/DgYSgoKMDNmzeKPc+5c6eRk5ONpk2bKyTkAKBFCxcMGjRUZTtPz6uIjY2BlVUNzJw5V56QA4Datetg+vTCZN7x40eQm5ur1F5HRwfff7+wRAm5spo9e748IQcAIpEIkydPA1CYcCwoKFDZbsKEqfKEHADo6xvgyy9HAgCePXuK2bPnyRNyAGBubo5Bg4YAAO7fv1fu10FERERERET0Pqq0I+UyMzMREBCgdDw2NhaxsbHyf+fl5ZX4nGPHjoWDgwN27tyJwMBAZGdnw9bWFoMHD8bkyZPLvPMqVU5paanw8fHGixfPkJGRIU8yRUa+BAC8fBlR7Dn8/B4AAHr06K2yvGfP3jh4cK+Kdr4AgM8/7wk9PT2l8i5dPoOpaTVkZKTjyZPHaN68hUJ5o0aOqF27TrHxlZVIJEK7dh2UjltaWsnjSktLhaWlcmKzfftPlY7VrVtXft5PPlFes7FOnXoAgMTExHcNnYiIiIiIiKhKqLRJubZt2+LJkyelavPtt9/i22+/LbJOhw4d0KGDcjKCqpYzZ05iw4Y1yMnJUVsnOzur2PMkJMQBAGrVslVZbmOj+nhCQgIAwNZWdblAIECtWrbIyEiX132Tuv7Ki6WllcLovTcZGxsjIyNdbcLb2tpG6ZihoVGR5zUyKizPy1MeFUhERERERET0Iaq0STmisgoJeYzff3eHUCjEN998i08/7QxraxsYGBhAIBDg9OkT+P13d0il0hKfUyBQd1xNwTvS19evkPPKCIVln7leVNt3OS8RERERERHRh4RJOapyPD2vQiqVYujQLzFy5NdK5ZGRr0p8rho1auLlywiV6xQCQExMtJp2NQAA0dFRas8tayurS0REREREREQfDg5roSonPT0dAGBtXUupLDc3F9evXyvxuZydWwIArl69rLL8778vqTzesmUrAMA///ytciOHGzc8kZGRDiMjYzg6Ni5xPMXR1dUFULiTKhHR+0goFEBHR6iVP0JhxYx+JiIiIiJShUk5qnLs7e0BAJcunVNYNy43Nxf//e9KxMSoH732tn79BsLAwAAPH/rj+PEjCmUPH/rj5MljKtt169Yd1tY2SExMwIYNaxV2Mo2OjsLGjX8AAIYMGVauU1Vr1KgJAHjx4nm5nZOISFOEQgHMLQxhYWGslT/mFoZMzBERERGRxnD6KlU5rq4DcPToIYSGPoGb2wA0b94SIpEQAQH+yM3NhZvbVzh69GCJzlWzpjXmz1+I5cuXYu3aVThz5iTq12+AxMQEPHzojy+/HImDB/cqbW6gp6eH//znN8ydOxOnTh3DnTu34OTUFNnZ2fD1vY+8vFy0adMe48dPLtdr79r1c1y4cBabNq3H/fv3YGFhAYFAgL59B6BZM+dy7YuIqLwJhQKIhCIsu7wM4SnhGu3b3sIeS3otgVAogERS8jVHiYiIiIjKikk5qnJMTU2xffte7NixBffu3cbduz6oVs0Mbdq0xbhxk/HwoX+pzterlytq1rTGnj07ERz8CFFRr1Cvnj2+//4nfPJJWxw8uBdmZuZK7Ro3doKHx37s378bd+74wMvrOnR19eDg4IjevV3Rr98gtTugllWHDh3xww+LcPLkMTx48C9ev34NAGjevAWTckT03ghPCUdoQqi2wyAiIiIiqlACaWm2oCS1EhMzUNydzM/PQ1JSDCwta0FXV08zgVGFunjxHJYvX4pPP+2E335bq+1wiOT4eUPvIx0dISwsjDHu0DiNJ+UcajjAY7gHUlKyUFAg0WjfRERERFR1CASAlZVpiepyTTmiYsTGxiIpKVHp+MOH/vjzz3UACqfMEhERERERERGVFKevEhXjwYN/sXLlr/joo0awtraBUChEVFQUnj4tHMXh6tofXbp003KURERERERERPQ+YVKOqBhOTs3g6tofAQF+8PPzRU5ODkxNTdG6dRv07TsAPXr01naIRERERERERPSeYVKOqBh2dvZYsGCxtsMgIiIiIiIioiqEa8oRERERERERERFpGJNyREREREREREREGsakHBERERERERERkYYxKUdERERERERERKRhTMoRERERERERERFpGJNyREREREREREREGsakHBERERERERERkYYxKUdERERERERERKRhTMoRERERERERERFpmI62AyBlQqEAQqFA22GUmEQihUQi1XYYRERERERERETvDSblKhmhUABzcyOIRO/PIEaxWILU1OxyT8x5e3vhwIE9ePo0DNnZWQCA9es3w9jYBL6+/+LJk8d48iQEUVGvIJVKsXjxL+jVy7VcY6gsli9fiosXz2HhwiVwde0vP75jxxZ4eGzDuHGTMGHClArrPyYmGm5uA2BjUwvHjp0tVduOHVsDALy971dEaERERERERETvJSblKhmhUACRSIhFB27iRXyatsMpVv2aZvjPiE4QCgXlmpQLC3uCRYu+h1QqhYtLa1haWkEgEMDS0gqbN2/AzZs3yq0vIiIiIiIiIiJNY1KuknoRn4aQqGRth6E1Xl7XUVBQgNGjx2HKlOkKZU2aNIO9fQM4On6MRo0csWLFL/D3f6ClSLVryJAv0b17L5iZmWs7FCIiIiIiIiIqBSblqFKKi4sFANStW0+pbPTosRqOpvIyNzeHubm5tsMgIiIiIiIiolJiUo4qFdkaaTLu7svg7r4MANCihQs2btxa4TGUZP20oUP7IzY2BkePnkGtWrYqj8fERGPvXg88fhyMvLw82Nvbw83tK/Tp00/lOdPT0+DhsR1eXp5ITk6ChUV1dOzYGRMnTlUbq7o15S5cOAt392Xo06cfvv12Djw8tuPWLS8kJMTDyanZO93HgoICHD68Hxcvnkd0dBQMDQ3g4vIJJk6cCjs7+yLbnjlzEqdOHcfLl+HQ0dFB06bNMXbsJDRt2kyp7rvcSyIiIiIiIqLKjkk5qlQaNXJEnz798PChP6KiItGsmTPq1KkLAKhXz167wZXC+fNnsHv3Djg4fIy2bdsjNjYGQUGBWL58KTIy0jFs2AiF+snJSZg2bRIiI1/C1LQaOnToCIlEiitXLuHu3duoX79BmeJIS0vFhAljkJmZAWfnFnB0bAxdXd13urYlS37ErVs30aKFCxo2/AiPHwfB0/Mq7tzxwdq1G9G0aXOV7TZsWIMjRw6iWTNndOzYBc+fP8WdOz7499+7+OWXlejSpZvKdqW9l0RERERERETvAyblqFLp3LkrOnfuiuXLlyIqKhL9+w9S2G30fbFv3y6sXLkGn37aSX5MNnpt586tGDhwMPT1DeRla9asQmTkSzg7t8Rvv62FiYkJgMLRc/PmzYK3t1eZ4vDx8UarVm3g7r4KxsYm73ZRAGJjY/D6dQ62b9+Ljz5qBAAQi8XYsGENjh07jKVLf8KBA8ehp6en1PbUqeP444+/0KrVJ/JjBw7swV9/rceKFcvQvLkzLCyqK7Ur7b0kIiIiIiIieh8ItR0AUVU0ZMiXCkkkAHB17Q87O3tkZmYiJOSx/HhcXCy8vDwhEAgwb96P8oQcAFSrZob5838scxw6Ojr4/vuF5ZKQkxkzZoI8IQcAIpEI06bNQo0aNREbG4Pr16+pbDdw4GCFhBwAjBgxBh9/3ASZmZk4e/aUynaluZdERERERERE7wsm5YgqwNtJJBk7u/oAgISEePmxgAA/SCQSODh8rHKaaqNGjmjYsJHS8ZJo1MgRtWvXKVNbdVSt46anp4fPPusBAPDz8y1xOwDo3du1yHaluZdERERERERE7wsm5YgqgLW1jcrjxsbGAIC8vDz5sfj4OABQ2DDibba26suKUtQ5y8LExBSmpqYqy2QxJiTEqYmldpHH1SXXSnMviYiIiIiIiN4XTMoRlYFEIimyXCisHG8tfX19jfcplZa1neqGleVeEhEREREREZUnftsleotsd9Ls7GyV5QUFBUhKSiy3/mrUqAmgcBMFdWJi1JdpUmZmBjIyMlSWyWKsWbOmmvIolcdjY6MB/O8+EBEREREREX0ImJQjeou5uQV0dXWRnp6GlJRkpfK7d29DLBaXW3/Ozi4QCAQIDQ1BRES4UnlYWCiePQsrt/7e1eXL55WO5efn49q1vwEALVu2Utnu0qULRR5X146IiIiIiIioKmJSjugtOjo6cHZuCQDYunWTwlTVsLBQrF27qlz7s7GxQefOXSGRSLB69QpkZWXKy9LT07FmzUq1Uzu1YdeuHXj+/Kn83xKJBJs2rUd8fBxq1rRGly6fqWx36tQxPHhwX+HY4cP78fhxEIyMjNGv38AKjZuIiIiIiIioMtHRdgBEpeXj441du7bL/x0e/gIAsHPnVhw/fkR+fOvWXWXuY9KkaQgI8MPZsyfh7++Lhg0bITExASEhwejRozf8/HyLnG5aWt999wOePg2Dn58v3NwGomVLF0ilwIMH92FmZoaOHTvD29ur3PorK2trGzg6Nsb48aPQsmUrVKtmhpCQYERFRcLQ0BBLlixXu47dwIGDMWvWN3B2bgkrqxp48eIZnj17CpFIhB9/XAxLSysNXw0RERERERGR9jApV0nVr2mm7RBKRBtxpqamIDj4kdLxqKhIREVFlksfTk5NsWHDVuzcuQVBQYGIj49D3bp2mDVrHgYNGgI3twHl0o+MpaUVtm7dBQ+PbfDyug4fH29YWFRH9+49MXHiN/jzzz/Ktb+yEggE+OWXFThwYA8uX76AgAA/GBgYomvXzzBhwlTUr99AbduZM+eiXj07nD59Ao8fB0FHRwdt23bA2LET0KyZswavgoiIiIiIiEj7BNLKNC/uPZaYmFHsrpP5+XlISoqBpWUt6OrqqawjFApgbm4Ekej9mVksFkuQmpoNiYQvJaLKoiSfN0SVjY6OEBYWxhh3aBxCE0I12rdDDQd4DPdASkoWCgqK3mGbiIiIiEgdgQCwsjItUV2OlKtkJBIpUlOzIRQKtB1KiUkkUibkiIiIiIiIiIhKgUm5SohJLiIiIiIiIiKiqo1JOarSIiLCsW/frhLXHzVqLOzs7CssnsogIMAf586dKnH96dNnw9zcvMLiISIiIiIiIvoQMSlHVVpSUiIuXjxX4vp9+vSr8km5qKhXpbon48dPZlKOiIiIiIiIqJwxKUdVmotLa3h739d2GJWKq2t/uLr213YYRERERERERB+092eLTyIiIiIiIiIioiqCSTkiIiIiIiIiIiINY1KOiIiIiIiIiIhIw5iUIyIiIiIiIiIi0jAm5YiIiIiIiIiIiDSMSTkiIiIiIiIiIiINY1KOiIiIiIiIiIhIw5iUIyIiIiIiIiIi0jAdbQdAREREVFmIRNr5faVEIoVEItVK30RERESkHUzKVUJCoQBCoUDbYZQYv0gQEdH7rrpRdUjFYlSrZqiV/iViMVJSc/jzlIiIiOgDwqRcJSMUCmBhbgihSKTtUEqMXySIiOh9Z6pvCoFIhMe//orsiAiN9m1kZ4fGixdDKBTwZykRERHRB4RJuUpGKBRAKBIh8cQC5Cc+13Y4xdK1agCrwSsr5IuEt7cXDhzYg6dPw5CdnQUAWL9+M4yNTeDr+y+ePHmMJ09CEBX1ClKpFIsX/4JevVzLNYbKYvnypbh48RwWLlwCV9f+8uM7dmyBh8c2jBs3CRMmTKnQGDp2bA0A8Pa+X6H9EBFpU3ZEBDJDw7QdBhERERF9AJiUq6TyE58jP/axtsPQmrCwJ1i06HtIpVK4uLSGpaUVBAIBLC2tsHnzBty8eUPbIRIRERERERERlRmTclQpeXldR0FBAUaPHocpU6YrlDVp0gz29g3g6PgxGjVyxIoVv8Df/4GWItWuIUO+RPfuvWBmZq7tUIiIiIiIiIioFJiUo0opLi4WAFC3bj2lstGjx2o4msrL3Nwc5ubm2g6DiIiIiIiIiEqJSTmqVGRrpMm4uy+Du/syAECLFi7YuHFrhccQExMNN7cBsLGphWPHzqqsM3Rof8TGxuDo0TOoVctW5fGYmGjs3euBx4+DkZeXB3t7e7i5fYU+ffqpPGd6eho8PLbDy8sTyclJsLCojo4dO2PixKlqY1W3ptyFC2fh7r4Mffr0w7ffzoGHx3bcuuWFhIR4ODk1K9f7mJiYiH37duHOHR/Ex8dCIBDAzMwcdevWQ9u2HTBixGgVbRJw8OBe3Lnjg9jYGAiFItjZ2aNPn74YOHAIdHT+99E0Zco4BAUFYunS5ejevZfKGI4fP4y1a39Hp05dsWLFaoWyq1cv4+zZUwgNfYLXr3NQvbolWrX6BKNGjUW9enbldh+IiIiIiIiISoNJOapUGjVyRJ8+/fDwoT+ioiLRrJkz6tSpCwCoV89eu8GVwvnzZ7B79w44OHyMtm3bIzY2BkFBgVi+fCkyMtIxbNgIhfrJyUmYNm0SIiNfwtS0Gjp06AiJRIorVy7h7t3bqF+/QZniSEtLxYQJY5CZmQFn5xZwdGwMXV3d8rhEAEBSUiImThyNxMQEWFvboG3b9tDT00NiYiLCwkLx5MljpaScv/8D/PjjPGRkpKNWLVt88klb5OXl4/HjIKxd+ztu3bqJVav+kCfmXF37IygoEBcunFOblDt/vjB52rfvAPkxqVSK5cuX4tKl8xCJRGjRwgXm5hYIDQ3BhQtnce3a3/jPf1ahXbsO5XY/iIiIiIiIiEqKSTmqVDp37orOnbti+fKliIqKRP/+gxR2G31f7Nu3CytXrsGnn3aSH5ONXtu5cysGDhwMfX0DedmaNasQGfkSzs4t8dtva2FiYgKgcPTcvHmz4O3tVaY4fHy80apVG7i7r4Kxscm7XZQKZ86cRGJiAgYM+ALz5y+EQCCQlxUUFCit9ZeUlIiffpqPzMwMzJ27AAMHDoZQKARQmED8+ecfce/eHezd64Fx4yYBALp374kNG9bg/v27SEiIR40aNRXO+fRpGEJDQ1C9uqVCgu306eO4dOk8zM3NsXbtn2jUyBFAYbJu586t8PDYhqVLf8LBgydgYWFR7veGiIiIiIiIqChCbQdAVBUNGfKlQkIOKBzxZWdnj8zMTISE/G9n3bi4WHh5eUIgEGDevB/lCTkAqFbNDPPn/1jmOHR0dPD99wsrJCEHAMnJyQCAtm07KCTkZH23bt1G4diRIweRlpaGwYPd8MUXQ+UJOQAwMzPHokXLoKOjg+PHj0AqlQIAjI1N0KXLZ5BIJLh06bxSDBcunAEA9OrlqjDt9eDBfQCAsWMnyhNyACAQCDB+/GQ0bNgImZkZOHv25LvcAiIiIiIiIqIyYVKOqAK8nZCTsbOrDwBISIiXHwsI8INEIoGDw8cqp6k2auSIhg0blSmORo0cUbt2nTK1LYkmTZwAAJs3b8CNG9eQnZ1dZP3bt70BAJ991lNleY0aNVGnTj2kpqbg1auX8uOyaakXL55TqF9QUIArVy4p1AGA+Pg4REVFAoDKNfwEAgH69i0cgfngwf0iYyYiIiIiIiKqCJy+SlQBrK1tVB43NjYGAOTl5cmPxcfHAYDChhFvs7W1xbNnYaWOo6hzlodevVzx7793ceXKRfz00/cQiUSwt6+PZs1aoFu3z9Gq1ScK9aOjowAA06dPLPbcqakp8o0YWrZsBVvb2nj5MgKBgQFo1swZAHDr1k2kpqagSZOmsLevL2+bkJAAADAzM1M7StDWtjBZmZiYUMqrJiIiIiIiInp3TMoRlYFEIimy/M1pmdqkr69foecXCoX4+edfMXr0ONy+7Y3AwAAEBgbg1KljOHXqGD79tBPc3VdDJBIBACSSwimpXbt+DkNDwyLPbWZmLv+7QCCAq2t/bN++GRcunJMn5WRTV9/HdQeJiIiIiIjow8akHNFbZLuTqpuKWVBQgKSkxHLrT7ZxQWxsjNo6MTHqyyqD+vUbyKfeSqVS+Pr+i2XLFuHWrZu4dOm8fGppzZrWiIx8iVGjvsbHHzcpVR+9e/fDzp1bce3aFcyePRdZWVm4c8cH+vr6Sruy1qhRAwCQlpaGrKxMlaPlZKP2rKxqlPp6iYiIiIiIiN5V5RjOQ1SJmJtbQFdXF+npaUhJSVYqv3v3NsRicbn15+zsAoFAgNDQEEREhCuVh4WFlmnqqrYIBAK0bt0GPXoUJsrCwkLlZbLdUa9d+7vU57WxsUGrVp8gKysLN2544vLlixCLxejS5TOFzTGAwuSfbC29CxfOKZ1LKpXi4sWzAAAXl9aljoWIiIiIiIjoXTEpR/QWHR0dODu3BABs3bpJYapqWFgo1q5dVa792djYoHPnrpBIJFi9egWysjLlZenp6VizZqV8J9LK5uLFcwo7ycpkZ2fBz88XQOH1yYwYMRomJqY4fPgADh7ch/z8fKW20dFRuHz5gsr+ZCPuzp8/W+zU1a++GgUA2L17u0JiUCqVYvfuHQgLC4WJiSn69/+iJJdKRERERBoiFAqgoyPUyh+hUKDtyyeiDwinr1ZSulbKu3BWRtqI08fHG7t2bZf/Ozz8BQBg586tOH78iPz41q27ytzHpEnTEBDgh7NnT8Lf3xcNGzZCYmICQkKC0aNHb/j5+RY53bS0vvvuBzx9GgY/P1+4uQ1Ey5YukEoLdwY1MzNDx46d4e3tVW79lRcvL08sX74UVlY10KiRA0xNqyEjIx2BgQHIzMxEgwYNMWDA/5JeNWtaY+XK/2LRou/x559/4MCBPWjQoCEsLa2QmZmJiIgXiIqKRJMmTdGrl6tSf506dYWpaTX4+t4DULiRxdubScgMHDgEgYEPcfnyBUycOBotWrSChYUFQkND8PJlBPT19bFkyX9gYWFRMTeHiIiIiEpNKBTA3MIQIqFIK/2LJWKkpuTI10ImIqpITMpVMhKJFBKxGFaDV2o7lBKTiMUa/aGVmpqC4OBHSsejoiIRFRVZLn04OTXFhg1bsXPnFgQFBSI+Pg5169ph1qx5GDRoCNzcBpRLPzKWllbYunUXPDy2wcvrOnx8vGFhUR3du/fExInf4M8//yjX/srL8OGjUKuWLQIDHyI0NATp6emoVq0a7O0boEePXnB1HaC0oUOLFi7Yu/cIjh8/Ah8fbzx+HIz8/DxYWFSHtbU1evbsg65dP1fZn2z9uJMnjwIAevfuC4FA9W8zBQIBFi/+Be3adcCZMyfx5MljvH6dg+rVLeHq2h+jRn2NevXsy/V+EBEREdG7EQoFEAlFWHZ5GcJTwjXat72FPZb0WgKhUMCkHBFphEBaWefFvWcSEzNQ3J3Mz89DUlIMLC1rQVdXT209oVDwXg2blkik/KFFVMmU9POGqDLR0RHCwsIY4w6NQ2hCaPENylEPhx5Y2mspfCdORGaoZtfxNHFohFbbtyMlJQsFBUXv7k1EVNVp82eBQw0HeAz34OcxEb0TgQCwsjItUV2OlKuEmOQiIiIiIiIiIqrauNEDERERERERERGRhnGkHFVpERHh2LdvV4nrjxo1FnZ29hUWT2UQEOCPc+dOlbj+9OmzYW5uXmHxEBEREREREX2ImJSjKi0pKREXL54rcf0+ffpV+aRcVNSrUt2T8eMnMylHREREREREVM6YlKMqzcWlNby972s7jErF1bU/XF37azsMIiIiIiIiog8a15QjIiIiIiIiIiLSMCbliIiIiIiIiIiINIxJOSIiIiIiIiIiIg1jUo6IiIiIiIiIiEjDmJQjIiIiIiIiIiLSMCbliIiIiIiIiIiINIxJOSIiIiIiIiIiIg1jUo6IiIiIiIiIiEjDdLQdACkTCgUQCgXaDqPEJBIpJBKptsMgIiIiIiIiInpvMClXyQiFAphbGEIkFGk7lBITS8RITclhYo6IiIiIiIiIqISYlKtkhEIBREIRll1ehvCUcG2HUyx7C3ss6bUEQqGg3JNy3t5eOHBgD54+DUN2dhYAYP36zTA2NoGv77948uQxnjwJQVTUK0ilUixe/At69XIt1xgqi+XLl+LixXNYuHAJXF37y4/v2LEFHh7bMG7cJEyYMKXC+o+JiYab2wDY2NTCsWNnS9V26ND+iI2NwdGjZ1Crlm2J26m7ZiIiIiIiIqKqgEm5Sio8JRyhCaHaDkNrwsKeYNGi7yGVSuHi0hqWllYQCASwtLTC5s0bcPPmDW2HSERERERERERUZkzKUaXk5XUdBQUFGD16HKZMma5Q1qRJM9jbN4Cj48do1MgRK1b8An//B1qKVLuGDPkS3bv3gpmZubZDUWvduk0oKChAjRo1tR0KERERERERUaXBpBxVSnFxsQCAunXrKZWNHj1Ww9FUXubm5jA3N9d2GEWqXbuOtkMgIiIiIiIiqnSYlKNKRbZGmoy7+zK4uy8DALRo4YKNG7dWeAwlWT9N3Tppbx6PiYnG3r0eePw4GHl5ebC3t4eb21fo06efynOmp6fBw2M7vLw8kZycBAuL6ujYsTMmTpyqNlZ1a8pduHAW7u7L0KdPP3z77Rx4eGzHrVteSEiIh5NTs3e6jwUFBTh8eD8uXjyP6OgoGBoawMXlE0ycOBV2dvYlvldlveaUlBRcvXoZd+/6ICIiHElJSdDR0UHduvXQrdvncHP7Cvr6+irbPn/+FDt2bIG//wO8fv0atWvXQb9+AzF06HAMGzawTGvfEREREREREZUFk3JUqTRq5Ig+ffrh4UN/REVFolkzZ9SpUxcAUK+evXaDK4Xz589g9+4dcHD4GG3btkdsbAyCggKxfPlSZGSkY9iwEQr1k5OTMG3aJERGvoSpaTV06NAREokUV65cwt27t1G/foMyxZGWlooJE8YgMzMDzs4t4OjYGLq6uu90bUuW/Ihbt26iRQsXNGz4ER4/DoKn51XcueODtWs3omnT5iU6T1mv+d6921i3bjVq1KiJ2rXroEmTpkhNTUVw8CNs3rwR3t5eWL9+M/T09BTa+fn5Yt68mcjNzUXt2nXQunVbpKenYdOmDQgKCnyne0JERERERERUWkzKUaXSuXNXdO7cFcuXL0VUVCT69x/0Xu68uW/fLqxcuQafftpJfkw2em3nzq0YOHAw9PUN5GVr1qxCZORLODu3xG+/rYWJiQmAwpFk8+bNgre3V5ni8PHxRqtWbeDuvgrGxibvdlEAYmNj8Pp1DrZv34uPPmoEABCLxdiwYQ2OHTuMpUt/woEDx5USYqqU9ZodHRtj82YPNG3aTOF4eno6li5diHv37uDYsUMYMWKMvCw39zV++WUxcnNzMXz4KEybNhNCoRAA8OLFc8ya9Q2Sk5PKdE+IiIiIiIiIykKo7QCIqqIhQ75USMgBgKtrf9jZ2SMzMxMhIY/lx+PiYuHl5QmBQIB5836UJ6cAoFo1M8yf/2OZ49DR0cH33y8sl4SczJgxE+QJOQAQiUSYNm0WatSoidjYGFy/fq3Yc7zLNdvb11dKyBW2q4bZs+cDADw9ryqUeXr+g4SEeNjY1MLUqTPkCTkAqF+/Ab7+ekKxMRMRERERERGVJ46UI6oAbyfkZOzs6iMiIhwJCfHyYwEBfpBIJHB0bKxyymajRo5o2LARnj0LK3UcjRo5lvtGC6rWxNPT08Nnn/XA4cP74efni549exd5jne9ZrFYDD8/Xzx69BCJiYnIy8uFVCqFVCoFALx8GaFQX7Y7b7du3aGjo/yx17NnH6xdu6rImImIiIiIiIjKE5NyRBXA2tpG5XFjY2MAQF5envxYfHwcABS5uYCtrW2ZknLlvWGBiYkpTE1NVZbZ2hb2lZAQV+x53uWaX716iYUL5+HFi+dq22ZlZb3VX2ES1Mamlsr6pqamMDExQWZmZrGxExEREREREZUHJuWIykAikRRZ/ub0SG1StwtpRfr/wWoVZtGiH/DixXN06NAJI0eOgb19fRgbm0BHRwf5+fno1q292rYCgaCIMxdVRkRERERERFS+mJQjeotsd9Ls7GyV5QUFBUhKSiy3/mrUqAmgcBMFdWJi1JdpUmZmBjIyMlSOlpPFWLNmzWLPU9ZrjogIx7NnYbCwqA5399+VpqK+evVSTX81/r+/aJXlmZmZyMzMKDZuIiIiIiIiovJSOYbzEFUi5uYW0NXVRXp6GlJSkpXK7969DbFYXG79OTu7QCAQIDQ0BBER4UrlYWGhZZq6WlEuXz6vdCw/Px/Xrv0NAGjZslWx5yjrNaenpwEArKysVK4Nd+XKRZX9tWjhAqBww4eCggKl8r//vlRszERERERERETliUk5orfo6OjA2bklAGDr1k0KU1XDwkLLfUMAGxsbdO7cFRKJBKtXr0BW1v/WNUtPT8eaNSvlGxhUBrt27cDz50/l/5ZIJNi0aT3i4+NQs6Y1unT5rNhzlPWa69a1g0gkwvPnz/DgwX2FMm9vLxw5ckBlf926dYelpRViYqKxdetfCs80IiIcu3ZtKzZmIiIiIiIiovLE6auVlL2FvbZDKBFtxOnj441du7bL/x0e/gIAsHPnVhw/fkR+fOvWXWXuY9KkaQgI8MPZsyfh7++Lhg0bITExASEhwejRozf8/HyLnHpZWt999wOePg2Dn58v3NwGomVLF0ilwIMH92FmZoaOHTvD29ur3PorK2trGzg6Nsb48aPQsmUrVKtmhpCQYERFRcLQ0BBLliwv8Tp2Zblmc3NzDB48DEePHsTs2dPQvHkLWFnVwMuXEQgNDcHXX0/A7t07lPoyMDDAzz//ivnzZ+PAgT3w8vKEo2NjZGSkw8/PFx07dkFw8CPExcXKpy8TERERERERVSQm5SoZiUQKsUSMJb2WaDuUEhNLxJBINDeSKzU1BcHBj5SOR0VFIioqslz6cHJqig0btmLnzi0ICgpEfHwc6ta1w6xZ8zBo0BC4uQ0ol35kLC2tsHXrLnh4bIOX13X4+HjDwqI6unfviYkTv8Gff/5Rrv2VlUAgwC+/rMCBA3tw+fIFBAT4wcDAEF27foYJE6aifv0GJT5XWa955szv0LDhRzh58hiePAnB06ehaNDgIyxb5o7PP++pMikHAK1afYKtW3dh586t8Pd/gJs3b8DWtjYmTZoGN7fh6NmzM4RCIUxNq5Xl1hARERERERGVikBamebFvccSEzOK3XUyPz8PSUkxsLSsBV1dPbX1hEIBhML3ZydIiUSq0aQcUXnz93+AGTMmo2HDj7B79yFth1MuSvp5Q1SZ6OgIYWFhjHGHxiE0IVSjffdw6IGlvZbCd+JEZIZqdh1PE4dGaLV9O1JSslBQUPTu3kREVZ02fxY41HCAx3APfh4T0TsRCAArK+XNEVXhSLlKiEkuovKXkpKCnJxs2NrWVjj+/PlT/PbbfwAArq79tREaERERERERfYCYlCOiD8KLF88wc+ZU2Ns3gK1tbejr6yMmJhqhoSGQSCT45JO2GDLkS22HSURERERERB8IJuWoSouICMe+fbtKXH/UqLGws7OvsHgqg4AAf5w7d6rE9adPnw1zc/MKi0dT6tWzw+DBbvD3f4DAwABkZ2fByMgYTZs2R48evdG//yDo6PAjkYiIiIiIiDSD30CpSktKSsTFi+dKXL9Pn35VPikXFfWqVPdk/PjJVSIpZ2VVA99994O2wyAiIiIiIiICwKQcVXEuLq3h7X1f22FUKq6u/bl2GhEREREREZGWCbUdABERERERERER0YeGSTkiIiIiIiIiIiINY1KOiIiIiIiIiIhIw5iUIyIiIiIiIiIi0jAm5YiIiIiIiIiIiDSMSTkiIiIiIiIiIiINY1KOiIiIiIiIiIhIw5iUIyIiIiIiIiIi0jAdbQdAyoRCAYRCgbbDKDGJRAqJRKrtMIiIiIiIiIiI3htMylUyQqEAFuaGEIpE2g6lxCRiMVJSc5iYIyIiIiIiIiIqoUqflLt48SIOHDiAkJAQ5Ofno169eujfvz/Gjh0LXV3dUp0rOzsbe/fuxeXLlxEeHo7c3FyYm5ujadOmGDZsGD7//PMKuoqSEwoFEIpEePzrr8iOiNB2OMUysrND48WLIRQKyj0p5+3thQMH9uDp0zBkZ2cBANav3wxjYxP4+v6LJ08e48mTEERFvYJUKsXixb+gVy/Xco2hsli+fCkuXjyHhQuXwNW1v/z4jh1b4OGxDePGTcKECVO0GCEBwIMH9zFz5lS0aOGCjRu3ajscIiIiIiIiqsQqdVJu+fLl2LNnD3R0dNCuXTsYGRnhzp07WL16NTw9PbFz504YGBiU6FwpKSkYNWoUnj59CiMjI7i4uMDU1BQvX77E9evXcf36dYwePRqLFi2q4KsqmeyICGSGhmk7DK0JC3uCRYu+h1QqhYtLa1haWkEgEMDS0gqbN2/AzZs3tB0iEREREREREVGZVdqk3NWrV7Fnzx4YGRlh3759cHJyAgAkJyfj66+/hq+vL9atW4cffvihROf7888/8fTpUzg5OWHnzp0wNzeXl924cQPTpk3D3r170a9fP7Ro0aICrohKw8vrOgoKCjB69DhMmTJdoaxJk2awt28AR8eP0aiRI1as+AX+/g+0FKl2DRnyJbp37wUzM3Nth0JEREREREREpVBpd1/dvHkzAGDy5MnyhBwAVK9eHUuWLAEA7Nu3DxkZGSU63927dwEAkyZNUkjIAUCXLl3Qtm1bAIC/v/87Rk7lIS4uFgBQt249pbLRo8diypTp6Nr1c9SuXUfToVUq5ubmsLOzV3pNExEREREREVHlVilHysXFxSEwMBAA0K9fP6Xy1q1bo1atWoiJicGNGzdU1nmbnp5eifpmckO7ZGukybi7L4O7+zIA0Ng6XTEx0XBzGwAbm1o4duysyjpDh/ZHbGwMjh49g1q1bFUej4mJxt69Hnj8OBh5eXmwt7eHm9tX6NNH9es1PT0NHh7b4eXlieTkJFhYVEfHjp0xceJUtbGqW1PuwoWzcHdfhj59+uHbb+fAw2M7bt3yQkJCPJycmpXpPoaEPMaBA3sQGBiAlJRk6Onpw8zMHA4ODujduy86deqqMq6+fQdg27ZNuH//LjIyMlCzpg169OiFUaO+hr6+4vTzgoIC/PPPFdy544MnTx4jMTERBQUFsLa2Rtu27TFq1FhYWdVQim3GjMnw93+A9es3QyQSYf/+PQgODkRaWhp+/PFnuLr2h0Qiwdmzp3Dp0jm8ePEcOTk5MDWtBktLK7Ro0RLDh49SeJayeC5ePIfLly/g2bOneP06B1ZWNdC2bXuMHj0O1tY2au/X69evsXv3Dly79jcSEuJhaloN7dp1wMSJU1GjRk2Fum++5o4ePYMzZ07i9OkTePkyHCKRCE2aNMWECVPQtGnzUj83IiIiIiIiqpwqZVIuODgYQGGCrG7duirrNG3aFDExMQgODi5RUq5z58549OgRtm3bhvbt2ytNX7179y5q1KhRKTZ7+JA1auSIPn364eFDf0RFRaJZM2fUqVP4GqhXz167wZXC+fNnsHv3Djg4fIy2bdsjNjYGQUGBWL58KTIy0jFs2AiF+snJSZg2bRIiI1/C1LQaOnToCIlEiitXLuHu3duoX79BmeJIS0vFhAljkJmZAWfnFnB0bFzqDVIA4P79e5g3byYKCgrw0UcOcHJqBolEgoSEeNy+fQsSiUQhKScTExONCRNGQSTSgbNzS+Tm5sLP7z48PLbh/v17+OOPv6Cvr69wH3799WeYmJjAzq4+GjZshNevcxAWFopjxw7j6tUr2Lx5p/w18TZPz39w+vRx1Ktnj1at2iAjI11+vStX/ooLF85CT08fzZs7w9zcAunp6YiOjsLx40fQqlUbhaRcdnYWfvjhO/j5+cLQ0AiOjh/D3NwCz58/xalTx+HpeRVr1/4JB4ePleIoKCjArFnf4NmzMLRs2QoODh/j4UN/nD9/Bnfu3MLGjdtUjgIFChPRf/99Cc7OLdGhQyeEhT3Bv//eRUCAHzZs2Aonp6aleXRERERERERUSVXKpFxkZCQAoFatWmrr2NjYKNQtzqRJk/Dw4UN4e3ujW7ducHFxQbVq1RAREYGgoCC4uLhg+fLlMDU1LVPMAkH51PnQde7cFZ07d8Xy5UsRFRWJ/v0HKew2+r7Yt28XVq5cg08/7SQ/Jhu9tnPnVgwcOFhhlNiaNasQGfkSzs4t8dtva2FiYgKgcPTcvHmz4O3tVaY4fHy80apVG7i7r4KxsUmZr2fPnp0oKCjAzz//ip49+yiUZWZmIjz8hcp2ly6dR6dOXbB06XL59cbHx2HWrG8QGBgAD49tmDp1hry+iYkJVq78L9q27aCQPCwoKMCOHVuwd68H1q1bjd9/X6eyv5Mnj+K7737A4MFuCsdjY2Nx4cJZ1KxpjW3bdsPS0kqhPDz8BQwMDBWO/f77Cvj5+aJDh0748cfFsLCoLi87cuQA1q9fg59/Xoj9+49CJBIptH306CHq1KmLffuOyT+rcnNz8euvi3H9+jX85z9LsGWLh1L8sbEx8PPzxZ49h1Gvnh0AQCwWY9Wq5Th//gx27NiMNWs2qrx2dQQCfvYQvU/4fiUiqhz4eUxEZVWaz49KmZTLysoCABgaGqqtY2xsrFC3OEZGRti8eTPWrFkDDw8PeHt7y8vMzc3RoUMHWFtblzlmS8vik3mvX79GcrIQIpEAOjqql/MTiSrtMn9FKu+4Bf//KhYK1d+rstQtiTevpbjziURClXXc3IajS5cuCscGDBiI/ft3IyIiHGFhIWjRwgVA4fp5Xl6eEAgEWLDgJ5ibV5O3qV7dAgsW/IQxY74CoHyNQqHqa5cd19HRwcKFi2Bm9r9zlkVKSjIAoGPHTkrXa25eDS1aOCsck/Wvr2+ABQt+grGxkbzM1rYWZs36DvPmzcbJk8cwadIU+Wi5atVM0bVrN6X+dXT0MH36t7h06Rzu3r2N3Nwc+WcA8L/XQOvWn2DYsC+V2qenpwAAHB0/hrV1TaXyjz5qqPDvFy+e4+rVy6hRowZ+/dVdoS8AGDFiFO7fvwcfH2/8++9tdOzYGYDia2fmzDmoU+d/I+90dAzx/fcLceeOD4KCAhEcHIjmzZ2V2s2d+z0aNKj/Rjshpk2bgfPnz/z/hiZi6OgUP9pRIhFAKBTCwsK4xLtUE5F2WVgYF1+JiIgqHD+PiUhTKmVSriLEx8dj2rRpePLkCWbPno2+ffvC0tIST58+xbp167Bx40ZcvXoV+/fvl49SKo2kpAxIpUXXyc/Pg0QigVgsRUGBpIxXUjmJxZJyvSbp/99MiaT4e1WauiUhFv/vHMWdT911t2/fUeVxO7v6iIgIR2xsnLzc19cXEokEjo6NUbeuvVK7Bg0aoWHDRnj2LEzpGiUS1dcuO96okSOsrW3f+b40buyEFy+e4+efF2L06PFwcmoKHR31Hx+y/tu0aQszs+pK/bdr1xFmZmZIS0tDcHAwmjVTTOqFhYXC1/ceYmKikZOTI3/GBQViSCQSREREKEwblZV36fK5ymutU6cejIyM4ePjjR07tqFHj96wta2tNn5vb29IpVK0bdsB+vqGKs/ZooULfHy8ERAQgHbtOgL432vHxMQU7dt3UmpXrZo52rZtjxs3PHH//r9o0qSZQjuRSIRPPmmv1M7MrDpMTashIyMdSUkpSiP9VBGLpZBIJEhJyYKubn6x9YkqA5FI+EF/EUpJyVL4GUQfHqFQADNzQ4iEouIrVwCxRIy01Bz5z3EibagMPwv4eUxE70IgKNnALaCSJuVko1JycnLU1pGNkHt7BIs6CxYsQGBgIObPn4+JEyfKjzdv3hybN2/G4MGDERISgp07d2LmzJmljlkqRbFJueLKqepQtwGA7PWal5cnPxYfHwcASpsMvMnW1hbPnoWVOo6izlkaU6ZMx9OnYbhzxwd37vhAX18fDg4fo2XLVujZsw/s7eurbFdU/zY2tkhLS0N8fLz8WE5ODn799Wd4eXkWGY+6EbI2NqqnvBsZGWPhwp/h7v4Ltm3bhG3bNsHS0gpOTs3Qtm179OjRG0ZG/xvNFx0dBQA4d+40zp07XWQsqakpSsdq1aolH72nXFaYDHzzumUsLa3UJjuNjY2RkZGu8NopiZJ8NhFR5cH364dNIBBAJBRh2eVlCE8J12jf9hb2WNJrCQQCgfyXXUQfMr4NiEgTKmVSrnbtwi+tMTExauvExsYq1C1KXFwcbt26BUD1bq66urro1asXQkND4ePjU6akHH1YJJKif3MmFFaOachvbqLwLiwtrbBjx174+fni/v17CAwMQHDwIwQGBmDvXg9MmTIdo0aNLePZ//c/ni1bNsLLyxN2dvaYOnUGGjd2gpmZuXx9ualTx+PRo4dqvywUdb1du36O1q3bwtv7BgIC/BEYGAAvL094eXlix44tWLv2TzRs+FFhRNLC59uokQM++sihyOibNCnrxgvK11BZXjdERKRd4SnhCE0I1XYYREREVMEqZVKuSZMmAIDU1FS8evVK5Q6sjx49AgA4OTkVe77o6Gj539VNTZVt8JCWllbqeKlqkSWAsrOzVZYXFBQgKSmx3PqrUaNwjbPYWPVJ6KIS1JoiEAjg4tIaLi6tARRuXHDx4lmsWbMKW7f+hW7duqN27ToKbWJiolWdCgAQG1tYJrt+ALh27SoAYNmyFfjoo0ZKbSIjX77TNZiYmKB3777o3bsvgML1/P7443fcvHkDa9euwsaNWwEANWsWri/ZrJkzvvvuh1L3U/QvFJSvm4iIiIiIiD48lXJYho2NDZo1K1xr6dy5c0rl9+/fR0xMDPT09JQW01flzQ0cAgICVNaRHa9Tp47KcvpwmJtbQFdXF+npafINDt509+5tiMXicuvP2dkFAoEAoaEhiIgIVyoPCwst09TViqavr49Bg4aiYcOPIJFI8PSpcoz37t1ReQ9v3/ZGWloajIyM4ejYWH48Pb0wKa5qGurdu7eRmppafheAwmnG48dPAQCEhT2RH2/XrgMAwNvbC7m5uaU+b2Zmhsodc1NSUnD37m0AQMuWrcoSMhEREREREVURlTIpBwBTp04FAGzduhVBQUHy4ykpKVi2bBkAYNSoUfIRbgDw999/o3fv3vj6668VzmVraytP8i1fvhyRkZEK5adPn8aFCxcAqJ7eSh8WHR0dODu3BABs3bpJYapqWFgo1q5dVa792djYoHPnrpBIJFi9egWysjLlZenp6VizZqXW13Y5cGCvfMr4myIiwhEZ+QqA6kRabm4uVq9egdzc1/JjiYkJ2LjxDwDAoEGDFaac2tkVrk137NghhfO8fBmO1atXlDn+0NAQ/PPPFYU4ZG7d8lKK38HhY3Tt+hni4+Pw00/zVY74y8nJwZUrF5GcnKSyz40b/5CvFwgUriO4Zs1vyMnJQePGTmjevEWZr4eIiIiIiIjef5Vy+ioAdO/eHaNHj8bevXvx5Zdfol27djAyMsLt27eRnp4OFxcXzJo1S6FNRkYGXrx4oXIhdHd3d4wZMwbPnj2Dq6srnJ2dYWFhgefPnyMsrHCEz4ABAzBgwACNXF9xjOzstB1CiWgjTh8fb+zatV3+7/DwFwCAnTu34vjxI/LjW7fuKnMfkyZNQ0CAH86ePQl/f180bNgIiYkJCAkJRo8eveHn51vkdNPS+u67H/D0aRj8/Hzh5jYQLVu6QCoFHjy4DzMzM3Ts2FnlyCtN2bNnB/76ax3s7OxhZ1cf+vr6SExMwMOH/hCLxejduy8cHT9Wate7d1/4+Hhj2LCBaN68JfLycvHgwX3k5OSgadPmmDBhikL98eMnYdGiH7B9+2Z4el6FvX0DpKamICDAD87OLWFlZYXAwIeljj82NhZLliyUb1BRs6Y1xGIxnj9/ipcvI6Crq4tvvlFcS3LhwiXIyMjEnTs+GDFiCD76qBFq1aoNqVSK2NhoPH0ahvz8fOzffwzVq1sqtG3atDkkEglGjBgCF5dPYGBggIcP/ZGYmAALi+pYvHhZqa+BiIiIiIiIqpZKm5QDgEWLFsHFxQUHDhyAn58fCgoKUK9ePUyaNAljx46Fnp5eic/l4OCAc+fOYdeuXfDy8sKjR4+Ql5eHatWqoWPHjhgyZAhcXV0r8GpKRiKRQiIWo/HixdoOpcQkYjEkEs2N5EpNTUFw8COl41FRkYiKilTRovScnJpiw4at2LlzC4KCAhEfH4e6de0wa9Y8DBo0BG5u5Zu8tbS0wtatu+DhsQ1eXtfh4+MNC4vq6N69JyZO/AZ//vlHufZXWt999wPu37+HkJBg+Ps/wOvXOahe3RKffNIWAwYMRqdOqqeR16pli+3b92Dr1r/w4MF9ZGSkw9raBj169MbIkV9DX99AoX6XLp9h48at2LlzG549C0VUVCRsbWtj/PjJ+Oqr0ZgzZ3qZ4ndyaoqpU2cgIMAP4eHhCAt7ApFIhBo1rDF4sBuGDv0S9erZK7QxMjLG2rUb8c8/V3DlykU8eRKCsLBQGBsbw9LSCj169EbHjl2U1tEDCkdb/v77Onh4bIWn5zUkJsbD1LQaXF37Y8KEKWp35yUiIiIiIqIPh0Cq7XlxVURiYkax22bn5+chKSkGlpa1oKurPqEoFAogFArKOcKKI5FINZqUo8pvx44t8PDYhnHjJimNhiPNKOnnDVFloqMjhIWFMcYdGqfxnSd7OPTA0l5L4TtxIjJDNbuOp4lDI7Tavh0pKVkoKCh6d2+q2rT5HnCo4QCP4R58HZLW8X1ARO87gQCwsjItviIq+Ui5DxWTXEREREREREREVVul3eiBiIiIiIiIiIioquJIOarSIiLCsW/frhLXHzVqLOzs7CssnsogIMAf586dKnH96dNnw9zcvMLioYolEgmgo1P87184QpeIiIiIiEizmJSjKi0pKREXL54rcf0+ffpV+aRcVNSrUt2T8eMnlzopN2HCFK4lVwYikRCCclpOUiIRQCgUwszMCAYGBsXWF0vESE3JYWKOiIiIiIhIQ5iUoyrNxaU1vL3vazuMSsXVtT9cXftrOwxSQSAAotOikSfOe+dzScUSZGalYPv5bUjNTy2yrr2FPZb0WgKhUMCkHBERERERkYYwKUdEVInkifPwuuD1u59IAuRLChCeEo74nPh3Px8RERERERGVK270QEREREREREREpGFMyhEREREREREREWkYk3JEREREREREREQaxqQcERERERERERGRhjEpR0REREREREREpGFMyhEREREREREREWkYk3JEREREREREREQaxqQcERERERERERGRhuloOwBSJhQKIBQKtB1GiUkkUkgkUm2HQURERERERET03mBSrpIRCgWwMDeCUPT+DGKUiCVISc0u98Sct7cXDhzYg6dPw5CdnQUAWL9+M4yNTeDr+y+ePHmMJ09CEBX1ClKpFIsX/4JevVzLNYbKYvnypbh48RwWLlwCV9f+8uM7dmyBh8c2jBs3CRMmTNFihEDHjq0BAN7e97UaBxEREREREdH7gEm5SkYoFEAoEuLK/gdIicvUdjjFsrA2Qc+RLhAKBeWalAsLe4JFi76HVCqFi0trWFpaQSAQwNLSCps3b8DNmzfKrS8iIiIiIiIiIk1jUq6SSonLREJUmrbD0Bovr+soKCjA6NHjMGXKdIWyJk2awd6+ARwdP0ajRo5YseIX+Ps/0FKk2jVkyJfo3r0XzMzMtR0KEREREREREZUCk3JUKcXFxQIA6tatp1Q2evRYDUdTeZmbm8Pc3FzbYRARERERERFRKTEpR5WKbI00GXf3ZXB3XwYAaNHCBRs3bq3wGGJiouHmNgA2NrVw7NhZlXWGDu2P2NgYHD16BrVq2ao8HhMTjb17PfD4cTDy8vJgb28PN7ev0KdPP5XnTE9Pg4fHdnh5eSI5OQkWFtXRsWNnTJw4VW2s6taUu3DhLNzdl6FPn3749ts58PDYjlu3vJCQEA8np2Zlvo+PHj2Eh8d2BAU9hFgsRr169vjii6Ho12+g2jbBwY9w/fo/8PPzRVxcHNLT02BqWg2NGzvBzW04Pvmkrcp2UqkU58+fwcmTxxAe/hz6+gZo3NgJY8dOQH5+PmbOnKr0mnjw4L78+Jo1G7Fv3y5cuXIJ8fGxMDe3QPfuvTBhwhTo6+sjMzMTu3Ztx40bnkhKSkD16pbo06cfvv56AnR0FD8aU1JScPXqZdy964OIiHAkJSVBR0cHdevWQ7dun8PN7Svo6+uX6Z4SERERERHRh4lJOapUGjVyRJ8+/fDwoT+ioiLRrJkz6tSpCwCoV89eu8GVwvnzZ7B79w44OHyMtm3bIzY2BkFBgVi+fCkyMtIxbNgIhfrJyUmYNm0SIiNfwtS0Gjp06AiJRIorVy7h7t3bqF+/QZniSEtLxYQJY5CZmQFn5xZwdGwMXV3dMp3r2rWrWLbsJ4jFYjRo0BANGnyE+Pg4/Pbbf/DixXO17bZs+Qt+fvdRv37hlGMDA0NERUXCx+cmfHxuYubMuRg27Culdv/97284deoYhEIhmjdvAUtLKzx//hQzZkyGm5ty/TcVFBTgu+9mICzsCVq2bIV69ezw8KEfDhzYg/DwF1i0aCmmTh2P9PR0tGjREtnZdeHv7wcPj21ISUnGvHk/Kpzv3r3bWLduNWrUqInateugSZOmSE1NRXDwI2zevBHe3l5Yv34z9PT0ynRviYiIiIiI6MPDpBxVKp07d0Xnzl2xfPlSREVFon//QQq7jb4v9u3bhZUr1+DTTzvJj8lGr+3cuRUDBw6Gvr6BvGzNmlWIjHwJZ+eW+O23tTAxMQFQOHpu3rxZ8Pb2KlMcPj7eaNWqDdzdV8HY2KTM15OUlIiVK3+FWCzGt9/OwZdfjpSX3b9/D99/P0dt2+HDR2Lx4l9gZWWlcPzRo4eYO/db/PXXOnTr9jlq1KgpL/P2voFTp47B0NAIa9ZsQLNmzvKyQ4f2YePGP4qM99Gjh2jc2AlHjpyWr7cXGxuDceNGwsfnJr79dgrq1q2HZctWwMCg8DmEhARjypRxOHPmJEaNGgcbGxv5+RwdG2PzZg80bdpMoZ/09HQsXboQ9+7dwbFjhzBixJgi4yIiIiIiIiKSEWo7AKKqaMiQLxUScgDg6tofdnb2yMzMREjIY/nxuLhYeHl5QiAQYN68H+UJOQCoVs0M8+crjtoqDR0dHXz//cJ3SsgBwLlzp5GdnQUnp2YKCTkAaN26DQYOHKy2bfv2nyol5ACgadPmGDx4GAoKCpR20z169BAAYOjQLxUScgAwfPgoNG7cpMh4BQIBfvxxscIGGDY2tdCrlysAIDo6GgsWLJYn5ADg44+boF27DpBIJPDzu69wPnv7+koJOQCoVq0aZs+eDwDw9LxaZExEREREREREb+JIOaIK8HZCTsbOrj4iIsKRkBAvPxYQ4AeJRAJHx8Yqp6k2auSIhg0b4dmzsFLH0aiRI2rXrlPqdm/z8/MFAPTs2VtleZ8+fXH06EG17dPSUuHj440XL54hIyMDBQUFAIDIyJcAgJcvI+R1CwoKEBj4EADQo4fq/nr06I3Hj4PV9mdtbYMGDT5SOl63buFUaEfHj2FhUV2pvE6dwo1FEhMTlcrEYjH8/Hzx6NFDJCYmIi8vF1KpFFKpVOkaiIiIiIiIiIrDpBxRBbC2tlF53NjYGACQl5cnPxYfHwcAChtGvM3W1rZMSbmizlkasiRirVq11fSj+jgAnDlzEhs2rEFOTo7aOtnZWfK/p6WlIi8v9//Pqzp+G5uir0vd/Tc0NCqy3MiosFzWv8yrVy+xcOG8ItfOy8rKUltGRERERERE9DYm5YjKQCKRFFkuFFaOmeHa3hE0JOQxfv/dHUKhEN988y0+/bQzrK1tYGBgAIFAgNOnT+D3393lo81KSiAorrzoCqV9PosW/YAXL56jQ4dOGDlyDOzt68PY2AQ6OjrIz89Ht27tS3U+IiIiIiIiIiblqFIQiYQKiRZZUkUoFEBHp+gESmnqqiKVAmLx/5Jsst1Js7OzVdYvKChAUpLy9Maykm1wEBsbo7ZOTIz6Mk2wsqqJiIhwxMREqyyPjVV93NPzKqRSKYYO/RIjR36tVB4Z+UrpmJmZOfT09JCXl4fY2BiVU3o1eT8iIsLx7FkYLCyqw939d+joKH5svnr1UmOxEBERERERUdXBpBxVCgIBEJ0WjTxx4bTOzLxMAEBiViLCk8OLbPu64HWJ675NT6QHWzPFqZDm5hbQ1dVFenoaUlKSldYeu3v3NsRican6KYqzswsEAgFCQ0MQEREOOzt7hfKwsNAyTV0tTy1busDX9x7+/vsShgwZplR+6dJ5le3S09MBANbWtZTKcnNzcf36NaXjOjo6cHJqBj8/X/z99yVMnjxNqc7Vq5dLewlllp6eBgCwsrJSSsgBwJUrFzUWCxEREREREVUdlWOOHRGAPHEeXhe8xuuC1xBLCpNe+ZJ8+TF1fyTSwlFu+eLi6779R5YEfJOOjg6cnVsCALZu3aQwVTUsLBRr164q1+u2sbFB585dIZFIsHr1CmRlZcrL0tPTsWbNylJP7yxv/foNhKGhER49eijfGVXmwYP7OHXquMp29vb2AIBLl84prBuXm5uL//53JWJiolS2Gzp0OADg2LHDePQoUKHsyJGDCA5+VNZLKbW6de0gEonw/PkzPHiguCurt7cXjhw5oLFYiIiIiIiIqOrgSLlKysLaRNshlIg24vT/1w9nDp+U/zv6VWFi59TB47h6/or8+M+rfylzH5MmTUNAgB/Onj0Jf39fNGzYCImJCQgJCUaPHr3h5+db5HTT0vruux/w9GkY/Px84eY2EC1bukAqLUx4mZmZoWPHzvD29iq3/krLyqoGfvjhJ/z6689Yt241zp07hfr1GyIxMQEBAX4YNuwrHD6snJxydR2Ao0cPITT0CdzcBqB585YQiYQICPBHbm4u3Ny+Urlra5cu3TBgwBc4c+Ykpk+fiObNW8DS0grPnz9FREQ4vvxyBA4fPiCfalyRzM3NMXjwMBw9ehCzZ09D8+YtYGVVAy9fRiA0NARffz0Bu3fvqPA4iIiIiIiIqGphUq6SkUikkIgl6DnSRduhlJhELIFEormRXBnp6Xge+lTpeHxsHOJj48qlDyenptiwYSt27tyCoKBAxMfHoW5dO8yaNQ+DBg2Bm9uAculHxtLSClu37oKHxzZ4eV2Hj483LCyqo3v3npg48Rv8+ecf5dpfWXTv3gs1alhj9+4dCAp6iKioSNSrZ4d5837EwIGDVSblTE1NsX37XuzYsQX37t3G3bs+qFbNDG3atMW4cZPx8KG/2v7mz1+Ixo2dcPLkMQQFPYKenh6aNHHC3LkL5GvbmZmZV9DVKpo58zs0bPgRTp48hidPQvD0aSgaNPgIy5a54/PPezIpR0RERERERKUmkGp7XlwVkZiYgeLuZH5+HpKSYmBpWQu6unpq6wmFAgiFxWwvWYlIJNJ3Tsrp6AgRnhwuXx9OUwx0DGBf3R4FBUXvpkqVi7v7Mly4cBYzZszG8OGjtB1OuSnX94EEyE5Lw8ZHGxCfE19kVYcaDvAY7oGUlCy+F0irdHSEsLAwxrhD4xCaEKrRvns49MDSXkvhO3EiMkM1u46niUMjtNq+ne9B0up7gD8LqLLg+4CI3ncCAWBlZVqiuhwpVwmVR5KL6H33/Pkz1KplC0NDQ/kxiUSCc+dO4+LFc9DT00f37r21GCERERERERFR2TEpR0SV0sGDe3Ht2t9wcHCElVVNvH6dg/DwF4iJiYZIJMLcuT/AyspK22ESERERERERlQmTclSlRUdG4fyxs2rLRUIRTPRM5Lubjho1FnZ29hqKTjsCAvxx7typEtefPn02zM3NKywedT77rAeysrLw5MljhIWFQiwWw8KiOj7/vAfc3EagadNmGo+JiIiIiIiIqLwwKUdVWlpKGm5dK/mupX369KvySbmoqFe4ePFcieuPHz9ZK0m59u0/Rfv2n2q8XyIiIiIiIiJNYFKOqrTGzZpg1xnlXUFlPsSNHlxd+8PVtb+2wyAiIiIiIiL6oAm1HQAREREREREREdGHhkk5IiIiIiIiIiIiDeP0VSIiIiIiIqL/JxJpZ+yKRCKFRCLVSt9EpB1MymkFP2iJqGIJ/v9jRrazMBEREREVrbpRdUjFYlSrZqiV/iViMVJSc5iYI/qAMCmnQUJh4W9cxGIxdHW1HAwRVW1SoEBSgKyCLG1HQkRERPReMNU3hUAkwuNff0V2RIRG+zays0PjxYshFAqYlCP6gDApp0EikQ50dPSQnZ0JfX1DCAQCbYdERFWRFJDmFeBJSgiyC7K1HQ0RERHReyU7IgKZoWHaDoM+cEKhAEKhdnIGnEqtOUzKaZixcTWkpSUiJSUBRkbGEIl0ADA5J5EIIBVLAIlm+5WKJcjLy4VYzA8c0r53fR8IpIBAAohz8xCbGQuvWK/yDZCI6AOhrS9C2lrHioiIKhehUABzC0OIhCKt9C+WiJGawqnUmsCknIYZGhoDALKy0pGamqjlaCoPoVCIzKwU5EsKNNpvvlAHifkiSCQazgYSqfCu7wOJVILMvAyEpobCK9YLqXmp5RsgEdEHQCgUwMLcEEKRdr4IERERCYUCiIQiLLu8DOEp4Rrt297CHkt6LeFUag1hUk4LDA2NYWhoDLG4gMkgACKRAGZmRth+fptWPnBW9F2BtLRsjpYjrXrn94EUyJPkIS0vDVJuJkNEVGZCoQBCkQiJJxYgP/G5Rvs2+KgjLD6bqdE+iYio8gpPCUdoQqi2w6AKxKScFolEOuAvYQEdHSEMDAyQmp+K+Jx4jfZtbmIOAwMD5OSIUVDABClpjzbfB0REpCw/8TnyYx9rtE8dy/oa7Y+IiIi0iwtXEBERERERERERaRiTckRERERERERERBrGpBwREREREREREZGGMSlHRERERERERESkYUzKERERERERERERaRiTckRERERERERERBrGpBwREREREREREZGGMSlHRERERERERESkYUzKERERERERERERaRiTckRERERERERERBrGpBwREREREREREZGGMSlHRERERERERESkYUzKERERERERERERaRiTckRERERERERERBrGpBwREREREREREZGGMSlHRERERERERESkYUzKERERERERERERaRiTckRERERERERERBrGpBwREREREREREZGGMSlHRERERERERESkYUzKERERERERERERaRiTckRERERERERERBrGpBwREREREREREZGGMSlHRERERERERESkYTrlcZKIiAgkJyfD3Nwc9evXL49TEhERERERERERVVllHiknFovx119/4dNPP0Xv3r0xYsQIbN26VV5+5swZDB8+HGFhYeUSKBERERERERERUVVRpqScWCzGlClTsGHDBqSlpaFhw4aQSqUKdVxcXODv748rV66US6BERERERERERERVRZmScocOHYK3tzfatm2Lf/75B+fOnVOqU6dOHdSrVw+3bt165yCJiIiIiIiIiIiqkjIl5U6ePAkzMzOsW7cO1tbWaus1aNAAMTExZQ6OiIiIiIiIiIioKipTUu758+do3rw5zMzMiqxnamqKpKSkMgVGRERERERERERUVZUpKSeRSKCnp1dsvYSEhBLVIyIiIiIiIiIi+pCUKSlna2uLJ0+eFFknPz8fYWFhsLOzK1NgREREREREREREVVWZknKdOnVCVFQUDh8+rLbOvn37kJycjK5du5Y1NiIiIiIiIiIioipJpyyNJkyYgJMnT2LZsmV4+vQp+vTpAwDIyclBUFAQLl68iF27dsHCwgIjR44s14CJiIiIiIiIiIjed2VKytWsWRN//vknZsyYgb1792Lfvn0QCAS4fPkyLl++DKlUimrVqmH9+vWoXr16ecdMRERERERERET0XitTUg4APvnkE5w/fx67du3CjRs3EBkZCYlEAhsbG3Tu3BkTJ06EtbV1ecZKRERERERERERUJZQpKRcdHQ2BQIBatWph3rx5mDdvXnnHRUREREREREREVGWVaaOHzz77DHPmzCnvWIiIiIiIiIiIiD4IZUrKmZiYoE6dOuUdCxERERERERER0Qfh/9i77/Aoqsbt4/emEBJa6ISqlA2hE5AmEGmC/LCAKCggXVABeVSqCPIgYEEQgUesIKAiIkWlhiK9J7SEIiBICaEkhJIAKfP+wbsrSzZhsySbkHw/18V1hZkzc84m5+zM3jtzxqlQrmLFijp//nx6twUAAAAAAADIEZwK5V544QWFhIRo//796d0eAAAAAAAAINtzKpR7/vnn9fLLL6t3796aOXOmTpw4odu3b6d32wAAAAAAAIBsyamnrwYEBFh/njp1qqZOnZpiWZPJpPDwcGeqAQAAAAAAALIlp0I5wzAypCwAAAAAAACQEzgVyh0+fDi92wEAAAAAAADkGE7NKQcAAAAAAADAeYRyAAAAAAAAgIs5dfuqRUJCglatWqUdO3YoMjJSklS8eHHVr19frVu3lofHA+0eAAAAAAAAyJacTs0OHTqkQYMG6cyZM8ke5vDLL79Yn8p695NaAQAAAAAAADgZykVGRqpXr16Kjo5WkSJF1LZtW5UtW1aSdPr0aS1btkz//POPevfurSVLlqhYsWLp2mgAAAAAAADgYeZUKPf1118rOjpaL7zwgt59913lzp3bZv1bb72lDz74QL/88ou++eYbjRw5Ml0aCwAAAAAAAGQHTj3oYdOmTSpZsqTef//9ZIGcJHl5eWnMmDEqWbKkNmzY8MCNBAAAAAAAALITp0K5iIgI1a5dW+7u7imW8fDwUK1atRQREeF04wAAAAAAAIDsyKlQLleuXLp+/fp9y924cUO5cuVypgoAAAAAAAAg23IqlKtYsaJ27NiR6lVw586d044dO1SxYkWnGwcAAAAAAABkR06Fcs8++6xu3rypHj162J0zbv369erZs6du3bql55577kHbCAAAAAAAAGQrTj199cUXX9Tq1au1bds29e/fXwUKFFDp0qUlSWfOnFFMTIwMw1CjRo304osvpmuDAQAAAAAAgIedU6Gcu7u7vvzyS33++ef68ccfdeXKFV25csW63sfHR126dNHAgQPl5ubUxXgAAAAAAABAtuVUKCfdedjDO++8o0GDBunAgQOKjIyUJBUvXlzVq1fnAQ8AAAAAAABACpwO5Sxy5cqlOnXqpEdbAAAAAAAAgByBe0sBAAAAAAAAF3MqlJs3b54CAgK0bt26FMusW7dOAQEBmj9/vtONAwAAAAAAALIjp0K5tWvXqlChQnriiSdSLBMUFKSCBQsqODjY2bYBAAAAAAAA2ZJTodyJEydUqVKlVJ+s6u7uLrPZrBMnTjjdOAAAAAAAACA7ciqUi4qKUpEiRe5brkiRIrp8+bIzVQAAAAAAAADZllOhXJ48eXThwoX7lrtw4YK8vb2dqQIAAAAAAADItjyc2ahy5cras2ePIiIi5OfnZ7dMRESEQkNDVatWrQdpHwAAcDE3N5Pc3Ewur9fdnYfCAwAAIOdwKpRr166dtm/frgEDBmjmzJkqWrSozfqLFy9q4MCBSkhIULt27dKloQAAIOO5uZlU0Ndbbu7umd0UAAAAIFtzKpRr3769Fi1apJCQELVq1UpBQUEqX768pDsPgdi4caPi4uJUq1YtPf/88+naYCAjZNbVGUlJhpKSjEypGwDscXMzyc3dXZcWDVf8Jdc+rCl3xcYq2HyQS+sEAAAAMotToZy7u7u++uorjRgxQsHBwVq1apVMpju3uRjGnYChRYsWmjhxojw8nKoCcIlCPoVkJCYqf/7MmfswKTFR0VfiCOYAZDnxl04o/vwhl9bpUfhRl9YHAAAAZCanE7O8efNq2rRpOnz4sDZt2qRz585Jkvz8/NS0aVNVrlw5XRq4YsUK/fjjjzp8+LDi4+NVtmxZPf300+rRo4c8PT2d2ueaNWu0cOFCHThwQDExMcqXL5/KlSunxo0ba8CAAenSbjwc8nnlk8ndXYfGjVPsqVMurdunXDkFvPee3NxMhHIAAAAAAOQwD3wZW+XKldMtgLvX+PHjNWfOHHl4eKhBgwby8fHR9u3bNWnSJK1fv17fffedcufO7fD+bt++rSFDhmjlypXKnTu3atWqpSJFiujixYs6duyY5s6dSyiXQ8WeOqXrR//K7GYAAAAAAIAcIsveW7pmzRrNmTNHPj4+mjdvnqpWrSpJioqKUvfu3bVnzx5NnTpVw4YNc3if7733nlauXKmWLVtq3LhxKlSokHVdUlKS9u/fn+6vAwAAAAAAALhXusxun5CQoO+++04vv/yynnrqKfXs2VMLFy58oH3OnDlTkvTqq69aAzlJKlSokMaMGSNJmjdvnq5du+bQ/rZt26YlS5bIbDbrs88+swnkJMnNzU21atV6oDYDAAAAAAAAjnAolFu9erUaNmyoKVOmJFuXlJSkfv366ZNPPlFISIj+/vtvbdu2Te+9956GDx/uVKMiIyN14MABSVK7du2Sra9bt678/Px0+/ZtbdiwwaF9zp07V5L0yiuvOD0XHQAAAAAAAJAeHLp9dceOHbpy5Ypat26dbN2CBQu0ZcsWSVLz5s31+OOPKyIiQj/88IOWLl2qdu3aqXHjxmlqVHh4uCTJ19dXZcqUsVumWrVqioiIUHh4uN3g7m6JiYnatm2bJOmxxx7TxYsXtWzZMv3999/KlSuXqlSpoieffFJ58uRJUzsBAAAAAAAAZzgUyu3bt09FixZVlSpVkq37+eefZTKZ1LZtW3366afW5TVq1NCgQYO0dOnSNIdyZ86ckXTnSa4pKVGihE3Z1Jw+fVqxsbGSpL1792rs2LHW/1t8/PHHmjx5sho2bJimtlqYTE5tBkii/2Qlbm4mmTLhD+Luni6zCTww+iKQuRiDyAroh0DmYgwiq6AvOictvzeHQrmLFy8qICAg2fKoqCgdOnRIJpNJffr0sVn35JNPqlSpUk49POHGjRuSJG9v7xTLWK5qs5RNzZUrV6w/jxo1SrVr19bQoUNVvnx5nT59WpMnT9aGDRv0+uuva/HixXrkkUfS3ObChfOleRtAkgoW5ArNrMRISpTJzT2zm5Ep6ItA5mIMIiugHwKZizGIrIK+6BoOhXLR0dHKnz9/suWWed8KFSpkN7SrWLGidu7c+YBNfHCGYVh/LlasmL799lvlypVLklS5cmV98cUXeu6553T06FF99dVXmjBhQprruHz5mu6qBmng7u6Wowd8dPQNJSYmZXYzoH/74qVFwxV/6YRL685dsbEKNh/k0jrvRV+ExHtyZmIMZg05fQzQD5HZGIOMQWSNcUBfdJ7J5PiFWw6Fcu7u7oqKikq23DL3m73bWiUpX758SkhIcKghd7NcBRcXF5diGcsVco7MA3d3mQ4dOlgDOQt3d3d16tRJ48aNs849l1aGIUI5OI2+k7XEXzqh+POHXFqnR+FHXVpfSuiLQOZiDCIroB8CmYsxiKyCvpjxHJrEqGTJkgoPD9ft27dtlm/btk0mk0k1a9a0u110dLSKFCmS5kaVKlVKkhQREZFimfPnz9uUvd/+LHNElS5d2m4ZywMlLl68mKa2AgAAAAAAAGnlUChXv359XblyRVOnTrUu2759u3bt2iVJCgoKsrvdoUOHVKxYsTQ3ynLl3ZUrV3T69Gm7ZQ4ePChJqlq16n33lydPHj366KPWfdoTHR0tSfLx8UlrcwEAAAAAAIA0cSiU6969uzw9PfXdd98pKChI7du3tz7YoWbNmqpevXqybUJDQxUVFaUaNWqkuVElSpSw7vOPP/5Itn737t2KiIhQrly5UgwE79WmTRtJ0tatW+2u37JliyTZfS0AAAAAAABAenIolCtXrpwmTZokb29vRUZG6tChQ0pISFCxYsX04Ycf2t3m559/liQ1bNjQqYb1799fkvTVV18pLCzMujw6Olpjx46VJHXt2lX58v07eV5wcLDatGmj7t27J9tft27dVKBAAW3YsEHz58+3Wbds2TL9/vvvkqRXXnnFqfYCAAAAAAAAjnLoQQ+S9OSTT6pOnTpav369Ll++LD8/P7Vs2TLF2z2rV6+ugIAANWjQwKmGtWzZUt26ddPcuXPVqVMnNWjQQD4+Ptq2bZuuXr2qwMBAvfnmmzbbXLt2TX///Xeyue+kO0+InTJlil577TWNGTNG8+bNU/ny5XX69GnrAytef/11h6+8AwAAAAAAAJzlcCgnSYULF1bHjh0dKtulSxenGnS3UaNGKTAwUD/++KNCQ0OVkJCgsmXLqm/fvurRo0eyp6jez+OPP66lS5fqyy+/1NatW7Vu3TrlyZNHQUFBeuWVV9S4ceMHbjMAAAAAAABwP2kK5TJD27Zt1bZtW4fKdujQQR06dEi1zKOPPpriLbcAAAAAAACAKzg0pxwAAAAAAACA9EMoBwAAAAAAALgYoRwAAAAAAADgYoRyAAAAAAAAgIsRygEAAAAAAAAuRigHAAAAAAAAuBihHAAAAAAAAOBiHpndAABA1uDunjnf0yQlGUpKMjKl7qzMzc0kNzeTy+vNrH4AAAAA5DQOhXItWrRwugKTyaQ1a9Y4vT0AIGMV8ikkIzFR+fN7Z0r9SYmJir4SRzB3Fzc3k3x9fQjIAAAAgGzMoVDu7NmzdpebTCYZhv0PUZZ1JpPrv+UHADgun1c+mdzddWjcOMWeOuXSun3KlVPAe+/Jzc1EKHcXNzeT3N3dNOrHTfr7QoxL627kX1JvPBXo0joBAACyIu5cQEZzKJRbu3ZtsmXz5s3T7Nmz1aJFC7Vv316lS5eWJJ05c0ZLlizR2rVr1bNnT3Xt2jV9WwwAyBCxp07p+tG/MrsZuMvfF2J0+GyUS+t8pGh+l9YHAACQFbm5mVTQ11tu7u6Z3RRkYw6FcqVKlbL5/9q1azV79mx98sknateunc26ypUrq2XLllq2bJneeecd1alTJ9n2AAAAAAAAWZWbm0lu7u66tGi44i+dcGnduSs2VsHmg1xaJzKHUw96+O6771StWrVkgdzd/u///k+zZ8/Wd999p5YtWzrdQAAAAAAAgMwQf+mE4s8fcmmdHoUfdWl9yDxO3ah85MgRlStX7r7lypUrpyNHjjhTBQAAAAAAAJBtOXWlXFJSkk6fPn3fcqdPn07xQRDIepjEEgAAAAAAwDWcCuWqVKmiPXv2KDg4WK1atbJbZs2aNdq3b5/q1q37QA2EazCJJQAAAAAAgOs4Fcr17t1bu3fv1uDBg9WmTRs988wzNk9f/f3337Vy5UqZTCb17t07XRuMjMEklgAAAAAAAK7jVCjXrFkzDR8+XJ988omWL1+u5cuX26w3DEPu7u5655131KxZs3RpKFyDSSwBAAAAAAAynlOhnCT16NFDDRs21Ny5c7Vr1y6dP39eklS8eHE99thj6tatmypXrpxuDQUAAAAAAACyC6dDOUny9/fXBx98kF5tAQAAAAAAAHIEHnsJAAAAAAAAuNgDXSlnGIY2btyokJAQRUdHq0aNGurYsaMkKSoqSjExMSpbtqzceaInAAAAAAAAYOV0KHf48GENHjxYp06dkmEYMplMio+Pt4ZyW7Zs0dChQzVjxgw1b9483RoMAAAAAAAAPOycun31/Pnz6tGjh06ePKmmTZtqyJAhMgzDpkzLli3l4eGhtWvXpktDAQAAAAAAgOzCqVBu5syZunLlikaOHKkvv/xSvXv3TlbG29tblStX1oEDBx64kQAAAAAAAEB24lQot2nTJpUvX16vvPJKquVKlSqlixcvOtUwAAAAAAAAILtyKpS7cOGCzGbzfcuZTCZdv37dmSoAAAAAAACAbMupUM7Hx0dRUVH3LXfmzBkVKFDAmSoAAAAAAACAbMupp6+azWaFhYUpKipKhQoVslvm7NmzOnz4sB5//PEHaiAAAAAAwLXc3ExyczO5vF53d6euGwGAh5JTodwzzzyjXbt2adSoUfr000/l7e1ts/727dsaO3asEhIS9Mwzz6RLQwEAAAAAGc/NzaSCvt5yc3fP7KYAQLbmVCjXoUMH/fbbb1q3bp2eeuopNWnSRJJ05MgRffDBB1q3bp3OnTunRo0aqW3btunaYAAAAABAxnFzM8nN3V2XFg1X/KUTLq07d8XGKth8kEvrBIDM4lQo5+7urpkzZ2r06NFavny5fvnlF0lSeHi4wsPDJUlPPvmkJk6cmH4tBQAAAAC4TPylE4o/f8ildXoUftSl9QFAZnIqlJOkPHny6NNPP9Xrr7+ujRs36vTp00pKSpKfn5+aNm2qgICA9GwnAAAAAAAAkG04HcpZVKhQQRUqVEiPtgAAAAAAAAA5glOPthkxYoQWLlx433KLFi3SiBEjnKkCAAAAAAAAyLacCuUWL16sPXv23LdcSEiIlixZ4kwVAAAAAAAAQLblVCjnqMTERLm5ZWgVAAAAAAAAwEMnQxOzU6dOKW/evBlZBQAAAAAAAPDQcfhBD9OnT7f5/+HDh5Mts0hMTNSxY8cUGhqqRo0aPVgLAQAAAAAAgGwmTaGcyWSSYRiSpEOHDunQoUOpbuPt7a3XX3/9wVoIAAAAAAAAZDMOh3JvvPGGNZSbMWOGAgIC1KJFC7tlPT09Vbx4cTVp0kSFCxdOt8YCAAAAAAAA2YHDodzAgQOtP8+YMUOVK1fWgAEDMqRRAAAAAAAAQHbmcCh3t8OHD6d3OwAAAAAAAIAcI0OfvgoAAAAAAAAgOadCuXnz5ikgIEDr1q1Lscy6desUEBCg+fPnO904AAAAAAAAIDtyKpRbu3atChUqpCeeeCLFMkFBQSpYsKCCg4OdbRsAAAAAAACQLTkVyp04cUKVKlWSm1vKm7u7u8tsNuvEiRNONw4AAAAAAADIjpwK5aKiolSkSJH7litSpIguX77sTBUAAAAAAABAtuVUKJcnTx5duHDhvuUuXLggb29vZ6oAAAAAAAAAsi2nQrnKlSsrNDRUERERKZaJiIhQaGiozGaz040DAAAAAAAAsiOnQrl27dopPj5eAwYM0MWLF5Otv3jxogYOHKiEhAS1a9fugRsJAAAAAAAAZCcezmzUvn17LVq0SCEhIWrVqpWCgoJUvnx5SXceArFx40bFxcWpVq1aev7559O1wQAAAAAAAMDDzqlQzt3dXV999ZVGjBih4OBgrVq1SiaTSZJkGIYkqUWLFpo4caI8PJyqAgAAAAAAAMi2nE7M8ubNq2nTpunw4cPatGmTzp07J0ny8/NT06ZNVbly5XRrJAAAAAAAAJCdPPBlbJUrVyaAAwAAAAAAANLAqQc9AAAAAAAAAHDeA4Vyu3fv1ptvvqmmTZuqWrVqGjlypHXdli1bNHnyZLtPZwUAAAAAAAByMqdvX/3f//6nadOmWR/sIMnm53z58unrr79W8eLF1aVLlwdrJQAAAAAAAFzC3T1zbqxMSjKUlGTcv2A24VQot2HDBn3++ecqUaKEhg8frnr16qlRo0Y2ZWrUqKFChQrpzz//JJQDAAAAAADI4gr5FJKRmKj8+b0zpf6kxERFX4nLMcGcU6HcnDlzlCtXLn399deqVKlSiuUqV66sU6dOOd04AAAAAAAAuEY+r3wyubvr0LhxinVxnuNTrpwC3ntPbm4mQrnUHDx4UDVq1Eg1kJOkggULKiQkxKmGAQAAAAAAwPViT53S9aN/ZXYzsj2nbhKOjY1VkSJF7lvu+vXrNvPMAQAAAAAAAHAylCtSpIj++eef+5b7+++/VaJECWeqAAAAAAAAALItp0K5wMBAHTp0SHv27EmxzPr163Xq1CnVr1/f6cYBAAAAAAAA2ZFToVyvXr1kMpk0cOBArVmzRgkJCTbrN27cqFGjRsnDw0PdunVLl4YCAAAAAAAA2YVTD3qoWrWqhg0bpg8//FADBw5U7ty5ZTKZtHr1agUHB+vGjRsyDEOjRo1SxYoV07vNAAAAAAAAwEPNqSvlJKl79+766quvVL16dd28eVOGYejGjRu6fv26zGazvvjiC3Xt2jU92woAAAAAAABkC05dKWfRpEkTNWnSRNHR0Tpz5owMw1CJEiVUrFix9GofAAAAAAAAkO04Fcq1aNFCjzzyiL799ltJUsGCBVWwYMF0bRgAAAAAAACQXTl1++rly5fl6+ubzk0BAAAAAAAAcganQrmSJUvq+vXr6d0WAAAAAAAAIEdwKpRr3bq1du3apaioqPRuDwAAAAAAAJDtORXK9evXTxUqVFCvXr0UEhKS3m0CAAAAAAAAsjWnHvTw6quvyt3dXQcOHFCXLl1UuHBhlSpVSl5eXsnKmkwmff/99w/cUAAAAAAAACC7cCqU27lzp/VnwzB06dIlXbp0yW5Zk8nkXMsAAAAAAACAbMqpUG7OnDnp3Q4AAAAAmczd3anZbR5YUpKhpCQjU+oGACCzOBXK1atXL73bAQAAACCTFPIpJCMxUfnze2dK/UmJiYq+EkcwBwDIUZwK5QAAAABkH/m88snk7q5D48Yp9tQpl9btU66cAt57T25uJkI5AECOQigHAAAAQJIUe+qUrh/9K7ObAQBAjuBQKDdixAiZTCa99dZbKlKkiEaMGOFwBSaTSRMmTHC6gQAAAAAAAEB241Aot3jxYplMJvXt21dFihTR4sWLHa6AUA4AAAAAAACw5VAoN3HiRElS0aJFbf4PAAAAAAAAIO0cCuXat2+f6v8BAAAAAAAAOM4tsxsAAAAAAAAA5DSEcgAAAAAAAICLOXT76vTp052uwGQy6Y033nB6ewAAAAAAACC7cTiUM5lMMgzDZrnJZEp1O8MwCOUAAAAAAACAezgUyg0YMCDZsjNnzmjJkiXy8vLS448/rtKlS0uSzp49qy1btujWrVtq3769SpUqlb4tBgAAAAAAAB5yToVyERERat++vVq2bKn3339fRYoUsVl/+fJljRkzRn/++ad+/fXX9GstAAAAAAAAkA049aCHqVOnysPDQ59++mmyQE6SChcurE8//VTu7u6aOnXqAzcSAAAAAAAAyE6cCuU2b96sxx57TF5eXimW8fLyUt26dbVlyxanGwcAAAAAAABkR06FcjExMbp58+Z9y926dUtXr151pgoAAAAAAAAg23IqlCtZsqR27NihS5cupVjm4sWL2rFjh/z8/JxuHAAAAAAAAJAdORXKtWvXTrGxserRo4fd21O3bt2qXr16KS4uTu3atXvgRgIAAAAAAADZiUNPX73Xq6++qs2bN2vfvn3q06ePChYsqFKlSkmSzp49q+joaBmGoZo1a+rVV19N1wYDAAAAAAAADzunrpTz8vLS999/r549e8rHx0dRUVE6cOCADhw4oKioKHl7e6tnz56aPXt2qg+DAAAAAAAAAHIip66Uk6TcuXNr2LBhGjx4sMLCwnT+/HlJUvHixVWtWjXCOAAAAAAAACAFTodyFl5eXgoMDEyPtgAAAAAAAAA5glO3rwIAAAAAAABw3gNdKRcZGakdO3YoMjJSt27dslvGZDLpjTfeeJBqAAAAAAAAgGzF6VBu4sSJ+uGHH5SYmChJMgzDZr3JZJJhGIRyAAAAAAAAwD2cCuVmzZql77//XiaTSY0bN1aFChWUN2/e9G4bAAAAAAAAkC05FcotXLhQHh4e+vbbb1W/fv30bhMAAAAAAACQrTn1oId//vlHgYGBBHIAAAAAAACAE5wK5fLkyaOiRYumd1sAAAAAAACAHMGpUK5u3bo6cuRIercFAAAAAAAAyBGcCuXeeOMNnTp1Sr/88kt6twcAAAAAAADI9px60MP169fVs2dPjR49Wps3b1azZs3k5+cnNzf7Gd9jjz32QI0EAAAAAAAAshOnQrlu3brJZDLJMAytXr1aq1evTrGsyWRSeHi40w0EAAAAAAAAshunQjmufAMAAAAAAACc51QoN3fu3PRuBwAAAAAAAJBjOPWgBwAAAAAAAADOI5QDAAAAAAAAXMyh21eXLFni0M7y5MkjPz8/BQQEyN3d/UHaBQAAAAAAAGRbDoVyw4cPl8lkcnin+fPnV+/evdW3b980bQcAAAAAAADkBA6FciVLlnRoZzdu3FBMTIxiYmI0ZcoUHT16VJMmTXqgBgIAAAAAAADZjUOh3Lp16xzeYUxMjFauXKkpU6Zo2bJlateunZ544gln2wcAAAAAAABkO+n+oIcCBQqoU6dO+uKLLyRJixYtSu8qAAAAAAAAgIdahj19tXbt2qpatar279//QPtZsWKFunXrpscee0y1atXSM888o6+//lrx8fEP3MYNGzbI399f/v7+6tGjxwPvDwAAAAAAAHBEhoVyklSuXDlFRUU5vf348eM1ePBghYSEqEaNGmrSpIkiIiI0adIkde/eXTdv3nR63zExMRo1ahQPogAAAAAAAIDLZWgol5CQIHd3d6e2XbNmjebMmSMfHx8tWLBA3377raZNm6ZVq1bJbDZrz549mjp1qtNtGzdunC5fvqzOnTs7vQ8AAAAAAADAGRkayoWHh6t48eJObTtz5kxJ0quvvqqqVatalxcqVEhjxoyRJM2bN0/Xrl1L876Dg4P1+++/q0ePHqpRo4ZT7QMAAAAAAACclWGh3MKFC3X69Gk99thjad42MjJSBw4ckCS1a9cu2fq6devKz89Pt2/f1oYNG9K076ioKI0ZM0aPPvqo3nzzzTS3DQAAAAAAAHhQHo4UOnfu3H3LGIahuLg4nTp1SqtXr9bvv/8uDw8PdenSJc2NCg8PlyT5+vqqTJkydstUq1ZNERERCg8PtxvcpeT9999XdHS0pk2bJi8vrzS3DQAAAAAAAHhQDoVyLVq0SNNODcOQyWTS8OHDVbly5TQ36syZM5IkPz+/FMuUKFHCpqwjli1bplWrVumVV15RnTp10tyu1PC8CDwI+g/AOAAkxgHAGAAYB4D0cI+DtLTdoVDOMAyHd+jj46N69eqpT58+qlu3ruMtucuNGzckSd7e3imWyZMnj03Z+7l48aL++9//qmzZsnrrrbecaldqChfOl+77RM5QsGCezG4CkOkYBwDjAGAMAIwDQMpZ48ChUG7t2rX3LWMymeTt7S1fX1+ZsmCk+d577ykmJkaff/55qmGfsy5fvqY0ZJdZjru7W47q+FlJdPQNJSYmZXYzIMZBZmIc2KIv5kyMg6yB8Zd5GANZB+Mg8zAOsg7GQeZ52MeByeT4hVsOhXKlSpV6oAalleUquLi4uBTLWK6Qs5RNzeLFi7V+/Xq99NJLql+/fvo08h6GoYc6lEPmou8AjANAYhwAjAGAcQBIOWccOBTKuZolBIyIiEixzPnz523KpiY4OFiSdODAAXXr1s1m3cWLFyVJYWFh1nWTJ09W0aJF095wAAAAAAAAwAFZMpSrUqWKJOnKlSs6ffq03SewHjx4UJJUtWpVh/dr2caeq1evaufOnZKkW7dupaW5AAAAAAAAQJpkyVCuRIkSql69ug4cOKA//vhDr732ms363bt3KyIiQrly5VJQUNB99/e///0vxXWLFi3SiBEj1LBhQ82ePftBmw4AAAAAAADcl1tmNyAl/fv3lyR99dVXCgsLsy6Pjo7W2LFjJUldu3ZVvnz/Tp4XHBysNm3aqHv37q5tLAAAAAAAAJAGWfJKOUlq2bKlunXrprlz56pTp05q0KCBfHx8tG3bNl29elWBgYF68803bba5du2a/v77b92+fTuTWg0AAAAAAADcX5YN5SRp1KhRCgwM1I8//qjQ0FAlJCSobNmy6tu3r3r06KFcuXJldhMBAAAAAACANMvSoZwktW3bVm3btnWobIcOHdShQ4c07d+ZbQAAAAAAAIAHkWXnlAMAAAAAAACyK0I5AAAAAAAAwMUyNJQbMGCAWrZsmZFVAAAAAAAAAA+dDA3lLl68qLNnz2ZkFQAAAAAAAMBDJ8s/6AHI7tzdM+cu8qQkQ0lJRqbUDQAAAABATudQKBcSEuLUzm/cuOHUdjmZm5tJbm4ml9ebWcFQTuZZqJCSkgzlz++dKfUnJSYp+koswRwAAAAAAJnAoVDu5ZdflsmU9qDIMAyntsup3NxM8vX1ISDLITzy5pWbm0mrfwhRdOR1l9ZdsHhePdklUG5uJkI5AAAAAAAyQZpuX/Xz80vTzi9evKiEhIQ0bZOTubmZ5O7uplE/btLfF2JcWncj/5J646lAl9aJO6Ijr+viWdf+vQEAAAAAQOZyKJQrWbKkIiIiNH/+fBUrVszhnXfq1En79+93unE51d8XYnT4bJRL63ykaH6X1gcAAAAAAJCTOXSfZLVq1SRJhw4dytDGAAAAAAAAADmBQ6Fc1apVZRiGDh48mNHtAQAAAAAAALI9h25frVOnjipXrqzY2Ng07bxjx45q0qSJUw0DAAAAAAAAsiuHQrm6detqyZIlad75Cy+8kOZtAAAAAAAAgOzOodtXAQAAAAAAAKQfQjkAAAAAAADAxRwK5ebMmaOtW7dmdFsAAAAAAACAHMGhUG7ChAn6/fff7a575ZVX9PXXX6drowAAAAAAAIDszKEHPaRm586dKlWqVHq0BQAAAAAAAMgRmFMOAAAAAAAAcDFCOQAAAAAAAMDFCOUAAAAAAAAAFyOUAwAAAAAAAFzM4Qc9/PPPP1qyZEma10nSc889l8ZmAQAAAAAAANmXw6FcSEiIQkJCki03mUwprrOsJ5QDAAAAAAAA/uVQKFeyZMmMbgcAAAAAAACQYzgUyq1bty6j2wEAAAAAAADkGDzoAQAAAAAAAHAxQjkAAAAAAADAxQjlAAAAAAAAABcjlAMAAAAAAABcjFAOAAAAAAAAcDFCOQAAAAAAAMDFCOUAAAAAAAAAFyOUAwAAAAAAAFyMUA4AAAAAAABwMUI5AAAAAAAAwMUI5QAAAAAAAAAXI5QDAAAAAAAAXIxQDgAAAAAAAHAxQjkAAAAAAADAxQjlAAAAAAAAABcjlAMAAAAAAABcjFAOAAAAAAAAcDFCOQAAAAAAAMDFCOUAAAAAAAAAFyOUAwAAAAAAAFyMUA4AAAAAAABwMUI5AAAAAAAAwMUI5QAAAAAAAAAXI5QDAAAAAAAAXIxQDgAAAAAAAHAxQjkAAAAAAADAxQjlAAAAAAAAABcjlAMAAAAAAABcjFAOAAAAAAAAcDFCOQAAAAAAAMDFCOUAAAAAAAAAFyOUAwAAAAAAAFyMUA4AAAAAAABwMUI5AAAAAAAAwMUI5QAAAAAAAAAXI5QDAAAAAAAAXIxQDgAAAAAAAHAxQjkAAAAAAADAxQjlAAAAAAAAABcjlAMAAAAAAABcjFAOAAAAAAAAcDFCOQAAAAAAAMDFCOUAAAAAAAAAFyOUAwAAAAAAAFyMUA4AAAAAAABwMUI5AAAAAAAAwMUI5QAAAAAAAAAXI5QDAAAAAAAAXIxQDgAAAAAAAHAxQjkAAAAAAADAxQjlAAAAAAAAABcjlAMAAAAAAABcjFAOAAAAAAAAcDFCOQAAAAAAAMDFCOUAAAAAAAAAFyOUAwAAAAAAAFyMUA4AAAAAAABwMUI5AAAAAAAAwMUI5QAAAAAAAAAXI5QDAAAAAAAAXIxQDgAAAAAAAHAxQjkAAAAAAADAxQjlAAAAAAAAABcjlAMAAAAAAABcjFAOAAAAAAAAcDFCOQAAAAAAAMDFCOUAAAAAAAAAFyOUAwAAAAAAAFyMUA4AAAAAAABwMUI5AAAAAAAAwMUI5QAAAAAAAAAXI5QDAAAAAAAAXIxQDgAAAAAAAHAxQjkAAAAAAADAxQjlAAAAAAAAABcjlAMAAAAAAABcjFAOAAAAAAAAcDFCOQAAAAAAAMDFCOUAAAAAAAAAFyOUAwAAAAAAAFyMUA4AAAAAAABwMUI5AAAAAAAAwMUI5QAAAAAAAAAXI5QDAAAAAAAAXIxQDgAAAAAAAHAxQjkAAAAAAADAxQjlAAAAAAAAABcjlAMAAAAAAABcjFAOAAAAAAAAcDFCOQAAAAAAAMDFCOUAAAAAAAAAF/PI7Abcz4oVK/Tjjz/q8OHDio+PV9myZfX000+rR48e8vT0dHg/4eHh2rRpk7Zu3aq//vpLMTEx8vHxUaVKlfR///d/evHFF9O0PwAAAAAAAMBZWTqUGz9+vObMmSMPDw81aNBAPj4+2r59uyZNmqT169fru+++U+7cue+7n4SEBLVv316S5OPjo+rVq6tIkSI6f/689u7dqz179mjJkiX69ttvlT9//ox+WQAAAAAAAMjhsmwot2bNGs2ZM0c+Pj6aN2+eqlatKkmKiopS9+7dtWfPHk2dOlXDhg1zaH9Vq1ZV37591aJFC+XKlcu6/MiRI+rdu7f279+viRMnauLEiRnyegAAAAAAAACLLDun3MyZMyVJr776qjWQk6RChQppzJgxkqR58+bp2rVr992Xh4eHFi1apKeeesomkJMkf39/DRkyRJK0fPlyxcfHp9dLAAAAAAAAAOzKkqFcZGSkDhw4IElq165dsvV169aVn5+fbt++rQ0bNjxwfVWqVJEk3bx5U9HR0Q+8PwAAAAAAACA1WTKUCw8PlyT5+vqqTJkydstUq1bNpuyDOHXqlCTJ09NTvr6+D7w/AAAAAAAAIDVZck65M2fOSJL8/PxSLFOiRAmbss4yDEPffPONJKlZs2bJbm91lMn0QM0AMg19F1kFfRFgHACMAYBxAEgP9zhIS9uzZCh348YNSZK3t3eKZfLkyWNT1lnTp09XaGiofHx89Pbbbzu9n8KF8z1QO4DMULBgnsxuAiCJvghIjAOAMQAwDgApZ42DLBnKucqSJUs0Y8YMubm5acKECXrkkUec3tfly9dkGA/WHnd3txzV+ZD5oqNvKDExKbObkWUwBjMPfdEWfTFnYhxkDYy/zMMYyDoYB5mHcZB1MA4yz8M+Dkwmxy/cypKhnOUquLi4uBTLWK6Qs5RNqxUrVmjkyJGSpHHjxumpp55yaj8WhqEHDuWAzEC/RVZBXwQYBwBjAGAcAFLOGQdZ8kEPpUqVkiRFRESkWOb8+fM2ZdNi9erVeuedd5SUlKT//ve/6tixo3MNBQAAAAAAAJyQJUO5KlWqSJKuXLmi06dP2y1z8OBBSVLVqlXTtO81a9borbfeUmJiot5//329+OKLD9ZYAAAAAAAAII2yZChXokQJVa9eXZL0xx9/JFu/e/duRUREKFeuXAoKCnJ4v+vWrdPgwYOVkJCg999/X507d063NgMAAAAAAACOypKhnCT1799fkvTVV18pLCzMujw6Olpjx46VJHXt2lX58v07eV5wcLDatGmj7t27J9vfhg0bNGjQICUkJGjs2LEEcgAAAAAAAMg0WfJBD5LUsmVLdevWTXPnzlWnTp3UoEED+fj4aNu2bbp69aoCAwP15ptv2mxz7do1/f3337p9+7bN8suXL2vAgAGKj49XiRIlFBoaqtDQULv1Dh06VIUKFcqw1wUAAAAAAABk2VBOkkaNGqXAwED9+OOPCg0NVUJCgsqWLau+ffuqR48eypUrl0P7iYuLswZ158+f1+LFi1MsO2DAAEI5AAAAAAAAZKgsHcpJUtu2bdW2bVuHynbo0EEdOnRItrx06dI6cuRIejcNAAAAAAAAcEqWnVMOAAAAAAAAyK4I5QAAAAAAAAAXI5QDAAAAAAAAXIxQDgAAAAAAAHAxQjkAAAAAAADAxQjlAAAAAAAAABcjlAMAAAAAAABcjFAOAAAAAAAAcDFCOQAAAAAAAMDFCOUAAAAAAAAAFyOUAwAAAAAAAFyMUA4AAAAAAABwMUI5AAAAAAAAwMUI5QAAAAAAAAAXI5QDAAAAAAAAXIxQDgAAAAAAAHAxQjkAAAAAAADAxQjlAAAAAAAAABcjlAMAAAAAAABcjFAOAAAAAAAAcDFCOQAAAAAAAMDFCOUAAAAAAAAAFyOUAwAAAAAAAFyMUA4AAAAAAABwMUI5AAAAAAAAwMUI5QAAAAAAAAAXI5QDAAAAAAAAXIxQDgAAAAAAAHAxQjkAAAAAAADAxQjlAAAAAAAAABcjlAMAAAAAAABcjFAOAAAAAAAAcDFCOQAAAAAAAMDFCOUAAAAAAAAAFyOUAwAAAAAAAFyMUA4AAAAAAABwMUI5AAAAAAAAwMUI5QAAAAAAAAAXI5QDAAAAAAAAXIxQDgAAAAAAAHAxQjkAAAAAAADAxQjlAAAAAAAAABcjlAMAAAAAAABcjFAOAAAAAAAAcDFCOQAAAAAAAMDFCOUAAAAAAAAAFyOUAwAAAAAAAFyMUA4AAAAAAABwMUI5AAAAAAAAwMUI5QAAAAAAAAAXI5QDAAAAAAAAXIxQDgAAAAAAAHAxQjkAAAAAAADAxQjlAAAAAAAAABcjlAMAAAAAAABcjFAOAAAAAAAAcDFCOQAAAAAAAMDFCOUAAAAAAAAAFyOUAwAAAAAAAFyMUA4AAAAAAABwMUI5AAAAAAAAwMUI5QAAAAAAAAAXI5QDAAAAAAAAXIxQDgAAAAAAAHAxQjkAAAAAAADAxQjlAAAAAAAAABcjlAMAAAAAAABcjFAOAAAAAAAAcDFCOQAAAAAAAMDFCOUAAAAAAAAAFyOUAwAAAAAAAFyMUA4AAAAAAABwMUI5AAAAAAAAwMUI5QAAAAAAAAAXI5QDAAAAAAAAXIxQDgAAAAAAAHAxQjkAAAAAAADAxQjlAAAAAAAAABcjlAMAAAAAAABcjFAOAAAAAAAAcDFCOQAAAAAAAMDFCOUAAAAAAAAAFyOUAwAAAAAAAFyMUA4AAAAAAABwMUI5AAAAAAAAwMUI5QAAAAAAAAAXI5QDAAAAAAAAXIxQDgAAAAAAAHAxQjkAAAAAAADAxQjlAAAAAAAAABcjlAMAAAAAAABcjFAOAAAAAAAAcDFCOQAAAAAAAMDFCOUAAAAAAAAAFyOUAwAAAAAAAFyMUA4AAAAAAABwMUI5AAAAAAAAwMUI5QAAAAAAAAAXI5QDAAAAAAAAXIxQDgAAAAAAAHAxQjkAAAAAAADAxQjlAAAAAAAAABcjlAMAAAAAAABcjFAOAAAAAAAAcDFCOQAAAAAAAMDFCOUAAAAAAAAAFyOUAwAAAAAAAFyMUA4AAAAAAABwMUI5AAAAAAAAwMUI5QAAAAAAAAAXI5QDAAAAAAAAXIxQDgAAAAAAAHAxQjkAAAAAAADAxQjlAAAAAAAAABcjlAMAAAAAAABcjFAOAAAAAAAAcDFCOQAAAAAAAMDFPDK7AfezYsUK/fjjjzp8+LDi4+NVtmxZPf300+rRo4c8PT3TvL+DBw/qq6++0u7du3Xt2jUVLVpUzZo10+uvv67ChQtnwCsAAAAAAAAAbGXpK+XGjx+vwYMHKyQkRDVq1FCTJk0UERGhSZMmqXv37rp582aa9rdy5Up16tRJq1atUsmSJdWiRQu5ublp3rx5euaZZ3Tq1KkMeiUAAAAAAADAv7LslXJr1qzRnDlz5OPjo3nz5qlq1aqSpKioKHXv3l179uzR1KlTNWzYMIf2FxkZqeHDhyshIUH//e9/1alTJ0lSYmKihg8frt9++01vv/22fvnlF5lMpgx7XQAAAAAAAECWvVJu5syZkqRXX33VGshJUqFChTRmzBhJ0rx583Tt2jWH9vf9998rLi5OjRo1sgZykuTu7q73339f+fLl04EDB7R58+Z0fBUAAAAAAABAclkylIuMjNSBAwckSe3atUu2vm7duvLz89Pt27e1YcMGh/a5Zs2aFPeXJ08eNW/eXJIUHBzsbLMBAAAAAAAAh2TJUC48PFyS5OvrqzJlytgtU61aNZuyqbl+/bp1vjjLdg+yPwAAAAAAAOBBZMlQ7syZM5IkPz+/FMuUKFHCpmxqzp49a/25ZMmSdstY6nJkfwAAAAAAAMCDyJIPerhx44YkydvbO8UyefLksSnryP5S26ePj4+kO1fVOcPNTTIMpzZNpnLJQvLO5do/Tbli+SVJuUoEyOSZ8u89I3gUeVSSZC5iVm6P3C6tu6xvWUlS3kqV5JbbtXX7lL1Td5FS+eWRy92ldfsWzWP92S0LRvMmkylTHrji7n7nl8E4cB2f/381tOV372qGYchIrzfvDMDxwHUYB1lvHHAsYAy4SlYdAxLjgHHgOoyD5BgHjANnpaW7mowsOPJmzpypKVOmKDAwUD/99JPdMlOmTNHMmTPVuHFjffvtt6nuLyQkRC+99JIkKSwsTB4eyT/gbNmyRb169ZKnp6cOHjz44C8CAAAAAAAASEEWvEbm36vg4uLiUixjufrNUtaR/aW2z9jYWElS3rx5HW4nAAAAAAAA4IwsGcqVKlVKkhQREZFimfPnz9uUdWR/knTu3Dm7ZSx1ObI/AAAAAAAA4EFkyVCuSpUqkqQrV67o9OnTdstYbjGtWrXqffeXN29elStXzma7B9kfAAAAAAAA8CCyZChXokQJVa9eXZL0xx9/JFu/e/duRUREKFeuXAoKCnJony1btkxxfzdu3ND69eslSa1atXK22QAAAAAAAIBDsmQoJ0n9+/eXJH311VcKCwuzLo+OjtbYsWMlSV27dlW+fPms64KDg9WmTRt179492f66d+8ub29vbd26VQsWLLAuT0xM1NixY3X16lVVr15djRs3zqiXBAAAAAAAAEjKok9ftfjggw80d+5ceXp6qkGDBvLx8dG2bdt09epVBQYGatasWcp91yN6Fy1apBEjRqhUqVJat25dsv2tWLFCb7/9thITE1WzZk2VKlVKBw4c0OnTp1WkSBH9+OOP1ttcAQAAAAAAgIzikdkNSM2oUaMUGBioH3/8UaGhoUpISFDZsmXVt29f9ejRQ7ly5UrT/p566imVKVNGX375pXbv3q3w8HAVK1ZMXbp00euvv64iRYpk0CsBAAAAAAAA/pWlr5QDAAAAAAAAsqMsO6ccAAAAAAAAkF0RyuUg/v7+8vf3d0ldZ86ckb+/v5o3b55sXfPmzeXv768zZ84kW3fs2DG9/vrratiwoQICAuTv769p06ZJkrp16yZ/f3/t2LEjw9sv3Zmj0N/fX8OHD3dJfci6XDl2sqJr165pxYoVGjlypNq2bauaNWuqevXqatGihUaMGKEjR45kdhORzbn6/R9wxvDhw+Xv769Fixal6343bdqkvn37qn79+qpWrZqaN2+u0aNH6/z58yluYznXSunfiy++mK5tBLKa1D6LAA8qpc+z6X2+ktM/g+QUWXpOOeQssbGxevXVV3X27FlVq1ZNjRs3lru7uwICAjK7aUCO9s0332jmzJmSpEceeURNmzZVYmKiwsLCtGjRIv3+++8aN26c2rdvn8ktBYDs5bPPPtMXX3whSapatapKly6tI0eO6Oeff9aKFSv0/fffq0qVKilu37p1a/n4+CRbXqZMmXRtZ7du3bRz507NmTNH9evXT9d9AwAy3pkzZ9SiRYsUH5qJjEMohwxRvHhxLV++XJ6eng5vc+DAAZ09e1a1a9fW/Pnzk63/6KOPFBcXp5IlS6ZnUwHch4+Pj3r27KnOnTvrkUcesS6Pj4/XpEmTNHv2bL333nsKDAzkCdbIELz/42Hw1ltvqW/fvipWrFi67G/Dhg364osv5ObmpilTpqhNmzaSJMMwNGPGDE2bNk0DBw7UihUrUnz42dChQ1W6dOl0aQ8AIHWcr8AZhHLIEJ6enqpQoUKatomIiJAkmw/9d+PNDcgc/fr1s7vc09NTw4YN059//qmTJ09q2bJlev31113cOuQEvP/jYVCsWLF0C+Qkac6cOZKkZ5991hrISZLJZNIbb7yhdevWKSwsTEuXLtULL7yQbvUCAJzD+QqcwZxyOdSqVav00ksvKTAwULVq1VLnzp21YcMGu2WPHTumzz//XJ07d1aTJk1UrVo11a9fXz169NDy5cvtbpOWeRx27Nghf39/DRs2TJK0ePFim3lPLFK6R//uOVxOnz6tIUOG6PHHH1e1atXUsmVLTZkyRbdv37Zbd0JCgmbPnq2nn35a1atXV4MGDTRw4ECH5sj6+++/NXr0aLVs2VLVq1dXnTp11KVLFy1dutRu+bvbv3v3bvXv318NGjRQ5cqV033+GWSctIwd6U4f++WXX9StWzfVq1fPOh/QmDFjrEH03SzjoVu3boqLi9PkyZPVqlUrVa9eXY0bN9bIkSMVGRlpt66tW7dq3LhxevbZZ61zDzVt2lSDBw/W/v377W4zbdo069yN586d08iRIxUUFKSqVas6NJ+im5ubdZymNr8RINnOjbJgwQJ16NBBtWrVUt26ddW3b1/t3bvX7napzdESGxurzz77TE8++aR16oMRI0YoMjLSpn/fLSkpST///LM6d+6sunXrqmrVqmrYsKGeeeYZjRs3zu6cp8h+DMNQ/fr1VblyZUVHR9us279/v7W//vDDD8m2bdGihfz9/XX69GnrspTmlLu7H0ZFRWns2LEKCgpStWrVFBQUpHHjxunq1avJ6jhw4IAkqWHDhsnWmUwmNWjQQNKd41J6SesYtRyzdu7cKUl65ZVXbM7h7v5dHDx4UIMHD1bTpk1VrVo1BQYGqkWLFho4cKDWrFmTbq8BmWf//v36+OOP1bFjR+u5eKNGjdS/f39t3brV7jZ3z+EcGxurTz/9VK1atVK1atX0+OOPa9iwYSme90jS+vXr1bVrV9WuXVt16tTRyy+/7FB/iomJ0eeff65nn31WtWvXVs2aNfX000/rf//7n+Li4pKV57iR8xw7dkyDBg1S/fr1VaNGDbVr107ffvutEhMTU9wmpfOVqKgozZkzR3379lXz5s1Vo0YNBQYGqkOHDvrqq69069at+7YnLedNUto+gwwfPlwtWrSQJJ09ezbZPKT3OnjwoN5++2098cQTqlatmurVq6fevXun+JnowoUL+uCDD9S6dWtVr15dNWvWVFBQkLp3765vv/32vq89u+NKuRzo888/1//+9z/Vrl1bQUFBOnHihEJDQ9WvXz9NmzZNrVq1sik/a9YsLVy4UOXLl5fZbFb+/PkVERGhHTt2aNu2bdq3b59GjBjhdHuKFCmi9u3b69SpUwoJCVHZsmVVp06dNO/n0KFDGj9+vAoUKKDHHntMMTExCgkJ0cyZM3Xs2DHNmDHDpnxSUpLefPNNrVmzRp6enqpfv77y58+vffv26YUXXtDzzz+fYl0rVqzQsGHDdOvWLZUvX15BQUG6du2a9u/fr6FDh2r79u2aOHGi3W1Xrlyp+fPnq3z58mrUqJFiYmJSvO0EWUtax87169f12muvaefOnfLx8VG1atVUsGBBHT16VPPnz9fKlSs1a9Ysu/MBxcfHq0ePHjpy5Ijq1aunKlWqaM+ePfr111+1ceNGzZs3L9lVpZaDbKVKlRQYGCgPDw+dOHFCK1asUHBwsCZPnqzWrVvbfW0nT55U+/bt5enpqcDAQBmGoYIFCzr0ezl16pQkqWjRog6VByZOnKjvv//e+qH86NGj2rhxo7Zu3arPPvss2VhKSWxsrF555RUdOHBAPj4+aty4sby8vLRp0yZt2LBBQUFBdrd79913tWjRInl5ealOnToqVKiQrly5ojNnzmjevHlq2LAht/zlAJZga+XKldq2bZvatm1rXXd3gLBt2zZ16dLF+v/Tp0/rzJkzKl26dJrmZouIiFD79u2VkJCgwMBA3bp1SyEhIZo3b5727dunn376yWbaj9jYWEmSr6+v3f1Z3qPDwsJSrHPRokWKiYlRQkKCihUrpnr16umxxx67b1sdHaOWc7hNmzbp0qVLaty4sc2xoGzZspLu/A779u2r+Ph4Va5cWbVq1VJSUpIiIyP1559/KjExUS1btrxvu5C1TZ48WTt27FDFihVVtWpVeXt76/Tp01q/fr3Wr1+vkSNHqnv37na3vXbtmjp37qyIiAjVqVNHlSpV0t69e7VkyRLt2rVLS5cuVb58+Wy2mT17tvV8u0aNGipbtqxOnjypN954Qz179kyxnceOHVOfPn0UERGhokWLqk6dOvLw8NCBAwc0depUrV69WnPnzrWpj+NGzrJ792717dtXsbGxKlOmjB5//HFFR0drypQp2rdvX5r3t2nTJo0fP17FixdXuXLlVKtWLUVFRWnfvn369NNPtW7dOs2ZMyfFz4RpPW9K62eQOnXqKDY2VqtWrZKPj0+Knxck6fvvv9eHH36opKQkBQQEqEaNGrp06ZJ27NihzZs3a+DAgRowYIC1/MWLF/X888/rwoULKlmypJo0aSIvLy9duHBBhw8fVlhYmHr37p3m32m2YiDHMJvNhtlsNurWrWvs3bvXZt3nn39umM1m48knn0y23Y4dO4x//vkn2fLjx48bTZs2Ncxms7Fv3z6bdadPnzbMZrPRrFmzZNs1a9bMMJvNxunTp22W//rrr4bZbDaGDRtmt/1du3Y1zGazsX37dpvlw4YNs762yZMnGwkJCdZ1R44cMWrVqmWYzWYjJCTEZrt58+YZZrPZaNSokXHs2DHr8vj4eGPMmDHWfd7bnsOHDxvVqlUzqlevbqxatcpm3ZkzZ4x27doZZrPZWLx4sd32m81mY968eXZfI7ImZ8fOW2+9ZZjNZqNfv37GpUuXbNbNmjXLut3dfXb79u3W+lq1amWcPXvWuu7mzZvGwIEDDbPZbLz44ovJ6gsODjauXLlid3mVKlWMevXqGXFxcXbbbzabjXfeece4deuWY7+U/2/Dhg2G2Ww2/P39jUOHDqVpW+Q8lr5Wo0YNY+vWrTbrvv76a8NsNht16tRJNl5Sev+fMGGCYTabjbZt2xqRkZHW5XePFbPZbHz++efWdWfPnjXMZrPRtGlT48KFC8naeOzYMZtxh+xt/vz5htlsNkaNGmWzvFu3bkbVqlWNNm3aGHXr1rV5n05pG8v5yK+//mqz/O732eHDh9u8z547d85o0qSJYTabjd9//91mO8vylM4Z3nvvPet+b9y4YbPOcq5l79/zzz9vnDx50u4+03uMWnTr1s0wm83G0qVLk627evWqERoaanc7PFz+/PNPm/dii5CQECMwMNCoWrWqcf78eZt1lvN/s9ls9OrVy7h27Zp13ZUrV4xnn33WMJvNxsyZM222O3TokBEQEGBUrlzZWLFihc26pUuXGv7+/nY/i8TFxRktW7Y0zGazMWXKFJvxGBsbaz13Gz58uHU5x42c5ebNm0ZQUJBhNpuN8ePH27z/Hzp0yKhfv761z977eTal98Jjx47ZfZ+7cuWK0atXL8NsNhtff/11svXOvic78xkktc/vFhs3bjT8/f2N+vXrGzt37rRZd/jwYWs2sGPHDuvyadOmGWaz2XjvvfeMpKQkm21u376d7HXlRNy+mgMNGjRINWvWtFnWr18/5cuXTydPnkx2OWu9evXsfhNcvnx56/xRK1euzLgGO6hq1aoaPHiw3N3drcvMZrOeeeYZSUp22fz3338vSRowYIDN/HceHh4aMWJEilf9zJw5U7dv39bgwYP15JNP2qwrVaqUxo8fL+nfuWDu1aBBA5tv3PHwSMvYOX78uJYtW6ZixYpp0qRJKly4sM12PXr0UFBQkE6ePKmNGzfarW/o0KE2c1N4eXlpzJgx8vb21t69exUSEmJTvmXLlipQoECy/bRs2VJt2rTRlStXUnxEu6+vr0aPHp2mqzYjIyP17rvvSpJefPFFVa5c2eFtkbN16tQp2S15ffr0UbVq1XTt2jX98ssv993HzZs3tWDBAknSiBEjbOby8vLy0vvvvy9vb+9k2126dEmSVKVKFbvv8xUqVGBOmBykUaNGkmzPEW7evKnQ0FDVrl1bzZo109WrV3Xw4EHrektZe7eVpqZEiRLJ3mf9/PzUtWvXZG2QZL09deHChTIMw2ZdTEyMzbnX9evXbdYHBQXp008/VXBwsPbv36+1a9fqo48+UsmSJXXgwAF169ZNly9fTrGt6TFG72apy97Vq/ny5VOtWrXStD9kTUFBQXbnVaxdu7a6dOmi+Pj4FG8t9fHx0cSJE5U3b17rsgIFCujVV1+VlHx8zJs3T4mJiWrTpo3NnIuS9Mwzz6Q4hc7ixYv1zz//qFmzZho8eLDNePT29tZ///tfFS5cWL/99ptiYmIkcdzIaVatWqWIiAj5+flpyJAhNp8tK1eurP79+6d5nxUqVLD7PlegQAGNGjVKUuqfp9Pynpwen0FSMm3aNBmGobFjxya76tpyG7p0Z3xaWN7/mzRpIpPJZLONp6dnmo+l2RGhXA7UrFmzZMty5cplDd7szdtw48YNrVixQpMnT9Z7772n4cOHa/jw4Vq9erWkO/OrZbZmzZolG+iSrIHb3a8rMjLSesudJbS7m5eXV7IDvHTnllfLm9fdt7ncrXr16vLx8dGhQ4fszg+Q2uXAyNrSMnY2bNggwzDUtGlTmxPMu9WrV0+SFBoammxd/vz5rXM73K1w4cJq0qSJJFnn8LlbZGSkFixYoA8//FDvvvuudaz+9ddfklIeqw0bNkx2W0hqrl+/rv79++vChQuqUaOGNZwDHNG+fXu7y5977jlJ9vv2vQ4ePKjY2FgVLFhQjRs3Tra+UKFC1sDlbuXLl1eePHm0ceNGffHFFzZzgiHnKVOmjEqXLq0zZ87on3/+kXTntqXbt2+rUaNGyUI7wzC0fft2mUymNH+QaNiwod2g2N55iiT17dtXXl5eCg8P14ABA3T06FHduHFDoaGh6tmzp/X2VunO/J53GzNmjNq1a6eyZcvKy8tLpUuX1nPPPafFixerVKlSioyM1MyZM1Nsa3qM0bvVqFFDkvTOO+9o9+7dSkhISNP2eHhER0dryZIl+vjjjzVq1CjreYilz6R0HlKtWjW7gV758uUlJR8flv3ZO4+XUu7DljmvnnrqKbvr8+TJo2rVqikhIcE6ryPHjZzF0reeeuopmykFLFLqW/eTmJiobdu2acaMGXr//fc1YsQIDR8+3PpenNrn6bS8Jz/oZ5CUREVFaf/+/cqdO7fdz0SSVL9+fUmyuXDA8v4/adIkrV69Wjdu3HC4zpyCOeVyoJS+ybEM2nuDpHXr1mnEiBG6cuVKivu89xvazODn52d3ueV13f2wB8uE9AULFlSePHnsbmdvXogrV65YX2tKcxXdW7548eI2y0qVKnXf7ZA1pWXsWE7YFi5cqIULF6a636ioqGTLSpUqZTdklv7tm/c+WGH69OmaOXOm4uPjU6wrpbGaln5548YN9enTR+Hh4apSpYq++eYbeXl5Obw9kNK8Oyn1bXssH9BS67v21uXNm1cTJ07UiBEj9Nlnn+mzzz5T0aJFVatWLTVp0kTt2rVL8biA7KlRo0ZasGCBtm7dqrJly1oDuMcff1xms1m5cuXS1q1b9dprryk8PFxXrlxRlSpVHJ530yIt5ymSVKlSJU2bNk3vvPOO1qxZY3OFka+vr4YPH65x48bJZDIpf/78DrXB19dX3bt314QJE7R+/foUv1BJjzF6t7feektHjhzRxo0btXHjRuXOnVtVqlRRvXr19Mwzz9jcsYCH14IFCzRx4kSbwPheKX0gT+v4sPTB+/XVe1nOz4YOHaqhQ4em2E7p3/Mzjhs5y/36VoECBZQvXz5du3bN4X2ePHlSAwYMsH5Jbk9qn6fT8p78oJ9BUnLmzBkZhqGbN2+qevXqqZa9++FJzz77rLZs2aLff/9dAwcOlLu7uypUqKA6deqodevWXCknQrkc6d5vU1MTGRmp//znP7p586b69Omjp59+WqVLl5aPj4/c3Ny0efPmLDMxY1pel7OSkpKsPzvyLYm9b1dy586drm2C66Slj1n6SkBAwH1v67z3llhH3X070+rVqzVt2jT5+PjovffeU4MGDVSsWDHlzp1bJpNJkydP1pdffpnsFigLR/tlbGys+vXrp9DQUPn7++u7776ze8ss8CBS6qf2pBRep7audevWatSokdauXas9e/YoJCREwcHBCg4O1ueff67vvvvO7tPGkD01bNjQGsp17txZ27ZtU4ECBVStWjW5ubmpdu3aCgkJUVxcnNO3rkrOnacEBQVp7dq1WrVqlY4cOaKEhARVrFhRbdu2VXh4uCTpkUceSdPUA5YA7EGemJ2WMSrdeRDQr7/+qp07d2rr1q0KCQnR/v37FRISoi+//FJvvfWW9TZFPJwOHjyo0aNHy93dXe+8846aN28uPz8/eXt7y2Qy6eeff9bo0aNT7DuuOI+X/j0/a9KkiYoUKZJq2bu/jOW4gQcxaNAg/fXXX2rWrJn69OmjChUqKG/evPL09NTt27fvG3Ldz93jKqM+g1jquN+DIO7l5uamSZMmqX///vrzzz8VEhKikJAQ/fTTT/rpp5/UrFkzzZgxw+Y24ZyGUA6pWrdunW7evKlWrVppyJAhydZbbgF92FiuXouOjtaNGzfsfrt19uzZZMsKFiyo3Llz6+bNmxo6dKgKFSqU4W3Fw8nyjW9gYKBGjx6d5u3t9b9715UoUcK6bMWKFZKk//znP+rUqVOybU6ePJnmNtwrLi5O/fr1065du+Tv76/Zs2en+UoRQLrzbWtAQECy5fb6dkos7+OOjBV78uXLp+eee85660dERITGjRuntWvXaty4cTbzoSB7a9iwoUwmk3bs2KHLly/r0KFDatWqlTUkaNSokXbs2KFdu3Zp27Zt1mWukj9/fr3wwgvJlu/evVvSnSv60sJy50NqV/akxxi9l8lkUv369a23N926dUuLFi3Sf//7X02ZMkVt2rSxPq0VD5+VK1fKMAx17dpVffv2TbY+Pc5D7la8eHH9888/Onv2rCpVqpRsfUrv/35+fjpx4oQ6duxod6qa1HDcyBks5xdnzpyxu/7q1atpukru+PHjOnLkiAoXLqzp06fLw8M2gnHk83Ra3pMf9DNISix1mEwmTZgwIc1BesWKFVWxYkVJ/04F8fbbb2v9+vVasmSJnn/++XRr68OGOeWQKssEp/Zu2zMMQ7///rurm5QuSpQoYZ0H7I8//ki2/vbt23Yn23R3d7eeiFtCEMCepk2bSroTbNubW/B+rl69qnXr1iVbHhUVpU2bNkn6dz4IKfWxevny5WQTJKfVzZs31a9fP+3cudMayBFKw1lLly5NdfndfTslVatWlbe3t6Kiouz275SWp8TPz0+DBg2SJB06dMjh7fDwK1iwoAICAnTlyhV98803MgzDJnSz/Pznn39qz549ypUrl+rWrZtZzZUkXbt2TQsXLpS7u7teeumlNG27bNkySf/O82NPWseo5c6AxMREh9vh5eWll156Sf7+/kpKStKRI0cc3hZZT2rnIbdu3bLOQ51eLJPMp/RZZMmSJXaXW87P0uM8nuNG9mTpWytXrrQ7JUxKfSsllrFRrFixZIGcJP3222/33Uda3pOd/QxieR9Pac7P4sWLy9/fXzdu3LB+FnGWZV7Wdu3aSWL8EMohVZZbHFatWqULFy5YlycmJmrq1Klpmhwyq+nevbukO0+ROX78uHV5YmKiPvroI5vXe7cBAwbI09NTn3zyiRYvXmxzS6vF0aNH0/3kAw+XKlWqqHXr1oqIiNCAAQPsftsWGxur3377zfpUr3t99NFHNrcX3b59W2PHjlVsbKxq1KihOnXqWNdZJkJesGCBzbwr165d07Bhw9L0jd69bt26pddee007duwgkEO6+Omnn5I9CXj27Nnav3+/8uTJo44dO953H97e3tZyEydOtBlHt2/f1rhx4+zOaxQeHq7ly5fr5s2bydZZgnCeopfzWG5H/eGHHyTZXn1WrVo15c+fXwsXLtTNmzdVu3Ztl01FsX///mS3+50/f16vvfaaLl68qD59+livPLBYs2aNzdNiLa5fv67x48db+3nPnj1TrDetY9RyZUlK8yV9++23OnfuXLLlx48ft14lwrh7uFk+MyxZssRmbqxbt27p/fffT/GqI2d169ZN7u7uWrFihYKDg23WLVu2LMWnvL744osqVaqUVq5cqU8++cTuPF4XL160Pt1b4riR07Rp00bFixfXuXPnNHnyZJvPekePHtUXX3yRpv098sgjcnd319GjR5O9r65bt06zZ8++7z7S8p7s7GeQQoUKydPTU5cuXUpxLvnBgwdLuvPUe3sXDxiGoX379mnz5s3WZUuWLEnxmGR5QEVOn3Od21eRqmbNmqlq1aoKCwtT69atVa9ePXl7e2v//v26cOGC+vbtq6+//jqzm+mULl26aMuWLVq/fr2effZZ1a9fXwUKFNC+fft08eJFvfTSS/rpp5+SbVe1alV98skn1ifmfPbZZ6pYsaIKFiyomJgYHT16VOfPn1fbtm315JNPZsIrQ1YxYcIEXb16VRs3blSbNm1UuXJllS5dWoZh6OzZszp8+LDi4+O1fPnyZPOa1K5dW0lJSWrTpo0aNGig3Llza8+ePbpw4YIKFy6sjz76yKZ89+7dtXTpUm3YsEEtW7ZUrVq1FB8fr127dil37tx6/vnn9euvvzr1OiZPnmy94qhkyZL6+OOP7ZarU6eO3VusgHt16tRJ3bt3V926dVW8eHEdPXpUR48elbu7uyZMmKCiRYs6tJ///Oc/CgkJUVhYmFq1aqUGDRrIy8tLe/bsUXx8vNq3b6/FixfbzO957tw5/ec//7FONO/n56eEhAQdPXpUf//9tzw9Pe1O14DsrVGjRvr2229169YtlS5d2uY2Sjc3N9WvX9/6wd+Vt6726tVL3t7eMpvN8vX11YULFxQaGqr4+Hh16tTJ+gHpbjt27NCcOXNUsmRJmc1m5cuXTxcuXNDhw4cVExMjDw8PDR06NNXXkdYx2rp1ay1atEiffPKJtm3bpkKFCslkMun5559XYGCgvvjiC3388ccqX768KlSoIC8vL124cEEhISFKSEjQc889p6pVq6b3rw8u1KFDB82ZM0fh4eFq0aKF6tatK3d3d+3evVs3b97UK6+8ojlz5qRbfQEBAXrrrbf0ySefaMCAAapZs6bKlCmjU6dO6cCBA+rRo4fdsMPHx0dffvml+vXrp2+++UYLFiyQv7+/ihcvrps3b+rkyZM6fvy4ChcurBdffFESx42cJnfu3Jo0aZJeffVVfffdd1qzZo2qV6+uK1euaOfOnWrWrJnCwsJSnSLjboUKFVKXLl00Z84c9ejRQ3Xr1lWxYsX0999/KywsTK+99tp9g760vic78xnE09NTzZs316pVq/Tcc8+pTp061i+gxo8fL0lq3ry53n33XX300Ud67bXXVK5cOT366KPKmzevoqOjdfjwYV2+fFl9+/ZV48aNJd2Z93rYsGEqVqyYAgIClD9/fl29elUhISG6du2azGZzjv/8QCiHVHl4eGju3Ln66quvtGrVKm3btk158+ZV7dq19fnnn+vGjRsPbSjn5uam6dOna+7cuVq4cKF27twpHx8f1alTRzNmzFB4eLjdUE6684js6tWra+7cudYJixMTE1WkSBGVLVtWXbp0SfM8Fch+8ubNq++++07Lly/Xb7/9prCwMB0+fFh58uRRsWLF9PTTT6tFixZ259Dx9PTUl19+qenTp2vVqlWKjIxUgQIF1KFDBw0aNCjZU8rKlCmjxYsX67PPPtOePXu0fv16FS1aVP/3f/+ngQMHptiXHWG57F6S1q9fn2rZnH5QhWNGjhypRx99VD///LMOHDggDw8PNWnSRK+//roCAwMd3k+ePHmsx6hly5Zp06ZN8vX1VaNGjTR48GBNnz5dkmzmPqxZs6befvtt7d69W8ePH9ehQ4fk7u6uEiVKqEuXLuratav1ylPkHHXr1lWuXLl0+/Ztu2FVw4YNMyWUe+WVV7RlyxaFhYXp+vXr8vX11RNPPKHOnTtbP/Dcq2XLloqNjVV4eLgOHjyomJgYeXp6ys/PT0899ZRefvnl+05In9Yx+sQTT+iDDz7QTz/9pO3btysuLk7SnS9rLPMabdu2TQcPHtSuXbsUGxurokWLqlGjRurUqZNatGjx4L8sZCrL1aTTpk3T5s2btXHjRvn6+urxxx/XgAEDtGfPnnSvs0+fPnr00Uf17bff6tChQ/rrr7/k7++vzz//XFWrVk3xCqRKlSrpt99+0/z587VmzRodOXJEe/fula+vr0qUKKFevXqpVatW1vIcN3KeevXqacGCBZo2bZp27typ4OBglSlTRoMGDVKvXr3SfOHFyJEj5e/vrx9//FEHDx6Uu7u7zGazpkyZorZt2943lEvre7Kzn0H++9//ytfXV5s2bdKqVaust+9aQjnpznGpQYMGmjdvnnbs2KFt27bJzc1NRYoUUUBAgJ544gmb30+vXr1UunRphYaGWp9g7uvrq4oVK6pdu3bq0KGDfHx80vT7zG5MRlofnwQAyDA7duzQK6+8onr16mnu3LmZ3RwgXVmCAFfMHRUfH6927drp5MmTWrRoEVfhAA5w5RgFAADMKQcAAB5iBw8eTDa3540bNzRu3DidPHlS/v7+BHIAAADIkrh9FQAAPLQGDRqkuLg4mc1mFS5cWJcvX9bhw4ett0d8+OGHmd1EAAAAwC5COQAA8NDq0aOHgoODdfz4cYWEhMjNzU0lS5bU008/rd69eyebfxEAAADIKphTDgAAAAAAAHAx5pQDAAAAAAAAXIxQDgAAAAAAAHAx5pTLgSIjI9WmTRvVr19fM2fOtFnXvHlznT17NsVta9asqQULFqS4/vbt25o/f75WrFih48ePKy4uTgULFpTZbFaHDh3Utm3bdHsdrmb53axdu1alS5d2ad3vvvuuFi9erMWLF8vf39+ldWdXKY2DRYsWacSIEffd3mQy6fDhw8mWJyUlacGCBfr111917NgxSVLFihXVsWNHvfjiizKZTOn3IjIB4+Dhk9p7/r0SExP18ssva+/evZKkH374QXXr1nWonjfffFMrV66UJH388cd69tlnk5U5ceKEtmzZorCwMIWFhen48eNKTEzUm2++qddffz1tLyyLYow8XDLqWCBJ169f19y5cxUcHKxTp04pPj5ehQsXVpUqVdSlSxc1atQo3V6Hq1n615EjR1xed48ePbR//36tWrVKRYsWdXn9D6uU+vq1a9e0efNmbdq0SXv37tXZs2eVlJSkYsWKqV69eurRo4fd95O4uDht375dmzZt0u7du3X69GlrHw8MDFTXrl1Vp04du2357bfftHnzZh0+fFgXL17U1atXlTt3bj366KNq1aqVunbtqjx58mTY78IVOBZkTRlxThQWFqbt27dbz21OnTolwzBSPBeyYBxkrIdpHBDK5UAff/yxbt68qf/85z8plmndurV8fHySLS9TpkyK25w/f169e/fWsWPHVLBgQQUGBsrb21sRERHavXu3fHx8HupQLjMNHDhQv//+uz744APNnTs3s5uTLaQ0DsqWLav27dunuN327dsVERGh+vXrJ1uXmJiowYMHa/Xq1fL29laDBg0kSdu2bdPo0aO1detWTZkyRW5uXKTsDMaBcxx5z7f49ttvtXfvXplMJqVlytnly5dr5cqV993up59+0pw5cxzeL9KGMZJ2GXEskKSjR4+qT58+ioyMVIkSJdSgQQO5u7srIiJCGzZsUNmyZR/qUC4zvf322+rYsaMmT56siRMnZnZzHhop9fVvvvnGGk488sgjatq0qRITExUWFqZFixbp999/17hx45KNhz/++EOjRo2SJJUqVUoNGzaUh4eHDh8+rOXLl2vFihV688039dprryVry08//aTQ0FBVqFBBVapUka+vry5duqS9e/fqwIED+vXXXzV37lwVL148g34b2RvHgpRlxDnRjBkztHbt2jS3hXGQsR6mcUAol8Ps379ff/zxh9q0aZNqYjx06NA0pdk3b95Uz549deLECQ0cOFD9+vWTp6endX1cXJxOnjz5IE3P0UqUKKEXXnhB8+bN09q1a9WiRYvMbtJDLbVxULdu3RSvDLp165aaNGkiSerYsWOy9XPnztXq1atVvHhx/fDDD9YQ+/Tp03r55Ze1cuVKPfbYY+ratWs6v6KcgXGQdo6+50vSX3/9pWnTpqlZs2Y6evRoqldN3+3SpUsaO3asqlSpoty5cyskJCTFsmazWb169VKVKlVUpUoVffnll1q6dGmaXhNSxhhJm4w6Fly6dEk9evRQTEyM3n//fXXu3NnmKumrV6/qwoUL6fhKcpbq1aurWbNmWrx4sbp3767KlStndpOyvNT6uo+Pj3r27KnOnTvrkUcesS6Pj4/XpEmTNHv2bL333nsKDAxUuXLlrOs9PDz0/PPPq2vXrqpSpYp1uWEYmj17tj788EN99tlnqlOnjurVq2dT5/Dhw1WuXDn5+vraLI+OjtYbb7yhPXv26KOPPtLkyZPT75eQg3AssC+jzolq1aqlSpUqWc9tRo4cqZ07d963PYyDjPUwjQMu18hhvv/+e0n2TyIfxJdffqkTJ06oU6dOGjBggE0gJ0ne3t4KCAhI1zpzGsvfzPI3hPOcHQfBwcGKiYlR/vz59eSTT9qsS0pK0jfffCNJeuedd2yuKi1TpozeeecdSXfGSlJS0oM0P0djHKSNo309ISFBw4YNU+7cuTV27Ng01fHee+/pxo0bmjhxojw8Uv+u74UXXtCwYcP09NNPq0KFClw1mgEYI47LiGOBdOdKjMuXL2vQoEF66aWXkk1bkD9/flWsWNH5hkMdO3aUYRj0cwel1tf79eun4cOH2wRykuTp6alhw4bpkUceUXx8vJYtW2azvn379powYYJNICfduaW7Z8+eatiwoSTZ/eKlZs2ayYIISSpYsKDeeustSdKWLVscfn1IjmNBchl1TvTqq6/qP//5j1q3bp3qXWX3YhxkvIdlHHClXA5y6dIlrVq1SsWKFdPjjz+ebvuNj4/XTz/9JEnq3bt3uu23W7du2rlzp+bMmaP8+fNrxowZ2rVrl27cuKGyZcuqY8eO6tmzp905uhISEvTLL79o6dKl+uuvv3T79m35+fmpadOm6tu3b4qXAR87dkyff/65duzYobi4OOvtKz169Ei1rQkJCVq8eLF+++03HTlyRLGxsSpWrJiaNGmi/v37y8/PL9k2W7du1Zw5c7R//37FxMTIx8dHBQsWVI0aNdSpUyc99thjNuUDAgJUuXJl7dixQ8ePH1eFChUc/2XC6kHGwa+//ipJevrpp+Xl5WWzLjQ0VBcvXlSuXLnUunXrZNu2bt1a7777ri5cuKB9+/apdu3aDtXJOGAcOCstfX3mzJkKCwvThAkT0nSbxJIlS7Ru3Tq98cYbmXa1CmOEMeKMjDoWXL58WcuXL1fu3LnVpUuXdGvv3fPynDlzRl999ZUOHDigW7duqUKFCurevbuee+45u9vGxcVp7ty5WrFihU6ePKmkpCSVLl1aLVu2VK9evVSgQAG724WGhmrGjBnau3evEhMT9eijj+rll1++7wfamzdv6scff9TKlSt14sQJ3bp1SyVLllSLFi3Ut29fFSxYMNk2K1as0M8//6xDhw7p+vXryps3r83cZPe+vwQFBalgwYJatmyZhg0bZveDLe54kL7u5uYmf39/nTx5UufPn0/TtgEBAdq2bVuat3N3d5ekZF/w3w/HAo4FqXHFOVF6YhzkrHFAKJeDbNiwQfHx8WrQoMF9r05YtGiRYmJilJCQYJ3o9d5ObhEeHq7o6GgVK1ZM5cqV05EjRxQcHKwLFy4of/78qlu3rpo2ber0FRGbN2/WrFmzVLZsWT3++OO6ePGi9XLeiIgIvfvuuzblb9++rX79+mnr1q3y8vJS/fr1lTdvXoWGhmru3Ln6448/9O2336pq1ao22+3evVt9+/ZVbGysypQpo8cff1zR0dGaMmWK9u3bl2L7rl+/rtdee007d+6Uj4+PqlWrpoIFC+ro0aOaP3++Vq5cqVmzZtl8k7h48WLrBNI1atRQ/fr1dfPmTUVGRmr58uUqWLCg3d93o0aNdPjwYa1ZsybLvqlkdWkZB3c7d+6ctm/fLsn+N2yHDh2SJFWqVCnZhzRJyp07typVqqTw8HCFh4c7HMpZMA7+xThwjKN9/dChQ5o5c6YaN26s559/3uH9R0ZGavz48TKbzerfv396NPmBMEb+xRi5v4w6FuzYsUPx8fGqVq2a8ubNq5CQEG3YsEHR0dEqVKiQGjVqlOxWvrT49ddf9cUXX6hKlSpq0qSJzp49q71792rYsGG6cuVKsg9BlmWHDh1S3rx51aBBA3l6emrnzp2aOXOm/vjjD33//ffJpixZsWKF3n77bSUmJspsNstsNisiIkKjRo2yPsTInsjISPXp00dHjx6Vr6+vqlevrjx58ig8PFzffvutVq5cqblz56pUqVLWbaZPn65p06bJw8NDtWvXVvHixXXt2jVFRERo4cKFqlixYrJQztPTU/Xq1dOqVau0efNmtWvXzunfaXbnbF+3OHXqlCSl+aEazmx3/fp1TZ8+XdKdINoZHAv+xbHgXxl9TpSeGAc5cBwYyDHeeecdw2w2G/PmzUuxTLNmzQyz2Wz33/PPP2+cPHky2TY///yzYTabjY4dOxqffPKJ4e/vn2zb5557zjh79mya2tu1a1fr9j/99JPNuq1btxr+/v5GQECAERERYbPuk08+Mcxms9GyZUvj9OnT1uW3b982Ro4caZjNZqN58+bGrVu3rOtu3rxpBAUFGWaz2Rg/fryRkJBgXXfo0CGjfv361rbcvU/DMIy33nrLMJvNRr9+/YxLly7ZrJs1a5ZhNpuNJ5980mafzZs3N8xms7Fr165kr/vSpUtGWFiY3d/J6tWrDbPZbHTv3j2F3xrux5FxYM+0adOsfdmeiRMnGmaz2Xj99ddT3Ef//v0Ns9lsfPjhhw7XyzhIjnHgGEf6+q1bt4ynn37aqF27ts17tOVYYO9vY9G7d28jICDA2L9/v3WZpb8uWbLEoTYOGzbMMJvNxowZMxwqbw9jJDnGyP1l1LFg8uTJhtlsNgYMGGCt495/PXr0MK5cuZKmei1jsmrVqsa6dets1v3666+G2Ww26tSpY8TFxdmsGzx4sGE2m40XXnjBiIqKsi6/fv260adPH8NsNhudOnWy2ebChQtG7dq1DbPZbMyaNctm3datW43q1atbX8vdkpKSjM6dOxtms9kYOXKkce3aNeu6+Ph448MPPzTMZrPRrVs36/Jbt24ZNWrUMGrVqmUcP3482es+c+aMcezYMbu/E8u4GTlypN31uMPZvm4YhrFhwwbDbDYb/v7+xqFDhxze7vDhw0aVKlUMs9lsrF27NsVymzZtMoYNG2YMGTLE6NWrl7Xf9e7d27h69Wqa2sqxIDmOBf/K6HOiu6X1XIhxwDhgMpccxHIlT2oJcVBQkD799FMFBwdr//79Wrt2rT766COVLFlSBw4cULdu3XT58mWbba5cuWLd/9dff22d0H7Pnj2aNWuWHnnkEYWHh6tfv36Kj49Pc7uffPJJde7c2WZZw4YN1bhxYyUmJlq/sZbuTL78ww8/SJJGjBhh882vp6enRo0apSJFiujMmTNatWqVdd2qVasUEREhPz8/DRkyxHrJsCRVrlw5xatAjh8/rmXLlqlYsWKaNGmSChcubLO+R48eCgoK0smTJ7Vx40br8suXLytfvnx2J5EuXLhwsvk5LCxz0ISHh9tdj/tzZBzcyzAMLVq0SFLK81DcuHFD0p35E1NieaKxpWxaMA7+xThwjCN9fcaMGTpy5IiGDh2qkiVLOrzvBQsWaNOmTerdu7eqV6/+wG1ND4yRfzFG7i+jjgXR0dGSpPXr12vZsmUaOHCg1q5dq507d2r69OkqWrSotm7dap0vKK26du2qZs2a2Szr0KGDypcvr2vXrungwYPW5efOnbM+Ffm///2vzW2jefLk0QcffCAvLy+FhobaPKBl4cKFunHjhmrVqpXsyruGDRuqU6dOdtu2adMmhYSEKCAgQGPHjlXevHmt6zw8PDRkyBCZzWbt2LFDR48elXTnaombN2+qTJkyKl++fLJ9lipVKsW/Ef3cMc70denOVY+Wq2lefPFFh6couHHjht555x0lJCSocePGqV7pc+zYMS1evFhLly7V5s2bdePGDbVr104ffvih8uXLl6b2WnAs+Bdj5F8ZeU70oBgHjANCuRzk0qVLkpTqvBtjxoxRu3btVLZsWXl5eal06dJ67rnntHjxYpUqVUqRkZHWx6ZbGP//EdHx8fFq166dRo8erUcffVR58+ZVo0aNNGvWLHl5eeno0aPJJol1xL0nnxaWN9W7n2B24MABxcbGytfX1+5JgLe3t9q2bSvpzi0mFpYn5Dz11FN2792/9zHwFhs2bJBhGGratKnNyefdLLephIaGWpdVr15d165d09ChQ3Xw4EGHJ/63/O1iYmJ0+/Zth7aBLUfGwb22bdums2fPysvLK9NukWEc/Itx4Jj79fX9+/fr66+/VoMGDVL8kG3P2bNn9eGHH6pChQoaOHBgejQ1XTBG/sUYub+MPhbEx8erT58+GjBggEqXLq0CBQqoVatWmjFjhkwmkzZv3qzdu3enud336+eRkZHWZbt27VJSUpKqVKliN1ApXry4GjduLMl+P3/66aft1pVaP5fufBi099AXNzc364cpSz8vVKiQSpUqpSNHjujDDz9M9dbYe1n+dpa/Jexzpq9fv35d/fv314ULF1SjRo1kt7qlJD4+Xm+++aaOHj2qMmXK6JNPPkm1fI8ePXTkyBEdPHhQwcHBGj58uDZt2qT/+7//065duxxu7904FvyLY8G/MuqcKD0wDpLLaeOAOeVykOvXr0tSip0/Nb6+vurevbsmTJig9evX2xyc8+TJY/3Z3ptYyZIl9cQTT2jVqlXatm1bihMRp8TeJI/Sv6/j1q1b1mWWN5i75yq5V9myZSXZnrhaJqG9d04ViwIFCihfvny6du2azfLTp09LuvOt8sKFC1N9HVFRUdaf33//ffXr109Lly7V0qVLlSdPHlWvXl0NGjTQs88+m+K3M3f/7a5du5bsmwXcnzPjwDKpd6tWrVKcENsyDuLi4lLcT2xsrE3ZtGAc/Itx4JjU+vqtW7c0fPhweXl56YMPPrA74a89hmFo5MiRiouL04QJE5QrV650bfODYIz8izFyfxl9LJDsnxPVrFlTVapUUVhYmLZu3Wr3G//U3O9vfnc/t/TdlPqr5Fw/T2m5pZ9PnTpVU6dOTbFOybaff/zxxxo0aJBmzZqlWbNmydfXVzVq1NDjjz+uZ555RoUKFbK7D8trvnr1aqp15XRp7es3btxQnz59FB4eripVquibb76xO1fuvRISEvTWW29p06ZNKlWqlL7//vsU/3b38vT0VNmyZdWzZ08FBgaqU6dOGjJkiFauXKncuXM7tA8LjgX/4ljwr4w4J0pvjIPkcso4IJTLQfLly6eoqCjrm1JaWZL1e5+idPejn1N6DLRlsF68eDHN9Tr7gAhXsCT0lie7pKZmzZrWnytUqKCVK1dqy5Yt2r59u0JDQ7Vnzx5t375dM2bM0Pjx4/Xss88m28fdb2r58+dPp1eRs6R1HFy9elXBwcGSUn+EuuVAFhERkWKZ+x28UsM4+BfjwDGp9fUTJ07o+PHjKliwoEaOHJlsveW9+oMPPlC+fPnUpEkTvfrqq7p27Zq2b98uHx8fffrpp8m2s9weMnPmTC1cuFCVK1d2+AqLB8UY+Rdj5P4y6lhgeX/38PBI8UNRmTJlFBYW5tQ5UWZ9WHSEpZ/XqVPH+gEuJZUqVbL+XLduXa1bt05//vmndu3apdDQUG3evFkbN27U559/rhkzZqhhw4bJ9mHp5/Tx1KWlr8fGxqpfv34KDQ2Vv7+/vvvuuxQD6LslJibqnXfe0erVq+Xn56fvv/8+1Q/4qalZs6YqVqyov/76SwcPHkxzcM2x4F8cC/6VEedEGYlxcEdOGQeEcjlI4cKFFRUVZZ0DLq0s2917lU+VKlVkMplkGIaio6PtnoRa5lixzKmVUYoVKybpzu1VKbGk83c/1tny85kzZ+xuc/Xq1WQpv/TvtxCBgYEaPXp0mtrq4eGhoKAgBQUFSbrzDc6sWbM0ffp0jRkzRq1atUr2+7L8DQoUKJDmR2TjjrSOg99//123bt1S6dKl1aBBgxTLWeYx+Ouvv3Tr1q1k3yrfvHlTf/31l03ZjMI4gORYX4+OjrbefmCPJWS798NVbGxsqtudOHFCJ06cSFuDXYgxgow6FlSrVk3SnauGrl+/bvcDgKvOiSz91dKX7Umpn584cSLF8ZHScks/b9GihXr37p2mtubOnVtt2rRRmzZtJN25OuKzzz7Tzz//rJEjR2r9+vXJtrH87YoUKZKmunIaR/t6XFyc+vXrp127dsnf31+zZ8+2mYcwJYmJiRoyZIhWrFghPz8/zZkzJ8Uv6R1lmZ/33nms0xvHgpwjI8+JMgrjILnsOg6yboSKdGcJAo4fP+7U9pb54GrUqGGzvGjRoqpTp44kaevWrcm2i4+Pt94Pf++26a169ery8fHRlStXtHbt2mTrb968qeXLl0uS6tevb11ueXzyypUr7T6MYsmSJXbra9q0qSRp3bp1NpcBOyNv3rwaOHCg8ufPr7i4OJ08eTJZGUuoc+/jqOG4tI4Dy+1KHTp0SPUKhdq1a6to0aK6ffu2zSSoFqtWrVJ8fLyKFStm861PRmAcQEq9rwcEBOjIkSMp/rOccP7www/WuZ6kO98wpradZQ6Qjz/+WEeOHNHcuXNd9GrThjGCjDoW1KhRw3oLzZYtW5Ktv3LlisLCwqxlM9Jjjz0mNzc3HTp0SIcPH062/sKFC9q0aZMk+/38999/t7vf+/XzlStXWucbdlahQoU0ZMgQSXceWBETE5OsDP3cMY709Zs3b6pfv37auXOnNZBz5NbTpKQkDR06VMuWLbMGcve7SvJ+oqKirP31kUceeaB93Q/HgpwjI86JMhLjwDHZZRwQyuUglkF094SJd1uzZo3NU7ssrl+/rvHjx2vdunWSpJ49eyYrM2DAAEnSV199pb1791qXJyQk6KOPPtLp06eVJ08edejQ4UFfRqq8vLzUpUsXSdJHH31kk/jHx8dr/PjxunjxokqXLq3WrVtb17Vp00bFixfXuXPnNHnyZJuJI48ePaovvvjCbn1VqlRR69atFRERoQEDBtj9piA2Nla//fabdYLRuLg4zZo1y+YeeYvdu3fr6tWrcnd3V4kSJZKtt/ztUvuWHqm73zi42+HDhxUWFiY3N7f79l03Nzf16dNHkjRp0iSbKxNOnz5tvdWvX79+GX5JOeMAUtr6ek7DGEFGHQtMJpPeeOMNSdInn3xic8VoXFycRo8erevXr6tkyZJq2bLlA7yC+ytZsqTatGkjwzA0evRo6xV60p3+Nnr0aN26dUu1a9dWYGCgdV3Hjh3l4+Oj0NBQzZkzx2afO3bs0Pz58+3W16JFC1WvXl379+/XiBEj7PbdmJgY/fTTT0pISJB058qMX375xe4tZZbzzgIFCtidB4p+7pj79fVbt27ptdde044dO9IcyI0YMUJ//PFHmgK5Y8eO6bfffrP7Yfzvv//Wm2++qdu3b6tWrVry9/e/7/4eBMeCnCOrnRMxDhgHd+P21RwkKChInp6e2r59uxITE20eWyzdOdGaM2eOSpYsKbPZrHz58unChQs6fPiwYmJi5OHhoaFDh6pRo0bJ9t2wYUO9+eabmjp1qrp06aLq1auraNGiCgsL09mzZ5U7d25NnjzZJbcYDBo0SAcPHtS2bdvUtm1b1a9fX3ny5NHevXt17tw5+fr6aurUqTYTlOfOnVuTJk3Sq6++qu+++05r1qxR9erVdeXKFe3cuVPNmjWzvpZ7TZgwQVevXtXGjRvVpk0bVa5cWaVLl5ZhGDp79qwOHz6s+Ph4LV++XEWKFFF8fLw+/PBDffzxxzKbzSpXrpw8PT119uxZa6DZv39/uydElisRW7RokTG/vBzgfuPgbpbJRx9//PEU5wa6W7du3bR7924FBwfr6aefts6Bs23bNsXFxal169Z6+eWX0+eF3AfjAGnp664SFhamsWPHWv//zz//SJJ+/vln/fnnn9bl06dPt95OkVEYIzlbRh4LOnbsqL179+qXX37Rc889p5o1aypfvnzav3+/Ll68aO1bjkye/6BGjx6tEydOaN++fWrVqpXq168vd3d37dq1S1FRUSpdurQmTZpks03x4sX1wQcfaMiQIRo/frx++eUXmc1mRUZGavfu3erevbtmz56drC43NzfNmDFD/fr10+LFi7Vq1Sr5+/urZMmSio+P1+nTp3X06FElJiaqQ4cO8vDw0NWrVzVq1CiNHTvWOiYk6dSpUwoPD5fJZNKQIUOS/X0sd2F4eXlZnyAL++7X1ydPnmx9zyhZsqQ+/vhju/upU6eOXnjhBev/582bZ70SpkyZMvrf//5nd7vy5cvbzL91+fJlDRkyRGPGjFFAQIBKlCih+Ph4nTt3TuHh4UpKSlKFChU0ZcqUB3nZDuNYkDNk5DnRn3/+adP/LU+Rnj59un744Qfr8gULFlh/ZhwwDu5GKJeDFClSRK1bt9Yff/yhzZs3W+/FtmjZsqViY2MVHh6ugwcPKiYmRp6envLz89NTTz2ll19+OdWk/vXXX1eNGjX0/fffa//+/Tp48KCKFCmiDh06qE+fPtYHRWS0XLly6ZtvvtGCBQu0dOlS7d69W7dv35afn5+6deumvn372twPb1GvXj0tWLBA06ZN086dOxUcHKwyZcpo0KBB6tWrl5588km79eXNm1ffffedli9frt9++01hYWE6fPiw8uTJo2LFiunpp59WixYtrN8e+vj4aOzYsdq1a5fCw8O1detW622NTz75pF566SW7ExqHh4fryJEjql+/vipWrJi+v7Qc5H7jwOL27dvWW3eef/55h/bt7u6uzz//XAsWLNAvv/yi7du3S5IqVqyojh07qlOnTi6bpJtxAEf7uitdv35d+/btS7b8/PnzNg8RcsUj6xkjOVtGHgukOxOCN2zYUPPnz9ehQ4d08+ZN+fn5qWvXrurbt6/db/MzQsGCBTV//nzNnTtXy5cv15YtW5SUlKTSpUvrxRdfVK9evexO5P9///d/Kl68uL744gvt3btXp0+f1qOPPqqxY8eqU6dOdkM56U6gt2DBAi1atEjLly/XkSNHdODAARUoUEDFihVT586d1bx5c2sgWaZMGY0cOVK7du3SX3/9pQ0bNki6M8fRc889p27dulnn6bvbn3/+qejoaHXo0EG+vr7p9vvKju7X1+++Ndje3H13uzuUu3u71Obhqlevnk0oV6lSJf3nP//R7t27deLECR06dEjx8fHy9fVVw4YN1apVKz3//PMue7o3x4KcISPPiaKiouye2/zzzz/WLx/vxThgHNzNZDzopA94qOzfv18vvPCCnnzySU2bNi2zm4M0GDdunObNm6f//e9/WTrpfxgwDh5ejIO0oa/nPIwRxzE+Hl79+/fXn3/+qcWLFysgICCzm5Pl0ddzHo4FyTEOcp6HZRwwp1wOU6NGDbVr107BwcF2J/1F1hQREaFffvlF9erVy9JvKA8LxsHDiXGQdvT1nIUxkjaMj4fT/v37tX79erVv355AzkH09ZyFY4F9jIOc5WEaB4RyOdDQoUPl7e3tsnvU8eCmT5+uhIQEvfvuu5ndlGyDcfDwYRw4h76eczBG0o7x8fCZPHmy8uTJo7feeiuzm/JQoa/nHBwLUsY4yDkepnHA7asAAAAAAACAi3GlHAAAAAAAAOBihHIAAAAAAACAixHKAQAAAAAAAC5GKAcAAAAAAAC4GKEcAAAAAAAA4GKEcgAAAAAAAICLeWR2AwAAAHB/zZs319mzZ63/N5lM8vb2Vr58+VSuXDlVq1ZNTz31lGrUqJGJrQQAAICjuFIOAADgIRIYGKj27dvrueeeU1BQkB599FEdOXJE3333nV544QV169ZNp0+fTpe6zpw5I39/fzVv3jxd9udq06ZNk7+/v6ZNm5bZTQEAAEiGK+UAAAAeIi+88II6dOhgs8wwDG3cuFETJkzQzp071blzZ82fP19lypTJpFYCAADgfrhSDgAA4CFnMpkUFBSkX375RY888oguXbqkUaNGZXazAAAAkAqTYRhGZjcCAAAAqbPMKTdx4sRkV8rdbcOGDXr11VclSb/++quqVasmSTp27JiWL1+urVu36uzZs4qOjlaePHkUEBCgF198UW3btrXZz/Dhw7V48eIU6zly5Igk6fr161q+fLk2btyoo0eP6sKFC5KkMmXKqHnz5urdu7fy58+fbPsLFy7oq6/+X3t3G1p1+cdx/L2jh/I05qZZTifOadN1o7jBYi2FbmxIlqx1YwlRZGDmiSJ6IBEaUo0QoUIwC1tUDqMHKx/Yicw2zW48bt24tkhrJW60G8/uG9t0/wfS6b//Zn+T//+Y4/16eF3ne13Xb3v24fv7XdvYt28fjY2NBAIBUlNTyczMZPHixTz00EMjan777Te2b99OVVVVvCYrK4vi4mJWrFjB+PF/vgQyd+7cM569uLiY0tLSM85LkiQlgq+vSpIkjSGLFy8mNTWV9vZ2Dhw4EA/l3njjDd577z2ysrLIzs4mJSWFpqYmvvzySz7//HO++eYb1q1bF18nLy+P3t5eIpEIoVCIoqKiUferr6/nmWeeYdKkScyaNYurrrqKzs5ODh8+zNatW9m9ezc7d+4kLS0tXtPS0kJJSQnNzc1MmzaNRYsWcdFFF9Hc3Ex9fT21tbUjQrmDBw/y6KOP0tHRwfTp07nuuuvo7+/nu+++Y+PGjezdu5etW7cSDAaB08FbXV0d9fX1zJs3j5ycnGHPJkmSdL4ZykmSJI0hSUlJXHnllRw4cIAff/wxPr58+XJWr1494jtzP/30Ew8++CBlZWXceuut8dtb77rrLgoKCohEIqSlpZ2xsywjI4OysjKuvfZaAoE/v4zy+++/s2HDBioqKnj55ZdZv359fG7nzp00Nzdzzz338Oyzz5KUlBSfGxgYIBqNDtujpaWFtWvX0tnZyfr161mxYkV8r1gsxuOPP87+/ft59dVXWbt2LQClpaW88sor1NfXc/PNNxMOh8/lzylJkvR/4zflJEmSxpg/utLa29vjY/n5+aNe/JCVlcWaNWsA+PDDD//2XlOnTqWgoGBYIAcwYcIENmzYwPjx40es29bWBsCiRYuGBXIAwWCQgoKCYWNvvvkm7e3trFy5kvvuu2/YXmlpabz44osEg0HeL+nh/wAABXVJREFUeecd/DKLJEm6UNgpJ0mSNMacOnUKYETg1dPTQ1VVFXV1dcRiMQYGBoDTnWgAP//88znvWV1dTTQapampib6+vng4FgwGOXHiBB0dHUycOBGA+fPns2PHDjZt2sTQ0BCFhYVccsklZ1y7srISgKVLl446f/nllzNz5kyOHDlCQ0MDs2bNOufnkCRJShRDOUmSpDEmFosBxEMwgE8++YR169YN6577T93d3X97r7a2NsLhMIcOHfrL33V3d8fPs3z5cj777DN27dpFOBxm3LhxzJ49m7y8PIqKikZ0yh07dgyAlStX/tfznDhxwlBOkiRdEAzlJEmSxpChoSHq6uoAyM7OBk7fWvrEE0/Q19fHqlWruO2228jIyCAUChEIBNi/f/+ot52ejaeffppDhw6xcOFCwuEw8+bNIyUlJX7hwvXXX09LS8uw10oDgQCbNm1i9erVfPrpp1RXV1NdXU15eTnl5eXccMMNbNmyhXHjxgF/dv4VFRURCoX+8jypqann9BySJEmJZignSZI0hlRWVtLR0QGcDsTgdJdcX18fS5Ys4amnnhpR88svv5zTXr29vVRVVREIBNi2bRspKSkj5ltbW89YP2fOHObMmQOcDhO/+OILnnzySfbu3UtFRQUlJSUApKen09DQwMMPP8w111xzTmeVJEn6p/GiB0mSpDGiq6uLF154AYDCwkJycnIA4iHdtGnTRtQMDQ2xa9euUdf7o9ttcHDwjPudPHmS5OTkEYEcwAcffHDWFy8kJSVRUFDAsmXLAOLdfnD6QgiA3bt3n9VaZ3t+SZKk88lQTpIk6QI3NDREZWUld955Jw0NDUyZMoWNGzfG52fPng1AJBKhubk5Pn7y5EleeuklampqRl130qRJBINBWltbR/0W3aWXXsrEiRPp7OykoqJi2NzXX3/N5s2bR123oqKCw4cPjxjv7u7mq6++AmD69Onx8VWrVpGSkkJZWRnbt2+nv79/RO2xY8d4//33h41NnToVgCNHjox6DkmSpPMpach74yVJkv7xbrzxRo4fP05ubi4zZ84EoL+/n1gsxvfffx8PzfLz83n++eeZMWNGvHZwcJC7776b2tpaQqEQ+fn5TJgwgW+//Zbm5mYeeOABXnvtNfLz83nrrbeG7fvYY48RiURIT08nLy+Piy++GIDnnnsOgLKysnh33oIFC5gxYwaNjY3U1NRw++23E41GOX78OHv27CEjIwOANWvWsGfPHi677DJycnJISUmhs7OT6upqurq6yM7Opry8nOTk5Pg5Dh48SDgcJhaLMXnyZK644gqmTJlCd3c3R48e5ddff2XBggW8++678ZrW1laWLFlCb28vubm5ZGZmEggEyM3Njb8aK0mSdL4YykmSJF0A/gjl/l0oFCI5OZnMzEyuvvpqli5dyvz580et7+npYdu2bUQiERobG0lOTmbhwoU88sgj9PT0cP/9948ayrW3t7N582b27dtHS0sLAwMDAPzwww/x33z88ce8/vrrHD16lMHBQbKysigpKeHee+/lpptuGhHKRaNRPvroI2pqamhqaqK9vZ3U1FQyMjJYtmwZd9xxx6gXOrS1tfH2229TWVlJQ0MD/f39TJ48mfT0dAoLC7nllluYO3fusJpoNMqWLVuora2lq6uLU6dOUVxcTGlp6d//J0iSJP0PGcpJkiRJkiRJCeY35SRJkiRJkqQEM5STJEmSJEmSEsxQTpIkSZIkSUowQzlJkiRJkiQpwQzlJEmSJEmSpAQzlJMkSZIkSZISzFBOkiRJkiRJSjBDOUmSJEmSJCnBDOUkSZIkSZKkBDOUkyRJkiRJkhLMUE6SJEmSJElKMEM5SZIkSZIkKcH+BeKrR7M/NPP2AAAAAElFTkSuQmCC" + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "execution_count": 6 + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Score-function convergence" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import matplotlib.pyplot as plt\n", + "\n", + "# Fitness data\n", + "fitness = {'run_1': [51775.062, 46950.692, 46950.692, 43898.075, 43898.075, 43353.780, 43353.780, 43353.780, 43353.780, 43353.780, 43353.780, 43160.255, 43160.255, 43160.255, 43160.255, 43160.255],\n", + "'run_2': [52388.294, 46896.789, 45909.773, 43365.164, 42920.946, 42677.318, 42552.277, 42551.150, 42551.150, 42551.150, 42551.150, 42551.150, 42551.150, 42551.150, 42551.150, 42551.150, 42551.150],\n", + "'run_3': [49253.299, 46508.138, 45203.675, 45119.051, 43483.064, 42150.214, 42150.214, 42064.240, 41949.812, 41802.151, 41802.151, 41750.152, 41550.527, 41550.527, 41550.527, 41550.527, 41550.527, 41550.527, 41550.527, 41550.527, 41550.527, 41550.527],\n", + "'run_4': [49325.263, 46488.362, 45729.883, 45500.434, 43101.351, 42511.529, 42462.224, 42227.904, 42159.111, 42145.738, 42145.738, 42145.738, 42145.738, 42145.738, 42145.738, 42145.738, 42145.738, 42145.738, 42145.738], \n", + "'run_5': [49115.110, 45379.152, 44387.612, 42441.959, 41787.585, 41726.348, 41369.006, 41172.714, 40860.771, 40860.771, 40848.762, 40848.762, 40848.762, 40848.762, 40818.368, 40669.969, 40669.969, 40669.969, 40669.969, 40669.969, 40669.969, 40669.969, 40669.969, 40669.969, 40669.969]}\n", + "\n", + "# Plotting the graph\n", + "for key in fitness.keys():\n", + " plt.plot(fitness[key], label=key)\n", + "plt.title('LSevoBN: Fitness vs Generations (win95pts dataset)')\n", + "plt.xlabel('Generations')\n", + "plt.ylabel('Fitness (K2 Score)')\n", + "plt.legend()\n", + "plt.show()" + ], + "outputs": [], + "execution_count": null + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Different number of clusters in K-means" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import pandas as pd\n", + "\n", + "def different_kmeans_test(datasets, num_runs=5):\n", + " results_df = pd.DataFrame(columns=['dataset', 'data_type', 'hidden_nodes_clusters', 'shd', 'f1', 'time'])\n", + "\n", + " for dataset_name, (data, reference_dag_edges, datatype, max_local_structures) in datasets.items():\n", + " for hidden_nodes_clusters in range(4, 17):\n", + " for i in range(num_runs):\n", + " start_time = time.time()\n", + "\n", + " custom_dag_edges = get_edges_by_localstructures(data,\n", + " datatype,\n", + " max_local_structures,\n", + " hidden_nodes_clusters,\n", + " time_m=5)\n", + "\n", + " end_time = time.time()\n", + "\n", + " structure_learning_time = end_time - start_time\n", + "\n", + " f1 = f1_score(custom_dag_edges, reference_dag_edges.values.tolist())\n", + "\n", + " results_df = results_df.append({\n", + " 'dataset': dataset_name,\n", + " 'data_type': datatype,\n", + " 'hidden_nodes_clusters': hidden_nodes_clusters,\n", + " 'shd': float(precision_recall(custom_dag_edges, reference_dag_edges.values.tolist())['SHD']),\n", + " 'f1': f1,\n", + " 'time': structure_learning_time\n", + " }, ignore_index=True)\n", + "\n", + " print(results_df)\n", + "\n", + " return results_df\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "datasets = {\n", + " \"ecoli70\": (ecoli70, ecoli70_true, \"continuous\", 4),\n", + " \"magic_irri\": (magic_irri, magic_irri_true, \"continuous\", 4),\n", + " \"magic_niab\": (magic_niab, magic_niab_true, \"continuous\", 4),\n", + "# \"pigs\": (pigs, pigs_true, \"discrete\", 16),\n", + " \"win95pts\": (win95pts, win95pts_true, \"discrete\", 4),\n", + " \"hailfinder\": (hailfinder, hailfinder_true, \"discrete\", 8),\n", + " \"hepar2\": (hepar2, hepar2_true, \"discrete\", 4)\n", + "# \"arth150\": (arth150, arth150_true, \"continuous\", 8),\n", + "}\n", + "\n", + "results_df_kmeans_lsevo = different_kmeans_test(datasets)\n", + "\n", + "# results_df_kmeans_lsevo.to_csv('results_df_kmeans_lsevo.csv', index=False)\n", + "\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# results_df_kmeans_lsevo = pd.read_csv('results_df_kmeans_lsevo.csv')" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import pandas as pd\n", + "import seaborn as sns\n", + "import matplotlib.pyplot as plt\n", + "\n", + "df = pd.DataFrame(results_df_kmeans_lsevo)\n", + "\n", + "sns.set(style='whitegrid')\n", + "g = sns.FacetGrid(df, col='dataset', height=6, aspect=1)\n", + "g.map(sns.boxplot, 'hidden_nodes_clusters', 'f1', order=None)\n", + "g.set_axis_labels('Number of Clusters', 'F1 Score')\n", + "g.set_titles('{col_name}')\n", + "\n", + "plt.show()" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Group by dataset, data_type, and hidden_nodes_clusters\n", + "grouped_results_df = results_df_kmeans_lsevo[results_df_kmeans_lsevo['dataset']=='win95pts'].groupby(['dataset', 'data_type', 'hidden_nodes_clusters'], as_index=False)\n", + "\n", + "# Extract the necessary data\n", + "time_data = [group['time'].values for name, group in grouped_results_df]\n", + "f1_data = [group['f1'].values for name, group in grouped_results_df]\n", + "\n", + "# Create the boxplots\n", + "fig, ax = plt.subplots()\n", + "ax.boxplot(time_data, positions=grouped_results_df['hidden_nodes_clusters'].unique(), widths=0.6)\n", + "ax.boxplot(f1_data, positions=grouped_results_df['hidden_nodes_clusters'].unique() + 0.4, widths=0.6)\n", + "\n", + "# Add labels and legends\n", + "ax.set_xlabel('Number of Hidden Nodes Clusters')\n", + "ax.set_ylabel('Time (s) and F1')\n", + "ax.legend(['Time', 'F1'])\n", + "\n", + "plt.show()\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "results_df_kmeans_lsevo.groupby(['dataset', 'data_type', 'hidden_nodes_clusters']).mean()\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Group by dataset, data_type, and hidden_nodes_clusters and calculate the mean of each group\n", + "grouped_results_df = results_df_kmeans_lsevo[results_df_kmeans_lsevo['dataset']=='win95pts'].groupby(['dataset', 'data_type', 'hidden_nodes_clusters'], as_index=False).mean()\n", + "\n", + "# Plot the results\n", + "fig, ax1 = plt.subplots()\n", + "ax1.set_xlabel('Number of Hidden Nodes Clusters')\n", + "ax1.set_ylabel('Time (s)', color='tab:red')\n", + "ax1.plot(grouped_results_df['hidden_nodes_clusters'], grouped_results_df['time'], color='tab:red')\n", + "ax1.tick_params(axis='y', labelcolor='tab:red')\n", + "\n", + "ax2 = ax1.twinx() # instantiate a second axes that shares the same x-axis\n", + "\n", + "ax2.set_ylabel('F1', color='tab:blue') # we already handled the x-label with ax1\n", + "ax2.plot(grouped_results_df['hidden_nodes_clusters'], grouped_results_df['f1'], color='tab:blue')\n", + "ax2.tick_params(axis='y', labelcolor='tab:blue')\n", + "\n", + "fig.tight_layout() # otherwise the right y-label is slightly clipped\n", + "plt.title('win95pts (discrete)')\n", + "plt.show()" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Group by dataset, data_type, and hidden_nodes_clusters and calculate the mean of each group\n", + "grouped_results_df = results_df_kmeans_lsevo[results_df_kmeans_lsevo['dataset']=='ecoli70'].groupby(['dataset', 'data_type', 'hidden_nodes_clusters'], as_index=False).mean()\n", + "\n", + "# Plot the results\n", + "fig, ax1 = plt.subplots()\n", + "ax1.set_xlabel('Number of Hidden Nodes Clusters')\n", + "ax1.set_ylabel('Time (s)', color='tab:red')\n", + "ax1.plot(grouped_results_df['hidden_nodes_clusters'], grouped_results_df['time'], color='tab:red')\n", + "ax1.tick_params(axis='y', labelcolor='tab:red')\n", + "\n", + "ax2 = ax1.twinx() # instantiate a second axes that shares the same x-axis\n", + "\n", + "ax2.set_ylabel('F1', color='tab:blue') # we already handled the x-label with ax1\n", + "ax2.plot(grouped_results_df['hidden_nodes_clusters'], grouped_results_df['f1'], color='tab:blue')\n", + "ax2.tick_params(axis='y', labelcolor='tab:blue')\n", + "\n", + "fig.tight_layout() # otherwise the right y-label is slightly clipped\n", + "plt.title('ecoli70 (continuous)')\n", + "plt.show()" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Group by dataset, data_type, and hidden_nodes_clusters and calculate the mean of each group\n", + "grouped_results_df = results_df_kmeans_lsevo[results_df_kmeans_lsevo['dataset']=='ecoli70'].groupby(['dataset', 'data_type', 'hidden_nodes_clusters'], as_index=False).mean()\n", + "\n", + "# Plot the results\n", + "fig, ax1 = plt.subplots()\n", + "ax1.set_xlabel('Number of Hidden Nodes Clusters')\n", + "ax1.set_ylabel('Time (s)', color='tab:red')\n", + "ax1.plot(grouped_results_df['hidden_nodes_clusters'], grouped_results_df['time'], color='tab:red')\n", + "ax1.tick_params(axis='y', labelcolor='tab:red')\n", + "\n", + "ax2 = ax1.twinx() # instantiate a second axes that shares the same x-axis\n", + "\n", + "ax2.set_ylabel('F1', color='tab:blue') # we already handled the x-label with ax1\n", + "ax2.plot(grouped_results_df['hidden_nodes_clusters'], grouped_results_df['f1'], color='tab:blue')\n", + "ax2.tick_params(axis='y', labelcolor='tab:blue')\n", + "\n", + "fig.tight_layout() # otherwise the right y-label is slightly clipped\n", + "plt.show()" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import itertools\n", + "\n", + "def generate_random_dag(nodes, num_edges):\n", + " all_edges = list(itertools.permutations(nodes, 2))\n", + " random_edges = np.random.choice(len(all_edges), num_edges, replace=False)\n", + " random_dag_edges = [all_edges[i] for i in random_edges]\n", + " return pd.DataFrame(random_dag_edges, columns=[\"source\", \"target\"])\n", + "\n", + "def calculate_f1_shd_random(networks):\n", + " \n", + " results_df = pd.DataFrame(columns=['dataset',\n", + " 'shd_random',\n", + " 'f1_random',\n", + " 'f1_undir_random'])\n", + " \n", + " for dataset_name, (reference_dag_edges) in networks.items():\n", + " \n", + " nodes = np.unique(reference_dag_edges).tolist() # Updated to convert to a list\n", + " num_edges = len(reference_dag_edges)\n", + " \n", + " random_dag = generate_random_dag(nodes, num_edges) # Updated to pass a list of nodes\n", + " \n", + " f1_random = f1_score(random_dag.values.tolist(), reference_dag_edges.values.tolist()) \n", + " f1_undir_random = precision_recall(random_dag.values.tolist(), reference_dag_edges.values.tolist())['F1_undir'] \n", + " shd_random = precision_recall(random_dag.values.tolist(), reference_dag_edges.values.tolist())['SHD']\n", + " \n", + " results_df = results_df.append({\n", + " 'dataset': f\"{dataset_name}\",\n", + " 'shd_random': shd_random,\n", + " 'f1_random': f1_random,\n", + " 'f1_undir_random': f1_undir_random\n", + " }, ignore_index=True)\n", + " \n", + " return results_df\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "datasets = {\n", + " \"ecoli70\": (ecoli70_true),\n", + " \"magic_irri\": (magic_irri_true),\n", + " \"magic_niab\": (magic_niab_true),\n", + " \"pigs\": (pigs_true),\n", + " \"win95pts\": (win95pts_true),\n", + " \"hailfinder\": (hailfinder_true),\n", + " \"hepar2\": (hepar2_true),\n", + " \"andes\": (andes_true),\n", + " \"diabetes\": (diabetes_true)\n", + "# \"arth150\": (arth150, arth150_true, \"continuous\", 8),\n", + "}\n", + "\n", + "random_generated_res = calculate_f1_shd_random(datasets)\n", + "\n", + "print(random_generated_res)" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import matplotlib\n", + "matplotlib.rcParams.update({'font.size': 16})\n", + "matplotlib.rcParams['pdf.fonttype'] = 42\n", + "matplotlib.rcParams['ps.fonttype'] = 42\n", + "\n", + "data_lsevo = {'dataset': ['ecoli70', 'hailfinder', 'hepar2', 'magic_irri', 'magic_niab', 'pigs', 'win95pts', 'andes', 'diabetes'],\n", + " 'shd_lsevo': [39.66, 63.33, 85.0, 77.333333, 54.666667, 490.0, 74.0, 297.33, 594.0],\n", + " 'f1_lsevo': [0.572215, 0.479836, 0.505263, 0.328704, 0.282016, 0.510259, 0.589862, 0.382214, 0.287690],\n", + " 'f1_undir_lsevo': [0.7732, 0.563233, 0.600000, 0.576600, 0.718400, 0.615500, 0.718900, 0.542433, 0.570167]}\n", + "\n", + "# Random DAGs data\n", + "data_random = {'dataset': ['ecoli70', 'magic_irri', 'magic_niab', 'pigs', 'win95pts', 'hailfinder', 'hepar2', 'andes', 'diabetes'],\n", + " 'shd_random': [134, 200, 125, 1179, 211, 129, 238, 669, 1200],\n", + " 'f1_random': [0.028571, 0.009804, 0.015152, 0.003378, 0.035714, 0.015152, 0.032520, 0.005917, 0.001661],\n", + " 'f1_undir_random': [0.0571, 0.0294, 0.0909, 0.0051, 0.0804, 0.0303, 0.0325, 0.0148, 0.0050]}\n", + "\n", + "# Combine the data into a single DataFrame\n", + "df_lsevo = pd.DataFrame(data_lsevo)\n", + "df_random = pd.DataFrame(data_random)\n", + "df_combined = pd.merge(df_lsevo, df_random, on='dataset')\n", + "\n", + "df_combined = df_combined[~df_combined['dataset'].isin(['ecoli70', 'magic_niab', 'magic_irri'])]\n", + "\n", + "# Plot the comparison\n", + "fig, axs = plt.subplots(1, 3, figsize=(20, 6))\n", + "\n", + "fig.suptitle('Comparison of SHD and F1 scores between LSevoBN and Random DAGs')\n", + "\n", + "# SHD comparison\n", + "sns.barplot(x='dataset', y='value', hue='type', data=df_combined.melt(id_vars='dataset', value_vars=['shd_lsevo', 'shd_random'], var_name='type', value_name='value'), ax=axs[0])\n", + "axs[0].set_title('SHD Comparison')\n", + "axs[0].set_xlabel('Dataset')\n", + "axs[0].set_ylabel('SHD')\n", + "axs[0].set_xticklabels(axs[0].get_xticklabels(), rotation=45)\n", + "\n", + "sns.barplot(x='dataset', y='value', hue='type', data=df_combined.melt(id_vars='dataset', value_vars=['f1_lsevo', 'f1_random'], var_name='type', value_name='value'), ax=axs[1])\n", + "axs[1].set_title('F1 Comparison')\n", + "axs[1].set_xlabel('Dataset')\n", + "axs[1].set_ylabel('F1')\n", + "axs[1].set_xticklabels(axs[1].get_xticklabels(), rotation=45)\n", + "\n", + "sns.barplot(x='dataset', y='value', hue='type', data=df_combined.melt(id_vars='dataset', value_vars=['f1_undir_lsevo', 'f1_undir_random'], var_name='type', value_name='value'), ax=axs[2])\n", + "axs[2].set_title('F1 Undirected Comparison')\n", + "axs[2].set_xlabel('Dataset')\n", + "axs[2].set_ylabel('F1 Undirected')\n", + "axs[2].set_xticklabels(axs[2].get_xticklabels(), rotation=45)" + ], + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "cell_type": "code", + "source": [ + "# import pandas as pd\n", + "# import numpy as np\n", + "# from pgmpy.estimators import HillClimbSearch, K2Score\n", + "# from sklearn.preprocessing import LabelEncoder, KBinsDiscretizer\n", + "# from sklearn.metrics import mutual_info_score\n", + "# import math\n", + "# import logging\n", + "# import time\n", + "# \n", + "# # Setup logger\n", + "# logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')\n", + "# logger = logging.getLogger()\n", + "# \n", + "# # Load datasets\n", + "# datasets = {\n", + "# \"pigs\": pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/pigs.csv'),\n", + "# \"win95pts\": pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/win95pts.csv'),\n", + "# \"hailfinder\": pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/hailfinder.csv'),\n", + "# \"hepar2\": pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/hepar2.csv'),\n", + "# \"arth150\": pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/arth150.csv'),\n", + "# \"ecoli70\": pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/ecoli70.csv', index_col=0),\n", + "# \"magic_irri\": pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/magic-irri.csv', index_col=0),\n", + "# \"magic_niab\": pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/magic-niab.csv', index_col=0),\n", + "# \"diabetes\": pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/sparsebndata/main/data/diabetes.csv'),\n", + "# \"andes\": pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/sparsebndata/main/data/andes.csv')\n", + "# }\n", + "# \n", + "# # Load true structures with headers V1 and V2 for the first eight datasets, and no headers for the last two\n", + "# true_structures = {\n", + "# \"pigs\": pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/pigs_true.csv', header=0),\n", + "# \"win95pts\": pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/win95pts_true.csv', header=0),\n", + "# \"hailfinder\": pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/hailfinder_true.csv', header=0),\n", + "# \"hepar2\": pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/hepar2_true.csv', header=0),\n", + "# \"arth150\": pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/arth150_true.csv', header=0),\n", + "# \"ecoli70\": pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/ecoli70_true.csv', header=0),\n", + "# \"magic_irri\": pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/magic-irri_true.csv', header=0),\n", + "# \"magic_niab\": pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/magic-niab_true.csv', header=0),\n", + "# \"diabetes\": pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/sparsebndata/main/truenetworks/diabetes_true.txt', header=None, names=['V1', 'V2']),\n", + "# \"andes\": pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/sparsebndata/main/truenetworks/andes_true.txt', header=None, names=['V1', 'V2'])\n", + "# }\n", + "# \n", + "# # BigBraveBN Class\n", + "# class BigBraveBN:\n", + "# def __init__(self):\n", + "# self.possible_edges = []\n", + "# \n", + "# def set_possible_edges_by_brave(\n", + "# self,\n", + "# df: pd.DataFrame,\n", + "# n_nearest: int = 5,\n", + "# threshold: float = 0.3,\n", + "# proximity_metric: str = \"MI\",\n", + "# ) -> list:\n", + "# df_copy = df.copy(deep=True)\n", + "# proximity_matrix = self._get_proximity_matrix(df_copy, proximity_metric)\n", + "# brave_matrix = self._get_brave_matrix(df_copy.columns, proximity_matrix, n_nearest)\n", + "# \n", + "# threshold_value = brave_matrix.max(numeric_only=True).max() * threshold\n", + "# filtered_brave_matrix = brave_matrix[brave_matrix > threshold_value].stack()\n", + "# self.possible_edges = filtered_brave_matrix.index.tolist()\n", + "# return self.possible_edges\n", + "# \n", + "# @staticmethod\n", + "# def _get_n_nearest(data: pd.DataFrame, columns: list, corr: bool = False, number_close: int = 5) -> list:\n", + "# groups = []\n", + "# for c in columns:\n", + "# close_ind = data[c].sort_values(ascending=not corr).index.tolist()\n", + "# groups.append(close_ind[: number_close + 1])\n", + "# return groups\n", + "# \n", + "# @staticmethod\n", + "# def _get_proximity_matrix(df: pd.DataFrame, proximity_metric: str) -> pd.DataFrame:\n", + "# encoder = OrdinalEncoder()\n", + "# df_coded = df.copy()\n", + "# columns_to_encode = list(df_coded.select_dtypes(include=[\"category\", \"object\"]))\n", + "# df_coded[columns_to_encode] = encoder.fit_transform(df_coded[columns_to_encode])\n", + "# \n", + "# if proximity_metric == \"MI\":\n", + "# df_distance = pd.DataFrame(\n", + "# np.zeros((len(df.columns), len(df.columns))),\n", + "# columns=df.columns,\n", + "# index=df.columns,\n", + "# )\n", + "# for c1 in df.columns:\n", + "# for c2 in df.columns:\n", + "# dist = mutual_info_score(df_coded[c1].values, df_coded[c2].values)\n", + "# df_distance.loc[c1, c2] = dist\n", + "# return df_distance\n", + "# \n", + "# elif proximity_metric == \"pearson\":\n", + "# return df_coded.corr(method=\"pearson\")\n", + "# \n", + "# def _get_brave_matrix(self, df_columns: pd.Index, proximity_matrix: pd.DataFrame, n_nearest: int = 5) -> pd.DataFrame:\n", + "# brave_matrix = pd.DataFrame(\n", + "# np.zeros((len(df_columns), len(df_columns))),\n", + "# columns=df_columns,\n", + "# index=df_columns,\n", + "# )\n", + "# groups = self._get_n_nearest(proximity_matrix, df_columns.tolist(), corr=True, number_close=n_nearest)\n", + "# \n", + "# for c1 in df_columns:\n", + "# for c2 in df_columns:\n", + "# a = b = c = d = 0.0\n", + "# if c1 != c2:\n", + "# for g in groups:\n", + "# a += (c1 in g) & (c2 in g)\n", + "# b += (c1 in g) & (c2 not in g)\n", + "# c += (c1 not in g) & (c2 in g)\n", + "# d += (c1 not in g) & (c2 not in g)\n", + "# \n", + "# divisor = (math.sqrt((a + c) * (b + d))) * (math.sqrt((a + b) * (c + d)))\n", + "# br = (a * len(groups) + (a + c) * (a + b)) / (divisor if divisor != 0 else 0.0000000001)\n", + "# brave_matrix.loc[c1, c2] = br\n", + "# \n", + "# return brave_matrix\n", + "# \n", + "# def preprocess_data(self, data_cluster):\n", + "# encoder = LabelEncoder()\n", + "# discretizer = KBinsDiscretizer(n_bins=5, encode='ordinal', strategy='quantile')\n", + "# for col in data_cluster.columns:\n", + "# if data_cluster[col].dtype == 'object':\n", + "# data_cluster[col] = encoder.fit_transform(data_cluster[col])\n", + "# discretized_data = pd.DataFrame(discretizer.fit_transform(data_cluster), columns=data_cluster.columns)\n", + "# info = {col: 'discrete' for col in data_cluster.columns}\n", + "# return discretized_data, info\n", + "# \n", + "# # Evaluation Functions\n", + "# def f1_score(custom_dag_edges, reference_dag_edges):\n", + "# tp = fp = fn = 0\n", + "# for edge in custom_dag_edges:\n", + "# if edge in reference_dag_edges:\n", + "# tp += 1\n", + "# else:\n", + "# fp += 1\n", + "# \n", + "# for edge in reference_dag_edges:\n", + "# if edge not in custom_dag_edges:\n", + "# fn += 1\n", + "# \n", + "# if tp + fp == 0:\n", + "# precision = 0\n", + "# else:\n", + "# precision = tp / (tp + fp)\n", + "# \n", + "# if tp + fn == 0:\n", + "# recall = 0\n", + "# else:\n", + "# recall = tp / (tp + fn)\n", + "# \n", + "# if precision + recall == 0:\n", + "# f1 = 0\n", + "# else:\n", + "# f1 = 2 * precision * recall / (precision + recall)\n", + "# \n", + "# return f1\n", + "# \n", + "# def child_dict(net: list):\n", + "# res_dict = dict()\n", + "# for e0, e1 in net:\n", + "# if e1 in res_dict:\n", + "# res_dict[e1].append(e0)\n", + "# else:\n", + "# res_dict[e1] = [e0]\n", + "# return res_dict\n", + "# \n", + "# def precision_recall(pred_net: list, true_net: list, decimal = 4):\n", + "# pred_dict = child_dict(pred_net)\n", + "# true_dict = child_dict(true_net)\n", + "# corr_undir = 0\n", + "# corr_dir = 0\n", + "# for e0, e1 in pred_net:\n", + "# flag = True\n", + "# if e1 in true_dict:\n", + "# if e0 in true_dict[e1]:\n", + "# corr_undir += 1\n", + "# corr_dir += 1\n", + "# flag = False\n", + "# if (e0 in true_dict) and flag:\n", + "# if e1 in true_dict[e0]:\n", + "# corr_undir += 1\n", + "# pred_len = len(pred_net)\n", + "# true_len = len(true_net)\n", + "# shd = pred_len + true_len - corr_undir - corr_dir\n", + "# return {'SHD': shd}\n", + "# \n", + "# def f1_score_undirected(custom_dag_edges, reference_dag_edges):\n", + "# custom_dag_edges_set = {frozenset((str(a), str(b))) for a, b in custom_dag_edges}\n", + "# reference_dag_edges_set = {frozenset((str(a), str(b))) for a, b in reference_dag_edges}\n", + "# \n", + "# tp = fp = fn = 0\n", + "# for edge in custom_dag_edges_set:\n", + "# if edge in reference_dag_edges_set:\n", + "# tp += 1\n", + "# else:\n", + "# fp += 1\n", + "# \n", + "# for edge in reference_dag_edges_set:\n", + "# if edge not in custom_dag_edges_set:\n", + "# fn += 1\n", + "# \n", + "# if tp + fp == 0:\n", + "# precision = 0\n", + "# else:\n", + "# precision = tp / (tp + fp)\n", + "# \n", + "# if tp + fn == 0:\n", + "# recall = 0\n", + "# else:\n", + "# recall = tp / (tp + fn)\n", + "# \n", + "# if precision + recall == 0:\n", + "# f1 = 0\n", + "# else:\n", + "# f1 = 2 * precision * recall / (precision + recall)\n", + "# \n", + "# return f1\n", + "# \n", + "# # Main Evaluation Loop\n", + "# def evaluate_bn_structure_learning(datasets, true_structures):\n", + "# results = {\n", + "# \"dataset\": [],\n", + "# \"data_type\": [],\n", + "# \"shd_lsevo\": [],\n", + "# \"f1_lsevo\": [],\n", + "# \"time_lsevo\": [],\n", + "# \"f1_undir_lsevo\": [],\n", + "# }\n", + "# \n", + "# thresholds = {\n", + "# \"pigs\": 0.5,\n", + "# \"win95pts\": 0.3,\n", + "# \"hailfinder\": 0.3,\n", + "# \"hepar2\": 0.3,\n", + "# \"arth150\": 0.3,\n", + "# \"ecoli70\": 0.3,\n", + "# \"magic_irri\": 0.3,\n", + "# \"magic_niab\": 0.3,\n", + "# \"diabetes\": 0.6,\n", + "# \"andes\": 0.5\n", + "# }\n", + "# \n", + "# data_types = {\n", + "# \"pigs\": \"discrete\",\n", + "# \"win95pts\": \"discrete\",\n", + "# \"hailfinder\": \"discrete\",\n", + "# \"hepar2\": \"discrete\",\n", + "# \"arth150\": \"discrete\",\n", + "# \"ecoli70\": \"continuous\",\n", + "# \"magic_irri\": \"continuous\",\n", + "# \"magic_niab\": \"continuous\",\n", + "# \"diabetes\": \"discrete\",\n", + "# \"andes\": \"discrete\"\n", + "# }\n", + "# \n", + "# for name, data in datasets.items():\n", + "# logger.info(f\"Starting evaluation for dataset: {name}\")\n", + "# \n", + "# true_structure = true_structures[name]\n", + "# true_edges = list(zip(true_structure['V1'], true_structure['V2']))\n", + "# \n", + "# brave_bn = BigBraveBN()\n", + "# metrics = {\"F1\": [], \"F1_undirected\": [], \"SHD\": [], \"Time\": []}\n", + "# \n", + "# for i in range(5):\n", + "# logger.info(f\"Run {i+1} for dataset: {name}\")\n", + "# start_time = time.time()\n", + "# \n", + "# preprocessed_data, _ = brave_bn.preprocess_data(data)\n", + "# possible_edges = brave_bn.set_possible_edges_by_brave(preprocessed_data, threshold=thresholds[name])\n", + "# hc = HillClimbSearch(preprocessed_data)\n", + "# model = hc.estimate(scoring_method=K2Score(preprocessed_data), white_list=possible_edges)\n", + "# \n", + "# end_time = time.time()\n", + "# elapsed_time = end_time - start_time\n", + "# \n", + "# predicted_edges = model.edges()\n", + "# \n", + "# metrics[\"F1\"].append(f1_score(predicted_edges, true_edges))\n", + "# metrics[\"F1_undirected\"].append(f1_score_undirected(predicted_edges, true_edges))\n", + "# metrics[\"SHD\"].append(precision_recall(predicted_edges, true_edges)['SHD'])\n", + "# metrics[\"Time\"].append(elapsed_time)\n", + "# \n", + "# results[\"dataset\"].append(name)\n", + "# results[\"data_type\"].append(data_types[name])\n", + "# results[\"shd_lsevo\"].append(np.mean(metrics[\"SHD\"]))\n", + "# results[\"f1_lsevo\"].append(np.mean(metrics[\"F1\"]))\n", + "# results[\"time_lsevo\"].append(np.mean(metrics[\"Time\"]))\n", + "# results[\"f1_undir_lsevo\"].append(np.mean(metrics[\"F1_undirected\"]))\n", + "# \n", + "# logger.info(f\"Completed evaluation for dataset: {name} with Average Time: {results['time_lsevo'][-1]:.2f} seconds\")\n", + "# \n", + "# return results\n", + "# \n", + "# # Run the evaluation\n", + "# results = evaluate_bn_structure_learning(datasets, true_structures)\n", + "# results_df = pd.DataFrame(results)\n", + "# print(results_df)\n" + ], + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "cell_type": "code", + "source": [ + "import numpy as np\n", + "import scipy.linalg as slin\n", + "import scipy.optimize as sopt\n", + "from scipy.special import expit as sigmoid\n", + "\n", + "\n", + "def notears_linear(X, lambda1, loss_type, max_iter=100, h_tol=1e-8, rho_max=1e+16, w_threshold=0.3):\n", + " \"\"\"Solve min_W L(W; X) + lambda1 ‖W‖_1 s.t. h(W) = 0 using augmented Lagrangian.\n", + "\n", + " Args:\n", + " X (np.ndarray): [n, d] sample matrix\n", + " lambda1 (float): l1 penalty parameter\n", + " loss_type (str): l2, logistic, poisson\n", + " max_iter (int): max num of dual ascent steps\n", + " h_tol (float): exit if |h(w_est)| <= htol\n", + " rho_max (float): exit if rho >= rho_max\n", + " w_threshold (float): drop edge if |weight| < threshold\n", + "\n", + " Returns:\n", + " W_est (np.ndarray): [d, d] estimated DAG\n", + " \"\"\"\n", + " def _loss(W):\n", + " \"\"\"Evaluate value and gradient of loss.\"\"\"\n", + " M = X @ W\n", + " if loss_type == 'l2':\n", + " R = X - M\n", + " loss = 0.5 / X.shape[0] * (R ** 2).sum()\n", + " G_loss = - 1.0 / X.shape[0] * X.T @ R\n", + " elif loss_type == 'logistic':\n", + " loss = 1.0 / X.shape[0] * (np.logaddexp(0, M) - X * M).sum()\n", + " G_loss = 1.0 / X.shape[0] * X.T @ (sigmoid(M) - X)\n", + " elif loss_type == 'poisson':\n", + " S = np.exp(M)\n", + " loss = 1.0 / X.shape[0] * (S - X * M).sum()\n", + " G_loss = 1.0 / X.shape[0] * X.T @ (S - X)\n", + " else:\n", + " raise ValueError('unknown loss type')\n", + " return loss, G_loss\n", + "\n", + " def _h(W):\n", + " \"\"\"Evaluate value and gradient of acyclicity constraint.\"\"\"\n", + " E = slin.expm(W * W) # (Zheng et al. 2018)\n", + " h = np.trace(E) - d\n", + " # # A different formulation, slightly faster at the cost of numerical stability\n", + " # M = np.eye(d) + W * W / d # (Yu et al. 2019)\n", + " # E = np.linalg.matrix_power(M, d - 1)\n", + " # h = (E.T * M).sum() - d\n", + " G_h = E.T * W * 2\n", + " return h, G_h\n", + "\n", + " def _adj(w):\n", + " \"\"\"Convert doubled variables ([2 d^2] array) back to original variables ([d, d] matrix).\"\"\"\n", + " return (w[:d * d] - w[d * d:]).reshape([d, d])\n", + "\n", + " def _func(w):\n", + " \"\"\"Evaluate value and gradient of augmented Lagrangian for doubled variables ([2 d^2] array).\"\"\"\n", + " W = _adj(w)\n", + " loss, G_loss = _loss(W)\n", + " h, G_h = _h(W)\n", + " obj = loss + 0.5 * rho * h * h + alpha * h + lambda1 * w.sum()\n", + " G_smooth = G_loss + (rho * h + alpha) * G_h\n", + " g_obj = np.concatenate((G_smooth + lambda1, - G_smooth + lambda1), axis=None)\n", + " return obj, g_obj\n", + "\n", + " n, d = X.shape\n", + " w_est, rho, alpha, h = np.zeros(2 * d * d), 1.0, 0.0, np.inf # double w_est into (w_pos, w_neg)\n", + " bnds = [(0, 0) if i == j else (0, None) for _ in range(2) for i in range(d) for j in range(d)]\n", + " if loss_type == 'l2':\n", + " X = X - np.mean(X, axis=0, keepdims=True)\n", + " for _ in range(max_iter):\n", + " w_new, h_new = None, None\n", + " while rho < rho_max:\n", + " sol = sopt.minimize(_func, w_est, method='L-BFGS-B', jac=True, bounds=bnds)\n", + " w_new = sol.x\n", + " h_new, _ = _h(_adj(w_new))\n", + " if h_new > 0.25 * h:\n", + " rho *= 10\n", + " else:\n", + " break\n", + " w_est, h = w_new, h_new\n", + " alpha += rho * h\n", + " if h <= h_tol or rho >= rho_max:\n", + " break\n", + " W_est = _adj(w_est)\n", + " W_est[np.abs(W_est) < w_threshold] = 0\n", + " return W_est\n", + "\n", + "from scipy.special import expit as sigmoid\n", + "import igraph as ig\n", + "import random\n", + "\n", + "\n", + "def set_random_seed(seed):\n", + " random.seed(seed)\n", + " np.random.seed(seed)\n", + "\n", + "\n", + "def is_dag(W):\n", + " G = ig.Graph.Weighted_Adjacency(W.tolist())\n", + " return G.is_dag()\n", + "\n", + "\n", + "def simulate_dag(d, s0, graph_type):\n", + " \"\"\"Simulate random DAG with some expected number of edges.\n", + "\n", + " Args:\n", + " d (int): num of nodes\n", + " s0 (int): expected num of edges\n", + " graph_type (str): ER, SF, BP\n", + "\n", + " Returns:\n", + " B (np.ndarray): [d, d] binary adj matrix of DAG\n", + " \"\"\"\n", + " def _random_permutation(M):\n", + " # np.random.permutation permutes first axis only\n", + " P = np.random.permutation(np.eye(M.shape[0]))\n", + " return P.T @ M @ P\n", + "\n", + " def _random_acyclic_orientation(B_und):\n", + " return np.tril(_random_permutation(B_und), k=-1)\n", + "\n", + " def _graph_to_adjmat(G):\n", + " return np.array(G.get_adjacency().data)\n", + "\n", + " if graph_type == 'ER':\n", + " # Erdos-Renyi\n", + " G_und = ig.Graph.Erdos_Renyi(n=d, m=s0)\n", + " B_und = _graph_to_adjmat(G_und)\n", + " B = _random_acyclic_orientation(B_und)\n", + " elif graph_type == 'SF':\n", + " # Scale-free, Barabasi-Albert\n", + " G = ig.Graph.Barabasi(n=d, m=int(round(s0 / d)), directed=True)\n", + " B = _graph_to_adjmat(G)\n", + " elif graph_type == 'BP':\n", + " # Bipartite, Sec 4.1 of (Gu, Fu, Zhou, 2018)\n", + " top = int(0.2 * d)\n", + " G = ig.Graph.Random_Bipartite(top, d - top, m=s0, directed=True, neimode=ig.OUT)\n", + " B = _graph_to_adjmat(G)\n", + " else:\n", + " raise ValueError('unknown graph type')\n", + " B_perm = _random_permutation(B)\n", + " assert ig.Graph.Adjacency(B_perm.tolist()).is_dag()\n", + " return B_perm\n", + "\n", + "\n", + "def simulate_parameter(B, w_ranges=((-2.0, -0.5), (0.5, 2.0))):\n", + " \"\"\"Simulate SEM parameters for a DAG.\n", + "\n", + " Args:\n", + " B (np.ndarray): [d, d] binary adj matrix of DAG\n", + " w_ranges (tuple): disjoint weight ranges\n", + "\n", + " Returns:\n", + " W (np.ndarray): [d, d] weighted adj matrix of DAG\n", + " \"\"\"\n", + " W = np.zeros(B.shape)\n", + " S = np.random.randint(len(w_ranges), size=B.shape) # which range\n", + " for i, (low, high) in enumerate(w_ranges):\n", + " U = np.random.uniform(low=low, high=high, size=B.shape)\n", + " W += B * (S == i) * U\n", + " return W\n", + "\n", + "\n", + "def simulate_linear_sem(W, n, sem_type, noise_scale=None):\n", + " \"\"\"Simulate samples from linear SEM with specified type of noise.\n", + "\n", + " For uniform, noise z ~ uniform(-a, a), where a = noise_scale.\n", + "\n", + " Args:\n", + " W (np.ndarray): [d, d] weighted adj matrix of DAG\n", + " n (int): num of samples, n=inf mimics population risk\n", + " sem_type (str): gauss, exp, gumbel, uniform, logistic, poisson\n", + " noise_scale (np.ndarray): scale parameter of additive noise, default all ones\n", + "\n", + " Returns:\n", + " X (np.ndarray): [n, d] sample matrix, [d, d] if n=inf\n", + " \"\"\"\n", + " def _simulate_single_equation(X, w, scale):\n", + " \"\"\"X: [n, num of parents], w: [num of parents], x: [n]\"\"\"\n", + " if sem_type == 'gauss':\n", + " z = np.random.normal(scale=scale, size=n)\n", + " x = X @ w + z\n", + " elif sem_type == 'exp':\n", + " z = np.random.exponential(scale=scale, size=n)\n", + " x = X @ w + z\n", + " elif sem_type == 'gumbel':\n", + " z = np.random.gumbel(scale=scale, size=n)\n", + " x = X @ w + z\n", + " elif sem_type == 'uniform':\n", + " z = np.random.uniform(low=-scale, high=scale, size=n)\n", + " x = X @ w + z\n", + " elif sem_type == 'logistic':\n", + " x = np.random.binomial(1, sigmoid(X @ w)) * 1.0\n", + " elif sem_type == 'poisson':\n", + " x = np.random.poisson(np.exp(X @ w)) * 1.0\n", + " else:\n", + " raise ValueError('unknown sem type')\n", + " return x\n", + "\n", + " d = W.shape[0]\n", + " if noise_scale is None:\n", + " scale_vec = np.ones(d)\n", + " elif np.isscalar(noise_scale):\n", + " scale_vec = noise_scale * np.ones(d)\n", + " else:\n", + " if len(noise_scale) != d:\n", + " raise ValueError('noise scale must be a scalar or has length d')\n", + " scale_vec = noise_scale\n", + " if not is_dag(W):\n", + " raise ValueError('W must be a DAG')\n", + " if np.isinf(n): # population risk for linear gauss SEM\n", + " if sem_type == 'gauss':\n", + " # make 1/d X'X = true cov\n", + " X = np.sqrt(d) * np.diag(scale_vec) @ np.linalg.inv(np.eye(d) - W)\n", + " return X\n", + " else:\n", + " raise ValueError('population risk not available')\n", + " # empirical risk\n", + " G = ig.Graph.Weighted_Adjacency(W.tolist())\n", + " ordered_vertices = G.topological_sorting()\n", + " assert len(ordered_vertices) == d\n", + " X = np.zeros([n, d])\n", + " for j in ordered_vertices:\n", + " parents = G.neighbors(j, mode=ig.IN)\n", + " X[:, j] = _simulate_single_equation(X[:, parents], W[parents, j], scale_vec[j])\n", + " return X\n", + "\n", + "\n", + "def simulate_nonlinear_sem(B, n, sem_type, noise_scale=None):\n", + " \"\"\"Simulate samples from nonlinear SEM.\n", + "\n", + " Args:\n", + " B (np.ndarray): [d, d] binary adj matrix of DAG\n", + " n (int): num of samples\n", + " sem_type (str): mlp, mim, gp, gp-add\n", + " noise_scale (np.ndarray): scale parameter of additive noise, default all ones\n", + "\n", + " Returns:\n", + " X (np.ndarray): [n, d] sample matrix\n", + " \"\"\"\n", + " def _simulate_single_equation(X, scale):\n", + " \"\"\"X: [n, num of parents], x: [n]\"\"\"\n", + " z = np.random.normal(scale=scale, size=n)\n", + " pa_size = X.shape[1]\n", + " if pa_size == 0:\n", + " return z\n", + " if sem_type == 'mlp':\n", + " hidden = 100\n", + " W1 = np.random.uniform(low=0.5, high=2.0, size=[pa_size, hidden])\n", + " W1[np.random.rand(*W1.shape) < 0.5] *= -1\n", + " W2 = np.random.uniform(low=0.5, high=2.0, size=hidden)\n", + " W2[np.random.rand(hidden) < 0.5] *= -1\n", + " x = sigmoid(X @ W1) @ W2 + z\n", + " elif sem_type == 'mim':\n", + " w1 = np.random.uniform(low=0.5, high=2.0, size=pa_size)\n", + " w1[np.random.rand(pa_size) < 0.5] *= -1\n", + " w2 = np.random.uniform(low=0.5, high=2.0, size=pa_size)\n", + " w2[np.random.rand(pa_size) < 0.5] *= -1\n", + " w3 = np.random.uniform(low=0.5, high=2.0, size=pa_size)\n", + " w3[np.random.rand(pa_size) < 0.5] *= -1\n", + " x = np.tanh(X @ w1) + np.cos(X @ w2) + np.sin(X @ w3) + z\n", + " elif sem_type == 'gp':\n", + " from sklearn.gaussian_process import GaussianProcessRegressor\n", + " gp = GaussianProcessRegressor()\n", + " x = gp.sample_y(X, random_state=None).flatten() + z\n", + " elif sem_type == 'gp-add':\n", + " from sklearn.gaussian_process import GaussianProcessRegressor\n", + " gp = GaussianProcessRegressor()\n", + " x = sum([gp.sample_y(X[:, i, None], random_state=None).flatten()\n", + " for i in range(X.shape[1])]) + z\n", + " else:\n", + " raise ValueError('unknown sem type')\n", + " return x\n", + "\n", + " d = B.shape[0]\n", + " scale_vec = noise_scale if noise_scale else np.ones(d)\n", + " X = np.zeros([n, d])\n", + " G = ig.Graph.Adjacency(B.tolist())\n", + " ordered_vertices = G.topological_sorting()\n", + " assert len(ordered_vertices) == d\n", + " for j in ordered_vertices:\n", + " parents = G.neighbors(j, mode=ig.IN)\n", + " X[:, j] = _simulate_single_equation(X[:, parents], scale_vec[j])\n", + " return X\n", + "\n", + "\n", + "def count_accuracy(B_true, B_est):\n", + " \"\"\"Compute various accuracy metrics for B_est.\n", + "\n", + " true positive = predicted association exists in condition in correct direction\n", + " reverse = predicted association exists in condition in opposite direction\n", + " false positive = predicted association does not exist in condition\n", + "\n", + " Args:\n", + " B_true (np.ndarray): [d, d] ground truth graph, {0, 1}\n", + " B_est (np.ndarray): [d, d] estimate, {0, 1, -1}, -1 is undirected edge in CPDAG\n", + "\n", + " Returns:\n", + " fdr: (reverse + false positive) / prediction positive\n", + " tpr: (true positive) / condition positive\n", + " fpr: (reverse + false positive) / condition negative\n", + " shd: undirected extra + undirected missing + reverse\n", + " nnz: prediction positive\n", + " \"\"\"\n", + " if (B_est == -1).any(): # cpdag\n", + " if not ((B_est == 0) | (B_est == 1) | (B_est == -1)).all():\n", + " raise ValueError('B_est should take value in {0,1,-1}')\n", + " if ((B_est == -1) & (B_est.T == -1)).any():\n", + " raise ValueError('undirected edge should only appear once')\n", + " else: # dag\n", + " if not ((B_est == 0) | (B_est == 1)).all():\n", + " raise ValueError('B_est should take value in {0,1}')\n", + " if not is_dag(B_est):\n", + " raise ValueError('B_est should be a DAG')\n", + " d = B_true.shape[0]\n", + " # linear index of nonzeros\n", + " pred_und = np.flatnonzero(B_est == -1)\n", + " pred = np.flatnonzero(B_est == 1)\n", + " cond = np.flatnonzero(B_true)\n", + " cond_reversed = np.flatnonzero(B_true.T)\n", + " cond_skeleton = np.concatenate([cond, cond_reversed])\n", + " # true pos\n", + " true_pos = np.intersect1d(pred, cond, assume_unique=True)\n", + " # treat undirected edge favorably\n", + " true_pos_und = np.intersect1d(pred_und, cond_skeleton, assume_unique=True)\n", + " true_pos = np.concatenate([true_pos, true_pos_und])\n", + " # false pos\n", + " false_pos = np.setdiff1d(pred, cond_skeleton, assume_unique=True)\n", + " false_pos_und = np.setdiff1d(pred_und, cond_skeleton, assume_unique=True)\n", + " false_pos = np.concatenate([false_pos, false_pos_und])\n", + " # reverse\n", + " extra = np.setdiff1d(pred, cond, assume_unique=True)\n", + " reverse = np.intersect1d(extra, cond_reversed, assume_unique=True)\n", + " # compute ratio\n", + " pred_size = len(pred) + len(pred_und)\n", + " cond_neg_size = 0.5 * d * (d - 1) - len(cond)\n", + " fdr = float(len(reverse) + len(false_pos)) / max(pred_size, 1)\n", + " tpr = float(len(true_pos)) / max(len(cond), 1)\n", + " fpr = float(len(reverse) + len(false_pos)) / max(cond_neg_size, 1)\n", + " # structural hamming distance\n", + " pred_lower = np.flatnonzero(np.tril(B_est + B_est.T))\n", + " cond_lower = np.flatnonzero(np.tril(B_true + B_true.T))\n", + " extra_lower = np.setdiff1d(pred_lower, cond_lower, assume_unique=True)\n", + " missing_lower = np.setdiff1d(cond_lower, pred_lower, assume_unique=True)\n", + " shd = len(extra_lower) + len(missing_lower) + len(reverse)\n", + " return {'fdr': fdr, 'tpr': tpr, 'fpr': fpr, 'shd': shd, 'nnz': pred_size}\n" + ], + "outputs": [], + "execution_count": null + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2024-06-19T13:45:38.888500Z", + "start_time": "2024-06-19T13:44:33.840106Z" + } + }, + "cell_type": "code", + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "from pgmpy.estimators import HillClimbSearch, K2Score\n", + "from sklearn.preprocessing import LabelEncoder, KBinsDiscretizer, OrdinalEncoder\n", + "from sklearn.metrics import mutual_info_score\n", + "import math\n", + "import logging\n", + "import time\n", + "import torch\n", + "from dagma import utils as dagma_utils\n", + "from dagma.linear import DagmaLinear\n", + "\n", + "# Setup logger\n", + "logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')\n", + "logger = logging.getLogger()\n", + "\n", + "# Load datasets\n", + "datasets = {\n", + " # \"pigs\": pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/pigs.csv'),\n", + " # \"win95pts\": pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/win95pts.csv'),\n", + " # \"hailfinder\": pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/hailfinder.csv'),\n", + " # \"hepar2\": pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/hepar2.csv'),\n", + " # \"arth150\": pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/arth150.csv'),\n", + " # \"ecoli70\": pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/ecoli70.csv', index_col=0),\n", + " # \"magic_irri\": pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/magic-irri.csv', index_col=0),\n", + " # \"magic_niab\": pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/magic-niab.csv', index_col=0),\n", + " # \"diabetes\": pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/sparsebndata/main/data/diabetes.csv'),\n", + " \"andes\": pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/sparsebndata/main/data/andes.csv')\n", + "}\n", + "\n", + "# Load true structures with headers V1 and V2 for the first eight datasets, and no headers for the last two\n", + "true_structures = {\n", + " # \"pigs\": pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/pigs_true.csv', header=0),\n", + " # \"win95pts\": pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/win95pts_true.csv', header=0),\n", + " # \"hailfinder\": pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/hailfinder_true.csv', header=0),\n", + " # \"hepar2\": pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/hepar2_true.csv', header=0),\n", + " # \"arth150\": pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/arth150_true.csv', header=0),\n", + " # \"ecoli70\": pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/ecoli70_true.csv', header=0),\n", + " # \"magic_irri\": pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/magic-irri_true.csv', header=0),\n", + " # \"magic_niab\": pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/BAMT-old/main/data/magic-niab_true.csv', header=0),\n", + " # \"diabetes\": pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/sparsebndata/main/truenetworks/diabetes_true.txt', header=None, names=['V1', 'V2']),\n", + " \"andes\": pd.read_csv('https://raw.githubusercontent.com/jrzkaminski/sparsebndata/main/truenetworks/andes_true.txt', header=None, names=['V1', 'V2'])\n", + "}\n", + "\n", + "# BigBraveBN Class\n", + "class BigBraveBN:\n", + " def __init__(self):\n", + " self.possible_edges = []\n", + "\n", + " def set_possible_edges_by_brave(\n", + " self,\n", + " df: pd.DataFrame,\n", + " n_nearest: int = 5,\n", + " threshold: float = 0.3,\n", + " proximity_metric: str = \"MI\",\n", + " ) -> list:\n", + " df_copy = df.copy(deep=True)\n", + " proximity_matrix = self._get_proximity_matrix(df_copy, proximity_metric)\n", + " brave_matrix = self._get_brave_matrix(df_copy.columns, proximity_matrix, n_nearest)\n", + "\n", + " threshold_value = brave_matrix.max(numeric_only=True).max() * threshold\n", + " filtered_brave_matrix = brave_matrix[brave_matrix > threshold_value].stack()\n", + " self.possible_edges = filtered_brave_matrix.index.tolist()\n", + " return self.possible_edges\n", + "\n", + " @staticmethod\n", + " def _get_n_nearest(data: pd.DataFrame, columns: list, corr: bool = False, number_close: int = 5) -> list:\n", + " groups = []\n", + " for c in columns:\n", + " close_ind = data[c].sort_values(ascending=not corr).index.tolist()\n", + " groups.append(close_ind[: number_close + 1])\n", + " return groups\n", + "\n", + " @staticmethod\n", + " def _get_proximity_matrix(df: pd.DataFrame, proximity_metric: str) -> pd.DataFrame:\n", + " encoder = OrdinalEncoder()\n", + " df_coded = df.copy()\n", + " columns_to_encode = list(df_coded.select_dtypes(include=[\"category\", \"object\"]))\n", + " df_coded[columns_to_encode] = encoder.fit_transform(df_coded[columns_to_encode])\n", + "\n", + " if proximity_metric == \"MI\":\n", + " df_distance = pd.DataFrame(\n", + " np.zeros((len(df.columns), len(df.columns))),\n", + " columns=df.columns,\n", + " index=df.columns,\n", + " )\n", + " for c1 in df.columns:\n", + " for c2 in df.columns:\n", + " dist = mutual_info_score(df_coded[c1].values, df_coded[c2].values)\n", + " df_distance.loc[c1, c2] = dist\n", + " return df_distance\n", + "\n", + " elif proximity_metric == \"pearson\":\n", + " return df_coded.corr(method=\"pearson\")\n", + "\n", + " def _get_brave_matrix(self, df_columns: pd.Index, proximity_matrix: pd.DataFrame, n_nearest: int = 5) -> pd.DataFrame:\n", + " brave_matrix = pd.DataFrame(\n", + " np.zeros((len(df_columns), len(df_columns))),\n", + " columns=df_columns,\n", + " index=df_columns,\n", + " )\n", + " groups = self._get_n_nearest(proximity_matrix, df_columns.tolist(), corr=True, number_close=n_nearest)\n", + "\n", + " for c1 in df_columns:\n", + " for c2 in df_columns:\n", + " a = b = c = d = 0.0\n", + " if c1 != c2:\n", + " for g in groups:\n", + " a += (c1 in g) & (c2 in g)\n", + " b += (c1 in g) & (c2 not in g)\n", + " c += (c1 not in g) & (c2 in g)\n", + " d += (c1 not in g) & (c2 not in g)\n", + "\n", + " divisor = (math.sqrt((a + c) * (b + d))) * (math.sqrt((a + b) * (c + d)))\n", + " br = (a * len(groups) + (a + c) * (a + b)) / (divisor if divisor != 0 else 0.0000000001)\n", + " brave_matrix.loc[c1, c2] = br\n", + "\n", + " return brave_matrix\n", + "\n", + " def preprocess_data(self, data_cluster):\n", + " encoder = LabelEncoder()\n", + " discretizer = KBinsDiscretizer(n_bins=5, encode='ordinal', strategy='quantile')\n", + " for col in data_cluster.columns:\n", + " if data_cluster[col].dtype == 'object':\n", + " data_cluster[col] = encoder.fit_transform(data_cluster[col])\n", + " discretized_data = pd.DataFrame(discretizer.fit_transform(data_cluster), columns=data_cluster.columns)\n", + " info = {col: 'discrete' for col in data_cluster.columns}\n", + " return discretized_data, info\n", + "\n", + "# Evaluation Functions\n", + "def f1_score(custom_dag_edges, reference_dag_edges):\n", + " tp = fp = fn = 0\n", + " for edge in custom_dag_edges:\n", + " if edge in reference_dag_edges:\n", + " tp += 1\n", + " else:\n", + " fp += 1\n", + "\n", + " for edge in reference_dag_edges:\n", + " if edge not in custom_dag_edges:\n", + " fn += 1\n", + "\n", + " if tp + fp == 0:\n", + " precision = 0\n", + " else:\n", + " precision = tp / (tp + fp)\n", + "\n", + " if tp + fn == 0:\n", + " recall = 0\n", + " else:\n", + " recall = tp / (tp + fn)\n", + "\n", + " if precision + recall == 0:\n", + " f1 = 0\n", + " else:\n", + " f1 = 2 * precision * recall / (precision + recall)\n", + "\n", + " return f1\n", + "\n", + "def child_dict(net: list):\n", + " res_dict = dict()\n", + " for e0, e1 in net:\n", + " if e1 in res_dict:\n", + " res_dict[e1].append(e0)\n", + " else:\n", + " res_dict[e1] = [e0]\n", + " return res_dict\n", + "\n", + "def precision_recall(pred_net: list, true_net: list, decimal = 4):\n", + " pred_dict = child_dict(pred_net)\n", + " true_dict = child_dict(true_net)\n", + " corr_undir = 0\n", + " corr_dir = 0\n", + " for e0, e1 in pred_net:\n", + " flag = True\n", + " if e1 in true_dict:\n", + " if e0 in true_dict[e1]:\n", + " corr_undir += 1\n", + " corr_dir += 1\n", + " flag = False\n", + " if (e0 in true_dict) and flag:\n", + " if e1 in true_dict[e0]:\n", + " corr_undir += 1\n", + " pred_len = len(pred_net)\n", + " true_len = len(true_net)\n", + " shd = pred_len + true_len - corr_undir - corr_dir\n", + " return {'SHD': shd}\n", + "\n", + "def f1_score_undirected(custom_dag_edges, reference_dag_edges):\n", + " custom_dag_edges_set = {frozenset((str(a), str(b))) for a, b in custom_dag_edges}\n", + " reference_dag_edges_set = {frozenset((str(a), str(b))) for a, b in reference_dag_edges}\n", + "\n", + " tp = fp = fn = 0\n", + " for edge in custom_dag_edges_set:\n", + " if edge in reference_dag_edges_set:\n", + " tp += 1\n", + " else:\n", + " fp += 1\n", + "\n", + " for edge in reference_dag_edges_set:\n", + " if edge not in custom_dag_edges_set:\n", + " fn += 1\n", + "\n", + " if tp + fp == 0:\n", + " precision = 0\n", + " else:\n", + " precision = tp / (tp + fp)\n", + "\n", + " if tp + fn == 0:\n", + " recall = 0\n", + " else:\n", + " recall = tp / (tp + fn)\n", + "\n", + " if precision + recall == 0:\n", + " f1 = 0\n", + " else:\n", + " f1 = 2 * precision * recall / (precision + recall)\n", + "\n", + " return f1\n", + "\n", + "def reassign_edges(edges, columns):\n", + " return [(columns[i], columns[j]) for i, j in edges]\n", + "\n", + "def evaluate_bn_structure_learning(datasets, true_structures):\n", + " results = {\n", + " \"dataset\": [],\n", + " \"data_type\": [],\n", + " \"shd_bbn\": [],\n", + " \"f1_bbn\": [],\n", + " \"time_bbn\": [],\n", + " \"f1_undir_bbn\": [],\n", + " # \"shd_dagma\": [],\n", + " # \"f1_dagma\": [],\n", + " # \"time_dagma\": [],\n", + " # \"f1_undir_dagma\": []\n", + " }\n", + "\n", + " thresholds = {\n", + " # \"pigs\": 0.5,\n", + " # \"win95pts\": 0.1,\n", + " # \"hailfinder\": 0.3,\n", + " # \"hepar2\": 0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001,\n", + " # \"arth150\": 0.3,\n", + " # \"ecoli70\": 0.3,\n", + " # \"magic_irri\": 0.3,\n", + " # \"magic_niab\": 0.3,\n", + " # \"diabetes\": 0.6,\n", + " \"andes\": 0.00000000000000001\n", + " }\n", + "\n", + " data_types = {\n", + " # \"pigs\": \"discrete\",\n", + " # \"win95pts\": \"discrete\",\n", + " # \"hailfinder\": \"discrete\",\n", + " # \"hepar2\": \"discrete\",\n", + " # \"arth150\": \"discrete\",\n", + " # \"ecoli70\": \"continuous\",\n", + " # \"magic_irri\": \"continuous\",\n", + " # \"magic_niab\": \"continuous\",\n", + " # \"diabetes\": \"discrete\",\n", + " \"andes\": \"discrete\"\n", + " }\n", + "\n", + " for name, data in datasets.items():\n", + " logger.info(f\"Starting evaluation for dataset: {name}\")\n", + " \n", + " true_structure = true_structures[name]\n", + " true_edges = list(zip(true_structure['V1'], true_structure['V2']))\n", + "\n", + " brave_bn = BigBraveBN()\n", + " metrics_lsevo = {\"F1\": [], \"F1_undirected\": [], \"SHD\": [], \"Time\": []}\n", + " # metrics_dagma = {\"F1\": [], \"F1_undirected\": [], \"SHD\": [], \"Time\": []}\n", + "\n", + " for i in range(1):\n", + " logger.info(f\"Run {i+1} for dataset: {name}\")\n", + "\n", + " # BigBraveBN + LSEvo\n", + " start_time = time.time()\n", + " \n", + " preprocessed_data, _ = brave_bn.preprocess_data(data)\n", + " possible_edges = brave_bn.set_possible_edges_by_brave(preprocessed_data, threshold=thresholds[name])\n", + " hc = HillClimbSearch(preprocessed_data)\n", + " model = hc.estimate(scoring_method=K2Score(preprocessed_data), white_list=possible_edges)\n", + "\n", + " end_time = time.time()\n", + " elapsed_time = end_time - start_time\n", + "\n", + " predicted_edges = model.edges()\n", + "\n", + " metrics_lsevo[\"F1\"].append(f1_score(predicted_edges, true_edges))\n", + " metrics_lsevo[\"F1_undirected\"].append(f1_score_undirected(predicted_edges, true_edges))\n", + " metrics_lsevo[\"SHD\"].append(precision_recall(predicted_edges, true_edges)['SHD'])\n", + " metrics_lsevo[\"Time\"].append(elapsed_time)\n", + "\n", + " # # DAGMA\n", + " # start_time = time.time()\n", + " # X = preprocessed_data.values\n", + " # model = DagmaLinear(loss_type='l2')\n", + " # W_est = model.fit(X, lambda1=0.02)\n", + " # end_time = time.time()\n", + " # elapsed_time = end_time - start_time\n", + " # \n", + " # predicted_edges_dagma = [(i, j) for i in range(W_est.shape[0]) for j in range(W_est.shape[1]) if W_est[i, j] != 0]\n", + " # predicted_edges_dagma = reassign_edges(predicted_edges_dagma, data.columns)\n", + " # print(predicted_edges_dagma)\n", + " # metrics_dagma[\"F1\"].append(f1_score(predicted_edges_dagma, true_edges))\n", + " # metrics_dagma[\"F1_undirected\"].append(f1_score_undirected(predicted_edges_dagma, true_edges))\n", + " # metrics_dagma[\"SHD\"].append(precision_recall(predicted_edges_dagma, true_edges)['SHD'])\n", + " # metrics_dagma[\"Time\"].append(elapsed_time)\n", + "\n", + " results[\"dataset\"].append(name)\n", + " results[\"data_type\"].append(data_types[name])\n", + " results[\"shd_bbn\"].append(np.mean(metrics_lsevo[\"SHD\"]))\n", + " results[\"f1_bbn\"].append(np.mean(metrics_lsevo[\"F1\"]))\n", + " results[\"time_bbn\"].append(np.mean(metrics_lsevo[\"Time\"]))\n", + " results[\"f1_undir_bbn\"].append(np.mean(metrics_lsevo[\"F1_undirected\"]))\n", + " # results[\"shd_dagma\"].append(np.mean(metrics_dagma[\"SHD\"]))\n", + " # results[\"f1_dagma\"].append(np.mean(metrics_dagma[\"F1\"]))\n", + " # results[\"time_dagma\"].append(np.mean(metrics_dagma[\"Time\"]))\n", + " # results[\"f1_undir_dagma\"].append(np.mean(metrics_dagma[\"F1_undirected\"]))\n", + " \n", + " logger.info(f\"Completed evaluation for dataset: {name} with Average Time LSEvo: {results['time_bbn'][-1]:.2f}\")\n", + "\n", + " return results\n", + "\n", + "# Run the evaluation\n", + "results = evaluate_bn_structure_learning(datasets, true_structures)\n", + "results_df = pd.DataFrame(results)\n", + "print(results_df)\n" + ], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-06-19 16:44:34,811 - INFO - Starting evaluation for dataset: andes\n", + "2024-06-19 16:44:34,811 - INFO - Run 1 for dataset: andes\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 0 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 1 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 2 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 3 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 4 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 5 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 6 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 7 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 8 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 9 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 10 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 11 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 12 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 13 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 14 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 15 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 16 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 17 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 18 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 19 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 20 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 21 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 22 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 23 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 24 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 25 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 26 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 27 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 28 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 29 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 30 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 31 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 32 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 33 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 34 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 35 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 36 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 37 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 38 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 39 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 40 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 41 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 42 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 43 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 44 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 45 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 46 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 47 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 48 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 49 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 50 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 51 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 52 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 53 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 54 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 55 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 56 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 57 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 58 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 59 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 60 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 61 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 62 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 63 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 64 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 65 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 66 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 67 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 68 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 69 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 70 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 71 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 72 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 73 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 74 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 75 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 76 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 77 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 78 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 79 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 80 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 81 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 82 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 83 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 84 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 85 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 86 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 87 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 88 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 89 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 90 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 91 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 92 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 93 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 94 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 95 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 96 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 97 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 98 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 99 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 100 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 101 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 102 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 103 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 104 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 105 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 106 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 107 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 108 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 109 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 110 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 111 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 112 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 113 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 114 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 115 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 116 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 117 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 118 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 119 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 120 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 121 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 122 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 123 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 124 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 125 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 126 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 127 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 128 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 129 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 130 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 131 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 132 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 133 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 134 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 135 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 136 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 137 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 138 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 139 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 140 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 141 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 142 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 143 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 144 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 145 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 146 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 147 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 148 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 149 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 150 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 151 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 152 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 153 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 154 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 155 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 156 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 157 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 158 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 159 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 160 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 161 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 162 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 163 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 164 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 165 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 166 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 167 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 168 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 169 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 170 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 171 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 172 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 173 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 174 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 175 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 176 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 177 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 178 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 179 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 180 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 181 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 182 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 183 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 184 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 185 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 186 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 187 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 188 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 189 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 190 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 191 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 192 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 193 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 194 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 195 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 196 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 197 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 198 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 199 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 200 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 201 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 202 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 203 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 204 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 205 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 206 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 207 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 208 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 209 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 210 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 211 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 212 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 213 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 214 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 215 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 216 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 217 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 218 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 219 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 220 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 221 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n", + "/home/jerzy/Documents/GitHub/GOLEM/venv/lib/python3.10/site-packages/sklearn/preprocessing/_discretization.py:307: UserWarning: Bins whose width are too small (i.e., <= 1e-8) in feature 222 are removed. Consider decreasing the number of bins.\n", + " warnings.warn(\n" + ] + }, + { + "data": { + "text/plain": [ + " 0%| | 0/1000000 [00:00\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    datasetdata_typeshd_lsevof1_lsevotime_lsevof1_undir_lsevoshd_dagmaf1_dagmatime_dagmaf1_undir_dagma
    0pigsdiscrete688.00.319936899.0762050.573955593.00.0000003549.3470850.000000
    1win95ptsdiscrete112.00.0000006.8883120.000000112.00.0000002212.3995160.000000
    2hailfinderdiscrete81.00.2393166.4369810.37606865.00.1777784.1874040.377778
    3hepar2discrete123.00.0000006.2711000.000000123.00.000000989.0881150.000000
    4arth150discrete247.00.00000027.8796890.688259217.00.0000002028.9505440.617512
    5ecoli70continuous46.00.5354335.9294610.74015751.00.4628103.9232990.694215
    6magic_irricontinuous79.00.30573210.7646100.68789896.00.0960003.9996410.368000
    7magic_niabcontinuous51.00.2884625.3640170.73076964.00.0571431.8709040.114286
    8diabetesdiscrete961.00.000000558.3422600.0000001052.00.0000004177.8053520.000000
    9andesdiscrete338.00.00000063.0811390.000000338.00.0000003685.0702660.000000
    \n", + "" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 5 + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2024-06-18T22:12:45.653737Z", + "start_time": "2024-06-18T22:12:45.650067Z" + } + }, + "cell_type": "code", + "source": "results", + "outputs": [ + { + "data": { + "text/plain": [ + "{'dataset': ['pigs',\n", + " 'win95pts',\n", + " 'hailfinder',\n", + " 'hepar2',\n", + " 'arth150',\n", + " 'ecoli70',\n", + " 'magic_irri',\n", + " 'magic_niab',\n", + " 'diabetes',\n", + " 'andes'],\n", + " 'data_type': ['discrete',\n", + " 'discrete',\n", + " 'discrete',\n", + " 'discrete',\n", + " 'discrete',\n", + " 'continuous',\n", + " 'continuous',\n", + " 'continuous',\n", + " 'discrete',\n", + " 'discrete'],\n", + " 'shd_lsevo': [688.0,\n", + " 112.0,\n", + " 81.0,\n", + " 123.0,\n", + " 247.0,\n", + " 46.0,\n", + " 79.0,\n", + " 51.0,\n", + " 961.0,\n", + " 338.0],\n", + " 'f1_lsevo': [0.319935691318328,\n", + " 0.0,\n", + " 0.2393162393162393,\n", + " 0.0,\n", + " 0.0,\n", + " 0.5354330708661418,\n", + " 0.3057324840764331,\n", + " 0.28846153846153844,\n", + " 0.0,\n", + " 0.0],\n", + " 'time_lsevo': [899.0762053966522,\n", + " 6.8883123874664305,\n", + " 6.436980581283569,\n", + " 6.2710999011993405,\n", + " 27.879689264297486,\n", + " 5.929460620880127,\n", + " 10.764610052108765,\n", + " 5.364017057418823,\n", + " 558.3422603130341,\n", + " 63.081138610839844],\n", + " 'f1_undir_lsevo': [0.5739549839228295,\n", + " 0.0,\n", + " 0.37606837606837606,\n", + " 0.0,\n", + " 0.6882591093117408,\n", + " 0.7401574803149606,\n", + " 0.6878980891719745,\n", + " 0.7307692307692308,\n", + " 0.0,\n", + " 0.0],\n", + " 'shd_dagma': [593.0,\n", + " 112.0,\n", + " 65.0,\n", + " 123.0,\n", + " 217.0,\n", + " 51.0,\n", + " 96.0,\n", + " 64.0,\n", + " 1052.0,\n", + " 338.0],\n", + " 'f1_dagma': [0.0,\n", + " 0.0,\n", + " 0.17777777777777776,\n", + " 0.0,\n", + " 0.0,\n", + " 0.46280991735537197,\n", + " 0.09599999999999999,\n", + " 0.05714285714285715,\n", + " 0.0,\n", + " 0.0],\n", + " 'time_dagma': [3549.34708480835,\n", + " 2212.3995155334474,\n", + " 4.187403678894043,\n", + " 989.0881148815155,\n", + " 2028.950544166565,\n", + " 3.9232993602752684,\n", + " 3.999641466140747,\n", + " 1.8709041118621825,\n", + " 4177.805352163315,\n", + " 3685.0702658176424],\n", + " 'f1_undir_dagma': [0.0,\n", + " 0.0,\n", + " 0.37777777777777777,\n", + " 0.0,\n", + " 0.6175115207373272,\n", + " 0.6942148760330579,\n", + " 0.368,\n", + " 0.1142857142857143,\n", + " 0.0,\n", + " 0.0]}" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 6 + }, + { + "metadata": {}, + "cell_type": "code", + "outputs": [], + "execution_count": null, + "source": [ + "{'dataset': ['pigs',\n", + " 'win95pts',\n", + " 'hailfinder',\n", + " 'hepar2',\n", + " 'arth150',\n", + " 'ecoli70',\n", + " 'magic_irri',\n", + " 'magic_niab',\n", + " 'diabetes',\n", + " 'andes'],\n", + " 'data_type': ['discrete',\n", + " 'discrete',\n", + " 'discrete',\n", + " 'discrete',\n", + " 'discrete',\n", + " 'continuous',\n", + " 'continuous',\n", + " 'continuous',\n", + " 'discrete',\n", + " 'discrete'],\n", + " 'shd_lsevo': [688.0,\n", + " 112.0,\n", + " 81.0,\n", + " 123.0,\n", + " 247.0,\n", + " 46.0,\n", + " 79.0,\n", + " 51.0,\n", + " 961.0,\n", + " 338.0],\n", + " 'f1_lsevo': [0.319935691318328,\n", + " 0.0,\n", + " 0.2393162393162393,\n", + " 0.0,\n", + " 0.0,\n", + " 0.5354330708661418,\n", + " 0.3057324840764331,\n", + " 0.28846153846153844,\n", + " 0.0,\n", + " 0.0],\n", + " 'time_lsevo': [899.0762053966522,\n", + " 6.8883123874664305,\n", + " 6.436980581283569,\n", + " 6.2710999011993405,\n", + " 27.879689264297486,\n", + " 5.929460620880127,\n", + " 10.764610052108765,\n", + " 5.364017057418823,\n", + " 558.3422603130341,\n", + " 63.081138610839844],\n", + " 'f1_undir_lsevo': [0.5739549839228295,\n", + " 0.0,\n", + " 0.37606837606837606,\n", + " 0.0,\n", + " 0.6882591093117408,\n", + " 0.7401574803149606,\n", + " 0.6878980891719745,\n", + " 0.7307692307692308,\n", + " 0.0,\n", + " 0.0],\n", + " 'shd_dagma': [593.0,\n", + " 112.0,\n", + " 65.0,\n", + " 123.0,\n", + " 217.0,\n", + " 51.0,\n", + " 96.0,\n", + " 64.0,\n", + " 1052.0,\n", + " 338.0],\n", + " 'f1_dagma': [0.0,\n", + " 0.0,\n", + " 0.17777777777777776,\n", + " 0.0,\n", + " 0.0,\n", + " 0.46280991735537197,\n", + " 0.09599999999999999,\n", + " 0.05714285714285715,\n", + " 0.0,\n", + " 0.0],\n", + " 'time_dagma': [3549.34708480835,\n", + " 2212.3995155334474,\n", + " 4.187403678894043,\n", + " 989.0881148815155,\n", + " 2028.950544166565,\n", + " 3.9232993602752684,\n", + " 3.999641466140747,\n", + " 1.8709041118621825,\n", + " 4177.805352163315,\n", + " 3685.0702658176424],\n", + " 'f1_undir_dagma': [0.0,\n", + " 0.0,\n", + " 0.37777777777777777,\n", + " 0.0,\n", + " 0.6175115207373272,\n", + " 0.6942148760330579,\n", + " 0.368,\n", + " 0.1142857142857143,\n", + " 0.0,\n", + " 0.0]}" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "lsevo", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.9" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "4dbeb5dcf851b2f7c45d83bc5b9b92c5e7fba8de5ef2ca4003b978e17cdaa303" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/paper_experiments/large_bayesian_networks_experiments/log_time_lbn.png b/paper_experiments/large_bayesian_networks_experiments/log_time_lbn.png new file mode 100644 index 0000000000000000000000000000000000000000..aeeb06ed4a4ec7aa15ab74a84bf7c4c5a9b3b964 GIT binary patch literal 65886 zcmd?Rc{rBq-!FViv6ND&B%zU%%q4TJgp5TQ5-L+NW}Yd9l4Qs{R6-((khwx;l8`y` zJWrYT=PK*BpLaj|efB<{zJ*L_~+`Tc&T@AcqeHFk;)y;1S5I2;4TT{NG%XI@Od--u=0me0^&*7!=|oKqCTql~EQTw#@Z;{Uu3 zsnZ)Xx_Yj?q%>a^hW+a=TP}3U{p;r_k4&%}SbO=A2iVeXTYHs~N#{KH=XbhEhui-7 z6>)Rd?WX_y$^n7X5BUH272mzrOovGLIi~L^y5+ydTvj)IY{|4R`1bOF_rWrz%L&R( z&t{_>C#jqkrr%BbFZ2}`)kI59%*^OEBxzjDvV8FB)se}ba>OpDV@5 z?_gK`TK`=1FE8&AqfnsrNRx;))jr?Fr%#@owzWOAfB*hO zwN!Tn(fNC`Bh9?#-BOE-h1k6`W>%s9+#DQRwgDU(jit>T3XtI7@45) zsVVLG$)0cl3-|n$1uqeY+3{Khv5Ofd8Vr)`2M+iNu@QIOPe%9sdi?fMU^3(0y`!0z zCV#xsZrr0{@-!fzaJ-{{xDpDtJF6>m`8mHjy#@v|v$JcTogsJM!dQm49-o+acjmoM zoPyYL0RDWhzb?hnT^%J>Ssi%b+`|i5mIizHO)3(S^U9L*SIShhoQiN4Y`eVqRkJMF zMn^|aK2jAIKV8*dG9>Q-O#X9WW@u?;aV$|I zJ8%;n$2C(^gO;$(9#hX<**8`c!7z}ll^?lb(=M~={)^jbY1y>$gvlwW ze}?EszH2eXCba0SxLzA8chB9uv%6c|pG9uBfcYT>;R*5BFKU_Qhh+lU_a8cR@7}%j zWr4?KGE7=`^WD4<8se9ceY=Zdb!m!5J>6(O7uS=oUoT*p?x&{mo-nNMj*pAX8tOCe z6PliDcS$wt`1`J#+k;1s3d_sOCmUOa%}Ji29mkX( z4b~;prs&FK*^HUYWEK|{al5QIo-}DG8K{kW;N?|NRCFI})ip9Am!SM6@rz5x=lgu^ zQJR{X6;WbQPmigXFH9Sdj$VE(p6|T;%~6wP&{;HL=P_k2y^7tW`XmjRqK7+qhv%^O zE&IQc`Ii>{n&`4yneR(ItZAs}Jg>WpQxkVdU-jLM4bhT5vG}LK;MF|GkmKsvYoxDBxoed>|?dk_ETv)VXjP!e>Mn5JI z`wP!blO>GI4L7>^_|Q;p*^8~Z&cVTVcFx}y>Yyw1#v4Hk~MS7+Fe#Fi9fF|Cj<*x9hH}tpWC_Emg|sdKXa6X zs-3C-Vp=n$D_82o_0Q{faa;}VZNibI7jqWIq3Z7P5$i4SJxEQ@bEr~chKh=6X{JGI z2@8!oyc!wT(l#$6OdJFO=cQ9b&B|Y%Wk^a&8ffyS=XJ*##Ac(4RpFF2W?3D>HtTIR z%}+a|#mi$gcuB7!_;!OJy}<3x^{eyME}oAb$y$i;@c5OKNd5iy->PZ1cD}iiCLPp5 z{J>dv>JID^gCwm?^ZQmq_3unt1taYT*;o`JAGx`mb>}WDE}n{7UGgBO-10J$X~zzo zdDF@6Qnjr7=GHO&x)>Qc7P-*EnV|+X6sob@IZbybPW5zZ|I&NW(FajW9e=8heYwW< z6@?{lVeptqTTUs~2!%l=R8YizdLIi5i$*hwfmz4r4Ma&of3eakH8;c_EK^btc{;lA zMm>YQ+8TY%ajw3Q{D8Xh}vz%GjD&6BDQLvJ$JOP|I? z+CRKQPN^6xOV6Q|_Xs-@t+NZIJ!fTMXlbIDdBE8~D@EtQ;Q{3Y<>M0Gd(Ksco_OHn zqmQy88+Ov0M)cRu=j{#4P1!b_=seHRj&qhK%L6#IBC+r&WJUPWjp@b=GC>@=rGAGC z3k$h!cZy$0yLE15X}&SvMU0kJ{*bV6INrz=3mPZmc>n%=ZhTLCN4*f=`j_WvuU@?h znDMcpA&A)1IIdjSTcV4@sf!O?R#%n-lNnCjn*Lbbb*A0x5*Dc&jh%Womn#`L#m|n8 zGpee)+a0Giqm*68a=tfd4Mk60&&tY5oQORqZr!W7sneWp9H`@!*4;XyoT&NyazF zj$bM4tqhwV%^qWy+4QC{(}ER8EY`R=J!i7)*rh~O-yc7ISfXA&ViqHoC_Frzcw<1B zPjz)qySsHbG;>ODM^tm|t+K2yUc5*wCsC#x=O-tc43le60svVAaO-R16!eQ9GbUcm z^!t2&6EAuXwRSL{!FFF?-$zU$q4E)e7ZX*v<)cI^>9ia_tt>B2OiVln#z7t8QJT|{ zQx;9YQq@LF(%_($;6(2hv43CJoziHOerQ_IdRWT$z`2yY`g$jBe*HLUb^7$_tU`5^ zGK0pH!!v_*1(lVXfjl}pJJDvSz34dgTC=PimuDI-C2L0Dh^jZqg`e`h-Sq_sj)p@c zOJ`QjK0ji$Y}lQ$N)Aa z_b`4_^PeBdV&%fV#4E+w;;Pi%-MFtioY+EC)w>ly1U?@CgF+K7#otI$jCp8>f*N9y zalL>-CYwzuR`m_9!>o}^gxyqc44aIU6c@iq%a5xu(mO3s1S@XezCF-1)}G&f6(xne zD9CrJuUdT}?4)s%$WUCp25?PQ3Z16yC&iTx59TYzV%I*BES?^{Tyf&ocdmwHt#-$S z=>bctm6b(P88JXMo2*G-lHZpy+LoIwaP(-|;3@SCleFn=d-pn4FJ`sk3?DZxC@2_9 z7>Qt-&IOR~1>RIEi&Kb-OcLhiu4&10awxYju8s8;;NjuX$i=QzTMz(7)F`GGvQDg1 zjgybK6`zVHCTy1yGvdCP;oDOjov?P>uFWzLXofLt`OAy(C_ufr4s)?X2Tz`?Z$T?8 z7tt$nb5=&pe;_W7S2e*s4XX+$$rhXRLuF0!2QKPJdxARn_~!u93x8AjLK;z_0Zs2HJBZrl{3Tl|$WGQQH9rqq@)Rs1$ zv%@A)D!>YFs)E@sJag4$Q86Uoy7#gpBGR z*&09yb~t8as;PQ%p*>5u0dofN@i=UO6?(kR&d#Lwh7DUJB_$JPs42*CL^^<}`hd{T z5LBIOWR0W5T$~deC}IY`J|{PfD$zt|uH?FY8NI2zf!_FnDc9~J>gnvLi3YQ5 z7== zTj$IjwzEZZyScfkrRY$ttgH|O_``=&vMfrWf&lBCV9s+b<>Q(Jdk8&Yc*?SWD|nK! zzg@hCUP}_rzD1)@-o@Y`ou;Jm-HBx!RQw+0JVg5 z%L8+xnzw`Z2LHhaS+~2~b*&`^ImPMEALkFN#D|yqT_eaeiTK$ey%qWhsgST18K3iHE%URpY>jN-xBz zKo=hc1<@V9^8T%0EzgX@*3o5Z5Q0L0tJ?ROwXNGd;(4zyTPoMPIVuB&b)%24ii$>p zJ4^!ds+SQg!O>AzBAUB(ph#d~*5-4QQ9T;OB*Eft1va#{B|8w{=EvLXIAaH8x^k9h zn~6@3My=^EasrLl;QOm>s1$sL^^b1Vy^b-oSXr_|jiP#4(d)FjJjc6ps4hX}DNwuJ zqu;`bPfW8u3lxg+Br8+S{9KJ&FJ7F!v(CoYC3&*trzWqEBE9!qTTEw_*4xy{a!WjqhJd7W1ttO+$BIUH7O(OujhQ^~a>&)T`4Sphk!jYk4jhJsn|qsrf&#_n zJ>ts?Gq)`)3Q9{$dEDaBJAg)F)VABlEs8vtmB(?rUKK8Y^>o9kYz2gla#<52W?R4M324UN=Rv1?t&%MN?nY5b5GOOF0TkLR@>;pgi&mTfZ}Eo* zW~XZvY5|}lZ9Azp(s39Gc-!=pJzatnlqx8Xt5M6b zBvlBF<($QTK5m>6dL&1EyN0;l6LGuX}j7 zB2vV`T<1}9(BaY6?1mJTmgTa?bDwvO?lhF3dX4_?7aT10{O|j!IkvamsmdAnO>`8V z&3%1uphUnFkba!HUyF;WPK(xC7zOX;=Zk?5bfetIY3ACWt`|*~oJp1L85yZWh5n0* zYX6ZVo=^|* zE(95guxWdDWlWQ58Gv-WfJSU99i8O0Yx_{~33fExo*(t($yQ-{|@&?n*h zVb7YHn!ti1DeBP@-e$LNlPR7It{$ui8Mfs#>gX=@4>_8@yh3`sS76=CLpeOsS=c=T z&kwgL?qpNE2Z+N3w1Ca<1^SMerDZp;?hl1S>+2 zEYU8##-*yKZ`7J~q9A0eUTO&12-$@g>8E^)wlV1nQc^#O-EWo%j;^cn!{_C9H&(PL z$an%g61Ycpz&@@4PWg&K8MLzeFA*yZ!6_4^IdruCE zYY*>WV``kPT2BF&VLdyJC;90pQKxT=dwds`E&9%E%*bw7e46Fb+uBysoUg$9nvQ`X ztS73d?)Bx=aCxW2MHOh5gY|DVy>#5A_A~@4qOWAo@6!Eaa(0$N@nlw{pw;U-L5ses zv_pCeb7N7liDx~w1Zwd??=#%KefzgGn!o-kd8bnw-T8nu@)__M$q^F6T=;qGt}mOD zu4bCIcY&xErjRXQ!&jr(j9n39V|##x!fo{9Enfrkbne^xg@xi^V!R2uAR`&!p7wn_ zh2QD5iDs2;eO#lttJr4>8(81s{eVWUecj-%ct3yt;~ocIRvFz3$emE6*-k7d>XlqW zy$%nz!M8)@eqZz>?N}HN=DVmZCz{M*Z*~3r`4I&QB7$RVlMXACo#DpRdB_rX0&-j1 zaM6}Ciu&0o!i|GW_km}$vE6MLkIhWwHCNwj))&1ZlN3^^2ePIkaH>Zh;2nBc+lj~D z>5Xkz=qNYuS%3Q4wQHe`!SxUc6L_ETS^z#y_J3mnPxgxoKXrSZq-1e+J=>M{M|K{) z^bCtgkjRnd41yv6KiWbJVgp!HNxJ%}xYz^bk?caMUZrLJTOmQg$$E9ui5f}f+ubGG zG;KRJ5DeOT&j~3hscQfyj*BB15RXJ>%b$oY^dBUIG*3@LzPtYO;~nAoZvOxdjXiFk zKBA@IEbMO1Fx5wilnvpz4?@+_up5Zc5lx4nh~Yvuk4ybo@xgnr$vn^{LDdf8et~#T zREw>?1?dB|3^p0)aLc1hC&X8rVu`>1UI(7bqgxixf#b-2<-I55L=v0Kjxonr@HuTj z9PxAKxNXO;wK*>ugCi0^6zr-O=kM9IPj|bq!;1r{&Xy$>K)-#;8s+E*l5r8_^{R%( zd0E+R=t@xCic!EsY=>3tvg_E<*rc=NHT>}H9C_QR0a|Wn5 z&=wA*-Ku*b{$M-Ab^;tg5YQ3$&`}kW7$Ia63x1lYnHvhdI~1+tYrc!it@_0CpYCqR zgyagQOH>*{DMnd|k@>3I%XCh>yC#P4X1dv+l;r3~U2 zv=TfHT5?R?VN<{?LXeZouzZJ45uR(YU{`(Xit`d9aF|xn?>iyG5MtS5mbQI|Ylxj% zk6U>?0=;yi#iBYUvlk5;N1L8U=PzLDPP|tS@qP`nkemrL*vK@ujGkh}YHDg)ZmwUl z(r9ACa%xNmk~%omub*x-JT1)QP&2OsCakBVj1zc%@-pgk2#@YV-N5)0)gp6y3C$Y> zd(wih4!{Cd!YL9hi|kKm2L#;s?GEKFa>%t3z5s|UV`6-1vy+9tY^QFFRCG#=d_l4vw3jzk5ysYtkJOonL5H&EUQHwXsTZAcw} z%12b+?!(_=Q2M5a8mdz$eNHetj+3F!b)nk!!R0x0=n&+qPxy?|C;QKSaGW;}onTVI zzSYOUCbkIL8uWn*)CTUbN+9R)LRy8dZ?B()a&uFSCv+zV2M3|h0Vhs^TBm@^plFCg zvEH&}OTdwfcR?Axr5mdQu-x{I4_VaP&T{$UswCLfySr zZf@lGQU%S;VMShasiy6s#NvZ{puD~W-G*Kp29d0mC~3cy{!nT?Lk14$oKoqHZ?C)Q zGQWcd7}ppr-F@c0X?rBj;w|M0?bkp`#6rPF7_U>&Y82dy&wuwf0|kDu`|bD2JP{F* zr{TjwzWHL)4X_#C@@#jHcB;WONk#6LT#p$Qv2Kr$lFYD z5V9D3nFxbF-c9(0{dE5s97Dh^axB8TsLFE9TE(p`|IeQEvkhb0F& zPM!?J*6bP@+C(^%V|H8J@E$)N6M*CZ%@O=x935`cHsORGqSm8%l#F+aZS!@2L8OpH z%L&pHG!hr`5MO`x_p#|FbDC`=Fx=CRA5Y`<5Z2g+jXRzhHKq{VmyoPc5MoFd!zQA3nV0zWj-uSe@UmQnKIp_>V(<=>KVHzb|ff@_dx@cvmkz z{`X5bkD9{F>Uem3mUK=lWh(vuw6+f^z5jmcAD{bwd7poWH9ncu-K`!h%cAbF|A1XE zzF3vKSk<}TA85|792dTVQI*H87=A4o;&um}i*eE7fs-c33(|Hs((lQ}XGNDi;6B&eE- zE@=C1YUvLTq^iu#NRpDQNgA72Po9)m^YDGI{_Sy0ClwdjlJ!*l&bs>n*lZrKI$&y@ zF=>@PATS^JreJN+18mE!<0;u*vUBd-W(PSy_3IrDYzMqD-EfH#sjcGO*K4a8fB(2| zi}-)c2|W9+;s`#6#qyV-Vn4ZFpqXZMB;;F)?%6BpMz-xEf1l;7!C!!YLdY)yOA)V< zRuJ*YzZgyqU&ar4MzuQ=OZVqx^EShZbimwns&Da!;QF=e&Qt)85MBoyZ#uMf?Phn2 z!VtIn+)+RY>xp({g%Sms2g2(Pj;r?u>kKk2KHqxe zR;=tivqtvpwO`{Izh)XjfIX}~7bF8$8t1L9=(}2KFU*hV7kV}vUv61B@o{m?MLfgA zV9ts+tP+Pi{OPfv+oLV?I18^XM(=W?%$Im`#p+vY`G|Isdd4p1sB<81dr(Ysa`y0A z_W#w=+KLv!uAUx@W@6s)Szs?CBf8v9(;V(~bU`=fW?I&^*5vlXZHE+o4a6}|%(W}r z->7gQ?beCOKAX{^lJrS_jHQ5ss z^;Wysdaz;c#U(a{yKdwYGa3bOC4&VlMm5IHt`!kZrB1oYdt~I1`Yj(GK>HJ}oN=Rc zvt1nNSo+nK0k!iE8eUskl&7be=%Nu18ldmN_D{_Yt}*QUKKj_SGa966C@OA8lRyX{ zv@Lg$=hpY^AI))&4jMVOcP>V^T^noDsEAzVP0pRw%bFFk8U0wEz^xSPDF5m1{ZYGo z;F>-+8Wt|29|>1;_mpnp%#m`VppUZOvuoGG{8h5vs%1C8Lb4~};T3&R3cb~0D%^(k z)G)mq5{y-nHLEH@re0mm+(>%xfGTIEp6l-2b-P8EE`u1VzWqpPV?#qCsLk@snQ&qH z&w^j@3|Loi+Be$@wg@Q-Zy5E~|ZHz$xpaC33Zi|0E<- zn{A^-njLEo8=2ex=ZsIj`ud2!l=nq+^p8vnN?0b{C=Y?+-|lbfBb~At9Tr&}pPM6p zrd`aTTrOla`1$kwsl@j}q|+W-A|PUEWlci#Ku~B~l9o>gL_rW5PC2`tvda_G>`v9c zQp(^>L6wQi9v!^WvRwtc6^6BcP-SSEl+7DZwLoXslc7mDmIEn7@=5q+|61+wVJ8j~ zjA;Sv%MMBl==9I$Z93f6?Q1jL zzb)_Ajg81Tz;vp8an2Ajb!n8C7{of1jfo;UCD=)NheLN>Zdst zxaTKo!ZO?B3b{sAL>S2a_|<%-vv<~QKx)fAwkwkk6?9+EJ;gu8+qZ&i&*mS^491gD zHNm{cO3jZEUs@&4(T)V!LoU>*1v#1d!2|`W&6T#hnj-9{Z6CtMq200LBd)JDV8ezD zZyhsz>cIa*9)yJnZQIrX2mU)B>pjgZOJ5L43fDZRsM31f7`T_~e@+i5HP*-d`99qm z647LI`m9&4Zfxom(~m1YIv*PoJ6QsUo(`;|^Vfv7t;5_4`G|!djr3I1)cc^`nvUs1 z+`k^Xa8&J^Hx%$FPofnFoJtD0>>c@Zt7uZwlGiim`%~BPerBcCf_tdMoTI zLT6|}<8M?)M(kiy48VtR-Ta!ce!~VdK6J2Xo%8&?RS}iwl7Pm9r_h%>C;IGSGzO3X^HIpU7EV zk%S#vTi>zvw!-ECwyF3?v;$unx~0m2V>Rhp`XY}8u$R5;yKU(CKh!G+lC&AayAbH_%yeWF8)wIevKUXV?M`Z!M*mot?$Xw$)!>`y?rj7lmQn zN1-mUtEG4zS3hY68Amo&M_5=G63=mRik#C(MkJn}2V>3rny=W6n0)X$b_rvn$n;S6{xXzuz-R zMH{inJK5PnsHcdSrP|L(b6h>;fkaVdSVpHezr@c@#fn>wsH?g)FWM2E;Y4vdb?VfG zc*ULeGxnj-_*>_e_1sH!Zp;ksM6zlRyvuW-oH)Fn>e5ajv!VadmG0Z|;#{ktT{su2 z879YB6(aZT-yhjoB=hy^c2P~Qbah&Z_86nA~R;{^P1eq~9XIK`T|3tyj~E#7PQPiD9t;5$=7R+jIeS8D2}Kgy#o;7eCe z-Kk3b9dvZ%!Q9akb7HRkN@M`5?)V{yQ`d?f#(=x}#w)ozlYV-OBs{O9j|{LOO7mIU zUpu7(j)ZW)KwIgApMurkko#ubSHM70`-R4rS#Zpn78)t6?6Qft=CEArUUevCcI2#3S@2cN&+c2#@Y!AOD2&>xyu;ri;^7Rt5LBy5$nkm2M3Tb(hl0 zhVn6SaoJ7w;d`|y?%K7hESTFAQK?%5z{Y7&b0OLMwP};~NK;(^`@k2-(SaNjw!IYw zAQsXVBVL@MpW0VR#2Lt6Wen1rZ3Ec|J~?9e#m5D0Bl$$SVctTv&c zwsA)4PhS-0(RmMI_{8rk6d_tP&bhY5kA9ZqN8m+ie{qz`b9@Ya5%SwjHuY4!!xIy? zzrViBDhDjAnM*M}+B$Z<^|Q!krlqXI0~9~|gQHfKqbL=(8~krzYHV_*g}yK1)VQ)* zAmWx*k=;Vpdr6|AE;>UZSF?9fore1IrfT+>R^GbN(Nzux5sO|vkfCVXiS4q%Tx_j= zjWA3e>s4Msnot00ut6s5aOuA*8~8_3G+duN>4LON4Sgz<&pgRX@y;D*s1dmB(r9Y6PlJwJ^n;JG?q}cJn5OZ{ ziZh95Q@8RX1l96QHvJ3ol0ai?UR|uU?19C)frw(8qe-awU7{aRL)>w5c=*Px+sz*# z`a$_vFujdU;`WXeEq~iYA}>Nynsu&0ON~O_fv`%=?}kwaPyQc4^=z6~A$Z6q|zd z0>9lZMn*X!BX$H_RTX$zIapYl21O0jE<*EYb8!(?ik0mS(u$ob=d8VLbzC0HfPn^j zMj`8Wh19CsEs$o>sSJH;Ga__!Fto?e?bGRoB#~f|Mg5f}ixcn)$P~92{CN9`?w5`7 z)z_Y$0X8KC1+}+UQMF^Gd|$v|v*;<)eMrZ7JIaRh&uKSz39$7S?nu9U-{xxdvyT*y zzC29GusCT>H6aAaRSFF z#jTt!dieBP;gwADod{8wXP6+AryMF~GgR-7qJJNa4_r&qI4&kDYx{Gu?Q_bEzQDYi7#=FMqMEPT_7CnZ9JP>3F(k3mh& zneRP<6a}+AH4(;<2^zH=@Z;$ko$TStoA2GUvf{v?6z5zu*S<=9y425v1er9bCb}3_ z*XSq{W*t1qDQ8oRn|Be#MNQAX&xqd@t;-ymFm?NL*!Np$X*WMH<|VD^6{8XkIp>q0 zI+J5GVE_JNDk=fwZP8`9IZ2d+vy|F22EjmA#&LFb3o#_2FXKXUKv(Po z3kmtIPhVp2T++$xGTS3HBZwBt^bYh^c3tp4$CY^@5iQrUlIOPwarZ!g}P0EBZ^^05ARp-8bcMp5I}m?I*xwD*{)TzCSyY+zE%&N6gtP zd2A2TL0N)C!)Z>hY!n=w=#p%FZy*AaKTw7gxX|fPe>EihB%0GlaPD~EIV9E5d*Z_r zwZ_Wp5_|{B5Mm?h>BLksJYPcSMRs)-8sia-R?XbVSX2qrlK|Xd?UP}(h0D#RsbsdJ znRn(~y4G5e8gbYRkwA4%)vq@HHGT%>v)uB+W+br=Liu?0%Fx!!r?j z1{_ykxwBC95UwP*NlP)f_8ritaU4SoLaeK*;vmk>;P>x(3F zt`D~3<=2-8#sWtgNy+uVRYlOapP!X{X)v|2$b1`@jA~kHQ9To*7*mF8Mrwg-Q+DA4 z+Wk=c{pY8gjwn~0kW+__7U`HmdrXmcFU;fVJ5!fAtbuWVZN4RSsBJ( z%{+=U-obr8NkEH$w*FYos`S1!mBO4~I;Zi%eZOUHrPDWaIsP;I+xs*mO6>PLPTHns@OO49J|R!e z`Rv`bAB^ws%SXFTSge$JPaG#YRjl8`2*)WTsrmI~b+~e2+VB1$P#owQl8~Q!5DC5-J18L` z@pCAfrPj%2r0Jl-p|;V!$ktYof4LZ?6w*1R+A(L`+!2LqufHyA>C+{Gs1fn{(G+cw z82jhnA+R6(znAXUzTbiW77P8@R!=6J^hknSA{uUnPH3sl7agAFf3aa-N%!rhC$I7C zF}rR3`KG@9&;_2#fbfd6SGI@S*I=DE&-p*xboa=xr|QYTux)Whj8!56J6!H|I*AtD z#)~sSC@VI>)+^akyvA)e%WjjdT03$%HQT4_Iy+DM7G56!QX3Jqtp&IgN_?i zVjh<~n%UWfK=2*n<-4P7yQdC0F%v{s5L|25eu(ZOx-9{?1KHIIA;}WCO%^2{CZM6%l!5Akfdzx^%J+Jy}njy`VblJuzqvFTQbBxt6DMNUA zTmr=$9Ok5zBC}Q-TW9~j@3Stp8Yp)GVFCx%#^muKh@ar`?KL4gMr*CLttEgvTzcZhjTiI+=10);+2FKmYHr4W04weoDrM^R0znA2ktRNA|Aj*_ zO*u7jpcy6)rX#MpA6;PIN-rhKF*k>3#BF8#n$$PNe7>*Gof0EOjYpobMoQ0Qrv* z)1rTE*`hP>t!S^PcS_+0B@z{~lbc`%8#g!!1=48`NUUUhX7vgCie}pZ&smhCL zqF1la4E9un3;1ADlAZ(ye^fW!d=QSfxvuLgV*m@usAfja(7{7{~~~ln^9Bb$I-t3p2Tx*O9DeCgFH`Vot;#_OM#g{37$u z2pfs7 zZS$PJ$2S2@HC>iak|L5^2R!8L* za%+nof=+-;7uNcHz#FwxzhA#jAv5gHGC*+wO&8&rxwj@Ak+?Cfpq6Hq6$+~>)#Pjs z-oyNP2Pz1heEdQD^5R_VfZ=G18cq(kPRUok^ZX>(GZ=QJboHrbrrB$QhC(n$(CN;B zxLh*T_x&~rBUd%Nc#VF%;!CY1>_s<<+{x9EUUg%Yl$Q)hgb;5J-G88Yj(*>ADiIsU ztMm;T#MS*=UupbH#bI#7Uy3)+g5@Xu?d5qT($dmOYeHpvJI%fV`U??)J&_?7{||%@ z@p*ZYpk@C}tHfJ%w|;hC0vXjtJY4ib3;jq}prAho?tOQ6w`I>rvrQp-qWTEE^Q(yezDan^?7{tlOYi=I^&67kN%nl9qvC6S9^zVl5=a z*{Wj~U+4thaDNnKJU9cuh{93JNb7!s#rHQa0-O^-xyUO>O37qIAST`Ief8h} zTnk|Vu0Q55y}CL>z0v67%?jJQ1FdDFIw!-}wePV ziE0KJyIjA;lnZ~|uKLdhax}2C;Ybzb*iAvgU`N#aB?4*0kTtW*(nO@O4N@~uIc|P? zNwt>Pk&O_nOW_Rc7R9hP987yvIga`X;A?cG2PfP+h09Cqi z^U_CcaPoW5EM+W?b8)%Co*;eAU$szfJ_ldAfTsQBdF4m}i%;Uiy!Fj%`GM}8 znbz?Gn6L|h2kVp(#;d=Dctin2AqaM`8p=>@g7IVk_&nMO@&P^%w=s&W-%=Fy*6Od2 z$+w+*MeM1O`|Ck@pN28HFvD;In$>An=8G2_#>dBXmVTW#XdeoJYmBh-ffja`JuSz0v@bkOv9GG2I6vW7{6rk_<&C1UKz&1=knvgPK!{0Kj2qyj zDkp1}Rz<8{h?bB`(#n6Fzv_&b?fIel5!;0UXPO=V=l~``hP0-jQyt*QvAE~oUtM$q z)O~j$ph(JBPl|vX@=4bH_L29!mRo^ty$$H)tm z9T00FdV53m&m8L50>Ao&N^%^5SqNDd$~)w*=aN2g{mmKqTQ2^ovcAIA*Qy3@-=k@S zf#YCY-o6YUJSQSp`svZG-rmPG zH@h*4k0DZX!CU{yK`WMvJfEZS!Hmq6c$5UPfC*OcXTE*U)e33!lWtZPaRhCRRnLfA zPqEN7=HiNIAra>*aQ8;wn2r$VzN0`<<4GW_wNCltpJkx@i!Qo3mtY3`$v_qcJ7cVd z(gk*Tbql+l5%c<8@ekQJ1i0vDM*WzQ?L2<+)rK$))d?9i19=&az@aU}a_TmOq_naq&bXvt((h9%47(v7JB7{;|-cICS*&-`jN#HKnQg`7^h+ z3vG6Dod?^}OgG&+ULu z9QFZX<3uGRyM{+yFqCQ`+ld$^DbjJW)ML-T9z-MTJSp3``6q&oK(R!C9gmbS$z(!j zpk#JDZEns(B&w0*Jo!j*jU$OOQD)AX-ACqlB7b$cda@6!gA54J1t4Cze1-^uBiXtR zBfS`$E=OAZGaeu!P0YbqS?R8>EaTBGO3(k^z#HO9^g{J~ zXA!DRn=p-k+156oEKt;#jcG^DZ7dQdL!zzGdx( z6!sE0m0%2qWp?3d928YiVm_q-k32kB;gadX<2<7H(zDL`|1FB=7I<7~mbLkv_Nr4c zMxu$VT~rUyPwuyXxKS7D43RbyW|!8iv|qp2J7GPZ5)|z0sSAcM2-)^=qmf=?N)a;r zevsEX5G|hSeS=5VZ2RzGN9vw-&`UMIZ&XJFEKUK3TzY-!J|UZ;JIy)e)590U1CLS# z?5F>N*@y?RoLiV3Q3&E_t*y&#U^0b-0A#}rWtX51Xh)@9oOzB76UcH@Bx$Ug^a0;y5kud5()dbsPx*BsoB^K(bm0PkL}W~E|JwQBk@^1- zIp%3?;BR+)fQ}CJABeL6COG_VWGWlbkzL$JzWA(i;vNNXuC9kd0I9q8d%QovC}Dj} z&NQ?uAu7XX4DL_Q-FX58@DVIs4Cp)x7_|lzrSu4|`wQk^q@_x3 zBMx4K=PqbB9%Sd>c#QZU68Q2g@=FBYR&OGrkd6BO+##}$_ReogmsV7IRexF&3XTZ3Q)uMQ#tx?qK103yAxsY@2ZSD}pmrv#n3e3^~-K z8TurOk#O?2A!*2h$bQPzDQr`ru}44gxR%-vx0T==T30(G^iH57Jg@@Kk1@anwkw`A zZA-@V9}Rsvqu7cz!j|{_{r!7FaYp9NnGrSQ6O_AN|LeO#Fsu;?8yC%S7k_ND{~=?ieT1b5JXKkfrSC%VVW^rEBFh5jf(i*n#y5{I|3|}B z98@xOZ$_F^3yH{5yf~MSPU z_5xEw8~S|(JyD~hYU;Z}X(33|YRI&5R^gn^&JN@=Zu;%NO25DA_4qLh@RqBnQzC1T z2^}r19HvQ9ckt>g87U>Rvalf27tuksHJ-L(sg zimv3zJ8_QA&iBRItaXw}79xWA!PqcaC`tdqHF62-@W%Z?|#uv&%o1?`Jv^fDmO^IqQcJ>~7 zgbYx~r<0tBa*n4IASEDvv;Q#koEz@Ttp*6KkWerdVXm-f%!A=z@tmS~>^Vm-uR(4< zM#~R(yEj@5o&0!{Z~J!hmO)W1G+E4VY;uL7+E&k@>%RgGRBgltIm?v8q7Dv}`*a(l z-^2`4Hmr9AMCtxzJ5)b^C3mGk%VpgKP^E-&9aDXbUo{4CMpt(y=ht7!#pB7yUBkj8 zvr;g!j!$84oq;1q%nH4Dk@OC;E0#TYWW#WK)JoPLs3T>K5fAeXHa0e3_O6Xay>4+xMNX|iIc+Up88{{S982hsDbcOhL_4S#`Z8Ni<#mxCd4L06A&+as%^xehdLBz&Y0rak~EVX~Tin!J_Ji3N7J+^u^ zc41=Hd6$%c!OX0`0aM4`9(^@>7<3g#O)m{9ul&ogpNNkm+yoQ#@u z{``5EmGtPiArIC6l&nJu@4tU{FUtnm|9?w->cB|MefMe4}X!W zMfjf}-7R*ZUF{vqSt%?+J z%a}`r8_A`SeTQ6257P_4Kzs4@8lr6d_@TOP-Ifcl0|)@7jex;GQ|`hb0|2r4@Q-bX zukPVYcJGEUR!)d?cpv5nUplya5K++o(^e_)^N(azxN(R05d+3_OM*x`phe9iD1e z4Mp}e)C<)QW=zl##4Z>qh_(8FX=Zgtre*R}(YuF1c$Hy=Lfmf4gmQ2rw1JR(6xjEdV>83;+iWFw@OZc{6Hhz@t)Cx#IuR=1Z`g1n z)9ToA;CY{oZe&QNU{ES_Lhs=JNe4J-jqEw*dLP#O8nX^Wh!C^Omw??Ru@R?azpXuT z)SQ{HKKuS}qyY6_;{|gVrG9p!Ncxme_ST(N-f)vpN|jzpUBu*OjoRq(KOM*i4~jqx z{NLlJ3d<6D-0X-t9(=U}4^b+<#s%nb2G!ASwvi4|kqj%(KOgJofvF*%91QdFS`UAa zt@D2L=wpM8T}-ebkixKxITCVsKmPGD?PQ5xe2OBSqFs!6rHQX1 zkLU1QPU3aL@QZ`WKYV+e@8a+2;I%{a-+FSZgE}QLS_dZI!rFc$;U@RxWJpD@tX|lo z-;0+HsT%JtybHc7N0mH`QzM={Hh~bsixbj$1o9OoPVP4K6)y+neyst_D_(TROuH@NGPSme zZWR?lz!n6QpokzyGLunqwpD^;$vLPkDx#7EQ3O$fNKz1xjG~f3i6W9DC>hB?VD>|m z-TmFa@148uomu~yS;Jc2`dX-Z>#gT~!a4iwy^k|054%2Bgmu3C_6yOyk-sk@e-9#m z7ygzs`afk2A7_4;J$$D-;yiIpLTLSl(b6aXP1K#* zNt6gm5xJ#!%&k%^cvwD|B;{pS@!XFon?014<0J{sf*7HlPClXjVi*}S{eX|>n!FQaFv z3q<<=mV7rbYDNLL;tipn2%{T2oV``Ar;8zIm7^0f#xHCzlgyloExCczkjL7YseU}Jm%wg z51i6ElhF6?Pp)SE&H~1$0^tQ~rDk>t_qXH)J2W%A#8D3~HS)~C!uXZh{jzS>9I3a+ zX7lJ}`wr%Xdl&1&9%Uzr0oRc)5a?H@*+*oEn174$@drn`cpw;Aj0~8N z!cA%pjP?rUn`D9IJc8;igjVteUh;7Cgf`HZ#eLFimpU&Tl^Z9fUr<*uTQG?zZvb)& z!~sA)5K3<^2u>CetpEGuGw_ZAuC15QpB`-p5+s_Q@dkYWq}z|@BCN?PzVZKG*}BlJ zFcWNP1ZvVi+fXM-s_xS@;tyG|dXxi^Z}&$Oy8V5BsU&hel~{hl9YcUnKRNmREC;9Xn2V-Fqk>d*b@S4P@zwNxeQE+^!2i)?HTt$?w(l*K{J zI%HfzggSVHnvJ^gAq?nec*+@kf0Dl+v06D?ncDh|v?ISl3onL?Fxw&t;rMw;24 zO7WS+?6R3rrW|C4hXy$Oechs~6k?lcP`U%jGh}}kK$j-=xNm)?d|juQK-K?k|NeJM zX@=!8Kw%!M#wn1DJjZ#>*QTtNg3QYNBs5&+9ruMR=@ zYjby+)Vzajm*8I3x=o7~U<4;;2F=w9lVL!aGtsPwzrAqY*U1d(vR`66Or}xkI8dId znd1J+OmGAPBOicwYdl*Ah;XLaGmr9tLZ?2{;zAGs&kFQuShSiG*cA3=XG!6Z6s zuR>}Fv z>a}YXw%Vq9`!L&%&x~_Do1S5jQfJ<}^)fi8uv*cE#&ul)hvM%*o0cw(tPyh^kR!`s z5R?W#5UWq-deuAA{?s#l+m_h)oJ|xs94xJJ zAEz#m;d9`74gcY@*1b%g$xq5lkCr z^~>kR5^Z2tw9MWupLcirPv|~dn!gHy!pv~99q_VdN>>}|W}u&}V`3_Tqz|i6zd8Gq zQ-7)uJhFl*yF-+w9jtdsf=ax%Z{(o<`BfP`Tj`xN2tTjAz+GrQ~)_7Vsb*mxwy;*r%0;6nB}Jz6>u9#sBo5@^^pns?l9g2iVnsXmMY;Ot?$vj z2k1)T#CO!{@r+ToJWRpmA?Gd!o>2i}t`r!qpSv}oKo^poou6~~?#6yd`LdUuOv;s( zy!z^v^rH5%a{{e>pZ~c3zM?K-A0SOffhgfBGHS8YPHLMkZ^T za9IJYfFbLU=`Lh8xl8+VO1h%36sxeXu=cdeFI+P(_nj`h*-z-@J#JqDA(0*)t!|dP zH%<@UReYUh`hcSBL*80IFb&yGOC;~EKL|LHa^7a7RRL?Wd+iy=6L?Zkn%qKq|LZ#I zP)xfaoRi-^vtx7NCmI|ec969=wYVi|PmsJ^-eDGX{-sZy*F7m}FG7B3^t|1pcdjtR z)QGv~gM6uAXKGhcQ`STBC8Y%4-(6Kq4*{}qec}>CV0^iE4%ylUMsdSVu~H&j4#YKZ z)M1m98I=6*f;7P*+TEkw0S#3xm$nFeRb^sgn*3I1iQz|~T1Nd#P3(HJ#-P;KwQ!+X zv0_NJeh_!b{qws<#ou$9LWNWI! z6i;4;qDzMLHgexNuA!lk*^8|CSy^P(EHiN3mHA*oG=Qt0X^(G#x?nQQ=i}PVTXJvX z`9x);;p|0m+Xswh)S@bfbFOwAHA=kS<~aH7%JiwPcD*T&7PJJ`-iDxOU$E{pqtdhC ztX%-_5<)c~1TFVjFKAwV{iXG|O3gmhp&2CHmF^1Sxcdel=)DMHqZMXDRpWw`BVMfs z^iM!ngloitAa41lPq#pTX#bKPer}#lp(6~e zmhkv@@SU@FkEk0iTRQnnzxWV{V}xza%lii5Pv^UcOE6=*4IbC1)|m>+B31fG?*Ef4k(3$-kQFc(Y+$!R+qoW-0*(ztK>}m9 zC=f&v#Far6e0?n((VGrVLs{^0NyH6j-~$23HUJqX z25S9JYn*RCc;LzXhK}pnC(J!-$k?FXlajg`14=BCvqMRGZs%&B$Y%EHn-HLu^F91+53WfuN9t)^8LppK7VCo9qKlZUrvoxOci?yKm9e~*pdzi8ns z396ZAsh(eNaQ`Lc&Gi@wm3Ldp9aH>{@AJFOh)B7u_4iwGrr!kq5^1yFpq{3%AqpUd z1tb3~(v}L5exIPNh^Wh`os*^>>acqB`kv@81orZHqCc-5Ed-4jurJ9xLH(-Eap8m4 z)|pt;0vX)r-4vo{;F}%H35C@7@>yT`Fwt$*cG{^Xvb}ZYHr&*}4ZR#Lle>UTz3ACO z!+{6hMMW#9j1Vx5JFdNMoc)xNl~w4*(G2c6wBBPwp0>8t%HV&_Rk&`m$jAOx#UF>2ldqYcv}y=)?X6GZfC~t) zZ=pyg|-ENKC*I;A_dfvN0|)m2Wk zf9zx=8gQLYE8lld{j~XpSD}5Kww~+;nrvRGLK#p{qHgZN$v5W!Ej2tia?QUieDG3E zPHq%6b3DR!IeF=C4f)N@VL<8r(h?LO{cm#&7#A;)a&8(y*eVw?EKwSslj9P_6)tFw zSVSPZm)SD?R-Q$ZmaFSVE2v#jnfo?uQU;{{yn{obnP7@p!x=)qLt%$R$xb6`+27u- z(9!YMyfKI(61UdqftiUxuxglKfPpX*%U-cM!zO(ux8=Z_*GEQrllC^`x~)=AItGo4 z4-n~*l3htgAG%8z`AU=}U2+J%wHUI(GY~A1$!O!wWPRwtVZLlOg~H#MGe0 z6%m;MyojJzIGF1?RrQ=!nJ3;Mcp(vYqRJDkIDhl-m5YN6gJ5^wk(d|4hJF~nMvyzc z1y~3rfPP)P6Z%91O3Y~t%~$3z5PT118bbfiu$h+%u0j1ik9Y~7I1d>C8?3AF_>z%G z5abBKvLXs{Dc@bGJ0sg5(S80`;TDRlCa6V&xd(y4SDpXBlcs^7r-4zqm{sLLAC3NFaNY=su%3gXU)1TM zbimssKkZPQLz97z+9gQa8TNMZ0E2}BQ4fw$+hxB&Px1w@>P>Rep%lWU^7t2De^$3I z_x&JrYBI?}=mNTF(n8)o26LJna*p9ZqTqvn^-I{yttf7=FA%Bu-MLfbxrlG*E#vmx zTa*=M#oVShAW7-$sV&NSn*}rrAs=A~FCh+01gb%#IXBaij@BPiBj4ArTj$uDAOvZ^ znW0EWC!37_%$soUiULuL0H$a-3-6rxF(s)kg9?Fc=mdX0fXUiBb z1t7YdGNSDrpzaf*1)g0BND}}cf74Yey9)y$ZRmYLLs-o;xsC#+vfRXXI73!qh$-h= z^K)%y?Y_R_g2Eo-@D9R52&dcNzC$Pah{pXJ7A*fuy?~k=(6YbO3q)A#?mojK`yKB0 zl&>G2U9WYgc|BS44$Yw1(RJH;bYNN%$#(5OO{RFgdP1dC3c(vv6igazT_s4VRl(CO zgLFl$5z#auo(`V9p9Zs^l*Xg8bR{KAzI;e*3Q$w@>aU621+bMxXY=i}V;UYv)2xjh`&mH}b_SBMhS-~gR!`T61m+JO((hkwwuv;7-g z``_S@PbRNoB`*XHM@AkgO@(LxwttI2SDX{J_zoc^C*;^`feHi%^?fhnKEd%fQJ1}hu4K!_Kaky!*5FIZZGC9QS$<% zO<`M=08gv(r*gE@7MuEd{*Q*cg!=RBW`>JrxUvEo%#vO2hs-uIF%37RDA-RA73>KW zck_k?5P|$7fSN#jyaHl4Qftk!FXuONnW*d?2=B}ZYnv5q2L&+s%o`=R4KoP#@tD*n zUcWB8r?l+l&+EmBxWPfbgvQPy+^j=AKJidv8)|T*#>1i4_#;5kAh=q_nZr2H;A~*V z;r9#xPQA7_seA}&ZLg-TY@~@NMKk}El@SUIp}Pcp zgAYJoDI_#V(52a5wZIXiBxtg)f2(I8`_;6dWEP$E6p*fbx1X&^Xrv&EfW8||2<0+| zaS^rPkhiaoEIE<=Vk{~JUKm!lal=dPxhC%R( zCMEmj!^wY~8VKO|cAGnde@Wjzru=6cM(6Emx^&H+mZm232x>gT9|A6!;?*0o4wk9d zYC#!Ojw|IZ4dl9lgdDGaU1{lYHyQo~DcDfs+EYy@*bp)Nye8*0z?(Z=?%*X9gbL^B zcZ(-K0YioY4?{M2ZhRxyxtvUt%BOCZQUftj(SjYH)r#u2>6a9E+l+qQ6DsBs1e!oq zo`+>mz2#MW9DT_Jrcx6=ZS*@uVY0s;Ul$w_QNoH6 zndxzdpn(}|Sh>*sVL)n6pio8$bFpBByu{-{ZpuRt0DKrHVB9B1uu(X5+m-|;c`jJT zyoZ&mKbA=~xTVNE!lU*a1FQgjIqGitjWb-hG0%q8*tfXgJ&uWF2*k+vBQd#)soabB z*%v~w2MYI{iJ>0QH0$q^C-q#dcIV9s?rPG2bAZwJ$y-hnO(#hy4wj09nE(?z;--R& zwLy-elgs3VerG`gU)-_n@)J@c6nX%(W8hKq`t93wznu4kWe!LWTj3m0{B1vV8F##g zImjB6h~T+M#HCzNE=lG}vWq4s#RGAsw> z@e80cu<2&OtLP-gmuN!CZDc!!A^IC>c3}Y)zkVZc=y9w=x($uhiWA#Go&?61Y+A2S z%Jm31YO3_-H4ofoIASrk#l^3GNVlYeY#&^BPPO>WVPR3-)rI%$`@_nt=P^^zOGudT zmfa5=*J(9&@=JZN)KPXsrQm<_>JVZVJJCx-&lmunn5!5x=}Ii7;rdP|C) z-i=9Igcvbmf_$Hy5R|z+j8;BEL*vYnF$xLjwi6U-r0_&DWg{Auw8Lu%GL8a768HVt zn?N&_;L8!X6RVt8^lkHx!86+3tC1`BZyn35>Z{pq06IN8AZ#_FC%41qCP%aNMWE|| zq$RI!OqlFTiA`I3Jed+T|z8Cxc^g1y{SS)_dm+07eU_~%yTGU=6wIheJILss+R06A49M> z)L_IV>yu^IYkd73ZMRQG72T<0YO22-{Z*o2q5y~HvjSU)Z8C>KWLYFG#Q|DUidCRB zA2`qY@H6E)HTN?Hm_X-n27}YY251q%-yss3gaVp@(AQsz+6aG>14{nGM?Uwv4>UA2 z73{c{4pWld&ZBp(h||0dr4)eWFC_2!&|jymZI5ta8q}l?lQqYI1CQk2roM5X`73vq zx@p6yhK7Q2e!nRQGPDGmeb!R>bXYzYJcb{5hHyphiERMed%w(B^$*{;B9eApdmEM` z<t-_JZxC~)L7OL;v0Z0LOb>g!j(exYhhqy`?o))w5mMVZ;y+B*5h zb|4h;*26rlE#Zf$A9O~0x}W=ALqo$^%Wx2<81Izdg`*_hl?JmNyu2IXKm4J=c+;YV zL&WcsWZwNw&x^~hgUYY8zr=F$2zO!e-w-UK7K z1!y5n)6q~y(28fLLa~tF6}fBaOrJYce&L9@h!1D5ww>G{>_~m|>B_tE*dEBwB+R{Y0tfGSXzya;!=P+Dw(+kp@qL4<3zbpeehSM9J$YiJW% z4sm}K`fznshAZM+hO6NN_+o-1jgAKg;p~l8$*FdCe#{X6!p;l^OE}D;EnCps>9kcBFw}3mjIBqu$Cp*%y$2Vr3CDhWTwz``(1vB~3);~%=4o*A2sXhJu;1)r%75QM} z7oeiSV}O8}IPJsA01g#sLU1MF6>Vu_`bma0w)c}xq0s$w-jbO69V{6nW2?Y(Mn+r) zn{lU}*qF`e1`;$axH(yv=uiEwNX_#hgqqHh?0A_+HxT0%CYvqx?e%(~$t(~7x?n@T zM4GXb04X_w5kb~>B52wkCf~ZJ$;1#&}@qV*M4?!8JxDJ2rs_~n9oTL{+?qJvx3(BTHFv3 z5>C_bhmtiO%XpC7F>G*OmaxWi>JONMF{0 z-UyL?)M!ubbpQa(>hp$s7j}rE?XvSMv~Szy&e9QEK%ZvOR08v6v@Vpp8>#&S_XM`? zRtglBV^0fywCwgPlnRr{zU<|-_>&!7`m4*z(Tk5AJu2qo`{Q|a>>+n#VPQcbCi4L5 zVs_;@jqCtg@Du3~N4F2$3F?nic1P@MZ9ExRcTwP!2&cL$2pF_V;4LuO5uixAQA1jR zH@)%lqefwK}P-|sGguj+qvHqD1=Q&U;kb&{bt|Y|EEoS^_RC5x2Gfar=AG@}235toI zbtsSSn{WJd$6sFOy0`K$X%ZnqvzTW)&hU3|lg`d;Q~DuW=UkK;QSo17CNxu3aqODW zug?--9*i~;(2y?2WcduI9Q2SwZSyb$;n1>0LH2`=eu04f2syUzCiSdgKKSQh;9ICs z(kIK|1D*to z1t~8oT@aMiXyt}vIy=!{i?{OMcI*Xg`Rv)V=jM$;z!OkU(eCTtYw-~+r?H>DMsble z8$$kPfJKEydms{z8)y&x*e!p{P>o(^37t?IX*P)goT$&p5ra}R5`8YgK;MHuaWA7f z^voAFyInbca$t}p&mll^oz|h-@2)eh_~nTJSJhH6$4D;Di>$1VA28oi`L!dC^|v#N zmeL>ferU^>Bju;Jgo~AxSw@O;o8-#KvOzlcEJNwV#s2b*6TO+)ydmSR_IoQU`f~PG zQYfC;I}X2WB#oD{BJ%U|xf|9}7Jhs;?H^T9rg)yU?)f(69~f%JDcta4;v)H*CwE(5 zE9o+|Q|HA~x#-X->z^qZehp^+w~~TCl?jhbPDggI>E7_(?z)7$+U)a*A^Y!>R{bAS zWswr7z)8Za8g@WYXaveBDyoAAR*g12(E6lv@7hQi=1*y06H2+aBmVw_>XR`ksr@Cn zxA*0?JBF0+xF|$Ezi7wWWtOWpa=$}3Ujqho<8wUI95lZ5zIM%z4K1KlF_x-mfVAP4 zUwrq~E|wHOFVge;<|)yzY6J3qN@!VMkqnw*pMYDbYMPm5<8$c9rLjknW?2ws$DmKJ zBmf32oGC%{s}|lv>7d7_2J`CrhlhtZJ;xI(DleDE){==zYCJ@qO6a2odd&&7nxAQJ z;KCblQ8G$R&yfbkL9{-Mvr``4X#SWG14)OduVkgO=5*Yp^4M4Rfj!!~!95}lQ!6&8 zvpAaig7ZglDRAIw(7SdFNQta}5U=jefZ|A~qZ@e~#``oes6BIWOQ|m(-h3s@i4}p@HQ$Rok&1A}-Xf0ZJm#1h!#I9y$m%cq~wu-1g ze*m#{0KTjERL6RH&J+1*W$1@=`_7%mSD2kSOTCxTK5FHaM8|+gBr-h^qSb&Ds)9C* zDkC7!olelnP=ZMJMbcv0CuA43waWX$$II97>keu3>l_{#$qFwgvjjg*L5I`tE~ftX zi>%eU-vpgU&s|}}T81>tzo7TW%T9V-Oo!Khq;QiyzqJ4A0x1WPmyFc6>5#u+d?ae0 zL-GC<-9Oc_Yma?6dn*=+yMH7$9%FRB@%HhlNvRZ%gTbG^`5Wq6C7FFAS`%J^ku6Ux<3P_3&aioE;L^*)KUXQ@B^gWu_*p)s`F6a=zDzs==sa&Z-~QGX$$DE zPu|IA&OWbBZ+4btmkp}%Vc0v6B}IKLYmybGV`G*>5=?7$r3lZ~mK&zvKjW}R2 zuBR@%AsYw^r-;aOSSjYhF0{M#+d=)Krzp?iv)N5h|H|O`-a`9qKGj$y_HqMI`=Zdx zB@Xqa31yaHz<_Ki4T0OuIXD<)6a)uySQp+exe=e=3xXuly?AlA>92@>YS$}@ZsX(Q zlUCBv2?cDeE*~zg3(`Xm3_gQ4`eYZI2^JUD%UT$wGZ{;82}`R#`n;lrdbKoiwd$B# z;{#1`c=9ISC%6#I+pJ@-LD+r&{4JE1Iyv&YnN~Kl73=fIqM24tQ&u`w8DhR|(mR!-eSo|M z7%ZbrFekr)m)bIam@j-R#Z$~iVH5Q~DILcr?A|oOP9pVeA?bGV75)pxywHLOz;JWM zn0lj>A{%@XIu=FY_VPZWmri7(;%<4mKPRFeHdT;=5`ij|mX^w*V^<{5+i2jKMs~Ch zl?>}VF@$bmdBu1EyxUJ=H%iH*J!sZSo3CxyS#(ZZ`f1IICfaShFZi~e$3(ON#SbbU5;N$1z%j=;D& zx+8!q&XoE)%J3R|G)xkW?6y)oUCS_k%ql`y%=`n^uyc&;W~F2-55GpCyjXjMZ0GzX z_!qWvW}dko+#~mf5my?&;Ih>>CN8W z-u`S;larA%zRM_{y?gG-^Jj&m4rDFy_I}})U_KMX6@T1C$hX1lv+NE}otOBO^1kSw zO*y{W)^wYhXC+%2+!)*+R(XU%Nq+CP%o4UMHE3A!98M8U*-py-D%kr$D3n5nGty`; zIZ}!u*}RF~>hPw@qhD+pD89QwZ;B+eZ@NYP;mH)Je3<_Zcm(ZFt|a<5Z2>LNaC=dX+OSuk8UPtM_8!$U(S(B7#Q*_Z0$ z69@fmyUG}NbTd`ZnW+w~I92o9-4b!CvIe(pQp3?28W3=pP9Wif%B6dB6?KBTA((X# z5r8K#ivHGd*sAPDAQ6rLkkI3IocTR;_5vPru4LS;{Tk`Ahx{9EoYKH!vsO}AEq0E+!S+2;rz^nO%|EYT?5zg-Y=<36N^jt zwQN*RNtMTxVU59jK$k~(Z0-{}6_u5W2M->+DS%H1{dqG&=zgOjWdFkWLRb$wXgS^AArCBH?g`l0EtP@8sIJRlf6_ zj*#ospyTVK`I)@=Y80(gEt*8Kro2uUH8trXfm24$#-I%=sc-0D$L;u!!ej>h1)L$4 z2qXI*d_<*{U?^Af!s;wGI`xg$*-KacI2_kqxg;c=tigSEzttJFTH4mr!H5M&TPpcY z*z7M~xnhbK6@#FXSsE(pqy*wd4V-zIcxka%Q5+u0MsAG<&@+_o_Q~QELVHv<^kiI1 zPuq?<3~+B`ijhqQuu@c(95QP|9`r=x*=MofIE`lyUlkN= zHKyHIQ8hm1^XJbU-Q7-=ooKI0zkYqY)OzZBCBwQ-45>X);X-EDY?JZy>25?5ul_oh z%CQNZd~{62ZTdM1q4x&(k+xN%>oF4h?N_eT_MYLLnCiK1GbQ@`#y~L17^1Xu`kn)9 zFOah<;?NFkFjK?kloGE&{an_h^A7q^@Nnsdf6Y5k0QNZz>&u7mAB9b+KO4H(oJ63; zzxuFc_BStZ$5Q^~@diY~Ht7j~VeEjJsAJlq$kmvQGyNPIj*%v1K^uMqYx~wNxAaGf zcmRw;1%!E3fVSC5jIt9bbI^w6v^k(UL7%O8n_JWW=^E;9q}sslM-tmxZ_~Q2JgopF_S=gARaO(IQaxZ`0;t zI<^fmouJTBMx$du6T29^PN$b`za{z*5QAxf*BVVFPUbQ^vTwEij!SOnC+O@&^o4|JrFm#S#Xydkh2UA zA`bk^2~}3=J*Kefh(!IdDx%?Iw9B=f4Ii1UZytULZ<5*Pu2a@-2qsppSyOxVme>yI zOxLMo(Bxu(Z+V-ZUo`D6Jtn; zFbOeT?uxkxp8at;vg4~tE8$!p>rKit0d|En@jA*$aq!H%&7+OvHcbWFeJ5I(-%kAz znRpevyvK%qJ0~hdM|{GZx$tflpy99B&Ma-&5ZxY4PPf1x zBh=u{kMD6AhY{GTs;MTbS;j~>0*m#luzqXPEl-t)iD`p+S&tnzCGrIILs zjfi68f#o^gtf6hd^{pp{x&{UkA;oRX1N=yGr`~6U%qiN5Km#(7WX-C6^<1@y_guc` zA`z?4e;{@k3YPn>U~;S2t+U_Eeo)2`l36d?&MGM95$24q#>h8`{CnWS4KiREiwSXOMnDU7Te4Ry4TdRAfa zlH>R2v13ngQ8KZxW3#6;M68;nUf6TB4X0xO8@3Qci|kEPs1&MWc(*oXABI)Y4r%As z3(Ey?0G5aBJ??e-Lzl-`t@_FpE8d}XWPf4>Z4D;hMbK_xi`$S^LQH@lr3SeGnZB5a zN+(YE!cyr025b$ps0KDk#eH*)Id|j~IJY!-=WT1voXuA~CFHwqmZFxJ$@ikC_a~)? zER=~BI?$w$Hllwm+FqBwcTRRa(EUuWDahCN0eDTOpI%s{IpJ`s2S!qhdiGWccb&Z1 z@kZUG53;h?)*Vew*3sX4s^yTd?qc6|om3Xuh4V{PCzh$ZHA%{hE@7RSIfzey9T!N> z3R^^MALEz~>P92*5ykR5dk*8MR6cj^Ty%Zq<3pF{oJL6dQIlNc-po!Nzry)l4NI@g zcF2Be*e>dY4Sh&!mtn;>x~a#HH?y;|e>u12ym{hBI-N1QFNPUymKXbuOB8LjK0M#9 zK~WE49DfsfMB?d}o|#K=CEo78^Ba8s9KU**rQ7uAd(4j{fC`Th&5&FGVxlMLhZ31p z0R!$)z{KZ}W~^Mf(jN&K4214IQ*V6`I2Ng{jeU$=8cM#h(eX8hms&b8c}V2j&0L7f zwP0nRUKFomcj|m?{hG)l5|re07JW`f#=V9<_)4+BjnCn$0wnQei*@|=`H5EQ6D&9c zy4y&;yK|=uSuiMkne~E8pasOl;j2kT*@_h_e2ew{7b1QobjncXL7M<}X*3|WH{qeq zuJh&$?k^4@;u!@9@fAx<>&5v9aI+#o?&(Fvz?gUN%a<5zB_uC=f{uu4wJz#p&fpkL zMcdT=56=z*zOf$xW9qrXpq{i60>88x8k|xe*d22hd!i;_rCNZYb_mKZ|MADyswmk^ z^jL>U`a@JyI9XX+6VoF^ssWSMCY_RSL&QoJ0#FkUX&iJ5-~p<)kFoP$xV3NA(028D z-|kkPD2#lEMt)EjPO&I_ zIUV`j)y-_Nkm8wOFHi`+p!48`gB63c^iHjsUB^EvD#$3+)b4+Ud;vc?a=B}oDRI9o zyI$1M75&bkMYXTR!^pI;`S#9O^kgnI6a1BW@U@;<(cH>^W`0gv=;nL~4RoVy^4oh! zj}~1_G5y$7TbC}iT(~;LbNw&0a}zm63!VB4_t{J+aULQXEPFL2V#ZCzMVjT99+1^w0RlqyPFuNkPqJi_E=gwWY-?+6L z{c(997cfJQQ%TmBhOLe?S_^mcs%Kamz&-HgzsBF#(pIist%3u1Ff%kJCI&4KnRc$G zeQGH_wahRDAp2m3&XFUpb(60Ch@E#KYy-BvlbgFQu}xb}&IcQ3G(Ph0nb3xX;{U9nj$snVv#=;mwLbVIw!Y#4!J8a$f1xjW2K?qJD z`vTSpCa-$XM%vAFEJN6ZsZxtP84(a4dL77r%yvqV+YtbD3pzNYA()t&nlgq>ozFK~ zW)j?SVrYjnE>jPz^a)1U6d|pM%XmCaP=dXtnl&h)1Qi$@tgcq~Bhz9a!8pml#n9!` zxhupd`3W=i?paTr2QEVWZ_2>UR$3H(e(@E{edt-^lRMqBtN&AmG8!-=Vo_$U7x%t#12K<-?zr9vby|9UFN$b2$9V@J`8}GFKiPKJ>oK6V zov7of)&EGfG%*Ux>6i(QJ=z@;j$FpTj^w`hkqrCtu7DoW930B>hp4ND!nzf2!0v(w z>5y7IZN4B0TxQ9aJ#-PG$VdQpMs{MfOXRk`+iZzIxJuhHCkkkRCN!<+)MpX}w7bMi z;XvUCK+~k(Xm_0bHlkHN(=E#w)dmup;{4=j>ckU4jf2UYcFRQpm{kPV#!*k;EPpl@ z78WNL7bD}-v=7xoayO1I(}TbEwB+SM)bx-QkOOX#2=G+^jrH+~f5x>;TiSWH zZ951Yv=*fRkezm5Gnc_Sy-gsgiFb$Tn$-P%+-v)Zp}H7F7Cc$RUL;;efZfdtS6Kul zpU+?T@iI)jxAbPze=X$MJTPgG>!tpJGEBb!KjB|Y2;^X}xu$Pw^SV?g|3k&>D;rX|Ql`{-^gQP=8`DgR7| z!GW4H3*Xw)l%Qny_6pu~RkOA!eU~bg?^b+*hQ0dMk}Sp8_AOiTN@{xI#0)-|X6-mg61z^>gGuk++wTH5mZ*Que@VfI|F zKk1#0l-dMAYluZY@_rn-A$oMwKNpqqGiASd`*x?GpkOt)UJdj5>!^QWjWO8oX!_x4 z1R8|5l>zEJ&xPSNv>Bh5e4|`|*Le2%wY4T-j`pC-z+OO)`385&cT7z>uD?h|b855V zzGW2F<@jAjM##SL8T08~{36ddCT3Fc%WCSJt`;>k3suK)a1g>EqhK6vNd zgnn{?MDH1G{2%$@V5QCnfLN?3QQ+7)iS4~2k(Oi>3Gr{pM$}T#U`KsDWoqT0g$xBr z+9*->FZz#P1Mx}#Y_z++*acs=n#RTvE^=;M*2oD zcO@R6)n?TmGyOtt!^IrVU+z29ptsL**&nbJkk@p;8>WEQ?;dbSBmq7m-8k&`xB?uG z7Rmr}ly z?$Lt%F+Qjl2E!^iarS{4LSP;ix6${rb+oO!!vt_x+Xz|*lSE>5Q+$}moD+%~N0Ha~ zvx$w>D3e?q>0s-L2Y<{zal;0->u;Usa@3R^eQ z4(GmSg`+SsX*u~tKULJdG*0@yZc zSU~N4BFA|Qv*;8e-R3k`f=0~E47PYsuSG{)HKpa{5sZEV02GqZ_ktZ5Gl%tlT#&@f zz;k6}WRz4?0@;cS3y%*BOr&a(`zV^gj`8VJ*_N4E!C88S`w8aiT)J|DVvS0MDc7*D zLOEsE@k8_sxQu_7;KJnN?Mx~ti{l zatz=U5aa)z!+F>az5i(n_#G97S3G3H-IR0NS~2`U_kH zIy*NNvqi`v9zj-2mK?r8&2wYrS;Sv!|MpdvSt4Lp0=;4&LkIn_&?AE9_+9GvqYMjR z-eHsPX?=MK&qKxmY-zTwBDPX=qC=j4$L(HfU*E{Hftfk++Io(izyxc*W01Sy8=>sz z_yUoH-QS~D#eHr-2Nv)tWB?yuSUqC)_-@tcXo(4kd^m0c0|UW=jA7=lskNJWE8qrT z$HO5AzS3>-$HrgyzD~9Y@8Z=ozgX*s6uo z2tl}T*4mg!Ok>;WQDc&dYM~aO5KJ4IP5y{;5xrL=ZZ8EktPKQA1(*tLNk>R2zHo`ao(!kP`A+5X78DJaXQC-b zimXS`@uP1zxwwR5N+s8pu9;=!7(WUyX2PGbdDDxaPao)XPFj4OVOnK<>(fwfe(F0q z&-m`;JCxt@Rg|c|Y=p-NsT8<@*SAwI+cC(pWj89+SDp>Brk}5Md;bq8QbQx%uF6;TN9xBGO9e1%ksnVBkB*31qX! zj-?~0wy;}LDCz-87t;|HM;)+4AqAVtWj&YveI$r((4ePYM#oP$h8AH{hPVHCeEC09 zr!QW^QKSyaKsin*lukN+l*%c*7k4d8aPo@30vyjj*T>?2y43%B;hzcTKMl|M%I?KF zgbCGPR^pcyKnmWB!c~F`56W}bHpfy!1hL}c%?Q3juCEF3_qQ7};clanS3CoL%{JIe zxKBbUZ{73cJ&qnj4xuWusoJG74OOw7M^rz3{~s+eOGJ+OhxV zrIY<1Kfa6@8_!jo1G+lLl>EInj+JyPk>BSVP=BxV^s1bN#Y(pc-8XZy4*15|((6?j z1UR)D$ywm}iL50ZDPUy1Z@)<(3rzB+Pf1-!^NV;kJJRBt^Z&hdMrItjCw&&7^uGCi zF^-Q*DXc=&WvU}-HTap|-(ThVV|9h##T3udAHVmzcvRooM)_;j;Lh>EJW%XD%in!m z^59^@#~l<(1~pM>J@f;>OE7mjLEA%enwIztV4rVhEs580Qs=%`*0HHf-1U7^mPF(lCjL4 ziiS_h@qOpsNx}@eIE01}wXPek-E2u9Qlu&ed6p^?xzE5W}~c zt=>pWrrpT81#@?BedaC~zNbL#077kiQO#&0};%7+_T z2s=EPup938b7_0Ue@|X(qG{35qo0cE(%ecn)IG{!Ec&zl;=5!;v!~fFf05P5#aS7@ zou@^We-94-UqtsGMs)vgc>a$c9ugvj8UXg12JA(@abq+NYQjHK6#e!Gih3M$bhkoz zX@T*S4(g#h6haAVSqcsykoMc@>FERj#lC9-pl7=4o+Su!sA^IJAy$@BhnJ0DfDmFO zYAU8}uPy^N31aJpV;P|*L=q$jWn>~eu@9yNlbihUQeYs`{UbeoOd@jN5T9T1MCpCJh?})?$7BB}MuChs4KvwC0ccf??XDMV0K zP#aRRM5*`j!k6UsRcJ)XCJn-G6w3GJ)sG%N6gd5AsdSZ1rC1D%);grzsINZ1W~wJP zS`FT`VSv$D>FDV8nSWG4jp1X4O$@xPqrvDoho&CGWnIU+sM8KXOeqjGpruiSC?63K z0g8bN2peOs*DF2-atZ8AQh_!vJY9KPezK%~mZU<7!w-HM3H7zb+Xn~vJUr&jjC*n7 z4^K>}qfVt#0jx5Rbt%e8P8sYt2J8BIRFU@)n%LZAJG^)n>J3;?#-m@~!`e(kA^55*cb3G= zqT@bW_Q}l6&1r$T8)V^t+koM(FDQx+!4r!E3Jdr&6#Zd*2Sn;d;z#FP2q63X7;Ab) zg(w`G_*M z09k@cKz#vq?gE*HK+#|h)dSLd=+NS(jySozO=c16Jdqm`s-d(Jf^8MtAPC!X5_6M} zNU&3kiM){%k0vg7#yACkhqq9ek_%1D9SN^VuAMhiW7(M)7oM4_$VX)$vjRDQ{yC zUUdF2weAmU**;@h*HX?y!5Ik;L#t zaAu4&A^rg|Czwrasv{r`EH;#@^_)8C39At8I`CA(M{AT74K#su6k1g*KEC^{ws@L- zzZPKwpbg6feC-&Jmf&g3W6bOF`Ss6k;|)e+EVRQ4%h+VUpQ9B?JkiJ2JO=m z15KFO^?>%-rFW^tKV`iBdS{MHm-5XxYmj-?u!!r>Nqm0_>yZS2k-g9bV`ojTGCQzm z&q;!7hdF-!nE*N8@H$|5HUkNbdHg=0f;dK7{t>G2WfF$NB9o7Ry!ll#|D=;CGxz81W7?d)lf3 zWo_!Y_lM5D^#M?R>_Y$n1jCln8I1xAb_%8hS9))MGc;Z2Bo7~(i=1LSUTC0#5v9#uxL zO8wYCKH;tv1+x}{WSLAixsshB{g3L}Nm|V&jR8+R;2}GuhmN-@M1M!P5#*sCyz~{bTfvDkumO;__P#iC?~KY z$IwpH6s+{_uo0;$@F2!M*;JAhXo4zq7-rb&enn_$ru$MVM=?@Q@E5X~k*Pt5#NMz3 zN);_!Q*Vp;h`uy&CRLHwQ1GaQaXu*(EL3dnk5*_RIeJKnuPCSx3IKTt7^j4~N)8J6 z4>?sj^VhBv(~x#Qn&?rI*6CE0Sg$6N;1M-rX@)U3)PjzbTAqGOK7 zmQKji_NA5Jl0(6&icQhrnT84)R1=8;)(|<5z^aPJ*?2%l5k=@)I6==sV=Z#N)f1jn zk1_a$&IfQBLGkdAq|FcuN@3K<-_gz|VLgY!xvsJh`%?H;^%9yJ-heGH0(}G@gE&f< zwPDUZTy}%aVohmeYxUcHoN!V|*%{QN1n|3eadJv+Kj!le=i(>=x|oaskm^SZ=p-Iv zZkjY_XP}vOb&mYf3|YgO3XIPJQwi(?@CC`f+u7S22k@Ym6kPE}WZs_y3cpHANu2;I zoQNb*q}Bjihm?OIY>vi0V1kG&HgRn#vlNT#&_NP}pp!Jz8iC{n!L>I9Bd-U%z`IHP zpdJ!F0=7tq0G2R#p>W)?I)@J(jMCZA>0F6Hi#f3$<5iFL)n=!%Z?Q-uk`I6d0$F`Z zV~<9mZ+~Lhk}G1}aTUb;T4)0Z6&z<_HSPulnqGl=m)?YQ!e1wUz)V*`b=wY{SQ6Ng zLJsQYFT^UBX%xg^9i4-dmow@Z<|9g8O9_J;LU{9eaWJI_4V~=mCh^aQhR)leKL$V< zvb$mm#-c&rIecsKJkZQXUr6-0nOQuk{&JM}Yf6M)x^$`99Fpx$@0&N(P{6>kk^qWu zZ^RYmOrrz_EY=ZplT1U=RF)9&nyk<~$J^f`?+V0hN{0?z#7+_cM2t&VSQCZcX$;8i zz`#gsy)7xc9TwODs|T90R8g_Q43yQ=(-Xn{5Ke6qoju)11j0>L}k^Z{34Q$8#< z*?NgkO9^>B$7P}iJ25Gw_SH>6V`u*5;a?1pj(S}Rvye{)83`s<_Ey2c!rbkf0= zuXYsvQ(JJN@PkZYg*(o2LbsSb_{V$fz$(3|#?^0eowo=O`l9)?)5^#{Fi6$$^BG^l z_TqKVOzU~Gx*^|ro|A^4h5=J4%?Y!5FCLJQ2KUBx6gusDd30X9Blm-i_y$J)Lo^2) z+h#1rbn}r4XWYFHwsA%X;P*g5WGXa>aP%0a?;6}N%8S31PxIh|U|B41YP&VZ@tF!n z%k0y*_SvD&Zbt)R<=$7R8WSn` z`H~<4Z?zvGT*HDh&o*NX%GIh@&GdysO+!8elj0KuFi+(b8x%1(=!F&D(1j9U1^M#m z*634IRHU*mY`@j3#L8_RGl=lWL-E;qpw7Y>v<4rXXbjlH`Ej7?PcYnX7OAuG0~fAA zUU7=y0Hj}8l^<}_GlJ+LKrRpEz=s5*`(e#fuMqp0H(G!j+9cZRq^32GjkIaM+qCW7 z#q-t0#J2*iMrAv0~{8gu{|Ud=$gO~=-d?gQu- zTSFpcY?r2he|MgI_(zj7_)^M9DeX|dH*d<^S3Ztl{IrnT5%7lh!BB^k3ho&YwqWD)ASm?nKxXGy==XGBfkYX)R#Qe}bF9}q3bMWnO@LqtjjtG#6+yg+rLV8{N z*GF8MjWWGEYmiJ66+;O0T`>r1VuUMBWB7`?_f3fs&KU0bxxsmrhev*% zMc*SFd_k|R3h0SC1KpB$YNy4+!^dKxDH&tCgz1Qh;sO`iiK1gd8QQCO0-a*89^f8W95|ts)h`fs??N-{4 z{grU=6I}>qzACPj%=lJ5w#;zJb^SF4gwL_>SDA_)kfb3;#g*vCe=2CD=~N)atXJxw?n%^M7yMv14N@y z_^MsF^dJ*q&l!8!V=!+AW_oC(2iFN_2hNF`ZT0AZ2u;?;9TYE>EPHcECO^K_?0C`U z96wVJHKL)=QcqRqnfe?w;WowzNVMd{km3fjm=+#(vKww7?Ay6u1=z>4{F}Ozq`NQ_?i39S&i=Z>aw9$?j->;?sbe1 zEdvzAPZCvYlAag|eQ>HOOx*GO9t#XcE8FoI`dvm~#eF=Kh5%n<Wq}nG$#OkFv-wNRo;44N3ztQ^Uf-@O4h$@EF6{&^VB|q(he|_u)fgxoKq! zR}SB&kbagrw;{cmp8N4%3`rUTx_ecE^7NNaKL4Y&GlA+kZ~wk~%xK1z8DoaA#1KQ) zQj{fovQs2WA*4OpC>6snW-Li+RLYW4TBJ>-;x@=q(u!1wv?#QWQa!J$ndg4a|D5Oi zpXWU1Ip;s;@8;L9-|zeVe6QvGeqZma^gvZ&K^w^Q1W82TW)I(MFr^TIx5`tuFPGjg zN?mGNX7|x8ZC}tEN+ZTiSrY*Ekh$Jx@yD!lVnHJl;7rPYFBJ&_b(vBTV1n<163cIcy~tZrAU zwa{6U=DE|^d8^st^y#Ob+zD@4zytPfSGTg$2~k`1?-d@Cm`nF~pLjwZiP zk;6~uN;!|T^?0SX@}{I}S9n7OQJY&SpA?~b9I5lQx6&(Oe!$qSQ%HyW$ml0;1 zmMYVu)E)hp{*>hbl49*H$?M&_QX zIG0y~w`((-dv3BSHBvObii>f+48Vh7w9@%X>X+{cy=;c!%seOf2*bT* zP79+h4cWO*rr#JtMFl*Q{$#0oiF5K+Ba@V_V+d845}CEFAmAnH=P-cxx}qYKmmcXm zVHQKVPRm4mCo|t_b>-NU zfVCD_TYmFOgbF8^6Q=!hz!40ufiR+W{NOV@-cM;)#BqT~{%PFuQDOT^Mp2yU~UBkd*)n6&O- z!+=sALbJ>?aLM3E$0@sNgE%U|q(|}b*x{6L%uuz~RZA`KepAc9Vbk(8h$WyLUKQ7` z(fNiMr6dKpAF%$5IB`k3TEb=}_@Q^6Z^8K6{VhBH9_;Y@>5YAxnd-Ze95ng^w8P`u zubbgI^fJ<9u&ML9q{PH0ORh%DRPgYoGSy8QXcVHJUy{Na9*Qrk1?sIt;ivIdenIZH zO+9Lg4Nyq>n{Rme0G)aP65=@1hlAFZ&wJlp=M%lI)9HX6X-k$;vcUdaAeRLt+JsD+ zHjPxDcPT1$lrDqv8SkCbTV!C9;U;m}vMYPQ1$4aO+t3cc;l#>DW8p!_P&>lb>1KnB z#MxJ@KxtpWPMW*9NleYkhk2$8GIOz^vMnemsC&Wkf2O>Icp%V6% zPyB?lF~2!1YpPE@$Yuf6jEa1_nDEI5-&{TJZrCKlt3jH(7dp2XIES~Mr<4qxHA%M_ zN%gzLBa~WGng{;bY#2~H*&lHIcv@N-UUrrxa|di0s1gUd?pe<8>g6KO%iKW*m!AfY zP*$$MLpCFK&*8)j1`@_qH0BtJ38h~E=mcul>opC*(t|V1T@K)sOQLyetsTca(@yWe z&4X>eBKXr8_Cb10&<_51$q>}Ut?i>}Vegol`Wp&LkBfRr#j^O1@iPg7G6e_4e@ouz^w4(p!(!%8O^R6Io6J)`b=Huq0nBEWYAMH zjFiTFdk(-r94u$@F8M-9OP2~ox-xQ|rijDEJj~a`9CMFHjk%VGN)M<=PlJEcc|o`- zh#gBBjsl(GW3a{C1(d~`5oP`&pfbjRY|{+j^PdwC#Go;*I#~b&*#iq{-;90iTMpHl ztslIh?*YLw>75FLMG93^>e+C%SLF0AS_!_h;3AguFbJN)kqiSOsI9G)5p#B#fB+75 zMMk5!K6tisQ%wXk&}_d+Hu;n$p#o&mUkGp};wToJ&J)Hm&WJawr_p~JFsF_(<{~iw zE4Y<(J@Q3pL(0h)1SRYGK0T8f)dPF!UI2z9F$kC%0VTQ@wT^Sb5 zHp!?c+$4xZsPy#tV)iC&QxSXn+ON4aSB$WtwwF$kaS%3`o&hA2;RTLY z0bKF`=Yd?hnDx^{zt5z0P~tkyX3jO`uJgt2tiyNiwcI$F{~8(%VP7CO*t8Tx1LkK=*rhQ*=7hC=7ZE ze*Mii-{fL=SNQ!IFXnP1guZ8HnEH>+W6S@+#s&@?!zdkT?qy`u&=}WPP#mbn#Oa^j z&z6xS&87SQBR!eaay)hjtO}0EYfd8dK6@r2+b!-CYgH zSB8wY(Q^5!nI4rUoP;VDomx)|ZylK~BFB`&k#+Q+G7!$gA^mIv(A75Xnb1)}V>4BI zkt5NyXV26skMBm}O|gz!o-br1Woa&odvkHNXF;5X)xdRJbx-N=VD4s>liMbNIb ze7{x0Pu=8pcahT(vxEtHRb3rHSTcuB;L_`=?L1+vZIlExk89&zt256Tj$!a+j+ol$Wr?$t>xJLat(cibn4++N~fXrOKTvY4{q zJX5mepo6*48n5sr0n+c7+Xog`y>^;z>&eljk|i0Z9!?t~Rj@Ez*x?P)bu8^gnQb7Q zuUHnahGqr5yM)rtic`-B_Xrlt^V_xH5l4_|ni*t+1!^8tV1`FOw*eT+Q*!@wmV3<# ziP^|fonIZ-$-QJSuJC)`DZHbRWTdaYdIY!AhbM(AAjPZkc>~Q!jRaqrH+EdI81tMoMMLs=ekOsH9rafcU(JMxMIKlQZ zn*{`S5{1ej;=GtmGI#f|Rs%!Mi*kKL+=#n}m}3ZYh*6^+!81GMw{kLQF2O8Z%wL#! z$)fdw!5o4KTshcMR=082&M;2>*!#`Z7rruT#S?7-{3lL{f+bpUucSK)q$ zctL!$1ci9^?Ac~OJ;HDm|D~C}_kQwli%#0UPf|G2|6%y50os%Acqu}iv!>MHmw`){ z`9HiEy%W`aD4dd5twp=I3tWkHixIQpvC{@VBUZt@8qm%Q2`kgrm8}nK7R}qLwD$8( zSI4o+ReZ1Vuo06v<&gXTx^lSTzMJFW*AlCFCaT2NB6YW>K~ATB9;aQgDR_1p!G5Q~ z1$_E#VcRv>-C0k1ujF+UQzJU`osqX9g5@R7q$}~sL2N701KtOB&zPdytkGj4Y|p_4Q>=8#qunN+ggZhyT{h%ic70|=^*110q(oFhunNg@|9Y}C5q z*O2mru{75721Zz=HRP|AF&M1Khlps!YD(IID5cTHsq3IdQ{ObxZ}LQk2E#8}@GB4m zoI<=ZL{|!KMv2dY|F6NTu*zso&ofG|ra84dNYeuV~)Zo?{|!z2*N zxswo2!WLNfv(G*g%Nl{K?i=Yi#?%;7FWzsAZ{ezg;Ru?DEy=yc#=5ZjMs7ot1tvLC z#EVUUVNmP8%kQp3s-{dBW;a~+?r_L!ns>i-jAk%#um=|VP9l&E$eT4XmwrFpO2 z>kDo(-g_?<^EI-Obb%*oRNC?ilH#^3dQ(S!C~BaOzW^sm12-)18gV=Y0$YqJa66KrN!@H72bm=nNdz%1`oY$* zKRJHhdVNB_j}H=>o2#N^_LUBB%ho>%0N=#&wP~FV>Bm>w-R(fa?+?x%+Z>>F^b1dW zY>5Sjf`k+UB6)Liw+OmGsh6XsszL&tiH4dR zGA<_2IfMgBq8UVIdSp+z>2PY>FujqpL;+kH{+fDVBoFMz)OUV! zZ(-}6=VO$x>&Yf#gpme|)IScKuxU>4ii-cXwrC~Y>Hr(LI9LEO$IGUegPO~Lmm(m*?ehD}*oA=viX`ubd~`m)#OZ;CGe<$I%L zGiL1JS?RFz;gP+UYcZ?2eCVmtaFZJx|6J-9jB%?@I~6aFGD>+H>6i-FKxH2D{^V6} zfNiOJBudCAc^=s?WZ}Jrn)oQ(-~Vq^Q}G|+_vyxy<=H?L_&B)#fwSORs|4o;W)u5K z=?EOHj&gCBS{JotQT;zA)FK>^>WGgprmFb!@BbY=*eX+N1Ol3XLC6OR6LEYpiKV1= z$)Bxl8+k2P@#?UlN@9f=6ub1>cXwn8T)+liAx=^25lew6K?ra4j;}BecUP+_>2=gD zRtax$Ba{`R*2G?v>5JrD(8f|T4y&LJj7`>x6}wzuDdnOX75=G7FLmL0rOiv`H8oIk zi%%wKrSJHSvx1-Q1g{ZL9G$;$O^s7~Mh&=1wv0iGmXO1}uC?Ox`s0V~Zb#2Za-Dk% zu&|->QdA{fy{`J7GkC6qs=w3iAOUvB&GC;Ps+C)}Gy^eEWBw#AWmNW32h!dd5(k?Z zk$zi-0r*M)kt^Mbf(jFiX(x>Z`U#rozAlvNAnfu?PclIRP}_V@^1!j~Y14ylM-Nsv zJ>5Iyub01fKu|Ph#SJx=r#w>h9~V{MIdbQH)3NK5fYnU17 z%P_tOk<~WDAOwO`kliqmyipzE_aaZb*skK+rkycC40z(032y}&X(oQc5sr~^ND2{j zQ`sq-o1iO23l!_pWK9|L#9F;83A! ze-Nbvr?ldvfzjRQOH0a!)_<)txoWj7z#HFl0p|k?Whd`Eza9Sm{A48|TMAeEy0XPY z)y2|QN|Y)~0{)ZuByt#5xT%n%I3i;2*6{Ccu4DW7mWeN zYf4K?8%Q%^x+t0&xc3fO11_9oG0&(dn=l7?W8(aB>RmD10wcAYwCeHcz;VPN@2IFK zH=HiSl_q0c)YWRYMu(Uu(lwoCY6+2VdTtWUM=mKtEhPY)VAzG`uiK3#e`h=Ts^8nm zBktkPR#2(%xy{E1RO3=>$rg;dQTzA`D%>4WGTd1C?%DAO9Cg0!)8_)f_9dj1C|CcG z-kZZ)QckT}+gjdyl>yMmnGupeS(IyAGemDA} z00w9pq=%Izh<0ox|A~HYOF}VLsbG_XaYzZ1rxRDZpTq6~832Wyk*o2|H=Zw^XZ+^&Ua@)R z?alVaikN9q>WOGc6kIn8kNx^<7S9O<-9)xoh5z(1HA|;$rc&Z0N+fC1QR0h=jVd~y z=pVb~{&yIRiZvWMu3ce;HL510KAr_{shfC&gze!3No*40O5fN%JV$&rXct8>J!sG% zsr5x@Cf5-hVWIl9bdaWolJ+(O>^iXG<^NFo6!2(TDfV2j5Dx<3KoFtar$DD(T@EKF z=7wa^HImGzPxOJM3boX>KFA`0y{2swf_u>^RdLord?X;}O6aJS@5@lKQTJJJ;hv_t zCgY^?_PvX5iDpvNS`o>4dQ~VGPQFqMXE2doRyX11Rvp{Xe<|Gg*?n@0|4%B96xUr= z>YHvLT$Jb>sHGM;odU@3i>Yu%J4Qa%EG^W^(-PgW=r72{ed&eHirB)uh}hSc`ue_p z%EM_*{#LE<0#7muIcEH`jVLs}R8&mxYXX1x0bS6xlGGG70{1Jd=Ro#DW!fS4fx;*i z*mV~L-aOO?qP=0~+#@!m#%6Tu(L(lyy?K%ihfSmw z9qGu59yVWO9dxF8DK#|h{eXtdQb+Kvq8MWD;=_=-$QMSc*^1e!^o50~{@!Gu68Raa zrfW1Xq1q(+p4wM;`&s+AdHm2i*hKY8R0$p~J6|3eahTgs=sIuhu`kS8t-`x>@4KPn zOLpygWpPYR6cF}n|`4{}w-KTVLHA}tuzIIDJGkt;oZOk>tX zjEI3`SFtNzM5X)p?_X5V%KM0d*ijm)V2FOK8SXaTkFj^TUJ3=Y#B(AoY_hTKGztay zbHQyWfAWZDR&|2E&_6@nEsX*r!q9?ZL(E|DYgHS~Hs@cr5;V8a?i?a6x6`@Q9qi=*GMV@rJ% zoUwQk`~Zze&rMrgYagJh@GeWKcbYz{$Q;}m@|>sq%9I<6?m92LOA%$21B!ZH8ty-T z{aOH(0}6!+ZPjRrc?=@DM|5`SSE1k}`e#0U2uY#rarNl>eePL`s%1tk&JY8l1|$hh zEE<<>riq7^I0uo%%^qLgXt>RV`=;-($(;Tj%g%-EmEvB$z7W@NoTBl7B7}aA z1ghJTRO3?phPE``%%!PjCBzsRFuVI=;XrtN<}Yy3b|(a8TUJ@OB-*!hGhU;(W`bWv z{J2H=t?XZUDHoF}GZ=g1k_L_ZDTeKi`Dx18W$CGIP&iVF0%pTMEpk}9#u@Cq{;5S& z#a|yv!89e-E^4Jo3EU74EsF!#qrMzGa?KrmJkXqcm!0#LK7U}C?Oy;30pS2EF zH>grL{K5Mnez~(fvx{!LyG=rs8j%V|V9>&Wd%E=*wYjA|cKStulz=ZP^HG^~W-FmTAuEr7YvfBH9KKgbEuC1g)beLXJtD&zSmod{ZXH(w;0_Gwx5UWo( z&-85HcM7hrlq9({M$cH#;(rlVK(ruK+elK5myK(3vWrUFb+7eV(!1b(5>yoVY_qT^R(w@tZbI~FukDpU&{iP#v3OL^Y?$s4C#Ggx8)A!W6R3imXHDB*+pBJjqHiqPQsb~EssG=RSZ;^RE5NnW95Pb&hkji| zV4lp)i3zWu>5wP6LOcVB0kKv=s4a*KXjpX={gwrJ0yn#P2co>L8rL6p&(f*MFue0w z)f@N+mn7re!p4A(AIv_eG+kIrKnQa8Y4jZsh%O4-+kI=azopH`;u3S>h~3{yP(wT+G{nkSQatL%`&VHMMn>yu(>t!G`h<9j7!_pmDs?ex8VWm-^i3s2@R zZ2#oT1WISZ>sY5FMW9sfy@fEFdv8zoBd~3wUK(pn2TEJ+9`ynXZktHrYL? zb!bWj!Hv`O!}tv|g3TN1>zR={&3lZ{O^Dhd1A6rgDw$Isi9{>RA>2Kg?Fw7N#u%J< z|_@MVB9^}NNB^5mf5@K8Q!^{Vz;g${C4zphskyR z>4xPBp|1+Ql(6T#s?pTm;J%b2|kAv4LO}j?so~YC*n&k@<3;Cy$!w}CecJz6xb&j4; z2Jr->AkI+EXyvKqKQGJF(HvT;TBsPFAa3{7kp@2ih8cqyXH!JH?J_8CO|oMI0Ufsv zvt(l%|GjRVDr#DsX|8b$iMDT#3D9?IDsqaC_TffW=|}Z;Og;!;POUo#Rf; zR@xhL^b5u%;tSUc^rBw}#umntQFtX2>or9dLWyOt!0cr@^X4(cWaXJ%{Lmefv55&T zQLVfAW$_=}2d+fQDe)z{`rw>7Iyxb14zx~{`dzMqG{AIfW9;vzTb%=)`eDMR{ZcKh zEq9E%H1Hj=0hRdHI4(<5T!hPu)z7v5c?b^Q9Va+vV6>x`*S>FqX<54@hio(yMhki} zm%}(rS0+2+9-xg@?ijrVMBVHcz}~VByRuAwY4F5|qQ$kJ(@H9;hMJ;osTsA`LN3L5B6^+cb;BHbb-c?|bz+!#yWXts{dkR{J1NJSoqz zf36zm4ZJ=s*|bRZjEEK*f~g)EPmr%oYhh7!f#b8Hl;)Yjn8MI{Ke0@GlWGCUY5RPJ zuB=I@jvO!BH>%^WQJqX_t+i?XSI~p%s>lP=Kcjj6XJS$}8-!t-(`!?R6P}w83Qe)f zCM=~LGqAN`9fb%dy5G#Yy1mrW;HhpxQv$eU1u0ELa*VTV@GAiGl1NFQ5>Tv2gXA7T zQ#mA~k@7o);`S#7uBb|v*p-NhBFvxHAAkH*(b(rZc2?=uHIA#=Ws3GZdw|a&?r%+; z^k;hGQtCZv@K0A{Suy1+vL&@XNv9`opGB0g^EJNO&f>C^#-|oALEaRxBcO6ztlzoW zjBZk3@X{7$9{4(m-wH z90`n3f@dGzw+8~kBk6FkwaZ4ia?ni6+&3B zcYcwe0+OSP&O8^qRLddEX!nP?qFR)tUs5y5(@kU*`CWC~Gj@dO%FSjZK|uIWKs?bf za0?jW`}S}10Rh*8EdsmT_Itsqiw(v#-%iXHR7d!Sct9|Gl}kYwUBQ z*1^hEdjhEiY?DkHf^r$<;5IK6RU`}CHugWUmENBrL_Glj$wyl=x+N+!|4TI{%G8Hv z7o7b@_ZToPNXwjql;WF;h>4D4m8BVcM{p-)`ozONY5gs>?nqo^Pb0VZ4%2z zkxi51jF_vn3X6wdl#L{uaSMt)30@4Hv#TP6w)nKBooG!AxbvJ7;Z6wti^YVW-p0~( zmS9`6L#6#Pr=O>S(PTS_9s*>#( zWF>a}qZMaoeIcWpW|0Q6qvXTovBs(4^vM?Nv6BI;K|7!Si}`_a6cZxupa@4d5DfbE zmz$B}f$gJhdt^=>A;W9cjqX)Br%YvOU203FhAYise0vDp^a=$0rosvnB`T3f58^@u1C>1;l=<9Jq|HdDV|RYoBseuImIGxq z3Ppv8tYlcNsI_7h89txMUv(gs>L)Qe92f)2;(7t_AzHFTaSl4fWe*n(!ei^1yP0omc%b$$Nb^@)x_;?91l2_1 zWY`j%=;6}P`4SE86YH1?e(XSeHM$|0LKS5+30m08jvYB2==<9zYK2{H2P+Le-{R~j zV6T|?n>X|M-VSK+{2*JKNIQ8pesfhAD$=Y8_d)Cg#I@Nm;~} z)4dpr(ju;(Xh`M3aV5 z%bDr2EG@fGj}BZ!GVtFg=u*8?j6Xg~X>9OH_zLipY?vyAW{raJTq{#ES^X#Efu3Y3QC`UR;owbNb76^?E+b{fc=op*mVjdDqkgf5!wSN0=! zb!|zig2kTRytgVHMs)4p&gwW^KL70>U(^35@1uWQ_6GkWVTfrh+^i~F8%113?IklD z(;6R}a4jC(7=?$GN56Hl|G?vS(EAw#tqv6xsFa*6IY9ELg# zAen3SN-RPK554*l=bphnYNr-SSudso6jfDp%d(?%i4U$XdtAn|Q|y(!$Uh=8wQd)3 zJ_9R%OYq|Co`Z&5?itH&WC7?sWvE?FKhRZ2B7=p1kO&n`n>i774gLCNm0ZHXWGiIrq3oAt^)AW zO9bn5%QjOlPa2Jsy38qS5_epf-tt!Mh>~`(#yh^53>OPG`Bm@XWq_v_z(_8nr!dowO=b1*FeWxzfaQj^zreLVJHT*L!Q>~ z9h7w$gpI{Q=`NKb-!R6HQSqrFP#3u)@-d47Aqm-p_M^!9=zcbNp`P;avAu=jX~v)v z{{EwNLuW+*rHI}Zl99n~nNklbH1N{RhDmyD>xVck3pP%o8}!-QPD1rSF|ed}4Ry^Is{wQy)Km#@xAI zj2Sb=y=C37Zw>{R@|;a|ZT4zzb{;iq)UUt)K6L2NeV;z+4@;) ze@QWaJt3=GYLR1qC8dyy7k}j$$q3tpcvG=jT|?tGEzXypf2MC>(7Wp|TQi!Pn%cLx zZrwWk)TvX~6Ry7pKk}ct%M&f%G6Mr0L&JWZ=vw#R+(>!y0GPNfcb;4Qvm_Nsi|(64Q`M*7L32yXn@=`ZNza$J zJ^Jy$!GocfF8NJps(rn1l#0rmbqSG?M`i5nhhuYooa){-U=J|UygRHf{_2-mxw*Nv z2*-vGAHFru(ZOLms8>G|oRSpIoIh`pX(!*>vM#v#?pQ8&4v*2vni^}e`22+nJ8#&u z>E|0aW`c9|?r|G|(q#9xsiC2vkDlJWt4lqsX(d4Y^Xvgvz`i4*5+*`kasNN`ltF$Q)zy=M}tT#ZVvls4Jk(NR@X z^DciHsiI=S$+e{PT=hPm9So!~ZgnZpME4#&j(d2_Sg@e$g$oy84ox$MckSG{Gi}>G zPRzewjPKpMcfhRaYf1WPx^rgg4;wz*)8Bv5#*IU8nJ@)~Q#1GV^9v0NJC~Aj`uOp$ zQ{7tc@gWicpS9b+9;WNrc88T=x_n{JpuX?5|MSQ35(DIKH(i%dBY*tuUlI}Ik0ZPM hub=t<^}@DJ`-klCH+1&R;Hz}fox5aC-6qQ9waakT6kN>BgW_LAvY)bW+kN%_NkX)C3bmq)S>w zx)hjnoa=$_|Mxe}`1knFK6{^W7{jqxFq!j>=eh5z?)T9J`E#pRu&!WYVp=VA{){3M z)8CU!Op68oS&E-Xh`8wDKf*W8s^3tuGQ44b+1h|f?(&Um=2kb%O|I;)GqAQXv9c88 z6XFv%u*3MqjcYa{{QMUG`2s#GYa{->U0bihj)oW;g@;aeuAf$b+b7QX+^^bgK@{%hHZ@?{I(9b>vJzVO}KMQca?^#*Hh zhBGhx@EX(Kt_$DYT)d-e@xr&;Z`xRr-#6tc#mJisG-WkpSOm$3ipK3%OZxob!)aYz z-Jw5!E+%TGS0`u)9x*OIAulhlYU@7!W7*7jn-?Y0eRkqpl(cWQiQdXBeAUi_bZ)6Y zao22ir>>W>jXAov{#x?u$B&o0&OVMct%;9Uq56r~40*@%2IB`={Ow1YM)`=O3M&G3jP?9i!I<;>2Ai zji*NYoqj&z9%=X2!_WG%XKh^R(-XtRIO=;VHYxh@s3@lz%Gq|6pKSl)n=^LjZzd)U zNnES9i%lm-x+f=llWqGOj!@g)ym|B3!=t7m;#|7pu%TS=v5Uix=LH z#}6Nt2OcpSVbtmM-rJ&E-kY57RygyQCl43b*)4q9<@Z)@x$*Pka(syN)k05|RKt>y zigO&zHE#z`{&Rj%SvOOX>BIoJN#7TZ)aki*50^>!b52Z7_Dpn!)tzP*Me#~8r50nh-IZMQkxH2cx-*S4?$o^%P8|V8)tT3?IyyV==jD~_EDKY+cJ1|M zUd>1xCi%O+UEi*Do1HK^V*c#{MPDdr6Fvl&%WQ0*Su@w+&6@4U!g;kax*lv~(Dn?1iU2iL3J$5ZTS}rKsDt|^5cg$&`-9P=(yT2yJ$MthK zQtK0Ft!|C$hZuvcdRDoA?w6Q-`SfH9xrv7_y=CH6e|;k2oL64z$}@iSp)dQL+I`im zowt6zy-k9)WaVb%A_|vEvaT3?{O1#L65_MtS9P+i&syb8L>rccCSu{X|9rTJ>8;1N zr;C|xE|bNj+_h^Lr<8&i!$8&6hM-Fk%ubel7p<>QGWfF6_iL*Cz`~DPi)NRpo*(sH6-TpXlvWKe8zK&cl zL2Ph`x7xW56+<(OR4ewwGn0&}F!$NG-@mW%YNnk}&YyM4Fk+t_D(0tf9zA-LJ~fb| zk))Po-7Z@!Wd1E2C$^W9^9-()wT+Duc0_fat21MuSt}_i$*#9Kv!k{)QH#-Nm2WcG zLYF>&{z9B$RCS`(QTp%qE5C$Eqr*hK1H@*28v(por zZ?4?)+%Ej;?%lt!&eV^X)mGzFl1f947hgK~jeZ}MNScy2{&S`6V2cipU=lmbcBJcK zRAgkVd3{PxLprr)I9sXSQFU&1Dmm17Kv=5MXP;6v{!X@0PFh-VxTD1P#f#Y4nW^4> zT5IX^KMhq`Hby7{3R)TF7cX3RwuVhaUy#Z#B&51c$h>@#(VOl*=XUn&*&=;`qT*s1 zT$WlT1+ggXt#4_rZTH1-5ZI<^BAo?C`sG7JqVNLKs%SY8>t7GAT)pbcuP5%i_u?s! zZAWWY^5=#>-n8G)NsJ}`bg@6bn!5T6KE1pH_V)Iuo$)RcL#DL}0ogaYDc$*UxI3ua zscEiLcI3m&g{b;P6w~^YC>)VdYIk|R_HHJo`jhi@v19VYj=g)&yH1W2eX3{@zw<7f zJIqC~)om&`z-^Z%mk5ipXYm_M|wa7}M^`sL7eJR+T&##~>$i zbZ&Dq_Cal1_;tx2-oA5(6b88ffp)(!+tOg+$zR?IK7M|7Rl|Lc9zRwL6L;On&TeTd zdi8VwDQZ}dLt|sKdf5&%52x-Z{~P1KFH_qN9Xh0$??$gp)=S1t>Y5tNS9^DRi6Gl) zP0c`ESB45o%a@+>3r$<|=p}!!vb#S%+8}cM=iSYG+OcKgQ?kw3wyDDWhYnptZBQ~cj;;BF8-U9r zvx1qqF59-3?4Tg`ps$9h1y5NbaP2f*OS!tgy}2r4*Y{#}dRPI=Nj_1FmKV|cH`9*e z1XDP?RgS%Il1tjBd}6pW-L7BL6`jfH&!;UaFHhZTV%N;5N!Du}ay2$K&bXb|dz8hb zI<|t%s~y{icGm?Lr84~F9pyJy&VKsz$)rBTAn>pW1+}s|-`yRZp0kR{9Bo@O&G@;< zjjnLFnaMcqEbHuRKW<0Mhs5JLs~fH=#TNSW=MTAS)hPxtq!%2w>nq1feAU%u?6LLG zDXLnsUux>ioOWZLYYutyG!v!OY$D3&_=@J{37h$K6ZG6B&XQI2{oS2(hd)>FCU4`D z9z1y9G}865qa=uWqq}lwbW}CZW#Yy_lkg0Vo>aPxtgP|Rcb_xu`}QWFCAF>4tG7x) zLZ!bUoq-}tmGoehmXdn2ZRp!@$Lac|4pW$N0{`~YrMw)r()TvWxS3a!> z(s10jZy#w{YdFLYL{!8Auf(47V&~D#%_MufyDA2VV73y6!srOHGOdi13h=y)_A25$ znxN~{%_TPa?JCd3S9h@-7&x$=!Th;ey!eP!izvF~ewjM`vVL(YyO@*l*B6I0bL0G^0m9B19)CTz=kG3Ez-y`_&M_C$TXXD_-6p%a`&;N*$@I}coU>8DS(oE=|1h1fBbfI0 z%|lyPIZgG`c+HwhLXNB1be8&hd3m8lM`6E^b3~7_^qaes>%@pRki1~1sHiyL(07za zEh)Hts;jHZyhhBX<7xZU@zqChE|J1E9p@?J2I300tmi}hH2wDaa=c>HV_nDgO=l$~ zO@DrTd$wbJ?u1M9CGaXsT{lhN-WP^TefWB)y^#M zVZQ;)X=!QM)0&@;<(X#NtD2L1%y6VC*U@HXdbrH*V0l%Q(%e+*oN~PKKG)yxmXYns z;XeIx(c-1*H6`XYtN8i(Yho{+W;Z}%J-aZFiY0(-NdV@JUctv)0{(-qt~Yz^_IUpO^GDsnShR{g!9jL zk?Q++kmG)}pa_tYz7r2gd7$@WRizJC3>-=LVN=z#Hd zmdm$NQ&Yvqzc0?|%_h_XU>HUJ&KG@wdW+(k8Wk%#!>>8z9EYni#V0T@9(6(eo(oIh zQ7g-UEZ;Xl*9=?S=Uj6HXPrj-;;^9A3j%disH!U1^acRi8#mslvT1ef3~AK;z<1<` z3gtlRc#0|D4keODC0>8BXEMkt&m_z1XroJ%F5O9Wr{2^?IG-#*vovhne8a|Xb1{}o9B@nb#Nj>pRd3C%ap zM6DQ|9J}scMc#glS3ogGuKzrfsF5mJ6{Ao#D!FCsJ;O}PAj9N6{q@ztHBySGJ(iQ_ z+`;;UEE*z7EBpL>w~(2)lwyd8t$=Id*kr(UmX|*~RxQ2r#b;kcW@ctksr3Q9yj+~T zkBQ@hHEe0XSOg#f^$v}Ur15w?c`}cgaX_T)J@)dd1FkY}5XON-yKda4$XO+KI^{j+ zCV_dR2?1Eee4j{=(@FdnKIqURwkfGzY?^DGb&lhk>BXPfs37B&Rn4vM_p5v z4FjvKXQaCd&>$_xv+dCa?!Khm0c_yIq(#~dwj_dE?ccYr$gnvV&5w(dQ$4Y+Yj806 z!vkisAMfuGJT79};}cabvU|Gp8xxlxOIee?fXUqK3{Bf+1Jli5A`ZM=e87Enx~Kf* z9W>n?QUvRtKmU-BL2#sAK=c(WS5`gXa5GL__m)8*HX3P9v@2@ag+cn5uBYc5ww~Xh zVB=09!MbEMJJ;j1w!ZpQ%8jan9R))VAO4ebvbE&)``$Xe;xCRxMMVLjHq<)bWR70c zp2YVN<>ePvvj}`NUDo-vsjtOwRN6|F{-gX-BE<<6X1AKQL?b9N!;ygo(-eFT!Hi zL|%G#n;>Y-4D&ebvToAlq@^F%CTK)_{`~nsa)@2BwuIJ!oa8bc4q0{yw`os9KZCDd zvTB~j_7)2U71>Z(OjuGl>-_a3YQa?qj3GD{y|o6g@Q7uTdR|^0 z2wdt@v$r=F;a=6r*;m=x+S1#PG|8fXj&z2(_W&JNf|hOEvLyy3Sj&BG25jEC{fiTT z3IoTlctjF%uajh3nlI!C}8$`4mj2cs9rbpOGF7bIrKBUF=h6F+`DW6_jZkzyBy zFVb&|sRb0qpq^dRfwW~zJGj@O;eQW@fRPBsYzmk{soe4`zx8(BvuA6I$La=u1#DtD zdCMn{+Rbo(YUH1jlk>B!%_=MO2ctcUesi|p(-w{4@uNr2D~zkL%Xo8cwREWlrT+2( z0wI*5(s#dFn)8_Dz5Dk~v8Wo}Tz!BIf7rNu2O5Gdny(Kh_4U9>M_qf7Qo@8baLX%e zYX_4Xee)*Mr@}%rh*T7P`wtIux+lvla61yd7?uQb;YvZZAZnW3hwqIVb!DL%sfJS0 z?oqQjgL$dWrHMl8c$A`_=FLu;xsKKwH30OSI(>T6_U#GZzh44Z?rOGgRlx^Mj`r^_ z67iY&eur5=XZL!hn@#Txw=*#vd)=$I9QekR&-0^Sy?Msh^Wf|58AhBcu@~O0S4`B- z8uHX#G%`Hw%~{-F6_AxD8X<2fY}fZquGouR%yc3T3=rGdjtaPL8og6bTKbDlE9VbM zKo--6w7C1LSYF`{oqn>JHzhaJb;hg{OVF=wU+QEoAj1`Vp;o$eP8rV~+%xQ>J>B^l zE-2M1kimjOLI(H#%41^*dU?LPw_0Zq@S73|4aiWWZ5XB8Ui5xu<}rXO=}9Z^ZO*{P z!BL~6|1Vw58tXncI|(UoWUSSl3IDitnK0Fxi;vq6 z0M7O0u{zV_$ggb|B|7c5st*E})R# zO@TJn{wx4XC@0+erc%$^GysM7GWzP36elIu0xqZi?M6C2jb^{b>zK`c)r8mPk`-Vv zL9dUsjqOj>iQ_T>{yy5j`>OrVpczkH2_^p2JZN=kAS6Mfz5&pSH0`SCpzbJ*m-d-e zyt>CpAW2^P(|2F>+W}iL*@%iOdWq18H&+XJgoH+;JF!<)A;iW+j?}UB)+RbLOWK?5 z@s`S;oun3cvhCj|c5^#QzePF&lC74VoX+b@xAG=Bf`Vp$m)%*|>ODo|87qw=K#e2|tM#{&OonBCMTHK~Af@}nYjO)`(yqnh^ICe}OLUXQ( z1j}V3Wo6~)P6tOv4c&+t)WN<`!OQRO1`V*fQUm;0^4&qtpc1{_?hf|K6=k(*7~*N` z;p(pmVF^U^pxrsLAhFS%RJ^$rYST=rpjT?8TQ+L~GWz)Y`$NnleB7jJ&kWRJzk}C6 z4g5ZJ57ulItaT6!kR@;eJq|^gj{tuG?_^M_z=N0HTa*bM-nwcM)Q-rH5Ka{Gr-zRRse^f<3$2R#9#UJ_%U!yib3&V zlS)ph?qtt3ECRnqEoIjX4G&lO@oEuvy?UF_SAkkIbTdFBN+e#~lhqlT8ejM3>N&#l z66kVgz;0jW3Qws>I zNDnw?nb~Y)O9uyx13R09DxMCZT1xTFl~2|q4M}rNfhC4Ic`h16Z-$KFcR=IHa?a+a zro)y^$8u6N7ytDaZqA;24<4A6ho58>GJo;KpFbIkiB~UA8%*;e+Lo2bhc~Fb=*a;D z=;IE?N_p?`V|(Ty8G-v492{JYE3sthQd9I+EQ@mR11>8os}ONl$mb=BmoA0Rqewk3 z@yS@I7OzJ88>!f2es%A6V-4sRtuLOtXx(P^8AYKIbq!#Fvm`+9+yiEgTF3{4^KZp&a2Hn92irJ?9s>ZT$be}%chunRZ0JFWIypjBHmLm3YI?3UJ?dZ| z3>-^ul)r8zo*PHB$@H)_2*Bpz*p-Fd2CmSU^d`Gca9*c{M%g6_2iC9pxfYB<*)6n(K0wJXZ z^iO@~iY%0d)rLBp@FMQC7~M9veS^QG3C*E{SM1Sg(E0krUB z<`CCjvSMR7v&3{bpagZbv?ZWAqcKmCi7Dj?{4!|#(9sETw`fR{BGiBw8xTMDiHqw} zAdu|@a*+36JyJ>7Iod^gkAsf3w({#sCy4@!wi>UKlL2HNiEI2zodc3P(X-~gh=2KA zK7v6tWSr&7-%i z`gI@I8#`5#NaN%Hfu{(~1|=a|eZ>|&b6N{@RU*gD;0+sSR$-7se)ab!?vV@Z`m{xl zNM<929PUjuZ~PBvD8emLg>ns~@Diy0)5nipSk*&ALyDlEHR)z*wgXM|DH8<@X`j6> zA=ViaojuPbi|Fk5=y{#pw7Q(=?0`o(wl*I~cT`DGzX(&H9MU^ib90^DaC8-e^NQz6wPq{I}0!X-3wr=Ow3u(p|iRJ?s zR*l`cY11Zyu_ND69m!Q9LL&iYsHnxw%{ov_)cpB$M!pJ`8RIhoMpMz{mMmLlhW8_k z3HE{yRIvB=R#I`ddaC2RA(uv@K-E_t*}95VK>s21^H(VDq&yRIa{Y!4?L9s5XwdOa zj2;w`d+1%xjq}13%h%D(FX6Yob&Uf$Zpd>zp6WIf3o*wS0)D(&awwpycjE`VVvJ?8 zJ276_nNaFzxhtGgJ&rRmooG1=RD-V06s-`Jl$Lhb(ZQj~noX98>DkdYgjF+LY;ID<3{C9RU(|_Y&cQ&yzSp(FnYHG%RHH0_4ARdF8WZYe$4{`uOO7Dv&HP zKK!M*`Siq4)@S3x(&(M#H$}fB;6m$UQNm^XuW&wkJ%9e!>ek_J?%sucB}_H0SU<<{ zd(mf$i6JRIz9e|Ws#mUvE*$^&q}`jG-bO`5?c-{XZ)z6%XX576(|4Di1qJ0ODk;Hj z2%Nv&!0N{+FP8-u`RtLhwk!27-o5COLc#s*dp4LE&0nwqzT4st-Z{Cvoq1HYhaC0y zmj7YR|4**f15uN@BsQ$Tmw;z48c!EdHuJK^pM7$_K4}Y!qbjfFVP!9Nox%558~A!V zb2psX;U?h01^k9Ib!+Fn^S{~QFWfnbzSQtFVA#g_pZTAC7e80NOqA)1AFqGo>G_{9 zjS1ftKX!NVvQd+nN%xKGQ!UX5r;}$QuD80zw~Wq zhuYmu;KTJaWjz5SbPp8j-(pdpal`Y$Ozvt}hi110Yr@=6(CGUy(gEX7TQGC*w6zGfL96TRY=bijTO@Zol^+?Aeni zPf#TjcAnNH0y4C%&3w~J?lbA?0525$5*;oM^0Z25=-E`;*7-EW+y32McTT?$b-aD* zlwGXL#0pgUz!e*>shs)9vSf*c;)U`(^vRxWPCao(Db=-sN2nu3+#JBRaaTS`^XTQh zx^m?e&jO!HMt!kDxCwp%n)@)f%Ep zVX%Qn?N^DB7ekd*ynOiqq^8C+XZwe4<3EfbEmv9UdGDEtMS+X8Y@S3|!%)QGT0V^Z z^wELl#8$VU#6wr}$~#K7!5Clcd!Rc8@~YXV$5F5&U%O2hOv)uMo?jlnwv*KX(5lMe zHvWjXlD00bd16j&9%_d#zaN8xnV3s|(wY|X;)O9zs8b+7)~P@JXiaKom0wDExsFqB zVqIUYjFeQx1CBiE__$0@ZNJoNj#fp@aRYiB*?+$TH{KRcFxxgnKXIXPG00*QYlXbiWvwe6Q8 z5d8Z)SKB`I7r-YGz-O&PPV?8`ZOGL+onv>QTs7Wxp&)c(VLLeMgE|1_a9=AU^+xv< zE-symC4uRQr^Kb8+2x_vT&59i`o@5E9<kN2Mjx8x%tKMp6-Wp;fZkmI zCQ0W>*3OEbo|bO5cA=5pEf;t=Rd}FX<*PuFjQ`JH?FQTYAnG=CsKcJ*?H4kw{urIB z94A0QRN}>PHj()Q`@D-RCN7!37$2Cg62Y%qSS^XOB_*V*YnZ~R*GjP~c6I6F@bdKZ zZtjIv z7eP(KRN-O^)>Q#)Ws%?4z}aZ>wG-UxL7VfNE$L~I%9($=9Y*W$cCfNo14;}TT@^Bb zX}9Gj0t91HoBU|@*KuhTp=;U|@7`Tzs92^&xiM;fCfdTNy5!iFg*V_n#I}q%cYNGI zO7YS9JRt+zX#jZ(zec2WLOD%J0#)pa=VqOwn%fvXN1k!QwrsUK+6Yn_f=l3!-V?9N zC>wIHo1Lua-pFES;HzH`$HeN=4L-x zrhDS;Ra|OMV8?_|F%G5v1_sK1^+5AWY2?ewLd%R>a_corsSbbEqeP4`COUUBn(g(> ziYO-#g*bWWPzBkebSE(?O4v8^6VD~w-JqqZk-~#KhzlI~v+&QWiNa^oxh5)-lX|@^ zx~IC@0Isy_BrAM`mk%s=o&0 z19k3zG^z_?s|Jh?#(ooc)UUggMx$z^ZiXSrym9L>y&St4D%@~s3N%ugc?ijLGxFQ; z^2eT;K8+tTxmR`cnV)^^%(JH>#gcI&Nk^vQ-pc0y^oUMO43@kddcA4CU(ayaxp>E6 zKO~O^t(}iKF#TpB>*i+Dk3Mssn3$Xn&t`|RJK|RP?|T?d`#%-z;W<)sqp+W471%pF zP1Nhi3%1GY4r87C!VDZ$icYPAKw!g1r9oEeT2;SWVVP#k0l_ zpEr9K9q=v+Zv3!tNDe#Jj;x5o z-ai+vwbcKm0Sxs5L>I_JZ!FgXXil0UWYuzA%XPlT-kjQYQjeW%BW#dmsY2ans-v3N z<^6p4b=AWdmirWxa0`g#Np0(yn0ny{2}5Qtk72{ z@7({=n3LJ;hWtt-c3#)-AEyOaSI4WrNeOlA5LSs-<~o_(z(;-IQ}JKR;ic8wT;z(FnjS%vz-=!Ta9AL!ZSB zM|LW_5HyMg;bc_GoE~|Vuzp?lT%*5-mD>d`qnRpjG4R#Ap*uaMF`QA_-B0d0zu@ zR2BH52vDhP*dv>1&sxr;QCP-k5|6c@lm0VX- zQ=`w#781AwRV=bI)cGQTMt0QUEeBpuK?niaYI%NnM*`ka&puJLcVbZubK_7aUQA7X9CcHFu5Csv4_muO{+si;OAw^WG8lBTp4kDYq= zzYZ+a$V}QmqnbTcIqnt7ctJ3U*QIBZgMmQZnpO%`o5Z%i04EYp4H66V`PhT{pVtzJ z=GW)vBz2-fkBsD84XF;Li0E0m?lZPB^>FczI1cHPpCh8KeCw~;YI?mT_pc%-X>Sj!}NzJa!c7H2=$nQo*I3ymuY?CUHkK+18CApphL^^2(A ze%tH2_s3NlFS3+UiZFF+{=~+Pgck#4Hal_2G)TA|U$J@Q*@Fj#a#N2y#jR*fb#;T$ zg@P;lQk*5`Z|z&UZEI;4tnZi>$4)jR%vGTB`cyddzC@~C!!T6?<|vYKij(_aAOn*H zZ36;c6a-z38ZBh8UcGvS`Xh|fYl=4?|NVoMJ>o$aSL~vuz|-!*A3gW>_7Yb$?2--_QIaA`NkKE3i8QmWw zqA?MY6BbZH(<*cOv~0$0{?Na7ZSUY8^YPhe@$L0Vi2P6bsgEEb$LmSBvSolqs{0yD zPow`f++{)kUbLt?`7jckt*Qnr5_QRXdTwqRsgVK&Z!DWdpg{6A*4+&zh=xD!*ojuR zX#e~ByP#UwD1MyX z@S#K;hko%UFB(HyFqJ@86i#W%`}b>FFFLbqb4t*qzqg$D;#)irJF_iO)1a{hJSNS~ z!ih;IS@(`?!JBE^Sn9%h9`$eFN1>h+(OEG(uOh`DvJ7kFK6&vL%YG zN|;7_yMcS^y z7R>3!_qT4yC8h?VA}8S_A0fS}PYsIv;d*lTHrQx%ouj*8J-{=64HmYuWz zYJ!PLw3e=#YVJs7@ro4<2~fdEY!}I%XE8<>p`x5#v`D8W&y`zBGAcSyUm&XWMr!@6 zn4HYi;5UzU1&L-dMN@$pn~j_8)B$NofMTTHNH5M*#a*V{KBsO(uXU*hE8`wat|Tn* z{fBS(Lwc%#Ir3_DW*PF8D*{FjnSbkri}jQ!U^sx{`g#p!v1(cVyl;B2t%}lg4a;_2 z9bV(zU5oR*s2Qk%#q;LPqc8qm=^~_Ef7iWkEzmL?-Yx_!6g(`p!9snx2we!gXbq-wqW(^Q1wijQh)wTQrDGoz3@V|WBW>&!{lpw$cF>5Bx_#G>Wu7isT_E$4<_w1$i>5UZc05&@$q zK6#@p#59BpO#l>Z6jcTZdcrl`hir4J`y3ro*qp|fJV*|(UW>8wNZvdiBPIsv7LBTN z)5BhpPq)4#ISsp-rq`G6{Iz(o!C}@T*S^2aasJ`3+Q*w@QeE8g-4-ujxSS`V zR~Zh^WQk5=a02{%Z*f_Sd4a)F?f@X=^i#J0`P73scL_s%UkO;A^&xkex5h&rHEqVwbqC1bN zB9Wg`CO;w09+Y2l7RV3GW(=H^m94dFreE8ssH>X?HZf^oOSgmkI2tTvHr%!BIK?*le)G>twsL1dgN$tRmnj9NUCi4>S{^jW~rF7(lR4g+bS>$=M zY+msH<(YddqU^F=^MWkIG$imJ{>}fwxZuAr<6sOXW*&vO^i=Y+CG&1A>>t@ZaBLPj z_4ivR%KevZ^;qh-`MPgK5lk8iZ*PX%1-M*p=p7A??P{TtSbU*Fr?@^&~_GIPre zZK8%>Y0melFwX2e+4^sG0Pa%2)qY9IP?fS!ucE?2wc*Y=+s1y>j|-GE$6?Q4R#tMC zn8=)?zJAMWKpp39+v zcyp+OaPZF(QkkSHr&#ey*R(wRIVP9R`tDN_GO}7afA)@RPjU?1g}-Y0`PnR*lX8H7 z0f^N~77zKyo}P-v#t(t{EpF$LEFyc80=yn2VmZ7#b(YYK{c3zl#$9|clg&2)0>zg$ zWeGy7ZIz--&zOCNc#UzJL$?=8-y9kwgZ>-K&1Lx^VKl!RC2gW zE8~Ex_w(MtsCx6-&M>cPfH=5Ek=QEK^Xn8an{>X|e-0B=bG1fcuEgwaj>?%|mUD+# zS3|GJs-zX^=XGwA-(|J4>vZTrqf%L}J$rhh{O@<&+cK%;%f+o%gFJ?Lk9k9yE@Bfk zcb6;u8VF_YBJ4fAZt4C5eGmUy+zGcg&u`d^eRb8{d`q36Bse1poY{Ugw#MJ?bF3@Yw7i1*r93x# zN10~nO62~u|CShMH>HOsJ26-Zw~_L8d#0}n6bl~U*FEv>T@aTP*z!BeEZ6`j3X(s5 z6oQiND`s^<+UNxcU=oB_%_Xh~X(9g-)gHX+;>FHQ>#j>us3b`%yF4e49WmRQcW0_p$zQRU%(Ei5W6?? z1;@Z;xA4@ zcmE?lxl=AKvsyN<(E)kX%TI&pSN7GrlgN>fZCR*CIlTJ*QU61N*XmR9EhicV1j|IZ zi6*U=@Lm!LYZ5iUIgvlr$q|3t+RY8ymW+-Hz;y=G%H}@_c3ZMnH**Mu9 za~iWb^U|aDlduINO1*6LL*XNzYNQ8=lEL#~O_Ks=;s~1}P5`w1$0t}>Ol->;BnFtqBKKb|= z)*t>Uzk2(4%I(`iy>D;P8}i-5!1aH1YbcYCM_Yx@D^*%*KYN|j4`2mV#33)Jf4dB4 zSR2Kv5-l_;US(!bvwUo%s;mf77&)xn=hl!g92x1(vL>_T=a&2}YM9zB*__RQY#*li z`nv)$5w*B{H~KXni;6voKl5lkhEP(8$Iv`0P}{b0RWP?yPj&Z27_B*-tgDu~j{o2W zI|}eSu)O-mhh?@5Q_ADb45FcueS^rnwr9%%tr^;Nr{26YfOCm3-+$$835t~&AO38d z#__Q!8JN^_iR%f=q8_7-xK3}II~RPdP&S(KgI#pv;d_bbvb?Y}(#kxZg%|Cthu2U2OH;O8j7W}kwvg>Rq+qgF&4y9tgn_raCb0Yp$4YT#!a@m?CkgO z9E=SJE*`E5xQ>y2)FwephVO*?i$tSFndopZnkyxu&KrFKP0IAsV*?Rqp;JjoOCU_` zUcL38IucN+!gGdZBcBTO;6vmaw-wlCJN&s#_5!>G7-H9=X9^1nOaa{+GI-B(E8Ooz zAko^+PPMzzpgLMk3&!A6OxpC-in&e4Vmd7WiJ5E!0g*r?FTjQ8aH z`45TQ{+A*4Z08JlJb{407pSD01z1Bivav;#m#;m0?p)*FuK)7GO2Z^#fDT9-L%U7J zctkWlV6vZ9&!hf2TqT6`Uc_>|IVYy-o`vb{L%>iI6JgO#3@4#;$=v{dNS_B0jD$-f zMRYh^y);|1XMmsQz$%W*^yB#zC;9WIDIf-1sFg$r0#5|Iz=KF`LseOl!^p5=t{rRj zdJb#^)sK|kO&DY%fwRg;4m^@Y62mxB;0?6+EwF;|P>y)S6-aeny-hq4iJ`z_f0lt7 zjCl^OoWBT;n+U}bwlUc1jH-A+@Bnry78L_I6^r?AumU^7NYCrV1R=|8*XK!7%i6f5ATk5iGo+FFW z&)ay+eKuIgQkxr3hrpn4nd$v2AOnnAtw94#8YYi~cIr9NPqB(m^Ga1>HK6||PMt$~ z52NQb-`{;W00ID{^ls1Hg?6oRuKa=MyVm^soF-pi@TViph7YyUh=J+}-{fnD1E?^U z6#ibd@boD}kVJ0$^SV(tMqVf7NQYK;Q+)<`fhqQG8{$x_c95Nxb7-0pBL1M>g(-JUB{KM z5;mN_vX?lL2X_58$l7nmZznG4@oyF+cuYaC|B9_B50BV*vH9Xh6fPUftB-Ck5yMlE zu2;FeiYRMM34x_#W0}!?7SEj_ak7#>urs^iq*OoJP$-xf^x}oXY4`52&xM6`0~74* z>|`^-1jGo}!XbJ5FCb={8M7$nU8xfT8FTNJ4R(Ee)P^k>h5WE1)JWZdfh8n?4*4+- zkJbsOMfCZo$%LW7{O9Zt1k6(cu~#=7jEati70HO@dUs%FN_ys~ihVP(bFZ z!0I6J0Xau~{a+R|*ND4!i?dxOQV4dP*~qUu?bq+%91~_Upxtpb_2_^%_Xl#`sDq{> zT~_A0AH-ZImG{gz&bhlA_4{#ia_-&vV~DqK0oIs{eFNG({{27E?nr{)z)*4fTmxR9La+YN2t^|c?OAtIYBZ5BKlmkWv=y#)p; zUhH(DIDi-%f4Fa}YJh2}%4oUXL0CypN0c#xTs@Qu2^SZ96WYnAC;k_DYrUFMR%0?g z-IVcKc8_<#(}`1$9?d0sY{63$08xGFOOL?!2hR+I#cGkHJ7U+THpF0% z=l}c~(7kZ|`kMy&@iPGEgv%j-WqZyQexnK|V4-oe9l7#R3r=k``b+oM03{KJ-#6ji zCT7|6FS}H)f|M$XmRi1B8OC0h0?xiB1nuENL2Fyv8cY%JZQ0^~ARTiggI!$}2a&u~ zhEPX^N-hu-g_DWJ2pyD4UqoAsRbnvm+tV_$4-d@I0{uy7NbikTKV(Iq?K#iwl`Hv2 z>d=kTyQwfUF&?6T0`mbD6j12>CCctcY32KFBu4JOy zMJ9tcY9B7`X_{4`iY8T)g#HlGFH)bJh8Cmr&t$7F#$vZmPl)7ott7 zqS5*paD@&aHv#?b&eBy|Uxm4`8qr!1{7gb>cHhBkc^HR`1Zs*sbx#^fQX0aRmyULM zMsjf9S+{TKT!Gl@?uPvaLvq41tVMJ z`T74LSWevk-P1sH@c%A}^Av{C-t9mbIA>?4$BPR8b87>CxUJi^jT#xbpGTvztOsL* z=NL6=VNw*(@s5J>`GVsRj)C9I82R917%gH;Ktd)~UrK^JM*{E>G8ZR|H%Sk6JWFfz zOaJA0eE0k9&aTDQMTiY*X5YBzSH>Z7BNIca)^!Co$$0LNYIDw*GIX~7<{T{~Nxzho z=wK14PK}0Cr<>8XYw+VqTZp>l-p`M|rM-ULb%2TFz}P}*)h3{S>-q|Z(s{}0%? zea%UZXEHe6l+A4O$NTwE0dimqUpJ2*QkQg zk%!Q9j$WHT!Ax#&glGG?6?lG}SO5Q8UoU#zdkjw!BBm&`r~N{suZBA9FaCIcRyj&~ zlWosPECljz)&rRDL8NOVqkoD*;%yiryLjBryb4B=N^YiC18p!CQJ|B^EhS8zmy@fJ zm`lLZ(N03n4<~~{K=_#BB62-uDs>S7YZ1DNhkAvdI&E(by?h(k?ed%QPt zOwo8aLb9~a@1hYm;M-DS`DdiR8?&|N<-q1|HYI1kneDFi-NLc zLPY$Yn%?^3d2kamdP|P#shFC^6a5jz49`5n!+)gTz0=Yc2uQe{w+h+5n*2E%TscfM zFvujSbK!TZj%t zi#eadFzb2}IF^__b`}l%jq_^M@yb^BWM!Njp1vk??;aZ-j$>H`*=h2}gD{l{$&x9DhNKljV=YKsL8ypgvX?v9T@1orR(C7i(tcR z7$|!WUJJ#;^2~(i&~5R1U4(Mrld2*ya=-vGU^%8TZWPuXLh2ZHLk)bac=T(S8^Bkh z(|4Plh?`UFmq<({zj|jV&Wr}L(`ojDmI$An#q)krgY_`bh$~1V^C5mW<)xWR2?=Ep zjP9NqEbPLtD6LTs+9;`#JuUQ@8WPumu$qtOjxCo97|O}beXQa8;6d00^hVV}jGV6| zO&g4d(c;YUr7ATxs;^AK2z%%adi>ZCn+{P7UTM@c)2zJUlqO2dzToEw8_{qFUO{@1 znPl#em@9<+8CWu27V<MLK+=N3-TTG&OwJ66R_;OA@Y}bQ^AGT8AcsNp@*UieJ(4I4oMO;k~cv*Jm)Hl$jA|@5XMeN3bqZB&=;h=WJq%sv=&T%XyC9( zVS0OuUxtF+yYI-68cVVHYGA$cUq;9O#ol{BRhea5qg0uy)Y2+*AgKow2`V6n2&fo1 zq98eofgpl_k`yGAmQqIWh#(?hAVUEI;+Bk)}MR~46uQ-qds_6-mV zw}WtCB4vR(`f|hJXt2lE>!&^jtVr;r-)F$Gcg0%gI}bI~tdMc^B91Hl1D^wIe+m{E zV=`Z@^)m8cy0KPL(qYCdt1`k~Y7>5ufe`qa*xPuolF8|Cj!Z|-%5sF#r0#2Nd!E5z0JdR(HrJ_XTh zgsUC^PH_;0BK0>i21Hi^B>Cb6VZ4vfd!KU2Rl^|LLd2EscVq3Qvhi!OI94pa{H1oat#LlA(SZdy0{#=qvQC~l*WRszxAsZtd z9T{P=s_aZZeqPUSYC1ZxEj=THQenSVHY~c`k}8&ikglL-xr6W+JX^Snct9;14JPt& zP{Cl`6#$WhLsZAI>qAYyY2mir-D-CUy>GM?yYb5@^Eh9z)m|Wo6C~bT^{z`yFUxP+ z@Lo#30ct@8l|zxAL<^aP(r2vY&>P+hV@FHH8mq2TkEm;Y=_ekz+pUD+EQ|zY*H_k< z0sX=!#j{(JQQ88|yG(A8OU?R}`lM8duNydX23PdtnX_O%LcijxRR{ za6)a1V|gyofvy4f(JO(zs`4l2z>>r#2vp+}U#{xiK>uYA3qNU8q&>xzm?deWYBvRmwL zcH|~5{-6UE0Wq&vNGcgd3JZ08^pmtol;TFN4S%|A2V~7zNiRSdn=JgZD%0te8yQZ`kJIw z!Zu>}Yb_ohst8Ub5*B>K$pOw zr^JGBtV3kYNihjdUpdzs8q%Rk7wMNQoxV~vhjc1=#okLGNxZ(6jdPUBBm!C2?8!8C zFgXft1@#*dJYRM9d$PkZ_9av8r+Aqv+EZdO1pP(>A_8CQE6m;;!L!LODbVz%08|#p8)D(-= zsyH+02>bHs5g7-dnhk|EH7TMG5d(XCIzmqn;u!MU+i<^Hiw+oG+ksVs()s}09JRps z(6qy$A&fGKesOH%I;q&`%JBuxIxeYl=ebch2p=o_SZA+pE1E`>9W%4f6CZQxx%VaC zq4-?e)U_&O=Dc${J9hX{Mhj$B`SRr-KlckoKuinnUL>+*uXd5h+Oq$&Ms7OwatKHO z#P{7`zPnNW*ipq;EZh-HCEPI&J1wLh-PL^q!r_ol#n1-D7imK<0|EQdV7D}EwG2Ih zVn+7qrMT<@fd+oVc2|&GJ_O=`B2pPuTBTTNf7!&iWPft5A=>Rfac|p?gNdv;fe=Ce z%dqNQK#2*gZGw>*>QtSwv8jPjC=Bj{(FEv1zA}4`^lEa>g`Ae^!iT%f{p@}W?ypJ6 z9&T|u;5V26&QIi>JM7FGov49cxpfu(x7jOEDv!EM;`Mzl{`WpNUX7Y*V^U|by5{+x z`qj|^FOtj!(*H&za}8;2u6FNGnq(G!jv%=hNEsbKxFN8qEA@X1Mky550iAYgyaQznlcK>do=0J!{u;eeG8>kUIN0V9=l+f{XEBdTAOU zMfS4baG-MWwl$+GV6a^`q*$iA*98`GA6O7#CBYCuC{WI}&3dtb>W7%?ljZKVSM1i+ zUu)-L`?4@h1B&+3Ox(H|(``$>HJWDK92#6cNlD`11Se_o)Bl(Exp+%g zTZ(R#(O$UafEoYuT!XaMBmYm7v)jkI8^fTsX9bn`In*ajw#A-~xuZ|D5y;Bc~UH-dd&;%Zd^kYPOuC{4_UixJtC&YnHH9Ho;6 z=#fA>_T%8!0AaMZ+IX@h#<67H(ta{nbsinmK+!>N9`IQu(JuKG!j1%PRI1mPi;>nU zIb!H9G)#EsHG_tUKPSst0AIIJcn9MZulrMMT9wFsog$!qT0Am``*4OOza40Ikhk6i z72C-phFj7G#1{xq4xrjvyl|l~4F4fw(Z)ZOB+CS`!$fT@xSUk;68{~%cB*;b=N^DB z+d*_9QtC?4bRtbMq{j+a)aVmbgBuXK?{>t8S4mw+MJhtZ9*&I;ts@tLLa&O2%KZm@ zL_AD&Tc7^wteZW~L!xjrlos>2GSTOw;|0h^G~!acUa5tId2Ku&SUj;GKfXVGN`M!H zAlM&|!}}*tB9E(VfQ8C#^rH-4yr`bo_59!H{he;%XK$mAZl_Z3_k!(Bbs|97V3^k& zWS6*@13-3l+wNer14;VGBZ6qE;ih6_fv}PIR;ubBJXi%b zS~4UA5Go-VqAp|9YD2-*m91cP~5DFzWdT7d8FwF1#GoZvupj~3=stS{jz44<&Mg1k16Hf4vzpjYLH=M-vR^+gq&{B97x?v}PK`bO$8WuO6TL{^D@asd zLa0KJOWZNngVDNnLdG^NUmx(eaynFe>cKu9AKwgr%c~lx*1s^z(PF}BsRK?)xnS&E zRdmQHYxmyx_2q+UE5yTMi8NwUIzVO~!B4;aFn$n0sJjUys!&%6!Be3hkpa8Sl zI{8pIZ*EGN<#{V!>}nnynW=U65KBaX1+Dc_n*bOb!tZtilg4Xk=;W4?GuX8~oqA^C zD=@-fj1>G$3 z8okGdV^9ZEbDxTOq()Iti#D_yA4j>Dewt4O9MX8-RnYVw0~d5?eojg!4Bm}YNaV*}_9507ePx1RyQ4SPwRy1gJ zvA6C#wgVj+5R>eK`iT{UrN+m{$7BHrF-vK{HKK^*+G_pOv05v)R5H?G>}TnW!kvI+)6m0V-w7+RG z1t2YyW>Pl$Shf(17NvnYNtG@m+}UY#jI1kqEb`}~D@*MZi99h0Fe{VpZj(7#lO+U! z3`2mQKQ}}O2Lz5G{5ChtuGaDKkvJsL1r$Lge?fL|{FC`Xc^qFUN& z{l*RaiCFZ#^m&~7n*_bk>%ZwGyQVF7pjy=ExAV&(4eLw+j>)8h`fSBtv|;vnHSTp4 zNT12WfHurZtIX9n2%%rtjmW8%Vb6s_DJMh;))n@;EaO=%D25{6tEjX;3yh5&YjQ7k zQI21;M!$bxAVDvn_7qb$o^cQw-s%MwbvDiW9yX7{as!L`>qPf!o(zxDl30NRr?!p` z`-;!kO84!VSgsT(8w|g+z7X?Y`1$!wblHiR+ZvnH`>`-DxGU`tL$-_L^+z^UUbrqM z5QXH~{h~X+)6%NhUswDC7oxodG(bHA&!EHInS4R^!Yb1jzf#5V?|={WU$zcj=z=9N zOcKjLYgi%rVKH>X`{;=Q$`V~Rmk@0dR2ysRYc_NYm0Dk281LjE_vH;25z}C%QVC{e zEP`KA!;|}P9;IPEAZd#0bvG!^oE8YPP!5jvFLk_D1(#d>t0Xu@+UhV?gj{BUF}5SA z(YVEbzzfFqsM{3js3a8#N)VtkWoa^%day}IZI&VjnCDmbNJ%aK&}AVfe(JQN^|Z)) z+)hx?z$9^Gg%z#4)KgFPTpv;R_18Y~R|2Z1u;;;wcf7U3RYBX zZBdaL0+$xBsiQ|xcT*y;m7OM4@Cmbv{86(eCO99_ zMYtrYQe7>1G_@%>*th4M-SVo4sx>F-&!mxl9SLc;lK)Im-*T(;T8oK*kl-d@w2=xGyXrQP$DBZ?A(~eP6woT4~+k{MWxd z9yNWfI8PfUi?Ch_gW;MUOx$74AC2Tik%b8)e8IgDg2C>;)nwmwdIF>BNSomPH!<6CD|l@auE~OD$f!UTS#~N>F$q)VF-OIA;^JoPzor4)^Z*mka}EW(N#fAaXW? z(ULdUDX}g959m}p)&sC~niX_g&__2q#|J$XxZT5X<`?_UfwoVH47lW_5zR}ZHqiA& z^{>-LEr(R&qix&c2`0vnl|*x&0n1FB%9Z&0bkk#LTsmXh`rs0H4*sc=|Krfx>2kAf zXw`+$XO#ZR{Tog5ghcu2GBrt0?aWKXU(h1+LR44#bGKr zjyad$yDSK06J&H1Ydp5GnjO=gC+r$DUEJb*kGm6HtQFFOf^aaTNmOt*2un@XhN;*? z_+HSS0<$~)kt|5J5pbb^0m7#ho8Ac{Us9XwKnajh1Y8`e>(d;T>{%?>67A5q`~iZu z0_ACNC{{(vh>9Z`r7Sx~>|VqBW)mHwfT-X&N@|?id^)@ej)i7eoc3E<8-G_6k`s3RTk@nr`g!4n$24TND zIXDcA_Nw)c(7M8CCqpUqms-%eG3l<`4SzOP4D)%5l|x4gBV0XPl@$b1QzRBUI&Z<= zA_gZG_R|C<%+p~}fQgKU+kpV?BjEE2xOGQKEZOM0pPU$R`i%}sow;3WN;F`x@SKDJ z8#~Tg{HxBPx0z~ijf%Q|zXTpMv4g!$p@0549npeF`isFl?XGi(*XirF@9!DQr1l^z ztJev2Jw`V^22lVGuUVMgH15FJ&s4?v)ZA^blS;JFpqPcQpv*lOMkwLC7G|a@hd!thtePBNT(~26X_q<@=gGVWk*LpVKj2}CW`Zgu$2UjQutJ^aE z3iyC3S~PRk{MoREIktnR=LY)i)EpDHcx6rf-!z$fqpF&6k7!Nw$eOnH54`(loa8fx z(Sj~1#`S&v(`K%FtMCfC!wdsGjfu83B~#GP$)*0HzTI0+G8Az44bdd$f1T3XD~~rcqqjSo(*!j2MeEGQ(jA6tmVZ$Tt65bvJ{1kz|y^whJ%b*b4D2Lm>;8sYr`(d?u77E17n<^Ik>WPLktE_(agIP zi~Lhh5QJ2aij;u~69Fw}2R?hfPJ9rFR$(Y%lHOwo8dD<1{m@J){E(F_DDx=cB*|AF zRcl(E&)j+q*qpw6o#Yw`zyK}fRBF4Q`SK%(Rm8kPf5?BL>BmVAs1N+^xIAQzCq|&& zj5mnIra-EzliUeY4^2aiflBUNWmWe?S{?DF@bfStL`qlxxeYR(6`V*=28kQs#fJcn zX`|I8cUL8tS|Ne#4Dk4o9y`aY)A?{`{P`a^;Wm72fCkf06uOj-PO>CxP=%<` z-)x5F{c)#`ovh=1tXu!ZJV^H=FS@_uczpHm?vhSVq4i@0RaTm(ZCk)G7!(}5k9BR& zcpCeY>J7d&54)i+uR8v9hS!}l5Qvk#=9{fZONc;IE0ulk-m4F=55;FXeDVpMhBheX zlRYAa5#;Dv<6lK4*S|>UYAf_u4;#T6sLiDs&!mFH{1P$cRp{M4aaa_Mt)Jz z-XDKFg}Ha}3l?r@9`xi;fJtWT9~Z9{R8=86VZ&AV(|#D?{4K#o7SM`tf|N7=&D!Fb z9NR6CGiJuH3BReKUj!?kEt@-2O1ib^8@~94`O|TXIlsA@)Re1@cU9bA*0VSn0F<&_t z6{MyL{^m@N)L<6~T-*PH&z93`hv9b@2XuP9p&Dg>%WuDTdUXeD-fsQk$px1>hekuP ze@K2j`EPE28MFGo;cooDQ+NNL^*`&)y}D_sdeBLV!}y-k2BZXPh-ElaG>Bb>$&nbH zBw+{V264Each@HD1)fSr60x~(B*Al7rXq``>B1vnJrcHWqg`P0K_?Q@kSsr+$&&!F z@8$hoU<@=h>;|{YdSzv0jGHoqc=5`$Yn8zH$y8=xKN)$`VMIz-xEPQj06YfhI40;1 z=TkUl5J)(8*Qcog0@uRjB4dsMiF6%~Y0Q`;&>l!;2vDlVM}m_*ob892t#>q`A4cX- z;63TbW`4ecmjof3*D1toM;e%@0rY&@2nOMiuP(k1B)QezHtrY}0yFvRoKmVd@KYju zFtY%zXPQR%~K5z$iiL{u9sYnju$27b@^`Q;1{!!V8 z=8EfkiaMWRppJt?%+#CZ%ep(-4dGtE5LyFc95j+GbkK>=rl~vR-UJ#Efo1>*;%7Ji zgH@3|nzP}!@;1C!Vyq#eaiDdwC&b}!RepvuCJA`66H1Kqqe&jYB|@)I9uuh53Y)DL5d5A5&+8*;BpXKc| zmdUlgt?14Z2cR=xH0%acb~6~>9VUc%Q=c1xnd6u~fVPUB$aQgZj6I3!dh#frCq-A0U*g{(}ve<;?3vte#$N>EH)OBI==AqAL zrBRxt+nQxG8OE{;F%mAfXLRgS1@8`sSJ%wbXw8x9(=!810Hlda$R^3~-?zH2zrT3U zoV;Ki{-C~avHuJ=wkO8;d5lNnUc2^vv~v2KFHNnP&v5fm3S$y&zJ-LKghX}x^4tOR z20+(vA=)A!@d$XIfmNdn8{N3@1gj=lrYgn_Sd7d;Z!gPvqNdS#?_L#|9v)YFdEOs? zUf|ek!Q1a9F2J*O^|}K)zMC=Q{Ol_;1lAWW4PX7k??8>`C+hOYm(KZJo5Mg}H#{|X8`c4hZ2WQNf z(GHN3Cc_#@OgL)Et2?{J&W&0+tge8mT_X)L(1V0^fmsgMS`^ttyJTIM_14G-M z??n9$oV$tjJ}*rszR1YF;`wv=#SWj0E>(92WJi5=Sdlj@HD{wb8RN{Hdk{uGG^iV8 z`)e9c$RJ%Sf>{l$_HTm^1VpyrP>5l+3)K_k&w|B86RBC29w5Cs^$Nh;3pQ+;h&-Gf zG)aIa8Kd_}lPYM0v~wev=rr6K;YzN5!x$Dw?xM4@x=a6vHL{0?(NnH>A_404*{ggT z<}UMkdp--12>SiTmo#q}q-JtTq-hl}fk7M=!R!Pq3br852h&sIp~R0sODF;t=-3sP zM~Eg&^hiiZGp1xgtYp}ob#&|iT3UfJ80<|GNLGGAW9>W+6$t(g(iwt^9}cVePn9Rb zGKbF1x94S<=%;Dn<2O4`j17TZm<<747(o+o&jv$TnI+xF06p(eytdG{5fqqBN#iF; z$AKxLEN62|1VZvPat4S*jcN1(SDqUOnqb4z4N#U7nXw>(3*+iqkbJOTbWNC&jKx);h#G%(j(ef|*S zV8D^V8q!mr5cDSNd{%1e7H3QlrPCnh(31|^<0@>AlIX1a_+rQMrcUz1wNJJdT3OSVK9Op=~KyF;(@vkrR1hHf;y|~4>%G0-G*MtyLRGt!O%fKje7@@v7&4)1|WQeCv zfrNf|?eu4l#29AB+>^3N*T0M*OCZ?)4Rs+4TmP>IHmzrGgy(8jc5#XE{3P#T#7)kQVIb<|b}7#rtrpv=L~-z-#^Z zKFmCN=;p?O5!r*kuAHp;Ii7CL{8m+LpwB|*xY8lQGF@t z&{O}HKY3a-iDzzw5RN2q zL|%ZHDhj;m1MX)r&K)MG>KHu}r5VYdcUDdFT`&(1VK&$xa%!1wv2HJGAPcbd>-Qnx zDv4UbQG^D!8z<=kR1t&O!YrC93HXsB4jB2HOIA#3qBx%3RCMScLzGAM>GpMO>{c;4 zvA-fv&;&Ao#Fes7a27e@H)t{y0?@~2dQ<+%yg&N%0N8tb9>K)RXat&(#S0d!X3{6+ zWYSrn9d{JLablYIq~kfq_I>!4Fzsya?)EntK^?$gV#tx*is?A%Fw)?cq1q|$YL20I zsxL7B60=M;to3yvX29tYg;+-O3x7Q*f``hsO)cs0x2i9-Kg8bNzN)dazSM4bWWJ{KgKZJK&5LqGiy zrAl|VoO($96d0RHv)^%q($+`Y_@1{8?ldDDMzHf6b*H|NQkw1FKnAsu@lQEyX^HI$ zTG=j~{+-5xH!a>$*&D!yy?y6+iB~!ntlKrh6FAB>vg*0x`T6tf>crGKl=mwf zG*TBjLUgyeFuvj72#T!dt~D0TXn*(a!Ig|NqVk>grG}cr(${a*TC4_O!Qu9nvQ)RZ zzc_pfdN=-4L6-^Li%aZ*L7qy;%!*CaF4g$yykCX;z-vRl%n>vVd66NHvTsAZXodiO zJ%2U(8FWTXbN@nXU(*p+J*l-)co6b?7#mPcBD(L83#%LAWYXW=PJ`G(2&jvqnvSB3vz0+_vo?jjbgw#`n3`H{(As zVsC4-&9vIDl) zwS-5aBW!eN3g1~Lr${u0L5-}&EG{(il+gxtZhx#XdE4FA$j@@d^k*6pNLK71Hm=R6 z2l9Yx+Jcg}9fI5P%(H#aUn~OkAC^uvHRw1*gWHqeJ~9iK$E#P`#9BldMlOp8*)j0h z20etT;q+7Q_iVM~n_2)L@AslC*U;9!Lj^OS8+;OG;_*dRK{&8A9OcbmJKRPgfk`9> zCHOUhqqoIN55pcY7G51?CQ{t|BaIk=O#cB^r4rf*2Z~CjA{uL>#mf{7#zv1)8A*io z54}D1oz=oLl8*X-8H~QGAd(|#Ag6^+&X(HoF`B<9?a`JlHT1E<2|{7ILdXCi!HZ1 zE<-X47mr7%iYNmX5OE?iWL8Q_3OE?ew!W)(U)drg2&;xN5E@uudh3cmKaIzy3A?|{ z%uM~)Q;7#}(%hx}IGZ)*ED?GMe~WPNpXzOfk(w!{%WxZ>w)oD#*4CDFTHy8*u~%1b z+_0Bw%gDQm654IfmROT*vNniRiD~@%y0^}SbPwEj=q&?2x0FPpR+LNd>wEgSS#f;E zY6Qc9bIvv<_6y@V*{`{l%`d9{QqrV&*%;bP^lb4WzX(w&o5t~{C*Su=Vd(Q3<~)zV z*M-6A)tWs$@!>d>HtR%-ECp`-qokyb_q;=^*Tll(eUp6W2eHyGrJNjYP6ZvUrbd~G z>HMO;st)pZP}!rs1+}8ZT{+jRmeRo9jpSj^3IZTyhS_t4@PZ{Q40fwb7OIa301TnO zV(qjt3bLYXrBnMpeNw@>{USgn5;6gPy%#k#Ls_}43Az;Vx3746@9w8B#R$csgLm@} z?g(T+&1en)u;2yiLM^zP$|MS+?88aC7!nYO_#o$ZWEOIwIAW>8xYfNKMFiDh@MS{I zl%VeO?J3`tN?7Sh6}uhAZ-q{PjHto_W6ug&{ztE2?#d7V?U=l48E9fQGpm<#b3<}F zv`>}g=#O=X`XicyLYR?5PZgd*|!eUp2IR{#Ll5L}yGz_S3eCk~1gi$GGT zpn_Hgh;*VMJqi81)`KyqEy?hL1~Frgx}PNk3|kDInNcVjjs9exe|Wlc#Ofbq48gdo zaK%>JFq#8^45K|Rk6R7-!6M|27!w-?(^MKR$?^g?Qwbxh5KQ#aL7j$~3@g$Y;eDb2%dW8eNnlYI?3i(RdF&EX1w zyXPkKhlK>3VihedefzAncJ2f|d-Mla(ckf^SP!3K-~J(`{R-*~yh5IxIB{%Wazw)K zLyJlEsGMb4(=S@0i9Mn-`r6*Nt}L*S_~on7&;I~8E!R!4dZ2%zO2#c)((~D5X2W}R zSo+~-rFl~W>Y(<8~; zpSXTg*do^fJrdpFh3)$<~Z|9!ts>R`I%XJ+Zk!m=T~ z-F0HrFuwT$dVt7iiBp8^bG>e27&*S)n91Gr;;h zX}mb`+PYa@0vLbu{rBH9$idX+j_K$Sa#VqY+E0(f{YvP17-oP{%o3*_n468x`nEZ9 z=G;^(sv)^wkZ}sH=T$|hN%jSqpPCyp41v9DZS95y5}tn z^X8m*7VJ2p!@|}vVdK49O~?UH5)}yeGDX1>e)~m)kxHvRJ-hk+C*NsjmYuRFJaeuk zzoY!&{2+TSoz-nu0mPxPM}%k{p(XggD%oD#RP-zId^q*CFXiI$57OUf+&gy=g_gm* z5iH2=^3P~QNEtTW^KJM|$i0;FY0mG*)NfVw*N?Vtx2{ zJT@8^d$PGl#7|cU!8i;BBB3dE)mya%Uf^5b%{n|-Kw6VED>dwn+6-~X+ti*Fi)1Q|YVn|_= z0B};;ywm~fipQPOV($=ue9EG7(5AvH_BI24#eWWyk3!1^?j3~2j{2m0Wi z(lqjm7cWw4XI>w0z=!Bbp+%sLQc~d-b%W`Lbu7&1RFq-%Ap{Li4As<8g_`XV#_Z-X zionE$OZ=)qSIqp-0ky3PvzI_gkOLp~7)@HETH);`oO5=r0!Mv}*U9uEei0zo?dg}L?DmyC$P>+2PPf#P@1y7x0`Qb0l|62Jd-~hv z{ZPbWgfRU^`*|O+?>Ic;Cci<@oI@|Bb8OF@Oh!Co&PV}WUw*;o6#XiG{wtLco#^GTC^y#(yme#2`ML?u&-hlQ_ZOX8i~igzN>{784q+pV8`OVr3 zMn$(3k&I2Xs{dL!&`0jy@p2Y#(U*T(InXDFO_nDd@xK)>E96VK#R7k!vMk_wx`<2s8j{9@OW9T^E)Mz=b z2jkP#I+XS&etsCR-&4oqz~>>CEf@b0^PI(N&IVJrdmlz^^#1*AvuDbGFQuFsGpX=; z{pjwu8QhL zr~iI8>_6Wb|E2yKA zS|Qhe@%Ho#3kzHJ2YW+txY_P{=`-U_34dCe@4&c+UcbYix93J@?0>dx_|jZ#ITe}9zhid;J$HzjuUnkEe$ zm{jxBZL-%9Qdy!m#?9SwZ`jN8^0aLBO%PqOgGY3iNkhY}#qy{V(fZinE7mEdte|eB zzTbPao@bBAe5*9y2JzDa3Rv96#~TLrRjVDo@D$|^-Q;+fqXb1#VOsL1^pyqdkmo1x zK+U8YMUpzWpW+%skK8;m`^nFDAJ}<+7QX#pQr{1s zwHK$IjC=~ZLattQU*Gc%4U{?=SDofwaS^qhp>+;bXA8SpydMlS3Vc47DDs3zqsQTDv7(eHnIz-w~w=+MJiBznRNJ(3!AFk)qC{w23+1ynTKr8oQVa zUf5f&OvTznrBW83SRx+(N9$w7n)qt-*n6FK&SCnmYaTnq=nLcL-#^0ML}zT=CYf@r zG(qf<68p8DNx$?zdt5BlC$MVMj>g6S--b#1Z8uvifgI6)9eL>06iqI?{f{w*{k~7P zA7%glzp5^e5iFKtV^j2{Nn-L^j80OldHR357TyMSbo|e+CI2;r1h;>7EgR2FHGltC zHE%qH6ILE(V4`Iu5kkMB|~Ll>#wy19&H<_I;CghxWXg5oC&(igyDjEBv@S zNUnidF*Qg*lhC9nxhJIt=V0FkoUin=Y|#$G6=)s%w-o!X*|1>)wG@yjdNT~N&gmeD zdT8I<7%iX{avb@dZ@uc&ixwa25B?Q=Aq5u72h;h(3<0ag%#n{d^ZA&NI#mGZ77fPj zrynfctXZ~b{&K;1V5&O-;b1hn0&Sr7K})E%M{gR_1o^t;tWnV`Np7EhFjNaQ z9-J>;{x#V+mX*@^!mt>YLKz4SvNKMV)Gys?NpD_-bA|hR+d2v zdQ^7WOD6BNg86jk`T0#vhO9-9|AU7mZP0>xj}-&}-Bf#W7(It)($D=e_dyB<@!{o6fr1EJn`uB*<)Yc zd(8gvM-}8%$%TrX>DCACzm}poJ69Z z`})*iO~on(;O_hN>sOe{`7-)w%rF{on2lLO;}buC)eG-kCd+ZKOF&LeuKQKonufX! zOc+ghL)(989@pJf18?8nzUkb0#1_ReoGMg_LX0-ew+Y4ACpA#m*DV}OMZz}|@)^aA}SgBGZaw5aEatHR8KAE+Nv;2mdR$~V*m=AB>2$?4n@0?zZl z<5oZ2WLz=g$QS84k-wgL*KYkwP~Q+#gxZkaf?QTW3IU2>Y+vCf)D5jNWTiM#!aA~!xTQCnBllCSX<$1*87~I0y#R;Q03t~`{Oo)Ba#%!9 zi2EYkggOzdKR(|>4|iP_KM9(dOcMNCh+_uc$bkyGX&mKEn5l{3C^n#6?uXfFepic; zz;x#`L5nbQSWq*{Gb6((qKIrp-oAZHlUQBQpk?$y7#WKWDs}&a{bs-6M(3|G!@-K) z6<43E>~ZyK1!B}@ntcqq&g>t4I2f)PEE@Y@Bjd){nMIlKdG4r7iN)O1JN%t>iDtJ5 zHG>z%0c;{PK5sH~qBY_8Xn?_f6`)Z=6kKRz$QVMkAA!eVR?f|_y)gCAvoYSboHfs3 z$BshXCcK#4UR$M*x^UJ3UYqGNXKwZH>FJ5Vc;Ggy{XR z`fx-p=O~sTOuc^q+OQKO9(3%Jh*AoOp}76CL$chKE!w0KfNYiqn~U>Q%xcfOowcT+ zG0$au*mM9SKd@ipJmUIc{O~%bTe@~c9&3eKi|B3>DhcXy1bXdZfCM|i3{~o+-XheQ z>|2&+1B1vo5)$A;q=m&Gk{H##L!?IdpHzX1gf%Lq@m}q2iHVck0kh(DZ(bZ5LCC6t zJ62Vg3YorXqPY)$2HDVN)fIi9P3{SF9`prnXw38Z2uw2IRGXQT|#l?@3L5#4G;r_gHwWARLKneW9~ek3A&AOetlWJ+G2IA#F&) z3l=258@A|YHw^1g;}wtt{EM!WW@N_n>0#i4;pyDRspw-;=T@BHI^jZsMOX*N8M;dz zf4QBW2)J++q7FCQxeyx2fC5iU9cN@eBB(je=z>Fi3PhX2q~Q5vNI@(r5a9wN#%S&_ zfv*@DVMc$0^`(UzVgM5EbrKR11)uLp#s>iR=mZx|>l=v!&NA$g>kAgJgm2Xk6ymhl z49&|xcb!(r-Ocpwi+=w3K1dDh^8rDzx0YKg#*a~x{dC-g%tV9jEG$3RB;FWX$Xv=6#UdiRqzp7+E z@I_LhLTO3&zYXvgtUY2eT@)V9;y|#=j|_vHZU$@=jrJ&^C08-hlX$BL@bU=W6_`UG zd?e>mgM-uO*3ww8oC&rkVFR5ySUhQ&0iK>eU+0$5#enFtV;(XgNW6|?VPgv9B|1dF zagY`Y1m7GZ!sUSj-=br#I$96_I&lacTYOD1n61^QlLXqAAc28uKEk3lo|XxS86kLr zojB^NaKgcWUpv%T3wne=mk}m*40-L_A}ks4WIW4xUX`^FP_NdZv%7J|dV%={8_LzN z`@DGRQV0Bd%dwrQuMm1n2u>o@71-+7ABL~92Aq2G@XG%B`6~4$u&DUbMyu>U$AH(3 z)WZvAi~ifrs!~P?ARopJ40u4jZchdx7>;S9y+sQ);vyP>e1#WrZ9pblDu0G zBE3tjcj|o-N>Ds#r38W!I|FDo+*cZK-8gdi!+FctEs>}P69`X2t8zy1B!81 zks(Iw;yY_%#xaS|aUTV+60ZFI`%B2kZQEY1!YrF25!aAQM`v(*K0g|#E40KMVLKXe z%I1!wU59!k{xBIbFJ>AD8rl;OfHNwT?`=7PyBk+^G){X0kKO%jsB&O3y%f2?EiXu> z!|AV-17*86jv?UCyLNi6lE0Un4eq_>)#pbWvYaz=`f=*NN;wJpx@!1Bjb_5BTi3iY z>`0QfP_>NH;1(~et^j(YmEy+_W9(9@vm=yM7;-9?7~9t~h^79gE&{nZzI!xI1N%D; zph21AaYik<`aB(NozU(SycPDNRKuQa_j*!#DW{>2|7q^r#6y{9%8oRGchl9}Q*WAL z)npkPiNc}7BU+ZUUGH7 z-=Q(CG4rgL9Z%sz3c6Qx%P`bhM}Fr{{qyLYVCCgJvxn-vzEV$IN%_+QpK<8mxa%`s@`lytVW( zr1GCv0XC2slFIlb*ilteuaNW6Ex}_MONJp`;|ys!dkra4Sk5L01rn;U0(9u2AxE%T zXX+PFKk&yao^yu{dy%PgwbaHW+#ADyAyacK{R+DfJD#NW+JsPg2tLAk-bRf$D54P- z>~!aOc%wa$^H$peTZY>l0gMvBYDn*D&`WvUgjX0kw)1sj-*xqY>=8wsPkJBI;yZ5- ziZB>|>V~WGnS=BnjlgJoCk0&ZNR$El^EX^;Saan$z3 zN}VFO8by&BiB7O&g12}!q% z#>cpC^|p@g-PhD`l6X%8eSKsqT9b~H+x;wdMqKOnuv&Pm90*MuT`f$GrkTU$Pu^EA z#geZ!ChGUS4mV9;HEc(O?<2a058prC`Xc)AFnIVZoEZ`R?<|inooNWPB8bWe7CT|<_+(VxHn zBGO>s2o!;YRjeRv0!=DE>~xF_i0Suv%TXK*L{y~zlEVOUnHnereiGJCJ;;Ks#zyzJ zveO^(AAroM0C0d=XpnhE1@^(cz5UA!k!t3?D%gp(J8!|Kkv9lqlvGfTfwp-Ez9XjH z+;`48sblI<84?8b0`o>3ruV6!{$(ZZV2zd4o0aYDHHs{%HAqGz*F;^Qugk-&g!Oc4tCBHt ztSV&XIN|w5wWD#U5}OTPKDD9E1-JI%xV+_+=voZOhKO>wWjV{dS;TzhleT?HBv=fY z2}7ZaujVQ5+K_iu;B$5(V=Wdr6@9Qn&WbbsR)k5eRD4nGDC2g`SRx~M|sf&nCFQb7wRwRM4xq|O@ zv1IG`t>V7H!DJ7gQ=yPAhhroH%ZO)byAtAD4QdPG0n_Rl z%yLA{%q$qHqD20z5I-N7t8_PrnhRpH)0ST^c4%GGz?yd!d=SZt9S(;vB<3tjxp>sT!o-GP{(L-1 zm)uw&;PA=52jy0i*_W=a6}wp@gWZuhVBkuegW-ViOqRDB8?nRi8Z8hlljU9uHgF3L zC{ZUK%Pka7em!p0`a_wi#5Y9h5Qgq4RYXuI-L%lFc<9kc*)A&x!!7ncd-e=@kVE)3 zZZ0TTiBEw6c=xfDn_PZ6T4&12n29?4N`>cRd>a;PaDLs$J%b#@g5I3uO@9WQdG}*z zA7)g9LjnV7N29C~f9rLeF;+64{#-aON8lAB#i~IOZl_4*LXZh{LO+d>LODbRAC21* z+*=;Qd5foibnj`(IFKC|<>chF53OQis%Z>Es{Z%G(&ISaRe)l#8(caTMXYu+E8&O{$9TIYPza{9LXTK`-c-d-1p>@v(4rQH_DgZEbMe3)R z;EBPK3`3Vs$TiG?;DPp|m>lzABtmes8CO&ls?0;;8<1y5Ai#%E?uNS0-fHjqEqe;O zYhJv{>JLtA+Un;6p&k@mnz+;oVw*zZv>(zT9LRD0BG;STvRH~l^V>2Hji4+8(6dxGQol}{6+k6p|B%37Nr^iX;l+=ZVz)xvQ zq2R`5M4QP!LgLGpFKsCHD(`GMh5-b*My~gnS7jYTQyLu=kq1?Q^Gpf-*)wMdK}2;= zXG8Nx^%>j-Ak0<*f#BNNmcci~*Hkrrq;6JkQfy<1{mDT9EFGw54kY`RyVjT>oOnBR z>!Es5L2eF)Ns@Z382klrusoh<>)F4rDcC@YU#@M);{z07d%)R|w5&gflu1p0@WoPn zxF1!JZfgK_q8Ygt+t{c#U!6eyOp?R%43ug64@N`vZ4=LDfL zHZA)@J?2as5=?Wh^!@P1N3RyL)dj`qB|=D&(~u=OKpq?RqVT_hSL2 z%yLGtU4i}KwN{xM^CNb{Q3vkKl>k2%tz7BUF)j(JJ01yosN`u&z3?J!H4M*pR$5p3deeCK>@Ni&6P2qI> z%lr9sU$C^x01#~kB#yYR4%FJ=%&x7GtJ>s4fa6@StWSLaB=rOwwwNt-+^)`*uX^II z!uSIB&VPKtQf}@5qr0)Ov7n3D;>3raX$A;!R_Q5=J^1TlD?m+ydiJ-XX8&$ihf+Zd zBXNF>%8!pp0&KEw4%(00vgl;09tc_h5@;t54-cCmI~8fF`kodk@j*lzYF+#}>o#rM zJocwA&dNw^uED%RFZ`uLagLbbB*ri!ubeNU8shd>IuN;0xH0)%S97a;TVU`ZNLIk+udX5CJThyC!yGV-hib zymw+;8HyT=@j2OL)&BgEDYgyd#vNsk+;Kkm8M)}{U#KTf*%3%PAqbDblqQC6VEw0T zJkhOZKL_D@-r4>h;AP#IzrvzsDaz9}_7;xQP2+*rd{6s34#YB=yGqo3|sNdt!nXBTy6K zWFcsYMW(TWuwe;FqOD<5D$}Z3iz3&c-1R-b)cX@WNit77@zY_C{71f2X6PQ`fEREf zRE;6&Lfo90a*V}L*As=i;V%z!B9kkhC>?bL%wQxwwfm`QAZ2y}^0m9m? z-IsMdx>f3twXxv395vwDQIRr30=Jy{!$oq=SWlVB zXx3xfc2)PdT+7DHaMTX#YBL`7XaQlOE)2S)C$LoJ^`S*DWYp9%+LcfW{ryqgl-shy z9~Oqbe`t_u{cK#*;d0*|O+XPV;ZLYQ7&nNnoLQ{|TD}?K6ul{aX^{yOVhkp-2pdPS^HnFR2++g>c^wMqNW9jD)y=nB z$N8%xcf;E)@=fuIohCtf(%nj z7v2*AbS50?apyaSj@L<%ht7T44bOKyplzzhkB8xY({tQ(ESHRh!*xr6hJ@1r2}6w& zwei7-jG+WzLSsl=Zv=Fb8M?=*VgW>=ey3!BJw_?~zI8n_)ek4{XK+l7+t19Da( zivz=^ivIWCf8c@|AB;$qWQ*E;V8ChMaRdnTlhSEO1n8ITXno*@`BZQ`M?D4u#M?Mb2hfB(fxu5q zST_0ewlqdksTmb{A3)r!LinZuGUnI{$TNHhMu7OW16hHG#YZSFaVi2~#wvqVVyK5*T2 zz=~k5d=qx88t7pKIm+NN5(5LNZ%Xq@z^5iL8@2iXM(DFRb8kjr)sTxgEiWtu^!>>p zlw^E(wvi}B1~ZP`w1bY7PU9Lxw|g@WBiSqUy5CElCV{Tw;aCtDFrkY#A8DEx55=)j zo4unkl8El0Kpnmr^)~o8aij@7U|p%Wk#Uv_?Kg75Ez0RqlMHj zs7`MDRqkbIWBsU_M__>w{2nUNu$M6tRtvZE>oYkaqxE$LvD9P6&ZBY(8@LT7rQ}6~ z8`vWrn*f}d?hmG!NyF~|&fCYZ{3`&U(jGMG^`c_UWo$qTkyaUptv@W?BA}Zl#tY7? zYQ%;hEV8D*R$h;4uW8IqC2K~%dbO!@#*GE?Q(Iy=O=jV$#-rP(8 z-v8v|A9Lue!6;C_+LG_euLEOOtyJKS+0E>mC5CspU4&)H_isz^0a7i(d`~|;i zeT5|aZ|30)DMzJchn$uQ41``R3=Qga!lZ8P@}dR!-80%7_e}9t4s%>9`zP9bYT4TW zaU>x-LN^gDI9q6J4rmPp-iRydup++GMagVCHRQ0{=@TJoDXgi{0!N*w<+!I67Sf`3 zyAyQa2@c=dfCqNjh&_~fk?@9iE75hovpT2Ye6I#D(xcOek#7Jzn+08SU;oF0i9cQqw5K_<#pM z`$6;t5K;>W zVWJE@C@M?PBZ4I6Al_01`0zb$H5i-`>P6uKVANLd8Jv0MgCqffl`8+xlf#HsNp_u@ z6y+!*L40jSl7PWoVL02U075qDYZ`* zp@%9DZ=3vGUnn`Ku*4}YR%?u1D7;}qETclvYu!x?Uo;K0kWIv+-z$QKE20`a@ztQ4 zBDohDUfk{XH;Dbjx#eK?*q|RfjQC4SLSLS*7&G4sIth{vC)cCHOPyyTbrfXc@sQvM zDAvu;&`{i=wM>dMG+2NpH=z?}A|`pfY5vI5F=ol9#uhZaD{!|6x+#486r`G$$f|mD zEXThNgW9JQQ8TCJ<=!4;$9s54u#2KTATl(JLpg$5ql{URq+h~-kRU*TvgJDrO&n3{ z(Apt`Ff@5fx>~l0vg&H^4fLGaRNyf{&I|yNffss`lUb9?=m7w^`^tI9=Kx@!A&W?7 z#GRlnBf51&+al@^jJYr26ur$ji2;9E80koT&jKJw2+q_*#eE3HInjp3TNx@ujj)3c zY)8beUGMwVrv20bapw_Rz#5Is$uDP>*F3761)3~B0Tn1RBhfz+2&wDM8jVpb7)6yy zd?9LGA(r!-3KAr;w4K4c@LD9(+Of4aF#mB6*z9IVCGP4S-wUvSoR=_Zlm7A z>1X>wP^L3rlOL?e8m6U<%#(Z>$3R^iJovT0y~1&PxD~yU_0$i>!rHsSN~L-`kF~)G zk0F*VaA9220sNzJ{P^)J4)#s$h>a_xP6vTJMJp27BO@aUZ57J!(1;%%9qlBz1r1&O z*Zd3**>~yvTY2=+e4rSG3Ic*So^V8vc@DOQnCA>0gjp(|jv_LGDj+JtI`O-RgNfYv zPVE85w&28Ar?Y3nA?yP*;D@mSCj+^W1;cam1gff$Hy7p;8v{vNZL2BL5lREGhtTG* z3bafDay@8)yQoA&8FYh!5y7`qN&$$a1PD)S9#Q)+={MF4+nbUjv@pEvd3G-OW0*I* zKmGqI?o6YiJhLsFv`J_)$c=i9aROVED542YNFizxBLxZuM^pwQ7(q}p3<{R0q??#F zK?{XQ5Db>$ghWM95zsiGsGuMq7KlR>uox1Vq)OmEhuEvvx~p&Z&)aL|N5@zc-}k;} z*n6LSh6yjsFrjBo`upq?qIOwV(DL%FoqGK>pNAB{vA@3frD^JV%;CE5hTytGuXcOG zn&q`Jl80JmpU-aB@9I`9GxUGhQNGG#L1u6YRae*ep>J{C5_hPrY>uR|5G?9ZK-G#Z zJ%={sD;n(2R5)Y)Wuj?w?Wa07?qEgfBi2%&g#w&gP6Ixw`1kWhCe^(3emIH(x3o>T zDknkeHUS1#d?yl&16L#iMNc6Mv`bJhc;1bsk8PcN>YVH8t1HWHkU2YlKKZS4Ls4g-I zl(cGgf14TMBMXvOt$~snQ>=~DSesZoq^?L3LjvZu=FpYSUjM9aWz(jn)&oti)>mKX zmdTwFM-VA&DJ+~%G+(}Si46^HYK@3xZj@+GeCaHx9;mx?!26vODMW*7_|aXS=j`Hx z>QD@99Z6rpk3B)k-_Mn=pxiHvtF;8Y!q0iM3^V2y)?_Uo7I;N~6=x z^ASx6Hf(}}iZV^~N0?ixcXS7f1{zDFnUpHa-QE2eFgy1~DkZr(J+9411^`7T{3{b)aOIsSuUdKi%%TDSkQRXfPbV*)$=Xqtc~JK9@4V)N$lG^r)N zIvr1`Y&P_?&wY1mlK?w=r_UDm+BZ}`rQ~ThB$%E+gD&)uB$$u_#c&Dau-RR_Oy7oz z-MLTb@9+$kE&=>xxBz~DTys%lpp%-L8SGCMc%I#G@6MSZmcYmDBYVghETt~3%@~u2 zSVhJOFFXU;bs+%5QbDP%k4oKk^=V#E}h~f*^ zciIJr|Ee#yUmf4sv#9<5T#p#U0{W*6){;828JfBm`e>TU<4@2V!TO5>#@ zyI7Ppshu=m{qX6b57)OgB=(uP!L(_6=gsLZ=A1HFgTUiPnU%x~^`WsAr4t5kJ14Ai z7e?>u(yd43xHs`n9yt-%!UZc9$%(vdd_hKBjkVCGp*iDtIMSkto{Y$PkW;>#a!<5Y z(QlU?U-Q^Oyc!+s?NP;SC%^FBgpk#GjDLKc%r7hy7>uw!oI&BcWSbmownGGsq7*+2 zmsNIcJ;{xJI!;Cd$b9lZ1OgWnn+o|LmA!Ja?Dj%tIbDbbLI!wArlDiTcl8@ugQS*H zs;`ZN>`Wj0(iA^AT}7+vPg zPxVE;;?xPzd}7?lkt3D6aDfv|#Yu3{61Htzd|Fi%tMLPsI6{Fv>STsAg3ox%AwRs= zySJA}Be(=tL#e4E*Cnohwng+1i_(@anI!e1o;L9a%vgrVeh3&xCak)bR*<>+(ceyT z!9--kVNV+;4CQDMdt|J13tLpbIJwyOWcI&O;fQAEt_U82#;+*3?t5&(5lRS$;}!UH zrwy_Ryi)V!@ntvqb!lFnKm7aE^}hB>U;V~0G=?V{a`U$iRFufyVJzMExK8x*@?~?s z>mU0xw5FT#mXVA3ID;N>WC~F*0~eX12haT!#?}VyowEp_O30M1tqql{5|?Q`h-7hx z#`geIl}`jFa4vq9e{;GzDN5r+kxq{G5?K&=_Kxn9L2PMU_a-GRK2%yVn!E2;0flnm zEL*HD$zLZzAHRJ0vdwigOxxr;C9Zq&zK&X}u4wPxy(!d@UFx6GqizH9Q1~pMVD*tt zM^riSIu66faWyQ?whR$c4acb)@|p6~-7|x4*o_`FD&GbYU7iZTyvVMd09Zn$;VLq! zNr*z!b#@GIG=u0lP42a*Qp$*;wo`Q+t;5B>6#T?qLvV`Iij$ zCf2T@i8v-tf=<^{M~UTC+2FpR&09y&>*i}w6+wC}kGWU+X~>%9h2PjsG4V9*Z~LLv zZTj$iWfZ=*Xn+pE@A!6hv@fI^1J6!nc=)LxSE!2UD1V(bY{ZDX><+l0%N-9l0YdTs zTt~Q_N$X)^cSPL2E?qiZUN6>;QnM8?bDWA_(1^i<&v#rq6BxF!whNx_5}bG(sHmu5 zse6MMYRSwBN!>p&gXWvE%nAL-EhaPSGG;is5XGXimOcgw(X)|7R$&Sn#_j*=he|Zy zC^SD^vSi8Kv+GW%^wce)3{38(s$y#V>RR*h=^v^1a1t@orl3ZOG7pC@FDjy~PKo(PYB1xY0d}8IwHf*t;t( zwB4WFv|U@+7*{>@mQi(ArX>*U(DcszsEy8n7^#o}`H5?O6={bIn@i=+do3b1i(y@) zRY_-O-tSb7oIj@_pKYDB%gTEPB@szt3gOCyN~m|gesNx=mA6kBHkVuu{yDqxMi836 zS!B-j>(|G-FJOlL5Ogns5%;A_!xV+RoNXTj+I~%aW#fMV_TLMyQ((Htx9%#+2Hh&7 zSWzvckAQ<8C#0y{nL{2oJq=o^*vQ}xbKieqK2CLiC3VGYk`nr;?LJX&+%oE*QQjR^ zr;}!$26-=4;ZVcAid{$%rG1dHi7Qi-bkJwB-7^!t{ZmbAn<+zS(uD$p&4Gp&brQn# z+cZ?{*^s9S*q2#e&!={&3bpusQ_w#~fHhV#e9nPwg0x<3%f1J8nH)r$=EcL6V97AETz31dh^8fV({S*6xMK&Hyv-h(vcE9ZHkHg zh{MgVUZy6;Y%WIQrI_o$n-EdDFGMJi=-vz6@uY}v=?n=D7ztM2(VElgLlUR8UvxZT znt5)SXMAC}b~QYhVXStOF8`QE{XAqH%J1L1kiKbi!=SyFaeTAQ+pWmpxIg)@oyVnK z*{RQBQA6*HN068;ot>DJ_3qnOa1I|j?>yv0;xEHHsR4CzYKhe-I)k}rj?XOq>=Ha# zb#8ohDDvg#fj^g*7y+zAyE$`$E*w9)$M{90-8LJ#8dBqJDrlrCJ>(iX8F>d^M#u%O zi-Aos>X03@v19!kBlbk`SJXB&D!Xm*9=*iw714@w_`wSGbWMMA`w8?Dr$AmQXvgP`LWH&)MRSVsdS0<3B|_# z1@q@sJ^va$OE;MO$hgvyW=33M`o9U$)`ruXfhW-3b$J%$qwL&0pkTKmw$`fEX7K*6?S3=S zPkE|IOHBmRFn5aI1~mXm#*hQsoRgwWtkl8M|JkmEmk_m<-T`XntA+XIemvEBld3TOm)jl z7kM|@*)Cy%{7zg}+3!iMSIl{wi>HX0?&msPZPoCvygj0}Z26qZpRjEK>Wq$MOxY}h z6Tneu)$WiqN+Pl9g$Y!Bdw!||UGPy3u?`4au<2_{tX~$2la6jF>_w6}QV8Dj8@`O! zlF4S;+}mI6OewDjF#|S1LIG?y!pf?!=C3LK*ZXO$y=(S)p5H9tFJ^#tboY?#VdX8I zqG4G0gnmc4Vf(fY!zD9Q(apUT{3%WE+Si2G9Y8my(S}TzHhsFH{fq6NmwuYA zDoLNLhz1sghw(Dl%Y_LAuM%K7^Y%Q5FhQFkIkyKIoW2*4n0R5*sKCu$HuLri&!ip6 zoHKu(v?{o}hzx>IDjFofoygS#qod)A(vjl^R$;s8N7(Q(8Ox*0}@GDoxR8GG1z_W(znaq z#-|7`N3)NQXCbFD`g#rpHXQCBay`|Vx}b9Y{u=!++o)4)Qd0TSY7NU&tjpg}O%~P) zPAN~TT?E=dlkIqXtL4ZuhUO>s1Lqqy)_s%$FST#&d(#lT0wBLC18ht7Ec}tzVL^7G zT?l+0G-48V82iqq3c5gpFtcT{c3Lf-;hrWF9M7~nE_ZnBxAf2qI8RZu0XL3WzQXMj zlTAA|R7FsTmYEC|qC5I}6u*EBvtHYuoI5CN1`GierwytGbsI@UT(i)1>aoOs5}4I` zw-h?g+a#xLgV3uakC;HizP~`lDv!TSiJ@8MA&)_{Lr^qfS7PIGS27CIEW$W|(~QXV zt!I90bVuLlSLs2s(>_Ia=DmC)*H1!e-OY&refJsvSJQ({Wj0m2yDhrs!|&Z=)ru2a zzW=_2m~xVlLZu7rOZ3Pwp64)h6@HP9x-2;tf6o`se;2r1v=ltNN!80VAJ7I1QE2AF zpY_Obx(Im0jOrO&$_3D%qtjN&8G{2agkF>JpU7A380KlFMYTyOph=~b5?idX{J7*C z--{P7R`WAY8owqGi8KOR6`sAAC?RI&-MBFmY{1E?vL(Z@X62vE8r)U)oum=x;bKhD zaJrHSJ_+Y)*)<=TY0iFO|M2S{efZ&r6|0hyji;>f_bBkK9l9L6$(tRxhZcNFp{*o|uwKAt95A0x=`iomdghYSbjEGz$_;VO^%ZOgAP+-7Y(- zX#Dl+qd-5)mgd{1OS3opQ_B*0ZEz(-O&V6{u^(s4z#NY!NI$ewXlNl^Xbr^Q6Z|H# zNe+OOB(O?M6z(0op zMn(`TyY@RKOtKKKu3c7_mYA;lPu#n2WlZLG?~nP60W(pEvZ1{{TQ@(a$B0E?P1#80 znnYbI&aOPh`M@DE2PEduUmQEYG?C1c@t;kSt(rMSe;=tG7)_jYPBbpKUterg@73S5 zMPSb0_&#v-=p0ZGu#mP!EA^EDVX}%uKiiYrC`xqq4_9i=R*vM zxa3B{B!$wAXhK0SpEN0}8wg+5u*nt%l4ryDPhqjjP&2awB-p$d%OAz+SNIkBo3z%q zlz{R(=;ua8U)2!C&D#uj68c=FQEI<&KA_&+phr7YoBJIng*~_3ol0lJk4_mh(= zR~E6DhdkJYs*&sk)~7g=a=_X4_pcMeJKB1O&t3a(*TzJ(dk?CyL=P%T>XQKvKBk43 z!GYlL#(dj*coK@(L*S`BmcD)G-xK3X)E;XVGT3oiq`7i5(K5+q8KI0n;2wyNu*^hr zqP5|Vo2aT_Tt)gRQf}n3Ex9&}PWtYvN(jrZ*;7O)_-!^$cOH#b597S zEz@I!YC9B^dHS>u)0ZONmJlLUp_-l1>u`#m20VY`=c%`Ay4lxdCv3-BkBT^a=&lX| zhRxYu1(>h7*GK>7!p8Nc6@yM?Q35kDpV+%+&m`$;Q82!gcyxnlnNs9tvc6o9Hk3m$SrHgkE|~cNq--s#jT@R|1k%dN1-hFl zOXoP1`)7fvs`mWcq}|{Bb#1aDB3p57-`I3l{(QjZYyn<{`QXOL`a6(Mj-zNFh-~v| zTm*G}JM4s_s@o&Sf)z=Wg^H|n^ef@Dlb7H7PR?KSRsEg_w^!71iG}3hT^FK{u*#em za2~n)*j4vzWu=gDb81gxh5p>$zuVpK*V23>((YkD{gu6#D|ASt5T8ZCz~OCgF7+q~ zg_ub)_L>ysVB)#ePP2t#;UKR~LQ&DCUB3s&OP2QvGFtVM-KFrTwQU+faZ#4HU~$)*B zt&Z*F2ijZKi-q!)hh*B@$CgeHs=hgujP^NzhIHrRFH>OKxD{uOgB-eKr5o36JCk%$ zfDhAMTP`!1{9gczn6jBzx7csS)d@SIQ}Ql3BWr$G#t;n38p%40KF*vvK4 zD|ZVk&J9zlMIA44d$ouwBt~%W-^422%S$8m6u*28mKh>x99S|2mJ~?GtSl7#V1)8p zYBTO$8s8GseFOqN^&C1a5r>Fy05Tuwa=~HB6zGkWc`*LWXWz_dq~f|tK4>0c(Hsll&jC(!zdsENWYhEXyw@Vc~Y0%EmH zJD@V1yMM;}G9DyIfv2_GSW# zCQx=~@mrj54ksWM=Y)5fJ!%mF0PTJW&$JoUcIDW)o zdjU&saiT3}`!ax#W|(##9=}*1$n2rvEFFbtFS+K$M1OFq+D2L1HDEyPtXzIqsej$z zm-TfH*F zMgc0@Nq6Jqk&k@SRs2NM8gSOri{~Top~Y53ZgZCHq4Fh%17Rvl!dQEwbU}hK0JmGI zp-@K-+%N0aXuez^=%pZ1+c`^8Exjv7uK79Z?nn zmv_PAnJ?=53>;Xu#!Z-7;6$L5`rfesZRr_RB)<|WkAosYLF=&t%9AgX%<&Z=v$DQg zLC<3*egLh01&|lr2_WZ&Z0g$IG8{#}C<@IG*D5nA=NS+=Kcym-rA(6X$#rL2K2sJV{f~PZyYz<{!$kSR?T$7xxNRw{iw(9Hz1!@QM{lR2qI>B= zY1$q30S4e8U9OPmoBs5=SP&Ur(E2OYB5EL57@BB$WCJ3vFX#+mr!kMi3!ENklkP!T zxsxdbv42pxA0K6>m9>gb-+!gRmVysK$zEe{x zjtgkTBYn8VF=oK~rd@xVwp;l5;#ic$w|UVN8JyR~FLqiuM;SOB->D8hkkj zuK>qcepph8P#F}>9^~VHsptunqN0}xQ|yIHf4WD+cO$;eLnR^7HS{U=YI+4rrG%sQ zQNMsx9VU_4gtZ(IsZ?@8r~?HqX{QuCMZhOzA$p-h6hnsryC9a;B$FU8L=iFq^G3t{ z0s)DdB1{p?20;Q2Ael%5XVcP)69f&j$ZBN>XiipW5F^kx-JfuTMAFXb5`_@*C(N-e zOij;qEclRwvaHL4|4NeI)XLom+=Q6eIm&Gz_0cU@hMy>gCR&CA%*v6JC@VWiAiNKv z5fE$dK>kQ%pH30kjMebIl!P!DzCfaHt`w$8wTp{e!zfTOTBY91uEDh&qJj;rKV5~D zRHNBc4+=ur|3+wk3T`8tVzBHG$b#Pz0kxBp$C=f{MClzx<$yARm%UF0d1b_hwhYkM zg2}CkJGv9iKrSgv1{2$7vO`WRLWluXdJ|s0={qm|QLzvELw(BG_