Jax_federated_learning-predict_age.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Federated learning example with Weight Erosion scheme\n",
    "Based on an earlier version of [Sai Praneeth Karimireddy's Jax-based FL simulation framework](https://colab.research.google.com/drive/1tLFkhmWWbY5veOcqaeXkFbJh9M_4f0hQ) (GitHub: @saipraneet)\n",
    "\n",
    "# Install dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting git+https://github.com/deepmind/dm-haiku\n",
      "  Cloning https://github.com/deepmind/dm-haiku to /tmp/pip-req-build-4vc4nx34\n",
      "  Running command git clone -q https://github.com/deepmind/dm-haiku /tmp/pip-req-build-4vc4nx34\n",
      "Requirement already satisfied, skipping upgrade: absl-py>=0.7.1 in /opt/anaconda3/lib/python3.7/site-packages (from dm-haiku==0.0.1b0) (0.9.0)\n",
      "Collecting numpy>=1.18.0 (from dm-haiku==0.0.1b0)\n",
      "  Using cached https://files.pythonhosted.org/packages/1f/df/7988fbbdc8c9b8efb575029498ad84b77e023a3e4623e85068823a102b1d/numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl\n",
      "Requirement already satisfied, skipping upgrade: six>=1.12.0 in /opt/anaconda3/lib/python3.7/site-packages (from dm-haiku==0.0.1b0) (1.12.0)\n",
      "Requirement already satisfied, skipping upgrade: dm-tree>=0.1.1 in /home/grimberg/.local/lib/python3.7/site-packages (from dm-haiku==0.0.1b0) (0.1.5)\n",
      "Requirement already satisfied, skipping upgrade: wrapt>=1.11.1 in /opt/anaconda3/lib/python3.7/site-packages (from dm-haiku==0.0.1b0) (1.11.2)\n",
      "Requirement already satisfied, skipping upgrade: tabulate>=0.7.5 in /home/grimberg/.local/lib/python3.7/site-packages (from dm-haiku==0.0.1b0) (0.8.7)\n",
      "Building wheels for collected packages: dm-haiku\n",
      "  Building wheel for dm-haiku (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for dm-haiku: filename=dm_haiku-0.0.1b0-cp37-none-any.whl size=216854 sha256=c10165a110d178c91251d0d55bb70d66d90cfcd4f666af93ccbc5230e3c64654\n",
      "  Stored in directory: /tmp/pip-ephem-wheel-cache-ubg6r2ca/wheels/97/0f/e9/17f34e377f8d4060fa88a7e82bee5d8afbf7972384768a5499\n",
      "Successfully built dm-haiku\n",
      "\u001b[31mERROR: tensorflow-federated 0.13.1 has requirement numpy~=1.17.5, but you'll have numpy 1.18.4 which is incompatible.\u001b[0m\n",
      "Installing collected packages: numpy, dm-haiku\n",
      "  Found existing installation: numpy 1.17.5\n",
      "    Uninstalling numpy-1.17.5:\n",
      "      Successfully uninstalled numpy-1.17.5\n",
      "  Found existing installation: dm-haiku 0.0.1b0\n",
      "    Uninstalling dm-haiku-0.0.1b0:\n",
      "      Successfully uninstalled dm-haiku-0.0.1b0\n",
      "Successfully installed dm-haiku-0.0.1b0 numpy-1.18.4\n"
     ]
    }
   ],
   "source": [
    "!pip install --upgrade --user git+https://github.com/deepmind/dm-haiku"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already up-to-date: numpy in /home/grimberg/.local/lib/python3.7/site-packages (1.18.4)\n"
     ]
    }
   ],
   "source": [
    "!pip install --upgrade --user numpy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "!pip install --user setGPU"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import packages (etc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "yRDenD5M0rDY"
   },
   "outputs": [],
   "source": [
    "from typing import Any, Generator, Tuple, Mapping, Sequence, Optional, Callable, Union, NamedTuple\n",
    "from collections import namedtuple\n",
    "import functools, inspect, time, random\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from absl import app\n",
    "import haiku as hk\n",
    "import jax\n",
    "from jax.experimental import optix\n",
    "from jax.tree_util import tree_multimap, tree_map, tree_reduce, tree_leaves\n",
    "import jax.numpy as jnp\n",
    "from jax.lax import fori_loop\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "yRDenD5M0rDY"
   },
   "outputs": [],
   "source": [
    "from module.custom_types import *\n",
    "import module.utils as utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "setGPU: Setting GPU to: 4\n"
     ]
    }
   ],
   "source": [
    "import setGPU"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Misc type-related stuff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "S-RWB9G48uJA"
   },
   "outputs": [],
   "source": [
    "# extracts messages from a list of client outputs.\n",
    "# @jax.partial(jax.jit, static_argnums=[1])  # fix extractor.\n",
    "def extract_from_cout(\n",
    "    couts: Sequence[ClientOutput],\n",
    "    extractor: Callable[[ClientOutput], Any]\n",
    "    ) -> Sequence[Any]:\n",
    "#   print(\"compiling: {}\".format(inspect.currentframe().f_code.co_name))\n",
    "  # TODO: use pytrees.transpose?\n",
    "  # msgs = fori_loop(0, len(couts),\n",
    "  #                    lambda i, msgs: msgs.append(extractor(couts[i])), [])\n",
    "  msgs = [extractor(cout) for cout in couts]\n",
    "  return msgs\n",
    "\n",
    "def init_client_states(client_data: Sequence[tf.data.Dataset]) -> Sequence[ClientState]:\n",
    "  return [ClientState() for i in range(len(client_data))]\n",
    "\n",
    "def make_jax_batch(batch: Mapping[str, tf.Tensor]) -> Batch:\n",
    "    \"\"\"Transform a Mapping[str, tf.Tensor] (tensorflow batch)\n",
    "    into a Mapping[str, jnp.array] (jax-compatible batch).\"\"\"\n",
    "    jax_batch = {}\n",
    "    for key in batch:\n",
    "        jax_batch[key] = jnp.array(batch[key].numpy(), dtype=np.float32)\n",
    "    return jax_batch\n",
    "\n",
    "def make_jax_dataset(ds: tf.data.Dataset) -> Sequence[Batch]:\n",
    "    jax_ds = []\n",
    "    for batch in ds:\n",
    "        jax_ds.append(make_jax_batch(batch))\n",
    "    return jax_ds"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Batch the data sets and create the test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_dss(clients: Sequence[Tuple[np.ndarray, np.ndarray]],\n",
    "             hyperparams: ServerHyperParams,\n",
    "             requesting_client: int = 0,\n",
    "             test_ratio: float = 0.5\n",
    ") -> Tuple[DatasetSequence, tf.data.Dataset]:\n",
    "    \"\"\"Make batched `tf.data.Dataset`s based on the (X,y) tuples representing\n",
    "    each client's data. \n",
    "    \n",
    "    requesting_client: Index of the age group for which a test set is created.\n",
    "    The training set for the requesting client is placed at index 0 in the returned\n",
    "    DatasetSequence.\"\"\"\n",
    "    if requesting_client < 0 or requesting_client >= len(clients):\n",
    "        raise ValueError(\"Unexpected 'requesting_client' argument: \"\n",
    "                         \"Expecting a non-negative int lower than \"\n",
    "                         \"{}.\".format(len(clients)))\n",
    "    \n",
    "    dss, ds_test = [None], None\n",
    "    for i, (X,y) in enumerate(clients):\n",
    "        if i == requesting_client:\n",
    "            test_size = 1 + int(y.shape[0] * test_ratio)\n",
    "            ds_test = utils.make_ds(X[:test_size], y[:test_size], hyperparams.batch_size)\n",
    "            dss[0] = utils.make_ds(X[test_size:], y[test_size:], hyperparams.batch_size)\n",
    "        else:\n",
    "            dss.append( utils.make_ds(X, y, hyperparams.batch_size) )\n",
    "    \n",
    "    assert (ds_test is not None)\n",
    "    \n",
    "    return dss, ds_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [],
   "source": [
    "# hyperparams = ServerHyperParams(\n",
    "#     num_rounds = 30,\n",
    "#     max_batches_per_round = 3,\n",
    "#     max_epochs_per_round = 1,\n",
    "#     batch_size = 10,\n",
    "#     seed = 420\n",
    "# )\n",
    "# dss, ds_test = make_dss(split_by_age(*load_2017annie_predict_EVD()), hyperparams)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for batch in ds_test:\n",
    "#     print(batch)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model, loss function, training, etc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "GbTmc5DwjoON"
   },
   "outputs": [],
   "source": [
    "# Neural network model.\n",
    "def net_fn(batch: Batch) -> jnp.ndarray:\n",
    "  \"\"\"Logistic regression for 2-class classification.\"\"\"\n",
    "  x = batch[\"features\"]\n",
    "  linear_reg = hk.Sequential([\n",
    "      hk.Flatten(),\n",
    "      hk.Linear(1, with_bias=True)\n",
    "  ])\n",
    "  return linear_reg(x)\n",
    "net: hk.Transformed = hk.transform(net_fn)\n",
    "\n",
    "# Initialize neural network parameters \n",
    "def init(rng: jax.random.PRNGKey, batch: Batch) -> hk.Params:\n",
    "  return net.init(rng, batch)\n",
    "\n",
    "# get predictions from model.\n",
    "def forward(params: hk.Params, batch: Batch):\n",
    "  return jax.jit(net.apply)(params, batch)\n",
    "\n",
    "\n",
    "# Training loss (cross-entropy).\n",
    "@jax.jit\n",
    "def loss(params: hk.Params, batch: Batch) -> jnp.ndarray:\n",
    "  \"\"\"Compute the loss of the network, including L2.\"\"\"\n",
    "  print(\"compiling: {}\".format(inspect.currentframe().f_code.co_name))\n",
    "  preds = forward(params, batch)\n",
    "  # TODO: Put weight decay into optimizer\n",
    "  l2_loss = 0.5 * sum(jnp.sum(jnp.square(p)) for p in jax.tree_leaves(params))\n",
    "  mse = jnp.mean(jnp.square(batch[\"label\"] - preds), dtype=np.float32)\n",
    "  return mse + 1e-4 * l2_loss\n",
    "\n",
    "# Evaluation metric (RMSE).\n",
    "@jax.jit\n",
    "def rmse(params: hk.Params, batch: Batch) -> jnp.ndarray:\n",
    "  print(\"compiling: {}\".format(inspect.currentframe().f_code.co_name))\n",
    "  return jnp.sqrt(loss(params, batch))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "elJQ6h8MVeH6"
   },
   "outputs": [],
   "source": [
    "# one local update step.\n",
    "@jax.partial(jax.jit, static_argnums=[2,4])  # fix loss function and optimizer.\n",
    "def run_one_step(\n",
    "    params: hk.Params,\n",
    "    batch: Batch,\n",
    "    client_opt: optix.InitUpdate,\n",
    "    opt_state: OptState,\n",
    "    loss: LossFunction\n",
    "    ) -> Tuple[hk.Params, OptState]:\n",
    "  \"\"\"Learning rule (stochastic gradient descent).\"\"\"\n",
    "  print(\"compiling: {}\".format(inspect.currentframe().f_code.co_name))\n",
    "  grads = jax.grad(loss)(params, batch)\n",
    "  updates, opt_state = client_opt.update(grads, opt_state)\n",
    "  new_params = optix.apply_updates(params, updates)\n",
    "  return new_params, opt_state\n",
    "\n",
    "# perform client updates.\n",
    "def client_updater(\n",
    "    msg: ClientMessage,\n",
    "    ds: Sequence[Batch],\n",
    "    client_opt: optix.InitUpdate,\n",
    "    loss: LossFunction\n",
    "    ) -> ClientOutput:\n",
    "  opt_state = client_opt.init(*msg.opt_init_input)\n",
    "  # iterate through data making updates.\n",
    "  new_params = msg.params\n",
    "  train_loss, train_acc = [], []\n",
    "  for minibatch in ds:\n",
    "    new_params, opt_state = run_one_step(new_params,\n",
    "                                         minibatch,\n",
    "                                         client_opt, \n",
    "                                         opt_state, \n",
    "                                         loss)\n",
    "    train_loss.append(loss(msg.params, minibatch))\n",
    "    train_acc.append(rmse(msg.params, minibatch))\n",
    "  # compute and return the change in parameters.\n",
    "  params_update = tree_multimap(lambda x, y: x - y, new_params, msg.params)\n",
    "  \n",
    "  # TODO: replace with an function which constructs message  \n",
    "  msg_to_server = ServerMessage(\n",
    "      aggregator_input=params_update,\n",
    "      stateupdater_input=None\n",
    "  )\n",
    "\n",
    "  diagnostic_msg = DiagnosticsMessage(\n",
    "      train_loss=np.mean(train_loss),\n",
    "      train_acc=np.mean(train_acc),\n",
    "      test_loss=0,\n",
    "      test_acc=1,\n",
    "      weight=len(train_loss)\n",
    "  )\n",
    "  return msg_to_server, diagnostic_msg"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Make clients stateful (remember which batch is up next)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Absolutely not suitable for jit-compilation.\n",
    "def make_client_data(\n",
    "    ds: tf.data.Dataset,\n",
    "    client_state: ClientState,\n",
    "    hyperparams: ServerHyperParams\n",
    ") -> Tuple[Sequence[Batch], ClientState]:\n",
    "  \"\"\"Picks out and assembles the batches used in the next round\n",
    "  for one client, updating the client state along the way.\n",
    "  \"\"\"\n",
    "  batches_before, batches_after = [], []\n",
    "  \n",
    "  # Initially skip all the way to client_state.next_idx:\n",
    "  for i, batch in enumerate(ds):\n",
    "    if i < client_state.next_idx:\n",
    "      batches_before.append(\n",
    "        make_jax_batch(batch)\n",
    "      )\n",
    "    # Return once num_batches batches have been selected:\n",
    "    elif len(batches_after) >= hyperparams.max_batches_per_round:\n",
    "      client_state = client_state._replace(next_idx = i)\n",
    "      return batches_after, client_state\n",
    "    # Transform the tf batch to a Batch (Mapping[str, jnp.array])\n",
    "    else:\n",
    "      batches_after.append(\n",
    "        make_jax_batch(batch)\n",
    "      )\n",
    "  \n",
    "  # Increment the client's epoch count once, because we've reached\n",
    "  # the end of the dataset:\n",
    "  client_state = client_state._replace(\n",
    "                            next_idx = 0,\n",
    "                            epoch_count = client_state.epoch_count + 1\n",
    "                            )\n",
    "  \n",
    "  # Traverse the dataset up to max_epochs-1 more times, until num_batches\n",
    "  # batches are in batches_after\n",
    "  for epoch in range(hyperparams.max_epochs_per_round - 1):\n",
    "    \n",
    "    for i, batch in enumerate(ds):\n",
    "      # Return once num_batches batches have been selected:\n",
    "      if len(batches_after) >= hyperparams.max_batches_per_round:\n",
    "        client_state = client_state._replace(next_idx = i)\n",
    "        return batches_after, client_state\n",
    "      # Transform the tf batch to a Batch (Mapping[str, jnp.array])\n",
    "      batches_after.append(\n",
    "        make_jax_batch(batch)\n",
    "      )\n",
    "    \n",
    "    # Increment the client's epoch_count\n",
    "    client_state = client_state._replace(\n",
    "                            next_idx = 0,\n",
    "                            epoch_count = client_state.epoch_count + 1\n",
    "                            )\n",
    "  \n",
    "  # If num_batches has not yet been reached,\n",
    "  # add (some or all of) the batches_before at the end:\n",
    "  num_to_add = hyperparams.max_batches_per_round - len(batches_after)\n",
    "  batches_to_add = batches_before[:num_to_add]\n",
    "  client_state = client_state._replace(next_idx = len(batches_to_add))\n",
    "  batches_after += batches_to_add\n",
    "  return batches_after, client_state\n",
    "\n",
    "def make_all_client_data(\n",
    "    client_data: Sequence[tf.data.Dataset],\n",
    "    client_states: Sequence[ClientState],\n",
    "    hyperparams: ServerHyperParams\n",
    ") -> Tuple[Sequence[Sequence[Batch]], Sequence[ClientState]]:\n",
    "  \"\"\"Picks out and assembles the batches used in the next round\n",
    "  for all clients, updating their states along the way.\n",
    "  \"\"\"\n",
    "  if not (len(client_states) == len(client_data)):\n",
    "    raise ValueError(\n",
    "        \"Number of datasets ({}) does not fit number of client \"\n",
    "        \"states ({})!\".format(len(client_data), len(client_states))\n",
    "        )\n",
    "  dss_jax, client_states_after = [], []\n",
    "  for ds, client_state_before in zip(client_data, client_states):\n",
    "    batches_jax, client_state_after = make_client_data(ds, client_state_before, hyperparams)\n",
    "    dss_jax.append(batches_jax)\n",
    "    client_states_after.append(client_state_after)\n",
    "  return dss_jax, client_states_after"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Aggregators"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "zvRbtDM1wS_s"
   },
   "outputs": [],
   "source": [
    "# aggregate client updates.\n",
    "# TODO: make the aggregator stateful.\n",
    "@jax.jit\n",
    "def average_params(params_list: Sequence[hk.Params],\n",
    "                   client_states: Sequence[ClientState],\n",
    "                   hyperparams: AverageAggregatorHyperParams\n",
    "                  ) -> hk.Params: \n",
    "  print(\"compiling: {}\".format(inspect.currentframe().f_code.co_name))\n",
    "  num_params = len(params_list)\n",
    "  params_sum = functools.reduce(\n",
    "      lambda t1, t2: tree_multimap(sum, t1, t2), params_list)\n",
    "  params_avg = tree_map(lambda x: x/num_params, params_sum)\n",
    "  return params_avg, client_states\n",
    "\n",
    "@jax.jit\n",
    "def normalize_array_l2(x: jnp.array) -> jnp.array:\n",
    "    print(\"compiling: {}\".format(inspect.currentframe().f_code.co_name))\n",
    "    return x / jnp.sqrt(jnp.dot(x, x))\n",
    "\n",
    "@jax.jit\n",
    "def normalize_array_l1(x: jnp.array) -> jnp.array:\n",
    "    print(\"compiling: {}\".format(inspect.currentframe().f_code.co_name))\n",
    "    return x / jnp.sum(jnp.abs(x))\n",
    "\n",
    "@jax.jit\n",
    "def tree_norm(tree: hk.Params) -> jnp.array:\n",
    "    print(\"compiling: {}\".format(inspect.currentframe().f_code.co_name))\n",
    "    return jnp.sqrt(\n",
    "               jnp.sum(\n",
    "                   tree_leaves(\n",
    "                       tree_map(\n",
    "                           lambda x: jnp.sum(x*x),\n",
    "                           tree\n",
    "                       )\n",
    "                   )\n",
    "               )\n",
    "           )\n",
    "\n",
    "# @jax.jit\n",
    "def calc_rel_distances_of_others(reference_update: hk.Params,\n",
    "                                 other_updates: Sequence[hk.Params]\n",
    "                                ) -> Sequence[Any]:\n",
    "#   print(\"compiling: {}\".format(inspect.currentframe().f_code.co_name))\n",
    "  # Compare the distances to the norm of the reference update.\n",
    "  reference_norm = tree_norm(reference_update)\n",
    "  return [ tree_norm(tree_multimap(lambda x, y: x-y, update, reference_update))\n",
    "           / reference_norm \n",
    "           for update in other_updates ]\n",
    "\n",
    "# @jax.partial\n",
    "def sum_args(*args):\n",
    "#   print(\"compiling: {} with {} arguments\".format(inspect.currentframe().f_code.co_name, len(args)))\n",
    "  return sum(args)\n",
    "\n",
    "# @jax.jit\n",
    "def similarity_aggregator(updates_list: Sequence[hk.Params],\n",
    "                          client_states: Sequence[ClientState],\n",
    "                          hyperparams: SimilarityAggregatorHyperParams\n",
    "                         ) -> Tuple[hk.Params, Sequence[ClientState]]:\n",
    "#     print(\"compiling: {}\".format(inspect.currentframe().f_code.co_name))\n",
    "    assert len(updates_list) == len(client_states)\n",
    "    reference_update = updates_list[0]\n",
    "    other_updates = updates_list[1:]\n",
    "    rel_distances_of_others = calc_rel_distances_of_others(reference_update,\n",
    "                                                           other_updates)\n",
    "    print(\"rel_distances_of_others:\", rel_distances_of_others)\n",
    "    \n",
    "    # Each dataset starts with a similarity of 1, which can only decrease over time.\n",
    "    # At every round, each dataset's similarity could decrease by something like:\n",
    "    # some_factor * (1+client_state.epoch_count) * rel_distance\n",
    "    for client_id, rel_distance in enumerate(rel_distances_of_others, start=1):\n",
    "        c = client_states[client_id]\n",
    "        print(c.similarity)\n",
    "        decrement = hyperparams.distance_penalty_factor * (1+0.2*c.epoch_count) * rel_distance\n",
    "        similarity = jnp.max([0, c.similarity - decrement], dtype=np.float32).astype(np.float32)\n",
    "        client_states[client_id] = c._replace(similarity=similarity)\n",
    "    \n",
    "    print([c.similarity for c in client_states])\n",
    "    weights = normalize_array_l1(jnp.array([c.similarity for c in client_states], dtype=np.float32))\n",
    "    weighted_updates = [tree_map(lambda x: w*x, update) for w, update in zip(weights, updates_list)]\n",
    "    aggregated_update = tree_multimap(sum_args, *weighted_updates)\n",
    "    return aggregated_update, client_states\n",
    "\n",
    "def init_average_aggregator():\n",
    "    return Aggregator(aggregator_function    = average_params,\n",
    "                      aggregator_hyperparams = AverageAggregatorHyperParams()\n",
    "                     )\n",
    "\n",
    "def init_similarity_aggregator(distance_penalty_factor: float = 0.05) -> Aggregator:\n",
    "    similarity_hyperparams = SimilarityAggregatorHyperParams(\n",
    "            distance_penalty_factor=distance_penalty_factor\n",
    "    )\n",
    "    return Aggregator(aggregator_function    = similarity_aggregator,\n",
    "                      aggregator_hyperparams = similarity_hyperparams\n",
    "                     )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # initialize random generator, params, opt_state.\n",
    "# num_splits = 4\n",
    "# rng = jax.random.PRNGKey(hyperparams.seed)\n",
    "# rngs = jax.random.split(rng, num_splits+1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "# updates_list = [init(rngs[i], dss[0].as_numpy_iterator().next()) for i in range(num_splits+1)]\n",
    "# client_states = [ClientState() for i in range(num_splits+1)]\n",
    "# client_states = similarity_aggregator(updates_list, client_states, SimilarityAggregatorHyperParams(0.05))[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for i in range(3):\n",
    "#     reference_update = init(rngs[-1], dss[0].as_numpy_iterator().next())\n",
    "#     other_updates = [init(rngs[i], dss[0].as_numpy_iterator().next()) for i in range(num_splits)]\n",
    "#     print(calc_rel_distances_of_others(reference_update, other_updates))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " DeviceArray(1.3672365, dtype=float32), DeviceArray(1.3019154, dtype=float32), DeviceArray(1.5341283, dtype=float32), DeviceArray(1.6079143, dtype=float32)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for i in range(3):\n",
    "#     reference_update = init(rngs[0], dss[0].as_numpy_iterator().next())\n",
    "#     other_updates = [init(rngs[i+1], dss[0].as_numpy_iterator().next()) for i in range(num_splits)]\n",
    "#     print(calc_rel_distances_of_others(reference_update, other_updates))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# One round of training & communication"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "c9V22gJC2DIn"
   },
   "outputs": [],
   "source": [
    "# aggregate diagnositcs.\n",
    "def agg_diagnostics(\n",
    "    client_outputs: Sequence[ClientOutput]) -> DiagnosticsMessage:\n",
    "  d_msgs = extract_from_cout(client_outputs, lambda cout: cout[1])\n",
    "  # TODO: aggregate and report statistics\n",
    "  total_weights = np.sum([msg.weight for msg in d_msgs])\n",
    "\n",
    "  train_loss = np.sum([msg.train_loss for msg in d_msgs]) / total_weights\n",
    "  train_acc =  np.sum([msg.train_acc for msg in d_msgs])  / total_weights\n",
    "  return DiagnosticsMessage(train_loss=train_loss,\n",
    "                            train_acc = train_acc,\n",
    "                            test_loss=0,\n",
    "                            test_acc=1,\n",
    "                            weight=total_weights\n",
    "                           )\n",
    "\n",
    "\n",
    "# update server params.\n",
    "# @jax.partial(jax.jit, static_argnums=[3,5])  # fix optimizer and aggregator.\n",
    "def server_updater(\n",
    "    server_params: hk.Params,\n",
    "    client_outputs: Sequence[ClientOutput],\n",
    "    client_states: Sequence[ClientState],\n",
    "    server_opt: optix.InitUpdate,\n",
    "    opt_state: OptState,\n",
    "    aggregator: Aggregator\n",
    ") -> Tuple[hk.Params, Sequence[ClientState], OptState]:\n",
    "#   print(\"compiling: {}\".format(inspect.currentframe().f_code.co_name))\n",
    "  agg_inputs = extract_from_cout(client_outputs,\n",
    "                                 lambda cout: cout[0].aggregator_input)\n",
    "  agg_update, client_states_out = aggregator.aggregator_function(\n",
    "                                                agg_inputs,\n",
    "                                                client_states,\n",
    "                                                aggregator.aggregator_hyperparams\n",
    "                                                )\n",
    "  eff_grads = tree_map(lambda x: -1.0*x, agg_update)  # effective gradient.\n",
    "  updates, opt_state = server_opt.update(eff_grads, opt_state)\n",
    "  # TODO: allow opt_state to also be explicitly updated by state_updater\n",
    "  server_params = optix.apply_updates(server_params, updates)\n",
    "  return server_params, client_states_out, opt_state\n",
    "\n",
    "# one round of federated learning.\n",
    "def run_one_round(\n",
    "    server_params: hk.Params,\n",
    "    hyperparams: ServerHyperParams,\n",
    "    client_data: Sequence[tf.data.Dataset],\n",
    "    client_states: Sequence[ClientState],\n",
    "    client_opt: optix.InitUpdate,\n",
    "    server_opt: optix.InitUpdate,\n",
    "    opt_state: OptState,\n",
    "    aggregator: Aggregator,\n",
    "    loss: LossFunction,\n",
    "    rng: jax.random.PRNGKey\n",
    ") -> Tuple[hk.Params, Sequence[ClientState], OptState]:\n",
    "  \n",
    "  print(\"computing updates from active clients.\")\n",
    "  # TODO: replace with an function which constructs message\n",
    "  msg_to_clients = ClientMessage(\n",
    "      params=server_params, \n",
    "      opt_init_input=[server_params])\n",
    "  \n",
    "  dss, client_states_upd = make_all_client_data(client_data, client_states, hyperparams)\n",
    "  \n",
    "  client_outputs = list(map(lambda ds: client_updater(\n",
    "                          msg_to_clients,\n",
    "                          ds,\n",
    "                          client_opt,\n",
    "                          loss), dss))\n",
    "  \n",
    "  print(\"aggregating client updates.\")\n",
    "  server_params, client_states_out, opt_state = server_updater(\n",
    "      server_params,\n",
    "      client_outputs,\n",
    "      client_states_upd,\n",
    "      server_opt,      \n",
    "      opt_state,      \n",
    "      aggregator)\n",
    "  \n",
    "  # TODO: aggregate and incporate new diagnostics. needs state!\n",
    "  diagnostics = agg_diagnostics(client_outputs)\n",
    "  \n",
    "  return server_params, client_states_out, opt_state, diagnostics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 383
    },
    "colab_type": "code",
    "id": "BuZcD4dEdzF3",
    "outputId": "08edffde-2229-4389-e8b7-bcd7c00320e2"
   },
   "outputs": [],
   "source": [
    "# # Testing one round of federated averaging.\n",
    "# batch_size=10\n",
    "# dss, ds_test = split_by_age(*load_2017annie_predict_EVD(), batch_size)\n",
    "# client_states = init_client_states(dss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 383
    },
    "colab_type": "code",
    "id": "BuZcD4dEdzF3",
    "outputId": "08edffde-2229-4389-e8b7-bcd7c00320e2"
   },
   "outputs": [],
   "source": [
    "# params = init(jax.random.PRNGKey(42), dss[0].as_numpy_iterator().next())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 383
    },
    "colab_type": "code",
    "id": "BuZcD4dEdzF3",
    "outputId": "08edffde-2229-4389-e8b7-bcd7c00320e2"
   },
   "outputs": [],
   "source": [
    "# client_opt = optix.sgd(0.1)\n",
    "# server_opt = optix.sgd(1.0)\n",
    "# opt_state = server_opt.init(params)\n",
    "# rng = jax.random.PRNGKey(0)\n",
    "# hyperparams = ServerHyperParams(\n",
    "#     num_rounds = 12,\n",
    "#     max_batches_per_round = 5000,\n",
    "#     max_epochs_per_round = 1,\n",
    "#     batch_size = 10,\n",
    "#     seed = 7\n",
    "# )\n",
    "\n",
    "# run_one_round(\n",
    "#     params,\n",
    "#     hyperparams,\n",
    "#     dss,\n",
    "#     client_states,\n",
    "#     client_opt,\n",
    "#     server_opt,\n",
    "#     opt_state,\n",
    "#     average_params,\n",
    "#     loss,\n",
    "#     rng\n",
    "# );"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [],
   "source": [
    "def eval_test_accuracy(params: hk.Params, ds: Sequence[Batch]) -> np.float64:\n",
    "    accs = []\n",
    "    for batch in ds:\n",
    "        accs.append(rmse(params, batch))\n",
    "    return np.mean(accs)\n",
    "\n",
    "def one_hot_encode(labels, n_classes):\n",
    "    \"\"\"\n",
    "    One hot encode a list of sample labels. Return a one-hot encoded vector for each label.\n",
    "    : labels: List of sample labels\n",
    "    : return: Numpy array of one-hot encoded labels\n",
    "     \"\"\"\n",
    "    targets = np.array(labels, dtype=np.int8).reshape(-1)\n",
    "    return np.eye(n_classes)[targets]\n",
    "\n",
    "# Baseline classification accuracy (prevalence of most prevalent label)\n",
    "def eval_baseline_test_accuracy(ds: tf.data.Dataset) -> np.float64:\n",
    "    labels = np.concatenate([batch[\"label\"] for batch in ds], axis=0)\n",
    "    return np.sqrt(np.mean(np.square(labels - labels.mean(axis=0))))\n",
    "\n",
    "def eval_test_loss(params: hk.Params, ds: Sequence[Batch]) -> np.float64:\n",
    "    losses = []\n",
    "    for batch in ds:\n",
    "        losses.append(loss(params, batch))\n",
    "    return np.mean(losses)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Put it all together"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_diagnostics(diag_state: DiagnosticsState,\n",
    "                       diagnostics: DiagnosticsMessage,\n",
    "                       client_states: Sequence[ClientState],\n",
    "                       server_params: hk.Params,\n",
    "                       ds_test: Sequence[Batch]\n",
    "                      ) -> DiagnosticsState:\n",
    "    diag_state_upd = DiagnosticsState(\n",
    "                        train_loss_global     = diag_state.train_loss_global     + [diagnostics.train_loss],\n",
    "                        train_accuracy_global = diag_state.train_accuracy_global + [diagnostics.train_acc],\n",
    "                        test_loss             = diag_state.test_loss     + [eval_test_loss(    server_params, ds_test)],\n",
    "                        test_accuracy         = diag_state.test_accuracy + [eval_test_accuracy(server_params, ds_test)],\n",
    "                        similarities          = diag_state.similarities + [[float(c.similarity) for c in client_states]],\n",
    "                        epoch_counts          = diag_state.epoch_counts + [[int(c.epoch_count) for c in client_states]]\n",
    "                       )\n",
    "    return diag_state_upd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "MsHof54fApTp"
   },
   "outputs": [],
   "source": [
    "def federated_learning(\n",
    "    client_data: Sequence[tf.data.Dataset],\n",
    "     aggregator: Aggregator,\n",
    "        ds_test: Sequence[Batch],\n",
    "    hyperparams: ServerHyperParams,\n",
    "     client_opt: optix.InitUpdate,\n",
    "     server_opt: optix.InitUpdate,\n",
    "           loss: LossFunction,\n",
    "           init\n",
    "    ) -> hk.Params:\n",
    "  # initialize random generator, params, opt_state.\n",
    "  rng = jax.random.PRNGKey(hyperparams.seed)\n",
    "  # TODO: use synthethic data for init\n",
    "  server_params = init(rng, client_data[0].as_numpy_iterator().next())\n",
    "  client_states = init_client_states(client_data)\n",
    "  opt_state = server_opt.init(server_params)\n",
    "  rngs = jax.random.split(rng, hyperparams.num_rounds)\n",
    "  \n",
    "  diag_state = DiagnosticsState()\n",
    "  for round_num, rng in enumerate(rngs):\n",
    "    print(\"\\nrunning round {}\".format(round_num))\n",
    "    server_params, \\\n",
    "    client_states, \\\n",
    "    opt_state,     \\\n",
    "    diagnostics,     = run_one_round(server_params, hyperparams, \n",
    "                                    client_data, client_states, client_opt, \n",
    "                                    server_opt, opt_state, \n",
    "                                    aggregator, loss, rng)\n",
    "    diag_state = update_diagnostics(diag_state, diagnostics, client_states, server_params, ds_test)\n",
    "  \n",
    "  # At each round, the training loss and accuracy are computed based on the\n",
    "  # server_params *before* the round, while the test diagnostics are computed\n",
    "  # from the server_params *after* the round. This is corrected here:\n",
    "  diag_state = diag_state._replace(train_loss_global     = diag_state.train_loss_global[1:],\n",
    "                                   train_accuracy_global = diag_state.train_accuracy_global[1:]\n",
    "                                  )\n",
    "  return server_params, diag_state"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Run a federated learning simulation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 530
    },
    "colab_type": "code",
    "id": "iT-dXeovX2Fd",
    "outputId": "57c040e3-f579-497a-bb96-d30bed2d2004",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "baseline test RMSE: 18.187126997442544\n",
      "\n",
      "running round 0\n",
      "computing updates from active clients.\n",
      "compiling: run_one_step\n",
      "compiling: run_one_step\n",
      "aggregating client updates.\n",
      "compiling: loss\n",
      "compiling: rmse\n",
      "compiling: loss\n",
      "\n",
      "running round 1\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "compiling: average_params\n",
      "\n",
      "running round 2\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 3\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 4\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 5\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 6\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 7\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 8\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 9\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 10\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 11\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 12\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 13\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 14\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 15\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 16\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 17\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 18\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 19\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 20\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 21\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 22\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 23\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 24\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 25\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 26\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 27\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 28\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 29\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 0\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "compiling: tree_norm\n",
      "rel_distances_of_others: []\n",
      "[1.0]\n",
      "\n",
      "running round 1\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "rel_distances_of_others: []\n",
      "[1.0]\n",
      "\n",
      "running round 2\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "rel_distances_of_others: []\n",
      "[1.0]\n",
      "\n",
      "running round 3\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "rel_distances_of_others: []\n",
      "[1.0]\n",
      "\n",
      "running round 4\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "rel_distances_of_others: []\n",
      "[1.0]\n",
      "\n",
      "running round 5\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "rel_distances_of_others: []\n",
      "[1.0]\n",
      "\n",
      "running round 6\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "rel_distances_of_others: []\n",
      "[1.0]\n",
      "\n",
      "running round 7\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "rel_distances_of_others: []\n",
      "[1.0]\n",
      "\n",
      "running round 8\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "rel_distances_of_others: []\n",
      "[1.0]\n",
      "\n",
      "running round 9\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "rel_distances_of_others: []\n",
      "[1.0]\n",
      "\n",
      "running round 10\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "rel_distances_of_others: []\n",
      "[1.0]\n",
      "\n",
      "running round 11\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "rel_distances_of_others: []\n",
      "[1.0]\n",
      "\n",
      "running round 12\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "rel_distances_of_others: []\n",
      "[1.0]\n",
      "\n",
      "running round 13\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "rel_distances_of_others: []\n",
      "[1.0]\n",
      "\n",
      "running round 14\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "rel_distances_of_others: []\n",
      "[1.0]\n",
      "\n",
      "running round 15\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "rel_distances_of_others: []\n",
      "[1.0]\n",
      "\n",
      "running round 16\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "rel_distances_of_others: []\n",
      "[1.0]\n",
      "\n",
      "running round 17\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "rel_distances_of_others: []\n",
      "[1.0]\n",
      "\n",
      "running round 18\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "rel_distances_of_others: []\n",
      "[1.0]\n",
      "\n",
      "running round 19\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "rel_distances_of_others: []\n",
      "[1.0]\n",
      "\n",
      "running round 20\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "rel_distances_of_others: []\n",
      "[1.0]\n",
      "\n",
      "running round 21\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "rel_distances_of_others: []\n",
      "[1.0]\n",
      "\n",
      "running round 22\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "rel_distances_of_others: []\n",
      "[1.0]\n",
      "\n",
      "running round 23\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "rel_distances_of_others: []\n",
      "[1.0]\n",
      "\n",
      "running round 24\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "rel_distances_of_others: []\n",
      "[1.0]\n",
      "\n",
      "running round 25\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "rel_distances_of_others: []\n",
      "[1.0]\n",
      "\n",
      "running round 26\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "rel_distances_of_others: []\n",
      "[1.0]\n",
      "\n",
      "running round 27\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "rel_distances_of_others: []\n",
      "[1.0]\n",
      "\n",
      "running round 28\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "rel_distances_of_others: []\n",
      "[1.0]\n",
      "\n",
      "running round 29\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "rel_distances_of_others: []\n",
      "[1.0]\n",
      "\n",
      "running round 0\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 1\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 2\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 3\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 4\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 5\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 6\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 7\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 8\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 9\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 10\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 11\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 12\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 13\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 14\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 15\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 16\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 17\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 18\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 19\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 20\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 21\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 22\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 23\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 24\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 25\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 26\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 27\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 28\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n",
      "\n",
      "running round 29\n",
      "computing updates from active clients.\n",
      "aggregating client updates.\n"
     ]
    }
   ],
   "source": [
    "# testing federated learning code.\n",
    "\n",
    "hyperparams = ServerHyperParams(\n",
    "    num_rounds = 30,\n",
    "    max_batches_per_round = 300,\n",
    "    max_epochs_per_round = 1,\n",
    "    batch_size = 150,\n",
    "    seed = 42\n",
    ")\n",
    "\n",
    "sim_aggregator = init_similarity_aggregator(0.02)\n",
    "avg_aggregator = init_average_aggregator()\n",
    "\n",
    "constant_inputs = (\n",
    "        hyperparams,\n",
    "        optix.sgd(0.1),\n",
    "        optix.sgd(1.0),\n",
    "        loss,\n",
    "        init\n",
    ")\n",
    "\n",
    "random.seed(hyperparams.seed)\n",
    "client_sets = utils.load(utils.SetLoaders.ANNIE_EVD, utils.Splitters.PREDICT_AGE_SINGLE_CLIENT, hyperparams.seed)\n",
    "diagnostics = {}\n",
    "baseline_test_accuracies = []\n",
    "for requesting_client in range(len(client_sets)):\n",
    "    dss, ds_test = make_dss(client_sets, hyperparams, requesting_client)\n",
    "    jax_ds_test  = make_jax_dataset(ds_test)\n",
    "    \n",
    "    training_types = {\n",
    "        'global' : (dss,      avg_aggregator),\n",
    "        'similar': (dss,      sim_aggregator),\n",
    "        'local'  : (dss[0:1], avg_aggregator),\n",
    "    }\n",
    "    \n",
    "    baseline_test_accuracies.append(eval_baseline_test_accuracy(ds_test))\n",
    "    print(\"baseline test RMSE:\", baseline_test_accuracies[-1])\n",
    "\n",
    "    for key, varied_inputs in training_types.items():\n",
    "        opt_params_global, \\\n",
    "        diag_state          = federated_learning(*varied_inputs,\n",
    "                                                 jax_ds_test,\n",
    "                                                 *constant_inputs\n",
    "                                                );\n",
    "        if not key in diagnostics:\n",
    "            diagnostics[key] = []\n",
    "        diagnostics[key].append( diag_state )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Display results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DiagnosticsState(train_loss_global=[249.33798217773438, 210.3965301513672, 204.32823181152344, 201.55966186523438, 199.4267578125, 197.5773468017578, 195.92642211914062, 194.43798828125, 193.08966064453125, 191.86453247070312, 190.74884033203125, 189.73080444335938, 188.80035400390625, 187.94869995117188, 187.168212890625, 186.4521484375, 185.7945556640625, 185.19015502929688, 184.63418579101562, 184.12246704101562, 183.651123046875, 183.21682739257812, 182.81640625, 182.4471435546875, 182.10638427734375, 181.79190063476562, 181.50155639648438, 181.2333984375, 180.98568725585938], train_accuracy_global=[11.12569808959961, 10.235666275024414, 10.092191696166992, 10.025053024291992, 9.972354888916016, 9.926234245300293, 9.884834289550781, 9.847354888916016, 9.813285827636719, 9.782235145568848, 9.753880500793457, 9.727943420410156, 9.70418643951416, 9.682395935058594, 9.66239070892334, 9.64400577545166, 9.627097129821777, 9.61153507232666, 9.597203254699707, 9.58399772644043, 9.571823120117188, 9.56059455871582, 9.550235748291016, 9.54067611694336, 9.531850814819336, 9.523700714111328, 9.51617431640625, 9.509222030639648, 9.502796173095703], test_loss=[500.40283, 414.51678, 398.6528, 391.3775, 386.01025, 381.43588, 377.3684, 373.7001, 370.37253, 367.34436, 364.58243, 362.0588, 359.74933, 357.63306, 355.69153, 353.9085, 352.26956, 350.7619, 349.37396, 348.09555, 346.9173, 345.8308, 344.8286, 343.9037, 343.04984, 342.26126, 341.5329, 340.8598, 340.23773, 339.66275], test_accuracy=[22.359821, 20.35309, 19.961134, 19.77879, 19.643114, 19.526707, 19.4226, 19.328218, 19.242195, 19.163574, 19.091578, 19.02555, 18.96492, 18.90919, 18.857914, 18.810696, 18.767189, 18.727076, 18.690071, 18.655918, 18.624388, 18.595263, 18.568357, 18.543491, 18.520504, 18.49925, 18.479595, 18.461416, 18.444597, 18.429039], similarities=[[1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0]], epoch_counts=[[1], [2], [3], [4], [5], [6], [7], [8], [9], [10], [11], [12], [13], [14], [15], [16], [17], [18], [19], [20], [21], [22], [23], [24], [25], [26], [27], [28], [29], [30]])"
      ]
     },
     "execution_count": 190,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "diagnostics['similar'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEWCAYAAABxMXBSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3de5xVZdn/8c+XkyAHT6ChgGCZqQyMMmCaCp7APEVqCWGCJqilBv480UNKWmaAWqZPpaV4SskzaYlmoI8F2YyCiD6G5mnEJ1Q8AIoKc/3+WGvGzbBnz97D7JmB+b5fr/2ate/7Xmtdew3Mtde617pvRQRmZmb5atPcAZiZ2abFicPMzArixGFmZgVx4jAzs4I4cZiZWUGcOMzMrCBOHLbZk9RH0ipJbRthWzMl/bgx4jLbVDlx2GZD0iuSPkqTRPVrx4h4LSK6RMS6Iu9/nKQnGmlbr0g6NEf9MElV6WdcKekFSSfXahOS/iOpXUZZO0nLJUVG2Z6SHpb0rqT3JFVIOiLLfjJf+zbG57RNkxOHbW6OTpNE9WtZcwdURMsiogvQDZgEXC9pt1pt3gO+mvH+CODdWm3+CDwC7ABsD5wNfFB7P7Ve8xvzg9imxYnDNnuS+qbfvtul7+dJulTS39Jv6w9L6p7R/k5J/yfpfUmPS9ozj33sDvwa2Df9Rv5eWr6FpBmSXku//f9aUqe0rrukB9Jv+Ssk/Y+kNpJuAfoAf0y3dX6ufUfiT8AKYECt6luAkzLenwTcnBF3d6AfcH1EfJK+/hYRjXLmZJsnJw5rrb4FnEzyDbsDcG5G3Z+BXdO6p4Db6ttYRDwPnA7MT7+Rb51W/Qz4IlAKfAHYCbgorft/QCXQg+Tb/g+STcW3gdf47OxpWq59p8nmGKA78GKt6vuAAyVtLWlr4ADg/oz6d9J1bpU0UtIO9X1WMycO29zcl36Df0/SfTna3RgR/4qIj4A/kPxhByAiboiIlRHxMTAVGChpq0IDkSRgPDApIlZExErgMmBU2uRToCewc0R8GhH/E4UNHrdjembzEXAvcE5EPF2rzRqSS1EnpPudnZZVf9YADgJeAa4A3kzPsnatvZ9ar84FxGmbGScO29yMjIit09fIHO3+L2P5Q6ALgKS2ki6X9JKkD0j+oELybb5QPYAtgYrqP7jAQ2k5wHSSb/sPS/q3pAsL3P6y9MymG3A1cHAd7W4muUS13mWqahFRGRFnRsTngZ2B1bXaLcs4ptWv1QXGapsRJw6z9X0L+BpwKLAV0DctVx7r1j5beJvkbGDPjD+4W6Ud2qRnNf8vInYBjgbOkXRIHduqe6fJmdEFQImkbMnyf0jObHYAcvZdRMTrwLVA/3z3b62PE4fZ+roCH5Nc+9+S5NJSvv4D9JLUASAiqoDrgaskbQ8gaSdJI9LloyR9Ib2k9QGwLn1Vb2uXfHccEZ+QXGq6KEtdkCSmY2pfCpO0jaQfpXG0STvLTwEWFPC5rZVx4jBb383Aq8AbwHMU9gf0r8AS4P8kvZ2WXUByOWpBeunrL0D1LbO7pu9XAfOB/46IeWndT4Ep6SWuzI77XG4A+kg6unZFRCyJiCVZ1vmE5KzqLyTJ61mSxDkuo82OWZ7jOC7PmGwzJE/kZGZmhfAZh5mZFcSJw8zMCuLEYWZmBXHiMDOzgrSrv8mmr3v37tG3b9/mDsPMbJNSUVHxdkT0qF3eKhJH3759KS8vb+4wzMw2KZJezVbuS1VmZlYQJw4zMyuIE4eZmRWkVfRxmNmGPv30UyorK1mzZk39jW2z1rFjR3r16kX79u3zau/EYdZKVVZW0rVrV/r27UsyzqK1RhHBO++8Q2VlJf369ctrHV+qMmul1qxZw3bbbeek0cpJYrvttivozNOJw6wVc9IwKPzfgROHmZkVxInDzJrFK6+8Qv/+xZtocN68eRx11FEAzJ49m8svv7xRtnvZZYXM7bW+mTNnsmzZsqx148aNo1+/fpSWljJw4EAeffTRmrphw4bRp08fMqfBGDlyJF26dAGgqqqKs88+m/79+1NSUsLgwYN5+eWXgeQB6JKSEkpLSyktLeXss89ucPzV3DluZpu9Y445hmOOOaZRtnXZZZfxgx/8oEHrzpw5k/79+7PjjjtmrZ8+fTrHH388c+fOZcKECSxdurSmbuutt+Zvf/sb+++/P++99x5vvvlmTd2sWbNYtmwZzzzzDG3atKGyspLOnTvX1M+dO5fu3bs3KOZsfMZhZs1m7dq1jB07lgEDBnD88cfz4YcfAnDJJZcwePBg+vfvz4QJE2q+aV999dXsscceDBgwgFGjRgGwevVqTjnlFAYPHsxee+3F/fffv8F+Zs6cyZlnngkk3+zPPvts9ttvP3bZZRfuuuuumnbTp09n8ODBDBgwgIsvvniD7Vx44YV89NFHlJaWMmbMGABuvfVWhgwZQmlpKaeddhrr1q1j3bp1jBs3ruYM4KqrruKuu+6ivLycMWPGUFpaykcffVTncdl3331544031isbNWoUd9xxBwD33HMPxx57bE3dm2++Sc+ePWnTJvmT3qtXL7bZZpt6jv5GiIjN/jVo0KAws/U999xzzbr/l19+OYB44oknIiLi5JNPjunTp0dExDvvvFPT7sQTT4zZs2dHRETPnj1jzZo1ERHx7rvvRkTE5MmT45Zbbqkp23XXXWPVqlUxd+7cOPLIIyMi4sYbb4zvfe97ERExduzYOP7442PdunWxZMmS+PznPx8REXPmzInx48dHVVVVrFu3Lo488sh47LHHNoi7c+fONcvPPfdcHHXUUfHJJ59ERMQZZ5wRN910U5SXl8ehhx5a06461qFDh8Y///nPrMdj7Nixceedd0ZExL333hujR4+uqRs6dGgsWLAgSkpKYu3atXHYYYfFyy+/XBPL66+/HjvvvHMMHDgwzjnnnHjqqadq1t15552jf//+MXDgwBg4cGBceeWVWfef7d8DUB5Z/qb6UpWZ5TZsWO76efMavOnevXvzla98BYATTzyRq6++mnPPPZe5c+cybdo0PvzwQ1asWMGee+7J0UcfzYABAxgzZgwjR45k5MiRADz88MPMnj2bGTNmAMltxq+99lrO/Y4cOZI2bdqwxx578J///KdmOw8//DB77bUXAKtWrWLp0qUceOCBdW7n0UcfpaKigsGDBwPw0Ucfsf3223P00Ufz73//m7POOosjjzyS4cOH53U8zjvvPM4//3yWL1/OggXrT3fftm1b9t9/f2bNmsVHH31E5ojfvXr14oUXXuCvf/0rf/3rXznkkEO48847OeSQQ4DGv1TlxGFmzab2baCSWLNmDd/97ncpLy+nd+/eTJ06teYZgwcffJDHH3+c2bNnc+mll7JkyRIigrvvvpvddtttvW1VJ4Rstthii5rlSC+DRQSTJ0/mtNNOyzv+iGDs2LH89Kc/3aBu0aJFzJkzh2uvvZY//OEP3HDDDfVub/r06Rx77LFcffXVjB07loqKivXqR40axde//nWmTp2a9TN99atf5atf/So77LAD9913X03iaGzu4zCz3ObNy/3aCK+99hrz588H4Pbbb2f//fevSRLdu3dn1apVNX0QVVVVvP766xx00EFMmzaN9957j1WrVjFixAh++ctf1iSAp59+ukGxjBgxghtuuIFVq1YB8MYbb7B8+fIN2rVv355PP/0UgEMOOYS77rqrpt2KFSt49dVXefvtt6mqquK4447j0ksv5amnngKga9eurFy5Mmccbdq04fvf/z5VVVXMmTNnvboDDjiAyZMnM3r06PXKn3rqqZq7taqqqnjmmWfYeeedG3AU8uMzDjNrNrvvvjs33XQTp512GrvuuitnnHEGW265JePHj6ekpIS+ffvWXAZat24dJ554Iu+//z4RwaRJk9h666354Q9/yMSJExkwYAARQd++fXnggQcKjmX48OE8//zz7LvvvgB06dKFW2+9le233369dhMmTGDAgAHsvffe3Hbbbfz4xz9m+PDhVFVV0b59e6699lo6derEySefTFVVFUDNGcm4ceM4/fTT6dSpE/Pnz6dTp05ZY5HElClTmDZtGiNGjFiv/Nxzz92g/fLlyxk/fjwff/wxAEOGDKm5GQDgoIMOom3btgAMGDCAm2++ueDjs1581Vm6sUm6ATgKWB4R/dOyUuDXQEdgLfDdiHiyjvW7Ac8D90bEmWnZPKAnUH07wvCI2PArQS1lZWXhiZzM1vf888+z++67N3cY1kJk+/cgqSIiymq3LealqpnA4bXKpgE/iohS4KL0fV0uBR7LUj4mIkrTV71Jw8zMGlfREkdEPA6sqF0MdEuXtwKyPkIpaRCwA/BwseIzM7OGaeo+jonAHEkzSJLWfrUbSGoDXAF8G8h2S8CNktYBdwM/jjqutUmaAEwA6NOnT+NEb2ZmTX5X1RnApIjoDUwCfpelzXeBP0XE61nqxkRECXBA+vp2XTuKiOsioiwiynr06NEIoZuZGTR94hgL3JMu3wkMydJmX+BMSa8AM4CTJF0OEBFvpD9XAr+vY30zMyuipr5UtQwYCswDDgaW1m4QEWOqlyWNA8oi4kJJ7YCtI+JtSe1J7tj6S1MEbWZmnynaGYek24H5wG6SKiV9BxgPXCFpEXAZaR+EpDJJv61nk1uQ9I88AywE3gCuL1b8ZtZ8xo0bt97gg9n07duXt99+O+9tZg50aBunaGccETG6jqpBWdqWA6dmKZ9JclsvEbE627pmVnxVUcXti2/nqgVX8foHr9O7W28mfXkSo0tG00YegKK18W/czHKqiiqOnXUspz1wGhVvVrB89XIq3qzgtAdO47g/HEdVVG3U9i+99FK+9KUvcdhhhzF69OiawQqrPfroo+y1116UlJRwyimn1DwdDcnYTkOGDGHIkCG8+OKLAPzxj39kn332Ya+99uLQQw/NOWaVNYwTh5nldPvi2/nLv//C6k9Xr1e++tPVPPLSI9zx7B0N3nZ5eTl33303Tz/9NPfccw+1R3hYs2YN48aNY9asWSxevJi1a9fyq1/9qqa+W7duPPnkk5x55plMnDgRgP33358FCxbw9NNPM2rUKKZNy/WcsTWEE4eZ5XTVgqs2SBrVVn+6mivnX9ngbT/xxBN87Wtfo1OnTnTt2pWjjz56vfoXXniBfv368cUvfhGAsWPH8vjjj9fUVw/2N3r06JrBEisrKxkxYgQlJSVMnz6dJUuWNDg+y86Jw8xyev2DbI9Ufabyg8oGb7u+sfLqq88clr16+ayzzuLMM89k8eLF/OY3v6kZbdcajxOHmeXUu1vvnPW9uvVq8Lb3339//vjHP7JmzRpWrVrFgw8+uF79l770JV555ZWa/otbbrmFoUOH1tTPmjWr5mf1qLbvv/8+O+20EwA33XRTg2OzujlxmDWhhx6C888Pbl10G11360qHz3eg7Loybl10G+efHzz0UHNHuKFJX55E5/ads9Z1bt+Zc/Y9p8HbHjx4MMcccwwDBw7k2GOPpaysjK222qqmvmPHjtx444184xvfoKSkhDZt2nD66afX1H/88cfss88+/OIXv+Cqq64CYOrUqXzjG9/ggAMOaNRZ7+wzRRtWvSXxsOrWUpx/fjB9umi33zWsfeOspHActHvkl6z9+5mcd14wbZpybqOx5DusevVdVbU7yDu378xhnz+Mu79590bdkrtq1Sq6dOnChx9+yIEHHsh1113H3nvv3eDtWcMUMqy6J3Iya0IDxvyedn97l7V/PxO6/hy2fQnmXMnaBWfSbr9rGHjitsC3mjvM9bRRG+454R7uePYOrpx/JZUfVNKrWy/O2fccRvUftdHPcUyYMIHnnnuONWvWMHbsWCeNTYDPOMyaUNfdurLq41Ww4vOw8qW0dCh0rYRtX2LQDwZRPqFp/q16IifL1FImcjKzWj5emz68tu1L61ek7zfmDiWzpuLEYdaEBlwwAMYBPa8kGe8zHfOz55UwbuPuUDJrKu7jMGtCE/eZxMnffZe1C86ErtcmZxo9r4IFk2jXpj2TfrVtc4doVi8nDrMm9Mxt32Lt36vvqkovV404h3Zt2rP272ey6NZgjEfIsBbOl6rMmtDBB4vzzgtm/mpbBv1gEDucuQODdhzEzF9ty3nnBQcf3DS34rZkp556Ks8991ze7cvLyzn77LMBD53eVHzGYdaEDj8cDj9cwLcYM2D9225b6pnGQw/B3Llw+eWQMcIHEXDhhXDQQcnnaiy//W19U/Osr6ysjLKyDW78ycvatWtp185/BgvlMw4zy2nuXJg2Dc45J0kWkPw855ykfO7chm979erVHHnkkQwcOJD+/fsza9Yshg0bVjNKbpcuXbjgggsYNGgQhx56KE8++STDhg1jl112Yfbs2QDMmzePo446aoNt1zW8+tSpU5kwYQLDhw/npJNOanjwrZgTh5nldPnlMHEi/PznnyWPc85J3k+cmNQ31EMPPcSOO+7IokWLePbZZzm81qnL6tWrGTZsGBUVFXTt2pUpU6bwyCOPcO+993LRRRfl3Hau4dUrKiq4//77+f3vf9/w4Fsxn6OZWU4SXJmOnP7znycvSJLGlVeuf/mqUCUlJZx77rlccMEFHHXUURxwwAHr1Xfo0KEmmZSUlLDFFlvQvn17SkpKeOWVV3Juu7KykhNOOIE333yTTz75hH79+tXUHXPMMXTq1KnhgbdyPuMws3plJo9qG5s0AL74xS9SUVFBSUkJkydP5pJLLlmvvn379jXDpbdp04YtttiiZnnt2rU5t51rePXOnbMP2mj5ceIws3pVX57KlNnn0VDLli1jyy235MQTT+Tcc8/lqaee2rgNZvDw6sXjxGFmOdXu06iq2rDPo6EWL17MkCFDKC0t5Sc/+QlTpkxptLg9vHrxeJBDs1Yq30EOL7gguXsqs08jM5mcfz787GdNELAVlYdVN7NGc9BByc/M5ziq+zw6dPis3loPJw4zyyl5aHHDcslnGq2V+zjMzKwgThxmZlaQoiYOSTdIWi7p2YyyUkkLJC2UVC5pSI71u0l6Q9I1GWWDJC2W9KKkq6WNvZPczMwKUewzjplA7auj04AfRUQpcFH6vi6XAo/VKvsVMAHYNX014vBqZmZWn6Imjoh4HFhRuxjoli5vBSzLtq6kQcAOwMMZZT2BbhExP5L7iG8GRjZ23GbWNLp06dKo25s6dSozZsxo1G3ahprjrqqJwBxJM0gS1361G0hqA1wBfBs4JKNqJyBzUubKtGwDkiaQnJnQp0+fRgncrLUbNmwYkIxIa61Xc3SOnwFMiojewCTgd1nafBf4U0S8Xqs8W39G1icYI+K6iCiLiLIePXpsVMBmVlwRwXnnnUf//v0pKSlh1qxZNXXTpk2jpKSEgQMHcuGFFwJw/fXXM3jwYAYOHMhxxx3Hhx9+2Fyht0rNccYxFvh+unwnkG3Wln2BAyR9F+gCdJC0CvgF0CujXS/quNRlZpuOe+65h4ULF7Jo0SLefvttBg8ezIEHHsjChQu57777+Mc//sGWW27JihXJle9jjz2W8ePHAzBlyhR+97vfcdZZZzXnR2hV6k0ckr4REXfWV1aAZcBQYB5wMLC0doOIGJOxr3FAWURcmL5fKenLwD+Ak4BfNjAOM8tD9eUpgMcee2yDssa4bPXEE08wevRo2rZtyw477MDQoUP55z//yWOPPcbJJ5/MlltuCcC2224LwLPPPsuUKVN47733WLVqFSNGjNjoGCx/+Vyqmpxn2QYk3Q7MB3aTVCnpO8B44ApJi4DLSPshJJVJymfOyDNIzlJeBF4C/pxPLGbWctU1Zl5EkO2O+3HjxnHNNdewePFiLr744vWGTLfiq/OMQ9JXgSOAnSRdnVHVDcg9EH4qIkbXUTUoS9ty4NQs5TNJbuvNbNc/n/2b2cbLPKMoVuf4gQceyG9+8xvGjh3LihUrePzxx5k+fTodOnTgkksu4Vvf+lbNpaptt92WlStX0rNnTz799FNuu+22muHTrWnkulS1DCgHjgEqMspXknRqm5k1iq9//evMnz+fgQMHIolp06bxuc99jsMPP5yFCxdSVlZGhw4dOOKII7jsssu49NJL2Weffdh5550pKSlh5cqVzf0RWpV6h1WX1J4kwfSJiBeaJKpG5mHVzTaU77DqmXw77uarsYdVPxyYAXQA+kkqBS6JiGMaI1gz23Q4YRjk1zk+FRgCvAcQEQuBvsULyczMWrJ8EsfaiHi/6JGYWZNrDTOAWv0K/XeQT+J4VtK3gLaSdpX0S+DvDQnOzFqOjh078s477zh5tHIRwTvvvEPHjh3zXiefPo6zgP8CPgZuB+aQjFprZpuwXr16UVlZyVtvvdXcoVgz69ixI7169aq/YarexBERH5Ikjv+S1BboHBF+2sZsE9e+fXv69evX3GHYJqjeS1WSfp9OqNQZWAK8IOm84odmZmYtUT59HHtExAck8178CehDMty5mZm1QvkkjvbpQ4Ajgfsj4lPqGMrczMw2f/kkjt8ArwCdgccl7Qx8UMygzMys5ao3cUTE1RGxU0QckU7X+hpwUPFDMzOzlqjgiZzS5JHX6LhmZrb5aY6pY83MbBOWz+24W+RTZmZmrUM+Zxzz8ywzM7NWINcMgJ8DdgI6SdoLqJ6/sRuwZRPEZmZmLVCuzvERwDigF3AFnyWOlcAPihuWmZm1VHUmjoi4CbhJ0nERcXcTxmRmZi1YPn0cvdKxqiTpt5KekjS86JGZmVmLlE/iOCUdq2o4sD1wMnB5UaMyM7MWK5/EUd23cQRwY0QsyigzM7NWJp/EUSHpYZLEMUdSV6CquGGZmVlLlc+QI98BSoF/R8SHkrYjuVxlZmatUD5nHAHsAZydvu8M5D85rZmZbVbySRz/DewLjE7frwSuLVpEZmbWouWTOPaJiO8BawAi4l2gQ30rSbpB0nJJz2aUlUpaIGmhpHJJQ7Kst7OkirTNEkmnZ9TNk/RCWrdQ0vZ5fUozM2s0+SSOTyW1JZ31T1IP8uscnwkcXqtsGvCjiCgFLkrf1/YmsF/aZh/gQkk7ZtSPiYjS9LU8jzjMzKwR5ZM4rgbuBbaX9BPgCeCn9a0UEY8DK2oXk4x1BbAVsCzLep9ExMfp2y3yjNHMzJpIvXdVRcRtkiqAQ0ie3xgZEc83cH8TSW7pnUGSEPbL1khSb+BB4AvAeRGRmWBulLQOuBv4cTqxVLZtTAAmAPTp06eB4ZqZWW35zMdxS0T8b0RcGxHXRMTzkm5p4P7OACZFRG9gEvC7bI0i4vWIGECSOMZK2iGtGhMRJcAB6evbde0oIq6LiLKIKOvRo0cDwzUzs9ryuQy0Z+abtL9jUAP3Nxa4J12+E9igczxTeqaxhCRJEBFvpD9XAr+vb30zM2t8dSYOSZMlrQQGSPogfa0ElgP3N3B/y4Ch6fLBwNIs++0lqVO6vA3wFeAFSe0kdU/L2wNHAc/WXt/MzIor17DqPwV+KumnETG50A1Luh0YBnSXVAlcDIwHfiGpHcntvRPStmXA6RFxKrA7cIWkIOlTmRERiyV1JukfaQ+0Bf4CXF9oXGZmtnFUR9/yZqWsrCzKy8ubOwwzs02KpIqIKKtd7ltdzcysIE4cZmZWkLwSh6T9JZ2cLveQ1K+4YZmZWUuVz3McFwMXANUd5O2BW4sZlJmZtVz5nHF8HTgGWA01z1Z0LWZQZmbWcuWTOD5Jh/WoHuSwc3FDMjOzliyfxPEHSb8BtpY0Hj8/YWbWquUzyOEMSYcBHwC7ARdFxCNFj8zMzFqkfOYcJ00UThZmZpbXXVXHSloq6f3q8aokfdAUwZmZWcuTzxnHNODojZiDw8zMNiP5dI7/x0nDzMyq1XnGIenYdLFc0izgPqB6Slci4p6sK5qZ2WYt16WqozOWPwSGZ7wPPpuQyczMWpFc83FUj031lYj4W2adpK8UOzAzM2uZ8unj+GWeZWZm1grk6uPYF9gP6CHpnIyqbiQz8JmZWSuUq4+jA9AlbZM5qOEHwPHFDMrMzFquXH0cjwGPSZoZEa82YUxmZtaC1dvH4aRhZmaZPHWsmZkVxInDzMwKUu9YVZJ6AOOBvpntI+KU4oVlZmYtVT6DHN4P/A/JBE7rihuOmZm1dPkkji0j4oKiR2JmZpuEfPo4HpB0RNEjMTOzTUI+ieP7JMljTTqJU94TOUm6QdJySc9mlJVKWiBpoaRySUOyrLezpIq0zRJJp2fUDZK0WNKLkq6WpHxiMTOzxpHPcxxdI6JNRHRMl7tGRLc8tz8TOLxW2TTgRxFRClyUvq/tTWC/tM0+wIWSdkzrfgVMAHZNX7W3b2ZmRZTXnOOSjgEOTN/Oi4gH8lkvIh6X1Ld2Mcl4VwBbAcuyrPdJxtstSBOcpJ5At4iYn76/GRgJ/DmfeMzMbOPlczvu5cBg4La06PuS9o+ICxu4z4nAHEkzSBLCfnXstzfwIPAF4LyIWCapDKjMaFYJ7NTAOMzMrAHy6eM4AjgsIm6IiBtILg1tTGf5GcCkiOgNTAJ+l61RRLweEQNIEsdYSTsA2fozItv6kiakfSjlb7311kaEa2ZmmfJ9cnzrjOWtNnKfY/ls9sA7gQ06xzNFxDJgCXAAyRlGr4zqXmS51JWud11ElEVEWY8ePTYyZDMzq5ZP4vgp8LSkmZJuAiqAyzZin8uAoenywcDS2g0k9ZLUKV3eBvgK8EJEvAmslPTl9G6qk0geUDQzsyZSbx9HRNwuaR5JP4eACyLi//LZuKTbgWFAd0mVwMUkw5f8QlI7YA3JHVKk/RenR8SpwO7AFZIi3eeMiFicbvYMkru1OpF0irtj3MysCSkiaxfBZqWsrCzKy8ubOwwzs02KpIqIKKtd7tFxzcysIE4cZmZWkDoTh6SDM5b71ao7tphBmZlZy5XrjGNGxvLdteqmFCEWMzPbBORKHKpjOdt7MzNrJXIljqhjOdt7MzNrJXI9x7GLpNkkZxfVy6Tv+9W9mpmZbc5yJY6vZSzPqFVX+72ZmbUSdSaOiHgs872k9kB/4I2IWF7swMzMrGXKdTvuryXtmS5vBSwCbiYZt2p0E8VnZmYtTK7O8QMiYkm6fDLwr4goAQYB5xc9MjMza5FyJUxwSSQAAA12SURBVI7MWfgOA+4DyHeAQzMz2zzlShzvSTpK0l4kw5o/BJCOatupKYIzM7OWJ9ddVacBVwOfAyZmnGkcQjKlq5mZtUK57qr6F8k0sbXL5wBzihmUmZm1XHUmDklX51oxIs5u/HDMzKyly3Wp6nTgWeAPJNO9enwqMzPLmTh6At8ATgDWArOAuyPi3aYIzMzMWqY676qKiHci4tcRcRAwDtgaWCLp200VnJmZtTy5zjgAkLQ3MJrkWY4/AxXFDsrMzFquXJ3jPwKOAp4H7gAmR8TapgrMzMxaplxnHD8E/g0MTF+XSYKkkzwiYkDxwzMzs5YmV+LwnBtmZraBXA8AvpqtXFJbYBSQtd7MzDZvuYZV7yZpsqRrJA1X4iySy1ffbLoQzcysJcl1qeoW4F1gPnAqcB7QAfhaRCxsgtjMzKwFyjnneDr/BpJ+C7wN9ImIlU0SmZmZtUi5hlX/tHohItYBLxeSNCTdIGm5pGczykolLZC0UFK5pCFZ1iuVNF/SEknPSDoho26mpJfT9RdKKs03HjMzaxy5zjgGSvogXRbQKX1ffTtut3q2PRO4hmS62WrTgB9FxJ8lHZG+H1ZrvQ+BkyJiqaQdgQpJcyLivbT+vIi4q74PZmZmxZHrrqq2G7PhiHhcUt/axUB1wtmKZPDE2uv9K2N5maTlQA/gvdptzcys6eW6VFUME4Hpkl4HZgCTczVOL2V1AF7KKP5JegnrKklb5Fh3Qno5rPytt95qjNjNzIymTxxnAJMiojcwCfhdXQ0l9SS5s+vkiKhKiycDXwIGA9sCF9S1fkRcFxFlEVHWo0ePxorfzKzVa+rEMRa4J12+E9igcxySZ0hIpqedEhELqssj4s1IfAzcWNf6ZmZWPE2dOJYBQ9Plg4GltRtI6gDcC9wcEXfWquuZ/hQwkmSiKTMza0L1DqveUJJuJ7ljqrukSuBiYDzwC0ntgDXAhLRtGXB6RJxK8lT6gcB2ksalmxuXPnR4m6QeJHd2LSSZpdDMzJqQIqK5Yyi6srKyKC8vb+4wzMw2KZIqIqKsdnlTX6oyM7NNnBOHmZkVxInDzMwK4sRhZmYFceIwM7OCOHGYmVlBnDjMzKwgThxmZlYQJw4zMyuIE4eZmRXEicPMzArixGFmZgVx4jAzs4I4cZiZWUGcOMzMrCBOHGZmVhAnDjMzK4gTh5mZFcSJw8zMCuLEYWZmBXHiMDOzgjhxmJlZQZw4zMysIE4cZmZWECcOMzMriBOHmZkVxInDzMwKUrTEIekGScslPZtRVippgaSFksolDcmyXqmk+ZKWSHpG0gkZdf0k/UPSUkmzJHUoVvxmZpZdMc84ZgKH1yqbBvwoIkqBi9L3tX0InBQRe6br/1zS1mndz4CrImJX4F3gO8UI3MzM6la0xBERjwMrahcD3dLlrYBlWdb7V0QsTZeXAcuBHpIEHAzclTa9CRhZhNDNzCyHdk28v4nAHEkzSJLWfrkap5eyOgAvAdsB70XE2rS6Etgpx7oTgAkAffr02fjIzcwMaPrO8TOASRHRG5gE/K6uhpJ6ArcAJ0dEFaAszaKu9SPiuogoi4iyHj16bGTYZmZWrakTx1jgnnT5TmCDznEASd2AB4EpEbEgLX4b2FpS9VlSL7Jc6jIzs+Jq6sSxDBiaLh8MLK3dIL1T6l7g5oi4s7o8IgKYCxyfFo0F7i9qtGZmtoFi3o57OzAf2E1SpaTvAOOBKyQtAi4j7YOQVCbpt+mq3wQOBMalt+0ulFSa1l0AnCPpRZI+jzovdZmZWXEo+SK/eSsrK4vy8vLmDsPMbJMiqSIiymqX+8lxMzMriBOHmZkVpKmf4zBr3YYNy10/b15TRGG2UXzGYWZmBfEZh1lT8hmFbQZ8xmFmZgVx4jAzs4I4cZiZWUGcOMzMrCBOHGZmVhAnDjMzK4gTh5mZFcSJw8zMCtIqRseV9BbwanPHkaE7ycRUrZmPQcLHIeHjkGhpx2HniNhgCtVWkThaGknl2YYqbk18DBI+Dgkfh8Smchx8qcrMzArixGFmZgVx4mge1zV3AC2Aj0HCxyHh45DYJI6D+zjMzKwgPuMwM7OCOHGYmVlBnDgaiaRtJT0iaWn6c5s62o1N2yyVNDaj/CeSXpe0qlb7LSTNkvSipH9I6lvcT7JxGuE4DJK0OP28V0tSWj5V0huSFqavI5rqMxVC0uGSXkjjvzBLfZ2/T0mT0/IXJI3Id5stTZGOwSvpv4uFksqb5pNsnIYeB0nbSZoraZWka2qtk/X/R5OLCL8a4QVMAy5Mly8EfpalzbbAv9Of26TL26R1XwZ6AqtqrfNd4Nfp8ihgVnN/1iIfhyeBfQEBfwa+mpZPBc5t7s9Xz2dvC7wE7AJ0ABYBe+Tz+wT2SNtvAfRLt9M2n222pFcxjkFa9wrQvbk/XxMdh87A/sDpwDW11sn6/6OpXz7jaDxfA25Kl28CRmZpMwJ4JCJWRMS7wCPA4QARsSAi3qxnu3cBhzTbt4z8NPg4SOoJdIuI+ZH8L7m5jvVbqiHAixHx74j4BLiD5Hhkquv3+TXgjoj4OCJeBl5Mt5fPNluSYhyDTVGDj0NErI6IJ4A1mY1b0v8PJ47Gs0P1H/705/ZZ2uwEvJ7xvjIty6VmnYhYC7wPbLfR0RbPxhyHndLl2uXVzpT0jKQb6roE1szy+f3W9fvMdUwK/TfTnIpxDAACeFhShaQJRYi7sW3Mcci1zVz/P5pMu+bY6aZK0l+Az2Wp+q98N5GlrL77oRuyTlEV8Tjk+qy/Ai5N318KXAGckuf+mko+v6tCP3u2L3ct+R76YhwDgK9ExDJJ2wOPSPrfiHh8I+Isto05DhuzzSbhxFGAiDi0rjpJ/5HUMyLeTE8pl2dpVgkMy3jfC5hXz24rgd5ApaR2wFbAikLibmxFPA6V6XJm+bJ0n//J2Mf1wAMNjb+Iqn9X1Wriz9Km9u8z17r1bbMlKcoxiIjqn8sl3UtyKaglJ46NOQ65tpn1/0dT86WqxjMbqL47aCxwf5Y2c4DhkrZJL7UMT8vy3e7xwF/T65stVYOPQ3ppa6WkL6fXvE+qXj9NQtW+DjxbrA+wEf4J7Cqpn6QOJB2es2u1qev3ORsYld5p0w/YlaQjNJ9ttiSNfgwkdZbUFUBSZ5J/Ly3x959pY45DVrn+fzS55r77YHN5kVybfBRYmv7cNi0vA36b0e4Ukk6/F4GTM8qnkXyjqEp/Tk3LOwJ3pu2fBHZp7s9a5ONQRvJH4SXgGj4b3eAWYDHwDMl/uJ7N/Vnr+PxHAP9K4/+vtOwS4Jj6fp8kl/peAl4g426ZbNtsya/GPgYkdyYtSl9LNoVj0AjH4RWSs49V6d+DPdLyrP8/mvrlIUfMzKwgvlRlZmYFceIwM7OCOHGYmVlBnDjMzKwgThxmZlYQJw6zIpM0TtKOGe9/K2mPIu5vqqRz0+VLJNX5wGY92yltqaMQW/Pyk+O22UsfllJEVDVTCONI7r2vfvr51KbacURctBGrl5I8N/CnRgrHNhM+47DNkqS+kp6X9N/AU0BvScMlzZf0lKQ7JXVJ2x4u6X8lPZHOcfBAWl7zzT19/2zGnAknSnoynR/iN5Lapq+ZabvFkiZJOp7kj+9tadtOkuZJKku3s0rJXCyLJC2QtENa/vn0/T/Ts4b15mnJiOmkdODHRZJuyVI/M42hei6Hx9KBAudUP42fxvOz9PP8S9IB6dPOlwAnpHGf0Di/GdscOHHY5mw34OaI2AtYDUwBDo2IvYFy4BxJHYHrgaOBA8g+eON6JO0OnEAy8F4psA4YQ/INfaeI6B8RJcCNEXFXuq8xEVEaER/V2lxnYEFEDCQZe2l8Wv4L4BcRMZg6xiOStCfJk9YHp+t/P0fM7YFfAsdHxCDgBuAnGU3aRcQQYCJwcSRDgV9EMkdEaUTMqu+4WOvhS1W2OXs1Ihaky18mmSjob8mVKzoA84EvAS9HxFIASbcC9Q3bfQgwCPhnuq1OJIM5/hHYRdIvgQeBh/OI8RM+G7CxAjgsXd6Xz+Za+D0wI8u6BwN3RcTbABGRa4C83YD+JCPLQjLRUOb8L/dkxNA3j7itFXPisM3Z6oxlkUweNTqzgaRS6h6aei3rn5V3zNjWTRExufYKkgaSTFT1PeCb1D/0+6fx2bg/6yjs/6TIf1htAUsiYt866j9uYAzWCvlSlbUWC4CvSPoCgKQtJX0R+F+gn6TPp+0yE8srwN5p+71JpjOFZPDG45XMDVE9z/rOkroDbSLibuCH1esCK4GuDYj3uHR5VB1tHgW+KWm76jhybO8FoIekfdO27dNLXbk0JG5rBZw4rFWIiLdI7m66XdIzJH+YvxQRa0guTT0o6Qng1YzV7ga2lbQQOINkpFMi4jmS/pKH0209QjJf/E7AvLT9TKD6jGQm8OvqzvE8Q55I0gfzZLrt97N8piUk/RSPSVoEXJnj839CMnT3z9K2C4H96olhLrCHO8etNo+Oa5ZB0jDg3Ig4qpnj2BL4KCJC0ihgdES05LnGrRXxtUyzlmkQcE36DMp7tLxpcq0V8xmHmZkVxH0cZmZWECcOMzMriBOHmZkVxInDzMwK4sRhZmYF+f9S/Y+D+Cv/dgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "markers = {k:v for k,v in zip(diagnostics.keys(), 'ox+svp'*8)}\n",
    "colors = {k:v for k,v in zip(diagnostics.keys(), 'gbkcm'*8)}\n",
    "\n",
    "def final_scatter(accuracies, **kwargs):\n",
    "    plt.scatter(np.arange(len(accuracies)), accuracies, s=60, **kwargs)\n",
    "\n",
    "final_scatter(baseline_test_accuracies, label='baseline test RMSE', marker='_', c='r')\n",
    "\n",
    "for key, val in diagnostics.items():\n",
    "    test_acc = [val[i].test_accuracy[-1] for i in range(len(val))]\n",
    "    x = np.arange(len(val))\n",
    "    final_scatter(test_acc, label=key, marker=markers[key], c=colors[key])\n",
    "    \n",
    "plt.legend()\n",
    "plt.xlabel('requesting client')\n",
    "plt.ylabel('RMSE on the test set')\n",
    "plt.title(\"Final test RMSE\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'global': DiagnosticsState(train_loss_global=array([249.33798218, 210.39653015, 204.32823181, 201.55966187,\n",
       "        199.42675781, 197.5773468 , 195.92642212, 194.43798828,\n",
       "        193.08966064, 191.86453247, 190.74884033, 189.73080444,\n",
       "        188.800354  , 187.94869995, 187.16821289, 186.45214844,\n",
       "        185.79455566, 185.19015503, 184.63418579, 184.12246704,\n",
       "        183.65112305, 183.21682739, 182.81640625, 182.44714355,\n",
       "        182.10638428, 181.79190063, 181.5015564 , 181.23339844,\n",
       "        180.98568726]), train_accuracy_global=array([11.12569809, 10.23566628, 10.0921917 , 10.02505302,  9.97235489,\n",
       "         9.92623425,  9.88483429,  9.84735489,  9.81328583,  9.78223515,\n",
       "         9.7538805 ,  9.72794342,  9.70418644,  9.68239594,  9.66239071,\n",
       "         9.64400578,  9.62709713,  9.61153507,  9.59720325,  9.58399773,\n",
       "         9.57182312,  9.56059456,  9.55023575,  9.54067612,  9.53185081,\n",
       "         9.52370071,  9.51617432,  9.50922203,  9.50279617]), test_loss=array([500.40283, 414.51678, 398.6528 , 391.3775 , 386.01025, 381.43588,\n",
       "        377.3684 , 373.7001 , 370.37253, 367.34436, 364.58243, 362.0588 ,\n",
       "        359.74933, 357.63306, 355.69153, 353.9085 , 352.26956, 350.7619 ,\n",
       "        349.37396, 348.09555, 346.9173 , 345.8308 , 344.8286 , 343.9037 ,\n",
       "        343.04984, 342.26126, 341.5329 , 340.8598 , 340.23773, 339.66275],\n",
       "       dtype=float32), test_accuracy=array([22.359821, 20.35309 , 19.961134, 19.77879 , 19.643114, 19.526707,\n",
       "        19.4226  , 19.328218, 19.242195, 19.163574, 19.091578, 19.02555 ,\n",
       "        18.96492 , 18.90919 , 18.857914, 18.810696, 18.767189, 18.727076,\n",
       "        18.690071, 18.655918, 18.624388, 18.595263, 18.568357, 18.543491,\n",
       "        18.520504, 18.49925 , 18.479595, 18.461416, 18.444597, 18.429039],\n",
       "       dtype=float32), similarities=array([[1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.]]), epoch_counts=array([[ 1.],\n",
       "        [ 2.],\n",
       "        [ 3.],\n",
       "        [ 4.],\n",
       "        [ 5.],\n",
       "        [ 6.],\n",
       "        [ 7.],\n",
       "        [ 8.],\n",
       "        [ 9.],\n",
       "        [10.],\n",
       "        [11.],\n",
       "        [12.],\n",
       "        [13.],\n",
       "        [14.],\n",
       "        [15.],\n",
       "        [16.],\n",
       "        [17.],\n",
       "        [18.],\n",
       "        [19.],\n",
       "        [20.],\n",
       "        [21.],\n",
       "        [22.],\n",
       "        [23.],\n",
       "        [24.],\n",
       "        [25.],\n",
       "        [26.],\n",
       "        [27.],\n",
       "        [28.],\n",
       "        [29.],\n",
       "        [30.]])),\n",
       " 'similar': DiagnosticsState(train_loss_global=array([249.33798218, 210.39653015, 204.32823181, 201.55966187,\n",
       "        199.42675781, 197.5773468 , 195.92642212, 194.43798828,\n",
       "        193.08966064, 191.86453247, 190.74884033, 189.73080444,\n",
       "        188.800354  , 187.94869995, 187.16821289, 186.45214844,\n",
       "        185.79455566, 185.19015503, 184.63418579, 184.12246704,\n",
       "        183.65112305, 183.21682739, 182.81640625, 182.44714355,\n",
       "        182.10638428, 181.79190063, 181.5015564 , 181.23339844,\n",
       "        180.98568726]), train_accuracy_global=array([11.12569809, 10.23566628, 10.0921917 , 10.02505302,  9.97235489,\n",
       "         9.92623425,  9.88483429,  9.84735489,  9.81328583,  9.78223515,\n",
       "         9.7538805 ,  9.72794342,  9.70418644,  9.68239594,  9.66239071,\n",
       "         9.64400578,  9.62709713,  9.61153507,  9.59720325,  9.58399773,\n",
       "         9.57182312,  9.56059456,  9.55023575,  9.54067612,  9.53185081,\n",
       "         9.52370071,  9.51617432,  9.50922203,  9.50279617]), test_loss=array([500.40283, 414.51678, 398.6528 , 391.3775 , 386.01025, 381.43588,\n",
       "        377.3684 , 373.7001 , 370.37253, 367.34436, 364.58243, 362.0588 ,\n",
       "        359.74933, 357.63306, 355.69153, 353.9085 , 352.26956, 350.7619 ,\n",
       "        349.37396, 348.09555, 346.9173 , 345.8308 , 344.8286 , 343.9037 ,\n",
       "        343.04984, 342.26126, 341.5329 , 340.8598 , 340.23773, 339.66275],\n",
       "       dtype=float32), test_accuracy=array([22.359821, 20.35309 , 19.961134, 19.77879 , 19.643114, 19.526707,\n",
       "        19.4226  , 19.328218, 19.242195, 19.163574, 19.091578, 19.02555 ,\n",
       "        18.96492 , 18.90919 , 18.857914, 18.810696, 18.767189, 18.727076,\n",
       "        18.690071, 18.655918, 18.624388, 18.595263, 18.568357, 18.543491,\n",
       "        18.520504, 18.49925 , 18.479595, 18.461416, 18.444597, 18.429039],\n",
       "       dtype=float32), similarities=array([[1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.]]), epoch_counts=array([[ 1.],\n",
       "        [ 2.],\n",
       "        [ 3.],\n",
       "        [ 4.],\n",
       "        [ 5.],\n",
       "        [ 6.],\n",
       "        [ 7.],\n",
       "        [ 8.],\n",
       "        [ 9.],\n",
       "        [10.],\n",
       "        [11.],\n",
       "        [12.],\n",
       "        [13.],\n",
       "        [14.],\n",
       "        [15.],\n",
       "        [16.],\n",
       "        [17.],\n",
       "        [18.],\n",
       "        [19.],\n",
       "        [20.],\n",
       "        [21.],\n",
       "        [22.],\n",
       "        [23.],\n",
       "        [24.],\n",
       "        [25.],\n",
       "        [26.],\n",
       "        [27.],\n",
       "        [28.],\n",
       "        [29.],\n",
       "        [30.]])),\n",
       " 'local': DiagnosticsState(train_loss_global=array([249.33798218, 210.39653015, 204.32823181, 201.55966187,\n",
       "        199.42675781, 197.5773468 , 195.92642212, 194.43798828,\n",
       "        193.08966064, 191.86453247, 190.74884033, 189.73080444,\n",
       "        188.800354  , 187.94869995, 187.16821289, 186.45214844,\n",
       "        185.79455566, 185.19015503, 184.63418579, 184.12246704,\n",
       "        183.65112305, 183.21682739, 182.81640625, 182.44714355,\n",
       "        182.10638428, 181.79190063, 181.5015564 , 181.23339844,\n",
       "        180.98568726]), train_accuracy_global=array([11.12569809, 10.23566628, 10.0921917 , 10.02505302,  9.97235489,\n",
       "         9.92623425,  9.88483429,  9.84735489,  9.81328583,  9.78223515,\n",
       "         9.7538805 ,  9.72794342,  9.70418644,  9.68239594,  9.66239071,\n",
       "         9.64400578,  9.62709713,  9.61153507,  9.59720325,  9.58399773,\n",
       "         9.57182312,  9.56059456,  9.55023575,  9.54067612,  9.53185081,\n",
       "         9.52370071,  9.51617432,  9.50922203,  9.50279617]), test_loss=array([500.40283, 414.51678, 398.6528 , 391.3775 , 386.01025, 381.43588,\n",
       "        377.3684 , 373.7001 , 370.37253, 367.34436, 364.58243, 362.0588 ,\n",
       "        359.74933, 357.63306, 355.69153, 353.9085 , 352.26956, 350.7619 ,\n",
       "        349.37396, 348.09555, 346.9173 , 345.8308 , 344.8286 , 343.9037 ,\n",
       "        343.04984, 342.26126, 341.5329 , 340.8598 , 340.23773, 339.66275],\n",
       "       dtype=float32), test_accuracy=array([22.359821, 20.35309 , 19.961134, 19.77879 , 19.643114, 19.526707,\n",
       "        19.4226  , 19.328218, 19.242195, 19.163574, 19.091578, 19.02555 ,\n",
       "        18.96492 , 18.90919 , 18.857914, 18.810696, 18.767189, 18.727076,\n",
       "        18.690071, 18.655918, 18.624388, 18.595263, 18.568357, 18.543491,\n",
       "        18.520504, 18.49925 , 18.479595, 18.461416, 18.444597, 18.429039],\n",
       "       dtype=float32), similarities=array([[1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.]]), epoch_counts=array([[ 1.],\n",
       "        [ 2.],\n",
       "        [ 3.],\n",
       "        [ 4.],\n",
       "        [ 5.],\n",
       "        [ 6.],\n",
       "        [ 7.],\n",
       "        [ 8.],\n",
       "        [ 9.],\n",
       "        [10.],\n",
       "        [11.],\n",
       "        [12.],\n",
       "        [13.],\n",
       "        [14.],\n",
       "        [15.],\n",
       "        [16.],\n",
       "        [17.],\n",
       "        [18.],\n",
       "        [19.],\n",
       "        [20.],\n",
       "        [21.],\n",
       "        [22.],\n",
       "        [23.],\n",
       "        [24.],\n",
       "        [25.],\n",
       "        [26.],\n",
       "        [27.],\n",
       "        [28.],\n",
       "        [29.],\n",
       "        [30.]]))}"
      ]
     },
     "execution_count": 192,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def average_diag_states(dstates: Sequence[DiagnosticsState]) -> DiagnosticsState:\n",
    "    args = [np.mean([ d[i] for d in dstates], axis=0)\n",
    "            for i in range(len(dstates[0]))]\n",
    "    return DiagnosticsState(*args)\n",
    "    \n",
    "avg_diagnostics = {}\n",
    "for key, dlist in diagnostics.items():\n",
    "    avg_diagnostics[key] = average_diag_states(dlist)\n",
    "avg_diagnostics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot the evolution of test accuracy and client similarities during training\n",
    "def plot_test_accuracy_and_similarities(diags: Mapping[str, DiagnosticsState], baseline, title=''):\n",
    "    fig, ax1 = plt.subplots(figsize=(7,5))\n",
    "    ax1.set_xlabel(\"Communication round\")\n",
    "    ax2 = ax1.twinx()\n",
    "\n",
    "    h = ax1.axhline(baseline, color='r', linestyle='dashed')\n",
    "#     ax1.axhline(1-baseline, color='r', linestyle='dashed')\n",
    "    \n",
    "    handles_methods, labels_methods, handles_sim = [h], ['baseline RMSE'], []\n",
    "    for key, dstate in diags.items():\n",
    "        m = markers[key]\n",
    "        c = colors[key]\n",
    "        h, = ax1.plot(dstate.test_accuracy, '-'+m+c)\n",
    "        handles_methods.append(h)\n",
    "        labels_methods.append(key)\n",
    "        if key not in ('global', 'local'):\n",
    "            handles_sim = ax1.plot(dstate.similarities, ':')\n",
    "            num_clients = np.array(dstate.similarities).shape[1]\n",
    "            ax2.plot(dstate.epoch_counts, ':')\n",
    "    ax1.set_ylabel(\"Similarity / Test RMSE\")\n",
    "#     ax1.set_ylim((-0.02,1.02))\n",
    "    ax2.set_ylabel(\"Epoch count\")\n",
    "    handles = handles_methods + handles_sim\n",
    "    labels = labels_methods + ['client {}'.format(i) for i in range(num_clients)]\n",
    "    ax1.legend(handles, labels)\n",
    "    plt.title(title)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAdgAAAFNCAYAAACjcn5pAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOydd3hUVdrAf++kN4IQeklAREACQY2IggRFQUXEVVBEBVFx9VNkLagsKhYUUVd0LSuggoooqKuCZQGliIIUpYqISEJvAUJ6mZzvj3sTJiGZTEImk/L+nuc+c++p7z33zH3ve6oYY1AURVEUpXJx+FoARVEURamNqIJVFEVRFC+gClZRFEVRvIAqWEVRFEXxAqpgFUVRFMULqIJVFEVRFC+gCrYGIiIjRGS5r+XwBBFZIiK3+1oOXyIi34jI8CrOs7DcRWSYiCyoyvxLQ0Rmi8ggD8Ilikhf+3yciEz3vnSVg4gYEWlXwbi9RGRrZctkp13ueigiMfb9+FeyLANF5KPKTLM6UicUrP2yOSoiQb6WpTYjIhNE5ANfy+FLSioDY8zlxpiZvpLJGDPLGHNZZaR1isqjC9AV+KI88YwxzxpjTvkjzVvKojIxxvxgjDnTS2n7pB6WVO7GmC+BznadqLXUegUrIjFAL8AAA72UR7X9w3oLX9+zr/NXKsSdwCxTS1a38YJVV9fq9GxglK+F8Ca1XsECtwArgRlAYfOIiJwvIvtFxM/F7RoR2WCfO0TkERHZLiLJIjJHRBrYfgVfZLeJyE7ge9t9rp1miogsE5GzXNJuKCLzROS4iKwWkWdcm3lFpIOILBSRIyKyVUSGFIv7pR13FXC6uxu2m182i8gx23rvaLs/IiKfFAv7ioi8ap9HisjbIrJPRPbYMvrZfiNE5EcReVlEjgATiqXTHxgHXC8iaSKy3sU72o6bKiILRCSq2HP4yZZ1vYgkuLmvRBF52H5G6SLiLyLNReRTETkkIjtEZLRL+BARmWG3XvwmIg+JyG4X/yLWmB32GZfrASKyzpbtJ9evbVuOPfY9bRWRS0orAynaXDtCRJaLyIu2XDtE5HKXdNvYdSdVRBaJyOviplVARK62ZTxu19X+JYQp0qVQRl2bYef5lS3DzyJyuu23zA623r6/60UkSkTm22V0RER+EJHS3iuXA0uLyXaHiGyx8/pNRM4uQf4irQLu6oxd1k+XUt8K5D9my9+jhLyCRGSKiOy1jylit3yJSIKI7Laf/X7gXdv9Ifs/s1dERpaQ3osislNEDojIf0QkpLT0Ctxc4ieKyIMiskGs98rHIhLs4j/WJe/bi9fpYrK41sN2IrLUTvOwiHxcUpwS0mgu1rvoiIj8KSJ3uPidJyJr7Lp4QET+VUa5LwGu9CTfGosxplYfwJ/A3cA5QC7QxMVvO3Cpy/Vc4BH7fAyWYm4JBAFvAbNtvxgsi/g9IAwIsd1HAhF2+CnAOpe0P7KPUKATsAtYbvuF2de3Av7A2cBh4CyXuHPscJ2BPQVxS7jf9kA6cCkQAIy1yyAQiAYygHp2WD9gH3C+ff25fZ9hQGNgFXCn7TcCyAPutWUMKSHvCcAHxdyW2OXcHgixryfZfi2AZOAKrI+9S+3rRqXcWyKwDmhlp+UA1gKP2/fXFvgL6GeHnwT8ADSw42wCdrukZ4B2LtczgGfs87OBg0B3u5yG2/kHAWfaz6u5S304vYwyuN2lHHOBO+x07wL2AmL7rwBetO+nJ3C8eHou6Z4HpNjl5rDLs0MpeXpa12YAR+y0/YFZwEduyuw54D9YdS0Aq7VISpA1zI7byMVtMFZdjgcEaAdEuzzrvsXLlDLqDO7rW4wtg7+b98VTWP/7xkAj4CfgadsvAes/8LxdD0KA/sABrP9lGPChaxlhvQe+xKqDEcA84Dk36SVQtI4mYv0Pm9tpbAH+bvv1B/YDZ2G9V94v/nzc1MPZwD/tMgwGepYSp0iZYX0gvWHHiQMOAZe41N2b7fNwTrxXSix3+34M9vuoNh4+F8CrN2e9oHKBKPv6d+AfLv7PAO/Y5xFYiinavt5SUHHs62Z2Wv4uFaatm7zr22EisV6kucCZxfIueOldD/xQLP5bwBMucTu4+D1L6Qr2MWCOy7UD6yWWYF8vB26xzy8FttvnTYBsXBQnMBRYbJ+PAHaWUd4TKFm5jHe5vhv41j5/GHi/WPj/AcNLST8RGOly3b24TMCjwLv2+V9Afxe/UXiuYN/EfrG6+G8FemMpgoNAXyDAwzJwVXZ/uviF2nI0BVpjvXBDXfw/KJ5esTrycil+xfMss665lMF0F78rgN/dlNlTWH2qJb7UXcK1sOMGF3vW97l51iUpWLd1poz6FkPZCnY7cIXLdT8g0T5PAHKK3cM72Arcvm5fUEZYHw3p2B9ftn8PYIeb9BI4WcHe5HI9GfiPS97Pufi1K/583NSJ94CpQMsynlthmWF9pDqBCBf/54AZ9vky4Ens921JaRRzD7DdW7uToSYftb2JeDiwwBhz2L7+EJdmYvv6b3YT0N+AX4wxSbZfNPBfuxnqGJbCdWIpogJ2FZyIiJ+ITLKb6Y5j/TEAorC+hP1dwxc7jwa6F+Rl5zcM66VbUtwkSqe5q78xJt+O28Llnofa5zfa1wUyBAD7XGR4C+tLviSZy8N+l/MMrK/bgjwHF7vvnlgfM6VRvNyaF4s/jhPPqDmel1txooEHiqXdCstq/ROrhWMCcFBEPhKR5uVIu7A8jDEZ9mm4Le8RFzdwX+atsBRCeXBX106Sj6LPqyRewGohWSAif4nII6WEO2b/Rri4VVT+supMeeQvTpH/j33u+mwPGWOyioUvrY41wvqAWusi67e2e2nplURp91M87/L8P8difQCsEqs7aWRZEThRP1Nd3JI48W65DesD43exusEGlJFeQV045jZUDabWdqrb/RxDAD+7fwOsZpj6ItLVGLPeGPObiCRh9Q25KhuwKutIY8yPJaQdY58aF+cbgauxrJpELMv1KFYlPoRlmbQE/rDDtyqW11JjzKUl5OVnx22FZYGDZemUxl4g1iW+2HH32E5zgZdEpCVwDdYXdYEM2Vhfn3mlpG1KcffUvzi7sKyRO8oMWXIeu7CsgTNKCbsP694329fFyy0D6wVYQFOgoP9rFzDRGDOxRCGM+RD4UETqYX2IPA/cTPnLoLi8DUQk1EXJtnITfhdl9MeXEqfEulYR7JftA1gfI2cBi0VktTHmu2Lh0kWkoOn2kIssFZG/vHWmUAwPwuzFUuKudWavmzQK6lgBrnXsMJCJ1fy+h5I51frS0uXaXV0pmqkx+7G6KRCRnsAiEVlmfzyWxl6s+hnhomRbY79bjDHbgKFi9cH/DfhERBpS+j12xGodOO6p3DWN2mzBDsKyODth9RXEYT3QH7AGPhXwITAauAhL+RTwH2CiiEQDiEgjEbnaTX4RWAoqGeul/WyBhzHGCXwGTBCRUBHpUEyG+UB7EblZRALsI15EOpYQtxNFrfDizAGuFGvQTQDWyy8bqy8JY8whrKaid7GU0xbbfR+wAEv51hNrkNfpItLbTV7FOQDESOmDXIrzAXCViPSzWwCC7UEeLcuMabEKOG4PEgmx0+gsIvG2/xzgURE5zU7z3mLx1wE32vH6YzX/FjAN+LuIdBeLMBG5UkQiRORMEbnYbvnIwnqJOitYBoXYrSdrsJ51oD0Y5Co3Ud4GbrWftUNEWth1yx2l1jUPxTyA1dcNFA4Ea2d/yB3HKgdnKXG/pmgZTwceFJFz7DJuV/B/c8Op1JlDQL6r/CUwGxhv/9+jsPr33U09mwOMEJFOIhKK1a0DFLYeTQNeFpHGAPYz6ueBrJ4wB+v5d7TzftzTiCIy2KXMjmIpwdKeGwDGmF1Y75Hn7HLvgmW1zrLTvElEGtn3XWCVOim93HsD33gqc02kNivY4Vh9cTuNMfsLDuA1YJicGBI/G6vf43uXpmSAV7AGJywQkVSsgQ/d3eT3HlZzyR7gNzu8K/dgWbX7sQYjzMZSfAVWwGXADVhfifs5MfChIG647T4De/RiSRhjtgI3Af/G+oK+CrjKGJPjEuxDLEv7w2LRb8EaXPMb1p/uE9w31xan4AMlWUR+KSuw/Ye9GqtZ9xCWdfIQHtZL++PjKqyPpx1Y9zsdq5zB6g9Ksv0WYJW7K/fZ8QuaST93SXsN1hf+a1hl8SdWXyZYz2WSnd9+rGb0cbZfucqgBIZhtSokY/XTf4xdT4pjjFmFNVjpZazBTkuxrK9S8aCulcUEYKbd5DkEOANYBKRhDXJ5wxizpJS4U7H+e2LLMheYiFUPU7HKv0EZ8le4ztitAhOBH235zy8h2DNYHzkbgI3AL7ZbaWl+gzWQ6XusOvJ9sSAP2+4rxeo6WoQ1SO6UsfN+FVhs57HC9iqxvhQjHvhZRNKw3nP3GWN2eBBvKFaf6l7gv1h99wttv/7AZjvNV4AbjDFZbsp9KFbrT62lYOSiUsWIyPNAU2OMO2tUqUTEms7xgTHGUwvZ54g1feJ3Y8wTZQauAYjIh1iD8D4vM7BSLuxWiE1AkJtunmqBiFyFNeJ4SJmBazCqYKsIu+kuEOurOB6ruex2fdFUHTVBwdrN20ewrO7LsKy6HsaYX30qmFItEZFrgK+wpgjNBPKNMWUuRalUDbW5ibi6EYHVl5qO1XfyEuVcMk6pEzTF6iNPw2r+u0uVq+KGO7Gayrdj9Xfe5VtxFFfUglUURVEUL6AWrKIoiqJ4AVWwiqIoiuIFasRCEw6Hw4SEhPhaDEVRFKUakZGRYYwx1dZQrBEKNiQkhPT0dF+LoSiKolQjRCTT1zK4o9pqfkVRFEWpyaiCVRRFURQvoApWURRFUbyAKlhFURRF8QKqYBVFURTFC6iCVRRFURQvoApWURRFUbxAjZgHeypMngwpDRcxK/V2kj5PInpQNMMiphOZ3JexY30tnaIoiuINRCQYWIa117E/8Ikx5gkRaQN8hLX38C9Y2+bllJ5Sxan1FmxKw0U8O7orSevawFJIWteGZ0d3JaXhIl+LpiiKoniPbOBiY0xXIA7ob2/2/jzwsjHmDOAocJu3BKj1CnZW6u1wzfXwwbWWw9w5MHiI5a4oiqLUSoxFmn0ZYB8GuBj4xHafCXht/9wasV1dWFiYqehSiZIgsLQEj95gllT/e1cURakNHM/KJSLIHxGptDRFJAfY6OI01Rgz1cXfD1gLtANeB14AVhpj2tn+rYBvjDGdK00oF2q9BRs9KBqGJ4DcZzkELofhCZa7oiiK4nV2Hcmg70tLmb1qV2UnnWeMOdflmOrqaYxxGmPigJbAeUDHEtLwmqVV6xXssIjpVrPw1VstB79HYO4cy11RFEXxOi3qh3DZWU3o0jLSJ/kbY44BS4DzgfoiUjDAtyWw11v51noFG5ncl3Gvrie69xaoHw2Zyzl/wHtEJvf1tWiKoii1lk17Urj57Z9JyczF4RCeGRRL5xZVp2BFpJGI1LfPQ4C+wBZgMXCdHWw48IXXZKjtfbCu/JG4lzPbdMU/NIas4z/j51frvy8URVF8wvpdx/i/D3/hrZvP4azm3lGsIpJhjAkrxa8L1iAmPyxjco4x5ikRacuJaTq/AjcZY7K9Il9dUrAAXRLuZePS1xj54Gu8/cL/VUqaiqIoCiQeTmd14hEGn9sKgFxnPgFeNGTcKdjqQJ1TsIeOHKdxo044/OuTk7FBrVhFUZRK4uFPNrBoywGWPJRARHCA1/Or7gq2zmmXRg3q0XPAzeTnbOZvt0/ytTiKoig1mkOp2Rw4ngXAuCs7Mn90zypRrjWBOmfBAqRlZBFRz5r2lHp8E+GhwZWWtqIoSl0h15nPxS8t4YzGEbwzIr7K81cLthoSHhrMwJtHgnM7lw990tfiKIqi1Ciy85wABPg5GH9lJx69vIOPJaqe1EkLFsDpzCcwtAv5ecc4eOg3GjWoV6npK4qi1EYSD6dz8zs/8+TAs7i4QxOfyqIWbDXFz8/BiNF3Qf4eLvnbP30tjqIoSo2gWf1gOjatR2RIoK9FqfbUWQsWLCs2uF538jIS2bpjPe1jmld6HoqiKDWdPw6k8uaS7Uy6NpYgfz9fi1OIWrDVGD8/Bw9OGAMcpv+1430tjqIoSrVk15EMfth2iB2HK9/Qqc3UaQu2gODTepN9bB0/r1/LeV3aeS0fRVGUmsLeY5ls3Z9Knw6NAUjPziMsyL+MWFWLWrA1gEn/GgukMnCwWrGKoigAT87bzNhPN5CVa40Yrm7KtSagFqxNRNP+pB1YyldLfuSK3md7NS9FUZTqyLGMHBwOoV5wAPtSMsnJyye6YbU1ENWCrSlMf/sxwMmwmyb4WhRFUZQqJzPHyZWvLuepeb8B0CwypFor15qAKlib66+8kAbRV3Js99fM+GSxr8VRFEWpEvLzrVbMkEA/7ko4neE9YnwrUC1Cm4hd+H7lJi7p0Z3QqAtIP7TQ6/kpiqL4kr8OpXH3rF+YfF0XurSs72txyo02EdcgLj6/My06DSLj8CKe/8/nvhZHURTFqzQMCyI4wI/MHKevRamVqAVbjH88tI8pL3aCsDPg3NVED4pmWMR0IpP7MnZslYigKIriNZKS0/nw55083L8DDodgjEFEfC1WhVALtoYR2mEzOO6H9NWwFJLWteHZ0V1JabjI16IpiqKcMj/+mczsVTtJTLaMlpqqXGsCasEWI2ZKDEkrzoA5m4F9INdCF6HljWvYNXZHlcigKIpSmRxKzWbPsUziWtXHGMPhtBwaRQT5WqxTprpbsF5TsCLSCngPaArkA1ONMa+ISAPgYyAGSASGGGOOukurKhWsJAgsLcknlMbtrmTswzfxwO0Dq0QWRVGUyuDGaSvZdTSDxQ8k4O9Xexou67KCbQY0M8b8IiIRwFpgEDACOGKMmSQijwCnGWMedpdWlVuw69rA3DmQ0RhCdkPbMZC0C9LWAnlIQEfO6nExr794NysXdyKl4SJmpd5O0udJ2merKEq1IC07j0A/B4H+Dv44kIoAZzSJ8LVYlUp1V7Be+5Qxxuwzxvxin6cCW4AWwNXATDvYTCylW20YFjHdUq6Dh1gOQ26CHW8wbsozrPh1M137jEbEwaZlr9P7vC48OqE3z/59L0mr2mmfraIo1YKUzFyueOUHXl70BwDtm0TUOuVaE6iStgIRiQG6AT8DTYwx+8BSwkDjqpDBUyKT+zLu1fVEx+2A3hAdt4Nxr64nMrkv58e1Z933r+DM3sSrM76m2ZmDyc/aCnnDYfYvVgLv94cOdzH99zE4nfknpT95Mvzz7UXETIlBEoSYKTH88+1FTJ5cxTeqKEqtJTIkgEFxzbmkQ7V6vdY5vD7ISUTCsXo1JxpjPhORY8aY+i7+R40xp5UQbxQwCiAwMPCc7Oxsr8pZUeQigR9K822If2gbIhu3om2HtlzUM5YUCWf6xIssC3nmEhieAHPnMO7V9Uy8rW/VCa4oSq3ir0NpPPLZRl4a3JVWDUJ9LU6VUN2biL26PYKIBACfArOMMZ/ZzgdEpJkxZp/dT3uwpLjGmKnAVLD6YL0p56kQ/bdoktq69NkG/wZnPUZA1jFCj4WQengXyYmLSE5MZfW3diRpDjM7W+cfXAWd7+bNFTu49+pFNI06eTWVyZPRfl5FUdwS6O/g4PEs9hzLrDMKtrrjzUFOgtXHesQYM8bF/QUg2WWQUwNjjFs1UZWDnMrLP99exLOju7q1SJ3OfD5fuIpPv1rB7FmvwNGk0hOUJjgCmxJcrzENGjehdZvmZBDJuoWXw3UPwKzFZVq9qpAVpW6wLyWTrzbs4/ZebQFw5hv8HHVnXmt1t2C9qWB7YjWebsSapgMwDqsfdg7QGtgJDDbGHHGXVnVWsOVVZieNUg7eCHETCDJHaZ4bTfK+A2SkHCQvaz/k7+VE0QH4AU6gO4QG46jvoGPbDkTHNOOsTjH0OPdMLr2wC8/NXl6m0j+Ve1AUpXrwyqJtvLVsOwvv702L+iG+FqfKqe4KFmNMtT9CQ0NNbWHc9IWG0IOG4QkGsH5DD5px0xeeFPbY8XTz0fzlhrZNrLAnHf4luDkM0sTgH2tgkOXmeMLQ7joT2u8i8+K0L8zyNVtMdk5uhWQyxpjnn7fiRL8cbeiNiX452oybvtA8/7zXik1RFJtj6Tlm+8FUY4wxOXlOszM53ccS+Q4g3VQDHVXaoSs5VTEVsRZPsnpDD8LgIbTssp15/Zfx/cqN/Lr+L/78czd7dx9k5x+LIWtXGZIEgDTCEdiQ/IBQ8DsNUs+B/Ing/yGc9wkN47bzy/hFtG4WVSSmJ83ilXHfiqIUxRjDNW/8RFauk69H98JRh5qDS6K6W7CqYGsA5VVoJzdDJ0Kve4kIT+aiiCvYtfMAhw8kk3r0CFlpx8hN3wpuF9MKB0dDHIGnERBcj5wAwUh9SL4InA9AwHzoOY3G520jccKvhAQHnvI9qEJWlBNk5ToJ8ncgIvz8VzKhgf7Etoz0tVg+p7orWK+OIlYqB2tu7iJmpe4gKdGamzuslzU3tySGRUzn2bkFygy4fgTMncO9bhVyV5jzAWS2hMDvoOObBPofJdqvLceSj5J+/Bg56SnkpO3D5O0GMoEvrARyB8BiOLgYQp8PBhqAX338AiMJCK5HcHg9Uh3pEFUfPrgZWAKzH4c+I3l311aecv6OX7Hl21IaFijkNrA0iaSYNjw7tyvjXl0EqEJW6g6H07K5YepKbukRzS09YujetqGvRVI8RBVsDcBSEH2ZSCKMKSMwp6CQh9gKeejTMHcOD7pTyGvawqevQdZZEPBfaDuLgICjNJXmpB1LITPtOLlZx8k6vousoynAYTv2p9ZPzsXwP9j3P/B/MhjkNMQvEkdABP5BEeT4GwiNhPcvAZbArNsg7i7eWLqL7tGRxHaIpnWzqELFrApZqa00DAvk7Nb1aRNVbQ01pRS0iVgpt7KpSB9sqxdj2P1jR/j6cci5APxnQfR/8fc/SiNpTEbqcbLS08jLTiU/NxWTtwfIKEPyAJBIcESCXwhIPciJBjMbHA9D820ENknh8q59adrkNFo2j6JtdBPObNOcj5b9wYsPnePVfmRV4kpF2ZmcwbNfb+G5v8VyWtjJXS6KhTYRK9Ueb1vIALecNp1nF3SFobaVPGwazJ3DWHdW8i8xMHcGZLWBwCUQ+2+Cw5KJCz+Xo0eOk5qSRkZqKtkZ6WSm/AbOTcBPVgL5z8NuyNkNX6z9rhSpImCmvbDHzBwIuprnHhDeePLfhISGEhoeRkRkGKedFkGDhpHs2O/kl297QEIXy0pu3o5nvyjdSva2Va0KvPZyPCuX1YlH+ONAqjYJ12DUglWqBG9bySfvgrQfrriFqGYHeCj2SXbtPsS+A0c5dOgYR48cZ+O6eZBS0oIfkSABYFIBT5bnDAAiwBGGOEJx+IfgFxCKX2AwWZKHyY+AY10g/znwmwLtlhPS/BC3nD+UJo3q07RxfVo0aUBMq8a8/92WclnVVTGaW5V41XE4LZsf/zzM1XEtAMjMcRIS6Odjqao37ixYN1umTgDuAA7ZQccZY772ioC+nifkyVGb5sEqnlHeubblncsb/XJ0YRigMG70y9GFYQ4mp5jla7aYmZ8tMU+9Orf0+cihrU14k34mqP5Fxj803jgCOxv82hoIK2X+srsj2ECUfX6Wwb+LIbibCYzsaUIb9TWRLa40UW2vNS3Pusn4n3GJofW1Br/XrfD+HxnOudbUG9zDvPT2l2bWlz+Y71ZsNFt37DEZmdkVKidvz5HWOdUneOKLTabD+G/M4dQsX4tSY8DNPFigGXC2fR4B/AF0AiYAD5YWrzIPtWCVWkFV9COXNh85Om4HiWMSSw8/5yPIbArBm6DnOCIbH2D4GaM4lJzC0aOppBxLJzU1nU0b5sOxEqxqR2PEryEmPxPyM8FkAmkUXeXLE4KAUJAQMBHAVqAHBDohUAgNjsA/IBD/wEACg4MIDApib9Ye8rLqw54+4LwH/N+Hs74gvPV+Rl90Fw3qh9OwfgRRDevRuGEk7y3czOuP9YAh19dYK7wqrfb07DzSs/NoXC+Y1Kxc9qVk0V63lfOY8vTBisgXwGvAhUCaMeZFrwqHNhErdZSKvETLqwxOuZnbjQJ3OvOJnnQGe9a0h2+eg+xuELgIurxJSP1DXNz8MlJTM0hLyyAjPYusjCyyM7PYt+snyNh58s1JA2uwWH4WmCysaVhZ5SvUIgRhNbE3s5rcHf6IIwjxC8LhCMDhH4jDL4Bsk4vJD4PUNmDeAnkImmzDPzKVs5qfRVBwACEhwYSEBBIWFkJ4eDAb/0znl2/Pge5vwA+fw6VXwQ8TufmhzTw6oidR9evRIDK8cIS5t59bRerS5MlwrMFCPkzcTlbyEYLbT2VYPd99JNTUrgBPFay9ZeoyoDNwPzACOA6sAR4wxu1CABVGBzkpdZLyDuyC8g/uOuX5y4OHwNw5DOu1/qSwfn4Ohjd9yxo4dkPB9KpnYO4c/lGmFZ5QihL/q0jYnNw8YiZ1YN8vZ8A3kyA7zhpsdvarhEUd5Oo215KWmkFaehYZ6ZlkZGSzYf2X9mYWBf3X+6zGb78WBIQ0Jz8vF+PMIS8rA5Ofi8k7AKScyNS8APshbz+s31ra4DSbgm0iF84D5vH+0/D+064BArEUfaB1zLTX6p15FPwv4dn/czD5viAcfgGIwx+Hnz9+/gGkOzMgMAze6wwsgffPhyajmPxCKvPe/4LAwACCguwjOJC/duXwx09doMu58EsSSdnn8OwvDi6+8TUiZp1BRHgIYaHBREaEEhkeSmhoEHuDfuWV0ecRevNcnF//RPYl7gfAlXfAnLfDQ7VR4v4issbleqqxdmIrxN4y9VNgjDHmuIi8CTyNVTOfBl4CRlZYAjeoBaso1YSqeGFVJyu81PDXXE+z9rv4fNA3HE1JI/lYGkeOpZKamsmx4xlMnvYg/HXg5JuLak2H9leQm51LTk4OeTm55Obmcnj/6pKtdr8mOAKaQX6upexNLpg8yD8CeOd94xfegEbXPk7KTx+RuW0lJzbwiAD8QfxP/EoAIv4gfhgcYAIhrx7W9tr9ISQNApyEBobh8PPD4fCzfv38OJaTQkpCrU0AACAASURBVH5OGBzpAOZFcDwGTTbhH3mcjk064u/vR0CgPwH+/vgH+PPzvp/ISmkIfwyCvJsgYC7EfkR4iwPccd5tBAUFEBwYQHBQIEFBAYQEB/LVT0nMm3YuXDIevv4KBvWH/73G35/YwPgRvQkPDSYkOJDAAMuOq0h3QFmUZcHaW6bOB/5njPlXCf4xwHxjTOcKCVCWfKpgFaXu4G0l7m0FDpWkxMsTPuQA/O0GmrXfxdfXLeRYWjrHj2dwPD2T1LRM7p54I2wpYUOwli3o33sU2Vk5ZGfnkpObR3ZOHmlnxLF/5Sdkbihh4GpIa+rV74zTmUe+04nJd5Kfn0dO+nbI23dyeDnNato3TuvjgDz7Nx3IKbH8qh7BaiwtOPyAY2U+B49Sdj+KuLQtU5sZY/bZ5/8AuhtjbqiQAGWgTcSKUocob9O4t+dIV2ROdXma0isl/JDrYe4cbn11PXGdYk4K//yhCJLWdSlRgX8z5nH2pWTy2vd/8tiATgQHWNNuYqY86aap/quT8nDftL+jlPCuHwl7YeDNNOuwk6+HLOB4egaZWTmkpWeRmZ3DffPu58j2lvDTPyAnAQL/B93eIrzxQW7oeCM5Oblk5+SRm5NHTm4uublO/rd4GuzbfXIBNmhN+zP648xz4nQ6ceY5yc93smfnj5DmMogvozHMhKREPO6mKScXAjcDG0Vkne02DhgqInFYTcSJwJ1eyR10mo6iKDULb08FquwpYku2HjQdH/vGrElM9jhOefOo6vDGeDbV7VTCewK6Xd2pE+bnZ9J79SrqOGQI3H03ZGTAFVecHGnECOs4fBiuu+5k/7vuguuvh1274OabT/Z/4AG46irYuhXuLOEDZ/x46NsX1q2DMSV8fj37LFxwAfz0E4wbd7L/lCkQFweLFsEzz5zs/9ZbcOaZMG8evPTSyf7vvw+tWsHHH8Obb57s/8knEBUFM2ZYR3G+/hpCQ+GNN2DOnJP9lyyxfl98EebPL+oXEgLffGOdP/00fFdsMErDhvCpvebwo4/CihVF/Vu2hA8+sM7HjLHK0JX27WGqPU5h1Cj444+i/nFxVvkB3HQT7C72Fd2jBzz3nHV+7bWQnFzU/5JL4LHHrPPLL4fMzKL+AwbAgw9a5wkJnITWPa17UFj3Ju+8gZTWK5h1/sckrcwmelA014VOI/vDw/ybtwA44h9Cg7zMwro3eTKkLLidWed+YMU5P4hhK68nMmwkY7/qbaXvUvcK87hkHknfHSX66lYM+6IPkTt7MLb1Ryfks+ve5CfSSVl69wmZCtJvcT9jZ3U9qe4Vpn/ZNyQtOET0lc0Z9lXfk9N3qXv/HPMyz65+seTm/Y6hJ9W9fxJdcvj4B5mIbdmWs+7J0qW6VKKiKEptpUABTVx5vvVxN+Y57p+zjiXxqaT+EkiEM8dSrq5xxgKL91hxAFYCJEGf1UDv0vNo+AwssT/uvrgCWpe0GhmMvTcTliadnP7A34Gupacf+zaMsz/uvrqz1PQBInf2YFz8g8xquoKk3hDddAXDxn5FZPIICpcsLR5+3AJmhdndAU1XMCz+QSJ39nCbT02mZliwOshJUZRqTlaukxxnPvWCA9ifksXB1Cy6tKzva7FqNdV9sX9H2UEURVEUd+Q687nmjZ/45383AdA0MliVq6JNxIqiKBXFGIOIEODnYOh5rYhuWG2NKcUHqAWrKIpSAfanZHHD1JX8stNaZe+WHjH0bt/Ix1Ip1QlVsIqiKBUgNMiPlMxcDqd6sq2hUhfRQU6Koigecjgtm/dXJHHfJWfgcAj5+QaHQ3wtVp1FBzkpiqLUEpZsPcSbS7ezee9xAFWuilvUglUURXFDenYeOw6n07lFJMYYdh/NpFWDUF+LpaAWrKIoSo3m/jnruHXGarJynYiIKlfFY3SajqIoSjFy8vIxGIL8/XjgsjM5mp5TuFC/oniKWrCKoiguZOU6ufbNn3hpgbUOcfsmEXRv29DHUik1EbVgFUVRXAgO8KPXGVF0aRnpa1GUGo4OclIUpc5z8HgW4z/fxLgrOhITVW3HzCjF0EFOiqIo1RynMWzck8LWA6m+FkWpRagFqyhKnSQlM5d56/dy0/nRAGTnOQny14FMNQm1YBVFUaohc1bv4okvN/PnQctqVeWqVDZqwSqKUmfIynVy4HgW0Q3DyHPms+1gGh2b1fO1WEoFUQtWURSlmvD3D9Zy67uryXXm4+/nUOWqeBWdpqMoSq3GmW8QrHWD7+p9Opm5TgL81LZQvI/WMkVRai2pWbncMHUFM35KBKB724YknNnYt0IpdQZVsIqi1FrCg/xpUT+EhuGBvhZFqYOUqmBF5CWX83uK+b3tTaEURVEqSnJaNmM/WU9yWjYiwpQbunF1XAtfi6XUQdxZsH1czkcW8+vmBVkURVFOmUNp2Xy9cT+/7Dzma1GUOo47BSulnCuKolQr0rPz+HbTfgA6NK3Hj49czKWdmvhYKqWu407BOkQkQkQiXc7riUg9QGdkK4pSbXhzyXbu+fAXdh/NACAyJMDHEimKm4UmRGQ3kE/J1qsxxrT2pmCu6EITiqIUJ9eZT0pmLlHhQaRn57Fl33HOjWnga7GUKqS6LzShKzkpilLjMMZw89uryMx1MvfOHjgc2otVF6nuCrbUhSZEpBWQYow5bl9fBFwNJAL/McbkVomEiqIoNsYYRAQR4cburQsXkFCU6oi7Pti5QD0AEekK/Bc4CHQHXve+aIqiKCdIycjllndW8fXGfQBcEduMy2Ob+VgqRSkdd0slhhpjdtvnNwHvGGOeFxEHsN77oimKopwgNMiPnLx8MnKcvhZFUTzC02k6FwPfARhj8oHq33GrKEqNJyUzl+e+2UJmjrV+8Eejzue6c1r6WixF8Qh3CnapiHxor+jUEPgeQESaAtr/qiiK19m8N4V3lu9g5Y5kAES0v1WpObhTsKOBr4H9QC9jTI7t3hx4rKyEReQdETkoIptc3CaIyB4RWWcfV5yK8Iqi1D6ycp2s2nEEgAtOj2LpQ33oowv0K+VERFqJyGIR2SIim0XkPtu9gYgsFJFt9u9pXpPBW9N07FHHacB7xpjOttsEIM0Y82J50tJpOopSd5jw5WY+Wr2TH8ZeTKOIIF+Lo1Rj3E3TEZFmQDNjzC8iEgGsBQYBI4AjxphJIvIIcJox5mFvyOdums5Riva1in0tWAtNuJ3RbYxZJiIxlSCjoii1HGe+ISvXSViQP3f3OZ2EMxupclVOCWPMPmCffZ4qIluAFljTTRPsYDOBJUDVKljgR6AR8CnwEbCnkvK8R0RuAdYADxhjjpYUSERGAaMAAgN1qylFqa3k5xuGv7OKiGB/3hh2No0jgml8ZrDX8svNzWX37t1kZWV5LQ+lcgkODqZly5YEBFRsCUzb2OsG/Aw0sZUvxph9IuK1/ge3TcR22/R1wPVY/bUfAXOMMR5tU2Hf1HyXJuImwGEsS/hpLPO9+E49J6FNxIpSu5n5UyJhQf5ce3YLrw9k2rFjBxERETRs2FAHTdUAjDEkJyeTmppKmzZtiviJSA6w0cVpqjFmarEw4cBSYKIx5jMROWaMqe/if9QY45V+WHcWLLZ1OU1EpmMp2deACOAld/HcpHeg4FxEpgHzK5KOoig1m2MZOTz62UZG9mxDfEwDhl8QU2V5Z2VlERMTo8q1hiAiNGzYkEOHDpXknWeMOddN3ACsVthZxpjPbOcDItLMtl6bYS2g5BXcjSJGRM4TkZeBX7HarAcD/6poZvbNFHANsKm0sIqi1F78/RxsO5jGjkO+aZlS5VqzqMjzEivS28AWY4yr3voSGG6fDwe+OGUBS6FUBSsifwJTgQPAbcAbQDIQKyJdykpYRGYDK4AzRWS3iNwGTBaRjSKyAWtD939Uwj0oilIDyMjJY+qy7TjzDeFB/nxzXy+GxLfytVhVTmJiIp07d/Za+kuWLGHAgAEAfPnll0yaNKlS0o2JiSE2NpYuXbrQu3dvkpKSCv1EhJtvvrnwOi8vj0aNGhXKceDAAQYMGEDXrl3p1KkTV1xhzdBMTEwkJCSEuLi4wuO9996rFHmBC4GbgYuLTQ2dBFwqItuAS+1rr+CuiXg/Vl/plcAVFF3ZyQAXuUvYGDO0BOe3yyugoii1g8W/H+K5b36nc4tILjg9igA/tw1oSiUwcOBABg4cWGnpLV68mKioKJ544gmeeeYZpk2bBkBYWBibNm0iMzOTkJAQFi5cSIsWLQrjPf7441x66aXcd999AGzYsKHQ7/TTT2fdunWVJmMBxpjllLzdKsAllZ5hCZRaw40xPY0xvezjIpfzXsYYt8pVURQFrD1b/ziQCsAVsU355r5eXHB6lI+l8j15eXkMHz6cLl26cN1115GRYW0U/9RTTxEfH0/nzp0ZNWoUBYNQX331VTp16kSXLl244YYbAEhPT2fkyJHEx8fTrVs3vvji5JbOGTNmcM899wAwYsQIRo8ezQUXXEDbtm355JNPCsO98MILxMfH06VLF5544oky5e/Rowd79hSdWHL55Zfz1VdfATB79myGDj1hY+3bt4+WLU8scdmlS5mNoLWCcn9CikgfEfnGG8IoilK7GP/fTVz/1gqOZ+UiInRoWs/XIp1MQsLJxxtvWH4ZGSX7z5hh+R8+fLKfB2zdupVRo0axYcMG6tWrxxt2fvfccw+rV68utAbnz7fGgU6aNIlff/2VDRs28J///AeAiRMncvHFF7N69WoWL17MQw89RFmzLfbt28fy5cuZP38+jzzyCAALFixg27ZtrFq1inXr1rF27VqWLVvmNp1vv/2WQYMGFXG74YYb+Oijj8jKymLDhg1079690O///u//uO222+jTpw8TJ05k7969hX7bt28v0kT8ww8/eFSGNQF3fbAJIvKbiBwTkRkicqaIrASmAO9WnYiKotQkjDHkOvMBuOOitky8JpZ6wRWbv1hbadWqFRdeeCEAN910E8uXLwesJtju3bsTGxvL999/z+bNmwHL4hs2bBgffPAB/v5Wz96CBQuYNGkScXFxJCQkkJWVxc6dO93mO2jQIBwOB506deLAgQOF6SxYsIBu3bpx9tln8/vvv7Nt27YS4/fp04fGjRuzaNEibrzxxiJ+Xbp0ITExkdmzZxf2sRbQr18//vrrL+644w5+//13unXrVjgquKCJuODo1atXeYqyWuOuD/ZlrPWIVwCXA6uAJ4uNxlIURSkkz5nPbTPX0LZRGE9cdRbtGofTrnG4r8Vyz5IlpfuFhrr3j4py718KxUfFighZWVncfffdrFmzhlatWjFhwoTCxTC++uorli1bxpdffsnTTz/N5s2bMcbw6aefcuaZZxZJq0BxlkRQ0InVsQqan40xPProo9x5551lyr148WLCwsIYMWIEjz/+OP/6V1F1MHDgQB588EGWLFlCcnJyEb8GDRpw4403cuONNzJgwACWLVvGOeecU2aevkREBhtj5pblVhpum4iNMYuMMenGmE+wRhC/XHFRFUWp7fj7OejQLIK2jaq5UvUxO3fuZMWKFYDVX9mzZ89CZRoVFUVaWlphH2l+fj67du2iT58+TJ48mWPHjpGWlka/fv3497//Xagof/311wrJ0q9fP9555x3S0tIA2LNnDwcPlj41NCQkhClTpvDee+9x5MiRIn4jR47k8ccfJzY2toj7999/X9jPnJqayvbt22ndunWF5K1iHvXQrUTcWbCRIlJ8+NlVBV9expgvPc1EUZTay/GsXCbO38Ko3m05vVE4j17e0dciVXs6duzIzJkzufPOOznjjDO46667CA0N5Y477iA2NpaYmBji4+MBcDqd3HTTTaSkpGCM4R//+Af169fnscceY8yYMXTp0gVjDDExMYV9tuXhsssuY8uWLfTo0QOA8PBwPvjgAxo3Ln0FwWbNmjF06FBef/11HnvsxOZqLVu2LBwp7MratWu555578Pf3Jz8/n9tvv534+HgSExML+2ALGDlyJKNHjy73fVQmInI51uyZFiLyqotXPSDP43RKWypRRN53E88YY27xNJNTRZdKVJTqy8HjWVzx6g+M7dehRsxr3bJlCx076kdATaOk5+ZuN51TQUS6AnHAU8DjLl6pwOLS1tAvTqkWrDHm5tL8FEWp22TnOflqwz6u6daCxvWCWfJQH8KD3K68qig1BmPMemC9iHxojMmtaDo601tRlHLzydrd3D9nPet3pwCoclVqK+fZm7L/ISJ/icgOEfnL08j6r1AUxSOc+YYDx7NoXj+EG+Jb065ROHGt6pcdUVFqLm9jLem7FnCWN3KZFqyInKSES3JTFKV289Dc9QydtpLMHCd+DqF724a+FklRvE2KMeYbY8xBY0xyweFpZE8U5SrgbA/cFEWpZRhjMAYcDmFo99Zc0C6K4ADtWVLqDItF5AXgMyC7wNEY84snkUtVsPYu782AEBGJ5cSiyfWA0AqLqyhKjSAr18mYj9ZxdnR9Rl10OvExDYiPaeBrsRSlKilY79F1z1kDXOxJZHcW7JXASKAl8DonFGwq8FhpkRRFqR0E+TsI8Hfg0L1Tq4wRI0YwYMAArrvuulLDxMTEsGbNGqKiPNs0YcaMGaxZs4bXXnutssSsMxhj+pxKfHfTdN4F3hWRIcaYOaeSiaIoNYOMnDymLNrGHb3a0igiiFdviKvzm5PP2jiLf373T3am7KR1ZGsmXjKRYbHDfC2WUgWIyOMluRtjnvIkviedKY1FpJ6d2X9EZJWIVMleeoqiVC17j2Xy/ooklv5hLcSuynUWo+aNIiklCYMhKSWJUfNGMWvjrFNO++mnn6ZDhw5ceumlDB06lBdffLGI/3fffUe3bt2IjY1l5MiRZGcXdgHywgsvcN5553Heeefx559/AjBv3jy6d+9Ot27d6Nu3r9s1iRWPSXc5nFjr8sd4GtmTQU6jjDGvichlWM3FdwFTgeq9SrOiKB6R68znp+3J9G7fiHaNI1g6NoHGEcG+FqtKGPPtGNbtL32z75W7V5LtzC7ilpGbwW1f3Ma0tdNKjBPXNI4p/ae4zXfNmjV8+umn/Prrr+Tl5XH22WcXWfg+KyuLESNG8N1339G+fXtuueUW3nzzTcaMGQNAvXr1WLVqFe+99x5jxoxh/vz59OzZk5UrVyIiTJ8+ncmTJ/PSSy95WhRKCRhjihSgiLwIeLxMsCcWbMFaipcD7xpj1noYT1GUGsD0H3Yw/J1VhRuj1xXl6gnFlWtZ7p6yfPlyrr76akJCQoiIiOCqq64q4r9161batGlD+/btARg+fHiRPVoLNjMfOnRo4aYBu3fvpl+/fsTGxvLCCy8UbnWnVCqhQFtPA3tiwa4Xka+B9sA/RSScE0pXUZQaiDGG45l5RIYGMOKCGNo1Dqd9kwhfi1XllGVpxkyJISkl6ST36MholoxYUuF8S1sD3lN/16b7gvN7772X+++/n4EDB7JkyRImTJhQYfkUCxHZyAl95wc0wlqf2CM8sURvBSYA5xljMoBg4LbyiakoSnXBGMN9H63jlndXkefMJyTQj0s7NfG1WNWSiZdMJDSg6KzE0IBQJl4y8ZTS7dmzJ/PmzSMrK4u0tDS++uqrIv4dOnQgMTGxsH/1/fffp3fv3oX+H3/8ceFvwS44KSkptGjRAoCZM2eeknxKIQOAq+zjMqC5Mcbj4dhlWrDGGKeItAUuBSYCIWgTsaLUWESEyzs35WBqNn6Ouj2IqSwKRgtX9iji+Ph4Bg4cSNeuXYmOjubcc88lMjKy0D84OJh3332XwYMHk5eXR3x8PH//+98L/bOzs+nevTv5+fnMnj0bgAkTJjB48GBatGjB+eefz44dO05JRgWMMUn2zjq9bKdlwAZP45e6XV1hAJHXgADgImNMRxFpAPzPGBNfQZnLjW5XpyinRmaOk/Gfb+Ki9lFcHdfC1+L4lOqyXV1aWhrh4eFkZGRw0UUXMXXqVM4+WxfIK42q3K7OJf37gDuwVnICuAaYaoz5tyfxPemDvcAYc7aI/ApgjDkiIoEVklZRFJ8Q4CfsOpLB/pQsX4ui2IwaNYrffvuNrKwshg8frsq1enIb0N0Ykw4gIs8DK4BKU7C5IuLA7ugVkYZAfsVkVRSlqsjOczL9hx3cemEMoYH+fHhHd/z9tHenuvDhhx/6WgSlbISiu+g4ObGqYZl4omBfBz4FGonIk8AQ4MnySKgoStWzac9xXlqwlZanhXB1XAtVropSft4FfhaR/9rXg7C2sPMId4v9+xtj8owx74nIWqAvluYebIzZdCoSK4riHZz5hs17U+jSsj7nRJ/Gwvt7c3qjcF+LpSg1EmPMv0RkCdATS//daoz51dP47izYwi3pjDGbAZ21rCjVnFcW/cF/lv7Fdw/0plWDUFWuinIKiMj5wOaC7elEJEJEuhtjfvYkvjsFq+P3FaUGYIwhOy+f4AA/brkghuiGYbQ8LcTXYilKbeBNiu59nl6CW6m465RpJCL3l3acgsCKolQSxhjun7Oee2f/ijGGqPAgrj2nZZ1fpL8mcvvtt/Pbb795HH7NmjWMHj0asLaku+eee7wlWl1GjMtcVmNMPp6NXYIyAvoB4aglqyjVFhGha8tIsvLyMQZUr1YukydDfDz0cdkVdPFiWL0axo6t3LymT59ervDnnnsu5557btkBSyAvLw9/f4/1RF3mLxEZjWW1AtwN/OVpZHcW7D5jzFPGmCdLOk5FYkVRKk5mjpPHv9jEyr+SARhxYRv+3vt0HLoqU6UTHw9DhlhKFazfIUMs91MhPT2dK6+8kq5du9K5c2c+/vhjEhISWLNmDQDh4eE8/PDDnHPOOfTt25dVq1aRkJBA27Zt+fJLazOXJUuWMGDAgJPSLm3bugkTJjBq1Cguu+wybrnlllO7gbrD34ELgD3AbqA7MMrTyNoHqyg1DINh+bbDNIsM4fy2DX0tTo1mzBhYV/pudQA0bw79+kGzZrBvH3TsCE8+aR0lERcHU9zvIcC3335L8+bNC9cgTklJ4c033yz0T09PJyEhgeeff55rrrmG8ePHs3DhQn777TeGDx/OwIEDS03b3bZ1a9euZfny5YSEaB+9JxhjDgI3VDS+OwWrm6orSjUhz5nP3LW7GXxOS0ID/flqdC9CAv18LVad4LTTLOW6cye0bm1dnyqxsbE8+OCDPPzwwwwYMIBevXoV8Q8MDKR///6FYYOCgggICCA2NpbExES3ae/evZvrr7+effv2kZOTQ5s2bQr9Bg4cqMq1CilVwRpjjlSlIIqilM4P2w7z6GcbOS00kP6dm6pyrSTKsjThRLPwY4/Bm2/CE08U7ZOtCO3bt2ft2rV8/fXXPProo1x22WVF/AMCAgoHqjkcDoKCggrP8/Ly3Kbtbtu6sDCvLdtbLRGRd7B2xDlojOlsu03AWl/4kB1snDHma2/kr73cilJNMcaw80gG0Q3D6NOhMZ/e1YNzohv4Wqw6RYFynTPHUqp9+hS9rih79+6lQYMG3HTTTYSHhzNjxoxKk1m3rSvCDOA14L1i7i8bY170dualDnISkf+JyD9EpIO3hVAU5WQmffs7A1/7kUOp2QCqXH3A6tVFlWmfPtb16tWnlu7GjRs577zziIuLY+LEiYwfP/7UhbUp2LauV69eREVFVVq6NRFjzDKgwq2xIhIkIjeKyDgRebzg8Dh+advViUhToL99tAd+Br4FvjPGpFVU4Iqg29UpdQlnvsHPIew4nM6yPw5xS49onddaiVSX7eqU8lHR7epEJAaYX6yJeARwHFgDPGCMOVpK3G+BFGAtLov+G2Ne8kRmd32w+7HM6xn2bjrdgcuBsSKSCSwwxkz2JBNFUcrGGMPYTzYQHODH04M60yYqjDZRdavPTFHKib+IrHG5nmqMmVpGnDeBp7F2iHsaeAkYWUrYlsaY/hUWzpNA9uoVK+zjcRGJAvpVNFNFUU5GRGgQFkiQvwNjjFqtilI2ecaYcq22YYw5UHAuItOA+W6C/yQiscaYjRURrkKDnIwxh4FZFYmrKMoJsvOcTFm0jWu6taB9kwgevUKbLhXFm4hIM2PMPvvyGuCk3eFEZCOWhesP3CoifwHZWOtDGGNMF0/y0lHEiuJDUrPymLN6F5EhAbRvEuFrcRSlViEis4EEIEpEdgNPAAkiEoelQBOBO0uIevISWRWgTAUrIn7GGGdZ4RRF8Yz8fMOC3w7Q76wmRIUHsfD+3jQIC/S1WIpS6zDGDC3BucwN040xSVBku7pU+zoC6AQkeZK/u7WIC/hTRF4QkU6eJKgoinu+XL+Xv3+wluV/HgZQ5aoo1Zc3AddZMwXb1XmEJ03EXbDWYpxujyZ+B/jIGHO8PFIqSl3GGENyeg5R4UFc1bU5YUH+9GxXt+co1mXCw8NJS6u82Y4TJkwgPDycBx98sNLSVIAStqsTEY+7Vsu0YI0xqcaYacaYC4CxWG3Y+0Rkpoi0q5DIilLHePbrLVz92o+kZuXi5xAu7dRERwnXMFyXHFTqDH+JyGgRCbCP+6ik7eoAqw9WRAaKyH+BV7DmDLUF5gFeWb9RUWoLBR+//Ts3Y9j5rQkN1HGFNZUnS9s+5xQwxvDQQw/RuXNnYmNj+fjjjwv9Jk+eTGxsLF27duWRRx4BYNq0acTHx9O1a1euvfZaMjIyKl0mpQiu29XtoRK3qytgG7AYeMEY85OL+yciclE5BFWUOkOeM5+n5v9G08hg7k5oxznRp3FOdCVsw6JUKmPGjGFdWfvVuZCQkFBmmLi4OKZ4sosA8Nlnn7Fu3TrWr1/P4cOHiY+P56KLLmLdunV8/vnn/Pzzz4SGhnLkiLXa39/+9jfuuOMOAMaPH8/bb7/Nvffe67H8Svnw5nZ1BdxijFnu6iAiFxpjfjTGjK5oxopSm/FzCEczcnXXmxpOYmIiSUknBowuXboUgOjoaGJiYk45/eXLlzN06FD8/PxoF9QeNAAAHhVJREFU0qQJvXv3ZvXq1SxdupRbb72V0NBQABo0sNah3rRpE+PHj+fYsWOkpaXRr5+u9+NNRKQl8G/gQqxpPcuB+4wxuz2J74mCfRU4u5jbv0twU5Q6TZ4zn6k//MW1Z7ekSb1gXrk+DodD+1mrM55ammCttFXa2u0VpbT0SlvJa8SIEXz++ed07dqVGTNmsGTJkkqVRzmJd4EPgcH29U2226WeRHa3m04PEXkAaCQi97scE4AyP8tF5B0ROSgim1zcGojIQhHZ9v/t3Xl4VfW1//H3ysnEEEDmIFRwACUOWBBbaxlUrLa9tnqpaau9em2v7f1ZW9vbX9V6a9Vef3W6XmurNlQpylUBca7WYmVUo8zIIIMICgTDJBDG5CTr98feCQEzHCBnyDmf1/Pkydln2HttNs9ZWd+993eFvzVmJmljw/a9PPjGKl5eVAag5CrNGjZsGBMnTqS6uprNmzczc+ZMhg4dyoUXXsjYsWPrzrHWDhFXVFRQWFhIVVUVTz6pyfQSoJu7/8Xdo+HPOKBbrB9u6iKnXKA9QZVbUO9nJzA6hnWPI+jEU99NBN14TgLeCJdFWi13Z+7a4MvvuC7tmHLDcH7w5eOTHJXEw29+85sWX+ell17K6aefzhlnnMF5553HPffcQ8+ePbnooou45JJLGDJkCIMGDeK++4LWpb/97W85++yzGTVqFCefrE6iCbDFzK4ML/aNmNmVwNZYP9xou7q6N5gdVzurxeFqoE3QCmCEu280s0JgursPaG49alcnqeqpdz/mV88v5vn/cw5nfk4DMq2B2tW1Tkfaru5omNnnCBq2fzF86i2Cc7Ax5cRGz8Ga2QPufgPwRzP7TBZ290uOIN4etZMsh0m2exPbv5bwcujcXM10I6llT2WUtrnZXPb5Y8mJGGf07pTskESkhbn7x8CR5Dqg6Yucxoe/7zvSlR+NsKffGAgq2GTEINKQO19ZxjsfbuPZfz+H/JwI3xrSJ9khiUgcmNnxBPM/fIHgKuJS4GfuHtNkE001XJ9nZhHg39z9ypYIFiivbRUUDhFvaqH1iiTM4OM6k5cdQRMxiaS9p4CHCNraQXBP7NMEE040q8mZnMIuOt3MrKXGaF8CrgofXwW82ELrFYmbquoa7vrbcp6bH9z6dtGpPfnFVwaQE4mlV4aItGLm7uPrXUX8vwSVbExiuQ92LfCWmb1E0EkAAHe/v8moGu7Ddxcwycy+D3zMgXuLRFJWlhnzP/qUaHUNl+nub5FMMs3MbgImECTWYuAVM+sM4O7bmvpwLAm2LPzJIrhNJyaN9OEDOD/WdYgkS02N89Tsj/nmmcfSPi+bJ74/lPwczcokkmGKw9+HNmW/hiDhNnlPXrMJ1t1bfoZrkRT3/ic7ufXFJVTXOFed01fJVeKufsu5W2+9lWHDhnHBBRcc9noWLlxIWVkZX/3qVxt8/Xe/+x2PPfYYkUiEBx98UNMtNsHd+x3N55tNsGbWjaBNXRGQX2/D5x3NhkVSjbuzevMuTuxeQFGvjrx43bmcemyHZIclGeiOO+444s8uXLiQuXPnNphgly1bxoQJE1i6dCllZWVccMEFrFy5kkhEf0DWZ2a/dPd7wsffcvdn6r32/9z9V7GsJ5arNJ4ElgP9gNsJzsnOOeyIRVLcX95ay8W/n8Wq8goATuvdUT1bJS6eeOKJuhmcvve9733m9auvvprJkycDMG/ePIYPH87gwYP5yle+wsaNG4Ggs8+NN97I0KFD6d+/P7NmzaKyspJbb72ViRMnMmjQoIPa3wG8+OKLfPvb3yYvL49+/fpx4oknMnv27PjvcOtTv4POzYe8dugMhY2KJcF2cffHgCp3n+Hu1xDcEySSFqLVNQBceuax3HjRyRzfrX2SI5JEKi4p5Zm564DgivHiklKeXxBcMb63spriktK6+aV37quiuKSU15YESW7b7kqKS0r5x7JyADZV7Gt2e0uXLuXOO+9k6tSpLFq0iN///veNvreqqorrr7+eyZMnM2/ePK655hpuueWWutej0SizZ8/mgQce4Pbbbyc3N5c77riD4uJiFi5cSHFx8UHr27BhA336HLhvu3fv3mzYsCGWf6ZMY408bmi5UbFc5FQV/t5oZl8juOCpd6wbEEllv3v1fVZt2sVjVw3hmHa5mkdY4m7q1KmMHj2arl27Agda0TVkxYoVLFmyhFGjguYt1dXVFBYW1r1+2WWXATB48GDWrl3b7LYbmhpXozQN8kYeN7TcqFgS7H+ZWUfgPwja1HUAfhbrBkRSWa9ObaisriFa4+RE9EWTiSb+8It1j3MiWQctt8mNHLTcIT/noOXO7XIPWu5eUHeZSqMaa0XX2HuLioooLS1t8PW8vDwAIpEI0Wi02fX17t2bdevW1S2vX7+eXr16xRRLhjnDzHYSVKttwseEy80f5FCzQ8Tu/ld33+HuS9x9pLsPdveXjixmkeSKVtfwhzdW8eaqLQBcdU5ffvNPRZo0QhLm/PPPZ9KkSWzdGjRlqW1F15ABAwawefPmugRbVVXF0qVLm1x/QUEBFRUVDb52ySWXMGHCBPbv38+aNWtYtWoVQ4cOPcI9SV/uHnH3Du5e4O7Z4ePa5ZxY19PUZP9/oIlS2N1/cpgxiyRdtMZ5YeEGtu+t4tyTuiY7HMlARUVF3HLLLQwfPpxIJMKZZ57JuHHjGnxvbm4ukydP5ic/+Qk7duwgGo1yww03UFRU1Oj6R44cyV133cWgQYO4+eabDzoPW1RUxOWXX87AgQPJzs7moYce0hXEcdRouzozu6rBF0Lu/nhcImqA2tXJ0XB3Xlm8ka8U9SQnksWOPVV0bBvzH6GSZtSurnVKRru6o9XUZP8JS6Ai8TR7zTZ+/NQC7hl9OpcP6aPkKiIJ0Ww/WDN7mQaGio+wH6xIwpTv3EePDvmcfXwXxv3rWQw7qVuyQxKRDJKy/WBFjsaYmav5w9QP+PsNw+jVqQ0jBnRPdkgikmGa7Acb/p6RuHBEjk7tLRAXFRWya1+Uru3zkh2SpKDDuVVGkq+xa4VSXbP3JpjZ181sgZltM7OdZlZR754gkZRx399X8J8vLAHgc13a8vMLB5Cbrdtv5GD5+fls3bq11X5pZxp3Z+vWreTnx3z7acqIZaKJB4DLgMWu/5GSwqI1TnX4E8lSdSIN6927N+vXr2fz5s3JDkVilJ+fT+/erW8CwUZv06l7g9k04Hx3r0lMSJ+l23SkITU1zl/eXss5J3ThlMIOGvYTyTCt9jaden4JvGpmM4D9tU+6+/1xi0okBjv3VfHI9NV8smMvt3xtoJKriKSUWCrYKcAuYDFQV8UmshG7Klip5e6Urt7KF0/ogplRtn0vhR3zlVxFMlA6VLCd3f3CuEciEoNpKzZxzbi5PHLF57n4tEJ6dWqT7JBEJEWZ2Vjg68Amdz81fK4zMBHoS9Df/HJ3/zQe24/lEst/mJkSrCRVxb6ga+KI/t25//IzGDWwR5IjEpFWYByfbZB+E/CGu58EvBEux0UsQ8QVQDuC869VBO163N07xCuoQ2mIOLP9acZqxpd+xKs//TId22iaQxEJxDJEbGZ9gb/Wq2BXACPcfaOZFQLT3X1APOJrdojY3QvisWGRWH3h+C6U79xHnu5pFZGj18PdNwKESTZu07w1NRfxye6+3Mw+39Dr7j4/XkFJZnN3Hpr2AWbGdSNPZFCfTgzq0ynZYYlI6sk2s7n1lse4+5ikRXOIpirYnwPXAv/dwGsOnBeXiCTjmRkfbNqFmeneVhFpStTdhxzmZ8rNrLDeEPGmeAQGMZyDTQU6B5v+3J1Jc9cxrH83Cju2oTJao2kORaRJR3gO9l5gq7vfZWY3Edwp88t4xNfoN5iZnWVmPest/4uZvWhmD4aXOYu0mPKd+7n95WWML/0IQMlVRI6amT0NlAIDzGy9mX0fuAsYZWargFHhcny231gFa2bzgQvcfZuZDQMmANcDg4BT3H10vII6lCrY9LWsbCcDewUXpK/4pIKTurcnS/MIi0gMUn2iiabKhIi7bwsfFxOcPH7W3X8NnBj/0CTdvfLeRr764Cze/mALAAN6Fii5ikjaaDLBmlntRVDnA1PrvRbLDFAiDaqMBjNuXjCwO//5tVMY0ldnHEQk/TSVYJ8GZpjZi8BeYBaAmZ0I7EhAbJKGSmas5rJH3mJ/tJq87Ag/+PLxOt8qImmp0UrU3e80szeAQmBKvV6wWQTnYkUO2wnd2jOwsAPVNal/9bqIyNHQbToSVzU1zri319KhTQ6jB7e+hskikrpa80VOIi3i9WXlzFy5OdlhiIgklCpYiYtXF2/k3JO60iE/h137o7TLjWhGJhFpUapgJeOs2bKb659ewLi31gLQPi9byVVEMo4qWGkxG7bv5diwAfq7H25l8HHHkB3R33AiEh+qYCUjPL9gPSPuncaSDcEdXGcf30XJVUQymr4B5ajUjoCcN6AHPxx2Aid2b5/kiEREUoOGiOWIPTrrQ95evZVH/2WIpjgUkYTTELGkrfycCG1yI+wPpz4UEZEDVMFKzNydiXPWUdipDcP7d6sbHtYVwiKSDKpgJW1UVtcw9q01PDd/PRAkViVXEZGGqYKVZr25agtD+3UmNzuLzRX76dIuV+dcRSTpVMFKq7a0bAdXPvYu49/5CIBuBXlKriIiMVAFKw3asaeKjm1zAPjb4o2cd0p38rIjSY5KROQAVbDS6jwzdx1fvmcq67btAeDi0wqVXEVEDpMSrHzGF0/owjcGHUunsIIVEZHDpyFiAeCJ0rWsLK/gv755WrJDERGJSaoPEWcnY6NmthaoAKqBqLsPSUYccsCmnfvZuH0fldEacrM1sCEicrSSUsGGCXaIu2+J5f2qYOPjpUVlHN+1Hace25HqGifLNGmEiLQeqV7BqlTJULv3R7nzlWX8JezZGsnSpBEiIi0pWRXsGuBTwIESdx/T1PtVwbacxet3cOqxHTAz1mzZTZ9j2qitnIi0SqpgG/Yld/88cDFwnZkNO/QNZnatmc01s7nRaDTxEaahOWu38U9/fJNn528AoF/XdkquIiJxkvSriM3sNmCXu9/X2HtUwR6dfVXV5OdEcHf+952PGD24D21ydV+riLRuqmAPYWbtzKyg9jFwIbAk0XFkiklz1nHB/TPYvqcSM+N7X+yr5CoikgDJuE2nB/B8eEFNNvCUu7+WhDgywsBeHRh83DEYuoBJRCSRkj5EHAsNER+eSXPWsXnXfq4beWKyQxERiRsNEUvCzV67jdLVW6muSf0/nkRE0pUq2DTxxvvl9O9RQJ/ObdlXVU1uJEtt5UQkramClbj7dHclP52wkEdmrAYgPyei5CoikmSqYFuxj7bu5rguwR9vi9Zt5+TCArWVE5GMoQpW4mLWqs2MvG86U5eXA3BGn05KriIi9ZjZWjNbbGYLzWxuoreflG46cuRqapysLOPsfl24/ryTOKtv52SHJCKSykbG2limpamCbUWembuOSx9+K7iIKTuLn43qT0G+mqKLiKQiJdhWpFtBHl3a57GvqjrZoYiIpILs2jnrw59rD3ndgSlmNq+B1+JOFzmluJcXlbE/WsPowb0BcHe1lRMRofmLnMysl7uXmVl34HXgenefmaj4VMGmMHdn0tx1PDd/PbV/CCm5iojExt3Lwt+bgOeBoYncvirYFPT26i0UFXakY9scduypol1eRG3lREQO0VQFGzaTyXL3ivDx68AdiZz7Xt/aKaZ85z6uHjuHP05bBUDHtjlKriIih68H8KaZLQJmA68kurGMKtgUsXXXfrq0zwNg+opNnN2vi9rKiYg0QRNNSLOmLi/nS3dPZcHHnwIwYkB3JVcRkVZOCTYFDOnbmW8N7kPfLin7h5iIiBwmDREnyQsLNvDK4o2UXDlYE/OLiBwBDRFLg/ZHq6nYV0XF/miyQxERkThQBZtAU5eXk2XGiAHdcXfcUfUqInKEUr2C1WT/CVJd49z795V0bZ/LiAHdMTM0Z4SISPpSBRtn763fzoCeQZ/WjTv20rldrtrKiYi0gFSvYHUONo7WbtnNpQ+/TcmMDwEo7NhGyVVEJEOogo2DPZVR2uYGo+/PzlvPqKIedFBbORGRFqUKNsO8vqycc++extotwR8E/zy4t5KriEgGUoJtYace24FzTuhC2zwNBYuIZDINEbeAVxdvZPaabdx2SVGyQxERyRgaIs4Aq8p3sWDddnZr0ggREQmpgj1Cb6/eQkFeDqf17ki0ugYHctRWTkQkYVTBpqH90Wr+7zPv8cA/VgKQHclSchURkYOogj0Mqzfvol+XdmRlGSvLK+h9TJu623FERCSxVMGmifc37uSiB2by5OyPAejfo0DJVUREGqUE24xodQ0AJ/cs4D8uHMDXTitMckQiItIaKME2YcrSTxj1PzPZtrsSM+NHw0+gc7vcZIclIiKtgBJsE/p0bkvvY9pQGa1JdigiItLK6CKnQ0xbvonln1Tw7yNOSMj2RETkyOgip1ZmyrJPeGlRGfuj1ckORUREWjFVsMCCjz+lS7s8PtelLXsqo2SZkZ+juYRFRFKZKtgUt6cyyjXj5nDP35cD0DY3W8lVRESOWsZWsJt27qN7h3wA5q7dRv+eBWorJyLSiqiCTUEL121n2L3T+NvijQAM6dtZyVVERFpURiXY2mq9qFcHrjz7OAYfd0ySIxIRkXSVMQn2H8vKOe22KTz17kfkRLK48eKTuf7pBTy/YD0AeyurKS4p5eVFZQDs3FdFcUkpry0JqtxtuyspLinlH8vKAdhUsY/iklKmr9gEQNn2vRSXlPLmqi0AfLx1D8Ulpbzz4VYgmMe4uKSUeR9tA2DFJxUUl5SyaN12AJaW7aC4pJSlZTsAWLRuO8Ulpaz4pAKAeR9to7iklNWbdwHwzodbKS4p5eOtewB4c9UWiktKKdu+F4DpKzZRXFLKpop9dftfXFLKtt2VALy2ZCPFJaXs3FcFwMuLyiguKWVvZXD19PML1lNcUkpVOJPVM3PXUVxSWvfv+fTsj7ni0XfqlseXruWqsbPrlse+uYYfPD6nbnnMzNX8aPy8uuWHp3/Aj5+aX7f84BuruGHCgrrl+6es4BfPLKpbvvu15dz83Ht1y3e+soxfv7Ckbvn2l5dy+8tL65Z//cIS7nxlWd3yzc+9x92vLa9b/sUzi7h/yoq65RsmLODBN1bVLf/4qfk8PP2DuuUfjZ/HmJmr65Z/8Pgcxr65pm75qrGzGV+6tm75ikff4elwWk2A4pJSnpm7DoCq6hqKS0r1f0//94DM/L+XKTImwebnRHCcPZW6/UZEROIvoy5yqqlxsrKsBSISEZFk00VOKUTJVUREEiWjEqyIiEiiKMGKiIjEgRKsiIikJTO7yMxWmNkHZnZTorevBCsiImnHzCLAQ8DFwEDgO2Y2MJExKMGKiEg6Ggp84O4funslMAH4RiIDSEqCTXbZLiIiae9YYF295fXhcwmT8ASbCmW7iIikhWwzm1vv59p6rzV0X2ZCJ37ITuTGQnVlO4CZ1Zbty5r8lIiIyMGi7j6kkdfWA33qLfcGyuIf0gHJGCJOetkuIiJpbw5wkpn1M7Nc4NvAS4kMIBkVbExle1jqXwuQm5sb75hERCSNuHvUzH4M/B2IAGPdfWkzH2tRyUiwMZXt7j4GGANgZjVmtrcFtp0NRFtgPa2J9jkzZOI+Q2but/b5gDZNfcjdXwVejUtEMUj4ZP9mlg2sBM4HNhCU8d9NxF8WZja3ifH6tKR9zgyZuM+QmfutfW49El7BpkLZLiIiEm/JGCJOetkuIiISb5k2k9OYZAeQBNrnzJCJ+wyZud/a51aiVTRcFxERaW0yrYIVERFJiIxIsJk697GZrTWzxWa20MzmJjueeDCzsWa2ycyW1Huus5m9bmarwt/HJDPGltbIPt9mZhvCY73QzL6azBhbmpn1MbNpZva+mS01s5+Gz6ftsW5in9P9WOeb2WwzWxTu9+3h8/3M7N3wWE8MJ49IaWk/RBzOfbwSGEVwD+4c4DvunvZTM5rZWmCIu29JdizxYmbDgF3AE+5+avjcPcA2d78r/IPqGHe/MZlxtqRG9vk2YJe735fM2OLFzAqBQnefb2YFwDzgm8DVpOmxbmKfLye9j7UB7dx9l5nlAG8CPwV+Djzn7hPM7E/AInd/JJmxNicTKtiktyyS+HH3mcC2Q57+BvB4+Phxgi+ltNHIPqc1d9/o7vPDxxXA+wRTrKbtsW5in9OaB3aFiznhjwPnAZPD51vFsc6EBJvJcx87MMXM5h3SZSLd9XD3jRB8SQHdkxxPovzYzN4Lh5DTZqj0UGbWFzgTeJcMOdaH7DOk+bE2s4iZLQQ2Aa8Dq4Ht7l47m1Or+B7PhASb9JZFSfQld/88QWvA68KhRUlPjwAnAIOAjcB/Jzec+DCz9sCzwA3uvjPZ8SRCA/uc9sfa3avdfRDBVLpDgVMaeltiozp8mZBgk96yKFncvSz8vQl4nuA/aiYoD89f1Z7H2pTkeOLO3cvDL6Ua4M+k4bEOz8c9Czzp7s+FT6f1sW5onzPhWNdy9+3AdOALQKdwql1oJd/jmZBgk96yKBnMrF14YQRm1g64EFjS9KfSxkvAVeHjq4AXkxhLQtQmmdClpNmxDi98eQx4393vr/dS2h7rxvY5A451NzPrFD5uA1xAcP55GjA6fFurONZpfxUxQHgZ+wMcmPv4ziSHFHdmdjxB1QrBlJhPpeN+m9nTwAigK1AO/AZ4AZgEfA74GPiWu6fNRUGN7PMIgiFDB9YCP6w9N5kOzOxcYBawGKgJn/4VwTnJtDzWTezzd0jvY306wUVMEYIicJK73xF+p00AOgMLgCvdfX/yIm1eRiRYERGRRMuEIWIREZGEU4IVERGJAyVYERGROFCCFRERiQMlWBERkThQgpW0ZGY9zWyCma02s2Vm9qqZ9U92XI0xs15mNrn5dzb42avNrFe95UfNbGDLRRd/Zrar+XeJtC66TUfSTniD/tvA4+7+p/C5QUCBu89KanBxYGbTgV+4e9xaEppZdr15YOOx/l3u3j5e6xdJBlWwko5GAlW1yRXA3Re6+ywL3GtmSyzolVsMYGYjzGyGmU0ys5VmdpeZXRH2pVxsZieE7xtnZo+EfTo/NLPh4YTr75vZuNrt1a/IzGx07Wvh5x80s7fDz48On+9rYX/XcKLz+8Ltvmdm14fP32pmc8LYx4T7MhoYAjxpQW/QNmY23cyGhJ/5TrieJWZ2d/34zOxOC3puvmNmPQ79R7Sg7+gYM5sCPGFBn86/hOtbYGYjw/ddbWZ/rPe5v5rZiKa2E86sVhruz2+P4liLpCwlWElHpxL0zmzIZQSz4JxBMAXbvfWmnjuDoO/kacD3gP7uPhR4FLi+3jqOIWid9TPgZeB/gCLgtLBSbk4hcC7wdeCuBl6/FugHnOnupwNPhs//0d3PCnvAtgG+7u6TgbnAFe4+yN331q4kHDa+O4x1EHCWmdW2+GoHvOPuZwAzgX9rJNbBwDfc/bvAdQDufhrBbEKPm1l+M/va2HZ+Dzzi7mcBnzSzDpFWSQlWMs25wNPhZOnlwAzgrPC1OWEPzv0E7bGmhM8vBvrWW8fLHpxbWQyUu/vicOL1pYe8rzEvuHuNuy8DPlM5EiT+P9UOydab+m+kmb1rZosJkmZRM9s5C5ju7pvDdT0J1HZUqgT+Gj6e10TcL9VL2ucC48OYlgMfAc2d125sO18Cng4fj29mHSKtkhKspKOlBJVXQxpqX1ir/rymNfWWawjmcz70fTUNfKb2ffUvbji0yqv/mYbisUM+T1gpPgyMDivIPzew3obW05gqP3ABRjUH7199u2NYX5SDv0vqx9XUdnQBiKQ1JVhJR1OBPDOrG/Y0s7PMbDjBMGVxeJ6zG0FFNzsOMZSb2SlmlkXQ8eRwTAF+ZGFrLjPrzIGktcWC/qCj672/AihoYD3vAsPNrKuZRQiGdWccZiz1zQSuCGPqTzDB/gqCCecHmVmWmfUhtvZpbxF0tqJ2nSLpRglW0k5YMV0KjApv01kK3EbQP/J54D1gEUEi/qW7x+Mc4E0EQ6NTCZpiH45HCTrDvGdmi4Dvhn0x/0wwLP0CQRvGWuOAP9Ve5FT7ZNhh5WaCNl+LgPnufjQtvh4GIuEQ9UTg6nA4/S1gTRjbfcD8GNb1U+A6M5sDdDyKmERSlm7TERERiQNVsCIiInGgBCsiIhIHSrAiIiJxoAQrIiISB0qwIiIicaAEKyIiEgdKsCIiInGgBCsiIhIH/x/xtiAV4IFoowAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 504x360 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# averaged over the various requesting clients\n",
    "plot_test_accuracy_and_similarities(avg_diagnostics, np.mean(baseline_test_accuracies), \"Averaged over the requesting clients (client ordering is lost)\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_diagnostics_for_requesting_client(dstates: Mapping[str, Sequence[DiagnosticsState]],\n",
    "                                          requesting_client: int\n",
    "                                         ) -> Mapping[str, DiagnosticsState]:\n",
    "    \"\"\"\n",
    "    INPUTS:\n",
    "        dstates: dict\n",
    "            key: data set selection strategy (e.g., 'global' or 'similar')\n",
    "            value: list of DiagnosticsState. Each DiagnosticsState\n",
    "                   corresponds to one requesting client.\n",
    "        requesting_client: int\n",
    "    \n",
    "    OUTPUTS:\n",
    "        res: dict\n",
    "            key: data set selection strategy (same keys as input `dstates`)\n",
    "            value: DiagnosticsState corresponding to client `requesting_client`.\n",
    "            \n",
    "    The DiagnosticsState.similarities and DiagnosticsState.epoch_counts are reordered,\n",
    "    to maintain the identity of each dataset. For example, the dataset\n",
    "    (, similarity, and epoch_count,) of client 1 are at index 1 irrespectively of\n",
    "    who the requesting client is. This is not the case in the raw training results.\n",
    "    \"\"\"\n",
    "    res = {}\n",
    "    for key, dlist in diagnostics.items():\n",
    "        dstate = dlist[requesting_client]\n",
    "        similarities = np.array(dstate.similarities)\n",
    "        num_clients = similarities.shape[1]\n",
    "        \n",
    "        indexes_reordered = list(range(1, requesting_client+1)) + [0] + list(range(requesting_client+1, num_clients))\n",
    "        if requesting_client >= num_clients:\n",
    "            indexes_reordered = list(range(num_clients))\n",
    "        \n",
    "        similarities_reordered = np.array(dstate.similarities).T[indexes_reordered].T\n",
    "        epoch_counts_reordered = np.array(dstate.epoch_counts).T[indexes_reordered].T\n",
    "#         print(num_clients, indexes_reordered)\n",
    "#         print(similarities_reordered)\n",
    "        res[key] = dstate._replace(similarities=similarities_reordered,\n",
    "                                   epoch_counts=epoch_counts_reordered\n",
    "                                  )\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAdgAAAFNCAYAAACjcn5pAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdd3hVVdbA4d9Kr4QSeklARVpCkCYqTVFQEbECAsKg4uinyFhQURF1sKCOyFhGFAUVC5YRxTKKAoqCFKWKiEqokU5ILzfr++PchARSbkKSm7Le57lPzj37nH3WvUZW9j777C2qijHGGGPKl4+3AzDGGGNqIkuwxhhjTAWwBGuMMcZUAEuwxhhjTAWwBGuMMcZUAEuwxhhjTAWwBGtMFSAik0XklUq+5lQRedO93UpEkkXEtzJjMKYmswRrqj0RiReRNHeC+EtE5ohImLfjKoqI9BORXfn3qeqjqnq9t2JS1R2qGqaqrpOtS0SWiEipPouIxInIGhFJdf+MO9k4jPE2S7CmprhEVcOAOKALcK+X4zEeEpEAYAHwJlAPmAsscO83ptqyBGtqFFX9C/gfTqIFQEQCReQpEdkhIntF5D8iEpyv/C4RSRCRPSIyTkRURE51lxVojYnIWBFZlu99OxH5SkQOicgWEbk6X9lFIvKLiCSJyG4RuVNEQoHPgWbuFneyiDQ7rrs22h3DGHfMB0Tkvnz1BovIXBE5LCKbRWTS8S3i/ESkY74Y94rI5EKOyb2mn/t9hIjMdn8vu0Xkn7ndx7nfgfs7PSwi20TkQnfZNKA38Jz7sz3nwX+2foAfMENVM1R1JiDAuR6ca0yVZQnW1Cgi0gK4EPg93+4ngLY4SfdUoDkwxX38IOBO4HzgNGBAKa4VCnwFvAU0AkYAL4hIR/chs4EbVTUc6AR8o6op7vj2uLtkw1R1TxGXOAc4HTgPmCIi7d37HwSigTbuuEcVE2M4sAj4Amjm/vxfe/Dx5gLZ7uO7ABcA+bt9ewJbgEhgOjBbRERV7wO+A25xf7Zb3HEsFJF7irhWR2C9Fpy3db17vzHVliVYU1N8JCJJwE5gH04SQkQEuAH4h6oeUtUk4FFguPu8q4HXVHWjO/lNLcU1BwPxqvqaqmar6k/AB8CV7vIsoIOI1FHVw+7y0nhIVdNUdR2wDuicL+ZH3XXuAmaWEONfqvq0qqarapKq/ljcRUWkMc4fARNVNUVV9wHPcOw7A9iuqi+779nOBZoCjYuqU1UHq+rjRRSHAYnH7UsEwouL05iqzhKsqSmGuluK/YB2OC0rgIZACLBGRI6IyBGc1lxDd3kznKSca3sprhkF9Myt1133SKCJu/wK4CJgu4gsFZFepfxMf+XbTsVJRIXFnH/7eC2BP0p53SjAH0jI97lewmmlnxCbqqa6N8s6sCwZqHPcvjpAUhnrM6ZKsARrahRVXQrMAZ5y7zoApAEdVbWu+xXhHhAFkICThHK1Oq7KFJwEnatJvu2dwNJ89dZ1d4ve5I5llapeipOYPgLm54Z5cp+SBKBFvvctizrQHeMppax/J5ABROb7XHVU1dMu29J+vk1ArLu3IVese78x1ZYlWFMTzQDOF5E4Vc0BXgaeEZFGACLSXEQGuo+dD4wVkQ4iEoK7azmftcDlIhLiHvh0Xb6yhUBbERktIv7uV3cRaS8iASIyUkQiVDULOArkPgKzF2ggIhFl/HzzgXtFpJ6INAduKebYhUATEZnoHuwVLiI9i6tcVROAL4GnRaSOiPiIyCki0tfD+Pbi3B/21BKc72aCO8bcz/NNKeowpsqxBGtqHFXdD7wOPODedTfOoKcVInIUZ9DP6e5jP8dJyN+4jzn+H/VngEycpDEXmJfvOkk4g3+GA3twuk2fAALdh4wG4t3X/DvuwUiq+ivwNvCnuwu2WSk/4sPALmCb+7O8j9PiLOy7SMIZCHWJO76tQH8PrnEtEAD8Ahx2X6Oph/E9C1zpHmE8E0BEPi9s9LI7xkxgqPuaR4BxOF3+mR5ez5gqSWzBdWMKEhEFTlPV30s8uAoQkZuA4arqaQvTGFMJrAVrTDUjIk1F5Gx31+3pwB3Af70dlzGmID9vB2CMKbUAnFG9rXG6VN8BXvBqRMaYE1gXsTHGGFMBrIvYGGOMqQCWYI0xxpgKUC3uwfr4+GhwcHDJBxpjjKk1UlNTVVWrbEOxWiTY4OBgUlJSvB2GMcaYKkRE0rwdQ3GqbOY3xhhjqjNLsMYYY0wFsARrjDHGVABLsMYYY0wFsARrjDHGVABLsMYYY0wFsARrjDHGVIBq8RzsyZg+HRIbLGJe0vVs/2g7UUOjGBn+ChEHBzBpkrejM8YYUxFEJAj4Fmd9Zj/gfVV9UERa4yyQUR/4CRhdUWsP1/gWbGKDRTw6oTPb17aGpbB9bWsendCZxAaLvB2aMcaYipMBnKuqnYE4YJCInAk8ATyjqqcBh4HrKiqAGp9g5yVdD5cNgzevcHa8Nx+uutrZb4wxpkZSR7L7rb/7pcC5wPvu/XOBoRUVQ7VYri40NFTLOlWi9BNYWkhBX9AlVf+zG2NMTXA0PYvwQD9EpNzqFJFMYEO+XbNUdVa+cl9gDXAq8DzwJLBCVU91l7cEPlfVTuUWVD41vgUbNTQKxvQDuc3ZEbAMxvRz9htjjKlwOw+lMuDppby9cmd5V52tqt3yvWblL1RVl6rGAS2AHkD7QuqosJZWjU+wI8NfcbqFL93i7PC9B96b7+w3xhhT4ZrXDeaCjo2JbRHhleur6hFgCXAmUFdEcgf4tgD2VNR1a3yCjTg4gMkz1xHVdzPUjYK0ZZw5+HUiDg7wdmjGGFNjbdydyOjZP5KYloWPj/DPoTF0al55CVZEGopIXfd2MDAA2AwsBq50HzYGWFBhMdT0e7D5/Ra/h9Nbd8YvJJr0oz/i61vj/74wxhivWLfzCP/31k+8NLorHZtVTGIVkVRVDS2iLBZnEJMvTmNyvqo+LCJtOPaYzs/AKFXNqJD4alOCBYjtdysblj7HuDufY/aT/1cudRpjjIH4Aymsij/EVd1aApDlysG/AhsyxSXYqqDWJdj9h47SqGEHfPzqkpm63lqxxhhTTu5+fz2LNu9lyV39CA/yr/DrVfUEW+uyS8P6dThn8GhyMjdx+fWPezscY4yp1vYnZbD3aDoAky9uz8IJ51RKcq0Oal0LFiA5NZ3wOs5jT0lHNxIWElRudRtjTG2R5crh3KeXcFqjcF4d273Sr28t2CooLCSIIaPHgesPLhzxkLfDMcaYaiUj2wWAv68P91/cgXsvbOfliKqmWtmCBXC5cggIiSUn+wj79v9Cw/p1yrV+Y4ypieIPpDD61R95aEhHzm3X2KuxWAu2ivL19WHshJsgZzfnXX6ft8MxxphqoWndINo3qUNEcIC3Q6nyam0LFpxWbFCdnmSnxrNl2zraRjcr92sYY0x199veJF5c8gePXxFDoJ+vt8PJYy3YKszX14c7p04EDjDoivu9HY4xxlRJOw+l8t3W/Ww7UP4NnZqsVrdgcwXV60vGkbX8uG4NPWJPrbDrGGNMdbHnSBpb/kqif7tGAKRkZBMa6FfCWZXLWrDVwOP/mgQkMeQqa8UaYwzAQ59sYtIH60nPckYMV7XkWh1YC9YtvMkgkvcu5dMl33NR3zMq9FrGGFMVHUnNxMdHqBPkT0JiGpnZOUQ1qLINRGvBVhevzH4AcDFy1FRvh2KMMZUuLdPFxTOX8fAnvwDQNCK4SifX6sASrNuwi8+mftTFHNn1GXPeX+ztcIwxplLk5Di9mMEBvtzU7xTG9Ir2bkA1iHUR5/PNio2c16snIZFnkbL/qwq/njHGeNOf+5O5ed5PTL8yltgWdb0dTqlZF3E1cu6ZnWjeYSipBxbxxH8+8nY4xhhToRqEBhLk70tapsvbodRI1oI9zj/uSmDGUx0g9DTotoqooVGMDH+FiIMDmDSpUkIwxpgKs/1gCm/9uIO7B7XDx0dQVUTE22GVibVgq5mQdpvA53ZIWQVLYfva1jw6oTOJDRZ5OzRjjDlp3/9+kLdX7iD+oNNoqa7JtTqwFuxxomdEs335aTB/E5AAcgXECi2uWc3OSdsqJQZjjClP+5My2H0kjbiWdVFVDiRn0jA80NthnbSq3oKtsAQrIi2B14EmQA4wS1WfFZH6wLtANBAPXK2qh4urqzITrPQTWFpYSQiNTr2YSXeP4o7rh1RKLMYYUx6ueXkFOw+nsviOfvj51pyOy9qcYJsCTVX1JxEJB9YAQ4GxwCFVfVxE7gHqqerdxdVV6S3Yta3hvfmQ2giCd0GbibB9JySvAbIR//Z07HUuzz91MysWdyCxwSLmJV3P9o+22z1bY0yVkJyRTYCvDwF+Pvy2NwkBTmsc7u2wylVVT7AV9qeMqiao6k/u7SRgM9AcuBSY6z5sLk7SrTJGhr/iJNerrnZ2XD0Ktr3A5Bn/ZPnPm+jcfwIiPmz89nn69ojl3ql9efTve9i+8lS7Z2uMqRIS07K46NnveGbRbwC0bRxe45JrdVApfQUiEg10AX4EGqtqAjhJGGhUGTF4KuLgACbPXEdU3DboC1Fx25g8cx0RBwdwZlxb1n7zLK6Mjcyc8xlNT7+KnPQtkD0G3v7JqeCNQdDuJl75dSIuV84J9U+fDvfNXkT0jGiknxA9I5r7Zi9i+vRK/qDGmBorItifoXHNOK9dlfrntdap8EFOIhKGc1dzmqp+KCJHVLVuvvLDqlqvkPPGA+MBAgICumZkZFRonGUlfQS+K6q0AX4hrYlo1JI27drQ55wYEiWMV6b1cVrIc5fAmH7w3nwmz1zHtOsGVF7gxpga5c/9ydzz4QaevqozLeuHeDucSlHVu4grdHkEEfEHPgDmqeqH7t17RaSpqia479PuK+xcVZ0FzALnHmxFxnkyoi6PYnubfPdsg36Bjg/gn36EkCPBJB3YycH4RRyMT2LVF+6TpBnM7eRsv3kJdLqZF5dv49ZLF9Ek8sTZVKZPx+7zGmOKFeDnw76j6ew+klZrEmxVV5GDnATnHushVZ2Yb/+TwMF8g5zqq2qxaaIyBzmV1n2zF/HohM7Ftkhdrhw++molH3y6nLfnPQuHtxddoTTGJ6AJQXUaUb9RY1q1bkYqEaz96kK48g6Yt7jEVq8lZGNqh4TEND5dn8D1vdsA4MpRfH1qz3OtVb0FW5EJ9hycztMNOI/pAEzGuQ87H2gF7ACuUtVDxdVVlRNsaZPZCaOUgzZA3FQC9TDNsqI4mLCX1MR9ZKf/BTl7OPbVAfgCLqAnhAThU9eH9m3aERXdlI4dounV7XTOPzuWx95eVmLSP5nPYIypGp5dtJWXvv2Dr27vS/O6wd4Op9JV9QSLqlb5V0hIiNYUk1/5SgnZp4zpp4DzM2SfTn7lqxOOPXI0Rd9ZuExp09g59oSXXyH7fBRprPjFKAx19vk8qJx6pYYM7KNPvbxAl63erBmZWWWKSVX1iSecc6KeiVL6olHPROnkV77SJ56osK/NGON2JCVT/9iXpKqqmdku3XEwxcsReQ+QolUgRxX1spmcKllZWosntHpD9sFVV9Mi9g8+GfQt36zYwM/r/uT333exZ9c+dvy2GNJ3lhCJP0hDfAIakOMfAr71IKkr5EwDv7egx/s0iPuDn+5fRKumkQXO9KRbvDw+tzGmIFXlshd+ID3LxWcTeuNTi7qDC1PVW7CWYKuB0ia0E7uh46H3rYSHHaRP+EXs3LGXA3sPknT4EOnJR8hK2QLFTqYVBj4N8Amoh39QHTL9BZW6cLAPuO4A/4Vwzss06rGV+Kk/ExwUcNKfwRKyMcekZ7kI9PNBRPjxz4OEBPgR0yLC22F5XVVPsBU6itiUD+fZ3EXMS9rG9njn2dyRvZ1ncwszMvwVHn0vN5kBw8bCe/O5tdiE3BnmvwlpLSDga2j/IgF+h4nybcORg4dJOXqEzJREMpMT0OxdQBqwwKkgazAshn2LIeSJIKA++NbFNyAC/6A6BIXVIcknBSLrwpujgSXw9hToP47Xdm7hYdev+B43fVtig9yE3BqWbmd7dGsefa8zk2cuAiwhm9rjQHIGw2et4NpeUVzbK5qebRp4OyTjIUuw1YCTIAYwjXiYWMLBnERCvtqdkEc8Au/N587iEvLqNvDBc5DeEfz/C23m4e9/mCbSjOQjiaQlHyUr/SjpR3eSfjgROOA++wPnR+a58D9I+B/4PRQEUg/xjcDHPxy/wHAy/RRCIuCN84AlMO86iLuJF5bupGdUBDHtomjVNDIvMVtCNjVVg9AAzmhVl9aRVbahZopgXcSm1MmmLPdgWz4Vza7v28NnUyDzLPCbB1H/xc/vMA2lEalJR0lPSSY7I4mcrCQ0ezeQWkLk/iAR4BMBvsEgdSAzCvRt8Lkbmm0loHEiF3YeQJPG9WjRLJI2UY05vXUz3vn2N566q2uF3ke2JG7KasfBVB79bDOPXR5DvdATb7kYh3URmyqvolvIANfWe4VHv+wMI9yt5JEvw3vzmVRcK/mnaHhvDqS3hoAlEPNvgkIPEhfWjcOHjpKUmExqUhIZqSmkJf4Cro3AD04FOU/ALsjcBQvWfF1EVOEw1z2xx9xMCLyUx+4QXnjo3wSHhBASFkp4RCj16oVTv0EE2/5y8dMXvaBfrNNKbnYqjy4oupVc0a1qS+A119H0LFbFH+K3vUnWJVyNWQvWVIqKbiWfuArSX3DRtUQ23ctdMQ+xc9d+EvYeZv/+Ixw+dJQNaz+BxMIm/IgA8QdNAjyZntMfCAefUMQnBB+/YHz9Q/ANCCJdstGccDgSCzmPge8MOHUZwc32c+2ZI2jcsC5NGtWleeP6RLdsxBtfby5Vq7oyRnNbEq88B5Iz+P73A1wa1xyAtEwXwQG+Xo6qaiuuBVvMkqlTgRuA/e5DJ6vqZxUSoLefE/LkVZOegzWeKe2ztqV9ljfqmai8Y4C8c6Oeico7Zt/BRF22erPO/XCJPjzzvaKfRw5ppWGNB2pg3T7qF9JdfQI6Kb5tFEKLeH65uFeQQqR7u6PiF6sEddGAiHM0pOEAjWh+sUa2uUJbdBylfqedp7S6QvF93jne7x2l6xVa56pe+vTsj3Xex9/p18s36JZtuzU1LaNM31NFPyNtz1Qf8+CCjdru/s/1QFK6t0OpNijmOVigKXCGezsc+A3oAEwF7izqvPJ8WQvW1AiVcR+5qOeRo+K2ET8xvujj578DaU0gaCOcM5mIRnsZc9p49h9M5PDhJBKPpJCUlMLG9QvhSCGtap9GiG8DNCcNctJA04BkCs7y5YlAIAQkGDQc2AL0ggAXBAghQeH4+QfgFxBAQFAgAYGB7EnfTXZ6XdjdH1y3gN8b0HEBYa3+YkKfm6hfN4wGdcOJbFCHRg0ieP2rTTz/QC+4eli1bYVXZqs9JSOblIxsGtUJIik9i4TEdNrasnIeK809WBFZADwHnA0kq+pTFRoc1kVsaqmy/CNa2mRw0t3cxSRwlyuHqMdPY/fqtvD5Y5DRBQIWQeyLBNfdz7nNLiApKZXk5FRSU9JJT00nIy2dhJ0/QOqOEz+c1HcGi+Wkg6bjPIaVXrovtYBAnC72pk6Xu48f4hOI+Abi4+OPj18APr7+ZGgWmhMKSa1BXwK5CxpvxS8iiY7NOhIY5E9wcBDBwQGEhgYTFhbEht9T+OmLrtDzBfjuIzj/EvhuGqPv2sS9Y88hsm4d6keE5Y0wr+j/bmX5XZo+HY7U/4q34v8g/eAhgtrOYmQd7/2RUF1vBXiaYN1Lpn4LdAJuB8YCR4HVwB2qxU4EUGY2yMnUSqUd2AWlH9x10s8vX3U1vDefkb3XnXCsr68PY5q85AwcG577eNU/4b35/KPEVni/IpL4nwWOzczKJvrxdiT8dBp8/jhkxDmDzc6YSWjkPi5tfQXJSakkp6STmpJGamoG69d97F7MIvf+dYLT+e3bHP/gZuRkZ6GuTLLTU9GcLDR7L5B47KL6JPwF2X/Bui1FDU5zy10m8qtPgE944xF445H8BwTgJPoA5zXXPVfv3MPgdx6P/p8P028LxMfXH/Hxw8fXD18/f1JcqRAQCq93ApbAG2dC4/FMfzKJT95YQECAP4GB7ldQAH/uzOS3H2Ihthv8tJ3tGV159Ccfzr3mOcLnnUZ4WDChIUFEhIcQERZCSEggewJ/5tkJPQgZ/R6uz34g47ziB8CVdsBcRR8PVSaJ+4nI6nzvZ6mzElse95KpHwATVfWoiLwIPILzm/kI8DQwrswRFMNasMZUEZXxD1ZVaoUXefxlw2jadicfDf2cw4nJHDySzKEjSSQlpXHkaCrTX74T/tx74oeLbEW7theRlZFFZmYm2ZlZZGVlceCvVYW32n0b4+PfFHKynGSvWaDZkHMIqJh/b3zD6tPwiikk/vAOaVtXcGwBj3DAD8Tv2E/xR8QPxBfFBzQAsuvgLK89CIKTwd9FSEAoPr6++Pj4Oj99fTmSmUhOZigcagf6FPg8AI034hdxlPaN2+Pn54t/gB/+fn74+fvxY8IPpCc2gN+GQvYo8H8PYt4hrPlebuhxHYGB/gQF+BMUGEBgoD/BQQF8+sN2Pnm5G5x3P3z2KQwdBP97jr8/uJ77x/YlLCSI4KAAAvyddlxZbgeUpKQWrHvJ1IXA/1T1X4WURwMLVbVTmQIoKT5LsMbUHhWdxCs6gUM5JfHSHB+8Fy4fTtO2O/nsyq84kpzC0aOpHE1JIyk5jZunXQObC1kQrEVzBvUdT0Z6JhkZWWRmZZORmU3yaXH8teJ90tYXMnA1uBV16nbC5comx+VCc1zk5GSTmfIHZCeceLzUc7r21eX8cUC2+2cKkFno91f5BKezNPflCxwp8b+DRzUXP4q4qCVTm6pqgnv7H0BPVR1epgBKYF3ExtQipe0ar+hnpMvyTHVputLL5firh8F78/nbzHXEdYg+4fgn9oezfW1soQn884lTSEhM47lvfueBwR0I8nceu4me8VAxXfWfnnCN4rv2txVxfP4/EvbAkNE0bbeDz67+kqMpqaSlZ5Kckk5aRia3fXI7h/5oAT/8AzL7QcD/oMtLhDXax/D215CZmUVGZjZZmdlkZmWRleXif4tfhoRdJ36B9VvR9rRBuLJduFwuXNkucnJc7N7xPSTnG8SX2gjmwvZ4PL5NU0pnA6OBDSKy1r1vMjBCROJwuojjgRsr5Opgj+kYY6qXin4UqLwfEVuyZZ+2f+BzXR1/0ONzSnuNyj5e1bNH3U7meE9gy9WdvFBfX03p3bvgzquvhptvhtRUuOiiE08aO9Z5HTgAV155YvlNN8GwYbBzJ4wefWL5HXfAJZfAli1wYyF/4Nx/PwwYAGvXwsRC/vx69FE46yz44QeYPPnE8hkzIC4OFi2Cf/7zxPKXXoLTT4dPPoGnnz6x/I03oGVLePddePHFE8vffx8iI2HOHOd1vM8+g5AQeOEFmD//xPIlS5yfTz0FCxcWLAsOhs8/d7YfeQS+Pm4wSoMG8IF7zuF774XlywuWt2gBb77pbE+c6HyH+bVtC7Pc4xTGj4fffitYHhfnfH8Ao0bBruP+iu7VCx57zNm+4go4eLBg+XnnwQMPONsXXghpaQXLBw+GO+90tvv14wT2u2e/e5D3uzd9x3ASWy1n3pnvsn1FBlFDo7gy5GUy3jrAv3kJgEN+wdTPTsv73Zs+HRK/vJ553d50zjkzkJErhhEROo5Jn/Z16s/3u5d3jfM+YfvXh4m6tCUjF/QnYkcvJrV651h87t+96Q+mkLj05mMx5dbf/HYmzet8wu9eXv0XfM72L/cTdXEzRn464MT68/3u3TfxGR5d9VTh3fvtQ0743buPqMKP734n03C3bEv5uydLl9pUicYYU1PlJqBpK850/rib+Bi3z1/Lku5JJP0UQLgr00mu+c+ZBCze7ZwDsAJgO/RfBfQt+hoN/glL3H/cLbgIWhU2GxlMujUNlm4/sf4hvwKdi64/ZjZMdv9x9+mNRdYPELGjF5O738m8JsvZ3heimixn5KRPiTg4lrwpS48/fvKXzAt13w5ospyR3e8kYkevYq9TnVWPFqwNcjLGVHHpWS4yXTnUCfLnr8R09iWlE9uirrfDqtGq+mT/PiUfYowxpjhZrhwue+EH7vvvRgCaRARZcjXWRWyMMWWlqogI/r4+jOjRkqgGVbYxZbzAWrDGGFMGfyWmM3zWCn7a4cyyd22vaPq2bejlqExVYgnWGGPKICTQl8S0LA4kebKsoamNbJCTMcZ46EByBm8s385t552Gj4+Qk6P4+Ii3w6q1bJCTMcbUEEu27OfFpX+wac9RAEuupljWgjXGmGKkZGSz7UAKnZpHoKrsOpxGy/oh3g7LYC1YY4yp1m6fv5a/zVlFepYLEbHkajxmj+kYY8xxMrNzUJRAP1/uuOB0Dqdk5k3Ub4ynrAVrjDH5pGe5uOLFH3j6S2ce4raNw+nZpoGXozLVkbVgjTEmnyB/X3qfFklsiwhvh2KqORvkZIyp9fYdTef+jzYy+aL2REdW2TEz5jg2yMkYY6o4lyobdieyZW+St0MxNYi1YI0xtVJiWhafrNvDqDOjAMjIdhHoZwOZqhNrwRpjTBU0f9VOHvx4E7/vc1qtllxNebMWrDGm1kjPcrH3aDpRDULJduWwdV8y7ZvW8XZYpoysBWuMMVXE399cw99eW0WWKwc/Xx9LrqZC2WM6xpgazZWjCM68wTf1PYW0LBf+vta2MBXPfsuMMTVWUnoWw2ctZ84P8QD0bNOAfqc38m5QptawBGuMqbHCAv1oXjeYBmEB3g7F1EJFJlgReTrf9i3Hlc2uyKCMMaasDiZnMOn9dRxMzkBEmDG8C5fGNfd2WKYWKq4F2z/f9rjjyrpUQCzGGHPS9idn8NmGv/hpxxFvh2JqueISrBSxbYwxVUpKRjZfbPwLgHZN6vD9PedyfofGXo7K1HbFJVgfEQkXkYh823VEpA5gT2QbY6qMF5f8wS1v/cSuw6kARAT7ezkiY4qZaEJEdgE5FN56VVVtVZGB5WcTTRhjjpflyiExLYvIsEBSMrLZnHCUbtH1vR2WqURVfaIJm8nJGFPtqCqjZ68kLcvFezf2wsfH7mLVRiyH1vEAACAASURBVFU9wRY50YSItAQSVfWo+30f4FIgHviPqmZVSoTGGOOmqogIIsI1PVvlTSBhTFVU3D3Y94A6ACLSGfgvsA/oCTxf8aEZY8wxialZXPvqSj7bkADARTFNuTCmqZejMqZoxU2VGKKqu9zbo4BXVfUJEfEB1lV8aMYYc0xIoC+Z2TmkZrq8HYoxHvH0MZ1zga8BVDUHqPo3bo0x1V5iWhaPfb6ZtExn/uB3xp/JlV1beDssYzxSXIJdKiJvuWd0agB8AyAiTQC7/2qMqXCb9iTy6rJtrNh2EAARu99qqo/iEuwE4DPgL6C3qma69zcDHiipYhF5VUT2icjGfPumishuEVnrfl10MsEbY2qe9CwXK7cdAuCsUyJZeld/+tsE/aaURKSliCwWkc0isklEbnPvry8iX4nIVvfPehUWQ0U9puMedZwMvK6qndz7pgLJqvpUaeqyx3SMqT2mfryJd1bt4LtJ59IwPNDb4ZgqrLjHdESkKdBUVX8SkXBgDTAUGAscUtXHReQeoJ6q3l0R8RX3mM5hCt5rFfd7wZlootgnulX1WxGJLocYjTE1nCtHSc9yERrox839T6Hf6Q0tuZqToqoJQIJ7O0lENgPNcR437ec+bC6wBKjcBAt8DzQEPgDeAXaX0zVvEZFrgdXAHap6uLCDRGQ8MB4gIMCWmjKmpsrJUca8upLwID9eGHkGjcKDaHR6UIVdLysri127dpGenl5h1zDlKygoiBYtWuDvX7YpMN2NvS7Aj0Bjd/JFVRNEpMLuPxTbRezum74SGIZzv/YdYL6qerRMhftDLczXRdwYOIDTEn4Ep/l+/Eo9J7AuYmNqtrk/xBMa6McVZzSv8IFM27ZtIzw8nAYNGtigqWpAVTl48CBJSUm0bt26QJmIZAIb8u2apaqzjjsmDFgKTFPVD0XkiKrWzVd+WFUr5D5scS1Y3K3Ll0XkFZwk+xwQDjxd3HnF1Lc3d1tEXgYWlqUeY0z1diQ1k3s/3MC4c1rTPbo+Y86KrrRrp6enEx0dbcm1mhARGjRowP79+wsrzlbVbsWc64/TCztPVT90794rIk3drdemOBMoVYjiRhEjIj1E5BngZ5w+66uAf5X1Yu4Pk+syYGNRxxpjai4/Xx+27ktm237v9ExZcq1eyvLfS5yTZgObVTV/3voYGOPeHgMsOOkAi1BkghWR34FZwF7gOuAF4CAQIyKxJVUsIm8Dy4HTRWSXiFwHTBeRDSKyHmdB93+Uw2cwxlQDqZnZzPr2D1w5SligH5/f1puru7f0dliVLj4+nk6dOlVY/UuWLGHw4MEAfPzxxzz++OPlUm90dDQxMTHExsbSt29ftm/fnlcmIowePTrvfXZ2Ng0bNsyLY+/evQwePJjOnTvToUMHLrrIeUIzPj6e4OBg4uLi8l6vv/56ucQLnA2MBs497tHQx4HzRWQrcL77fYUorov4L5x7pRcDF1FwZicF+hRXsaqOKGT37NIGaIypGRb/up/HPv+VTs0jOOuUSPx9i+1AM+VgyJAhDBkypNzqW7x4MZGRkTz44IP885//5OWXXwYgNDSUjRs3kpaWRnBwMF999RXNmzfPO2/KlCmcf/753HbbbQCsX78+r+yUU05h7dq15RZjLlVdRuHLrQKcV+4XLESRv+Gqeo6q9na/+uTb7q2qxSZXY4wBZ83W3/YmAXBRTBM+v603Z50S6eWovC87O5sxY8YQGxvLlVdeSWqqs1D8ww8/TPfu3enUqRPjx48ndxDqzJkz6dChA7GxsQwfPhyAlJQUxo0bR/fu3enSpQsLFpzY0zlnzhxuueUWAMaOHcuECRM466yzaNOmDe+//37ecU8++STdu3cnNjaWBx98sMT4e/Xqxe7dBR8sufDCC/n0008BePvttxkx4lgbKyEhgRYtjk1xGRtbYidojVDqPyFFpL+IfF4RwRhjapb7/7uRYS8t52h6FiJCuyZ1vB3Sifr1O/H1wgtOWWpq4eVz5jjlBw6cWOaBLVu2MH78eNavX0+dOnV4wX29W265hVWrVuW1BhcudMaBPv744/z888+sX7+e//znPwBMmzaNc889l1WrVrF48WLuuusuSnraIiEhgWXLlrFw4ULuueceAL788ku2bt3KypUrWbt2LWvWrOHbb78ttp4vvviCoUOHFtg3fPhw3nnnHdLT01m/fj09e/bMK/u///s/rrvuOvr378+0adPYs2dPXtkff/xRoIv4u+++8+g7rA6KuwfbT0R+EZEjIjJHRE4XkRXADOC1ygvRGFOdqCpZrhwAbujThmmXxVAnqGzPL9ZULVu25OyzzwZg1KhRLFu2DHC6YHv27ElMTAzffPMNmzZtApwW38iRI3nzzTfx83Pu7H355Zc8/vjjxMXF0a9fP9LT09mxY0ex1x06dCg+Pj506NCBvXv35tXz5Zdf0qVLF8444wx+/fVXtm7dWuj5/fv3p1GjRixatIhrrrmmQFlsbCzx8fG8/fbbefdYcw0cOJA///yTG264gV9//ZUuXbrkjQrO7SLOffXu3bs0X2WVVtw92Gdw5iNeDlwIrAQeOm40ljHG5Ml25XDd3NW0aRjKg5d05NRGYZzaKMzbYRVvyZKiy0JCii+PjCy+vAjHj4oVEdLT07n55ptZvXo1LVu2ZOrUqXmTYXz66ad8++23fPzxxzzyyCNs2rQJVeWDDz7g9NNPL1BXbuIsTGDgsdmxcrufVZV7772XG2+8scS4Fy9eTGhoKGPHjmXKlCn8618F08GQIUO48847WbJkCQcPHixQVr9+fa655hquueYaBg8ezLfffkvXrl1LvKY3ichVqvpeSfuKUmwXsaouUtUUVX0fZwTxM2UP1RhT0/n5+tCuaThtGlbxpOplO3bsYPny5YBzv/Kcc87JS6aRkZEkJyfn3SPNyclh586d9O/fn+nTp3PkyBGSk5MZOHAg//73v/MS5c8//1ymWAYOHMirr75KcnIyALt372bfvqIfDQ0ODmbGjBm8/vrrHDp0qEDZuHHjmDJlCjExMQX2f/PNN3n3mZOSkvjjjz9o1apVmeKtZPd6uK9QxbVgI0Tk+OFnl+T+5aWqH3t6EWNMzXU0PYtpCzczvm8bTmkYxr0Xtvd2SFVe+/btmTt3LjfeeCOnnXYaN910EyEhIdxwww3ExMQQHR1N9+7dAXC5XIwaNYrExERUlX/84x/UrVuXBx54gIkTJxIbG4uqEh0dnXfPtjQuuOACNm/eTK9evQAICwvjzTffpFGjomcQbNq0KSNGjOD555/ngQeOLa7WokWLvJHC+a1Zs4ZbbrkFPz8/cnJyuP766+nevTvx8fF592BzjRs3jgkTJpT6c5QnEbkQ5+mZ5iIyM19RHSDb43qKmipRRN4o5jxV1Ws9vcjJsqkSjam69h1N56KZ3zFpYLtq8Vzr5s2bad/e/giobgr771bcajonQ0Q6A3HAw8CUfEVJwOKi5tA/XpEtWFUdXVSZMaZ2y8h28en6BC7r0pxGdYJYcld/wgKLnXnVmGpDVdcB60TkLVXNKms99qS3MabU3l+zi9vnr2PdrkQAS66mpurhXpT9NxH5U0S2icifnp5s/1cYYzziylH2Hk2nWd1ghndvxakNw4hrWbfkE42pvmbjTOm7BnCV9uQSW7AickISLmyfMaZmu+u9dYx4eQVpmS58fYSebRp4OyRjKlqiqn6uqvtU9WDuy9OTPUmUK4EzPNhnjKlhVBVV8PERRvRsxVmnRhLkb3eWTK2xWESeBD4EMnJ3qupPnpxcZIJ1r/LeFAgWkRiOTZpcBwgpc7jGmGohPcvFxHfWckZUXcb3OYXu0fXpHl3f22EZU5ly53vMv+asAud6cnJxLdiLgXFAC+B5jiXYJOCBok4yxtQMgX4++Pv54GNrp1aasWPHMnjwYK688soij4mOjmb16tVERnq2aMKcOXNYvXo1zz33XHmFWWuoav+TOb+4x3ReA14TkatVdf7JXMQYUz2kZmYzY9FWbujdhobhgcwcHlfrFyeft2Ee9319HzsSd9AqohXTzpvGyJiR3g7LVAIRmVLYflV92JPzPbmZ0khE6rgv9h8RWSkilbKWnjGmcu05ksYby7ez9DdnInZLrvMY/8l4tiduR1G2J25n/Cfjmbdh3knX/cgjj9CuXTvOP/98RowYwVNPPVWg/Ouvv6ZLly7ExMQwbtw4MjLybgHy5JNP0qNHD3r06MHvv/8OwCeffELPnj3p0qULAwYMKHZOYuOxlHwvF868/NGenuzJIKfxqvqciFyA0118EzALqNqzNBtjPJLlyuGHPw7St21DTm0UztJJ/WgUHuTtsCrFxC8msvavohf7XrFrBRmujAL7UrNSuW7Bdby85uVCz4lrEseMQTOKve7q1av54IMP+Pnnn8nOzuaMM84oMPF9eno6Y8eO5euvv6Zt27Zce+21vPjii0ycOBGAOnXqsHLlSl5//XUmTpzIwoULOeecc1ixYgUiwiuvvML06dN5+umnPf0qTCFUtcAXKCJPAR5PE+xJCzZ3LsULgddUdY2H5xljqoFXvtvGmFdX5i2MXluSqyeOT64l7ffUsmXLuPTSSwkODiY8PJxLLrmkQPmWLVto3bo1bdu2BWDMmDEF1mjNXcx8xIgReYsG7Nq1i4EDBxITE8OTTz6Zt9SdKVchQBtPD/akBbtORD4D2gL3iUgYx5KuMaYaUlWOpmUTEeLP2LOiObVRGG0bh3s7rEpXUkszekY02xO3n7A/KiKKJWOXlPm6Rc0B72l5/q773O1bb72V22+/nSFDhrBkyRKmTp1a5viMQ0Q2cCzf+QINceYn9ognLdG/AVOBHqqaCgQB15UuTGNMVaGq3PbOWq59bSXZrhyCA3w5v0Njb4dVJU07bxoh/gWfSgzxD2HaedNOqt5zzjmHTz75hPT0dJKTk/n0008LlLdr1474+Pi8+6tvvPEGffv2zSt/9913837mroKTmJhI8+bNAZg7d+5JxWfyDAYucb8uAJqpqsfDsUtswaqqS0TaAOcD04BgrIvYmGpLRLiwUxP2JWXg61O7BzGVJHe0cHmPIu7evTtDhgyhc+fOREVF0a1bNyIiIvLKg4KCeO2117jqqqvIzs6me/fu/P3vf88rz8jIoGfPnuTk5PD2228DMHXqVK666iqaN2/OmWeeybZt204qRgOqut29sk5v965vgfWenl/kcnV5B4g8B/gDfVS1vYjUB/6nqt3LGHOp2XJ1xpyctEwX93+0kT5tI7k0rrm3w/GqqrJcXXJyMmFhYaSmptKnTx9mzZrFGWfYBHlFqczl6vLVfxtwA85MTgCXAbNU9d+enO/JPdizVPUMEfkZQFUPiUhAmaI1xniFv6+w81AqfyWmezsU4zZ+/Hh++eUX0tPTGTNmjCXXquk6oKeqpgCIyBPAcqDcEmyWiPjgvtErIg2AnLLFaoypLBnZLl75bht/OzuakAA/3rqhJ36+dnenqnjrrbe8HYIpmVBwFR0Xx2Y1LJEnCfZ54AOgoYg8BFwNPFSaCI0xlW/j7qM8/eUWWtQL5tK45pZcjSm914AfReS/7vdDcZaw80hxk/37qWq2qr4uImuAATiZ+ypV3XgyERtjKoYrR9m0J5HYFnXpGlWPr27vyykNw7wdljHVkqr+S0SWAOfg5L+/qerPnp5fXAs2b0k6Vd0E2FPLxlRxzy76jf8s/ZOv7+hLy/ohllyNOQkiciawKXd5OhEJF5GeqvqjJ+cXl2Bt/L4x1YCqkpGdQ5C/L9eeFU1Ug1Ba1Av2dljG1AQvUnDt85RC9hWpuJsyDUXk9qJeJxGwMaacqCq3z1/HrW//jKoSGRbIFV1b1PpJ+quj66+/nl9++cXj41evXs2ECRMAZ0m6W265paJCq81E8z3Lqqo5eDZ2CUo40BcIw1qyxlRZIkLnFhGkZ+egCpZXy9f06dC9O/TPtyro4sWwahVMmlS+13rllVdKdXy3bt3o1q1byQcWIjs7Gz8/j/NEbfaniEzAabUC3Az86enJxbVgE1T1YVV9qLDXyURsjCm7tEwXUxZsZMWfBwEYe3Zr/t73FHxsVqZy1707XH21k1TB+Xn11c7+k5GSksLFF19M586d6dSpE++++y79+vVj9erVAISFhXH33XfTtWtXBgwYwMqVK+nXrx9t2rTh44+dxVyWLFnC4MGDT6i7qGXrpk6dyvjx47ngggu49tprT+4D1B5/B84CdgO7gJ7AeE9PtnuwxlQzirJs6wGaRgRzZpsG3g6nWps4EdYWvVodAM2awcCB0LQpJCRA+/bw0EPOqzBxcTCj+DUE+OKLL2jWrFneHMSJiYm8+OKLeeUpKSn069ePJ554gssuu4z777+fr776il9++YUxY8YwZMiQIusubtm6NWvWsGzZMoKD7R69J1R1HzC8rOcXl2BtUXVjqohsVw7vrdnFVV1bEBLgx6cTehMc4OvtsGqFevWc5LpjB7Rq5bw/WTExMdx5553cfffdDB48mN69excoDwgIYNCgQXnHBgYG4u/vT0xMDPHx8cXWvWvXLoYNG0ZCQgKZmZm0bt06r2zIkCGWXCtRkQlWVQ9VZiDGmKJ9t/UA9364gXohAQzq1MSSazkpqaUJx7qFH3gAXnwRHnyw4D3Zsmjbti1r1qzhs88+49577+WCCy4oUO7v7583UM3Hx4fAwMC87ezs7GLrLm7ZutDQCpu2t0oSkVdxVsTZp6qd3Pum4swvvN992GRV/awirm93uY2polSVHYdSiWoQSv92jfjgpl50jarv7bBqldzkOn++k1T79y/4vqz27NlD/fr1GTVqFGFhYcyZM6fcYrZl6wqYAzwHvH7c/mdU9amKvniRg5xE5H8i8g8RaVfRQRhjTvT4F78y5Lnv2Z+UAWDJ1QtWrSqYTPv3d96vWnVy9W7YsIEePXoQFxfHtGnTuP/++08+WLfcZet69+5NZGRkudVbHanqt0CZe2NFJFBErhGRySIyJffl8flFLVcnIk2AQe5XW+BH4Avga1VNLmvAZWHL1ZnaxJWj+PoI2w6k8O1v+7m2V5Q911qOqspydaZ0yrpcnYhEAwuP6yIeCxwFVgN3qOrhIs79AkgE1pBv0n9VfdqTmIu7B/sXTvN6jns1nZ7AhcAkEUkDvlTV6Z5cxBhTMlVl0vvrCfL35ZGhnWgdGUrryNp1z8yYUvITkdX53s9S1VklnPMi8AjOCnGPAE8D44o4toWqDipzcJ4c5J69Yrn7NUVEIoGBZb2oMeZEIkL90AAC/XxQVWu1GlOybFUt1Wwbqro3d1tEXgYWFnP4DyISo6obyhJcmQY5qeoBYF5ZzjXGHJOR7WLGoq1c1qU5bRuHc+9F1nVpTEUSkaaqmuB+exlwwupwIrIBp4XrB/xNRP4EMnDmh1BVjfXkWjaK2BgvSkrPZv6qnUQE+9O2cbi3wzGmRhGRt4F+QKSI7AIeBPqJSBxOAo0Hbizk1BOnyCqDEhOsiPiqqquk44wxnsnJUb78ZS8DOzYmMiyQr27vS/3QAG+HZUyNo6ojCtld4oLpqrodCixXl+R+Hw50ALZ7cv3i5iLO9buIPCkiHTyp0BhTvI/X7eHvb65h2e8HACy5GlN1vQjkf2omd7k6j3jSRRyLMxfjK+7RxK8C76jq0dJEaUxtpqocTMkkMiyQSzo3IzTQj3NOrd3PKNZmYWFhJCeX39OOU6dOJSwsjDvvvLPc6jRAIcvViYjHt1ZLbMGqapKqvqyqZwGTcPqwE0RkroicWqaQjallHv1sM5c+9z1J6Vn4+gjnd2hso4SrmfxTDppa408RmSAi/u7XbZTTcnWAcw9WRIaIyH+BZ3GeGWoDfAJUyPyNxtQUuX/8DurUlJFntiIkwMYVVlcPFbV8zklQVe666y46depETEwM7777bl7Z9OnTiYmJoXPnztxzzz0AvPzyy3Tv3p3OnTtzxRVXkJqaWu4xmQLyL1e3m3Jcri7XVmAx8KSq/pBv//si0qcUgRpTa2S7cnh44S80iQji5n6n0jWqHl2jymEZFlOuJk6cyNqS1qvLp1+/fiUeExcXxwxPVhEAPvzwQ9auXcu6des4cOAA3bt3p0+fPqxdu5aPPvqIH3/8kZCQEA4dcmb7u/zyy7nhhhsAuP/++5k9eza33nqrx/Gb0qnI5epyXauqy/LvEJGzVfV7VZ1Q1gsbU5P5+giHU7Ns1ZtqLj4+nu3bjw0YXbp0KQBRUVFER0efdP3Lli1jxIgR+Pr60rhxY/r27cuqVatYunQpf/vb3wgJCQGgfn1nHuqNGzdy//33c+TIEZKTkxk40Ob7qUgi0gL4N3A2zmM9y4DbVHWXJ+d7kmBnAmcct+/fhewzplbLduUw67s/ueKMFjSuE8Szw+Lw8bH7rFWZpy1NcGbaKmru9rIqqr6iZvIaO3YsH330EZ07d2bOnDksWbKkXOMxJ3gNeAu4yv1+lHvf+Z6cXNxqOr1E5A6goYjcnu81FSjxz3IReVVE9onIxnz76ovIVyKy1f3T+sxMjbH7SBozv97KJ+v2AFhyNSXq06cP7777Li6Xi/379/Ptt9/So0cPLrjgAl599dW8e6y5XcRJSUk0bdqUrKws5s2zyfQqQUNVfU1Vs92vOUBDT08ubpBTABCG08oNz/c6ClzpQd1zcFbiye8enNV4TgO+dr83ptpSVVbHO//4RTUI5cuJfbm+dxsvR2UqwoMPPljudV522WXExsbSuXNnzj33XKZPn06TJk0YNGgQQ4YMoVu3bsTFxfHUU87SpY888gg9e/bk/PPPp107W0m0EhwQkVHuwb6+IjIKOOjpyUUuV5d3gEhU7qwWpVXIMkFbgH6qmiAiTYElqnp6SfXYcnWmqnrrxx1M/u8G/nvzWXRpZR0y1YEtV1c9lXW5upMhIq1wFmzv5d71Pc49WI9yYpH3YEVkhqpOBJ4TkROysKoOKUO8jXMnWXYn2UbFXH887uHQAQE2042pWlIzswkJ8OPyM5rj7yt0blHX2yEZY8qZqu4AypLrgOIHOb3h/vlUWSs/Ge41/WaB04L1RgzGFGbap7+w4s9DfHDTWQT5+3JVt5beDskYUwFEpA3O/A9n4owiXg78Q1U9mmyiuAXX14iIL3CDqo4qj2CBvblLBbm7iPeVU73GVJquUfUJ9PPFJmIypsZ7C3geZ1k7cJ6JfRtnwokSFTuTk3sVnYYiUl59tB8DY9zbY4AF5VSvMRUmy5XD45//yoc/OY++DerUhDsHno6/rydrZRhjqjFR1TfyjSJ+E6cl6xFPnoONB74XkY9xVhIAQFX/VWxUha/D9zgwX0SuA3Zw7NkiY6osHxF+2n6YbFcOl9vT38bUJotF5B7gHZzEOgz4VETqA6jqoeJO9iTB7nG/fHAe0/FIEevwAZznaR3GeEtOjvLWyh0M7dKcsEA/Xr+uB0H+NiuTMbXMMPfP4xdlH4eTcIt9Jq/EBKuq5T/DtTFV3Oa/jjJlwUZcOcqYs6ItuZoKl3/JuSlTptCnTx8GDBhQ6nrWrl3Lnj17uOiiiwotf+yxx5g9eza+vr7MnDnTplsshqq2PpnzS0ywItIQZ5m6jkBQvgufezIXNqaqUVX+2J/MqY3C6dgsggX/dw6dmtfxdlimFnr44YfLfO7atWtZvXp1oQn2l19+4Z133mHTpk3s2bOHAQMG8Ntvv+Hra39A5icik1R1unv7KlV9L1/Zo6o62ZN6PBmlMQ/4FWgNPIRzT3ZVqSM2pop77ft4Lnz2O7buTQIgpkWErdlqKsTrr7+eN4PT6NGjTygfO3Ys77//PgBr1qyhb9++dO3alYEDB5KQkAA4K/vcfffd9OjRg7Zt2/Ldd9+RmZnJlClTePfdd4mLiyuw/B3AggULGD58OIGBgbRu3ZpTTz2VlStXVvwHrn7yr6Bz73Flx89QWCRPEmwDVZ0NZKnqUlUdh/NMkDE1QrYrB4DLujTn7kHtaNMwzMsRmco07KXlvLd6J+CMGB/20nL++7MzYjwt08Wwl5bnzS99ND2LYS8t54uNTpI7lJLJsJeWs+iXvQDsS0ov8XqbNm1i2rRpfPPNN6xbt45nn322yGOzsrK49dZbef/991mzZg3jxo3jvvvuyyvPzs5m5cqVzJgxg4ceeoiAgAAefvhhhg0bxtq1axk2bFiB+nbv3k3Llsee227RogW7d+/25GuqbaSI7cLeF8mTQU5Z7p8JInIxzoCnFp5ewJiq7LHPNrN1XzKzx3SjXmiAzSNsKtw333zDlVdeSWRkJHBsKbrCbNmyhY0bN3L++c7iLS6Xi6ZNm+aVX3755QB07dqV+Pj4Eq9d2NS41ktTKC1iu7D3RfIkwf5TRCKAO3CWqasD/MPTCxhTlTWrG0ymK4fsHMXf1/6hqY3evbFX3ra/r0+B98EBvgXe1wnyL/C+fmhAgfeNwvOGqRSpqKXoijq2Y8eOLF++vNDywMBAAHx9fcnOzi6xvhYtWrBz586897t27aJZs2YexVLLdBaRozit1WD3Nu73Jf9Hdiuxi1hVF6pqoqpuVNX+qtpVVT8uW8zGeFe2K4d/f72VZVsPADDmrGgevKSjTRphKs15553H/PnzOXjQWZQldym6wpx++uns378/L8FmZWWxadOm/2/vzqOzqu88jr+/2QhL2KJgFCqKghKpWEC7WAVbndp2ug01Y7XV6XRs56jVdjptnbYz2hnP0Wptx9Yl1FIso1D3SrWKyqYSZRUCCCICgmACQSBsCUm+88e9iY9pNiD3uc/yeZ2Tw3Of5d7v5eY833x/997ft8P1FxUVUVtb2+ZrX/jCF5gxYwZ1dXVs2LCBdevWcfbZZx/hnmQud891977uXuTueeHj5uX8rq6no8n+f0MHpbC7f/cwYxaJXUOT88Rr77DrwCHOPfWYuMORLFRaWspPfvITzj//fHJzcznrkiFCVwAAFjNJREFUrLOYOnVqm+8tKCjgkUce4bvf/S67d++moaGB66+/ntLS0nbXP3HiRG655RbGjBnDDTfc8IHzsKWlpVxyySWMGjWKvLw87rrrLl1BHKF229WZ2RVtvhBy9/sjiagNalcnR8PdeapyG39Xehz5uTns3n+Ifr26/EeoZBi1q0tPcbSrO1odTfaftAQqEqWFG3ZyzYPL+MWkD3PJuKFKriKSFJ32gzWzmbQxVHyE/WBFkqZqz0EG9y3knJOLmfpP4znv1GPjDklEskjK9oMVORqT56/nN7Pf5Nnrz+P4/j2ZMHJQ3CGJSJbpsB9s+O+85IUjcnSab4H4TGkJew82cEyfHnGHJCnocG6Vkfi1d61Qquv03gQz+7yZLTOznWa2x8xqE+4JEkkZtz+7lp8+sRKADxX34vsXjaQgT7ffyAcVFhZSU1OTtl/a2cbdqampobCwy7efpoyuTDTxa+ArQKXrN1JSWEOT0xj+5OaoOpG2DRkyhC1btrB9+/a4Q5EuKiwsZMiQ9JtAsN3bdFreYDYH+JS7NyUnpL+l23SkLU1Nzh8WbOTjw4s5vaSvhv1Eskza3qaT4IfA02Y2D6hrftLd74gsKpEu2HPwEPfMXc+7uw/wk8+NUnIVkZTSlQp2FrAXqARaqthkNmJXBSvN3J2K9TV8bHgxZsbWXQco6Veo5CqShTKhgh3o7hdFHolIF8xZW803py7mnss+wsWjSzi+f8+4QxKRFGVmU4DPA9Xufkb43EDgT8Awgv7ml7j7e1FsvyuXWD5vZkqwEqvag0HXxAkjBnHHJWdy4ajBMUckImlgKn/bIP3HwAvufirwQrgcia4MEdcCvQnOvx4iaNfj7t43qqBa0xBxdrt33nqmVWzi6es+Sb+emuZQRAJdGSI2s2HAXxIq2LXABHffZmYlwFx3HxlFfJ0OEbt7URQbFumqj55cTNWeg/TQPa0icvQGu/s2gDDJRjbNW0dzEZ/m7mvM7CNtve7uS6MKSrKbu3PXnDcxM66eeApjhvZnzND+cYclIqknz8wWJyxPdvfJsUXTSkcV7PeBq4BftvGaAxdEEpFkPTPjzeq9mJnubRWRjjS4+7jD/EyVmZUkDBFXRxEYdOEcbCrQOdjM5+48tHgz5404lpJ+PalvaNI0hyLSoSM8B3sbUOPut5jZjwnulPlhFPG1+w1mZuPN7LiE5W+Y2Z/N7M7wMmeRblO1p46bZq5mWsUmACVXETlqZjYdqABGmtkWM/tn4BbgQjNbB1wYLkez/fYqWDNbCnza3Xea2XnADOBaYAxwurtPiiqo1lTBZq7VW/cw6vjggvS179Zy6qA+5GgeYRHpglSfaKKjMiHX3XeGj8sITh4/6u4/A06JPjTJdE+t2MZn73yRBW/uAGDkcUVKriKSMTpMsGbWfBHUp4DZCa91ZQYokTbVNwQzbn561CB++rnTGTdMZxxEJPN0lGCnA/PM7M/AAeBFADM7BdidhNgkA5XPW89X7nmZuoZGeuTl8q1PnqzzrSKSkdqtRN39ZjN7ASgBZiX0gs0hOBcrctiGH9uHUSV9aWxK/avXRUSOhm7TkUg1NTlTF2ykb898Jo1Nv4bJIpK60vkiJ5Fu8dzqKua/sT3uMEREkkoVrETi6cptnHvqMfQtzGdvXQO9C3I1I5OIdCtVsJJ1NuzYx7XTlzH15Y0A9OmRp+QqIllHFax0m3d2HeCEsAH6q2/VMPbEAeTl6m84EYmGKljJCo8v28KE2+aw8p3gDq5zTi5WchWRrKZvQDkqzSMgF4wczLfPG84pg/rEHJGISGrQELEcsftefIsF62u47xvjNMWhiCSdhoglYxXm59KzIJe6cOpDERF5nypY6TJ350+LNlPSvyfnjzi2ZXhYVwiLSBxUwUrGqG9sYsrLG3hs6RYgSKxKriIibVMFK516ad0Ozj5pIAV5OWyvraO4d4HOuYpI7FTBSlpbtXU3l//+Vaa9sgmAY4t6KLmKiHSBKlhp0+79h+jXKx+Av1Zu44LTB9EjLzfmqERE3qcKVtLOw4s388lfzGbzzv0AXDy6RMlVROQwKcHK3/jY8GK+OOYE+ocVrIiIHD4NEQsAf6zYyBtVtfzPl0bHHYqISJek+hBxXhwbNbONQC3QCDS4+7g44pD3Ve+pY9uug9Q3NFGQp4ENEZGjFUsFGybYce6+oyvvVwUbjSeXb+XkY3pzxgn9aGxyckyTRohI+kj1ClalSpbaV9fAzU+t5g9hz9bcHE0aISLSneKqYDcA7wEOlLv75I7erwq2+1Ru2c0ZJ/TFzNiwYx9DB/RUWzkRSUuqYNv2CXf/CHAxcLWZndf6DWZ2lZktNrPFDQ0NyY8wAy3auJO//+1LPLr0HQBOOqa3kquISERiv4rYzG4E9rr77e29RxXs0Tl4qJHC/Fzcnf97ZROTxg6lZ4HuaxWR9KYKthUz621mRc2PgYuAlcmOI1s8tGgzn75jHrv212NmfP1jw5RcRUSSII7bdAYDj4cX1OQBD7r7MzHEkRVGHd+XsScOwNAFTCIiyRT7EHFXaIj48Dy0aDPb99Zx9cRT4g5FRCQyGiKWpFu4cScV62tobEr9P55ERDKVKtgM8cLrVYwYXMTQgb04eKiRgtwctZUTkYymClYi996+eq6b8Rr3zFsPQGF+rpKriEjMVMGmsU01+zixOPjjbfnmXZxWUqS2ciKSNVTBSiReXLedibfPZfaaKgDOHNpfyVVEJIGZbTSzSjN7zcwWJ3v7sXTTkSPX1OTk5BjnnFTMtRecyvhhA+MOSUQklU3samOZ7qYKNo08vHgzX7775eAiprwcvnfhCIoK1RRdRCQVKcGmkWOLelDcpwcHDzXGHYqISCrIa56zPvy5qtXrDswysyVtvBY5XeSU4mYu30pdQxOTxg4BwN3VVk5EhM4vcjKz4919q5kNAp4DrnX3+cmKTxVsCnN3Hlq8mceWbqH5DyElVxGRrnH3reG/1cDjwNnJ3L4q2BS0YP0OSkv60a9XPrv3H6J3j1y1lRMRaaWjCjZsJpPj7rXh4+eAnydz7nt9a6eYqj0HuXLKIn47Zx0A/XrlK7mKiBy+wcBLZrYcWAg8lezGMqpgU0TN3jqK+/QAYO7aas45qVht5UREOqCJJqRTs9dU8YlbZ7Ps7fcAmDBykJKriEiaU4JNAeOGDeSrY4cyrDhl/xATEZHDpCHimDyx7B2eqtxG+eVjNTG/iMgR0BCxtKmuoZHag4eorWuIOxQREYmAKtgkmr2mihwzJowchLvjjqpXEZEjlOoVrCb7T5LGJue2Z9/gmD4FTBg5CDNDc0aIiGQuVbARW7FlFyOPC/q0btt9gIG9C9RWTkSkG6R6BatzsBHauGMfX757AeXz3gKgpF9PJVcRkSyhCjYC++sb6FUQjL4/umQLF5YOpq/ayomIdCtVsFnmudVVnHvrHDbuCP4g+IexQ5RcRUSykBJsNzvjhL58fHgxvXpoKFhEJJtpiLgbPF25jYUbdnLjF0rjDkVEJGtoiDgLrKvay7LNu9inSSNERCSkCvYILVi/g6Ie+Ywe0o+GxiYcyFdbORGRpFEFm4HqGhr594dX8Ovn3wAgLzdHyVVERD5AFexhWL99LycV9yYnx3ijqpYhA3q23I4jIiLJpQo2Q7y+bQ+f+fV8Hlj4NgAjBhcpuYqISLuUYDvR0NgEwGnHFfFvF43kc6NLYo5IRETSgRJsB2atepcLfzWfnfvqMTO+c/5wBvYuiDssERFJA0qwHRg6sBdDBvSkvqEp7lBERCTN6CKnVuasqWbNu7X864ThSdmeiIgcGV3klGZmrX6XJ5dvpa6hMe5QREQkjamCBZa9/R7FvXvwoeJe7K9vIMeMwnzNJSwikspUwaa4/fUNfHPqIn7x7BoAehXkKbmKiMhRy9oKtnrPQQb1LQRg8cadjDiuSG3lRETSiCrYFPTa5l2cd9sc/lq5DYBxwwYquYqISLfKqgTbXK2XHt+Xy885kbEnDog5IhERyVRZk2CfX13F6Btn8eCrm8jPzeFHF5/GtdOX8fiyLQAcqG+krLyCmcu3ArDn4CHKyit4ZmVQ5e7cV09ZeQXPr64CoLr2IGXlFcxdWw3A1l0HKCuv4KV1OwB4u2Y/ZeUVvPJWDRDMY1xWXsGSTTsBWPtuLWXlFSzfvAuAVVt3U1ZewaqtuwFYvnkXZeUVrH23FoAlm3ZSVl7B+u17AXjlrRrKyit4u2Y/AC+t20FZeQVbdx0AYO7aasrKK6iuPdiy/2XlFezcVw/AMyu3UVZewZ6DhwCYuXwrZeUVHKgPrp5+fNkWysorOBTOZPXw4s2UlVe0/H9OX/g2l933SsvytIqNXDFlYcvylJc28K37F7UsT56/nu9MW9KyfPfcN7nmwaUty3e+sI7rZyxrWb5j1lp+8PDyluVbn1nDDY+taFm++anV/OyJlS3LN81cxU0zV7Us/+yJldz81OqW5RseW8Gtz6xpWf7Bw8u5Y9baluXrZyzjzhfWtSxf8+BS7p77Zsvyd6YtYfL89S3L37p/EVNe2tCyfMWUhUyr2NiyfNl9rzA9nFYToKy8gocXbwbgUGMTZeUV+t3T7x6Qnb972SJrEmxhfi6Os79et9+IiEj0suoip6YmJyfHuiEiERGJmy5ySiFKriIikixZlWBFRESSRQlWREQkAkqwIiKSkczsM2a21szeNLMfJ3v7SrAiIpJxzCwXuAu4GBgFXGpmo5IZgxKsiIhkorOBN939LXevB2YAX0xmALEk2LjLdhERyXgnAJsTlreEzyVN0hNsKpTtIiKSEfLMbHHCz1UJr7V1X2ZSJ37IS+bGQi1lO4CZNZftqzv8lIiIyAc1uPu4dl7bAgxNWB4CbI0+pPfFMUQce9kuIiIZbxFwqpmdZGYFwD8CTyYzgDgq2C6V7WGpfxVAQUFB1DGJiEgGcfcGM7sGeBbIBaa4+6pOPtat4kiwXSrb3X0yMBnAzJrM7EA3bDsPaOiG9aQT7XN2yMZ9huzcb+3z+3p29CF3fxp4OpKIuiDpk/2bWR7wBvAp4B2CMv5ryfjLwswWdzBen5G0z9khG/cZsnO/tc/pI+kVbCqU7SIiIlGLY4g49rJdREQkatk2k9PkuAOIgfY5O2TjPkN27rf2OU2kRcN1ERGRdJNtFayIiEhSZEWCzda5j81so5lVmtlrZrY47niiYGZTzKzazFYmPDfQzJ4zs3XhvwPijLG7tbPPN5rZO+Gxfs3MPhtnjN3NzIaa2Rwze93MVpnZdeHzGXusO9jnTD/WhWa20MyWh/t9U/j8SWb2anis/xROHpHSMn6IOJz7+A3gQoJ7cBcBl7p7xk/NaGYbgXHuviPuWKJiZucBe4E/uvsZ4XO/AHa6+y3hH1QD3P1HccbZndrZ5xuBve5+e5yxRcXMSoASd19qZkXAEuBLwJVk6LHuYJ8vIbOPtQG93X2vmeUDLwHXAd8HHnP3GWZ2L7Dc3e+JM9bOZEMFG3vLIomOu88HdrZ6+ovA/eHj+wm+lDJGO/uc0dx9m7svDR/XAq8TTLGasce6g33OaB7YGy7mhz8OXAA8Ej6fFsc6GxJsNs997MAsM1vSqstEphvs7tsg+JICBsUcT7JcY2YrwiHkjBkqbc3MhgFnAa+SJce61T5Dhh9rM8s1s9eAauA5YD2wy92bZ3NKi+/xbEiwsbcsitEn3P0jBK0Brw6HFiUz3QMMB8YA24BfxhtONMysD/AocL2774k7nmRoY58z/li7e6O7jyGYSvds4PS23pbcqA5fNiTY2FsWxcXdt4b/VgOPE/yiZoOq8PxV83ms6pjjiZy7V4VfSk3A78jAYx2ej3sUeMDdHwufzuhj3dY+Z8Oxbubuu4C5wEeB/uFUu5Am3+PZkGBjb1kUBzPrHV4YgZn1Bi4CVnb8qYzxJHBF+PgK4M8xxpIUzUkm9GUy7FiHF778Hnjd3e9IeCljj3V7+5wFx/pYM+sfPu4JfJrg/PMcYFL4trQ41hl/FTFAeBn7r3l/7uObYw4pcmZ2MkHVCsGUmA9m4n6b2XRgAnAMUAX8F/AE8BDwIeBt4KvunjEXBbWzzxMIhgwd2Ah8u/ncZCYws3OBF4FKoCl8+j8Izklm5LHuYJ8vJbOP9YcJLmLKJSgCH3L3n4ffaTOAgcAy4HJ3r4sv0s5lRYIVERFJtmwYIhYREUk6JVgREZEIKMGKiIhEQAlWREQkAkqwIiIiEVCClYxkZseZ2QwzW29mq83saTMbEXdc7TGz483skc7f2eZnrzSz4xOW7zOzUd0XXfTMbG/n7xJJL7pNRzJOeIP+AuB+d783fG4MUOTuL8YaXATMbC7wA3ePrCWhmeUlzAMbxfr3unufqNYvEgdVsJKJJgKHmpMrgLu/5u4vWuA2M1tpQa/cMgAzm2Bm88zsITN7w8xuMbPLwr6UlWY2PHzfVDO7J+zT+ZaZnR9OuP66mU1t3l5iRWZmk5pfCz9/p5ktCD8/KXx+mIX9XcOJzm8Pt7vCzK4Nn/9PM1sUxj453JdJwDjgAQt6g/Y0s7lmNi78zKXhelaa2a2J8ZnZzRb03HzFzAa3/k+0oO/oZDObBfzRgj6dfwjXt8zMJobvu9LMfpvwub+Y2YSOthPOrFYR7s9/H8WxFklZSrCSic4g6J3Zlq8QzIJzJsEUbLclTD13JkHfydHA14ER7n42cB9wbcI6BhC0zvoeMBP4FVAKjA4r5c6UAOcCnwduaeP1q4CTgLPc/cPAA+Hzv3X38WEP2J7A5939EWAxcJm7j3H3A80rCYeNbw1jHQOMN7PmFl+9gVfc/UxgPvAv7cQ6Fviiu38NuBrA3UcTzCZ0v5kVdrKv7W3nf4F73H088G4n6xBJS0qwkm3OBaaHk6VXAfOA8eFri8IenHUE7bFmhc9XAsMS1jHTg3MrlUCVu1eGE6+vavW+9jzh7k3uvhr4m8qRIPHf2zwkmzD130Qze9XMKgmSZmkn2xkPzHX37eG6HgCaOyrVA38JHy/pIO4nE5L2ucC0MKY1wCags/Pa7W3nE8D08PG0TtYhkpaUYCUTrSKovNrSVvvCZonzmjYlLDcRzOfc+n1NbXym+X2JFze0rvISP9NWPNbq84SV4t3ApLCC/F0b621rPe055O9fgNHIB/cv0b4urK+BD36XJMbV0XZ0AYhkNCVYyUSzgR5m1jLsaWbjzex8gmHKsvA857EEFd3CCGKoMrPTzSyHoOPJ4ZgFfMfC1lxmNpD3k9YOC/qDTkp4fy1Q1MZ6XgXON7NjzCyXYFh33mHGkmg+cFkY0wiCCfbXEkw4P8bMcsxsKF1rn/YyQWcrmtcpkmmUYCXjhBXTl4ELw9t0VgE3EvSPfBxYASwnSMQ/dPcozgH+mGBodDZBU+zDcR9BZ5gVZrYc+FrYF/N3BMPSTxC0YWw2Fbi3+SKn5ifDDis3ELT5Wg4sdfejafF1N5AbDlH/CbgyHE5/GdgQxnY7sLQL67oOuNrMFgH9jiImkZSl23REREQioApWREQkAkqwIiIiEVCCFRERiYASrIiISASUYEVERCKgBCsiIhIBJVgREZEIKMGKiIhE4P8BfrISJqrqLrAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 504x360 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Individually per requesting client\n",
    "for requesting_client in range(len(diagnostics['similar'])):\n",
    "    plot_test_accuracy_and_similarities(\n",
    "        get_diagnostics_for_requesting_client(diagnostics, requesting_client),\n",
    "        baseline_test_accuracies[requesting_client],\n",
    "        \"Requesting client: {}\".format(requesting_client)\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "EaE23XWAxE2x"
   },
   "source": [
    "# Questions\n",
    "\n",
    "1. Does the net.init also initialize the output layer based on batch size?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "Jax federated learning",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}