Code/RL/retrace_without_bias.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6tw4dti2pfwK"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3SA0CDifItbZ"
      },
      "source": [
        "### Library"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QEOTTPlTKllj",
        "outputId": "b4ed3f0c-9b07-4dd6-d403-fe91ba307aed"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: gym in /usr/local/lib/python3.8/dist-packages (0.25.2)\n",
            "Requirement already satisfied: numpy>=1.18.0 in /usr/local/lib/python3.8/dist-packages (from gym) (1.21.6)\n",
            "Requirement already satisfied: cloudpickle>=1.2.0 in /usr/local/lib/python3.8/dist-packages (from gym) (1.5.0)\n",
            "Requirement already satisfied: gym-notices>=0.0.4 in /usr/local/lib/python3.8/dist-packages (from gym) (0.0.8)\n",
            "Requirement already satisfied: importlib-metadata>=4.8.0 in /usr/local/lib/python3.8/dist-packages (from gym) (4.13.0)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.8/dist-packages (from importlib-metadata>=4.8.0->gym) (3.11.0)\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting pygame\n",
            "  Downloading pygame-2.1.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (21.8 MB)\n",
            "\u001b[K     |████████████████████████████████| 21.8 MB 1.4 MB/s \n",
            "\u001b[?25hInstalling collected packages: pygame\n",
            "Successfully installed pygame-2.1.2\n"
          ]
        }
      ],
      "source": [
        "!pip install gym\n",
        "!pip install pygame"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "wsNO3FuyIuOZ"
      },
      "outputs": [],
      "source": [
        "import gym\n",
        "from gym.spaces import Discrete as DiscreteSpace\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import random\n",
        "import torch\n",
        "from collections import deque, namedtuple\n",
        "import numpy as np\n",
        "import torch.nn.functional as F\n",
        "from torch.autograd import Variable\n",
        "from torch import multiprocessing as mp\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bEwiIU77IbFw"
      },
      "source": [
        "### Replay Memory"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "fRFg13-oIWIf"
      },
      "outputs": [],
      "source": [
        "Transition = namedtuple('Transition', ('states', 'actions', 'rewards', 'next_states',\n",
        "                                       'done', 'exploration_statistics'))\n",
        "\n",
        "class ReplayBuffer:\n",
        "    \"\"\"\n",
        "    Replay buffer for the agents.\n",
        "    \"\"\"\n",
        "    def __init__(self):\n",
        "        self.episodes = deque([[]], maxlen=REPLAY_BUFFER_SIZE)\n",
        "\n",
        "    def add(self, transition):\n",
        "        \"\"\"\n",
        "        Add a new transition to the buffer.\n",
        "        Parameters\n",
        "        ----------\n",
        "        transition : Transition\n",
        "            The transition to add.\n",
        "        \"\"\"\n",
        "        if self.episodes[-1] and self.episodes[-1][-1].done[0, 0]:\n",
        "            self.episodes.append([])\n",
        "        self.episodes[-1].append(transition)\n",
        "\n",
        "    def sample(self, batch_size, window_length=float('inf')):\n",
        "        \"\"\"\n",
        "        Sample a batch of trajectories from the buffer. If they are of unequal length\n",
        "        (which is likely), the trajectories will be padded with zero-reward transitions.\n",
        "        Parameters\n",
        "        ----------\n",
        "        batch_size : int\n",
        "            The batch size of the sample.\n",
        "        window_length : int, optional\n",
        "            The window length.\n",
        "        Returns\n",
        "        -------\n",
        "        list of Transition's\n",
        "            A batched sampled trajectory.\n",
        "        \"\"\"\n",
        "        batched_trajectory = []\n",
        "        trajectory_indices = random.choices(range(len(self.episodes)-1), k=min(batch_size, len(self.episodes)-1))\n",
        "        trajectories = []\n",
        "        for trajectory in [self.episodes[index] for index in trajectory_indices]:\n",
        "            start = random.choices(range(len(trajectory)), k=1)[0]\n",
        "            trajectories.append(trajectory[start:start + window_length])\n",
        "        smallest_trajectory_length = min([len(trajectory) for trajectory in trajectories]) if trajectories else 0\n",
        "        for index in range(len(trajectories)):\n",
        "            trajectories[index] = trajectories[index][-smallest_trajectory_length:]\n",
        "        for transitions in zip(*trajectories):\n",
        "            batched_transition = Transition(*[torch.cat(data, dim=0) for data in zip(*transitions)])\n",
        "            batched_trajectory.append(batched_transition)\n",
        "        return batched_trajectory\n",
        "\n",
        "    @staticmethod\n",
        "    def extend(transition):\n",
        "        \"\"\"\n",
        "        Generate a new zero-reward transition to extend a trajectory.\n",
        "        Parameters\n",
        "        ----------\n",
        "        transition : Transition\n",
        "            A terminal transition which will become the new transition's previous\n",
        "            transition in the trajectory.\n",
        "        Returns\n",
        "        -------\n",
        "        Transition\n",
        "            The new transition that can be used to extend a trajectory.\n",
        "        \"\"\"\n",
        "        if not transition.done[0, 0]:\n",
        "            raise ValueError(\"Can only extend a terminal transition.\")\n",
        "        exploration_statistics = torch.ones(transition.exploration_statistics.size()) \\\n",
        "                                 / transition.exploration_statistics.size(-1)\n",
        "        transition = Transition(states=transition.next_states,\n",
        "                                actions=transition.actions,\n",
        "                                rewards=torch.FloatTensor([[0.]]),\n",
        "                                next_states=transition.next_states,\n",
        "                                done=transition.done,\n",
        "                                exploration_statistics=exploration_statistics)\n",
        "        return transition"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uzXtKkquIg_v"
      },
      "source": [
        "### Ornstein Uhlenbeck"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "srY32EPTImkw"
      },
      "outputs": [],
      "source": [
        "class OrnsteinUhlenbeckProcess:\n",
        "    def __init__(self, theta, mu, sigma, time_scale=1e-1,\n",
        "                 size=1, initial_value=None):\n",
        "        self.theta = theta\n",
        "        self.mu = mu\n",
        "        self.sigma = sigma\n",
        "        self.time_scale = time_scale\n",
        "        self.size = size\n",
        "        self.initial_value = initial_value if initial_value is not None else np.zeros(size)\n",
        "        self.previous_value = self.initial_value\n",
        "\n",
        "    def sample(self):\n",
        "        value = self.previous_value\n",
        "        value += self.theta * (self.mu - self.previous_value) * self.time_scale\n",
        "        value += self.sigma * np.sqrt(self.time_scale) * np.random.normal(size=self.size)\n",
        "        return value\n",
        "\n",
        "    def reset(self):\n",
        "        self.previous_value = self.initial_value\n",
        "\n",
        "    def sampling_parameters(self):\n",
        "        mean = self.previous_value + self.theta * (self.mu - self.previous_value) * self.time_scale\n",
        "        sd = self.sigma * np.sqrt(self.time_scale) * np.ones((self.size,))\n",
        "        return mean, sd"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vsR1jS81aVu0"
      },
      "source": [
        "### Actor critic"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "2gGuX7mCIzm6"
      },
      "outputs": [],
      "source": [
        "class ActorCritic(torch.nn.Module):\n",
        "    \"\"\"\n",
        "    Actor-critic network used in A3C and ACER.\n",
        "    \"\"\"\n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "\n",
        "    def forward(self, *input):\n",
        "        raise NotImplementedError\n",
        "\n",
        "    def copy_parameters_from(self, source, decay=0.):\n",
        "        \"\"\"\n",
        "        Copy the parameters from another network.\n",
        "        Parameters\n",
        "        ----------\n",
        "        source : ActorCritic\n",
        "            The network from which to copy the parameters.\n",
        "        decay : float, optional\n",
        "            How much decay should be applied? Default is 0., which means the parameters\n",
        "            are completely copied.\n",
        "        \"\"\"\n",
        "        for parameter, source_parameter in zip(self.parameters(), source.parameters()):\n",
        "            parameter.data.copy_(decay * parameter.data + (1 - decay) * source_parameter.data)\n",
        "\n",
        "    def copy_gradients_from(self, source):\n",
        "        \"\"\"\n",
        "        Copy the gradients from another network.\n",
        "        Parameters\n",
        "        ----------\n",
        "        source : ActorCritic\n",
        "            The network from which to copy the gradients.\n",
        "        \"\"\"\n",
        "        for parameter, source_parameter in zip(self.parameters(), source.parameters()):\n",
        "            parameter._grad = source_parameter.grad"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "igZXsvZ6aNaY"
      },
      "source": [
        "##### Discrete Actor critic"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "UCeJLhvxaMn8"
      },
      "outputs": [],
      "source": [
        "\n",
        "class DiscreteActorCritic(ActorCritic):\n",
        "    \"\"\"\n",
        "    Discrete actor-critic network used in A3C and ACER.\n",
        "    \"\"\"\n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "        self.input_layer = torch.nn.Linear(STATE_SPACE_DIM, 32)\n",
        "        self.hidden_layer = torch.nn.Linear(32, 32)\n",
        "        self.action_layer = torch.nn.Linear(32, ACTION_SPACE_DIM)\n",
        "        self.action_value_layer = torch.nn.Linear(32, ACTION_SPACE_DIM)\n",
        "\n",
        "    def forward(self, states):\n",
        "        \"\"\"\n",
        "        Compute a forward pass in the network.\n",
        "        Parameters\n",
        "        ----------\n",
        "        states : torch.Tensor\n",
        "            The states for which the action probabilities and the action-values must be computed.\n",
        "        Returns\n",
        "        -------\n",
        "        action_probabilities : torch.Tensor\n",
        "            The action probabilities of the policy according to the actor.\n",
        "        action_probabilities : torch.Tensor\n",
        "            The action-values of the policy according to the critic.\n",
        "        \"\"\"\n",
        "        hidden = F.relu(self.input_layer(states))\n",
        "        hidden = F.relu(self.hidden_layer(hidden))\n",
        "        action_probabilities = F.softmax(self.action_layer(hidden), dim=-1)\n",
        "        action_values = self.action_value_layer(hidden)\n",
        "        return action_probabilities, action_values"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "E_N8FWAlacq_"
      },
      "source": [
        "### Brain"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "WQWMtWLZTFNL"
      },
      "outputs": [],
      "source": [
        "\n",
        "class Brain:\n",
        "    \"\"\"\n",
        "    A centralized brain for the agents.\n",
        "    \"\"\"\n",
        "    def __init__(self):\n",
        "        self.actor_critic = None\n",
        "        self.average_actor_critic = None\n",
        "        self.train_logs = mp.Queue()\n",
        "\n",
        "class DiscreteBrain(Brain):\n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "        self.actor_critic = DiscreteActorCritic()\n",
        "        self.actor_critic.share_memory()\n",
        "        self.average_actor_critic = DiscreteActorCritic()\n",
        "        self.average_actor_critic.share_memory()\n",
        "        self.average_actor_critic.copy_parameters_from(self.actor_critic)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "akxotsgUI9Qk"
      },
      "source": [
        "### Agent"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "iOX6bbPBI-Ps"
      },
      "outputs": [],
      "source": [
        "class Agent:\n",
        "    \"\"\"\n",
        "    Agent that learns an optimal policy using ACER.\n",
        "    Parameters\n",
        "    ----------\n",
        "    brain : brain.Brain\n",
        "        The brain to update.\n",
        "    render : boolean, optional\n",
        "        Should the agent render its actions in the on-policy phase?\n",
        "    verbose : boolean, optional\n",
        "        Should the agent print progress to the console?\n",
        "    \"\"\"\n",
        "    def __init__(self, brain, render=False, verbose=False):\n",
        "        self.env = gym.make(ENVIRONMENT_NAME, render_mode=\"rgb_array\")\n",
        "        self.env.reset()\n",
        "        self.render = render\n",
        "        self.verbose = verbose\n",
        "        self.buffer = ReplayBuffer()\n",
        "        self.brain = brain\n",
        "        self.optimizer = torch.optim.Adam(brain.actor_critic.parameters(),\n",
        "                                          lr=LEARNING_RATE)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-Pv9UAMzbLhZ"
      },
      "source": [
        "#### Discrete Agent"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "id": "xMxHxq-NbEEf"
      },
      "outputs": [],
      "source": [
        "class DiscreteAgent(Agent):\n",
        "    def __init__(self, brain, render=True, verbose=True):\n",
        "        super().__init__(brain, render, verbose)\n",
        "        \n",
        "    def run(self):\n",
        "        \"\"\"\n",
        "        Run the agent for several episodes.\n",
        "        \"\"\"\n",
        "        for episode in range(MAX_EPISODES):\n",
        "            episode_rewards = 0\n",
        "            episode_length = 0\n",
        "            end_of_episode = False\n",
        "#             if self.verbose:\n",
        "#                 print(\"Episode #{}\".format(episode), end=\"\")\n",
        "            while not end_of_episode:\n",
        "                trajectory = self.explore(self.brain.actor_critic) \n",
        "                self.learning_iteration(trajectory)\n",
        "                end_of_episode = trajectory[-1].done[0, 0]\n",
        "                episode_rewards += sum([transition.rewards[0, 0] for transition in trajectory])\n",
        "                episode_length += len(trajectory)\n",
        "                for trajectory_count in range(np.random.poisson(REPLAY_RATIO)):\n",
        "                    trajectory = self.buffer.sample(OFF_POLICY_MINIBATCH_SIZE, MAX_REPLAY_SIZE)\n",
        "                    if trajectory:\n",
        "                        self.learning_iteration(trajectory)\n",
        "                        episode_length += len(trajectory)\n",
        "\n",
        "#             if self.verbose:\n",
        "#                 print(\", episode rewards {} - episode length {}\".format(episode_rewards,episode_length))\n",
        "            self.brain.train_logs.put([episode_rewards.numpy(),episode_length,episode])\n",
        "\n",
        "             \n",
        "    def learning_iteration(self, trajectory):\n",
        "        \"\"\"\n",
        "        Conduct a single discrete learning iteration. Analogue of Algorithm 2 in the paper.\n",
        "        \"\"\"\n",
        "        actor_critic = DiscreteActorCritic()\n",
        "        actor_critic.copy_parameters_from(self.brain.actor_critic)\n",
        "\n",
        "        _, _, _, next_states, _, _ = trajectory[-1]\n",
        "        action_probabilities, action_values = actor_critic(Variable(next_states))\n",
        "        retrace_action_value = (action_probabilities * action_values).data.sum(-1).unsqueeze(-1)\n",
        "\n",
        "        for states, actions, rewards, _, done, exploration_probabilities in reversed(trajectory):\n",
        "            action_probabilities, action_values = actor_critic(Variable(states))\n",
        "            average_action_probabilities, _ = self.brain.average_actor_critic(Variable(states))\n",
        "            value = (action_probabilities * action_values).data.sum(-1).unsqueeze(-1) * (1. - done)\n",
        "            action_indices = Variable(actions.long())\n",
        "\n",
        "            importance_weights = action_probabilities.data / exploration_probabilities\n",
        "            \n",
        "            naive_advantage = action_values.gather(-1, action_indices).data - value\n",
        "            retrace_action_value = rewards + DISCOUNT_FACTOR * retrace_action_value * (1. - done)\n",
        "            retrace_advantage = retrace_action_value - value\n",
        "\n",
        "            # Actor\n",
        "            actor_loss = - ACTOR_LOSS_WEIGHT * Variable(\n",
        "                importance_weights.gather(-1, action_indices.data).clamp(max=TRUNCATION_PARAMETER) * retrace_advantage) \\\n",
        "                * action_probabilities.gather(-1, action_indices).log()\n",
        "            # actor_loss = - ACTOR_LOSS_WEIGHT * Variable(\n",
        "            #                           naive_advantage*action_probabilities.data) * action_probabilities.log()\n",
        "            # actor_loss = - ACTOR_LOSS_WEIGHT * Variable(importance_weights.clamp(min=0.) *\n",
        "            #                           naive_advantage*action_probabilities.data) * action_probabilities.log()\n",
        "            # bias_correction = - ACTOR_LOSS_WEIGHT * Variable((1 - TRUNCATION_PARAMETER / importance_weights).clamp(min=0.) *\n",
        "            #                           naive_advantage * action_probabilities.data) * action_probabilities.log()\n",
        "            # actor_loss += bias_correction.sum(-1).unsqueeze(-1)\n",
        "            actor_gradients = torch.autograd.grad(actor_loss.mean(), action_probabilities, retain_graph=True)\n",
        "            # actor_gradients = self.discrete_trust_region_update(actor_gradients, action_probabilities,\n",
        "            #                                            Variable(average_action_probabilities.data))\n",
        "            action_probabilities.backward(actor_gradients, retain_graph=True)\n",
        "\n",
        "            # Critic\n",
        "            critic_loss = (action_values.gather(-1, action_indices) - Variable(retrace_action_value)).pow(2)\n",
        "            # critic_loss = (action_values.gather(-1, action_indices) - Variable(value)).pow(2)\n",
        "            critic_loss.mean().backward(retain_graph=True)\n",
        "\n",
        "            # Entropy\n",
        "            entropy_loss = ENTROPY_REGULARIZATION * (action_probabilities * action_probabilities.log()).sum(-1)\n",
        "            entropy_loss.mean().backward(retain_graph=True)\n",
        "\n",
        "            retrace_action_value = importance_weights.gather(-1, action_indices.data).clamp(max=1.) * \\\n",
        "                                   (retrace_action_value - action_values.gather(-1, action_indices).data) + value\n",
        "        self.brain.actor_critic.copy_gradients_from(actor_critic)\n",
        "        self.optimizer.step()\n",
        "        # self.brain.average_actor_critic.copy_parameters_from(self.brain.actor_critic, decay=TRUST_REGION_DECAY)\n",
        "        self.brain.average_actor_critic.copy_parameters_from(self.brain.actor_critic)\n",
        "        \n",
        "\n",
        "    def explore(self, actor_critic):\n",
        "        \"\"\"\n",
        "        Explore an environment by taking a sequence of actions and saving the results in the memory.\n",
        "        Parameters\n",
        "        ----------\n",
        "        actor_critic : ActorCritic\n",
        "            The actor-critic model to use to explore.\n",
        "        \"\"\"\n",
        "        state = torch.FloatTensor(self.env.env.state)\n",
        "        trajectory = []\n",
        "        for step in range(MAX_STEPS_BEFORE_UPDATE):\n",
        "            action_probabilities, *_ = actor_critic(Variable(state))\n",
        "            action = action_probabilities.multinomial(1)\n",
        "            action = action.data\n",
        "            exploration_statistics = action_probabilities.data.view(1, -1)\n",
        "            next_state, reward, done, _= self.env.step(action.numpy()[0])\n",
        "            next_state = torch.from_numpy(next_state).float()\n",
        "            if self.render:\n",
        "                self.env.render()\n",
        "            transition = Transition(states=state.view(1, -1),\n",
        "                                                  actions=action.view(1, -1),\n",
        "                                                  rewards=torch.FloatTensor([[reward]]),\n",
        "                                                  next_states=next_state.view(1, -1),\n",
        "                                                  done=torch.FloatTensor([[done]]),\n",
        "                                                  exploration_statistics=exploration_statistics)\n",
        "            self.buffer.add(transition)\n",
        "            trajectory.append(transition)\n",
        "            if done:\n",
        "                self.env.reset()\n",
        "                break\n",
        "            else:\n",
        "                state = next_state\n",
        "        return trajectory\n",
        "\n",
        "    @staticmethod\n",
        "    def discrete_trust_region_update(actor_gradients, action_probabilities, average_action_probabilities):\n",
        "        \"\"\"\n",
        "        Update the actor gradients so that they satisfy a linearized KL constraint with respect\n",
        "        to the average actor-critic network. See Section 3.3 of the paper for details.\n",
        "        Parameters\n",
        "        ----------\n",
        "        actor_gradients : tuple of torch.Tensor's\n",
        "            The original gradients.\n",
        "        action_probabilities\n",
        "            The action probabilities according to the current actor-critic network.\n",
        "        average_action_probabilities\n",
        "            The action probabilities according to the average actor-critic network.\n",
        "        Returns\n",
        "        -------\n",
        "        tuple of torch.Tensor's\n",
        "            The updated gradients.\n",
        "        \"\"\"\n",
        "        negative_kullback_leibler = - ((average_action_probabilities.log() - action_probabilities.log())\n",
        "                                       * average_action_probabilities).sum(-1)\n",
        "        kullback_leibler_gradients = torch.autograd.grad(negative_kullback_leibler.mean(),\n",
        "                                                         action_probabilities, retain_graph=True)\n",
        "        updated_actor_gradients = []\n",
        "        for actor_gradient, kullback_leibler_gradient in zip(actor_gradients, kullback_leibler_gradients):\n",
        "            scale = actor_gradient.mul(kullback_leibler_gradient).sum(-1).unsqueeze(-1) - TRUST_REGION_CONSTRAINT\n",
        "            scale = torch.div(scale, actor_gradient.mul(actor_gradient).sum(-1).unsqueeze(-1)).clamp(min=0.)\n",
        "            updated_actor_gradients.append(actor_gradient - scale * kullback_leibler_gradient)\n",
        "        return updated_actor_gradients\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dgMiAqHrI_7l"
      },
      "source": [
        "### Run agent"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "id": "kltwuUSnJEt2"
      },
      "outputs": [],
      "source": [
        "def run_agent(shared_brain, render=False, verbose=False):\n",
        "    \"\"\"\n",
        "    Run the agent.\n",
        "    Parameters\n",
        "    ----------\n",
        "    shared_brain : brain.Brain\n",
        "        The shared brain the agents will use and update.\n",
        "    render : boolean, optional\n",
        "        Should the agent render its actions in the on-policy phase?\n",
        "    verbose : boolean, optional\n",
        "        Should the agent print progress to the console?\n",
        "    \"\"\"\n",
        "    local_agent = DiscreteAgent(shared_brain, render, verbose)\n",
        "    local_agent.run()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def test(ENVIRONMENT_NAME=\"CartPole-v1\",model_path=\"model.pkl\",num_episodes=1000):\n",
        "  \n",
        "  # Create the environment\n",
        "  env = gym.make(ENVIRONMENT_NAME)\n",
        "  \n",
        "  ## Environment parameters\n",
        "  action_space = env.action_space\n",
        "  state_space = env.observation_space\n",
        "  ACTION_SPACE_DIM = action_space.n\n",
        "  STATE_SPACE_DIM = state_space.shape[0]\n",
        "\n",
        "\n",
        "  ## load model here\n",
        "  model = torch.load(model_path)\n",
        "  model.eval()\n",
        "\n",
        "  # Set the number of episodes to run\n",
        "  num_episodes = 1000\n",
        "\n",
        "  # Create lists to store the returns for each episode\n",
        "  returns = []\n",
        "\n",
        "  # Run the episodes\n",
        "  for episode in range(num_episodes):\n",
        "    # Reset the environment at the start of each episode\n",
        "    state = env.reset()\n",
        "    \n",
        "    # Initialize the episode return\n",
        "    episode_return = 0\n",
        "    episode_length = 0  \n",
        "    # Run the episode\n",
        "    done = False\n",
        "    while not done:\n",
        "      # Get the action from the model\n",
        "      action_prob,_  = model(torch.Tensor(state))\n",
        "      \n",
        "      # Step the environment with the action\n",
        "      next_state, reward, done, _ = env.step(torch.argmax(action_prob).numpy())\n",
        "      \n",
        "      # Update the episode return\n",
        "      episode_return += reward\n",
        "      episode_length += 1\n",
        "      # Update the state\n",
        "      state = next_state\n",
        "    # Add the episode return to the list of returns\n",
        "    returns.append([episode_return,episode_length]) \n",
        "#     print(episode,episode_return,episode_length)\n",
        "  return returns"
      ],
      "metadata": {
        "id": "vDvqXBjhlG8k"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GwJe8DyVblIM"
      },
      "source": [
        "### Init Configuration"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "id": "LlLEkzf2bq-9"
      },
      "outputs": [],
      "source": [
        "\n",
        "ENVIRONMENT_NAME = 'CartPole-v1'\n",
        "# ENVIRONMENT_NAME = 'MountainCarContinuous-v0'\n",
        "\n",
        "env = gym.make(ENVIRONMENT_NAME, render_mode=\"rgb_array\")\n",
        "action_space = env.action_space\n",
        "state_space = env.observation_space\n",
        "env.close()\n",
        "del env\n",
        "\n",
        "ACTION_SPACE_DIM = action_space.n\n",
        "CONTROL = 'discrete'\n",
        "STATE_SPACE_DIM = state_space.shape[0]\n",
        "\n",
        "# Parameters that work well for CartPole-v0\n",
        "LEARNING_RATE = 1e-3\n",
        "\n",
        "### this is hyperparameter\n",
        "REPLAY_BUFFER_SIZE = 100\n",
        "REPLAY_RATIO = 4\n",
        "MAX_EPISODES = 500\n",
        "OFF_POLICY_MINIBATCH_SIZE = 16\n",
        "MAX_REPLAY_SIZE = 200\n",
        "\n",
        "\n",
        "TRUNCATION_PARAMETER = 10\n",
        "DISCOUNT_FACTOR = 0.99\n",
        "MAX_STEPS_BEFORE_UPDATE = 20\n",
        "NUMBER_OF_AGENTS = 2\n",
        "TRUST_REGION_CONSTRAINT = 1.\n",
        "TRUST_REGION_DECAY = 0.99\n",
        "ENTROPY_REGULARIZATION = 1e-3\n",
        "ACTOR_LOSS_WEIGHT = 0.1\n",
        "\n",
        "# Not used for discrete agents\n",
        "ORNSTEIN_UHLENBECK_NOISE_SCALE = None\n",
        "INITIAL_ORNSTEIN_UHLENBECK_NOISE_RATIO = None\n",
        "NUMBER_OF_EXPLORATION_EPISODES = None\n",
        "INITIAL_STANDARD_DEVIATION = None"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JP0YfFDCbtTe"
      },
      "source": [
        "#### Training"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!mkdir retracewobias"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mQ18bOEvlRoI",
        "outputId": "45b4f82e-067e-4a66-eb32-0e5f8ec23adf"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "mkdir: cannot create directory ‘retracewobias’: File exists\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "Cu4ilTKobWZ9",
        "outputId": "8eb4ed94-c435-44d6-86a3-3d7d5dab6479"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "############################################################\n",
            "retracewobias/CartPole-v1_100_1_0_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_100_1_1_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_100_1_2_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_100_1_3_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_100_1_4_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_100_1_5_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_100_1_6_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_100_1_7_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_100_1_8_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_100_1_9_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_100_2_0_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_100_2_1_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_100_2_2_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_100_2_3_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_100_2_4_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_100_2_5_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_100_2_6_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_100_2_7_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_100_2_8_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_100_2_9_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_100_4_0_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_100_4_1_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_100_4_2_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_100_4_3_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_100_4_4_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_100_4_5_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_100_4_6_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_100_4_7_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_100_4_8_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_100_4_9_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_100_8_0_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_100_8_1_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_100_8_2_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_100_8_3_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_100_8_4_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_100_8_5_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_100_8_6_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_100_8_7_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_100_8_8_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_100_8_9_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_250_1_0_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_250_1_1_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_250_1_2_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_250_1_3_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_250_1_4_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_250_1_5_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_250_1_6_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_250_1_7_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_250_1_8_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_250_1_9_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_250_2_0_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_250_2_1_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_250_2_2_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_250_2_3_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_250_2_4_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_250_2_5_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_250_2_6_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_250_2_7_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_250_2_8_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_250_2_9_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_250_4_0_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_250_4_1_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_250_4_2_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_250_4_3_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_250_4_4_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_250_4_5_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_250_4_6_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_250_4_7_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_250_4_8_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_250_4_9_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_250_8_0_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_250_8_1_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_250_8_2_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_250_8_3_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_250_8_4_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_250_8_5_IS_\n",
            "############################################################\n",
            "retracewobias/CartPole-v1_250_8_6_IS_\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-20-813ae68f42f4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      5\u001b[0m                 \u001b[0mbrain\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mDiscreteBrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m                 \u001b[0mlocal_agent\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mDiscreteAgent\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbrain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m                 \u001b[0mlocal_agent\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m                 \u001b[0mlength\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbrain\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_logs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mqsize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-15-207f9068a695>\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     22\u001b[0m                     \u001b[0mtrajectory\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuffer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msample\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mOFF_POLICY_MINIBATCH_SIZE\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mMAX_REPLAY_SIZE\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     23\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0mtrajectory\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 24\u001b[0;31m                         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlearning_iteration\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrajectory\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     25\u001b[0m                         \u001b[0mepisode_length\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrajectory\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     26\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-15-207f9068a695>\u001b[0m in \u001b[0;36mlearning_iteration\u001b[0;34m(self, trajectory)\u001b[0m\n\u001b[1;32m     76\u001b[0m             \u001b[0;31m# Entropy\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     77\u001b[0m             \u001b[0mentropy_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mENTROPY_REGULARIZATION\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0maction_probabilities\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0maction_probabilities\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlog\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 78\u001b[0;31m             \u001b[0mentropy_loss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mretain_graph\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     79\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     80\u001b[0m             \u001b[0mretrace_action_value\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mimportance_weights\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgather\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maction_indices\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclamp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmax\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1.\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/torch/_tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m    485\u001b[0m                 \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    486\u001b[0m             )\n\u001b[0;32m--> 487\u001b[0;31m         torch.autograd.backward(\n\u001b[0m\u001b[1;32m    488\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    489\u001b[0m         )\n",
            "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m    195\u001b[0m     \u001b[0;31m# some Python versions print out the first line of a multi-line function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    196\u001b[0m     \u001b[0;31m# calls in the traceback and some print out the last line\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 197\u001b[0;31m     Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass\n\u001b[0m\u001b[1;32m    198\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    199\u001b[0m         allow_unreachable=True, accumulate_grad=True)  # Calls into the C++ engine to run the backward pass\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ],
      "source": [
        "for MAX_EPISODES in [100,250,500]:\n",
        "    for REPLAY_RATIO in [1,2,4,8]:\n",
        "            for itern in range(10):\n",
        "#                 try:\n",
        "                brain = DiscreteBrain()\n",
        "                local_agent = DiscreteAgent(brain, False,True)\n",
        "                local_agent.run()\n",
        "\n",
        "                length = brain.train_logs.qsize()\n",
        "                out = [brain.train_logs.get_nowait() for _ in range(length)]\n",
        "                train_result = pd.DataFrame(out,columns=[\"Expected Reward\",\"Episode length\",\"Episode number\"])\n",
        "\n",
        "                pathName = \"retracewobias/\" + ENVIRONMENT_NAME + \"_\" + str(MAX_EPISODES) + \"_\" + str(REPLAY_RATIO) \\\n",
        "                            + \"_\" + str(itern) + \"_\" + \"IS_\"\n",
        "                print(\"############################################################\")\n",
        "                print(pathName)\n",
        "                ## save logs\n",
        "                train_result.to_csv( pathName + \"train_result.csv\",index=None)\n",
        "\n",
        "                ## save model\n",
        "                torch.save(brain.actor_critic, pathName + \"model.pkl\")\n",
        "\n",
        "                returns = test(ENVIRONMENT_NAME, model_path=  pathName+\"model.pkl\", num_episodes=1000)\n",
        "\n",
        "                test_results = pd.DataFrame(returns,columns=[\"Expected Rewards\",\"Episode Length\"])\n",
        "                test_results.to_csv(pathName+ \"test_result.csv\",index=None)\n",
        "#                 except:\n",
        "#                     print(\"-------------------------------ERROR ------------------------------------------\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "M1qweDGocg9A"
      },
      "outputs": [],
      "source": [
        "250,2,3"
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1U2MDOxx5ILJ",
        "outputId": "d4e5e739-1a2f-4ccd-99ec-cdfc3f3d87a8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "5"
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import glob"
      ],
      "metadata": {
        "id": "OdDxXKis_k-b"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "id": "EUSou917jKBn"
      },
      "outputs": [],
      "source": [
        "out =[]\n",
        "for file in glob.glob(\"./retracewobias//*test*.csv\"):\n",
        "    data = pd.read_csv(file)\n",
        "    out.append(file.split(\"_\")[1:4] + [data[\"Expected Rewards\"].mean(),data['Expected Rewards'].std()])\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "resdf = pd.DataFrame(out,columns=[\"Max_episodes\",\"Replay\",\"itern\",\"mean\",\"std\"]).groupby(by=[\"Max_episodes\",\"Replay\"],as_index=False).mean()"
      ],
      "metadata": {
        "id": "YaPj5WQFjKBn"
      },
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "resdf"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 300
        },
        "id": "n-cgAxdMAON4",
        "outputId": "ad15faeb-0b40-41a7-cbe2-e35bd5b0c59c"
      },
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "  Max_episodes Replay        mean        std\n",
              "0          100      1   23.134400   7.130934\n",
              "1          100      2   25.765100   5.816208\n",
              "2          100      4   42.783000   9.600703\n",
              "3          100      8   64.311200  19.737669\n",
              "4          250      1   96.150600  11.591731\n",
              "5          250      2  139.863100  11.413524\n",
              "6          250      4  190.138200  15.746201\n",
              "7          250      8  171.630857  18.992106"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-1d44258b-e343-44c1-bdaa-c8199ad24a14\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Max_episodes</th>\n",
              "      <th>Replay</th>\n",
              "      <th>mean</th>\n",
              "      <th>std</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>100</td>\n",
              "      <td>1</td>\n",
              "      <td>23.134400</td>\n",
              "      <td>7.130934</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>100</td>\n",
              "      <td>2</td>\n",
              "      <td>25.765100</td>\n",
              "      <td>5.816208</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>100</td>\n",
              "      <td>4</td>\n",
              "      <td>42.783000</td>\n",
              "      <td>9.600703</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>100</td>\n",
              "      <td>8</td>\n",
              "      <td>64.311200</td>\n",
              "      <td>19.737669</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>250</td>\n",
              "      <td>1</td>\n",
              "      <td>96.150600</td>\n",
              "      <td>11.591731</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>250</td>\n",
              "      <td>2</td>\n",
              "      <td>139.863100</td>\n",
              "      <td>11.413524</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>250</td>\n",
              "      <td>4</td>\n",
              "      <td>190.138200</td>\n",
              "      <td>15.746201</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>250</td>\n",
              "      <td>8</td>\n",
              "      <td>171.630857</td>\n",
              "      <td>18.992106</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-1d44258b-e343-44c1-bdaa-c8199ad24a14')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-1d44258b-e343-44c1-bdaa-c8199ad24a14 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-1d44258b-e343-44c1-bdaa-c8199ad24a14');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for episode in [\"100\",\"250\"]:\n",
        "    plt.errorbar(resdf.loc[resdf[\"Max_episodes\"].isin([episode]),\"Replay\"],\n",
        "                resdf.loc[resdf[\"Max_episodes\"].isin([episode]),\"mean\"],\n",
        "                resdf.loc[resdf[\"Max_episodes\"].isin([episode]),\"std\"],label=\"Max episodes = \"+ episode)\n",
        "plt.xlabel(\"REPLAY RATIO\")\n",
        "plt.ylabel(\"Average Reward with std deviation\")\n",
        "plt.title(\"Max episodes for training vs Replay Ratio\")\n",
        "plt.legend()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 312
        },
        "outputId": "88f5304a-7a32-445d-c5a1-c7e6be53e9e7",
        "id": "_HqWYlbbjKBo"
      },
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.legend.Legend at 0x7f195e95c490>"
            ]
          },
          "metadata": {},
          "execution_count": 25
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd3yV5fn48c+VAWEEEvYMYe8poKAgQxyAolZFLA7QAhV/Wuss1WqtfrWt1Vq1CIoLK4LbuheoVBHZKENAVhgBAgkhELKu3x/3k8NJyDgkOTkZ1/v1Oq+c88zr5CTPde7x3LeoKsYYYwxAWKgDMMYYU3FYUjDGGONjScEYY4yPJQVjjDE+lhSMMcb4WFIwxhjjY0nBFEhE4kTkiIiEl/Fxt4nIOWV8zM4iskpEUkXk5rI8dkmJyBAR2VjW21Y1IvKiiDwY6jiKIyIzROS5UMdRHiwplCPvgpghIo3yLV8pIioi8aGJ7GSqukNV66pqdqhjCcCdwEJVjVbVf5X2YCJyv4i8UppjqOo3qtq5rLcNBRG5TkSyvS8Jh0VktYiMDXVcRRGRRSKS7sV8QETeEpHmAe47TEQS/Jep6v+p6g3BibZisaRQ/rYCE3JfiEhPoHbowqkS2gA/lWRHEYkowT4iItXtf+c7Va0LxAD/Bl4TkZgQx1Scm7yYOwB1gUdDHE+lUN3+sCuCucA1fq+vBV7230BExnilh8MislNE7vdbN15EtopIPe/1BSKyV0QaF3QyETlDRL4VkWTvG94wv3WLRORhEVnqnetdEWngrYv3Si8R3uvrROQXr4pmq4j82lseJiL3iMh2EdknIi+LSH2/c1ztrUsSkT/miy1MRO4WkS3e+gV+548SkVe85cki8oOINC3g/X0JDAee8r4VdhKR+l4c+71z35N7Effex/9E5HERSQLuz3e884EZwHjveKv9flcPicj/gKNAOxGZJCLrvd/JLyIy1e84eb5teqXE20VkjYikiMh8EYk61W299XeKyB4R2S0iN3ifU4cCfjfjRWRZvmW3ish73vPRIrLOi3+XiNye/xj5qWoO7m+4DtDRO05NEXlURHaISKKIPCMitfzfm7jqlwPee/t1QccWkVgRed/73A55z1t56y4XkeX5tv+9iLwbQMzJwDtAH799C/zsRKQO8BHQwvv8j4hIC8lXehSRi0TkJ+9vc5GIdC0ujkpDVe1RTg9gG3AOsBHoCoQDCbhvugrEe9sNA3riknYvIBG42O84/wFeBBoCu4GxhZyvJZAEjPaONcp73dhbvwjYBfTA/ZO/CbzirYv3Yorw1h0GOnvrmgPdveeTgc1AO9y3sbeAud66bsARYChQE3gMyALO8dbfAiwBWnnrZwHzvHVTgf/iSlHhwGlAvULe5yLgBr/XLwPvAtHe+/gZuN5bd50Xw//z3lutAo53f+7vId85dgDdvf0igTFAe0CAs3HJop/fZ5iQ77NfCrQAGgDrgWkl2PZ8YK8XR23gFe9z6lDA+6gNpAId/Zb9AFzpPd8DDPGex+bGXsBxrgMWe8/DgelABtDEW/Y48J4Xa7T3uT3s996yvM++pvd7SuPE39KLwIPe84bAr7y4o4HXgXe8dTWBg0BXv7hWAr8q7m/CO+7nwLt+6wP+7PL/TQCdvPcwyvs7uBP3P1Aj1NeYMrlOhTqA6vTgRFK4B3jY+wf/DHeR8SWFAvb7J/C43+sY3AVqLTCriPPdhXeB9lv2CXCt93wR8Ijfum7eP3s4JyeFZO8ftla+430B3Oj3ujOQ6e33J+A1v3V1vOPnJoX1wEi/9c399p0MfAv0CuD36n8BCPfO0c1v/VRgkff8OmBHMcfzXQDyneOBYvZ7B7jFe57nwuJ99hP9Xv8NeKYE2z6Pd8H1XnegkKTgrX8F+JP3vCMuSdT2Xu/wfjcFJlu/Y1yHu7Ane5/PMeAKb53gLpDt/bYfBGz1e29ZQB2/9QuAe73nL+IlhQLO2wc45Pd6JvCQ97w7cAioWcTfxFEgxfv9rALiSvLZ5f+bAO4FFvitC8N9uRpW3N9qZXhY9VFozAWuwv2zvZx/pYicLiILvWJ0CjAN8DVOqysOv477hv+PIs7TBrjcK+Imi0gycBbu4ptrp9/z7bhvPnkawlU1DRjvxbFHRD4QkS7e6hbefv7HiACaeut25jtOUr743vaLbT2Q7e07F5fAXvOqSf4mIpFFvNdcjbz3kD+mloW851ORZz9xVXdLROSgF/9o8v3u8tnr9/wormR1qtvm+Z3mj6kAr3KiDesq3Dfvo97rX3kxbxeRr0RkUBHHWaKqMbgSxXvAEG95Y9w3++V+n+PH3vJch7zPPtd2733kISK1RWSWV+V3GPgaiJETPeBeAq4SEQGuxl2YjxcR882qWh9X2o7FlUhzz3Wqn52/PH/z6qrUdpL3b6zSsqQQAqq6HdfgPBpX3ZLfq7h/vNbeH/UzuG9kAIhIH9w36XlAUb1tduJKCjF+jzqq+ojfNq39nsfhvgkeKCDmT1R1FC6hbACe9Vbtxl3c/Y+Rhavy2uN/fBGpjSvK+8d3Qb74olR1l6pmquqfVbUbMBgYS962mMIc8N5D/ph2+b+dYo5R2HrfchGpiatuexRo6l0wP8TvcwqSPfhd3Mj7+RXkM6Cx9zczAfe3BYCq/qCq44AmuG/KC4o7uaoeAX4LXC0ifXG/72O46sTcz7C+ugbeXLFeXX2uONzfTX634Uqap6tqPVy1I3i/U1VdgisFDsEluLnFxevttxZ4EHhanOI+u+L+PvL8zXtJqjV5/8YqLUsKoXM9MCLfN6hc0cBBVU0XkYG4fwDANcDiqgRmAJOAliJyYyHneAW4UETOE5FwcY23w3Ib7zwTRaSbd8F+AHhD83VDFZGmIjLO+8c+jmsnyPFWzwNuFZG2IlIX+D9gvqpmAW8AY0XkLBGp4R3f/2/uGeAhEWnjnaexiIzzng8XkZ7et8TDuAt9DsXwYl/gHTfaO/bvvd9FoBKBeCm6h1ENXD33fiBLRC4Azj2Fc5TUAmCSiHT1PrN7i9pYVTNxpcq/4+r8PwMQkRoi8msRqe9tc5gAfr/eMQ8Cz+GqpXJwXxAeF5Em3rFbish5+Xb7s3fOIbgE/3oBh47GJZhkcR0O7itgm5eBp4BMVV0cSLyel3Al0Iso/rNLBBqKX4eJfBYAY0RkpFd6vQ33f/HtKcRTYVlSCBFV3aKqywpZfSPwgIik4url/b/BPQzsVNWZXtF5IvCgiHQs4Bw7gXG4BLIf9838DvJ+7nNx9bp7gSigoJu/wnAX1t24xr6zcd8WwdVxz8UV9bcC6bhGXFT1J1yj5Ku4b7iHcA3ruZ7AlYg+9d7rEuB0b10zXFI5jKtW+ooAvxl6508DfgEWe+d/PsB94cQFK0lEVhS0gaqm4n5XC3Dv6yrvvQSVqn6EKx0uxDVuLvFWFVWN8iquLet1L1nnuhrY5lXVTAMK7BVUiH8Co0WkF67tajOwxDvW57hv/Ln24n5Hu3GdJKap6oZCjlkLV/pYgquGym8urtr0lO4jUdUM3N/bvcV9dl5s84BfvCqxFvmOtRH3f/ekF+uFwIXeOSo98RpKTDUkIotwjWfV4k7NqsjrCvkjrsE1q7jty5u4LtCvqGqr4rYN8Hi1gH24nkKbyuKYJi8rKRhTyYjIJeLuDYgF/gr8tyImhCD5LfCDJYTgOeW7OY0xITcVV+WXjatWK6xNqUoRkW24xuCLQxxKlWbVR8YYY3ys+sgYY4xPpa4+atSokcbHx4c6DGOMqVSWL19+QFULHC+tUieF+Ph4li0rrFenMcaYgojI9sLWWfWRMcYYH0sKxhhjfCwpGGOM8anUbQoFyczMJCEhgfT09FCHYiqoqKgoWrVqRWRkIIOuGlO9VLmkkJCQQHR0NPHx8bjBC405QVVJSkoiISGBtm3bhjocYyqcKld9lJ6eTsOGDS0hmAKJCA0bNrSSpDGFqHJJAbCEYIpkfx/GFK5KJoVTNX7Wd4yf9V2owzDGmJCzpBAEIsLEiRN9r7OysmjcuDFjx44NSTyjR48mOTm5VMdYtGhRuce/YcMGBg0aRM2aNXn00UfzrPv444/p3LkzHTp04JFHTkwkt3XrVk4//XQ6dOjA+PHjycioEkPch84LY9zDVBuWFIKgTp06/Pjjjxw7dgyAzz77jJYtQzd964cffkhMTEzIzl9SDRo04F//+he33357nuXZ2dlMnz6djz76iHXr1jFv3jzWrVsHwF133cWtt97K5s2biY2NZc6cOaEI3ZhKy5JCkIwePZoPPvgAgHnz5jFhwgTfuqVLlzJo0CD69u3L4MGD2bhxIwCPP/44kydPBmDt2rX06NGDo0eP5jludnY2d9xxBwMGDKBXr17MmjULcN/khw4dypgxY+jcuTPTpk0jJ8fNrhgfH8+BAwdIS0tjzJgx9O7dmx49ejB//nwAvvjiC/r27UvPnj2ZPHkyx4+7Sbw+/vhjunTpQr9+/XjrrRNTSaelpTF58mQGDhxI3759effddwH46aefGDhwIH369KFXr15s2lS6Ie+bNGnCgAEDTuo6unTpUjp06EC7du2oUaMGV155Je+++y6qypdffslll10GwLXXXss777xTqhiMqW6C1iVVRFrj5lNtipsIe7aqPuHNvTofiAe2AVeo6iFv8usncJPZHwWuU9UCp0IM1J//+xPrdh8udrt1e9w2gbQrdGtRj/su7F7sdldeeSUPPPAAY8eOZc2aNUyePJlvvvkGgC5duvDNN98QERHB559/zowZM3jzzTe55ZZbGDZsGG+//TYPPfQQs2bNonbt2nmOO2fOHOrXr88PP/zA8ePHOfPMMzn3XDe97NKlS1m3bh1t2rTh/PPP56233vJdIMFd5Fu0aOFLVikpKaSnp3PdddfxxRdf0KlTJ6655hpmzpzJtGnT+M1vfsOXX37pq4rJ9dBDDzFixAief/55kpOTGThwIOeccw7PPPMMt9xyC7/+9a/JyMggOzvPVM/udzx+vC8J+vv973/PNddcU+zvFWDXrl20bn1ivvpWrVrx/fffk5SURExMDBEREb7lu3ZVibnUjSk3wbxPIQu4TVVXiEg0sFxEPgOuA75Q1UdE5G7gbtwcrxcAHb3H6cBMTszXW+n06tWLbdu2MW/ePEaPHp1nXUpKCtdeey2bNm1CRMjMzAQgLCyMF198kV69ejF16lTOPPPMk4776aefsmbNGt544w3fsTZt2kSNGjUYOHAg7dq1A2DChAksXrw4T1Lo2bMnt912G3fddRdjx45lyJAhrF69mrZt29KpUyfAfbt++umnGTZsGG3btqVjRzf188SJE5k9e7Yvhvfee89Xz5+ens6OHTsYNGgQDz30EAkJCVx66aW+ff3llk6MMRVT0JKCqu7BTdaOqqaKyHqgJW4i+WHeZi8Bi3BJYRzwsrpZf5aISIyINPeOUyKBfKOHEyWE+VMHlfRUBbrooou4/fbbWbRoEUlJSb7l9957L8OHD+ftt99m27ZtDBs2zLdu06ZN1K1bl927dxd4TFXlySef5LzzzsuzfNGiRSd1tcz/ulOnTqxYsYIPP/yQe+65h5EjRzJu3LhTfl+qyptvvknnzp3zLO/atSunn346H3zwAaNHj2bWrFmMGDEizzZlUVJo2bIlO3fu9L1OSEigZcuWNGzYkOTkZLKysoiIiPAtN8YErlzaFEQkHugLfA809bvQ78VVL4FLGDv9dkvwluU/1hQRWSYiy/bv3x+0mMvC5MmTue++++jZs2ee5SkpKb6L1Ysvvphn+c0338zXX39NUlKSrzTg77zzzmPmzJm+0sXPP/9MWloa4KqPtm7dSk5ODvPnz+ess87Ks+/u3bupXbs2EydO5I477mDFihV07tyZbdu2sXnzZgDmzp3L2WefTZcuXdi2bRtbtmwBXLuIfwxPPvkkubP2rVy5EoBffvmFdu3acfPNNzNu3DjWrFlzUvzz589n1apVJz0CTQgAAwYMYNOmTWzdupWMjAxee+01LrroIkSE4cOH+35vL730UomSnjHVmqoG9QHUBZYDl3qvk/OtP+T9fB84y2/5F0D/oo592mmnaX7r1q07aVlxrnjmW73imW9Peb/C1KlT56RlCxcu1DFjxqiq6rfffqsdO3bUPn366B//+Edt06aNqqpOmjRJn3jiCVVV3bFjh7Zv314TExPzHCc7O1v/8Ic/aI8ePbR79+46bNgwTU5O1oULF+qQIUN09OjR2qlTJ506dapmZ2erqmqbNm10//79+vHHH2vPnj21d+/e2r9/f/3hhx9UVfXzzz/XPn36aI8ePXTSpEmanp6uqqofffSRdu7cWfv27as333yzL/6jR4/qlClTtEePHtqtWzff8ocffli7deumvXv31vPOO0+TkpJK9Xvcs2ePtmzZUqOjo7V+/frasmVLTUlJUVXVDz74QDt27Kjt2rXTBx980LfPli1bdMCAAdq+fXu97LLLfO8lv5L8nVRLz492D1OlAMu0kOtqUOdoFpFI72L/iao+5i3bCAxT1T0i0hxYpKqdRWSW93xe/u0KO37//v01/yQ769evp2vXrqcUZ7Cqj8rTokWLePTRR3n//fdDHUqlUJK/k2op9x6FSR+ENg5TpkRkuar2L2hdMHsfCTAHWJ+bEDzvAdcCj3g/3/VbfpOIvIZrYE4pKiGUpcqcDIwxpiwFs/fRmcDVwFoRWeUtm4FLBgtE5HpgO3CFt+5DXHfUzbguqZOCGFuVM2zYsDwN1sYYUxLB7H20GChs5LGRBWyvwPRgxWOMMaZ4dkezMcYYH0sKxhhjfCwpgI0EaYwxnmKTgohcKiKbRCRFRA6LSKqIFD+gUDVmQ2eXjf/85z/06tWLnj17MnjwYFavXu1bFx8fT8+ePenTpw/9+5/oWXfw4EFGjRpFx44dGTVqFIcOHSrXmKuknCwIYtd1U7EEUlL4G3CRqtZX1XqqGq2q9YIdWGVmQ2eXjbZt2/LVV1+xdu1a7r33XqZMmZJn/cKFC1m1ahX+96o88sgjjBw5kk2bNjFy5Mg8cy2YU7RrOST+CDuXwF/j4cWx8MkfYfVrkLgOsrNCHWH1FcTajUCSQqKqrg/K2aswGzq79ENnDx48mNjYWADOOOMMEhISit3n3Xff5dprrwVs6OwS2/sjzLsKnh0BGUegXivofjFkpMEPz8HbU2HmIPi/FjB7GLx3Myx9FnYudduYSi2QLqnLRGQ+8A5wPHehqr5V+C4VxEd3w961xW+31xujJ5DM26wnXFD8t08bOrtsh86eM2cOF1xwge+1iHDuueciIkydOtVXikhMTKR58+YANGvWjMTExKI/KHPC/p9h0cPw01tQsz4Mvwc2fw5hEXDhE26b7CxI2gR71rj/m71rYN27sOIl7yACDTtA817uf6VZL/eo2zhkb8ucmkCSQj3czWTn+i1ToOInhRCyobPLbujshQsXMmfOHBYvXuxbtnjxYlq2bMm+ffsYNWoUXbp0YejQoXn2E5GTRoo1BTi4Fb76G6x5DSJqwZDbYfBNUCsWflmUd9vwCGjS1T16e18UVCElwUsSa13C2LkUfnzzxH7Rzb0E0dNLGL0gNh7s86lwik0Kqlp57ywO4Bs9ELTxXWzo7NIPnb1mzRpuuOEGPvroIxo2bOhbnttG06RJEy655BKWLl3K0KFDadq0KXv27KF58+bs2bOHJk2anPL7qzZSdsHXf4eVc11p4Iwb4axboU6jUzuOCMS0do8ufqXtowddkvBPFps/B/VKkDXreaUJr0TRvBc06gwRNcruPZpTVmxSEJFWwJO4YSsAvgFuUdXiK3irucmTJxMTE0PPnj1ZtGiRb3kgQ2ffdNNNvPHGG3m+6cOJobNHjBhBZGQkP//8s+9YuUNnt2nThvnz55/UMLt7924aNGjAxIkTiYmJ4bnnnuPOO+/0DZ3doUOHAofObt++fYFDZz/55JOICCtXrqRv3755hs7esWMHa9asOSkpnEpJYceOHVx66aXMnTvXV5IB16aRk5NDdHQ0aWlpfPrpp/zpT38CXCJ+6aWXuPvuu23o7MIc2QffPAbLngfNgdOugyG3Qb0WZXue2g2g3dnukSvzGOxb51U/eQljxcuQ6bWdhdeAxl1OlCaa9YKm3SHK+raUl0Cqj14AXgUu915P9JaNClZQVUWrVq24+eabT1p+5513cu211/Lggw8yZsyJb1a33nor06dPp1OnTsyZM4fhw4czdOjQPN92b7jhBrZt20a/fv1QVRo3buxrTB0wYAA33XQTmzdvZvjw4VxyySV5zrt27VruuOMOwsLCiIyMZObMmURFRfHCCy9w+eWXk5WVxYABA5g2bRo1a9Zk9uzZjBkzhtq1azNkyBBSU1MBV9L53e9+R69evcjJyaFt27a8//77LFiwgLlz5xIZGUmzZs2YMWNGqX5/DzzwAElJSdx4440AREREsGzZMhITE33vLSsri6uuuorzzz8fgLvvvpsrrriCOXPm0KZNGxYsWFCqGKqUowfhf0/A0tmQdRz6TIChd0Jsm/KLIbIWtDzNPXLlZEPSlhNtFHvWwMaPYOUrJ7Zp0M6v+qm3+xndrPzirkaKHTpbRFapap/iloVCWQ2dXRWGB7ahs09NtRo6Oz0Fvvs3fPe0603U8zI4+25o1KH4fUP1v6EKqXv8ShSr3fPk7Se2qdMkb4N2894Q2xbCqsE9uaX8XEo7dHaSiEwEcusPJgBJRWxf+VTiZGBMoTLSXKngf0/AsUPQ9UIYNgOadgt1ZMUTcdVZ9VpA5/NPLD+W7O6dyG2j2LvGNYbnePdM1KgLTXvkbdBu0hUiaobkbVRGgSSFybg2hcdxvY6+xYa1rnBs6Gzjk5kOy1+Ab/4Bafuh47kwfAa06BvqyEqvVgzEn+UeubKOw771eRu0V8+DH55168MiXDuFf4N20x7uWOYkgfQ+2g5cVA6xlBlVta6IplDBnG0wpLIyYNUr8NXfIXU3tB0Kw/8DcaeHOrLgiqgJLfq4R66cHDi0FfasPtGgvfkLlyxyxbTJ26DdrKcrmVTza0ehSUFE7lTVv4nIk7gSQh6qenILagUQFRVFUlISDRs2tMRgTqKqJCUlERUVFepQyk52FqxdAIsecXXurQbCJc/k7fVT3YSFQcP27tHj0hPLUxPzNmjvXQPr/3tife2G+Rq0e7ljhIWX/3sIkaJKCrlDWywrYpsKp1WrViQkJLB///5Qh2IqqKioKFq1ahXqMEovJwfWvQ0LH3Z3GTfvDWP+AR3OqfbfdgsV3RSiR0FHv86Tx1Pd0B7+DdpLZkKOu6mUyNquW6x/9VOTbq4nVRVUaFJQ1dz0eVRVX/dfJyKXF7AL+bZ5HhgL7FPVHt6y+UDuHU8xQLKq9hGReFwSyr2raYmqTjuF9+ETGRlJ27ZtS7KrMZWDKmz8EBb+n2t0bdwVxr8CXcZaMiiJmtHQZpB75MrKgAMb895PsfYNd28HgIRDo055G7Sb9XT3ZlRygTQ0/wF4PYBl+b0IPAW8nLtAVX0D6IjIP4AUv+23VIRursZUWKqw5Qv48kHYvRIatIdfzYHul1Sr6o1yEVHjxN3WuVTh0La8DdrbFruqu1z1W+cbzqOnW1aJknVRbQoXAKOBliLyL79V9YBix8xV1a+9EkBBxxbgCmBEQeuNMfls+59LBju+dReZi56C3hPcWESmfIhAg7bu0c3vTvkj+08kCt/Ndx/ia4qtFZt3cMBmPV0po4J+dkVFtRvXnnARsNxveSpwaynPOwQ3JLf/2MptRWQlcBi4R1W/KWhHEZkCTAGIi4srZRjGVHAJy1wy+GUh1G0Gox+FftdYv/uKpG5j6DDSPXJlpEHiT3l7Py19FrK9gaYjoly7hH/1U9PuUKNOaN6Dn0DuaI5U1cwSHdyVFN7PbVPwWz4T2Kyq//Be1wTqqmqSiJyGG6a7u6oWOcNbQXc0G1Ml7Fnj2gx+/sj1iDnr9zDg+irbuFktZGfBgZ/9qp9Wu+fpubXoAo06+pUqvB5QBQ1QGOI7muNF5GGgG+Drx6eq7UoYTARwKeAb/ERVj+PN1aCqy0VkC9CJStbzyZhS27/RJYN170BUfRhxL5w+DWrWDXVkprTCI9zd5E27Qe8r3TJVSNmZt0F7x/f5hh1vcXKDtmrQ2ikCHRDvPtwdzcNxdzOXZnCRc4AN/qOsikhj4KCqZotIO6Aj8EspzmFM5XLwF1j0V9doGVnbDVQ3aLrddVvViUBMnHt09ZsD/ejBvA3ae9fA5s/cqLbgej/VbRqUkAJJCrVU9QsREe/u5vtFZDnwp6J2EpF5wDCgkYgkAPep6hzgSk6Mo5RrKPCAiGQCOcA0VT14iu/FmMoneac3p8ErbtjoQTfBmb+DOg2L39dUXbUbQLth7pEr85ibG3vvGnejYkRwbsAMJCkcF5EwYJOI3ATsAooty6rqhEKWX1fAsjeBN0/e2pgqKnWvm9Ng+Qvu9YAbYMjvbThoU7jIWtDqNPdY+0bQThNIUrgFqA3cDPwF14302qBFZExVlpYE//un1xMlA/pOhKF3uFnLjKkAAhkQ7wfv6RFsdFRjSuZYspvPYMm/XXfFXlfA2Xe5cXWMqUCKunntn6r6OxH5LwUPiFepRk41JiSOH4Gls+B//4L0ZHfT07AZ0KRLqCMzpkBFlRTmej8fLY9AjKlSMo+5cXK+eQyOHoBO57s5DZr3DnVkxhSpqAHxcu9ibgh84N1LYIwpSlYGrHwZvn7UTSfZbhgMvwdaDwh1ZMYEJJCG5guBx0Xka2A+8LGqFjv2kTHVSnYWrHnN3WuQsgNanwGXPgtth4Q6MmNOSSANzZNEJBK4ADc/89Mi8pmq3hD06Iyp6HJy4Ke33F3IB7e4KS8vfBzaj6xUI2MakyugYfpUNVNEPsI1ONcCLgYsKZjqSxU2vO+Swb510KQ7XPkqdB5tycBUasUmBW8I7fG4u5MXAc/hhr02pvpRhc2fu5FL96yChh3gsueh2yVuCkhjKrlASgrX4NoSplpjs6nWtn7tksHO791YNeP+Db3GV9hx8Y0piUDaFCaISBvcHAifi0gtIEJVU4MenTEVwbS9SawAACAASURBVM6lLhls/Qqim8OYx6Dv1W52LmOqmECqj36Dm9SmAdAeaAU8A4wsaj9jKr3dq1ybwaZPoE5jOO9h6D/J5jQwVVog5d7pwEDgewBV3SQiTYIalTGhtG+9Swbr34OoGBh5HwycYnMamGohoFFSVTVDvB4V3iQ5RU/XZkxllLTFDUm89nWoURfOvhsG3egmuzGmmggkKXwlIjOAWiIyCrgR+G9wwzKmHCXvgK/+BqtedXManHmLe9RuEOrIjCl3gSSFu4HrgbXAVOBDXLdUYyq3w3vgm3/A8hfdvQUDp8BZt0J0cGa0MqYyCKT3UQ7wrPcwpvJLOwCLH4cfnoOcLNeTaOjtUL9VqCMzJuSKGjp7LUW0Hahqr6IOLCLPA2OBfaraw1t2P/AbYL+32QxV/dBb9wdciSQbuFlVPwn8bRgTgGOH4NunYMlMyDrm7jE4+y5o0DbUkRlTYRRVUsidRXq69zN3KO2JBNbQ/CLwFPByvuWPq2qe4bhFpBtu7ubuQAvc/RCdVDU7gPMYU7TjqfD9M/Dtk5CeAt0vgWF/gMadQx2ZMSUz6YOgHbqoobO3A4jIKFXt67fqLhFZgWtrKJSqfi0i8QHGMQ54zbtjequIbMZ1g/0uwP1NVfDCGPezrP7gM47CsjmuquhokhuXaPgMaNazbI5vTBUUSEOziMiZqvo/78VgoDSDvNwkItcAy4DbVPUQ0BJY4rdNgresoGCm4G6mIy4urhRhmCor6zis8OY0OLIX2o9wcxq0Oi3UkRlT4QWSFK4HnheR3M7aycDkEp5vJvAXXPXTX4B/nOqxVHU2MBugf//+dr+EOSE7E1bPc91LU3ZC3GA3WF38maGOzJhKI5DeR8uB3rlJQVVTSnoyVU3MfS4izwLvey93Aa39Nm3lLTOmeDnZ8OObsOhhOPgLtDwNLvoXtBtuw1gbc4oCHt6xNMkgl4g0V9U93stLgB+95+8Br4rIY7iG5o7A0tKez1RxOTmw4b9uSIr9G6BpT5jwmpsP2ZKBMSUStDF/RWQebg6GRiKSANwHDBORPrjqo224m+FQ1Z9EZAGwDsgCplvPI1MoVdj0qRu5dO8aaNQJLn8Ruo6zOQ2MKaWgJQVVnVDA4jlFbP8Q8FCw4jFVxC+LXDJI+AFi2sDFz0CvKyAsPNSRGVMlFHXz2qVF7aiqb5V9OMYUYscSlwy2fQP1WsLYf0LfiRAeGerIjKlSiiopXOj9bAIMBr70Xg8HvgUsKZjg270SvnwINn8GdZrA+X+F066DyKhQR2ZMlVTUzWuTAETkU6BbbgOxiDTH3a1sTPAk/uQakDe8D7Vi4Zw/w8DfQI06oY7MmCotkDaF1n49hgASAbtrzARH5jF443rXxbRmNAybAWf8FqLqhToyY6qFQJLCFyLyCTDPez0e+Dx4IZlqae9aOLAR0vbD/vVuCOvB/8/mNDCmnAVy89pNInIJMNRbNFtV3w5uWKZaUIXNn7uB6rZ+BRIG0S1g6tdQt3GoozOmWio2KYjIX1X1LuDtApYZc+oy02HtAvjuaXfTWXRzOOd+2PAxhEdYQjAmhAK502dUAcsuKOtATDWQlgSL/gr/7AHv/T8Ii4RLZsEta1x1UXjQbpsxxgSoqPsUfoubj7mdiKzxWxUN/C/YgZkq5MAmVypYPQ+y0qHjuTDoJmg71IajMKaCKeqr2avAR8DD5J07IVVVDwY1KlP5qcK2xS4Z/PwRhNeE3uPhjOnQpEuoozPGFKKo+xRSgBQRuQfYq6rHRWQY0EtEXlbV5PIK0lQi2Znw0zvw3ZOwZzXUbuimvBxwA9RtEurojDHFCKQS902gv4h0wM1j8C6uFDE6mIGZSiY9BZa/BN/PgsMJ0LCjG4qi95UQWSvU0RljAhRIUshR1SxvLKQnVfVJEVkZ7MBMJXFou5v/eMXLkHEE4ofAmH+4dgMbsdSYSieQpJApIhOAazgxHpKNQlbdJSx3VUTr3nX3F3S/FAZNhxZ9Qh2ZMaYUAkkKk4BpwEOqulVE2gJzgxuWqZBysmHjR/DdU7DjO6hZ3/UiOn0q1G8V6uiMMWUgkDua1wE3+73eCvw1mEGZCiYjDVa9Ckv+7aa7rB8H5z0M/a524xMZY6oMu1vIFC51LyydDcueh2OH3NzHl70AXS+yG82MqaKCOR3n88BYYJ+q9vCW/R3XLpEBbAEmqWqyiMQD64GN3u5LVHVasGIzxdj7o7u/YO3rkJMFXca4welanx7cm80mfRC8YxtjAhLMr3svAk8BL/st+wz4g9eb6a/AH4DcMZS2qKq1UoaKKmz5Ar59Cn5ZCJG1of8kOH0aNGwf6uiMMeUkkAHxOgF3AG38t1fVEUXtp6pfeyUA/2Wf+r1cAlx2CrGaYMg67koE3z0N+9ZB3WYw8k9w2iQbttqYaiiQksLrwDPAs0B2GZ57MjDf73Vb7/6Hw8A9qvpNQTuJyBRgCkBcnM31U2JHD8IPc1ybQdo+aNIdLp4JPX4FETVDHZ0xJkQCSQpZqjqzLE8qIn8EsoD/eIv2AHGqmiQipwHviEh3VT2cf19VnY27s5r+/ftrWcZVLSRtcaWCVa9C1jHocI7rVtpumA1OZ4wpcpTU3LqD/4rIjbj5FI7nri/poHgich2uAXqkqqp3rOO5x1bV5SKyBegELCvJOUw+qrD9W3d/wcaPIDwSel3hkkGTrqGOzhhTgRRVUlgOKJD79fEOv3UKtDvVk4nI+cCdwNmqetRveWPgoKpmi0g7oCPwy6ke3+STnQXr3nHJYPdKqNUAht4OA34D0U1DHZ0xpgIqapTUtgAiEqWq6f7rRCSquAOLyDxgGNBIRBKA+3C9jWoCn4mrqsjtejoUeEBEMoEcYJoNz10K6YfdWETfPwMpO6FBexjzGPSeADVqhzo6Y0wFFkibwrdAvwCW5aGqEwpYPKeQbd/EjcZqSiN5p0sEy1+CjFRocyZc8DfodL4NTmeMCUhRbQrNgJZALRHpy4lqpHqAfd2sSHatcFVEP73jXne/2LUXtCwybxtjzEmKKimcB1wHtAIe81ueCswIYkwmEDk58PPHLhls/x/UiIYzfutuNotpHerojDGVVFFtCi8BL4nIr7zqHVMRZByF1a/Cd/+Gg1ugfms49yHodw1E1Qt1dMaYSq6o6qOJqvoKEC8iv8+/XlUfK2A3EyypifDDs+6Gs2MHoUU/uOx56DrOBqczxpSZoq4mdbyfdcsjEFOIxHXe4HQL3PzHnUfD4JsgbpDdbGaMKXNFVR/N8p7+NX+XVBNkqm5Qum+fcoPURdSCvlfDGTdCow6hjs4YU4UFUu/wo4gkAt94j8WqmhLcsKqprOPw45uuZJD4I9RpAiPugf7X2+B0xphyEcjMax1EJA4YAowBnhaRZBvmugwdPegmsln6LBzZC026wbinoeflNjidMaZcBTJ0divgTFxS6A38BCwOclzVQ9IWWDITVv0HMo9C+xFw8dPQfqS1FxhjQiKQ6qMdwA/A/9lsaGVAFXYscfcXbPgAwiK8wemmQ9PuoY7OGFPNBZIU+gJnAVeJyN3AJuArVS1wyApTiOwsWP+eSwa7lkNUDAz5PQycAtHNQh2dMcYAgbUprPaGst6Cq0KaCJxNIeMYmXyOp7rB6ZY8Ayk7oEE7GP0o9LkKatQpfn9jjClHgbQpLMONbPotrvfRUFXdHuzAgu6FMe5nsCaLT0k4MTjd8cPuvoLzH4bOF0BYeHDOaYwxpRRI9dEFqro/6JFUFbtXeYPTve3aD7qNc4PTtTot1JEZY0yxAqk+soRQnJwc2PSJu79g2zdQoy4MnAqnT4XYNqGOzhhjAmaD5pRG5jFYPc8NTpe0Ceq1hFF/gdOuhaj6oY7OGGNOmSWFkjiy3xuc7jk4mgTNe8Olz7l5DMIjQx2dMcaUWFGjpF5a1I6q+lZxBxeR54GxwD5V7eEtawDMB+KBbcAVqnpI3PycTwCjgaPAdaq6IrC3UU72bXDtBWsWQPZx6HSBG5yuzZl2s5kxpkooqqRwofezCTAY+NJ7PRzXE6nYpAC8CDwFvOy37G7gC1V9xLvv4W7gLuACoKP3OB2Y6f0MLVXY+pUbnG7zZxAR5bqTDpoOjTqGOjpjjClTRY2SOglARD4FuqnqHu91c9zFvliq+rWIxOdbPA4Y5j1/CViESwrjgJdVVYElIhIjIs1zz1vusjL8BqdbC3Uaw/A/Qv/JUKdRSEIyxphgC6RNoXW+C3MiEFeKczb1O95eoKn3vCWw02+7BG9ZnqQgIlOAKQBxcaUJoxDHDsGyF2DpbEjdA427wEVPQs8rIDKq7M9njDEVSCBJ4QsR+QSY570eD3xeFidXVRURPcV9ZgOzAfr3739K+xbp4FY3ON3KVyAzDdoNc8mgwznWXmCMqTYCuU/hJhG5BBjqLZqtqm+X4pyJudVCXlXUPm/5LsB/xvlW3rLg2vE9fPekG5xOwqHnZa69oFnPoJ/aGGMqmiKTgoiEAz+pahegNInA33vAtcAj3s93/ZbfJCKv4RqYU4LWnpCdBWkH4PAueP5cd0/Bmbe4wenqtQjKKY0xpjIoMimoaraIbBSROFXdcaoHF5F5uEblRiKSANyHSwYLROR6YDtwhbf5h7juqJtxXVInner5ArZ9MRzY4HoSXfB315uopk1FbYwxgbQpxAI/ichSIC13oapeVNyOqjqhkFUjC9hWgekBxFN6bc92s5tFxcLpU8rllMYYUxkEkhTuDXoU5U0Eatmcx8YYk18gDc1flUcgxhhjQi+suA1E5AwR+UFEjohIhohki8jh8gjOGGNM+So2KeCGqZiAm4azFnAD8HQwgzLGGBMagSQFVHUzEK6q2ar6AnB+cMMyxhgTCoE0NB8VkRrAKhH5G27YiYCSiTHGmMolkIv71d52N+G6pLYGfhXMoIwxxoRGICWFDrj5EA4Dfw5yPMYYY0IokJLCNcBqEVkiIn8XkQtFJDbYgRljjCl/gdyncC2AiLQALsP1PGoRyL7GGGMql2Iv7CIyERgC9AQO4LqofhPkuIwxxoRAIN/2/wlsAZ4BFqrqtqBGVF4mfRDqCIwxpkTGz/oOgPlTB5X5sYttU1DVRsBkIAp4SESWisjcMo/EGGNMyAUyzEU93PSbbYB4oD6QE9ywjDHGhEIg1UeL/R5PqWpCcEMyxhgTKoH0PuoFICK1VfVo8EMyxhgTKoFUHw0SkXXABu91bxH5d9AjM8YYU+4CuXntn8B5QBKAqq4Ghpb0hCLSWURW+T0Oi8jvROR+Ednlt3x0Sc9hjDGmZAK6AU1Vd4qI/6Lskp5QVTcCfQBEJBzYBbyNm5P5cVV9tKTHNsYYUzqBJIWdIjIYUBGJBG4B1pfR+UcCW1R1e76kY4wxJgQCqT6aBkwHWuK+1fcBbiyj818JzPN7fZOIrBGR5wsbX0lEpojIMhFZtn///jIKwxhjDAR289oBVf21qjZV1SbA/wN+W9oTe3M0XAS87i2aCbTHJZ09wD8KiWe2qvZX1f6NGzcubRjGGGP8FJoURKS1iMwWkfdF5HoRqSMijwIbgSZlcO4LgBWqmgigqonezG45wLPAwDI4hzHGmFNQVEnhZWA38CTQA1iGq0Lqpaq3lMG5J+BXdSQizf3WXQL8WAbnMMYYcwqKamhuoKr3e88/EZHLgV973+RLRUTqAKOAqX6L/yYifQAFtuVbZ4wxphwU2fvIa+zN7RaUBNQXr5uQqh4s6UlVNQ1omG/Z1SU9njHGmLJRVFKoDyznRFIAWOH9VKBdsIIyxhgTGoUmBVWNL8c4jDHGVACB3KdgjDGmmrCkYIwxxiegsY+MMcZUHFnZOWSrBuXYASUFETkL6KiqL4hIY6Cuqm4NSkTGGGN8snOUTftSWbkjmRXbD7FyZzKb9x2hYZ0aQTlfsUlBRO4D+gOdgReASOAV4MygRGSMMdXYobQMVu1MZsWOQ6zckcyqnckcOZ4FQGztSPrGxZKemU29qOBU9ARy1EuAvnjdUVV1t4hEByUaY4ypRrKyc/g58YgvAazccYhfDqQBECbQpVk9Lu7bgn5xsfSNiyW+YW1EhPGzvgtaTIEkhQxVVRFR8N2NbIwx5hQlHTnuqoG8JLA6IZmjGW56moZ1atA3LpbL+reib+tYerWqT52a5d/sG8gZF4jILCBGRH4DTMYNWGeMMaYQmdk5bNybyoodh3xtAduT3DT3EWFC1+b1uPy0VvRrE0vf1rG0blCLijCvTLFJQVUfFZFRwGFcu8KfVPWzoEdmjDGVyL7U9DylgDUJyaRnuqHiGkfXpF9cDFcNjKNvXCw9W9anVo3wEEdcsECn4/wMsERgjDFARlYO6/YcZuWOQ6zw2gISDh0DIDJc6NaiPhMGxnltATG0jKkYpYBABNL7KBU31pG/FNxQ2rep6i/BCMwYYyqKvSnpXgJwpYC1u1I4nuVKAc3rR9E3LobrBsfTNy6G7i3qExVZMUsBgQikpPBPIAF4FTc43pW4GdJWAM8Dw4IVnDHGlLfjWdn8tPuwawfwSgG7U9IBqBERRs+W9bn6jDauLSAuhub1a4U44rIVSFK4SFV7+72eLSKrVPUuEZkRrMCMMSbYVJXduaWA7cms3HmIn3YdJiPblQJaxtSiX5tYbvCqgbq1qEfNiMpbCghEIEnhqIhcAbzhvb4MSPeeB+c+a2OMCYL0zGzW7krJkwQSDx8HoGZEGL1a1WfSmfH0jYulX1wMTepFhTji8hdIUvg18ATwb1wSWAJMFJFawE1BjM0YY0pMVUk4dCzPjWE/7T5MVo77LhvXoDZntGvoawzu2rwekeE2RmggXVJ/AS4sZPXikp5YRLYBqUA2kKWq/UWkATAfiMdNyXmFqh4q6TmMMdXH0Yws1iSk5OkWeuCIKwXUigynd+v6/GZoO/rFxdKndQyNo2uGOOKKKZDeR1HA9UB3wFeWUtXJZXD+4ap6wO/13cAXqvqIiNztvb6rDM5jjCmB3OEU5k8dFOJI8lJVticdZeVOVw20YschNuxNJdsrBbRtVIehHRvRt00sfVvH0KVZNBFWCghIINVHc4ENwHnAA7jqpPVBimccJ3ozvQQswpKCMdVe2vEsVick5xkp9GBaBgB1aoTTJy6G357dnn5tYujTOpYGQRpBtDoIJCl0UNXLRWScqr4kIq8C35TBuRX41BtTaZaqzgaaquoeb/1eoGkZnMcYU4moKr8cSMtTDbRx72G8QgDtG9dhRJcmvraATk2jCQ+rHDeGVQaBJIVM72eyiPTAXayblMG5z1LVXSLSBPhMRDb4r/QfhM+fiEwBpgDExcWVQRjGmFBKTc9k1c5kXxJYtTOZ5KPushNdM4I+cTGMGtGRfnEx9GkdQ0xtKwUEUyBJYbaIxAL3AO8BdYF7S3tiVd3l/dwnIm8DA4FEEWmuqntEpDmwr4D9ZgOzAfr3729dYo2pRHJylC37j+QpBfy8LxVVEIGOTepyXrdm9GsTQ9+4WDo0rkuYlQLKVZFJQUTCgMNeD6CvgXZlcVJv+O0wVU31np+La694D7gWeMT7+W5ZnM8YExopRzNZufNQnlJAarqbMKZ+rUj6xsUwumdz+rWJoXfrGOpFRYY4YlNkUlDVHBG5E1hQxudtCrztDRAVAbyqqh+LyA+4obqvB7YDV5TxeY0xQVLYtJHgJozp1DSasb1a0C/OlQLaNapjpYAKKJDqo89F5Hbc/QNpuQtV9WBJT+rd+9C7gOVJwMiSHtcYU34OpWXkKQWs3pmSZ9rIfnGxXNzHzRrWq3UMdUMwYYw5dYF8SuO9n9P9lillVJVkjKn4srJz2JiYeqIaaEeyb9rI8DChS7No37SR/eJiaeNNG2mCI5j3jQRyR3PboJ3dGFPhpB3PYtO+I2zce5jtSWmkZWTT68+f+qaNbFT3xLSR/eLctJG1a1gpoKoI5I7m2sDvgThVnSIiHYHOqvp+0KMzxgRNZnYO2w6ksWFvKj8nprJhbyob96ay4+BR3zZh4oaIqIjTRprgCCS9vwAsBwZ7r3cBrwOWFIypBHKHh9649zAb97oSwIa9qfyyP803RHR4mNC2UR16tqzPZae1onOzaLo0i+aO11cjIvx5XI8QvwtTXgJJCu1VdbyITABQ1aNiXxOMqZCSj2awcW8qG71v/j97z3O7gYKbKaxzs2jO7tyYLs2i6dQ0mvaN6xY4W5j9q1c/gSSFDG+YbAUQkfbA8aBGZYwpUnpmNpv3HfGqfA6zMdGVAHLnBgCoFxVBl2b1uLhPSzp53/w7NY2mfi27F8AULpCkcD/wMdBaRP4DnAlcF8SYjDGe7Bxle1Janjr/jXtT2ZaU5hsLqEZEGB2b1OXM9o3o3Czaq/qpR9N6Ne2bvjllgfQ++lRElgNn4OZoviXfcNfGmFJSVfanHvdd+HMbfzftSyU909X7i0B8wzp0alqXsb1b+L75xzesbcNCmzITSO+j/wKvAu+palpx2xtjipaanun75v+zlwA2Jqb6BoEDaBxdky7Nopl4ehtf1U/HJtHUqlG15wc2oRdI9dGjuBvYHvGGoXgNeF9V04vezZjqLSMrhy37j5xU9bMr+Zhvm7o1I+jUtC4X9GhG56bRdG5Wj87Nom0+ABMygVQffQV8JSLhwAjgN8DzQL0gx2ZMpZCT4+YC3piY6uvu+XOi6/KZOx9wZLjQvnFdTmsTy1Wnx/mqflrFWp9/U7EEdBui1/voQlyJoR9uVjRjqp2kI8fz1Plv2JvKpsRU0ry7fQFaxdaiS7NoRnVrSqemrtG3baM61Iiwen9T8QXSprAAN9fBx8BTwFeqmhPswIwJpaMZWfyceMSvzt/d+JU7ETxAgzo16Nw0msv7t/b1+unUNLpKDfxW0eZmNsEXyF/vHGCCqmYDiMhZIjJBVacXs58xFV5Wdg7bktLy1PlvTHRDPajX5bNWZDidmtZleOfGvu6enZrVpXFd6/Jpqp5A2hQ+EZG+3h3NVwBbgbeCHpmpdsbP+g4IzrdTVWVPSvpJVT9b9h05aaiHHi3q86t+bqiHzk2jiWtQ28b9N9VGoUlBRDoBE7zHAdx8CqKqw8sptqAK5gXIhFbK0Uw27D2ct9dPIUM9DO3UyOv1U/hQD8ZUJ0WVFDYA3wBjVXUzgIjcWi5RGROA3KEeci/6udU/ew+f6C2dO9TDuD4t6Nysng31YEwxikoKlwJXAgtF5GPc/QmlLkOLSGvgZdyUnArMVtUnROR+XHfX/d6mM1T1w9Kez1R+2TnKjoNHT4zymei6fW47cPJQD4PbN/Q1+nZuFk2zelFW72/MKSg0KajqO8A7IlIHGAf8DmgiIjOBt1X10xKeMwu4TVVXiEg0sFxEPvPWPa6qj5bwuKaSU1Uys5VvNu331f1v3HvyUA9tGtSmczM3329u1Y8N9WBM2QikoTkNN8zFqyISC1wO3AWUKCmo6h5gj/c8VUTWAy1LcixTOlnZOWRmKxlZORzPziYjK8c9snNOPM/K4bj3OtN/uff8eAH75G53PN9xMgrY33/ZsUzX1//qOUsBG+rBmFA4pQ7VqnoImO09Sk1E4oG+wPe40VdvEpFrgGW40sShsjhPRZCVnfcCeLyMLqQnXaCzc8jIyvZd7E++gGf7XudWvZSFyHChRngYNSL8HuFhRIaHUdN7HRUZRr2oCG99uG/73PXvr95NZHgYj/yqlw31YEyIhOwuGxGpC7wJ/E5VD3vVUn/BtTP8BfgHMLmA/aYAUwDi4uJKdO7U9ExSjmWiqnz8417fRTL/N+HjeS68+b9Jq+8CXNiF+rjfMcvjAlwjItxdZMPDqBUZTv1akW7bAi7AefYPL2KZ3+vcfSPzbxceViZdNlfvTAZgUPuGpT6WMaZkQpIURCQSlxD+o6pvAahqot/6Zylkuk9V9ZVU+vfvX6JL7Zb97mYlgGmvLC9y24IulL4LrXcBrl0jgphCLqRFXWxr5jlmwRfggva3hlNjTLCUe1LwpvKcA6xX1cf8ljf32hsALgF+DFYMHZrUpWuzaMJE+Nvlvajpf1G2C7AxphoLRUnhTOBqYK2IrPKWzQAmiEgfXPXRNmBqsAKoWzOCel4/9e4t6gfrNMYYU+mUe1JQ1cUUfL+D3ZNgjDEhVnWGczSVng05Ykzo2d0+xhhjfCwpGGOM8bGkYIwxxqfatilY/bUxxpzMSgrGGGN8LCkYY4zxsaRgjDHGx5KCMcYYH0sKxhhjfCwpGGOM8bGkYIwxxseSgjHGGB9LCsYYY3xEtQzniSxnIrIf2F6KQzQCDpRROKZs2GdS8dhnUjGV5nNpo6qNC1pRqZNCaYnIMlXtH+o4zAn2mVQ89plUTMH6XKz6yBhjjI8lBWOMMT7VPSnMDnUA5iT2mVQ89plUTEH5XKp1m4Ixxpi8qntJwRhjjB9LCsYYY3yqXVIQkedFZJ+I/BjqWMwJItJaRBaKyDoR+UlEbgl1TMYRkXARWSki74c6FgMicqv3P/KjiMwTkaiyPH61SwrAi8D5oQ7CnCQLuE1VuwFnANNFpFuIYzLOLcD6UAdhQERaAjcD/VW1BxAOXFmW56h2SUFVvwYOhjoOk5eq7lHVFd7zVNxFqGVoozIi0goYAzwX6liMTwRQS0QigNrA7rI8eLVLCqbiE5F4oC/wfWgjMcA/gTuBnFAHYkBVdwGPAjuAPUCKqn5aluewpGAqFBGpC7wJ/E5VD4c6nupMRMYC+1R1eahjMY6IxALjgLZAC6COiEwsy3NYUjAVhohE4hLCf1T1rVDHYzgTuEhEtgGvASNE5JXQhlTtnQNsVdX9qpoJvAUMLssTWFIwFYKICDAHWK+qj4U6HgOq+gdVbaWq8bjGzC9VtUy/lZpTtgM4Q0Rqe/8zIynjTgDVLimIyDzgO6CziCSIyPWhjskA7lvp1bhvo6u8x+hQB2VMRaKq3wNvACuAtbhreJkOd2HDXBhjjPGpdiUFohOAJAAAAyhJREFUY4wxhbOkYIwxxseSgjHGGB9LCsYYY3wsKRhjjPGxpGCqHBHJ9rq0/igi/xWRGG95vIgc8+vyukpErvHWbRORtSKyRkQ+FZFmfssbFXKe34lIuojUF2exiFzgt/5yEfm4gP38z/WViLTJt/4dEVniPT/PL9YjIrLRe/6yiAzzH7lURC72jrneO/7FZfH7NNWLJQVTFR1T1T7eKJIHgel+67Z463IfL/utG66qvYBlwIwAzjMB+AG4VF3f7mnAYyLy/9u7d9aogjCM4/8HLRQE0UZIYzBaiLcihZVFugiiKChZhXwBQQRvKCraRJBolU9goxYSMZWkiKighWAI3pqwjWCrEBVR81rMnMPxkMSz4gWW5wdb7Ow5s2+aec/MZN9Zkct1jNS+u6r4rgfA+aIxJ7B+YLWkDRFxv4g1x3Ukvx+udiZpB6kmzr6I2AzsBUYlbW/wd5iVnBSs2z2h82qrD4GNS10gqQ9YRRrQWwAR8QKYAM4AF4EbETHbYXwHch+36Kwk8klgJCLaOZY2cAU41UEfZk4K1r0kLSOVAbhXae6rLR/tWuDWPaRfiy5liDRwPyL9On5dbr8MHAZ2A1cbhDkI3K28bwE386vV4P7CFqBeuO5ZbjdrbPn/DsDsL1gpaZr0BP4amKx8NpuXYhYyJek7MENlSWcRLWB/RMxLugMcBMYi4qOk28BcRHxZ4v4pSWuBOeACQE4sm4DHERGSvkrammcgZv+EZwrWjT7ngX89IBZf168bKNbrI+L9YhdJ2kYavCdzBdEhfn6qn+fX5w8M5PimSbMLgEPAGqCd++2l+WzhFWkvoqofeNnwfjPAScG6WER8Ih1deCKfUvWntIBLEdGbXz1AT/2/iBrE9w04DgznWUMLGCz6JQ3qTfcVRoGz+YCi4qCic8C1TmIyc1KwrhYRz0nLQcUTd31P4ViDbmZyRd23kq6TBurx2jXj/MZZuRHxjrR/cJQ0c3ha+awNfJC0s0E/06QN7glJb0ib1adzu1ljrpJqZmYlzxTMzKzkpGBmZiUnBTMzKzkpmJlZyUnBzMxKTgpmZlZyUjAzs9IPPkDHqzKT0poAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "LSUfGt6N5cPY"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}