PART_1_final_Q_learning (1).ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "PART 1 final Q learning.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "APPENDIX 1: Q learneing PArt 1"
      ],
      "metadata": {
        "id": "naQiLtMP533r"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -U gym\n"
      ],
      "metadata": {
        "id": "IVyAeIkhrz9D",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "88beca02-e811-4da1-dbce-cbce6cef5d4c"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: gym in /usr/local/lib/python3.7/dist-packages (0.17.3)\n",
            "Collecting gym\n",
            "  Downloading gym-0.23.1.tar.gz (626 kB)\n",
            "\u001b[K     |████████████████████████████████| 626 kB 4.0 MB/s \n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "    Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting gym-notices>=0.0.4\n",
            "  Downloading gym_notices-0.0.6-py3-none-any.whl (2.7 kB)\n",
            "Requirement already satisfied: importlib-metadata>=4.10.0 in /usr/local/lib/python3.7/dist-packages (from gym) (4.11.3)\n",
            "Requirement already satisfied: numpy>=1.18.0 in /usr/local/lib/python3.7/dist-packages (from gym) (1.21.6)\n",
            "Requirement already satisfied: cloudpickle>=1.2.0 in /usr/local/lib/python3.7/dist-packages (from gym) (1.3.0)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.10.0->gym) (3.8.0)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.10.0->gym) (4.2.0)\n",
            "Building wheels for collected packages: gym\n",
            "  Building wheel for gym (PEP 517) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for gym: filename=gym-0.23.1-py3-none-any.whl size=701376 sha256=befc925e9a882474b1e6cc22d3c8b675a815b9a882fd089f628ea2dcb7839b2a\n",
            "  Stored in directory: /root/.cache/pip/wheels/e3/33/04/6723848e46f0f1ebe794bb329b7c761c3329a0d7ffade99da7\n",
            "Successfully built gym\n",
            "Installing collected packages: gym-notices, gym\n",
            "  Attempting uninstall: gym\n",
            "    Found existing installation: gym 0.17.3\n",
            "    Uninstalling gym-0.17.3:\n",
            "      Successfully uninstalled gym-0.17.3\n",
            "Successfully installed gym-0.23.1 gym-notices-0.0.6\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import gym\n",
        "!python -m pip install pygame\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import random\n",
        "!pip install wandb\n",
        "import wandb"
      ],
      "metadata": {
        "id": "z_QxE8zWSvsl",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "113adde3-9354-458a-fd9e-281b5f5e26af"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting pygame\n",
            "  Downloading pygame-2.1.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (21.8 MB)\n",
            "\u001b[K     |████████████████████████████████| 21.8 MB 1.7 MB/s \n",
            "\u001b[?25hInstalling collected packages: pygame\n",
            "Successfully installed pygame-2.1.2\n",
            "Collecting wandb\n",
            "  Downloading wandb-0.12.15-py2.py3-none-any.whl (1.8 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.8 MB 4.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: promise<3,>=2.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (2.3)\n",
            "Requirement already satisfied: protobuf>=3.12.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (3.17.3)\n",
            "Collecting setproctitle\n",
            "  Downloading setproctitle-1.2.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (29 kB)\n",
            "Requirement already satisfied: six>=1.13.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (1.15.0)\n",
            "Requirement already satisfied: Click!=8.0.0,>=7.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (7.1.2)\n",
            "Collecting docker-pycreds>=0.4.0\n",
            "  Downloading docker_pycreds-0.4.0-py2.py3-none-any.whl (9.0 kB)\n",
            "Requirement already satisfied: python-dateutil>=2.6.1 in /usr/local/lib/python3.7/dist-packages (from wandb) (2.8.2)\n",
            "Collecting GitPython>=1.0.0\n",
            "  Downloading GitPython-3.1.27-py3-none-any.whl (181 kB)\n",
            "\u001b[K     |████████████████████████████████| 181 kB 53.8 MB/s \n",
            "\u001b[?25hCollecting shortuuid>=0.5.0\n",
            "  Downloading shortuuid-1.0.8-py3-none-any.whl (9.5 kB)\n",
            "Requirement already satisfied: psutil>=5.0.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (5.4.8)\n",
            "Collecting pathtools\n",
            "  Downloading pathtools-0.1.2.tar.gz (11 kB)\n",
            "Requirement already satisfied: PyYAML in /usr/local/lib/python3.7/dist-packages (from wandb) (3.13)\n",
            "Requirement already satisfied: requests<3,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (2.23.0)\n",
            "Collecting sentry-sdk>=1.0.0\n",
            "  Downloading sentry_sdk-1.5.10-py2.py3-none-any.whl (144 kB)\n",
            "\u001b[K     |████████████████████████████████| 144 kB 40.1 MB/s \n",
            "\u001b[?25hCollecting gitdb<5,>=4.0.1\n",
            "  Downloading gitdb-4.0.9-py3-none-any.whl (63 kB)\n",
            "\u001b[K     |████████████████████████████████| 63 kB 1.6 MB/s \n",
            "\u001b[?25hRequirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from GitPython>=1.0.0->wandb) (4.2.0)\n",
            "Collecting smmap<6,>=3.0.1\n",
            "  Downloading smmap-5.0.0-py3-none-any.whl (24 kB)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.0.0->wandb) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.0.0->wandb) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.0.0->wandb) (2021.10.8)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.0.0->wandb) (3.0.4)\n",
            "Building wheels for collected packages: pathtools\n",
            "  Building wheel for pathtools (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pathtools: filename=pathtools-0.1.2-py3-none-any.whl size=8806 sha256=ef4a0e221d397ed6e5be8c77fe462e0708b80627681d529ce45cf229ecbd5008\n",
            "  Stored in directory: /root/.cache/pip/wheels/3e/31/09/fa59cef12cdcfecc627b3d24273699f390e71828921b2cbba2\n",
            "Successfully built pathtools\n",
            "Installing collected packages: smmap, gitdb, shortuuid, setproctitle, sentry-sdk, pathtools, GitPython, docker-pycreds, wandb\n",
            "Successfully installed GitPython-3.1.27 docker-pycreds-0.4.0 gitdb-4.0.9 pathtools-0.1.2 sentry-sdk-1.5.10 setproctitle-1.2.3 shortuuid-1.0.8 smmap-5.0.0 wandb-0.12.15\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "The follwing enviorment is taken from https://github.com/openai/gym/blob/master/gym/envs/toy_text/taxi.py and adapted."
      ],
      "metadata": {
        "id": "Oo6IEUKb6Bzp"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#First we create the enviorment. \n",
        "from contextlib import closing\n",
        "from io import StringIO\n",
        "from os import path\n",
        "from typing import Optional\n",
        "\n",
        "import numpy as np\n",
        "\n",
        "from gym import Env, spaces, utils\n",
        "import numpy as np\n",
        "\n",
        "\n",
        "def categorical_sample(prob_n, np_random):\n",
        "    \"\"\"\n",
        "    Sample from categorical distribution\n",
        "    Each row specifies class probabilities\n",
        "    \"\"\"\n",
        "    prob_n = np.asarray(prob_n)\n",
        "    csprob_n = np.cumsum(prob_n)\n",
        "    return (csprob_n > np_random.random()).argmax()\n",
        "#Apapted the enviorment representation\n",
        "MAP = [\n",
        "    \"+---------+\",\n",
        "    \"|R: :O: :G|\",\n",
        "    \"| : : : : |\",\n",
        "    \"|O: :O: :O|\",\n",
        "    \"| : : : : |\",\n",
        "    \"|Y| :O|B: |\",\n",
        "    \"+---------+\",\n",
        "]\n",
        "WINDOW_SIZE = (550, 350)\n",
        "\n",
        "\n",
        "class WareEnv(Env):\n",
        "    \"\"\"\n",
        "    The Taxi Problem\n",
        "    from \"Hierarchical Reinforcement Learning with the MAXQ Value Function Decomposition\"\n",
        "    by Tom Dietterich\n",
        "    ### Description\n",
        "    There are four designated locations in the grid world indicated by R(ed),\n",
        "    G(reen), Y(ellow), and B(lue). When the episode starts, the taxi starts off\n",
        "    at a random square and the passenger is at a random location. The taxi\n",
        "    drives to the passenger's location, picks up the passenger, drives to the\n",
        "    passenger's destination (another one of the four specified locations), and\n",
        "    then drops off the passenger. Once the passenger is dropped off, the episode ends.\n",
        "    Map:\n",
        "        +---------+\n",
        "        |R: | : :G|\n",
        "        | : | : : |\n",
        "        | : : : : |\n",
        "        | | : | : |\n",
        "        |Y| : |B: |\n",
        "        +---------+\n",
        "    ### Actions\n",
        "    There are 6 discrete deterministic actions:\n",
        "    - 0: move south\n",
        "    - 1: move north\n",
        "    - 2: move east\n",
        "    - 3: move west\n",
        "    - 4: pickup passenger\n",
        "    - 5: drop off passenger\n",
        "    ### Observations\n",
        "    There are 500 discrete states since there are 25 taxi positions, 5 possible\n",
        "    locations of the passenger (including the case when the passenger is in the\n",
        "    taxi), and 4 destination locations.\n",
        "    Note that there are 400 states that can actually be reached during an\n",
        "    episode. The missing states correspond to situations in which the passenger\n",
        "    is at the same location as their destination, as this typically signals the\n",
        "    end of an episode. Four additional states can be observed right after a\n",
        "    successful episodes, when both the passenger and the taxi are at the destination.\n",
        "    This gives a total of 404 reachable discrete states.\n",
        "    Passenger locations:\n",
        "    - 0: R(ed)\n",
        "    - 1: G(reen)\n",
        "    - 2: Y(ellow)\n",
        "    - 3: B(lue)\n",
        "    - 4: in taxi\n",
        "    Destinations:\n",
        "    - 0: R(ed)\n",
        "    - 1: G(reen)\n",
        "    - 2: Y(ellow)\n",
        "    - 3: B(lue)\n",
        "    ### Rewards\n",
        "    - -1 per step unless other reward is triggered.\n",
        "    - +20 delivering passenger.\n",
        "    - -10  executing \"pickup\" and \"drop-off\" actions illegally.\n",
        "    state space is represented by:\n",
        "    (taxi_row, taxi_col, passenger_location, destination)\n",
        "    ### Arguments\n",
        "    ```\n",
        "    gym.make('Taxi-v3')\n",
        "    ```\n",
        "    ### Version History\n",
        "    * v3: Map Correction + Cleaner Domain Description\n",
        "    * v2: Disallow Taxi start location = goal location, Update Taxi observations in the rollout, Update Taxi reward threshold.\n",
        "    * v1: Remove (3,2) from locs, add passidx<4 check\n",
        "    * v0: Initial versions release\n",
        "    \"\"\"\n",
        "\n",
        "    metadata = {\"render_modes\": [\"human\", \"ansi\", \"rgb_array\"], \"render_fps\": 4}\n",
        "\n",
        "    def __init__(self):\n",
        "        self.desc = np.asarray(MAP, dtype=\"c\")\n",
        "\n",
        "        self.locs = locs = [(0, 0), (0, 4), (4, 0), (4, 3)]\n",
        "        self.box = box = [(0,2), (2,0), (2,2), (2,4), (4, 2)]\n",
        "        self.locs_colors = [(255, 0, 0), (0, 255, 0), (255, 255, 0), (0, 0, 255)]\n",
        "\n",
        "        num_states = 500\n",
        "        num_rows = 5\n",
        "        num_columns = 5\n",
        "        max_row = num_rows - 1\n",
        "        max_col = num_columns - 1\n",
        "        self.initial_state_distrib = np.zeros(num_states)\n",
        "        num_actions = 6\n",
        "        self.P = {\n",
        "            state: {action: [] for action in range(num_actions)}\n",
        "            for state in range(num_states)\n",
        "        }\n",
        "        for row in range(num_rows):\n",
        "            for col in range(num_columns):\n",
        "                for pass_idx in range(len(locs) + 1):  # +1 for being inside taxi\n",
        "                    for dest_idx in range(len(locs)):\n",
        "                        state = self.encode(row, col, pass_idx, dest_idx)\n",
        "                        if pass_idx < 4 and pass_idx != dest_idx:\n",
        "                            self.initial_state_distrib[state] += 1\n",
        "                        for action in range(num_actions):\n",
        "                            # defaults\n",
        "                            new_row, new_col, new_pass_idx = row, col, pass_idx\n",
        "                            reward = (\n",
        "                                -1\n",
        "                            )  # default reward when there is no pickup/dropoff\n",
        "                            done = False\n",
        "                            taxi_loc = (row, col)\n",
        "\n",
        "                            if action == 0:\n",
        "                                if pass_idx == 4 and (min(row + 1, max_row), col) in self.box: #if agent carrying target then cannot pass O cells\n",
        "                                    reward = -2 #also added a penalty of -2 \n",
        "                                    #pass\n",
        "                                else:\n",
        "                                    new_row = min(row + 1, max_row)\n",
        "                            elif action == 1:\n",
        "                                if pass_idx ==4 and (max(row - 1, 0), col) in self.box: #if agent carrying target then cannot pass O cells\n",
        "                                    reward = -2 #also added a penalty of -2\n",
        "                                    #pass\n",
        "                                else:\n",
        "                                    new_row = max(row - 1, 0)\n",
        "                            if action == 2 and self.desc[1 + row, 2 * col + 2] == b\":\":\n",
        "                                if pass_idx == 4 and (row, min(col + 1, max_col)) in self.box: #if agent carrying target then cannot pass O cells\n",
        "                                    reward = -2 #also added a penalty of -2\n",
        "                                    pass\n",
        "                                else:\n",
        "                                    new_col = min(col + 1, max_col)\n",
        "                            elif action == 3 and self.desc[1 + row, 2 * col] == b\":\":\n",
        "                                if pass_idx == 4 and (row, max(col - 1, 0)) in self.box:\n",
        "                                    reward = -2 #also added a penalty of -2\n",
        "                                    pass\n",
        "                                else:\n",
        "                                    new_col = max(col - 1, 0)\n",
        "                            elif action == 4:  # pickup\n",
        "                                if pass_idx < 4 and taxi_loc == locs[pass_idx]:\n",
        "                                    new_pass_idx = 4\n",
        "                                else:  # passenger not at location\n",
        "                                    reward = -10\n",
        "                            elif action == 5:  # dropoff\n",
        "                                if (taxi_loc == locs[dest_idx]) and pass_idx == 4:\n",
        "                                    new_pass_idx = dest_idx\n",
        "                                    done = True\n",
        "                                    reward = 20\n",
        "                                elif (taxi_loc in locs) and pass_idx == 4:\n",
        "                                    new_pass_idx = locs.index(taxi_loc)\n",
        "                                else:  # dropoff at wrong location\n",
        "                                    reward = -10\n",
        "                            new_state = self.encode(\n",
        "                                new_row, new_col, new_pass_idx, dest_idx\n",
        "                            )\n",
        "                            self.P[state][action].append((1.0, new_state, reward, done))\n",
        "        self.initial_state_distrib /= self.initial_state_distrib.sum()\n",
        "        self.action_space = spaces.Discrete(num_actions)\n",
        "        self.observation_space = spaces.Discrete(num_states)\n",
        "\n",
        "        #rest of cell is the same as in the gym ai base class\n",
        "        # pygame utils\n",
        "        self.window = None\n",
        "        self.clock = None\n",
        "        self.cell_size = (\n",
        "            WINDOW_SIZE[0] / self.desc.shape[1],\n",
        "            WINDOW_SIZE[1] / self.desc.shape[0],\n",
        "        )\n",
        "        self.taxi_imgs = None\n",
        "        self.taxi_orientation = 0\n",
        "        self.passenger_img = None\n",
        "        self.destination_img = None\n",
        "        self.median_horiz = None\n",
        "        self.median_vert = None\n",
        "        self.background_img = None\n",
        "\n",
        "    def encode(self, taxi_row, taxi_col, pass_loc, dest_idx):\n",
        "        # (5) 5, 5, 4\n",
        "        i = taxi_row\n",
        "        i *= 5\n",
        "        i += taxi_col\n",
        "        i *= 5\n",
        "        i += pass_loc\n",
        "        i *= 4\n",
        "        i += dest_idx\n",
        "        return i\n",
        "\n",
        "    def decode(self, i):\n",
        "        out = []\n",
        "        out.append(i % 4)\n",
        "        i = i // 4\n",
        "        out.append(i % 5)\n",
        "        i = i // 5\n",
        "        out.append(i % 5)\n",
        "        i = i // 5\n",
        "        out.append(i)\n",
        "        assert 0 <= i < 5\n",
        "        return reversed(out)\n",
        "\n",
        "    def step(self, a):\n",
        "        transitions = self.P[self.s][a]\n",
        "        i = categorical_sample([t[0] for t in transitions], self.np_random)\n",
        "        p, s, r, d = transitions[i]\n",
        "        self.s = s\n",
        "        self.lastaction = a\n",
        "        return (int(s), r, d, {\"prob\": p})\n",
        "\n",
        "    def reset(\n",
        "        self,\n",
        "        *,\n",
        "        seed: Optional[int] = None,\n",
        "        return_info: bool = False,\n",
        "        options: Optional[dict] = None,\n",
        "    ):\n",
        "        super().reset(seed=seed)\n",
        "        self.s = categorical_sample(self.initial_state_distrib, self.np_random)\n",
        "        self.lastaction = None\n",
        "        self.taxi_orientation = 0\n",
        "        if not return_info:\n",
        "            return int(self.s)\n",
        "        else:\n",
        "            return int(self.s), {\"prob\": 1}\n",
        "\n",
        "    def render(self, mode=\"human\"):\n",
        "        if mode == \"ansi\":\n",
        "            return self._render_text()\n",
        "        else:\n",
        "            return self._render_gui(mode)\n",
        "\n",
        "    def _render_gui(self, mode):\n",
        "        import pygame  # dependency to pygame only if rendering with human\n",
        "\n",
        "        if self.window is None:\n",
        "            pygame.init()\n",
        "            pygame.display.set_caption(\"Taxi\")\n",
        "            if mode == \"human\":\n",
        "                self.window = pygame.display.set_mode(WINDOW_SIZE)\n",
        "            else:  # \"rgb_array\"\n",
        "                self.window = pygame.Surface(WINDOW_SIZE)\n",
        "        if self.clock is None:\n",
        "            self.clock = pygame.time.Clock()\n",
        "        if self.taxi_imgs is None:\n",
        "            file_names = [\n",
        "                path.join(path.dirname(__file__), \"img/cab_front.png\"),\n",
        "                path.join(path.dirname(__file__), \"img/cab_rear.png\"),\n",
        "                path.join(path.dirname(__file__), \"img/cab_right.png\"),\n",
        "                path.join(path.dirname(__file__), \"img/cab_left.png\"),\n",
        "            ]\n",
        "            self.taxi_imgs = [\n",
        "                pygame.transform.scale(pygame.image.load(file_name), self.cell_size)\n",
        "                for file_name in file_names\n",
        "            ]\n",
        "        if self.passenger_img is None:\n",
        "            file_name = path.join(path.dirname(__file__), \"img/passenger.png\")\n",
        "            self.passenger_img = pygame.transform.scale(\n",
        "                pygame.image.load(file_name), self.cell_size\n",
        "            )\n",
        "        if self.destination_img is None:\n",
        "            file_name = path.join(path.dirname(__file__), \"img/hotel.png\")\n",
        "            self.destination_img = pygame.transform.scale(\n",
        "                pygame.image.load(file_name), self.cell_size\n",
        "            )\n",
        "            self.destination_img.set_alpha(170)\n",
        "        if self.median_horiz is None:\n",
        "            file_names = [\n",
        "                path.join(path.dirname(__file__), \"img/gridworld_median_left.png\"),\n",
        "                path.join(path.dirname(__file__), \"img/gridworld_median_horiz.png\"),\n",
        "                path.join(path.dirname(__file__), \"img/gridworld_median_right.png\"),\n",
        "            ]\n",
        "            self.median_horiz = [\n",
        "                pygame.transform.scale(pygame.image.load(file_name), self.cell_size)\n",
        "                for file_name in file_names\n",
        "            ]\n",
        "        if self.median_vert is None:\n",
        "            file_names = [\n",
        "                path.join(path.dirname(__file__), \"img/gridworld_median_top.png\"),\n",
        "                path.join(path.dirname(__file__), \"img/gridworld_median_vert.png\"),\n",
        "                path.join(path.dirname(__file__), \"img/gridworld_median_bottom.png\"),\n",
        "            ]\n",
        "            self.median_vert = [\n",
        "                pygame.transform.scale(pygame.image.load(file_name), self.cell_size)\n",
        "                for file_name in file_names\n",
        "            ]\n",
        "        if self.background_img is None:\n",
        "            file_name = path.join(path.dirname(__file__), \"img/taxi_background.png\")\n",
        "            self.background_img = pygame.transform.scale(\n",
        "                pygame.image.load(file_name), self.cell_size\n",
        "            )\n",
        "\n",
        "        desc = self.desc\n",
        "\n",
        "        for y in range(0, desc.shape[0]):\n",
        "            for x in range(0, desc.shape[1]):\n",
        "                cell = (x * self.cell_size[0], y * self.cell_size[1])\n",
        "                self.window.blit(self.background_img, cell)\n",
        "                if desc[y][x] == b\"|\" and (y == 0 or desc[y - 1][x] != b\"|\"):\n",
        "                    self.window.blit(self.median_vert[0], cell)\n",
        "                elif desc[y][x] == b\"|\" and (\n",
        "                    y == desc.shape[0] - 1 or desc[y + 1][x] != b\"|\"\n",
        "                ):\n",
        "                    self.window.blit(self.median_vert[2], cell)\n",
        "                elif desc[y][x] == b\"|\":\n",
        "                    self.window.blit(self.median_vert[1], cell)\n",
        "                elif desc[y][x] == b\"-\" and (x == 0 or desc[y][x - 1] != b\"-\"):\n",
        "                    self.window.blit(self.median_horiz[0], cell)\n",
        "                elif desc[y][x] == b\"-\" and (\n",
        "                    x == desc.shape[1] - 1 or desc[y][x + 1] != b\"-\"\n",
        "                ):\n",
        "                    self.window.blit(self.median_horiz[2], cell)\n",
        "                elif desc[y][x] == b\"-\":\n",
        "                    self.window.blit(self.median_horiz[1], cell)\n",
        "\n",
        "        for cell, color in zip(self.locs, self.locs_colors):\n",
        "            color_cell = pygame.Surface(self.cell_size)\n",
        "            color_cell.set_alpha(128)\n",
        "            color_cell.fill(color)\n",
        "            loc = self.get_surf_loc(cell)\n",
        "            self.window.blit(color_cell, (loc[0], loc[1] + 10))\n",
        "\n",
        "        taxi_row, taxi_col, pass_idx, dest_idx = self.decode(self.s)\n",
        "\n",
        "        if pass_idx < 4:\n",
        "            self.window.blit(self.passenger_img, self.get_surf_loc(self.locs[pass_idx]))\n",
        "\n",
        "        if self.lastaction in [0, 1, 2, 3]:\n",
        "            self.taxi_orientation = self.lastaction\n",
        "        dest_loc = self.get_surf_loc(self.locs[dest_idx])\n",
        "        taxi_location = self.get_surf_loc((taxi_row, taxi_col))\n",
        "\n",
        "        if dest_loc[1] <= taxi_location[1]:\n",
        "            self.window.blit(\n",
        "                self.destination_img,\n",
        "                (dest_loc[0], dest_loc[1] - self.cell_size[1] // 2),\n",
        "            )\n",
        "            self.window.blit(self.taxi_imgs[self.taxi_orientation], taxi_location)\n",
        "        else:  # change blit order for overlapping appearance\n",
        "            self.window.blit(self.taxi_imgs[self.taxi_orientation], taxi_location)\n",
        "            self.window.blit(\n",
        "                self.destination_img,\n",
        "                (dest_loc[0], dest_loc[1] - self.cell_size[1] // 2),\n",
        "            )\n",
        "\n",
        "        if mode == \"human\":\n",
        "            pygame.display.update()\n",
        "            self.clock.tick(self.metadata[\"render_fps\"])\n",
        "        else:  # rgb_array\n",
        "            return np.transpose(\n",
        "                np.array(pygame.surfarray.pixels3d(self.window)), axes=(1, 0, 2)\n",
        "            )\n",
        "\n",
        "    def get_surf_loc(self, map_loc):\n",
        "        return (map_loc[1] * 2 + 1) * self.cell_size[0], (\n",
        "            map_loc[0] + 1\n",
        "        ) * self.cell_size[1]\n",
        "\n",
        "    def _render_text(self):\n",
        "        desc = self.desc.copy().tolist()\n",
        "        outfile = StringIO()\n",
        "\n",
        "        out = [[c.decode(\"utf-8\") for c in line] for line in desc]\n",
        "        taxi_row, taxi_col, pass_idx, dest_idx = self.decode(self.s)\n",
        "\n",
        "        def ul(x):\n",
        "            return \"_\" if x == \" \" else x\n",
        "\n",
        "        if pass_idx < 4:\n",
        "            out[1 + taxi_row][2 * taxi_col + 1] = utils.colorize(\n",
        "                out[1 + taxi_row][2 * taxi_col + 1], \"yellow\", highlight=True\n",
        "            )\n",
        "            pi, pj = self.locs[pass_idx]\n",
        "            out[1 + pi][2 * pj + 1] = utils.colorize(\n",
        "                out[1 + pi][2 * pj + 1], \"blue\", bold=True\n",
        "            )\n",
        "        else:  # passenger in taxi\n",
        "            out[1 + taxi_row][2 * taxi_col + 1] = utils.colorize(\n",
        "                ul(out[1 + taxi_row][2 * taxi_col + 1]), \"green\", highlight=True\n",
        "            )\n",
        "\n",
        "        di, dj = self.locs[dest_idx]\n",
        "        out[1 + di][2 * dj + 1] = utils.colorize(out[1 + di][2 * dj + 1], \"magenta\")\n",
        "        outfile.write(\"\\n\".join([\"\".join(row) for row in out]) + \"\\n\")\n",
        "        if self.lastaction is not None:\n",
        "            outfile.write(\n",
        "                f\"  ({['South', 'North', 'East', 'West', 'Pickup', 'Dropoff'][self.lastaction]})\\n\"\n",
        "            )\n",
        "        else:\n",
        "            outfile.write(\"\\n\")\n",
        "\n",
        "        with closing(outfile):\n",
        "            return outfile.getvalue()\n",
        "\n",
        "    def close(self):\n",
        "        if self.window is not None:\n",
        "            import pygame\n",
        "\n",
        "            pygame.display.quit()\n",
        "            pygame.quit()\n"
      ],
      "metadata": {
        "id": "S-3_c22rTDjn"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "env =  WareEnv()\n",
        "env.reset()\n",
        "ALPHA = 0.7\n",
        "GAMMA= 0.7\n",
        "exploration = 0.7\n",
        "episodes = 1000"
      ],
      "metadata": {
        "id": "UogI3mzDsNcE"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "We use the third party plugin below for visualizing the average reward and number of steps/episode. It streams the values to it's website for convenient plotting."
      ],
      "metadata": {
        "id": "ZUz915uZ4ebz"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "wandb.login() #plug in used to visualize and plot"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 126
        },
        "id": "2qK2TNL1TlrZ",
        "outputId": "54a40f02-949f-4c64-f3f7-35d6a8143223"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/tornado/httputil.py:107: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since Python 3.3,and in 3.9 it will stop working\n",
            "  class HTTPHeaders(collections.MutableMapping):\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "\n",
              "        window._wandbApiKey = new Promise((resolve, reject) => {\n",
              "            function loadScript(url) {\n",
              "            return new Promise(function(resolve, reject) {\n",
              "                let newScript = document.createElement(\"script\");\n",
              "                newScript.onerror = reject;\n",
              "                newScript.onload = resolve;\n",
              "                document.body.appendChild(newScript);\n",
              "                newScript.src = url;\n",
              "            });\n",
              "            }\n",
              "            loadScript(\"https://cdn.jsdelivr.net/npm/postmate/build/postmate.min.js\").then(() => {\n",
              "            const iframe = document.createElement('iframe')\n",
              "            iframe.style.cssText = \"width:0;height:0;border:none\"\n",
              "            document.body.appendChild(iframe)\n",
              "            const handshake = new Postmate({\n",
              "                container: iframe,\n",
              "                url: 'https://wandb.ai/authorize'\n",
              "            });\n",
              "            const timeout = setTimeout(() => reject(\"Couldn't auto authenticate\"), 5000)\n",
              "            handshake.then(function(child) {\n",
              "                child.on('authorize', data => {\n",
              "                    clearTimeout(timeout)\n",
              "                    resolve(data)\n",
              "                });\n",
              "            });\n",
              "            })\n",
              "        });\n",
              "    "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Initial Run"
      ],
      "metadata": {
        "id": "hVIuvNVg3NCm"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#function to get moving average\n",
        "def moving_average(x, w):\n",
        "    return np.convolve(x, np.ones(w), 'valid') / w"
      ],
      "metadata": {
        "id": "Si2NsswW9qdZ"
      },
      "execution_count": 41,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#set inital hyper-parameters\n",
        "ALPHA = 1\n",
        "GAMMA = 0.7\n",
        "exploration = 0.7\n",
        "#number of episodes to train\n",
        "episodes = 500\n",
        "rewards = []\n",
        "steps_episode = []\n",
        "#initialize Q table at 0\n",
        "q_table = np.zeros([env.observation_space.n, env.action_space.n])\n",
        "for ep in range(episodes): #number of episodes\n",
        "  reward_sum = 0 #reward per episode\n",
        "  state = env.reset()\n",
        "  done = False\n",
        "  episode_score = 0\n",
        "  steps = 0 #steps per episode\n",
        "  while not done:\n",
        "            if (random.random() < exploration):\n",
        "                action = env.action_space.sample() # Explore a random action\n",
        "            else:\n",
        "                action = np.argmax(q_table[state]) # Use the action with the highest q-value\n",
        "              \n",
        "            next_state, reward, done, _ = env.step(action)\n",
        "            reward_sum += reward #add reward to total\n",
        "            steps += 1 #add step by 1\n",
        "          \n",
        "            prev_q = q_table[state, action]\n",
        "            next_max_q = np.max(q_table[next_state])\n",
        "            new_q = (1 - ALPHA) * prev_q + ALPHA * (reward + GAMMA * next_max_q)\n",
        "            q_table[state, action] = new_q\n",
        "          \n",
        "            state = next_state #next state\n",
        "      ##if done == New Episode\n",
        "  episode_score +=1\n",
        "  #wandb.log({\"Rewards\": reward_sum, \"Steps\": steps, \"Episodes\": episode_score, \"EPSILON\": exploration\n",
        "      \n",
        "  # })\n",
        "  #if done == New Episode\n",
        "  rewards.append(reward_sum)\n",
        "  print('done')\n",
        "  steps_episode.append(steps)\n",
        "  #all episodes finished"
      ],
      "metadata": {
        "id": "zvFusKPi3CF3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "plt.plot(moving_average(rewards,10))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 283
        },
        "id": "f--J2LCz36aw",
        "outputId": "45813d8a-86a0-40dd-dd8c-0a513c7e52cc"
      },
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<matplotlib.lines.Line2D at 0x7faef2884210>]"
            ]
          },
          "metadata": {},
          "execution_count": 40
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAD4CAYAAAAD6PrjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxc9Xnv8c+jmdG+2Za829gYsxgI2AgHErJACDa0iUlKEtP24iw3bhPobZp7k0B43dKblDakS1paQuMWEsgrraFkwZeaEGgC3NAYbMDgDbCwDbawJXnRrhnN8tw/5kiMzcgCj6QZeb7v12teOvOcMzOPjuzzzO/3O+f8zN0REREBKMl3AiIiUjhUFEREZIiKgoiIDFFREBGRISoKIiIyJJzvBHLV0NDg8+bNy3caIiITyrPPPnvQ3RuPjU/4ojBv3jw2bdqU7zRERCYUM3stW1zdRyIiMkRFQUREhqgoiIjIEBUFEREZUnBFwcyWm9nLZtZsZjfmOx8RkWJSUEXBzELAHcCVwCLgWjNblN+sRESKR0EVBWAp0Ozuu9x9AFgLrMhzTiIiRaPQrlOYBezNeL4PePexG5nZamA1wNy5c8cnM5EC5u70DiQJlxjlkdCYfEZH3wDd0QThkDGpsnTMPud43J1YIkVvLEEkXEJteSSn92rtitE7kCBSUkIoZCSTTu9Agt5Ygt6BZPpnLEEskaK6LExtRZjqsgiJVIpoPEn/QPrnQDJFZWmI6rIwVWVhqsvCVJSGcE9/TjzptPfEaO2K0t4dIxKyoe0AUu5MqymnpjzCQDJJLJ4ilkwxkEgRS6R/ppeTR8X+5MOnEyqx0dq9QOEVhbfF3dcAawCampo0IYSMukQyRWt3bOig0BtL0juQoLoszOxJFcyoq6A0PPYNbXfnYM8A+zv7KTGjKxrn1fZe/uPFN9h9sJf6ilJ6BxIc7IkRjacAmFQZoTRcQrikhFCJMbWmjLmTK5kzuZL6yggGlJQYg4eSQ70DdPTFqa+M0NWfYNfBHnpjCaLx9AEvmkjS1Z+gsz9+VG5VpSEaaso4e2Yti+dMIulOJFTCpMoIiZQHB80k/fEkqZQzra6c6rIwKXeSKeiOxtl9sJcX9nXS3NpNKChoZZESysIhyoOfZeESUu7sOdhHW3eUVMb/+FMbqmioLks/sTd/mA3uv+BnsH0y5UMH2IM9MQ71Doz2n2zclBhcf+lpVJSObnEutKLQAszJeD47iIkA0BtL8JPnWzjcM8DkqgjvPa2BlDt9A0nmN1RR8w6/OaZSjhmYGe7OG51Rfr2znX/4ZTP7jvQP+zozWNBYzRWLplFixhud/XT1JwCoLA1REQlRUmIkkilqKyLUlkfoG0jQE0sQT6ZYOn8KqZTzwHP7qCoNcdrUauorS+mNJXjtcB97D/fR1hWjKxqnbyD5ls8/tbGK9y1spKs/TmVpiIbqMhpqyognUrR2R4knnETKSaRStHZF2bDrED/d3EK2ObXMoLo0THcsQXmkhAWN1dRVRKgpj1AWLqE8EqKqLMQpk6uoq4yQTDmHewc41DNAa3eU5147wvotB467n83I+tnVZWEWzazlE03p//axRJJoPP2NOBZPfyOOJZKkHN5z2hRm1JVTVRamqjRMTyzBC3s76I4m8OCw754uAO6kFywoEkEOkUgJdRURSkMlnD+nnrNm1DCpqpRE0kkG/xYGv+2nHyGqSsOUhUvoiSXoiibojsYJl5RQUZouXJWRMJGw0TeQpCea/hLRHUsQjScxSxffcInRWFPGtNpyGmvKSKSc3lj638Og1s4ovQNJysIllAaPsuBRGkoXy9LQm/FwaGy+lFghzbxmZmHgFeBDpIvBRuB33X3bcK9pampy3eaiOLR2RfnsDzay7Y2uYbepr4wwvbac95/eyMULpjC1poxXWrvZd7ifOZMrmddQxcz6cnpjSX7y3D7u/vVuUg7V5WE6++IMJNPfts+dVcfKpXOoq4gMHYQqS0N0ReO0HOmnpaOfp3cd5undhzAzptWUUVsRwczoH0jQN5Ak5U64pITO/jj98fR/9pryMCmHw8E31IVTqwmVGLsP9hJLpAiXGLMnVTBnciXTa9PdCXMmVzCrvgJIH7BmTapg7uRKzN5Zt0EskaQvlsRJd1ekD6BOXUWEsnCIeDJFyIySd9gd4e509scpDZcwkEhxuHeASKgkXRxLQ5SHQzjQ1h2lN5YkVGKEzKgsCzGlqvQd/x4yOszsWXdvOjZeUC0Fd0+Y2Q3AI0AIuPt4BUEmlq0tnSRTzjmz6o7qB40lkmxt6eKNjn72B9+4L5w/mXgixaPbW4e+Wd238XViiRR3rWriA6c3svdIPxt2HaKyNERZOMSr7T0c6Iyy51Av339qN2ue3DViTledO52ZdRX0xBLUV5YyZ3IFZ06vZcnc+rd1sOqJJSgf4Vube/pbeyTYxt3Z2tLFQDJ11OdE4+kxgbH6Bpjuihm+qyFygp9rZtRXlgJQWcrQ8rFm1FWc0PvL+CqolsKJUEthYuiOxll663/SH09SEQlRXxlham0586ZU8uQr7Rzpe7O/OrOroaY8TN9AEnfnw4um8ScfPp0zp9eO+Hld0TivHOimtSvG/IYq5jdUse9IH7sP9rK/M0p1WZizZ9W+rfcSORlNiJaCnLweenE//fEkX1l2Bgd7YvREE+w90sdTzYe4eMEUPnreLOY3VDGjvpzSUAm/2XUIgEtOayCRdPrjSSZXZf8Gmk1teYSmeZOPii2cVsPCaTWj+nuJnGxUFGRcPPDsPk6bWs0XP7jgbXXLXHrG1KHlSIhRP8NCRLIrtIvX5CT0ansPz752hGsumK1BRZECp5aCvEVbV5RvPfwSrd1RLpg7iZ1tPYRDJdy+8vx3dFB/o6Ofv3z4JZ54uY1QifHxxbPGMGsRGQ0qCnKUR7e38uX7NxNLpJg/pYrbf9lMfWWEjr44n2qawyULG4Z9bXNbN81tPXT0xWnvjnHXU7uJJ1L81rtmcPXiWUytLR/H30REToSKggz5/lO7+cZD2zl3Vh1/v3Ix8xuq6BtIUGLGe7/1S+75zZ6sRWFnazd//587eejF/UfF3zX7zfcRkYlBRaEIHeqJ8fWfbuGZ3YcB+PiS2SRTzg/+aw/Lzp7G331q8dDAbmVp+p/IyqVzuPPxV9l7uI+Z9RX87PkWntl9mF0He9i45wgVkRD/47LTWHbOdCZVllJfGaEiEtIYgsgEo6JwEhtIpLj3N3vYe7iPyrIwHzpzKjsOdPOPv9zJkb44H188i55Ygu8/lb6q93OXzOfrV52V9QZbv/fuU7jz8Vf52Hf/i5ryMLsP9jKlqpQZ9eV8ZdkZrLxwDlMG70EjIhOWisJJYO/hPra90cWys6cd9c38mw9t54cbXqO2PEx/PMmdj78KwDmzarn70xdy9sw6AF4+0M2+I3186Kxpw37GzPoK/mVVE+s2v8EbHVH+5xWn81vnzlBLQOQko6JwEviL9Tt4eOsBPvPeeXz2vfNp6ejn/+1s54cbXuMP3n8qN111Fp39cZ58pZ25kyt51+y6ow7mZ0yv4YzpI1/UddmZ07jszOELh4hMfCoKE1w8meLXOw/SUF3G95/aw/ef2jO07rIzp/KVZWcAUFcR4SPnzcxTliIyUagoTHCb93bQHUvwrd95F+GQcaR3gBn1FZwxrYZptWXq3hGRd0RFYYJ78pV2QiXGJQsbqKs48VmoRERAt7mY8J54pZ3Fc+pVEERkVKgoTGBb9nWypaWTD5zemO9UROQkoe6jCebR7a38759t5cwZNfzm1UNMry3nY0t0TyERGR05tRTM7BNmts3MUmbWdMy6m8ys2cxeNrNlGfHlQazZzG7MiM83s6eD+H1m9vZvnl8kOvvi3PSTLYRKjD0He7nktAb+7x9dwuxJlflOTUROErm2FLYCHwe+lxk0s0XASuBsYCbwmJmdHqy+A/gwsA/YaGbr3H07cBvwHXdfa2b/BHwOuDPH/E4qf7F+B0f6Bnjw+vdyzqy6fKcjIiehnFoK7r7D3V/OsmoFsNbdY+6+G2gGlgaPZnff5e4DwFpghaXPm7wMeCB4/T3A1bnkdrJpbuvhvk17+ex756kgiMiYGauB5lnA3ozn+4LYcPEpQIe7J46JZ2Vmq81sk5ltam9vH9XEC9Vdv95FWbiEP/jAgnynIiInsRG7j8zsMWB6llU3u/uDo5/SyNx9DbAGoKmpyfORw3hq747x4+dauOaC2TTopnMiMoZGLArufvkJvG8LMCfj+ewgxjDxQ0C9mYWD1kLm9kVv7TOvE0+m+O+XzM93KiJykhur7qN1wEozKzOz+cBC4BlgI7AwONOolPRg9Dp3d+BXwDXB61cBeWmFFKKXW7s5ZXIlpzZW5zsVETnJ5XpK6sfMbB9wMfAfZvYIgLtvA+4HtgM/B65392TQCrgBeATYAdwfbAvwNeDLZtZMeozhrlxyO5m0dcU0laWIjIucTkl1958CPx1m3a3ArVni64H1WeK7SJ+dJMdo645y7uz6fKchIkVAt7kocO5Oa1eMaTUaYBaRsaeiUOC6Ywn640mm1qooiMjYU1EocG1dMQCmaUxBRMaBikKBa+uKAjC1RkVBRMaeikKBa+tOtxTUfSQi40FFocC1Bi0FdR+JyHhQUShwrV0xqkpDVJdp6gsRGXsqCgWurTuqC9dEZNyoKBS4tq4YU3WNgoiMExWFAtfaHdV4goiMGxWFAubuaimIyLhSUShgg1czq6UgIuNFRaGANbf1ALpGQUTGj4pCgUqmnG8+tJ1JlRHet7Ax3+mISJFQUShQ339qN8+/3sEtHzmbyVWl+U5HRIpErpPs/JWZvWRmL5rZT82sPmPdTWbWbGYvm9myjPjyINZsZjdmxOeb2dNB/L5gZrai9Oj2Vv7y4Ze4/KyprDh/Zr7TEZEikmtL4VHgHHd/F/AKcBOAmS0iPdXm2cBy4LtmFjKzEHAHcCWwCLg22BbgNuA77n4acAT4XI65TUhbWzq5/l+f45yZtfzdysWYWb5TEpEiklNRcPdfBFNsAmwAZgfLK4C17h5z991AM+lZ1ZYCze6+y90HgLXACksf+S4DHghefw9wdS65TVR3/Xo3ZeESfvCZpbq1hYiMu9EcU/gs8HCwPAvYm7FuXxAbLj4F6MgoMIPxotIVjbN+y34+et5MJmkcQUTyYMSvomb2GDA9y6qb3f3BYJubgQTwo9FNb9icVgOrAebOnTseHzkuHnphP7FEik80zcl3KiJSpEYsCu5++fHWm9mngd8GPuTuHoRbgMwj2+wgxjDxQ0C9mYWD1kLm9tlyWgOsAWhqavLhtpto7t+0l4VTqzlvdl2+UxGRIpXr2UfLga8CH3X3voxV64CVZlZmZvOBhcAzwEZgYXCmUSnpweh1QTH5FXBN8PpVwIO55DbRPL3rEJv3dnDt0rkaXBaRvMl1JPMfgTLg0eBAtsHd/9Ddt5nZ/cB20t1K17t7EsDMbgAeAULA3e6+LXivrwFrzezPgeeBu3LMbUL5zmOv0FhTxrVLT57uMBGZeHIqCsHpo8OtuxW4NUt8PbA+S3wX6bOTis5/NR9kw67D3PKRRVSUhvKdjogUMV3RnGeplPOtn7/EjLpytRJEJO9UFPLsZ5tbeHFfJ19ZdgblEbUSRCS/VBTyqH8gyW0/f4nzZtdx9flFd1mGiBQgFYU82vTaYVq7Yvzx5QspKdEZRyKSfyoKebRjfxcAi+dMynMmIiJpKgp5tGN/N9Nry3VLCxEpGCoKebRjfxdnzajJdxoiIkNUFPIklkjS3NbDWTNq852KiMgQFYU8aW7rIZFyFQURKSgqCnmyY383gIqCiBQUFYU82bG/i/JICfMbqvKdiojIEBWFPNmxv4szptUQ0vUJIlJAVBTyIJVytuzr5JxZmjdBRAqLikIe7GzroTuWYMlcXbQmIoVFRSEPnn/9CABLTlFREJHCoqKQB8+9foRJlRHmTanMdyoiIkfJdTrOb5rZi2a22cx+YWYzg7iZ2e1m1hysX5LxmlVmtjN4rMqIX2BmW4LX3G4n2ZyU7s7HvvsUP/zNHp57vYPFcydp2k0RKTi5thT+yt3f5e7nAw8BfxrEryQ9L/NCYDVwJ4CZTQZuAd5Nepa1W8xssA/lTuDzGa9bnmNuBaU/nuT51zv45kM7aG7rYfGc+nynJCLyFjkVBXfvynhaBXiwvAK419M2APVmNgNYBjzq7ofd/QjwKLA8WFfr7hvc3YF7gatzya3QdPUnABhIpgCNJ4hIYcppjmYAM7sVuA7oBC4NwrOAvRmb7Qtix4vvyxIf7jNXk26BMHfuxJjCsisaB+ATF8xm98FeFs9VS0FECs+ILQUze8zMtmZ5rABw95vdfQ7wI+CGsU44+Mw17t7k7k2NjY3j8ZE56w6Kwm+fN5MHvvAeKktzrsciIqNuxCOTu1/+Nt/rR8B60mMGLcCcjHWzg1gL8MFj4o8H8dlZtj9pDHYf1ZarGIhI4cr17KOFGU9XAC8Fy+uA64KzkC4COt19P/AIcIWZTQoGmK8AHgnWdZnZRcFZR9cBD+aSW6EZ7D6qrYjkORMRkeHl+rX1W2Z2BpACXgP+MIivB64CmoE+4DMA7n7YzL4JbAy2+4a7Hw6Wvwj8AKgAHg4eJ42uaLqlUKOWgogUsJyOUO7+O8PEHbh+mHV3A3dniW8Czskln0LW1R+0FMrVUhCRwqUrmsdJVzROabiE8kgo36mIiAxLRWGcdEcTGmQWkYKnojBOuvrj6joSkYKnojBOuqIJanTmkYgUOBWFUXT/xr28dqg367ruaFzdRyJS8FQURklbd5Sv/vhFvnTfZtInXx1N3UciMhHoq+soee61DgCef72DtRv30tUfp7M/zoXzJvOB0xvpiiaordDuFpHCpqPUKHl+7xFKQyWcNrWam36yBYBwifHdx1/l+5++MOg+UktBRAqbuo9GyfOvdXD2rFq+fc27+Mh5M/nxF97D01//EABbWzqJxlO6mllECp6OUqMgnkzxYksHv7v0FM6ZVcc/XLt4aF1DdRmb96a7lnTfIxEpdGopjIId+7uIxlMsOeWtcySc2lj1ZlFQ95GIFDgVhVHw/Ovpg/6SuW+dTW1BYxWHegcA3QxPRAqfisIo2NrSSUN1GTPqyt+y7tSG6qFldR+JSKFTURgFb3T2M2dyBempII52amPV0LK6j0Sk0KkovEN7Dvbyl+t3kEimhmL7O6NZWwkACxrfbCmo+0hECp2Kwju0fut+vvfkLp597QgA7s6BzijTayuybj97UgWRULoFoe4jESl0o1IUzOx/mpmbWUPw3MzsdjNrNrMXzWxJxrarzGxn8FiVEb/AzLYEr7ndsvXFFIC2rhgAT7zSDqRvdNc3kBy2pRAOlXDKlCpKDKpKNZeCiBS2nIuCmc0hPdfy6xnhK4GFwWM1cGew7WTgFuDdwFLglmCuZoJtPp/xuuW55jYWDnRGAXj85fajnk8fpigAnNpQRU15JOuYg4hIIRmNlsJ3gK8CmXeBWwHc62kbgHozmwEsAx5198PufgR4FFgerKt19w3BVJ73AlePQm6j7kBXughs399FW1eU/Z39AMO2FACuu3ge11+6YFzyExHJRU4jn2a2Amhx9xeO+RY8C9ib8XxfEDtefF+W+HCfu5p0C4S5c+fm8Bu8c21dURbNqGX7/i6eeKWdZCpdC2fUZx9TALhkYQOXLGwYrxRFRE7YiEXBzB4DpmdZdTPwddJdR+PK3dcAawCampreep/qMZJKOW3dMVYsnsWh3hhPvNLOgsZqzGBqTdl4pSEiMmZGLArufnm2uJmdC8wHBlsJs4HnzGwp0ALMydh8dhBrAT54TPzxID47y/YF5WBvjETKmVFXznsXNPDkznYqIiEaq8uIhHQil4hMfCd8JHP3Le4+1d3nufs80l0+S9z9ALAOuC44C+kioNPd9wOPAFeY2aRggPkK4JFgXZeZXRScdXQd8GCOv9uoGzzzaGpNORedOoWDPQM81XzwuOMJIiITyVhdTbUeuApoBvqAzwC4+2Ez+yawMdjuG+5+OFj+IvADoAJ4OHgUhBf3dbDnUB+VkfQppdPryjl7Zi0Ab3RGOXd2XT7TExEZNaNWFILWwuCyA9cPs93dwN1Z4puAc0Yrn9H0vSd28diOVr62/EwApteWM72unFn1FbR09DOjbvhBZhGRiUQd4W/D/s5+YokUP992gBKDhupSAC46dQpw/GsUREQmEhWFt2HwArVndh+mobqMcDCofPGCdFHQmIKInCxUFEaQTDmt3bGh55mtgg+fNY3fOncGFwctBhGRiU637RxBe3eMZMqZVBnhSF+cqTVvFoW6ygh3/N6S47xaRGRiUUthBIO3sfjIeTMBmF6ni9RE5OSlojCCwfGEj543k5qyMAun1uQ5IxGRsaPuoxG8ERSF06ZW8+RXL9VEOSJyUtMRbgQHOvspj5RQV6FbX4vIyU/dRyNIT7WZff5lEZGTjYrCCPZ3Rpleq+sQRKQ4qCiM4EBnlBn1KgoiUhxUFI4jmXJau6K6YllEioaKwnEc7EnPnzBdN7wTkSKhonAcg9coaExBRIqFisJxdEXjANRXRvKciYjI+MipKJjZn5lZi5ltDh5XZay7ycyazexlM1uWEV8exJrN7MaM+HwzezqI32dmpbnkNhq6owkAXbAmIkVjNFoK33H384PHegAzWwSsBM4GlgPfNbOQmYWAO4ArgUXAtcG2ALcF73UacAT43CjklpOeoChUl6koiEhxGKvuoxXAWnePuftu0tNyLg0eze6+y90HgLXAimBe5suAB4LX3wNcPUa5vW3dsaClUKbuIxEpDqNRFG4wsxfN7G4zmxTEZgF7M7bZF8SGi08BOtw9cUw8KzNbbWabzGxTe3v7KPwK2Q22FKrKQmP2GSIihWTEomBmj5nZ1iyPFcCdwALgfGA/8DdjnC8A7r7G3ZvcvamxsXHMPqcnFqciEhqaaU1E5GQ3Yme5u1/+dt7IzP4ZeCh42gLMyVg9O4gxTPwQUG9m4aC1kLl93vTEElRrkFlEikiuZx/NyHj6MWBrsLwOWGlmZWY2H1gIPANsBBYGZxqVkh6MXufuDvwKuCZ4/SrgwVxyGw1d0YTOPBKRopLrEe/bZnY+4MAe4A8A3H2bmd0PbAcSwPXungQwsxuAR4AQcLe7bwve62vAWjP7c+B54K4cc8tZTzRBjc48EpEiktMRz93/23HW3QrcmiW+HlifJb6L9NlJBUPdRyJSbDSCehw90YSuURCRoqKicBw9sQTVukZBRIqIisJxdEfjGmgWkaKiojAMd6cnprOPRKS4qCgMo28gScp13yMRKS4qCsPoCe57pLOPRKSYqCgMo1t3SBWRIqSiMIzBloLGFESkmKgoDOPNuRR0SqqIFA8VhWH0xNJTcaqlICLFREVhGF0aUxCRIqSiMIwezc8sIkVIRWEYgwPNVWopiEgRUVEYRk8sQXmkhIhmXRORIqIj3jC6o7oZnogUHxWFYfTEEtRqPEFEikzORcHM/sjMXjKzbWb27Yz4TWbWbGYvm9myjPjyINZsZjdmxOeb2dNB/L5gus686Y7GdYsLESk6uc7RfCmwAjjP3c8G/jqILyI9//LZwHLgu2YWMrMQcAdwJbAIuDbYFuA24DvufhpwBPhcLrnlShPsiEgxyrWl8AXgW+4eA3D3tiC+Aljr7jF33w00k55qcynQ7O673H0AWAusMDMDLgMeCF5/D3B1jrnlJD3BjoqCiBSXXIvC6cD7gm6fJ8zswiA+C9ibsd2+IDZcfArQ4e6JY+JZmdlqM9tkZpva29tz/BWy645qfmYRKT4jHvXM7DFgepZVNwevnwxcBFwI3G9mp45qhlm4+xpgDUBTU5OPxWf0xBLUqKUgIkVmxKOeu18+3Doz+wLwE3d34BkzSwENQAswJ2PT2UGMYeKHgHozCwethcztx92bs67plFQRKS65dh/9DLgUwMxOB0qBg8A6YKWZlZnZfGAh8AywEVgYnGlUSnowel1QVH4FXBO87yrgwRxzO2H98STJlKv7SESKTq5HvbuBu81sKzAArAoO8NvM7H5gO5AArnf3JICZ3QA8AoSAu919W/BeXwPWmtmfA88Dd+WY2wnr0c3wRKRI5XTUC84g+v1h1t0K3Jolvh5YnyW+i/TZSXnXrQl2RKRI6YrmLNRSEJFipaKQxeAdUlUURKTYqChk0T00l4LOPhKR4qKikEV3VFNxikhxUlHIQt1HIlKsVBSyGBxo1qxrIlJsVBSy6IklKAuXUBrW7hGR4qKjXhbdsYTGE0SkKKkoZNET1X2PRKQ4qShk0R2Na5BZRIqSikIWmmBHRIqVikIWmmBHRIqVikIWmmBHRIqVikIWPTG1FESkOKkoHMPdg7OPVBREpPjkVBTM7D4z2xw89pjZ5ox1N5lZs5m9bGbLMuLLg1izmd2YEZ9vZk8H8fuCmdnGXTSeIpFyqst0SqqIFJ+cioK7f8rdz3f384EfAz8BMLNFpKfaPBtYDnzXzEJmFgLuAK4EFgHXBtsC3AZ8x91PA44An8sltxPVHUvfDE/dRyJSjEal+8jMDPgk8G9BaAWw1t1j7r4baCY9q9pSoNnddwWztq0FVgSvvwx4IHj9PcDVo5HbOzV43yMNNItIMRqtMYX3Aa3uvjN4PgvYm7F+XxAbLj4F6HD3xDHxcac7pIpIMRvxyGdmjwHTs6y62d0fDJav5c1Wwpgzs9XAaoC5c+eO6nsPTcWp7iMRKUIjHvnc/fLjrTezMPBx4IKMcAswJ+P57CDGMPFDQL2ZhYPWQub22XJaA6wBaGpq8pF+h3eiOzY465qKgogUn9HoProceMnd92XE1gErzazMzOYDC4FngI3AwuBMo1LSg9Hr3N2BXwHXBK9fBTxIHgxNxamzj0SkCI1GUVjJMV1H7r4NuB/YDvwcuN7dk0Er4AbgEWAHcH+wLcDXgC+bWTPpMYa7RiG3EaXr0ZsGp+JU95GIFKOcj3zu/ulh4rcCt2aJrwfWZ4nvIn120rh56UAXn/reBn78hYs5bWoNAAd7YoRKjPoKtRREpPgU9RXN//r063f1OHkAAAgnSURBVHT2x3n5QM9QrL07RkN1KSUllsfMRETyo2iLQiyR5MHNbwBwuG9gKN7eHaOxpixfaYmI5FXRFoVf7mijsz89ftDRm1EUemI0VqsoiEhxKtqi8MCz+5hWW0ZlaYgjffGhuFoKIlLMirIouDt1lRFWXjiXyVWldATdR6mUc7BnQEVBRIpWUZ53aWb87SfPB+CXL7UNjSkc6RsgmXJ1H4lI0SrKlkKmSVWlQ91H7T0xABpryvOZkohI3qgoVEaGuo/au9NFYWqtWgoiUpxUFCpLOdx7dFFQ95GIFCsVhcpSuqMJEsnUm0VBA80iUqRUFKrSt7Po6I/T3h2jsjREleZSEJEiVfRFob4yPRX0kd6B9IVraiWISBEr+qIwqTLdUjjSl24paDxBRIqZisJgS6FvQFczi0jRU1GoSheFjj51H4mIqCgE3Ud7D/fT0RdnqoqCiBSxnIqCmZ1vZhvMbLOZbTKzpUHczOx2M2s2sxfNbEnGa1aZ2c7gsSojfoGZbQlec7uZjcuEBhWREGXhEta9kL6N9tL5U8bjY0VEClKuLYVvA//H3c8H/jR4DnAl6XmZFwKrgTsBzGwycAvwbtKzrN1iZpOC19wJfD7jdctzzO1tMTMmVZby+uE+6isjLJlbPx4fKyJSkHItCg7UBst1wBvB8grgXk/bANSb2QxgGfCoux929yPAo8DyYF2tu2/w9KTJ9wJX55jb21YfdCFdesZUwqGi71ETkSKW61VaXwIeMbO/Jl1g3hPEZwF7M7bbF8SOF9+XJZ6Vma0m3QJh7ty5uf0GwORgsPnys6bl/F4iIhPZiEXBzB4DpmdZdTPwIeBP3P3HZvZJ4C7g8tFN8a3cfQ2wBqCpqclzfb9JlaVEQsb7T2/IOTcRkYlsxKLg7sMe5M3sXuCPg6f/DvxLsNwCzMnYdHYQawE+eEz88SA+O8v24+L3LzqFixdMoaY8Ml4fKSJSkHLtQH8D+ECwfBmwM1heB1wXnIV0EdDp7vuBR4ArzGxSMMB8BfBIsK7LzC4Kzjq6Dngwx9zetosXTOH3LzplvD5ORKRg5Tqm8Hng780sDEQJ+vmB9cBVQDPQB3wGwN0Pm9k3gY3Bdt9w98PB8heBHwAVwMPBQ0RExpGlT/aZuJqamnzTpk35TkNEZEIxs2fdvenYuM6/FBGRISoKIiIyREVBRESGqCiIiMgQFQURERmioiAiIkMm/CmpZtYOvHaCL28ADo5iOqNN+eVG+eVG+eWm0PM7xd0bjw1O+KKQCzPblO083UKh/HKj/HKj/HJT6PkNR91HIiIyREVBRESGFHtRWJPvBEag/HKj/HKj/HJT6PllVdRjCiIicrRibymIiEgGFQURERlSlEXBzJab2ctm1mxmNxZAPnPM7Fdmtt3MtpnZHwfxPzOzFjPbHDyuynOee8xsS5DLpiA22cweNbOdwc9JecjrjIx9tNnMuszsS/nef2Z2t5m1mdnWjFjW/RVMSHV78G/yRTNbkqf8/srMXgpy+KmZ1QfxeWbWn7Ev/ylP+Q37NzWzm4L997KZLctTfvdl5LbHzDYH8XHffyfM3YvqAYSAV4FTgVLgBWBRnnOaASwJlmuAV4BFwJ8B/yvf+ywjzz1AwzGxbwM3Bss3ArcVwN/3AHBKvvcf8H5gCbB1pP1FelKqhwEDLgKezlN+VwDhYPm2jPzmZW6Xx/2X9W8a/H95ASgD5gf/x0Pjnd8x6/8G+NN87b8TfRRjS2Ep0Ozuu9x9AFgLrMhnQu6+392fC5a7gR3ArHzm9A6sAO4Jlu8Brs5jLgAfAl519xO9yn3UuPuTwOFjwsPtrxXAvZ62Aag3sxnjnZ+7/8LdE8HTDRw9d/q4Gmb/DWcFsNbdY+6+m/Ssj0vHLDmOn18wrfAngX8byxzGQjEWhVnA3ozn+yigA7CZzQMWA08HoRuCpvzd+eiaOYYDvzCzZ81scOrVaZ6eYxvS39Cn5Se1ISs5+j9iIe0/GH5/FeK/y89y9LS4883seTN7wszel6+kyP43LbT99z6g1d13ZsQKZf8dVzEWhYJlZtXAj4EvuXsXcCewADgf2E+6OZpPl7j7EuBK4Hoze3/mSk+3k/N2jrOZlQIfBf49CBXa/jtKvvfX8ZjZzUAC+FEQ2g/MdffFwJeBfzWz2jykVtB/0wzXcvSXk0LZfyMqxqLQAszJeD47iOWVmUVIF4QfuftPANy91d2T7p4C/pkxbg6PxN1bgp9twE+DfFoHuzmCn235y5ArgefcvRUKb/8FhttfBfPv0sw+Dfw28HtB4SLoljkULD9Lus/+9PHO7Th/00Laf2Hg48B9g7FC2X9vRzEWhY3AQjObH3yzXAmsy2dCQf/jXcAOd//bjHhmn/LHgK3Hvna8mFmVmdUMLpMekNxKet+tCjZbBTyYnwyBY76dFdL+yzDc/loHXBechXQR0JnRzTRuzGw58FXgo+7elxFvNLNQsHwqsBDYlYf8hvubrgNWmlmZmc0P8ntmvPMLXA685O77BgOFsv/elnyPdOfjQfpMj1dIV+ubCyCfS0h3I7wIbA4eVwE/BLYE8XXAjDzmeCrpszteALYN7jdgCvCfwE7gMWBynvKrAg4BdRmxvO4/0gVqPxAn3cf9ueH2F+mzju4I/k1uAZrylF8z6b75wX+H/xRs+zvB330z8BzwkTzlN+zfFLg52H8vA1fmI78g/gPgD4/Zdtz334k+dJsLEREZUozdRyIiMgwVBRERGaKiICIiQ1QURERkiIqCiIgMUVEQEZEhKgoiIjLk/wNaJIlAItF//gAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We use a simple for loop to change the hyper-parameters value for a specific number of episodes for each value."
      ],
      "metadata": {
        "id": "NZ1w-_Lg4RnK"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#Gamma/alpha hyperparameter search\n",
        "#initialize the enviorment\n",
        "env =  WareEnv()\n",
        "env.reset()\n",
        "#set initial parameter values\n",
        "exploration = 0.7 #epsilon\n",
        "episodes = 1000\n",
        "rewards = []\n",
        "steps_episode = []\n",
        "episode_score = 0\n",
        "ALPHA=0.7\n",
        "GAMMA = 0.7\n",
        "episodes = 100\n",
        "epsilon = 0.3\n",
        "episodes = 400\n",
        "for GAMMA in np.linspace(0.1, 1, 20): # Gamma/alpha change for variable gridsearch\n",
        "    wandb.init(\n",
        "      # Set the project where this run will be logged\n",
        "      project=\"Q Learning Part 4\", \n",
        "      # We pass a run name (otherwise it’ll be randomly assigned, like sunshine-lollypop-10)\n",
        "      name=f\"gammaaa2{GAMMA}\", \n",
        "      # Track hyperparameters and run metadata\n",
        "      )\n",
        "  \n",
        "  # This simple block simulates a training loop logging metrics\n",
        "    #initialize q-table at zero\n",
        "    q_table = np.zeros([env.observation_space.n, env.action_space.n])\n",
        "    #ALPHA = ALPHA\n",
        "    #GAMMA = 0.6\n",
        "    #exploration = 0.1\n",
        "    #episodes = episodes\n",
        "    rewards = []\n",
        "    steps_episode = []\n",
        "    for ep in range(episodes): #number of episodes\n",
        "      reward_sum = 0 #reward per episode\n",
        "      state = env.reset()\n",
        "      done = False\n",
        "      steps = 0 #steps per episode\n",
        "      while not done:\n",
        "            if (random.random() < exploration):\n",
        "                action = env.action_space.sample() # Explore a random action\n",
        "            else:\n",
        "                action = np.argmax(q_table[state]) # Use the action with the highest q-value\n",
        "              \n",
        "            next_state, reward, done, _ = env.step(action)\n",
        "            reward_sum += reward #add reward to total\n",
        "            steps += 1 #add step by 1\n",
        "          \n",
        "            prev_q = q_table[state, action]\n",
        "            next_max_q = np.max(q_table[next_state])\n",
        "            new_q = (1 - ALPHA) * prev_q + ALPHA * (reward + GAMMA * next_max_q)\n",
        "            q_table[state, action] = new_q\n",
        "          \n",
        "            state = next_state #next state\n",
        "      ##if done == New Episode\n",
        "      episode_score +=1 #count number of episodes\n",
        "      wandb.log({\"Rewards\": reward_sum, \"Steps\": steps, \"Episodes\": episode_score, \"EPSILON\": exploration\n",
        "          \n",
        "      }) #visualize here.\n",
        "      #if done == New Episode\n",
        "      rewards.append(reward_sum)\n",
        "      print('done')\n",
        "      steps_episode.append(steps)\n",
        "  #all episodes finished\n",
        "  # Mark the run as finished\n",
        "    wandb.finish()"
      ],
      "metadata": {
        "id": "9IhsuQaIk1sH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Epsilon decay"
      ],
      "metadata": {
        "id": "kuibcAT45E8p"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#set inital hyper-parameters\n",
        "ALPHA = 1\n",
        "GAMMA = 0.7\n",
        "exploration = 0.7\n",
        "#number of episodes to train\n",
        "episodes = 200\n",
        "rewards = []\n",
        "steps_episode = []\n",
        "#Epsilon decay:\n",
        "min_epsilon = 0.1\n",
        "max_epsilon = 1\n",
        "decay_rate = 0.005\n",
        "exploration_list = []\n",
        "#initialize Q table at 0\n",
        "q_table = np.zeros([env.observation_space.n, env.action_space.n])\n",
        "for ep in range(episodes): #number of episodes\n",
        "  reward_sum = 0 #reward per episode\n",
        "  state = env.reset()\n",
        "  done = False\n",
        "  episode_score = 0\n",
        "  steps = 0 #steps per episode\n",
        "  while not done:\n",
        "            if (random.random() < exploration):\n",
        "                action = env.action_space.sample() # Explore a random action\n",
        "            else:\n",
        "                action = np.argmax(q_table[state]) # Use the action with the highest q-value\n",
        "              \n",
        "            next_state, reward, done, _ = env.step(action)\n",
        "            reward_sum += reward #add reward to total\n",
        "            steps += 1 #add step by 1\n",
        "          \n",
        "            prev_q = q_table[state, action]\n",
        "            next_max_q = np.max(q_table[next_state])\n",
        "            new_q = (1 - ALPHA) * prev_q + ALPHA * (reward + GAMMA * next_max_q)\n",
        "            q_table[state, action] = new_q\n",
        "          \n",
        "            state = next_state #next state\n",
        "  ##if done == New Episode###\n",
        "  episode_score +=1\n",
        "  #reduce epsilon after every episode\n",
        "  exploration = min_epsilon + (max_epsilon - min_epsilon)*np.exp(-decay_rate*ep)\n",
        "  exploration_list.append(exploration)\n",
        "  #wandb.log({\"Rewards\": reward_sum, \"Steps\": steps, \"Episodes\": episode_score, \"EPSILON\": exploration\n",
        "      \n",
        "  # })\n",
        "  #if done == New Episode\n",
        "  rewards.append(reward_sum) #episode rewards\n",
        "  steps_episode.append(steps) #number of steps/episode\n",
        "  #all episodes finished"
      ],
      "metadata": {
        "id": "fCkdCLyR5bUq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "fig, (ax1, ax2) = plt.subplots(1, 2)\n",
        "fig.suptitle('Moving average of rewards/10 episodes and epsilon value/episode')\n",
        "ax1.plot(moving_average(rewards, 10))\n",
        "ax2.plot(exploration_list)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 312
        },
        "id": "XJD8lpxh63Gy",
        "outputId": "09d8a5fa-b450-4095-8e5e-7967c4f1ab5d"
      },
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<matplotlib.lines.Line2D at 0x7faef2986150>]"
            ]
          },
          "metadata": {},
          "execution_count": 38
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaEAAAEVCAYAAABJ81qhAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd5wU5f3A8c/3eueAo9ejKqgUEbCjUUFjxBgbGsVKNJqiSYzGxJYfSUxMTIzGxBbFhsTEQBQ1GnsBARWkCJwUadKOehxXv78/5jkYll3ujt272dv9vl+vfd3uM7Mz352bne88zz7zjKgqxhhjTBBSgg7AGGNM8rIkZIwxJjCWhIwxxgTGkpAxxpjAWBIyxhgTGEtCxhhjAhM3SUhEjheRxUHHkShE5JsiskpEdorIkKDjaSgRuUNEngo6joMlIi+LyPgYLzOut4mI9BQRFZG0Zlznz0TkkaDWXx8RuUxE3gs6jlDueNArxst8S0SuOtj3NzgJicgKEakUkaKQ8k/cDtDzYIMAUNV3VbV/NMsw+7gHuF5V81T1k6CDaQoikiEim0QkT0TOF5EPRGSXiLwVZt7BIjLHTZ8jIoObIiZVPV1Vn2iKZZu9VPVXqnrQB76WRkSOFpEPol2OOx4si0VMsdLYmtByYFzdCxE5HMiJaUQtlHjipmYJ9AAWNGTGoM4gY7DeE4BPVXUnUAr8EfhNmPVkAFOBp4DWwBPAVFduTEvwdWB60EE0hcYeNJ8ELvW9Hg9M8s8gIq1EZJKIbBSRlSLycxFJEZFMEdkqIof55m0nIuUi0l5ERonIat+0FSLyYxGZJyLbROQ5EcnyTb9JRNaJyFoRucrVxvqEC1pELheRRSKyQ0SWich3fNMWiciZvtdpLvah7vVId4a9VUTmisgo37xvichEEXkf2AX0OtC66ovbbaN7RORLEVkvIn8VkewInynFbduVIrLBbfNWbhk7gVRgroh8EeH9KiLXichSYKkrO1NEPnWf9QMROcK3/f7je+9SEfmH7/WqupqFiPzJvd7uahzH++a7Q0SeF5GnRGQ7cJmIFIvI2257vQYU+ebPcvNudjHNEpEOvo9xBu6Lqaqvq+oUYG2YjzsKSAP+qKoVqnofIMDJEbZNKxF51P2f1ojI/4lIqpt2mYi8LyL3u/3ycxH5mu+9e5omRKSP+2zbxKuxPeeb7xj3eba5v8f4pkXcJm76gfbJy9x+t0NElovIxRE+43AR+dAtY537PBm+6Soi17j/9VYReUBExE1LdfvpJhFZhneAjEhEOovIP8X7Xi0Xke/7ptXtE8+5mD8WkUG+6T91/4MdIrK4blvLAZoo3fqmiUipiJSIyNUh65vivi87RGSBiAyLsJwHReSekLKpInKje36ziHzhlrNQRL4ZYTn7NRdKSBOWiFwh3nFji4i8KiI9QhazZ18XkUNE5DX3+RaLyPm+5Twu3nHjNRfX2/5lyb7HmzNc3DvcNv6xb76r3bYrdduys2/aqW6/3yYi9+N9l/yft77Psi9VbdADWAGcAiwGDsU7yK3GO+NWoKebbxLeWWc+0BNYAlzppj0GTPQt8zrgFfd8FLA6ZH0fAZ2BNsAi4Bo3bQzwFTAQryb2lIuhT4TYvw70dhvrRLyEMdRNuw14OmTeRe55F2Az3g6QApzqXrdz098CvnRxpAHp9azrgHED9wLT3OfNB/4D/DrCZ7oCKAF6AXnAv4AnfdMjbg/f9NfcurKBIcAGYIT73453/4NMt46tbht0BlbW/a/ctC1Ainv9baCt2x4/cp83y027A6gCznbLygY+BP7g1nMCsAN4ys3/HbcNclxMRwIFvs/wOdA/5HNdBbwVUnYD8HJI2YvAjyJsmxeAvwG5QHu8/fA7btplQLVbZjpwAbANaOPbJ65yz58FbnWfNQs4zpW3cdvsEredxrnXbd30A22TiPuki3d73TYBOgEDI3zGI4GRbv098b5fPwzZP14ECoHuwEZgjJt2jdv23dxnedPNnxZmPSnAHLzvWYbbX5YBo0P2iXPd9vwxXotLOtAfWAV0dvP2BHr73veUr3zP+oF3gL+4bT7YxX6y73273fZLBX4NzIiwjU5w6xf3ujVQ7ovnPLzvQ4rbD8qATr795L1w8YXZT8bifZcPdf+PnwMf+ObtBKzBO6bkupgud/MOATYBA9y8j+PtLyfg7T9/qosj9LgArAOO9322uuPUyW6ZQ90y/gy846YVueXX/b9uwPs+NOizhN3OB5GEfu7+cWPwDmJp7oP1dP/UyroN4juQvOWenwJ84Zv2PnDpAZLQt32vfwv81ZfMfu2b1od6Drohn+XfwA98790B5LjXTwO3uec/xXdgd2WvAuN9O9JdjVhXxLjdDlaG+5K56UcDyyMs93/Ad32v++N9meu+iA1JQif7Xj8I/DJknsXAie75KrdTXgg8hHdgPgTvyzDtAOvZAgzyHQDe8U3r7nbgXF/ZM+w9uFwBfAAcEWa5vYGSMOXhktAvgMkhZU8Dd4R5fwegAsj2lY0D3tS9B5e1uAOTK/sIuMS3T9R9ISe5bdU1ZB2XAB+FlH3oll3fNom4T+IdoLYC3/LH38DvxA+BF0L2j+N8r6cAN7vnb+BOCN3r04ichEYAX4aU3QL83bdPzPBNS8EdHPG+FxvwjhvpIcu4gzBJCC8x1gD5vnl/DTzue9/rvmkDgPII20TwTjJPcK+vBt44wDb8FBjr208amoRexp2o+7bBLqCHe30l8Kh7fgHwbsh6/wbc7p4/jm9fxztBrQG6+f6vdUnoS7zjc0HI8h4FfhuyjCr3OS4N+X8JXmWkQZ8l3ONgfsN4ErjIbeRJIdOK8LLjSl/ZSryzN/DOmHJEZIR4HRkG4511RvKV7/kuvI0B3tnHKt80//P9iMjpIjLDVS234p0FFQGoagneWeA3RCQHOAvvSw9eLe881xyx1b33OLwzk7DrPtC66om7Hd4Z/xzful5x5eHU1UjqrMT7EnYIP3tY/vX3AH4U8lm7ufUAvI13onCCe/4WXk3vRPcaAPGaUBe5qvpWoBX7Nif519kZ2KKqZSGfo86TeAfYyeI1X/5WRNLdtDPwdviG2AkUhJQV4J18hOqBtw+v822Hv+HViOqsUfcN88Xcmf3dhPcl/cg1+1zhykP/d3XL6EL92yTiPunecwFeTWWdiLwkIoeEiQsR6SciL4rIV+I1jf6KkGY/Gv79C/0sfj2AziHx/ox999M9y1LVWryDWmf33fwhXuLYICKT/c1CEXQGSlXV/7/1H4PCfa4sCfP7pPsfT2bv7+AX4Z28ACAil8re5uutwGHsvw0bogfwJ99ySvH2m7qY9zTFuXlHhGzPi4GOvuX5t2fd76Xhttu33LJXuma7o135PvunW8Zm9u6f/uUr+x9HDvRZ9tPoJKSqK/Gqy2fgNQH5bcLLmD18Zd3xqpKoag3eGdU493gxZGdpqHVAV9/rbpFmFJFM4J94vcU6qGoh3j/U3475rItnLLDQ7fzgbdwnVbXQ98hVVf+P33sORg1Y14Hi3oRX1R/oW1crVc0jvLXsv52rgfWRtkUY/gPpKrymUv9nzVHVZ930uiR0vHv+NiFJSLzff24Czgdau8+/jX23tX+d64DWIpIb8jm8GVWrVPVOVR0AHAOcyd7fJP1fzPosAI4QEX8cRxC+48YqvJpQkW87FKjqQN88XUKW1Z0wv0Wp6leqerWqdsY74/yLa48P/d/VLWMN9WwT6tknVfVVVT0V70Tpc+DhCNvkQTe9r6oW4CUGiTBvqHXsu+92jzSji3d5SLz5qnqGb549yxKvc09X3PZU1WdU9Tj2NvvfXU9sa4E2IpIfEt+a+j5UBM8C57rfNUbgfb9xrx8GrsdrRi0E5hN+G9adUPg7cYUmje+EbKNsVf3AnXSdiNfqVDfv2yHz5qnqtb7l+bdnHl6Tabj9c5aqjsU7wfo33rEZQvZPty+2Ze/+6V++sO++EPGzhNkuwMFfJ3QlXlOO/2zNn2Qmiki++0fdiPfbR51n8M7WLmZvjaOxpgCXi8ihrvbyiwPMm4HXrrkRqBaR0/GaD/wmu7JrQ2J6Cq+GNFq8H2OzxOtA0ZXw6ltXxLjdGeDDwL0i0h5ARLqIyOgI63oWuEG8H7Hz8M5kn1PV6gNsiwN5GLjG1VJFRHJF5Ou+L/PbwEl4zTyrgXfxmmTbAnVdwPPxEuFGIE1EbmP/Gsge7oRmNnCneN2tjwO+UTddRE4SkcPF6xSwHe8Ep9Ztu+F4Neu6eVPF67iSBqS4/1VdrektvCaJ74vXceN6V/5GmJjWAf8Ffi8iBeJ1AOktIif6ZmvvlpUuIufhtX/vlxBF5DzfvrIF7yBa6+btJyIXidcR5gK8ZqEX69smHGCfFJEOIjLWHTQq8GqAtRE2f77bpjtdbenaCPOFM8V9/q4i0hq4+QDzfgTsEK+DQbaL+TAROco3z5Eico6rjfzQxT5DRPqLyMnu5G433klapM8DgKquwmvC/bXbNkfgHa8O6jor9S5v2AQ8AryqqlvdpFy8/+dG8Drv4NWEwi1jI94B/Nvu81+B15xc56/ALSIy0C2rlduvwKvlzlPV7e71i3j7ziVu/0sXkaNE5FDf8s4QkePE62jyS7zms9AWmwwRuVhEWqlqFd6+ULdtn8U7Tg122/5XwExVXQG8BAz0/b++z74J9UCfJayDSkKq+oWqzo4w+Xt4mX8Z8B7eQf0x33tnuumdaXhzSuj6XwbuwzsIlQAz3KSKMPPuwNtQU/AOBBfh/fjvn2cdXpv8McBzvvJVeLWjn+HtbKuAnxBhu9W3rgbE/dO6cvGaSF7H+60nnMfwmqvewauZ7sbb9gfF/T+vBu53sZfgNbnWTV+Cd1B7173ejvc/ft+dfIDXdPYKXmeUlS6mAzaV4m2jEXjV9tvZt4m3I/A83hdkEV4ifBLvh9MPVXW3b95L8A5SD+LV1spxtQBVrcTrDHEp3m8mVwBnu/JwLsU7oVjotsXz7NsEOxPoi3dwmgicq6qbwyznKGCmeL0Vp+H9NrjMzXsmXseNzXi1xzNVdVN926SefTIF76RvrXvviUROLj9269nhttNzEeYL52G8//Vc4GP2bxHZw+0bZ+I1vS9n7wG9lW+2qXgnpnWdNc5xB8ZMvC73m/Ca0Nrj/Z5Un3F4v1+sxWvuv11VX2/wp9vfM3i/S+05QVXVhcDv8Y4b64HD8X7jjuRqvP/TZryOSXtqBqr6Al4Nb7L73s8HTneT9+ma7Y4xp+H9NrsWb7vcjbet/PHejrcPHInXWSicS4AVbp3X4FUMcNvqF3i1vnV4CfNCN20TXoeM37jP0tf/uev5LGHV9fpo0dxZwHwgM4qaQLNrqXEHTUT+AsxX1b8EsO7L8H6EPa65152IROQOvB/KIx0ok5qILMQ7yVnYwPkfx+vg9fMmDSyG4uniykYRb1iaTNcccDfwn5ZwIG+pcceZTzlwhxZjWjzXnDapoQmopWqxSQjvh94NwBd47f2NadMOUkuNO26o6kOuCdWYhKWqlSGdoBJSQjTHGWOMaZlack3IGGNMC2dJyBhjTGAsCRljjAmMJSFjjDGBsSRkjDEmMJaEjDHGBMaSkDHGmMBYEjLGGBMYS0LGGGMCY0nIGGNMYCwJGWOMCYwlIWOMMYGxJGSMMSYwloSMMcYEJi3oAKJVVFSkPXv2DDoMk6DmzJmzSVXbNfd6bb82TWnOnDnbgQ9VdUzQsbT4JNSzZ09mz54ddBgmQYnIyiDWa/u1aUoisjQeEhBYc5wxxpgAWRIyxhgTGEtCxhhjAmNJyBhjTGDiLgmJyBgRWSwiJSJyc9DxGGOMaTpxlYREJBV4ADgdGACME5EBwUZlzIGJyGMiskFE5keYLiJynzuxmiciQ5s7RmPiVVwlIWA4UKKqy1S1EpgMjA04JmPq8zhwoO6upwN93WMC8GAzxGRMixBv1wl1AVb5Xq8GRoTOJCIT8L7MdO/evXkiS1K7q2pISxHSUus/X9lWXkV5ZQ0dCjIRkUatp7ZW2VZeRUF2Oqkp4d+7payS7burAFi/vYJ128qpqKqlS+tsKqpr2LyzkuyMVDoXZlOQlU5ZRTXz1myjsrqWorwM8jLTKMxJZ3t5Ndt3V7G9vIpt5VV0LszmnKFdGxWvn6q+IyI9DzDLWGCSqiowQ0QKRaSTqq5r7LrmrCzlnSWbuOHUfgcZrTHxJd6SUIOo6kPAQwDDhg3TgMOJK5t3VrBqSzmbdlTQp30ePdrmNDgh1NYqi9fvYFXpLv70v6XsrKhmVekustJTaZ2TQefCLI7r047CnHRUldVbyindVcmKTWVs2VXFys1l1CrkZ6ZxQr92DO5WSKucdErLKqmsrqVbm2y2l1fzr0/WsHzjTgpzMijMSSczLYW5q71k0aUwmy6F2YiACKhCrSplFTUsWb+D6trY/7tH9W8XVRJqgHAnV12AfZJQQ06uZiwr5U//W8qYwzpyaKeCponWmGYUb0loDdDN97qrK0tas1aUMmtFKUO7t6Zbmxy6FGbvN09trSIC/5i9mlte+Iwa34F6YOcCRvZqy5elu2id49UyWudksGVXFau37EIVBncr5NNVW5m3eivbd1cDUJSXwbAebTjziE7sqqxhe3k1Szfs4N7Xl+xZdmqKkJOeyqBuhXRpncOZR3SifX4mC9ft4IVPVvPSZ+FP9HsV5XL2kC5sL69ia3kVO3ZXc9Hw7nQuzGLmslLKKqu95FPrJaLUFPESYN8i+nfIR4EOBZl0apVFRmoqa7aWk5WeQpvcDMqrali7tZwdu6vJTEtlYOcC8jLT2FxWyeadFeyurqUgK42C7HRaZaeTn5VGZlpqbP9pB6khJ1cXj+jO/W+U8Mi7y/n9+YOaNT5jmkK8JaFZQF8RKcZLPhcCFwUbUnDWbC3n4odnUllTu6esb/s8rj6+F2WV1XRrnUOtKj987lMEKKusYURxG64+vhetczOYv2Ybf36jhKdmrKRH2xzmra6iqkbZuquSrPRUdlXWAPBeySb6ts/jzEGdGdKtkM6F2Qzt3prsjP0PzjsrqqmoqqFWoW1uBgphm89u/8YAdlfVsK28irZ5mQCs376b7PRUOrXKilg7m3BC70Zvp+5tc/Z5fUjH/WsIrXMz6NM+r9HLjpGYnVwV5mRwwVHdeHrmSn4yuj8dW2XFJEBjghJXSUhVq0XkeuBVIBV4TFUXBBxWk1q6fgc5mWl0KcxmzdZy3l68kQ07drNu627eK9kEwOQJI/mydBebd1by70/WcNM/5+2zjPysNM48ojNDuhdy1qDOZKV7yePIHq25eER3FEj3/aZTU6tUVtcyZ+UWjundlqra2gbXBvIy08jLrH+3yUpPJSs9lcKcjL3vbRdYEgjaNOB6EZmM9xvntoP5PajOFccWM+nDFTz+wQpuPv2QWMVoTCDiKgkBqOp0YHrQcTQHVeWs+9+nvKpmTxIK9etzDmdkr7aM7NUWgKuOL+aj5aW0zslg007vx/lTDu2wp7YRKlyHgtQUITsjleP6FgGQmRIfzVEtlYg8C4wCikRkNXA7kA6gqn/F25/PAEqAXcDl0ayve9scTj+sE8/MXMn1J/dp0EmBMfHK9t4Ard9eQXlVDbkZqRTlZ9KzKIc7zxpIaVkV+VlptM/P3C+5pKemcGyfooAiNuGo6rh6pitwXSzXedXxxbz02TqmzFrFFccVx3LRxjQrS0IBWrJ+BwCPXnbUnpqOMQ0xpHtrjurZmkffW86lR/doUBd6Y+KR7bkBqktC/TrkBxyJaYmuPr4Xa7aW8/L8r4IOxZiDZkmoicxctpmzH3if8//2IS/OW7vf9OWbynjiwxV0LMiiTW7G/gswph6nHNqB4qJcHnl3GV6LnzEtjyWhJrBjdxXfn/wJn67aykfLS3l6xpd7ppVX1vDK/HXc8NynbC+v5t4LBgcYqWnJUlKEq44vZu7qbXy0vDTocIw5KJaEmsAj7y5n/fYKXvjuMXxraFeWbtgJeEPgXPrYTK556mM+W7ONO88ayNG97bcgc/C+NbQrbXIzePjdZUGHYsxBsY4JMVJWUc3lj89iwZptlFXWcPphHRnSvTWffLmVf368mimzV/Hkhyv5bM027v7W4Zx5RGdyrWutiVJWeiqXjOzBn/63lJINO4O8INeYg2I1oShNm7uWu/6zkAlPzuaj5aWUuVEIfnSaN8Dk4V1bAXDT8/NYt62cey8YxAVHdbcEZGLm0qN7kJmWwiNWGzItkB0JD8KHX2ymIDuNjgVZ3PDcp3vGarvyuGKuPr4XKzaX0ae91+NtWI/W/OBrfZm3eiv3XjB4nxEEjImFtnmZnD+sG5NnfckPT+lnQ/mYFsWS0EEY9/AMACac0IuaWmXSFcMpr6rhtAEdEJF9DgIiYsPumyY34YRePPPRlzz63jJu/brdB9K0HNYc10jVvsFEH3pnGcf1KeKEfu0YPbBjo++hY0ysdGuTwzeO6MQzM79k667KoMMxpsEsCTXSum27ATi2T1v+ee3RPHHF8IAjMsZzzajelFXWMOnDlUGHYkyDWRJqpFVbdgFw3ag+HNmjTcS7gBrT3A7pWMDXDmnP399fzq7K6qDDMaZBLAk10upSb6Trbm1y6pnTmOZ37ajebNlVxXOzVtU/szFxwJJQIy3fXEaKYD2QTFwa1rMNw3u24eF3llFZXVv/G4wJmCWhBqqsruXMP7/Lg299wTG9i/a5SZwx8eTak3qzdttups3df8xCY+JNVEdSETlPRBaISK2IDAuZdouIlIjIYhEZ7Ssf48pKRORmX3mxiMx05c+JSFxdUDN7ZSnz12wH4BdnWhdYE79G9WvHoZ0K+OvbX1BbawObmvgW7en8fOAc4B1/oYgMAC4EBgJjgL+ISKqIpAIPAKcDA4Bxbl6Au4F7VbUPsAW4MsrYYurNzzeQkZrCgjtH07+j3XrBxC8R4dpRvSnZsJPXFq0POhxjDiiqJKSqi1R1cZhJY4HJqlqhqsvxbms83D1KVHWZqlYCk4Gx4l1gczLwvHv/E8DZ0cQWax98sZkje7S24XZMi3DGYR3p3iaHv7z1hd3mwcS1pvphowvg756z2pVFKm8LbFXV6pDysERkgojMFpHZGzdujGng4eyuqmHxVzsY0r2wyddlTCykpabwnRN7MXfVVt4v2Rx0OMZEVG8SEpHXRWR+mMfY5ggwHFV9SFWHqeqwdu3aNfn6Fq7bTnWtckRXS0Km5Tj3yK50LMjivjeWBh2KMRHV27akqqccxHLXAN18r7u6MiKUbwYKRSTN1Yb88wdu/pptAAzq1irgSIxpuMy0VL5zYi/u/M9CZizbzMhedu8qE3+aqjluGnChiGSKSDHQF/gImAX0dT3hMvA6L0xTr9H6TeBc9/7xwNQmiq3RNu2oQAQ6Fti1QaZlGTe8O0V5mfzZakMmTkXbRfubIrIaOBp4SUReBVDVBcAUYCHwCnCdqta4Ws71wKvAImCKmxfgp8CNIlKC9xvRo9HEFktllTXkpKfaAKWmxclKT+U7J/Ti/ZLNzFlptwA38Sfa3nEvqGpXVc1U1Q6qOto3baKq9lbV/qr6sq98uqr2c9Mm+sqXqepwVe2jquepakU0scXSrsoacqxXnGmhLh7ZnTa5Gdz3v5KgQzFmP3bZfwPsqqwmJyM16DCMOSg5GWlcdXwxby/ZyNxVW4MOx5h9WBJqgLKKGnIyrCZkWq5Lj+5Jq+x0/vyG1YZMfLEk1ADlVdXkWk3IRBBpKCrf9B4i8j8RmScib4lI1+aOMS8zjSuPK+b1RetZsHZbc6/emIgsCTVAWUUN2ZaETBj1DEVV5x5gkqoeAdwF/Lp5o/SMP6Yn+Zlp3G+1IRNHLAk1QHllDbnWHGfCCzsUVcg8A4A33PM3w0xvFq2y07n82J68PP8rFn+1I4gQjNmPJaEGKKusJifTakImrEhDUfnNxRvoF+CbQL6I7HflaHMMR3XFccXkZqTaKAomblgSaoBdlTXWO85E48fAiSLyCXAi3mggNaEzNcdwVIU5GVx2bE9emreOz7/a3iTrMKYxLAk1wK7KamuOM5EcaIgqAFR1raqeo6pDgFtdWWB9pa8+vhf5mWnc+9qSoEIwZg9LQvWoqVV2V9VaF20TSdihqPwziEiRiNR9124BHmvmGPdRmJPBlccX8+qC9XvGRTQmKJaE6rGr0ru7hDXHmXAiDUUlIneJyFlutlHAYhFZAnQAJoZdWDO64rhiWmWnW23IBM5O7+tRXuk13VvHBBOJqk4HpoeU3eZ7/jx7b9gYFwqy0plwQi9+9+piPvlyC0O6tw46JJOkrCZUjzKXhOw3IZNoLjumJ21yM/iD1YZMgCwJ1WPH7ioAu1jVJJzczDSuObEX7y7dxKwVNsK2CYYloXp8tNz7cg7oVBBwJMbE3iUje1KUl8nv/7s46FBMkrIkVI83Pt9Avw55dGuTE3QoxsRcdkYq3x3VmxnLSvmgZFPQ4ZgkFO1N7X4nIp+7gRlfEJFC37Rb3ICOi0VktK887GCProvrTFf+nOvuGqi1W8uZubyUUwd0CDoUY5rMRSO607Egi9+/tgTvJsfGNJ9oa0KvAYe5gRmX4F0DgRvA8UJgIDAG+IuIpNYz2OPdwL2q2gfYAlwZZWxRe/ajL1FVLjyqe9ChGNNkstJTue7kPsxZuYW3lzTNcEHGRBLtnVX/666TAJiBd7U4eAM0TlbVClVdDpTgDfQYdrBH8e6bfTJ7u7E+AZwdTWyxMHNZKUO6t7amOJPwLhjWjS6F2fzBakOmmcXyN6ErgLrbeEca1DFSeVtgqy+hhRsEslmpKp9/tZ1DOuYHGYYxzSIjLYUffK0v81Zv45X5XwUdjkki9SYhEXldROaHeYz1zXMrUA083ZTB+tbX5KMNr99ewfbd1fS3JGSSxDlDu9CnfR6/++9iqmtqgw7HJIl6k5CqnqKqh4V5TAUQkcuAM4GLdW89PtKgjpHKNwOFIpIWUh4ppiYfbbhuhOH+HSwJmeSQlprCT0b3Z9nGMv4xZ3XQ4ZgkEW3vuDHATcBZqrrLN2kacKGIZIpIMdAX+IgIgz265PUmcK57/3hgajSxReuz1dsQgUM62hNxCl8AACAASURBVPVBJnmcNqADQ7sX8sfXl+wZssqYphTtb0L3A/nAayLyqYj8FUBVFwBTgIXAK8B1qloTabBHt6yfAjeKSAneb0SPRhlbVGYuL6V/h3xa5aQHGYYxzUpE+OmYQ1i/vYLHP1gRdDgmCUQ1IJrrTh1p2kTCjBYcbrBHV74Mr/dc4KpqapmzcgsXHNWt/pmNSTAjerXlpP7tePCtEi4a3t1OxEyTshETwvhszTbKq2oYXtwm6FCMCcRNYw5hR0U1D779RdChmARnSSiMuvHijuppScgkp0M7FXD24C78/f3lfLVtd9DhmARmSSiMmcs207tdLu3yM4MOxZjA3HhqP2pV+dP/7FYPpulYEgpjyfqdHN6lVdBhGBOobm1yuHhED6bMXs0XG3cGHY5JUJaEwthcVkFRntWCjLn+5D5kpaVwz6t2qwfTNCwJhdhVWc3uqlraWhIyhqK8TK46vhcvz/+Kj7/cEnQ4JgFZEgqxeWclAG1zA7+ThDFxYcIJvWiXn8nElxbZ4KYm5iwJhdhc5pJQniUhY8C7DfiPTu3HnJVbeNkGNzUxZkkoRGlZBQBtrCZkzB7nDevGIR3z+fXLi6iotuF8TOxYEgpR1xxnHROM2Ss1RfjZGYeyqrScSR+sDDock0AsCYWoa46zmpAx+zqhXztO7NeOP7+xlC3ue2JMtCwJhSgtqyQzLYWcjNSgQzEm7tz69UPZWVHNn/63NOhQTIKwJBRi444K2uVn4t1x3Bjj169DPhcc1Z2nZqxkmV3AamLAklCI9dt306EgK+gwTAsiImNEZLGIlIjIzWGmdxeRN0XkExGZJyJnBBFnrNx4aj8y01L4zcufBx2KSQCWhEJ4Scg6JZiGEZFU4AHgdGAAME5EBoTM9nO8e2cNwbuR41+aN8rYapefyXdP6sN/F65nxrLNQYdjWjhLQiE27Kigfb7VhEyDDQdKVHWZqlYCk4GxIfMoUHeL3lbA2maMr0lceVwxnVtl8X8vLaS21i5gNQcv2tt7/9I1L3wqIv8Vkc6uXETkPtc8MU9EhvreM15ElrrHeF/5kSLymXvPfdLMP8rU1irH/uYNduyupr3VhEzDdQFW+V6vdmV+dwDfFpHVeDd0/F64BYnIBBGZLSKzN27c2BSxxkxWeio/GdOf+Wu288Ina4IOx7Rg0daEfqeqR6jqYOBF4DZXfjrQ1z0mAA8CiEgb4HZgBN4Z5O0i0tq950Hgat/7xkQZW6PsqqphzdZyAKsJmVgbBzyuql2BM4AnRWS/756qPqSqw1R1WLt27Zo9yMYaO6gLg7oV8ptXPmdnRXXQ4ZgWKqokpKrbfS9z8ZodwGuOmKSeGUChiHQCRgOvqWqpqm4BXgPGuGkFqjpDvcGpJgFnRxNbY1VW1+55npcZ1V3PTXJZA/jvA9/VlfldCUwBUNUPgSygqFmia0IpKcId3xjAxh0V3P9GSdDhmBYq6t+ERGSiiKwCLmZvTShSE8WByleHKY+0zpg3W/iTkN3W2zTCLKCviBSLSAZex4NpIfN8CXwNQEQOxUtC8d3e1kBDurfm3CO78uh7y1i+qSzocEwLVG8SEpHXRWR+mMdYAFW9VVW7AU8D1zd1wG6dMW+2qEtC95w3yEZLMA2mqtV4+/2rwCK8XnALROQuETnLzfYj4GoRmQs8C1ymCTQc9U1j+pOZlsovX1wYdCimBaq33UlVT2ngsp7G+9H1diI3UawBRoWUv+XKu4aZv9nUDcqYkWYdBk3jqOp0vH3fX3ab7/lC4Njmjqu5tM/P4gdf68vE6Yt44/P1nHxIh6BDMi1ItL3j+vpejgXqrl6bBlzqesmNBLap6jq8s8XTRKS165BwGvCqm7ZdREa6XnGXAlOjia2xKlxNKCPVkpAxjTX+mJ70apfLL1+0UbZN40R7xP2Na5qbh5dQfuDKpwPLgBLgYeC7AKpaCvwSrx19FnCXK8PN84h7zxfAy1HG1iiVNV4Syky3JGRMY2WkpXDbmQNYvqmMv7+/IuhwTAsSVTcwVf1WhHIFrosw7THgsTDls4HDooknGnW/CWVaTciYgzKqf3tOObQ9f/7fUs4Z0oX2NvyVaQA74jp1Sch+EzLm4P386wOoqlF+84qNK2caxo64jiUhY6LXsyiXq44v5l8fr2HOyi1Bh2NaADviOnW/CVkSMiY6153Uhw4Fmdw+bT41Nq6cqYcdcZ1K6x1nTEzkZqbx868PYP6a7Tw1w24Fbg4sqY+481Zv5c3FGwBrjjMmls48ohPH9y3inlcXs2H77qDDMXEsqY+4Z93/Ppf/fRY1tUqFNccZEzMiwp1nDaSiupaJ0xcFHY6JY3bEBS55dCYz3c25MlNTA47GmMTQq10e14zqzdRP1/J+yaagwzFxypIQ8MEXm3lx3jrALlY1Jpa+O6o3Pdrm8Iup820kBRNW0h5xq2tqqbttnn/UbOuYYEzsZKWncudZA1m2sYyH31kWdDgmDiXtEXfTzkpUYeI3D+PiEd33lKekNOsNXY1JeKP6t+eMwzvy5zdK+HLzrqDDMXEmaZPQetdjp0N+FoU5dusGY5rSbWcOJC1FuH3afBLoLhYmBiwJFWRRmJ0ecDTGJLaOrbK44dR+vLl4I68uWB90OCaOJG0S2rSzEoCi/AxaWRIypslddkxPDumYz53/WcDOiuqgwzFxImmT0M6KKgDys9IpzLEkZExTS0tN4VfnHM5X23dzz6uLgw7HxImkS0KzV5Ty6HvL2VnhdRfNSU8lP8uSkDHNYWj31ow/uidPfLjCBjg1QIySkIj8SERURIrcaxGR+0SkRETmichQ37zjRWSpe4z3lR8pIp+599zn7rAac//8eDV3v/w5O3dXk5uRSkqKkGo94oxpNj8e3Z9OBVnc/M95e4bLMskr6iQkIt3w7qr6pa/4dKCve0wAHnTztgFuB0YAw4Hb3W2+cfNc7XvfmGhjC6esoobKmlpWbC4jJzOqe/oZYw5CXmYa//fNw1i6YScPvvVF0OGYgMWiJnQvcBPg73c5FpiknhlAoYh0AkYDr6lqqapuAV4DxrhpBao6w92VdRJwdgxi20+Z+0F08Vc7yLMkZEwgTj6kA2cN6sz9by5l6fodQYdjAhRVEhKRscAaVZ0bMqkLsMr3erUrO1D56jDlMVdW6SWhNVvLyc3cO07cvRcM4k8XDm6KVRpjwrjtGwPIzUzj5n99Rq3ddyhp1VsVEJHXgY5hJt0K/AyvKa5ZicgEvGY+unfvXs/c+9pVuXf8qtyMvR//m0O6xiY4Y0yDFOVl8ouvD+BH/5jL0zNXcsnRPYMOyQSg3pqQqp6iqoeFPoBlQDEwV0RWAF2Bj0WkI7AG6OZbTFdXdqDyrmHKI8X0kKoOU9Vh7dq1a8jn3KPMd31CrjXHGROoc4Z24fi+Rdz9ymLWbSsPOhwTgINujlPVz1S1var2VNWeeE1oQ1X1K2AacKnrJTcS2Kaq64BXgdNEpLXrkHAa8Kqbtl1ERrpecZcCU6P8bGHtUxOyJGRMoESEX33zcGpqlV/824b0SUZNdZ3QdLyaUgnwMPBdAFUtBX4JzHKPu1wZbp5H3Hu+AF5uisD8NaG8TLt3kImeiIwRkcXu8oKbw0y/V0Q+dY8lIrI1iDjjVbc2OfzotH68vmgD0+auDToc08xiVhVwtaG65wpcF2G+x4DHwpTPBg6LVTzhPDljJdt3+5rjMqwmZKIjIqnAA8CpeK0Bs0RkmqourJtHVW/wzf89YEizBxrnLj+2mJc+W8ft0xZwdO+2tM/PCjok00ySZsSE7bur+MW/5wPQNtcbNduuEzIxMBwoUdVlqloJTMa7RCGSccCzzRJZC5KaIvzu3EHsqqzh5y9Ys1wySZok5G+G69chH7DmOBMTkS472I+I9MDrzPNGM8TV4vRpn8ePT+vHfxeut2a5JJJESWhvh4R+HfIA65hgmt2FwPOqGvY+1yIyQURmi8jsjRs3NnNo8eHK43oxtHsht01dwAZ3uxWT2JIoCe2tCQ3uXsg3h3ThmN5FAUZkEkSkyw7CuZADNMVFc+lBokhNEe45bxC7q2r4mTXLJYWkTEKF2Rnce8FgiotyA4zIJIhZQF8RKRaRDLxEMy10JhE5BGgNfNjM8bU4vdrl8ZPR/Xl90Xr+/WnEywVNgkiaJOS/iVZFddjWEGMaTVWrgevxroFbBExR1QUicpeInOWb9UJgstqpfYNcfmwxw3q05vapC/bcBdkkpqRJQnVjxoF3XYIxsaKq01W1n6r2VtWJruw2VZ3mm+cOVd3vGiITXmqK8LvzBlFZU8vP/vWZNcslsKRJQnU3sXv5B8czsHOrgKMxxtSnuCiXm0Yfwv8+38Dzc1bX/wbTIiVNEtrlmuO6Wy3ImBbjsmN6MqK4DXf+ZyGrSncFHY5pAkmThMoqqhGBnAy7NsiYliIlRfj9+YMQ4IbnPqXGbvmQcJImCe2sqCE3I40mumu4MaaJdG2dw11nD2T2yi389W27E2uiSZokVFZRvc9N7IwxLcfZg7tw5hGduPe1JXy2elvQ4ZgYSpoktLOy2kZIMKaFEhEmnn04RXmZ/PC5TyivtMssEkXSJKFdFdXkWRIypsVqlZPO788fxBcby/jNy4uCDsfESNIkobKKGuuUYEwLd2yfIq44tpgnPlzJW4s3BB2OiYGkSUIVNbVkpVsSMqalu2lMf/p1yOMnz8+jtKwy6HBMlKJKQiJyh4is8d018gzftFvcnSYXi8hoX3nYu1C6sbdmuvLn3DhcMVNZXUt6atLkXGMSVlZ6Kn+8YAjbdlVx8z/n2WgKLVwsjsr3qupg95gOICID8MbKGgiMAf4iIqm+u1CeDgwAxrl5Ae52y+oDbAGujEFse1TV1JJhSciYhDCgcwE3jenPfxeu56kZK4MOx0ShqY7KY/EGa6xQ1eVACd4dKMPehVK8i3dOBp53738CODuWAVVW15KRZknImERxxbHFjOrfjl++tIiFa7cHHY45SLE4Kl8vIvNE5DERae3KIt1tMlJ5W2CrG5HYXx7Wwdz8q6qmlvRUu1DVmESR4u49VJidzvee/ZhdvkGKTctRbxISkddFZH6Yx1jgQaA3MBhYB/y+ieMFDu7mX1YTMibxFOVl8scLBrNsUxl3TFsQdDjmINR74YyqntKQBYnIw8CL7uWB7jYZrnwzUCgiaa42dKC7Ux6UyhrrmGBMIjqmTxHXjerD/W+WcGyfIsYOjtiIYuJQtL3jOvlefhOY755PAy4UkUwRKQb6Ah8R4S6U7kZfbwLnuvePB6ZGE1so65hgTOL64Sl9GdajNbe+MJ+Vm8uCDsc0QrRH5d+KyGciMg84CbgBQFUXAFOAhcArwHWqWhPpLpRuWT8FbhSRErzfiB6NMrZ9WHOcMYkrLTWFP144mBSB7z/7CZXVtUGHZBooqnFsVPWSA0ybCEwMUz4dmB6mfBle77mYq6lVahVrjjMmgXVtncNvzz2Ca576mHv+u5ifnXFo0CGZBkiKo3LdWZHVhIxJbGMO68QlI3vw0DvLeH3h+qDDMQ2QFEflyhovCVlNyJjEd+vXD+WwLgXcOOVTvtxsd2ONd0lxVLaakDHJIys9lQcvPhKA7z4zh91VdtuHeJYUR+UqVxPKsItVjUkK3drk8IfzBzN/zXbuenFh0OGYA0iKJFRXE7LmOGOSxykDOnDtqN48M/NL/vXx6qDDMREkxVF5T03ImuOMSSo/OrUfI4rb8LMXPmPxVzuCDseEkRRHZeuYYExySktN4c8XDSE/K51rn5rDzgobXy7eJMVR2TomGJO82udn8edxQ1ixuYyf2v2H4k5SHJWrarydzobtMU0h0o0aQ+Y5X0QWisgCEXmmuWNMdiN7teUnow/hpXnrePS95UGHY3yiGjGhpbCakGkqvhs1nop3C5JZIjJNVRf65ukL3AIcq6pbRKR9MNEmt2tO7MXcVVv51fRFHNqpgGP7FAUdkiFpakL2m5BpMmFv1Bgyz9XAA6q6BUBVNzRzjAYQEe45fxC92+Vx/TMfs6rULmSNB0lxVK7Y00XbrhMyMRfpRo1+/YB+IvK+iMwQkTHNFp3ZR15mGg9fOoyaWmXCk3Mor7QLWYOWFEmoriaUac1xJhhpeLczGQWMAx4WkcLQmQ7mjsGm8XoW5XLfuCF8/tV2brKOCoFLiqOyXaxqmtCBbuBYZzXefbOqVHU5sAQvKe3jYO4YbA7OqP7t+fFp/fnP3LU8/O6yoMNJaklxVLaLVU0TCnujxpB5/o1XC0JEivCa5+zIF7DvjurNGYd35Dcvf867S63mGZSkOCpbxwTTVCLdqFFE7hKRs9xsrwKbRWQh3h2Ef6Kqm4OJ2NQREX537iD6ts/n+mc+sRG3AxL1UVlEvicin7vrH37rK7/FXTexWERG+8rDXlPhziRnuvLn3FllTFRYF23ThFR1uqr2U9Xe7maOqOptqjrNPVdVvVFVB6jq4ao6OdiITZ3czDQeutQbcfvKJ2axY3dVwBEln6iOyiJyEl531EGqOhC4x5UPwGuWGAiMAf4iIqm+aypOBwYA49y8AHcD96pqH2ALcGU0sfnZxarGmEh6tM3lwYuHsnxTGd979hOqa+zW4M0p2qPytcBvVLUC9rn+YSwwWVUr3A+xJXjXU4S9pkJEBDgZeN69/wng7Chj28M6JhhjDuSYPkXcOXYgby3eyMTpi4IOJ6lEe1TuBxzvmtHeFpGjXHmkaycilbcFtrr2dX95WI3tylpVU0tqipCaYtcJGWPCu3hEDy4/tid/f38FT89cGXQ4SaPeYXtE5HWgY5hJt7r3twFGAkcBU0SkV0wjDENVHwIeAhg2bFi9nfwra2rtQlVjTL1uPeNQlm8q4/apCyhum8sxNrRPk6u3JqSqp6jqYWEeU/FqLP9yP7x+BNQCRUS+diJS+WagUETSQspjorK61n4PMsbUKy01hfvGDaG4KJdrnprDso07gw4p4UV7ZP43cBKAiPQDMoBNeNdJXCgimSJSjHdh3kdEuKZCvUuW3wTOdcsdD0yNMrY9KmtqrWecMaZBCrLSeeyyo0hLTeHKJ2azbZf1mGtK0R6ZHwN6ich8vE4G412taAEwBVgIvAJcp6o1ka6pcMv6KXCjiJTg/Ub0aJSx7VFlNSFjTCN0a5PD3y45ktVbdnHNU3P2dG4ysRfVrRxcD7dvR5g2EZgYpnw6MD1M+TK83nMxV1VTS7rVhIwxjXBUzzbc/a0juHHKXH76z3n84fxBeB15TSwlx/2Eamqte7YxptHOGdqVNVvK+f1rS+hSmM2PR/cPOqSEkxxJqFqtOc4Yc1CuP7kPa7aWc/+bJXQuzOaiEd2DDimhJEcSsuY4Y8xBEhH+7+zDWLdtN7+YOp9OrbI46RC7OW6sJMWRuaq6lkyrCRljDlJaagoPXDyUQzrmc90zH/PZ6m1Bh5QwkuLI7NWE7AdFY8zBy8tM4++XHUXrnAyueGKW3R48RpIiCVXVWBdtY0z02hdk8fjlR1FRVcNlf/+ILWWVQYfU4iXFkbmy2nrHGWNio2+HfB6+dBirtpRz+eOzKKuorv9NJqKkODJbxwRjTCyN6NWWP48bwrzVW+1i1iglxZG5qsY6JhhjYmv0wI785pwjeHfpJn70j7nU1tY7lrIJIzm6aFtznDGmCZx/VDc2l1Vy9yuf0yYnnTvOGmijKjRSUiShqhq1AUyNMU3imhN7UVpWwcPvLqdNbiY/OKVv0CG1KEmRhKwmZIxpKiLCz844lNKyKu59fQlt8jK4ZGSPoMNqMZIjCdmtHIwxTUhEuPtbh7OtvJLbps6nICuNsYMj3hza+CT8kVlV3U3trJ3WGNN00lJTuP+ioYwobsONU+byyvyvgg6pRUj4JFTteqxYc5wxpqllpafyyPijOKJrK7737Me8uXhD0CHFvaiOzCLynIh86h4rRORT37RbRKRERBaLyGhf+RhXViIiN/vKi0Vkpit/zt15NWpVNV7/fWuOM8Y0h7zMNB6/fDj9OuRzzZNz+OCLTUGHFNeiOjKr6gWqOlhVBwP/BP4FICID8G7dPRAYA/xFRFJFJBV4ADgdGACMc/MC3A3cq6p9gC3AldHEVqfuIjKrCRljmkur7HSevHIEPdrmcNUTs5mzsjTokOJWTI7M4nWMPx941hWNBSaraoWqLgdK8O6aOhwoUdVl7q6sk4Gx7v0nA8+79z8BnB2L2CqtJmSMCUCb3AyeumoEHQqyuOyxWTbydgSxOjIfD6xX1aXudRdglW/6alcWqbwtsFVVq0PKo1ZXE7IBTE1TidTE7Jt+mYhs9DVdXxVEnKb5tc/P4umrRlCQnc4lj81k0brtQYcUd+o9MovI6yIyP8xjrG+2ceytBTU5EZkgIrNFZPbGjRsPOG9VjdcxwWpCpinU08Ts91xd07WqPtKsQZpAdS7M5tmrR5KVlsrFj1giClXvkVlVT1HVw8I8pgKISBpwDvCc721rgG6+111dWaTyzUChW5a/PFJMD6nqMFUd1q5duwPGb78JmSYWtok54JhMnOneNofJE0aSmZbCRQ/PYOFaS0R1YnFkPgX4XFVX+8qmAReKSKaIFAN9gY+AWUBf1xMuA6/zwjRVVeBN4Fz3/vHA1BjEtqd3XLpdJ2SaRqQm5lDfEpF5IvK8iHQLM71RNXzT8vQsymXyhJFkpady8SMzWLDWfiOC2CShCwlpilPVBcAUYCHwCnCdqta433yuB14FFgFT3LwAPwVuFJESvN+IHo1BbFRUW8cEE7j/AD1V9QjgNbyON/tpTA3ftEw92nqJKDvda5qbv8YSUdRHZlW9TFX/GqZ8oqr2VtX+qvqyr3y6qvZz0yb6ypep6nBV7aOq56lqRTRxTZm1ip43v8TGHd5istNTo1mcMZFEamLeQ1U3+/bnR4Ajmyk2E4e8RHQ0uRlplohI4BETJk5fBMCareUA5GQkxTB5pvmFbWL2zyAinXwvz8JrBTBJrO43orzMNC56eEZSd99O2CS0rbxqn7/ZGVYTMrEXqYlZRO4SkbPcbN8XkQUiMhf4PnBZMNGaeNKtjZeICrLTufiRGcxdtTXokAKRsEmozpaySgByLAmZJhKuiVlVb1PVae75Lao6UFUHqepJqvp5sBGbeFGXiFrlpHPxIzOZuWxz0CE1u4RMQnW1H4AtuywJGWPiV9fWOfzjO8fQsVUWlz72UdINepqQSWh7eRWpKV6X7LokZM1xxph41bFVFs9NGEmf9nlMmDSbl+atCzqkZpOQSahbmxz+de0xAGwp8xKSDdtjjIlnbfMyeXbCSAZ1LeR7z37MlNmr6n9TAkjYI3NmuvfRtuyqJCc9FW+MVGOMiV8FWelMunI4x/Yp4qbn5/H395cHHVKTS9gklJXmNb+VllVaU5wxpsXIyUjjkfHDGD2wA3f+ZyH3v7EUb1CZxJS4SchdnFpRXWudEowxLUpmWioPXDSUc4Z04Z7/LuH/XlpEbW1iJqKEvYIzK31vfs22C1WNMS1MWmoK95w3iILsdB59bzkbd1Rwz3mDEm4IsoQ9Omf5humxmpAxpiVKSRFu/8YA2hdk8ttXFlNaVsmD3x5KflZ60KHFTGKlVB9/bzhLQsaYlkpE+O6oPvzu3CP4cNlmLnxoxp4xMRNBwiahlBTZU221wUuNMS3decO68cj4YSzbWMa3HvyA5ZvKgg4pJhI2CQFk1SUhqwkZYxLASf3b8+yEkeysqObcBz9IiPHmEjsJuRqQNccZYxLF4G6FPH/N0WRnpDLu4Rm8+XnLHuYnoZNQ3QWrdhsHY0wi6dUuj39dewzFRblc+cQsJn24IuiQDlpUSUhEBovIDBH51N2WeLgrFxG5T0RK3C2Nh/reM15ElrrHeF/5kSLymXvPfRKDIQ7KK727qnYuzI52UcYYE1faF2Qx5TtHc/Ih7blt6gLumLaAmhZ4LVG0NaHfAneq6mDgNvca4HSgr3tMAB4EEJE2wO3ACGA4cLuItHbveRC42ve+MVHGxqadXg+SHm1yol2UMcbEndzMNP52yTCuOLaYxz9YwYRJsymrqA46rEaJNgkpUOCetwLWuudjgUnqmQEUurtLjgZeU9VSVd0CvAaMcdMKVHWGeuNTTALOjjK2PXoWWRIyxiSm1BThtm8M4JdjB/Lm4g2c99cPWbetPOiwGizaJPRD4Hcisgq4B7jFlXcB/EPArnZlBypfHaY8LBGZ4Jr/Zm/cuLHeILu2tiRkjElslxzdk0cvO4qVm8s4+4H3mb+mZdwyvN4kJCKvi8j8MI+xwLXADaraDbgBeLSpAwZQ1YdUdZiqDmvXrl2982fZdULGmCRwUv/2PH/tMaSKcP7fPuSdJfWfpAet3m5jqnpKpGkiMgn4gXv5D+AR93wN0M03a1dXtgYYFVL+livvGmb+qEy6Yvie34WMMSYZHNqpgH9fdyw/fn4e3VrA7+HRNsetBU50z08Glrrn04BLXS+5kcA2VV0HvAqcJiKtXYeE04BX3bTtIjLS9Yq7FJgaZWyc0K8d5wztWv+MxhiTQNoXZDHpiuEUF+UGHUq9or2A5mrgTyKSBuzG6wkHMB04AygBdgGXA6hqqYj8Epjl5rtLVUvd8+8CjwPZwMvuYYwxJoFFlYRU9T3gyDDlClwX4T2PAY+FKZ8NHBZNPMYYY1qWhB4xwRhjTHyzJGSMMSYwloSMMcYExpKQMcaYwFgSMsYYExhLQsYYYwIjXm/qlktENgIrI0wuAjY1Yzj1sXjqF28x9VDV+seGirEWtF9bLOHFSyyR4ugLfKiqUd+tIFotPgkdiIjMVtVhQcdRx+KpXzzGFG/iaRtZLOHFSyzxEseBWHOcMcaYwFgSMsYYE5hET0IPBR1ACIunfvEYU7yJp21ksYQXL7HESxwRJfRvQsYYY+JboteEjDHGxLGETEIiMkZEFotIiYjcHGAcK0TkMxH5VERmu7I2VniroAAAAyhJREFUIvKaiCx1f1s34fofE5ENIjLfVxZ2/e7eT/e5bTZPRIY2Uzx3iMgat40+FZEzfNNucfEsFpHRsY6nJQpy3w5yf46nfTme9mMR6SYib4rIQhFZICI/cOWBfc8bTVUT6gGkAl8AvYAMYC4wIKBYVgBFIWW/BW52z28G7m7C9Z8ADAXm17d+vPs/vQwIMBKY2Uzx3AH8OMy8A9z/LhModv/T1KD3ryAfQe/bQe7P8bQvx9N+DHQChrrn+cASt87AvueNfSRiTWg4UKKqy1S1EpgMjA04Jr+xwBPu+RPA2U21IlV9BygNKY60/rHAJPXMAApFpFMzxBPJWGCyqlao6nK8GyQOj2U8LVA87tvNsj/H074cT/uxqq5T1Y/d8x3AIqALAX7PGysRk1AXYJXv9WpXFgQF/isic0Sk7q6zHdS7nTnAV0CHZo4p0vqD3G7Xu6aBx3zNOfH0f4wXQW+TeNuf421fDnQ/FpGewBBgJvG3bSJKxCQUT45T1aHA6cB1InKCf6J69ePAuicGvX7nQaA3MBhYB/w+2HDMAcTt/hwH+3Kg+7GI5AH/BH6oqtv90+Jg2xxQIiahNUA33+uurqzZqeoa93cD8AJeNXx9XfXX/d3QzGFFWn8g201V16tqjarWAg+zt6kibv6PcSTQbRKH+3Pc7MtB7sciko6XgJ5W1X+54rjZNvVJxCQ0C+grIsUikgFcCExr7iBEJFdE8uueA6cB810s491s44GpzRxapPVPAy51vWdGAtt81fkmE9Ie/U28bVQXz4UikikixXgDLn7U1PHEucD27Tjdn+NmXw5qPxYRAR4FFqnqH3yT4mbb1CvonhFN8cDrAbIEryfKrQHF0AuvV8xcYEFdHEBb4H/AUuB1oE0TxvAsXtNAFV7b75WR1o/XW+YBt80+A4Y1UzxPuvXNw/uCdPLNf6uLZzFwetD7VTw8gtq3g96f42lfjqf9GDgOr6ltHvCpe5wR5Pe8sQ8bMcEYY0xgErE5zhhjTAthScgYY0xgLAkZY4wJjCUhY4wxgbEkZIwxJjCWhIwxxgTGkpAxxpjAWBIyxhgTmP8HHbczQ3RgMn4AAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#plots moving average of steps/episode\n",
        "a = moving_average(steps_episode, 10)\n",
        "plt.plot(a)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 283
        },
        "id": "QttLTg3r9YKk",
        "outputId": "283b7c8e-cc85-4052-a1dc-c93a64e970a2"
      },
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<matplotlib.lines.Line2D at 0x7faef29170d0>]"
            ]
          },
          "metadata": {},
          "execution_count": 39
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de3xc5X3n8c9v7pJmdL9YtoVtwMaYFHMxhCTcUhJuDSHd3aTJdgtNk6Xbkr7SbbcN9Jbbdpu2m+xu+krTQEMDaS5NNqGYlAYImyZpwYANtrEhji9Y2LJsybpLI81oZp79Y47EGEu+SZoz1vm+X695zdEzRzO/OZK+8+g5zznHnHOIiEgwhPwuQEREykehLyISIAp9EZEAUeiLiASIQl9EJEAifhdwIs3NzW7lypV+lyEiclbZsmXLUedcy0yPVXTor1y5ks2bN/tdhojIWcXMOmd7TMM7IiIBotAXEQkQhb6ISIAo9EVEAkShLyISIAp9EZEAUeiLiARIIEL/J7t72dMz4ncZIiK+C0To/963t/Pp773idxkiIr5b9KHvnKN/LMtzr/aTzRX8LkdExFeLPvTT2TzZfIHxyTxbDwz6XY6IiK8WfegPpLPTy/+656iPlYiI+G/Rh/5genJ6+WmFvogE3ElD38w6zOyHZvayme00s4967Y1m9qSZ7fbuG7x2M7PPm9keM9tuZpeVPNed3vq7zezOhXtbr5sK/Q0rGth6YJCxTK4cLysiUpFOpaefA37XObcOuAq428zWAfcATznnVgNPeV8D3AKs9m53AV+E4ocE8HHgzcCVwMenPigW0tTwzo0XtZErOHYeGl7olxQRqVgnDX3nXLdz7gVveQR4BVgG3A486K32IPAeb/l24CFXtAmoN7N24CbgSedcv3NuAHgSuHle380MBseLPf117XUA9IxMLPRLiohUrNMa0zezlcClwLNAm3Ou23voMNDmLS8DDpR820Gvbbb2N77GXWa22cw29/b2nk55MxocK/b017QlAegZzsz5OUVEzlanHPpmlgS+A/y2c+6YMRLnnAPcfBTknLvPObfBObehpWXGq32dloH0JDWxMC2pONGw0TOi0BeR4Dql0DezKMXA/5pz7rte8xFv2Abvvsdr7wI6Sr59udc2W/uCGhzPUl8dw8xoScbpVeiLSICdyuwdA74MvOKc+1zJQxuBqRk4dwKPlLTf4c3iuQoY8oaBHgduNLMGbwfujV7bghpMT1JfHQWgpTahMX0RCbRTuTD624BfAV4ys61e2x8AnwG+ZWYfAjqB93mPPQbcCuwB0sAHAZxz/Wb2aeB5b71POef65+VdnMBAOktDdQyA1lScA/3phX5JEZGKddLQd879K2CzPHzDDOs74O5ZnusB4IHTKXCuhtKTLK2vAqAlFeeFzoFyvryISEVZ1EfkOue8nn5xeKc1FadvLMtkXideE5FgWpSh3zU4zrV/8UM2bjvE0PhkyfBOAoCjo9qZKyLBtChDf0ltgoGxLE/sPELBQV2VtyM3FQfQDB4RCaxFGfrhkHHlqkZ+uKs4i7R0Ry7oAC0RCa5FGfoAbzmviXQ2DzA9ZbO11gt99fRFJKAWbehfdW7T9HK919NvqpkKfc3VF5FgWrShf2F7LbWJ4ozUqdk7sUiIxpqYxvRFJLAWbegXx/WLvf2pnj4Ux/U1vCMiQXUqR+SetW6/ZCmHBsenZ+9AcQaPQl9EgmpRh/5t65dy2/qlx7Q1VMd0KgYRCaxFO7wzm1QiwsiELpkoIsEUuNBPJiKM6Dq5IhJQgQv92kSUbK5AJpf3uxQRkbILXOgn48XdGKMa4hGRAApc6Ke8ufsa1xeRIApc6E/39DWuLyIBFLjQTyWKc/aHJyZ9rkREpPwCGPoa3hGR4Aps6GtHrogEUeBCf2pMf0TDOyISQMEL/YR25IpIcAUu9OORMLFISGP6IhJIgQt9gFqdikFEAiqQoZ+M66RrIhJMgQz9VCLKqHbkikgABTT01dMXkWAKZOgn4xHN3hGRQApk6KcSUfX0RSSQAhr6ER2cJSKBFNjQH83kcM75XYqISFkFMvST8QgFB+msrp4lIsESyNCfOr2yxvVFJGgCGfqvn39H4/oiEiyBDP2p0ysPq6cvIgETzNCP60IqIhJMwQx9b0xfF1IRkaAJZOgnE7qQiogEUyBDP6ULqYhIQAUy9Gti2pErIsF00tA3swfMrMfMdpS0fcLMusxsq3e7teSxe81sj5ntMrObStpv9tr2mNk98/9WTl04ZMWTrin0RSRgTqWn/xXg5hna/5dz7hLv9hiAma0D3g9c5H3PX5tZ2MzCwBeAW4B1wAe8dX1TvJCKxvRFJFgiJ1vBOfdjM1t5is93O/BN51wGeNXM9gBXeo/tcc7tAzCzb3rrvnzaFc+TqfPviIgEyVzG9D9iZtu94Z8Gr20ZcKBknYNe22ztxzGzu8xss5lt7u3tnUN5J6YLqYhIEJ1p6H8ROA+4BOgGPjtfBTnn7nPObXDObWhpaZmvpz1OMhHVxdFFJHDOKPSdc0ecc3nnXAG4n9eHcLqAjpJVl3tts7X7RufUF5EgOqPQN7P2ki9/EZia2bMReL+Zxc1sFbAaeA54HlhtZqvMLEZxZ+/GMy977lKavSMiAXTSHblm9g3geqDZzA4CHweuN7NLAAfsB34dwDm308y+RXEHbQ642zmX957nI8DjQBh4wDm3c97fzWnQmL6IBNGpzN75wAzNXz7B+n8K/OkM7Y8Bj51WdQsoGY8yPpknly8QCQfyGDURCaDApp1OxSAiQRTY0H/9pGsKfREJjsCGfq1CX0QCKLChn4xPXSdX0zZFJDgCG/oa0xeRIAps6GtMX0SCKLChP9XT16kYRCRIghv6GtMXkQAKbOgnoiEiIdOpGEQkUAIb+mZGUqdiEJGACWzogy6kIiLBE+zQj0c1pi8igRLo0NfwjogETaBDv1ahLyIBE+jQT8Y1pi8iwRLo0E8lNKYvIsES6NBPerN3nHN+lyIiUhaBDv26qiiTeUc6m/e7FBGRsgh06Lem4gAcGZ7wuRIRkfIIdOgvqU0AcFihLyIBEejQb6srhr56+iISFIEO/eme/lDG50pERMoj0KFfE4+QikfU0xeRwAh06ENxiOfwUDH0B8ayPlcjIrKwAh/6S2oTHB6eYEvnAJf99yfZdXjE75JERBZM4EO/rTbBkeEJNu3rwznY3aPQF5HFK/Chv6QuTs9Ihu0HBwGmh3oARjM5nt/f71dpIiLzTqFfmyBfcDy9pw84NvTv+9Fe3n/fJp2fR0QWjcCHfps3bXPEO9tm6YFaT+/tI19wdA9pdo+ILA6BD/0l3gFaALFIaLqnP57Ns80b8jk0OO5LbSIi802hX/t66L/1vKbpnv6Lrw0wmS+effOwevoiskgEPvSbknHCIaOjsYoLlqToGc5QKDg2vdpPyMAMDin0RWSRCHzoh0PG8oYqLuloYEltgmy+QH86y6Z9fVy0tI6WZJzDQyce3tH5+EXkbBH40Af48p0b+ON3XUi7N77f2Zdm64FB3ryqkfa6xEl35H7g/k382T+/Uo5SRUTmJOJ3AZXg/NYU8PpMnke3HSKbK/DW85s4ODDOnt7RWb/XOceLrw1SE9OmFJHKp55+ifa6KgAefrGLWCTEVec2saTk3Dwz6R3JkMkVGJnQBdZFpPIp9Es0J2OEDIbGJ3nzqkaqYxGW1icYzeQYnuUArQMDaYBZHxcRqSQK/RKRcIgW7xKK161pAWCJ1/ufrbd/oL+4k3c0o56+iFQ+hf4bTIX8VOhP7dyd7QCt1/qLPX0N74jI2eCkoW9mD5hZj5ntKGlrNLMnzWy3d9/gtZuZfd7M9pjZdjO7rOR77vTW321mdy7M25m7lU3VdDRWcX5rEng99Gfv6RdDfzST09RNEal4p9LT/wpw8xva7gGecs6tBp7yvga4BVjt3e4CvgjFDwng48CbgSuBj099UFSaP37XOr7+4aswM6A4o+dEB2hNjennC450Nl+2OkVEzsRJQ98592PgjecXvh140Ft+EHhPSftDrmgTUG9m7cBNwJPOuX7n3ADwJMd/kFSE5mScjsbq6a+j4RAtyfiswzsH+sfxPh80xCMiFe9Mx/TbnHPd3vJhoM1bXgYcKFnvoNc2W/txzOwuM9tsZpt7e3vPsLz5tba9lh1dQ8e1T+YLdA+Ns7KpBkCnYBaRijfnHbmuOJA9b4PZzrn7nHMbnHMbWlpa5utp5+TycxrYdWTkuGmZ3YMTFBysa68FXj89s4hIpTrT0D/iDdvg3fd47V1AR8l6y7222drPCpevaMA52PraIM458oXiZ9zUzJ0L24tH9Gp4R0Qq3ZmeO2AjcCfwGe/+kZL2j5jZNynutB1yznWb2ePA/yjZeXsjcO+Zl11el5xTT8hgc+cAG7cd4rGXunnLuU30p7MArFvq9fQ1vCMiFe6koW9m3wCuB5rN7CDFWTifAb5lZh8COoH3eas/BtwK7AHSwAcBnHP9ZvZp4HlvvU85586ai88m4xHWLqll49YuOvvTXNJRz6t9Y4TM+IWL21nTpp6+iJwdThr6zrkPzPLQDTOs64C7Z3meB4AHTqu6CnL5iga+uqmTZDzCA3deQUNNbPqxqR6+evoiUul0RO4punxFcWTq19628pjAB6iJRTCDUfX0RaTC6XzAp+jGi9r4r+9Yw4euWXXcY6GQkYxHGFboi0iFU+ifoupYhI++Y/Wsj9cmohrTF5GKp+GdeZKMRzSmLyIVT6E/T1KJiHr6IlLxFPrzJJWIMJJRT19EKptCf56kElHN3hGRiqfQnyca3hGRs4FCf56kNHtHRM4CCv15kkpEyOYLDKUnGUprbF9EKpNCf56kEsVDHv7Tl5/lvV962udqRERmpoOz5slU6L/kXWyldyRDSyruZ0kiIsdRT3+epOLR4r0X/i+8NuBnOSIiM1Loz5OVzdXEIiH+9y9dQiwc4oVOhb6IVB4N78yT81tT7PzkTUTDId60rJYtCn0RqUDq6c+jaLi4OS9f0cD2riGyuYLPFYmIHEuhvwAuX9FANldg56Ehv0sRETmGQn8BXHZO8YIrL7w26HMlIiLHUugvgNbaBC2pOC8fGva7FBGRYyj0F8iF7bW80q3QF5HKotBfIBe2p9jTM8pkXjtzRaRyKPQXyLr2WrL5Ant7R/0uRURkmkJ/gaxdUgugIR4RqSgK/QVybksNsXCIV7pH/C5FRGSaQn+BRMMhVrcl1dMXkYqi0F9AxRk86umLSOVQ6C+gi5fXcXQ0o/n6IlIxFPoL6Pb1y6iOhfnbn+zzuxQREUChv6DqqqP80hUdbNx2iO6hcb/LERFR6C+0X3vbKhzwd/+23+9SREQU+guto7Ga2y5u5+83ddI/lvW7HBEJOIV+GXzk589nfDLP/RrbFxGfKfTL4PzWFO+6eCkPPb2fAfX2RcRHCv0yufvt5zGWzbNx2yG/SxGRAFPol8naJbWsXZLiUYW+iPhIoV9Gt61fyubOAboGNX1TRPyh0C+j2y5eCsA/bVdvX0T8odAvo3OaqlnfUc83njvAYFo7dEWk/BT6ZfZ7N15A1+A4//H+ZzWTR0TKbk6hb2b7zewlM9tqZpu9tkYze9LMdnv3DV67mdnnzWyPmW03s8vm4w2cba5e3cz9d2xgT88on3x0p9/liEjAzEdP/+3OuUuccxu8r+8BnnLOrQae8r4GuAVY7d3uAr44D699VrpuTQv/+dpV/OPWQ2w9MOh3OSISIAsxvHM78KC3/CDwnpL2h1zRJqDezNoX4PXPCr9x/fk0J+N8+nsv45zzuxwRCYi5hr4DnjCzLWZ2l9fW5pzr9pYPA23e8jLgQMn3HvTajmFmd5nZZjPb3NvbO8fyKlcyHuHD16xiS+cAvSMZv8sRkYCYa+hf7Zy7jOLQzd1mdm3pg67YhT2tbqxz7j7n3Abn3IaWlpY5llfZVjRWA9A7qtAXkfKYU+g757q8+x7gYeBK4MjUsI133+Ot3gV0lHz7cq8tsJpTcQCOjmoWj4iUxxmHvpnVmFlqahm4EdgBbATu9Fa7E3jEW94I3OHN4rkKGCoZBgqk5qQX+hreEZEyiczhe9uAh81s6nm+7pz7vpk9D3zLzD4EdALv89Z/DLgV2AOkgQ/O4bUXheZkDICjGt4RkTI549B3zu0D1s/Q3gfcMEO7A+4+09dbjJLxCPFISKEvImWjI3J9ZGY0J+Ma0xeRslHo+6w5FVdPX0TKRqHvs5ZkTD19ESkbhb7PmmrU0xeR8lHo+6w5FaN/LEuhoFMxiMjCU+j7rDkZJ19wDOj8+iJSBgp9n00foKVxfREpA4W+z14PfY3ri8jCU+j7rCWlo3JFpHwU+j6b6unr9MoiUg4KfZ/VVUWJhk1j+iJSFgp9n5kZTTVx9fRFpCwU+hVg3dJafvSzXiYm836XIiKLnEK/Anz4mlUcHc3wf7cc9LsUEVnkFPoV4C3nNrG+o577fryPXL7gdzkisogp9CuAmfEb153Ha/1p/nHrIb/LEZFFTKFfIW5c18bFy+v47BO7NLYvIgtGoV8hQiHjD269kO6hCe797kt87old7D4y4ndZIrLIzOUauTLPrjq3iZsuauPhF7sA+MrT+/m7D17B5Ssafa5MRBYLK166tjJt2LDBbd682e8yyiqTy9M3miVfcNzxwHP0DE/w9D03UFcd9bs0ETlLmNkW59yGmR7T8E6FiUfCLK2voqOxmv/53osZy+b5yZ5ev8sSkUVCoV/B1i+vp64qyo92KfRFZH4o9CtYJBzi6tXN/OhnvRQKjh1dQ4xnNbNHRM6cduRWuOvXtPBP27v53W9v4+EXu6iOhXn72lZufVM7N17URjSsz20ROXUK/Qp33ZoWAB5+sYsb17XRkorz+M7D/NP2bq5b08L9d2wgFpmf4P/+jm4e2XqIbK7AR9+xmouX18/L84pI5dDsnbPAL//tJgAe+NUriEfC5AuOrz3byZ88spPr1rTwjnVtjGVyHB6a4IYLW7n6/GbM7LRe45m9fXzg/k20porX7O0by7KsvopzGqv5zbefxzWrWxbirYnIAjjR7B2F/lkgly8QDtlxQf53//Yqf/bYT8l65+uJR0JkcgV+blkdn33feta0pTjQn+ZT33uZZfVVfPy2dbN+GPzJIzv41uYDvPDH7yRfcDz0TCd7e0Z5bn8/BwfGuW5NC/feupa1S2oX/P2KyNwo9BexfMFxdDRDIhomEQ2xceshPvPPP2Ukk+P8liT7+8bI5R3ZfIFfv/Zc7r31wuOeo1BwvPUz/4/1HXV86VeO/T3J5PJ89ZlOPv/UbkYyOS7tqOfq85tZ0VTDz69tpaEmVq63KiKn6EShrzH9s1w4ZLTVJqa/fu+GDq6/oJXPPbmL3pEsFy+v47duWM2XfrSXL/14H3t7R/mjX1jHyuaa6e/ZdnCQw8MTfOxNFxz3/PFImA9fcy7/4fLlPPRMJ0++fIS/+uEenIOWVJzPvnc9167R0I/I2UI9/YAoFBz3/2Qfn39qN5lcgV+8dBnt9VUMj0/S2TfGT3YfZcsfvfOUjvzN5PLs6BriY995iT09o1yxsoE/uPVCLj2noQzvRERORsM7Mq1neIK//pe9fP2515jMF6iKhkln89ywtpUv/+oVp/VcE5N5/n5TJ/f9eB+1VVF+8DvXLVDVInI6NLwj01prE3zi3RfxsZvXEg4Z0bBxeHiCuqrTP7dPIloc+gmHjE8++jL7j44dM2wkIpVHR/YEVFUsTCwSwsxor6uiOnbmn//vuLANgB+8cmS+yhORBaLQlznraKxm7ZIUT7z8eug753jxtQG+9mwnfaMZH6sTkVIa3pF58c51bXzhh3v45KM72dE1xP6+NL0jxbD/9Pde5trVLbTXJaiKRegdyfDc/j7Ob0lyzeoWnnz5CIloiN+6YTWXaWewyILSjlyZFzu6hnjXX/0rsXCI9R11nNNYw5vPbWTtkhRffaaTFw8McmR4gkyuQDIe4fIVDWw9MEjvSIaVTdWMTOToG8vy+zdfwG9ef/5JX29gLEtnf5rxbJ5Lz6knEQ3jnGPnoWFe6R4mHDKuWNlIR2N1Gd69SGXRjlxZcG9aVscPfuc6ltYnjts/8JfvnfkcPtlcgdf6xzivJUk6m+fe777EX3x/F0PpSa5Z3ULIYHwyTyoRpX8sw85DwxwdzfKzIyO88NoAU/2VqmiYcxqr6RvLcrRkKCkSMm79uXZikRBhM1pr46xuS3H5igaW1Vct2LYYGp/k8R2HWdZQxVvPawJgNJOjfyxLaypBVSy8YK8tcjLq6UvFyOUL/O63t/HI1kMzPh4OGQ3VUZbWV/Hza1t509I6QiH4l129HB6aoLYqypUrG7liVSPZXIGvPdvJxm2HSETC5J2jbzRDwYEZ3HbxUq46t4lYJMTKpmqWNVRRHY0wms0x4H14PL7zCE/vPco5jdWc15KkrirKZL7AWCbHaCbPWCbHWDY3Hei9I8Ujo8cyOTK54qkxLmhL0TeW4ehoFii+9tK6KhprYtRXR2mqibFmSYoltQkm8wU6Gqv5uWV1pBK6UhoU9w0Bp30uqaDTPH05azjn6B6aYH/fGIZRFQszMjFJMh7hwvZaEtEz7yVncwV294zw6LZuHnpmP+mTXJugKhrmbec3c2hwnAMDaUYmckRCRk08QjIeoSYenl6ur47RkoyTzeeJhkO8e/1SdnQN8ej2bs5prGZ1a5KGmhiHBsfp7EszkM4ymJ6kdyRD1+D4Ma8bi4R47+XLWdteS99ohvqqKK21CRprYhwZnqBnOENNPEIkZGRyeQ4OjnOwf5yDA2kmJgsUnCPvHIlImPrqKPXVUUJmDE/kaK9NsLK5BjMoOIdhtKbirG5LctHSOsIho1BwdPan6RmeoHc0w7/s6uXVo2OEzRhIZxkaL/48kokIqUTx/QP0jmTIFxzhkBEOGYlomMaaGJd21LNmSYpCASYLBfJ5R65Q4OholgMDaQ72j5PJ5elorKam5L/El7qGeGZv3/S5pcwgZIZRvK+Oh2mqidGUjBMLhzg6miEaDlFfHaW2Ksp5LUmuW9NMczJONBwiFvFu4eItFHr9g8S54kkGe4YzpLM5GmtitKTiJOMRzIxcvsBErkDP8ARHhjOMZXKkJ/NMZPOEQ0ZTMsYVKxupib9e/1gmR99ollgkxPDEJM+92k8sHGJlcw2JaMibMl28j4SMSDhEJGTEIyHqqqJz+qCrqNA3s5uB/wOEgb91zn1mtnUV+rJQxrN5hsYnmZjMs+/o6PQfck08QkN1lPrqGBctrT2mx50vOEI2/73OofQkA+ks4ZCx7+gY39/RzXe2dE2H3cnEwiGWNVSxvKGK6lh4+uR8E957HByfJF9wJOMRDg6kGUhPzvg8tV6AD6QnGZ/MH9N+0dI6HI76qhi1VRHS2TwjE8X/ckYncjgcLak4kVCo+KFTcKSzeXqGJzg0NDFr7dGwsbyhmlg4xIGBNJlcAeccDjinsZq3X9BKXVUURzGYnSt+WBUcpLM5jo4W/4vK5go0J+PkCwUGxycZTBePNC+cIN4iISMWCVEdC5PNFRieyM24bQvOkTvRE5W8l9ZUgkjYODqSYWwOFzyqiRU7HPfdMWNun1TFjOmbWRj4AvBO4CDwvJltdM69XM46RKpi4emx9VM9oCwcWpghhrrq6PTpLzoaq7luTQsfu3kt2VyBxpoYg+PF/wj6RrO01sZZUpcgncmTKxSIhUM0JeOnXJtzjvHJPIZN9/YPD03wUtcQz77aTzZXoDYRZe2SFEvrq6itKv6HNZeL9RzoT3NgIE0kFCISNqKhYu+2vjpKW21iwbbrwFiW5/b3M5bJkc0VyOYLx957t7FsnkjIWNVc480wCzOQLg7X9Y1lvd53mHgkRHMyTntdgmQiQnUsQiIaolCA1/rT/Nveo/QMZ5jMF2hKxmhNJWhOxsgVHNFwiCtXNpJ3jgP9aSbzBSbzxQ/HXKFQvM8XP1zS2Rxdg+PUVy3MyQzL2tM3s7cAn3DO3eR9fS+Ac+7PZlpfPX0RkdN3op5+uQ/OWgYcKPn6oNc2zczuMrPNZra5t1cXBBcRmU8Vd0Suc+4+59wG59yGlhadsldEZD6VO/S7gI6Sr5d7bSIiUgblDv3ngdVmtsrMYsD7gY1lrkFEJLDKOnvHOZczs48Aj1OcsvmAc25nOWsQEQmysp+GwTn3GPBYuV9XREQqcEeuiIgsHIW+iEiAVPS5d8ysF+icw1M0A0fnqZyFoPrmRvXNjeqbm0qub4VzbsY57xUd+nNlZptnOyqtEqi+uVF9c6P65qbS65uNhndERAJEoS8iEiCLPfTv87uAk1B9c6P65kb1zU2l1zejRT2mLyIix1rsPX0RESmh0BcRCZBFGfpmdrOZ7TKzPWZ2TwXU02FmPzSzl81sp5l91Gv/hJl1mdlW73arjzXuN7OXvDo2e22NZvakme327ht8qu2Ckm201cyGzey3/d5+ZvaAmfWY2Y6Sthm3mRV93vud3G5ml/lQ21+a2U+913/YzOq99pVmNl6yHf9mIWs7SY2z/kzN7F5v++0ys5t8qu8fSmrbb2ZbvXZftuEZKV53cvHcKJ7IbS9wLhADtgHrfK6pHbjMW04BPwPWAZ8A/pvf28yraz/Q/Ia2vwDu8ZbvAf68AuoMA4eBFX5vP+Ba4DJgx8m2GXAr8M+AAVcBz/pQ241AxFv+85LaVpau5/P2m/Fn6v29bAPiwCrvbzxc7vre8PhngT/xcxueyW0x9vSvBPY45/Y557LAN4Hb/SzIOdftnHvBWx4BXuENVwyrULcDD3rLDwLv8bGWKTcAe51zczlSe144534M9L+hebZtdjvwkCvaBNSbWXs5a3POPeGcm7r69yaK17PwzSzbbza3A990zmWcc68Ceyj+rS+YE9VnZga8D/jGQtawEBZj6J/0kox+MrOVwKXAs17TR7x/tx/wa/jE44AnzGyLmd3ltbU557q95cNAmz+lHeP9HPuHVinbb8ps26zSfi9/jeJ/HlNWmdmLZvYjM7vGr6I8M/1MK237XQMccc7tLmmrpG04q8UY+hXLzJLAd4Dfds4NA3veXq0AAAIdSURBVF8EzgMuAbop/rvol6udc5cBtwB3m9m1pQ+64v+wvs7v9S68827g215TJW2/41TCNpuJmf0hkAO+5jV1A+c45y4Ffgf4upnV+lReRf9MS3yAYzsflbQNT2gxhn5FXpLRzKIUA/9rzrnvAjjnjjjn8s65AnA/C/zv6ok457q8+x7gYa+WI1NDEN59j1/1eW4BXnDOHYHK2n4lZttmFfF7aWa/CrwL+GXvQwlvyKTPW95Ccbx8Tblr815/tp9pRWw/ADOLAP8O+IeptkrahiezGEO/4i7J6I3/fRl4xTn3uZL20jHdXwR2vPF7y8HMaswsNbVMcYffDorb7U5vtTuBR/yor8QxvatK2X5vMNs22wjc4c3iuQoYKhkGKgszuxn4feDdzrl0SXuLmYW95XOB1cC+ctZWUstsP9ONwPvNLG5mqyjW+Fy56/O8A/ipc+7gVEMlbcOT8ntP8kLcKM6U+BnFT9s/rIB6rqb4b/52YKt3uxX4KvCS174RaPepvnMpzozYBuyc2mZAE/AUsBv4AdDo4zasAfqAupI2X7cfxQ+gbmCS4hjzh2bbZhRn7XzB+518CdjgQ217KI6LT/0O/o237r/3fu5bgReA23zcfrP+TIE/9LbfLuAWP+rz2r8C/Jc3rOvLNjyTm07DICISIItxeEdERGah0BcRCRCFvohIgCj0RUQCRKEvIhIgCn0RkQBR6IuIBMj/B2AcqvztN/PyAAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    }
  ]
}