0 - Introduction to Gym.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## introduction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.15.4\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import gym\n",
    "\n",
    "print(gym.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://gym.openai.com/envs/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 859 gym environments. Such as ['Copy-v0', 'RepeatCopy-v0', 'ReversedAddition-v0', 'ReversedAddition3-v0', 'DuplicatedInput-v0', 'Reverse-v0', 'CartPole-v0', 'CartPole-v1', 'MountainCar-v0', 'MountainCarContinuous-v0', 'Pendulum-v0', 'Acrobot-v1']\n"
     ]
    }
   ],
   "source": [
    "all_envs = gym.envs.registry.all()\n",
    "env_ids = [env.id for env in all_envs]\n",
    "\n",
    "print(f'There are {len(env_ids)} gym environments. Such as {env_ids[:12]}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://gym.openai.com/envs/CartPole-v1/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## discrete action space environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = gym.make('CartPole-v1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "observation space is: Box(4,)\n",
      "is observation space discrete? False\n",
      "is observation space continuous? True\n",
      "observation space shape: (4,)\n",
      "observation space high values? [4.8000002e+00 3.4028235e+38 4.1887903e-01 3.4028235e+38]\n",
      "observation space low values? [-4.8000002e+00 -3.4028235e+38 -4.1887903e-01 -3.4028235e+38]\n"
     ]
    }
   ],
   "source": [
    "print('observation space is:', env.observation_space)\n",
    "\n",
    "print('is observation space discrete?', isinstance(env.observation_space, gym.spaces.Discrete))\n",
    "print('is observation space continuous?', isinstance(env.observation_space, gym.spaces.Box))\n",
    "\n",
    "print('observation space shape:', env.observation_space.shape)\n",
    "\n",
    "print('observation space high values?', env.observation_space.high)\n",
    "print('observation space low values?', env.observation_space.low)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "action space is: Discrete(2)\n",
      "is action space discrete? True\n",
      "is action space continuous? False\n",
      "action space shape: 2\n"
     ]
    }
   ],
   "source": [
    "print('action space is:', env.action_space)\n",
    "\n",
    "print('is action space discrete?', isinstance(env.action_space, gym.spaces.Discrete))\n",
    "print('is action space continuous?', isinstance(env.action_space, gym.spaces.Box))\n",
    "\n",
    "print('action space shape:', env.action_space.n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "500\n",
      "475.0\n",
      "False\n"
     ]
    }
   ],
   "source": [
    "print(env.spec.max_episode_steps)\n",
    "print(env.spec.reward_threshold)\n",
    "print(env.spec.nondeterministic)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## continuous action space environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = gym.make('Pendulum-v0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "observation space is: Box(3,)\n",
      "is observation space discrete? False\n",
      "is observation space continuous? True\n",
      "observation space shape: (3,)\n",
      "observation space high values? [1. 1. 8.]\n",
      "observation space low values? [-1. -1. -8.]\n"
     ]
    }
   ],
   "source": [
    "print('observation space is:', env.observation_space)\n",
    "\n",
    "print('is observation space discrete?', isinstance(env.observation_space, gym.spaces.Discrete))\n",
    "print('is observation space continuous?', isinstance(env.observation_space, gym.spaces.Box))\n",
    "\n",
    "print('observation space shape:', env.observation_space.shape)\n",
    "\n",
    "print('observation space high values?', env.observation_space.high)\n",
    "print('observation space low values?', env.observation_space.low)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "action space is: Box(1,)\n",
      "is action space discrete? False\n",
      "is action space continuous? True\n",
      "action space shape: (1,)\n",
      "action space high values? [2.]\n",
      "action space low values? [-2.]\n"
     ]
    }
   ],
   "source": [
    "print('action space is:', env.action_space)\n",
    "\n",
    "print('is action space discrete?', isinstance(env.action_space, gym.spaces.Discrete))\n",
    "print('is action space continuous?', isinstance(env.action_space, gym.spaces.Box))\n",
    "\n",
    "print('action space shape:', env.action_space.shape)\n",
    "\n",
    "print('action space high values?', env.action_space.high)\n",
    "print('action space low values?', env.action_space.low)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "200\n",
      "None\n",
      "False\n"
     ]
    }
   ],
   "source": [
    "print(env.spec.max_episode_steps)\n",
    "print(env.spec.reward_threshold)\n",
    "print(env.spec.nondeterministic)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## atari environments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = gym.make('Freeway-v4')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "observation space is: Box(210, 160, 3)\n",
      "is observation space discrete? False\n",
      "is observation space continuous? True\n",
      "observation space shape: (210, 160, 3)\n"
     ]
    }
   ],
   "source": [
    "print('observation space is:', env.observation_space)\n",
    "print('is observation space discrete?', isinstance(env.observation_space, gym.spaces.Discrete))\n",
    "print('is observation space continuous?', isinstance(env.observation_space, gym.spaces.Box))\n",
    "print('observation space shape:', env.observation_space.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "action space is: Discrete(3)\n",
      "action space shape: 3\n",
      "is action space discrete? True\n",
      "is action space continuous? False\n",
      "action meanings: ['NOOP', 'UP', 'DOWN']\n"
     ]
    }
   ],
   "source": [
    "print('action space is:', env.action_space)\n",
    "print('action space shape:', env.action_space.n)\n",
    "print('is action space discrete?', isinstance(env.action_space, gym.spaces.Discrete))\n",
    "print('is action space continuous?', isinstance(env.action_space, gym.spaces.Box))\n",
    "print('action meanings:', env.unwrapped.get_action_meanings())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100000\n",
      "None\n",
      "False\n"
     ]
    }
   ],
   "source": [
    "print(env.spec.max_episode_steps)\n",
    "print(env.spec.reward_threshold)\n",
    "print(env.spec.nondeterministic)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Pong-v0 => 10k steps, randomly skips 2-4 frames, repeat action probability of 25%\n",
    "- Pong-v4 => 100k steps, randomly skips 2-4 frames\n",
    "- PongDeterministic-v0 => 100k steps, always skips 4 frames, repeat action probability of 25%\n",
    "- PongDeterministic-v4 => 100k steps, always skips 4 frames\n",
    "- PongNoFrameskip-v0 => 100k steps, returns every frame, repeat action probability of 25%\n",
    "- PongNoFrameskip-v4 => 100k steps, returns every frame\n",
    "\n",
    "information about environments: https://github.com/openai/gym/blob/master/gym/envs/__init__.py#L604\n",
    "spaceinvadersdeterministic always skips 3 frames instead of 4: https://github.com/openai/gym/blob/master/gym/envs/__init__.py#L620\n",
    "default frameskip when one not provided: https://github.com/openai/gym/blob/master/gym/envs/atari/atari_env.py#L30\n",
    "when skipping frames, you repeat the last action: https://github.com/openai/gym/blob/master/gym/envs/atari/atari_env.py#L94"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## wrappers\n",
    "\n",
    "not exclusive to atari, but most commonly used for atari\n",
    "\n",
    "commonly used atari wrappers: https://github.com/openai/baselines/blob/master/baselines/common/atari_wrappers.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ClipRewardEnv(gym.RewardWrapper):\n",
    "    def __init__(self, env):\n",
    "        gym.RewardWrapper.__init__(self, env)\n",
    "\n",
    "    def reward(self, reward):\n",
    "        \"\"\"Bin reward to {+1, 0, -1} by its sign.\"\"\"\n",
    "        return np.sign(reward)\n",
    "    \n",
    "class ScaledFloatFrame(gym.ObservationWrapper):\n",
    "    def __init__(self, env):\n",
    "        gym.ObservationWrapper.__init__(self, env)\n",
    "        self.observation_space = gym.spaces.Box(low=0, high=1, shape=env.observation_space.shape, dtype=np.float32)\n",
    "\n",
    "    def observation(self, observation):\n",
    "        return np.array(observation).astype(np.float32) / 255.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = gym.make('Pong-v0')\n",
    "\n",
    "env = ClipRewardEnv(env)\n",
    "\n",
    "env = ScaledFloatFrame(env)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## interacting with an environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "state type: <class 'numpy.ndarray'>\n",
      "state shape: (4,)\n",
      "state: [-0.01315549 -0.04012824  0.04801855 -0.0044493 ]\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('CartPole-v1')\n",
    "\n",
    "env.seed(1234)\n",
    "\n",
    "state = env.reset()\n",
    "\n",
    "print('state type:', type(state))\n",
    "print('state shape:', state.shape)\n",
    "print('state:', state)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://github.com/openai/gym/blob/master/gym/envs/classic_control/cartpole.py#L21\n",
    "\n",
    "cart pos, cart velocity, pole angle, pole velocity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "selected action: 1\n"
     ]
    }
   ],
   "source": [
    "action = env.action_space.sample() #select random action, uniformly between high and low for continuous\n",
    "\n",
    "print('selected action:', action)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://github.com/openai/gym/blob/master/gym/envs/classic_control/cartpole.py#L29\n",
    "\n",
    "0 = left, 1 = right"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "state: [-0.01395805  0.15427334  0.04792957 -0.28160352]\n",
      "reward: 1.0\n",
      "done: False\n",
      "info: {}\n"
     ]
    }
   ],
   "source": [
    "state, reward, done, info = env.step(action) #perform action on environment\n",
    "\n",
    "print('state:', state)\n",
    "print('reward:', reward)\n",
    "print('done:', done)\n",
    "print('info:', info)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## interacting with the atari environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "state type: <class 'numpy.ndarray'>\n",
      "state shape: (210, 160, 3)\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('FreewayNoFrameskip-v4')\n",
    "\n",
    "env.seed(1234)\n",
    "\n",
    "state = env.reset()\n",
    "\n",
    "print('state type:', type(state))\n",
    "print('state shape:', state.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAANEAAAD8CAYAAADpCEEHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAERFJREFUeJzt3X2sHNV5x/HvrxD8B6EBYuRcAzEGGYSDqLlFJHIBUcVJjIswVOjWVp2QgmpAQBy5DrWD1NBWEQkvjgxpXRlhYRdiuA2hsRA4YJQXVQ4E23GMsWMwxIhrLnYhLdCkIgGe/rGzdFjv+s7es29z7+8jje7umTNzZtb7eGbOznlGEYGZjd4fdHsDzMrOQWSWyEFklshBZJbIQWSWyEFklqhtQSRptqTdkvZIWtqudsy6Te34nUjSYcBzwGeAIeBpYH5E7Gx5Y2Zd1q4j0TnAnoh4MSJ+B9wPzG1TW2ZddXib1ns88HLu/RDwyUaVJfm2CetFr0XEcSNValcQjUjSQmBht9o3K+ClIpXaFUT7gBNz70/Iyt4XEauAVeAjkZVbu66JngamSZoq6QhgHrC+TW2ZdVVbjkQR8Y6k64AfAIcBqyPi2dGub/ny5S3bNrOiFi9eXKhe266JIuIR4JF2rb+XDbz0wVPpwSlTurQl3TVePgffsdBi1S/O4JQp739par9M48F4+hwcRGaJHERmiRxEZokcRGaJHERmiRxELZbvicr3UI034+lz6Nq9c2PZWP2yNGu8fA4+EpklasugvKY3wjegWm/aEhFnj1SpFKdzg4OD3d4EG4cGBgYK1fPpnFkiB5FZIgeRWSIHkVkiB5FZIgeRWaJRB5GkEyX9UNJOSc9KWpSV3yRpn6Rt2TSndZtr1ntG/WOrpD6gLyK2SjoK2AJcAgwA/xMRtzWxLv/Yar2ovT+2RsQwMJy9fkvSLipJG1vOiUqsG4omKmnJNZGkk4CzgKeyouskbZe0WtIxrWjDrFclB5GkDwMPAl+OiDeBlcApwAwqR6rbGyy3UNJmSZtTt8Gsm5LunZP0ISoBdF9EfA8gIvbn5t8FPFxvWWdAtV5V9DSuKqV3TsDdwK6IWJ4r78tVuxTYMdo2zLph3eTJrJs8uXD9lCPRnwCfB56RtC0r+yowX9IMIIC9wFUJbZh1VN9thTuV35fSO/cfgOrMGpdZT21sGV6yBDrZO2c2njmIzBI5iMwSOYjM6mimg8FBZJYzvGRJ08uUIlGJWSfNf+WVpur7SGRWY/ny5U3d9OwgMktUytO52bNnH1S2YcOGQ9Zp9fyx0oY/y8bzOzoUwmw8K0UaYQ/Ks25YvHhxoZGtPhKZJXIQmSUqZceCWTt1bFCe2Vj18PBXeHj4K4XrO4jMck6dP9z0Mg4iszqeW9c3cqVM8jWRpL3AW8C7wDsRcbakY4EHgJOoDBEfiIj/Sm3LrBe16kj0pxExI9envhR4IiKmAU9k783GpHadzs0F1mSv11BJL2w2JrUiiAJ4TNIWSQuzsklZmmGAV4FJLWjHrGOa6WBoRRCdGxH9wIXAtZLOz8+Myn1FB93W4wyo1oua6VCoSu5YiIh92d8Dkh4CzgH2S+qLiOEsmeOBOss5A6r1pIv6bm2qftKRSNKR2WNVkHQk8FkqGU/XA5dn1S4Hvp/SjlknNTsoL/VINAl4qJJRmMOB70TEBklPA4OSrgReovLMIrMxKSmIIuJF4I/qlL8OfDpl3YfigWTtm9+JNsryWRa9h87jicwa8Hgisw5xEJkl8ngisxoeT2SW6JprPsI113ykcH0HkVnO88//fdPLOIjM6pg27WuF6zqIzBI5iMwSOYjMEjmIzBI5iMzqaKaXzkFkltNMr1yV71gwq7Fy5RtN1feRyKyGn5Rn1mGlPJ3zQLL2ze9EG2X5LNs+KE/SaVSynFadDPwdcDTw18B/ZuVfjYhHRliXB+VZzyk6KG/UR6KI2A3MAJB0GLAPeAj4K+BbEXHbaNdtViatuib6NPBCRLzUovWZlUarronmAety76+T9AVgM/A3TmZvZdLxQXmSjgAuBv4tK1oJnELlVG8YuL3Bcs6Aaj1paNEihhYtKly/FadzFwJbI2I/QETsj4h3I+I94C4qGVEPEhGrIuLsIhduZp0y8FLzVyStCKL55E7lsrTBVZdSyYhqViqDU6YUrpt0TZSlDv4McFWu+BZJM6gksd9bM89szEnNgPob4KM1ZZ9P2iKzkvFtP2aJHERmdTTTweAgMstppkOhqpQ3oJq10wkrVjRV30cisxoeT2TWYaU8nfMYmPbN70QbZfks/ZAvs0R+yJdZhziIzBKV8prIrJ38kC+zDnMQmdX42OR1fGzyupErZhxEZjm33NY3cqUaDiKzOm5YMly4roPILJGDyCxRoSCStFrSAUk7cmXHSnpc0vPZ32Oyckm6Q9IeSdsl9bdr4816QdEj0T1A7U1WS4EnImIa8ET2HirZf6Zl00IqKbTMSqWZDoZCQRQRPwF+XVM8F1iTvV4DXJIrXxsVTwJH12QAMutZzXQoVKXcsTApIqotvgpMyl4fD7ycqzeUlX1g6yQtpHKkMuspr74yv6n6LbntJyJipDux6yyzClgFI9/FbdZJ1VEDRW//SQmi/ZL6ImI4O107kJXvA07M1TshK2uZL31p00Fld9wx85B1Wj1/rLThz7Lx/KK30BUeTyTpJODhiDgje38r8HpEfEPSUuDYiLhB0p8B1wFzgE8Cd0RE3VTCuXV7PJH1nJY+n0jSOuACYKKkIeBrwDeAQUlXAi8BA1n1R6gE0B7gt1SeV9RS/t+zffM70UZZPsuWH4nayUci60Ue2WrWIR6UZ1bDg/LMEkzccSdrr5jK2iumFl7GQWRWx2tnXF+4roPIrMZrZ1zPxB13Fq7vIDJL5CAyS+QgMkvkIDKrMXHHne5YMEvVTMeCf2w1y3ntjOv9Y6tZKj/ky6zDHERmiUp5TeSnu7VvfifaKMtn6SflmSXyeCKzDhkxiBpkP71V0i+zDKcPSTo6Kz9J0v9K2pZN/9LOjTfrBUWuie4Bvg2szZU9DiyLiHckfRNYBvxtNu+FiJjR0q0066CW/05UL/tpRDwWEe9kb5+kkhbLrPQ2bZ3NZQt2ctmCnYWXacU10RXAo7n3UyX9XNKPJZ3XaCFJCyVtlrS5Bdtg1lIz+w/u3WskKYgk3Qi8A9yXFQ0DH4+Is4DFwHck/WG9ZSNiVUScXaT3w6yTZvZvYNPWg7v+Gxl1EEn6InAR8JeR9ZNHxNsR8Xr2egvwAnDqaNswK4NRBZGk2cANwMUR8dtc+XGSDsten0zl8SovtmJDzXrViL1zDbKfLgMmAI9LAngyIq4Gzgf+QdLvgfeAqyOi9pEsZmPKiEEUEfWeM3F3g7oPAg+mbpRZN23aOpuZ/Rv47r3F6vuOBbM6mulYKOUNqGbtMrN/gwflmaXyoDyzDivl6ZzHwLRvfifaKMtn6fFEZok8nsisQxxEZolKeU1k1k7u4jZLMP3ee9nQ38+G/v7CyziIzOrYuWBB4boOIrMaOxcsYPq9BW+cw0FklsxBZJbIQWSWyEFklshBZFZj+r33trZ3rkEG1Jsk7ctlOp2Tm7dM0h5JuyV9ruk9MOsBre6duweoN8zvWxExI5seAZA0HZgHfCJb5p+riUvMymDnggXM3rqV2Vu3Fl5mVBlQD2EucH+WOutXwB7gnMJbY9YDOjko77osof1qScdkZccDL+fqDGVlB3EGVBsrRnsD6krgH4HI/t5OJZ1wYRGxClgFI48nquWBZO2b34k2yvJZtnRQnqSTgIcj4oxDzZO0DCAibs7m/QC4KSJ+OsL6PSjPek5bB+VJ6su9vRSo9tytB+ZJmiBpKpUMqD8bTRtmZTHaDKgXSJpB5XRuL3AVQEQ8K2kQ2Ekl0f21EfFuezbdrDe0NANqVv/rwNdTNsqsmzwozyzB26dv4uZHL+PmRy8rvIyDyKyOCbtmFq7rIDKrMWHXTN4+fVPh+g4is0QOIrNEDiKzRA4isxpvn77JHQtmqZrpWHAGVLOcCbtm+sdWs1R+yJdZh5XydM5jYNo3vxNtlOWz9EO+zBL5IV9mHeIgMkvkIDJLVGRk62rgIuBANceCpAeA07IqRwP/HREzsnwLu4Dd2bwnI+LqVm+0WTs1+ztRkd65e4BvA2urBRHxF9XXkm4H3sjVfyEiZjS1FWY9Ytas9WzffgEAZ575o0LLJCVvlCRgAFhXcBvNSmHjxosL1029JjoP2B8Rz+fKpkr6uaQfSzovcf1mHbdx48XMmrW+cP3UH1vn88Gj0DDw8Yh4XdIfA/8u6RMR8WbtgpIWAgsT2zfrulEfiSQdDvw58EC1LMvB/Xr2egvwAnBqveUjYlVEnF3kxyyzXpZyOjcL+GVEDFULJB1XfQqEpJOpJG98MW0TzXpbkecTrQN+CpwmaUjSldmseRzcoXA+sF3SNuC7wNURUfSJEmY9Ydas9U11LIw2eSMR8cU6ZQ8CDxZu3axHNdOx4DsWzHI2bryYM8/8UeHfiMBBZHYQD8oz6zAHkVkiB5FZolKMbDXrkkIjW0uRY2FwcLDbm2Dj0MDAQKF6Pp0zS+QgMkvkIDJL5CAyS+QgMkvkIDJL5CAyS+QgMkvkOxbMGhs7dyy0ytCiRYecf8KKFUn1bXwqMjz8REk/lLRT0rOSFmXlx0p6XNLz2d9jsnJJukPSHknbJfW3eyfMuioiDjkBfUB/9voo4DlgOnALsDQrXwp8M3s9B3gUEPAp4KkCbUS7pqFFiwqV13vfaFlP42baPNJ3NyIKZUAdjoit2eu3qOTaPh6YC6zJqq0BLslezwXWRsWTwNGS+kZqpx2qp2P5v7WnaPVO2apl1dO16nL55WvXWVveaN029jTVO5clrD8LeAqYFBHD2axXgUnZ6+OBl3OLDWVlXVF73TLSdUw+gPJBcMKKFe9PI7Xh4BlfCncsSPowlUw+X46INytpuCsiIprtYetUBtRmvtC1R6BG6ynSoeBAGj8KHYkkfYhKAN0XEd/LivdXT9Oyvwey8n3AibnFT8jKPqBTGVCb7UFrdKRpdBRqRZtWbkV65wTcDeyKiHwKlPXA5dnry4Hv58q/kPXSfQp4I3fa13WHOkI0+vLXXvcUWacDaRwp0HN2LpWeiu3AtmyaA3wUeAJ4HtgIHJvVF/BPVPJwPwOc3c3eOU+eEqZCvXO+Y8GsMT893KwTHERmiRxEZokcRGaJHERmiXplKMRrwG+yv2PFRMbO/oylfYHi+zOlyMp6oosbQNLmsfT81rG0P2NpX6D1++PTObNEDiKzRL0URKu6vQEtNpb2ZyztC7R4f3rmmsisrHrpSGRWSl0PIkmzJe3OEpss7fb2jIakvZKekbRN0uasrG4il14kabWkA5J25MpKm4imwf7cJGlf9m+0TdKc3Lxl2f7slvS5phsscqt3uybgMCpDJk4GjgB+AUzv5jaNcj/2AhNryuomcunFCTgf6Ad2jLT9jCIRTY/sz03Akjp1p2ffuwnA1Oz7eFgz7XX7SHQOsCciXoyI3wH3U0l0MhY0SuTScyLiJ8Cva4p7PhFNIw32p5G5wP0R8XZE/ArYQ+V7WVi3g6inkpokCOAxSVuy3BHQOJFLWZQiEU2TrstOQVfnTq+T96fbQTRWnBsR/cCFwLWSzs/PjMp5Q2m7Qcu+/ZmVwCnADGAYuL1VK+52EBVKatLrImJf9vcA8BCV04FGiVzKIikRTa+JiP0R8W5EvAfcxf+fsiXvT7eD6GlgmqSpko4A5lFJdFIako6UdFT1NfBZYAeNE7mURSkT0TRSc912KZV/I6jszzxJEyRNBaYBP2tq5T3QkzKHSmriF4Abu709o9j+k6n07vwCeLa6DzRI5NKLE7COyinO76lcE1zZaPsZRSKaHtmff822d3sWOH25+jdm+7MbuLDZ9nzHglmibp/OmZWeg8gskYPILJGDyCyRg8gskYPILJGDyCyRg8gs0f8B/uzQ/ucwgfkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(state);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "selected action: 2\n",
      "action meaning: DOWN\n"
     ]
    }
   ],
   "source": [
    "action = env.action_space.sample() #select random action, uniformly between high and low for continuous\n",
    "\n",
    "print('selected action:', action)\n",
    "print('action meaning:', env.unwrapped.get_action_meanings()[action])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "reward: 0.0\n",
      "done: False\n",
      "info: {'ale.lives': 0}\n"
     ]
    }
   ],
   "source": [
    "state, reward, done, info = env.step(action) #perform action on environment\n",
    "\n",
    "print('reward:', reward)\n",
    "print('done:', done)\n",
    "print('info:', info)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAANEAAAD8CAYAAADpCEEHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAEVRJREFUeJzt3X/sVfV9x/Hna1j5w7qqxdCvP4po0EiNw++MbZgal9IWmRE1hkFGa6cZatTSMOuwJqvb0tj6g0bblQUjUYZFWdWVGKWK6Y8sVCtQighF0WIEv8K0m7p2sVXf++Oe646Xe/me+/3cX+fe1yO5+d77OZ/zk/vmfM7nns/7KCIws7H7o25vgFnZOYjMEjmIzBI5iMwSOYjMEjmIzBK1LYgkzZS0Q9JOSYvbtR6zblM7fieSNA54DvgMsBt4GpgXEdtavjKzLmvXmegMYGdEvBgRvwfuA2a3aV1mXXVQm5Z7NPBy7vNu4JONKkvybRPWi16LiCNHq9SuIBqVpAXAgm6t36yAl4pUalcQ7QGOzX0+Jit7X0QsA5aBz0RWbu26JnoamCJpsqSDgbnAmjaty6yr2nImioh3JF0N/BAYByyPiGfHurwlS5a0bNvMilq0aFGhem27JoqIR4BH2rX8XjbnpQ82pVdPmtSlLemuQTkOvmOhxapfnNWTJr3/pan9Mg2CQToODiKzRA4is0QOIrNEDiKzRA4is0QOohbL90Tle6gGzSAdh67dO9fP+vXL0qxBOQ4+E5klasugvKY3wjegWm/aGBGnj1apFM251atXd3sTbADNmTOnUD0358wSOYjMEjmIzBI5iMwSOYjMEjmIzBKNOYgkHSvpR5K2SXpW0sKs/EZJeyRtzl6zWre5Zr1nzD+2ShoChiJik6RDgY3ABcAc4H8i4tYmluUfW60XtffH1ogYAUay929J2k4laWPLOVGJdUPRRCUtuSaSdBxwGvBUVnS1pC2Slks6vBXrMOtVyUEk6cPAA8CXI+JNYClwAjCNypnqtgbzLZC0QdKG1G0w66ake+ckfYhKAN0bEQ8CRMTe3PQ7gYfrzesMqNarijbjqlJ65wTcBWyPiCW58qFctQuBrWNdh1k3rDrqKFYddVTh+ilnoj8DPg88I2lzVvZVYJ6kaUAAu4DLE9Zh1lFDtxbuVH5fSu/cfwCqM2kgs55afxm59lroZO+c2SBzEJklchCZJXIQmdXRTAeDg8gsZ+Taa5uepxSJSsw6ad4rrzRV32cisxpLlixp6qZnB5FZolI252bOnLlf2dq1aw9Yp9XT+2UdPpaNp3d0KITZICtFGmEPyrNuWLRoUaGRrT4TmSVyEJklKmXHglk7dWxQnlm/enjkKzw88pXC9R1EZjknzhtpeh4HkVkdz60aGr1SJvmaSNIu4C3gXeCdiDhd0hHA/cBxVIaIz4mI/0pdl1kvatWZ6M8jYlquT30x8ERETAGeyD6b9aV2NedmA/dk7++hkl7YrC+1IogCeEzSRkkLsrKJWZphgFeBiS1Yj1nHNNPB0IogOjMihoFzgasknZ2fGJX7iva7rccZUK0XNdOhUJXcsRARe7K/+yQ9BJwB7JU0FBEjWTLHfXXmcwZU60nnDd3SVP2kM5GkQ7LHqiDpEOCzVDKergEuyapdAvwgZT1mndTsoLzUM9FE4KFKRmEOAr4XEWslPQ2slnQZ8BKVZxaZ9aWkIIqIF4E/qVP+OvDplGUfiAeStW96J9ZRlmNZ9B46jycya8Djicw6xEFklsjjicxqeDyRWaIrr/wIV175kcL1HURmOc8//w9Nz+MgMqtjypSvFa7rIDJL5CAyS+QgMkvkIDJL5CAyq6OZXjoHkVlOM71yVb5jwazG0qVvNFXfZyKzGn5SnlmHlbI554Fk7ZveiXWU5Vi2fVCepJOoZDmtOh74e+Aw4G+A/8zKvxoRj4yyLA/Ks55TdFDemM9EEbEDmAYgaRywB3gI+GvgWxFx61iXbVYmrbom+jTwQkS81KLlmZVGq66J5gKrcp+vlvQFYAPwt05mb2XS8UF5kg4Gzgf+LStaCpxApak3AtzWYD5nQLWetHvhQnYvXFi4fiuac+cCmyJiL0BE7I2IdyPiPeBOKhlR9xMRyyLi9CIXbmadMuel5q9IWhFE88g15bK0wVUXUsmIalYqqydNKlw36ZooSx38GeDyXPHNkqZRSWK/q2aaWd9JzYD6W+CjNWWfT9ois5LxbT9miRxEZnU008HgIDLLaaZDoaqUN6CatdMxt9/eVH2ficxqeDyRWYeVsjnnMTDtm96JdZTlWPohX2aJ/JAvsw5xEJklKuU1kVk7+SFfZh3mIDKr8bGjVvGxo1aNXjHjIDLLufnWodEr1XAQmdVx3bUjhes6iMwSOYjMEhUKIknLJe2TtDVXdoSkxyU9n/09PCuXpDsk7ZS0RdJwuzberBcUPRPdDdTeZLUYeCIipgBPZJ+hkv1nSvZaQCWFllmpNNPBUCiIIuKnwG9qimcD92Tv7wEuyJWviIongcNqMgCZ9axmOhSqUu5YmBgR1TW+CkzM3h8NvJyrtzsr+8DWSVpA5Uxl1lNefWVeU/VbcttPRMRod2LXmWcZsAxGv4vbrJOqowaK3v6TEkR7JQ1FxEjWXNuXle8Bjs3VOyYra5kvfWn9fmV33DH9gHVaPb1f1uFj2Xh60VvoCo8nknQc8HBEnJJ9vgV4PSK+IWkxcEREXCfpL4CrgVnAJ4E7IqJuKuHcsj2eyHpOS59PJGkVcA4wQdJu4GvAN4DVki4DXgLmZNUfoRJAO4HfUXleUUv5f8/2Te/EOspyLFt+Jmonn4msF3lkq1mHeFCeWQ0PyjNLMGHrt1lx6WRWXDq58DwOIrM6XjvlmsJ1HURmNV475RombP124foOIrNEDiKzRA4is0QOIrMaE7Z+2x0LZqma6Vjwj61mOa+dco1/bDVL5Yd8mXWYg8gsUSmvifx0t/ZN78Q6ynIs/aQ8s0QeT2TWIaMGUYPsp7dI+lWW4fQhSYdl5cdJ+l9Jm7PXv7Rz4816QZFroruB7wArcmWPA9dHxDuSvglcD/xdNu2FiJjW0q0066CW/05UL/tpRDwWEe9kH5+kkhbLrPTWb5rJxfO3cfH8bYXnacU10aXAo7nPkyX9QtJPJJ3VaCZJCyRtkLShBdtg1lLTh/fv3WskKYgk3QC8A9ybFY0AH4+I04BFwPck/XG9eSNiWUScXqT3w6yTpg+vZf2m/bv+GxlzEEn6InAe8FeR9ZNHxNsR8Xr2fiPwAnDiWNdhVgZjCiJJM4HrgPMj4ne58iMljcveH0/l8SovtmJDzXrVqL1zDbKfXg+MBx6XBPBkRFwBnA38o6Q/AO8BV0RE7SNZzPrKqEEUEfWeM3FXg7oPAA+kbpRZN63fNJPpw2v5/spi9X3HglkdzXQslPIGVLN2mT681oPyzFJ5UJ5Zh5WyOecxMO2b3ol1lOVYejyRWSKPJzLrEAeRWaJSXhOZtZO7uM0STF25krXDw6wdHi48j4PIrI5t8+cXrusgMquxbf58pq4seOMcDiKzZA4is0QOIrNEDiKzRA4isxpTV65sbe9cgwyoN0rak8t0Ois37XpJOyXtkPS5pvfArAe0unfubqDeML9vRcS07PUIgKSpwFzgE9k8360mLjErg23z5zNz0yZmbtpUeJ4xZUA9gNnAfVnqrF8DO4EzCm+NWQ/o5KC8q7OE9sslHZ6VHQ28nKuzOyvbjzOgWr8Y6w2oS4F/AiL7exuVdMKFRcQyYBmMPp6olgeStW96J9ZRlmPZ0kF5ko4DHo6IUw40TdL1ABFxUzbth8CNEfGzUZbvQXnWc9o6KE/SUO7jhUC1524NMFfSeEmTqWRA/flY1mFWFmPNgHqOpGlUmnO7gMsBIuJZSauBbVQS3V8VEe+2Z9PNekNLM6Bm9b8OfD1lo8y6yYPyzBK8ffJ6bnr0Ym569OLC8ziIzOoYv3164boOIrMa47dP5+2T1xeu7yAyS+QgMkvkIDJL5CAyq/H2yevdsWCWqpmOBWdANcsZv326f2w1S+WHfJl1WCmbcx4D077pnVhHWY6lH/JllsgP+TLrEAeRWSIHkVmiIiNblwPnAfuqORYk3Q+clFU5DPjviJiW5VvYDuzIpj0ZEVe0eqPN2qnZ34mK9M7dDXwHWFEtiIi/rL6XdBvwRq7+CxExramtMOsRM2asYcuWcwA49dQfF5onKXmjJAFzgFUFt9GsFNatO79w3dRrorOAvRHxfK5ssqRfSPqJpLMSl2/WcevWnc+MGWsK10/9sXUeHzwLjQAfj4jXJf0p8O+SPhERb9bOKGkBsCBx/WZdN+YzkaSDgIuA+6tlWQ7u17P3G4EXgBPrzR8RyyLi9CI/Zpn1spTm3AzgVxGxu1og6cjqUyAkHU8leeOLaZto1tuKPJ9oFfAz4CRJuyVdlk2ay/4dCmcDWyRtBr4PXBERRZ8oYdYTZsxY01THwliTNxIRX6xT9gDwQOG1m/WoZjoWfMeCWc66dedz6qk/LvwbETiIzPbjQXlmHeYgMkvkIDJLVIqRrWZdUmhkaylyLKxevbrbm2ADaM6cOYXquTnXQhdd5GCHwTsODqIWG7QvUCODdBwcRC1S/dI8+GCxJkC/GsTj4CBKUP3CDNL/uvUM+nFwECUa1C9OrUE+Dg6iBIPUZDmQQT8ODqJE+S/QIH+ZBvk4OIhaoPqlGeQmDQzucfAdC2aN9c8dC62ye+HCA04/5vbbk+rbYCoyPPxYST+StE3Ss5IWZuVHSHpc0vPZ38Ozckm6Q9JOSVskDbd7J8y6KiIO+AKGgOHs/aHAc8BU4GZgcVa+GPhm9n4W8Cgg4FPAUwXWEe167V64sFB5vc+N5vVrYF4bRvvuRkShDKgjEbEpe/8WlVzbRwOzgXuyavcAF2TvZwMrouJJ4DBJQ6Otpx2qzbH839omWr0mW7Ws2lyrzpefv3aZteWNlm39p6neuSxh/WnAU8DEiBjJJr0KTMzeHw28nJttd1bWFbXXLaNdx+QDKB8Ex9x++/uv0dbh4BkshTsWJH2YSiafL0fEm5U03BUREc32sHUqA2ozX+jaM1Cj5RTpUHAgDY5CZyJJH6ISQPdGxINZ8d5qMy37uy8r3wMcm5v9mKzsAzqVAbXZHrRGZ5pGZ6FWrNPKrUjvnIC7gO0RkU+Bsga4JHt/CfCDXPkXsl66TwFv5Jp9XXegM0SjL3/tdU+RZTqQBkiBnrMzqfRUbAE2Z69ZwEeBJ4DngXXAEVl9Af9MJQ/3M8Dp3eyd88uvhFeh3jnfsWDWmJ8ebtYJDiKzRA4is0QOIrNEDiKzRL0yFOI14LfZ334xgf7Zn37aFyi+P5OKLKwnurgBJG3op+e39tP+9NO+QOv3x805s0QOIrNEvRREy7q9AS3WT/vTT/sCLd6fnrkmMiurXjoTmZVS14NI0kxJO7LEJou7vT1jIWmXpGckbZa0ISurm8ilF0laLmmfpK25stImommwPzdK2pP9G22WNCs37fpsf3ZI+lzTKyxyq3e7XsA4KkMmjgcOBn4JTO3mNo1xP3YBE2rK6iZy6cUXcDYwDGwdbfsZQyKaHtmfG4Fr69Sdmn3vxgOTs+/juGbW1+0z0RnAzoh4MSJ+D9xHJdFJP2iUyKXnRMRPgd/UFPd8IppGGuxPI7OB+yLi7Yj4NbCTyveysG4HUU8lNUkQwGOSNma5I6BxIpeyKEUimiZdnTVBl+ea18n70+0g6hdnRsQwcC5wlaSz8xOj0m4obTdo2bc/sxQ4AZgGjAC3tWrB3Q6iQklNel1E7Mn+7gMeotIcaJTIpSySEtH0mojYGxHvRsR7wJ38f5MteX+6HURPA1MkTZZ0MDCXSqKT0pB0iKRDq++BzwJbaZzIpSxKmYimkZrrtgup/BtBZX/mShovaTIwBfh5UwvvgZ6UWVRSE78A3NDt7RnD9h9PpXfnl8Cz1X2gQSKXXnwBq6g0cf5A5ZrgskbbzxgS0fTI/vxrtr1bssAZytW/IdufHcC5za7PdyyYJep2c86s9BxEZokcRGaJHERmiRxEZokcRGaJHERmiRxEZon+Dxo19DYHRdd9AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(state);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAANEAAAD8CAYAAADpCEEHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAEu1JREFUeJzt3X+sXGWdx/H3RxT+ULOAuN0rYCmmGkvj1krUdJW4EbV2DRVi7tJs/bGarRj5YZrKFt2sqDG6WDBUd9mtoRG2bPFGQBuCVUrcZTddlFKxltbS8qOh5dJucVdc3VSB7/4xZy7nzp25d+4858zMmfm8ksmd85wzc54znW+f5zxznu9RRGBmnXtRrytgVnUOIrNEDiKzRA4is0QOIrNEDiKzRKUFkaSlkvZJOiBpbVn7Mes1lfE7kaQTgIeBdwGHgPuBFRGxp/CdmfVYWS3Rm4EDEfFoRPwOuBVYXtK+zHrqxSW97+nAE7nlQ8BbWm0syZdNWD86FhGvnGmjsoJoRpJWAat6tX+zNhxsZ6OygugwcGZu+YysbEJEbAA2gFsiq7ayzonuB+ZLmifpROBiYEtJ+zLrqVJaooh4VtKlwA+AE4CNEfFQp+933XXXFVY3s3atXr26re1KOyeKiLuAu8p6/342enByV3ps7twe1aS3huVz8BULBat/ccbmzp340jR+mYbBMH0ODiKzRA4is0QOIrNEDiKzRA4is0QOooLlR6LyI1TDZpg+h55dOzfIBvXLMlvD8jm4JTJLVMqkvFlXwhegWn96ICLOnWmjSnTnxsbGel0FG0Kjo6NtbefunFkiB5FZIgeRWSIHkVkiB5FZIgeRWaKOg0jSmZJ+JGmPpIckXZGVXy3psKQHs8ey4qpr1n86/rFV0ggwEhE7Jb0ceAB4PzAK/G9ErJvFe/nHVutH5f7YGhHjwHj2/NeS9lJL2lg4JyqxXmg3UUkh50SSzgLeCPw4K7pU0i5JGyWdUsQ+zPpVchBJehlwG/CpiHgGuAF4DbCIWkt1bYvXrZK0Q9KO1DqY9VJSEEl6CbUAuiUibgeIiCMR8VxEPA98k1py+ykiYkNEnNtOn9OsCKtXr57URWtc7lTH50SSBNwI7I2I63LlI9n5EsCFwO60KpoVY/OrXlV7sm7dlOXxNWs6ft+Uq7j/BPgg8HNJD2ZlnwFWSFoEBPA48PGEfZgVYmTd5MHietA0lnciZXTuPwA1WTWUWU+tGhqDZ3zNmuRA8hULZokcRGaJHERmiSoxPdysKPlzofxyCrdENhQah7BH1q0rJIDALZENkRVPPgm8cC1m/YfW1GszHUQ2NBqDpagLm92dM0tUyZZo6dKlU8q2bt067TZFrx+UffizbL2+q1MhzIZZJdIIe1Ke9cLq1avbmtnqlsgskYPILFElBxbM2pEfGCj6t6E8B5ENrDvHP51bqs0TvXPFpyeWH948Ush+3J2zgfTaFeNTyh7ePFJY4OS5JbKBlg+aemA9vHmkaZB1KjmIJD0O/Bp4Dng2Is6VdCrwbeAsalPERyPiv1P3ZdaPiurO/WlELMqNqa8F7omI+cA92bLZQCrrnGg5cFP2/CZq6YXNBlIR50QB/DC76uCfImIDMCeXNuspYE4B+zGbtfy5T/38qMjzISgmiN4WEYcl/SFwt6Rf5FdGRDS7rEfSKmBVAfs3m6LZ4EHRwVOXHEQRcTj7e1TSHdQynh6pJ3HM7h5xtMnrNgAbwHeFsHK8b+SrE8/79sdWSS8FXpTdFeKlwLuBLwBbgA8DX8n+fi+1omaz1SxQyriYObUlmgPcUcsozIuBf4mIrZLuB8YkfQw4SO2eRWYDKSmIIuJR4I+blD8NvDPlvafjiWTlre/GPqryWbY7Kc/zicxa8Hwisy5xEJkl8gWoNjAah6+bzScqg4PIBsYnPvEHAOzf//lJy/Wy+fM/V8p+3Z2zgVAPnLr58z9XWtA0cktkA6UeOPWgalwug1sis0QOIrNEDiKzRA4is0QeWLCB0s0BhTq3RDYQGoez9+//fFcCCNwS2QC54YZfAb5iwaxjZd0JbybuzpklqmRL5Ilk5a3vxj6q8lmWPilP0uuoZTmtOxv4W+Bk4K+A/8rKPxMRd83wXp6UZ32n3Ul5HbdEEbEPWAQg6QTgMHAH8JfA1yJiXafvbVYlRZ0TvRN4JCIOFvR+ZpVRVBBdDGzOLV8qaZekjZJOKWgfNuRWr1496TylcblXkhOVSDoReBI4JyKOSJoDHKOWXviLwEhEfLTJ6/IZUN803T58TmQAowdbd3TG5s4tfH/dTFTyXmBnRBwBiIgjEfFcRDwPfJNaRtQpImJDRJzbTiXNGgNobO7cUgKnE0UMca8g15Wrpw/OFi8EdhewDzPghRanHlRjc+dO20J1QxFphN8FfDxXfI2kRdS6c483rDMbOKkZUH8DvKKh7INJNTKrGF/2Y5aokpf92PDKnwvll3vJLZFVQuNI3OjBg30RQOCWyCrkjOuvB8q9YVcnHERWGb2aLzQTd+fMElWyJfIcmPLWd2MfVfksfZMvs0S+yZdZlziIzBI5iCxZv87z6RYHkVmiSo7OWX/5o1fVZsJcs27q8pVrxlu9bGC4JbIk16wbmbR85ZrxoQicPLdEVoh64NSD6so141MCbFC5JTJL5CAyS9RWEGWpr45K2p0rO1XS3ZL2Z39Pycolab2kA1narMVlVd6sH7R7TvQt4BvAzbmytcA9EfEVSWuz5b+mlv1nfvZ4C3BD9tcGWP5cKL88DNpqiSLiXuCXDcXLgZuy5zcB78+V3xw19wEnSxqeT3TINI7EXbNuZKgCCNJG5+bkUmM9BczJnp8OPJHb7lBWNunTbkjeaBX21JMrgP6bLNcthQxxR0TMdCV2k9dsADbAzFdxW3/r18ly3ZISREfqiRqz7trRrPwwcGZuuzOyssJcfvn2KWXr1y+Zdpui1w/KPvxZtl7f7uV/bc8nknQWcGdELMyWvwo8nRtYODUirpT0Z8ClwDJqAwrrI6JpKuHce3s+kfWdQu9PJGkz8A7gNEmHgM8BXwHGJH0MOAiMZpvfRS2ADgC/pXa/okL5f8/y1ndjH1X5LAtvicrklsj6kWe2mnWJg6hAwz45bVj5Ku6CnLb769z80Xm1hd1fB5hYPtarSllXuCUq2LGFl3Fs4WWTyk7LgsoGk1uiAh1beNlEwNQDyQE0+NwSmSVyEJklchCZJfI5UYFO2/11nwsNIQdRwZoFT+NonQ0Wd+cKcmzhZXxo42N8aONjE8Pc9WUbbG6JCjTs82qGlVsis0QOIrNElezO+e5u5a3vxj6q8ln6TnlmiTyfyKxLZgyiFtlPvyrpF1mG0zsknZyVnyXp/yQ9mD3+sczKm/WDdlqibwGNHee7gYUR8QbgYeCq3LpHImJR9rikmGqm8WQ5K9OMAwsRcW+W6Sdf9sPc4n3AB4qtVnG271zKB1YuzZ7XyurLMPXk02y2ijgn+ijw/dzyPEk/lfRvkt7e6kWSVknaIWlHAXWY0ZLFW1myeHLQbN85dWTKbLaShrglfRZ4FrglKxoHXh0RT0t6E/BdSedExDONr+1mBtQli7dOBEw9kBxAVpSOWyJJHwHeB/xFZOPkEXE8Ip7Onj8APAK8toB6mvWtjoJI0lLgSuCCiPhtrvyVkk7Inp9N7fYqjxZRUbN+NWN3rkX206uAk4C7JQHcl43EnQd8QdLvgeeBSyKi8ZYsZgOlndG5FU2Kb2yx7W3AbamVKtr2nUt9LmSlqeS1c51oFjyNo3VmnRj4y36WLN7KdzYt4DubFkwMc9eXzYowFC2RJ8tZmQa+JTIrWyVbIs+BKW99N/ZRlc/S84nMEnk+UY80u/OcDTYHUQkcSMNlIIKoX+YL1YOn2f1CbXBVcmAhb8GmTWxdvLi2sGkTwMTynpL3ffnl21m/fsmklscBNHwGoiUC2LNyJXtWrpxUtiALqjK562YDEUR7Vq5kwaZNLNi0qWkwlcWtjsEAdOf6hQNqeA1ES9RL9eBxt254OYjMErk7VwB35YbbQARRfUCh/tysmzrNgHq1pMO5TKfLcuuuknRA0j5J7ymr4o3qo3N53Rqls+HWaQZUgK/lMp3eBSBpAXAxcE72mn+oJy4py56VK1m6cydLd+6cGN6uL5t1Q0cZUKexHLg1Io4Dj0k6ALwZ+M+Oa9gGT7qzXkoZnbs0S2i/UdIpWdnpwBO5bQ5lZVN0OwOqWVk6HVi4AfgiENnfa6mlE25bSgZUTyQrb3039lGVz7LQSXlZd+7OiFg43TpJVwFExJezdT8Aro6IabtznpRn/ajUSXmSRnKLFwL1kbstwMWSTpI0j1oG1J90sg+zqug0A+o7JC2i1p17HPg4QEQ8JGmM2iyEZ4FPRsRz5VTdrD8UmgE12/5LwJdSKjWdej8138VrVmbWLZW6YuH467fz5e/X7id2nBcu+KyXsbcXtbJhV8kLUE/au2TikXf89b6S2rqvUi0R1AIoHyz1QHIAWa9UsiUy6ycOIrNEDiKzRJU7Jzr++u2TBhR8LmS9VrkggtaB0zhaZ9YNlQqik/YumeHHVgeRdV+lggiaX5XgKxWslzywYJaoci0ReA5Mmeu7sY+qfJa+yZdZIt/ky6xLHERmiRxEZonamdm6EXgfcLSeY0HSt4HXZZucDPxPRCzK8i3sBfZl6+6LiEtSK9n421D+hM/nS9Zr7YzOfQv4BnBzvSAi/rz+XNK1wK9y2z8SEYuKqiDArl3vyJ5taViGbduK3JPZ7M3YnYuIe4FfNlsnScAosLngek2xbdsFbNt2wZTy88/fUvauzaaV+jvR24EjEbE/VzZP0k+BZ4C/iYh/T9wH27ZdMBEs9UBy8Fi/SA2iFUxuhcaBV0fE05LeBHxX0jkR8UzjCyWtAlYl7t+s5zoenZP0YuAi4Nv1sog4HhFPZ88fAB4BXtvs9RGxISLObefHLLN+ljLEfT7wi4g4VC+Q9Mr6XSAknU0teeOjaVU0628dJW+MiBup3UKlcUDhPOALkn4PPA9cEhFNByVm4/zzt/hcyPpWp8kbiYiPNCm7DbgtvVpTtQqeZiN2Zt1Uiau43/CGfwVa/djqILLeqkQQ+SZe1s987ZxZIgeRWSIHkVmiSsxsNeuRtma2VmJgYWxsrNdVsCE0Ojra1nbuzlnhLrpouP7TcxBZKYYpkBxEVqh68Nx+e3tdoUHgILJk9cAZptYnz0FkhRjWAAIHkRVgmLpuzTiIrBD5QBq2oHIQWWHqwTNsXTtfsWDW2uBcsVCUQ1dcMe36M66/Pml7G04zducknSnpR5L2SHpI0hVZ+amS7pa0P/t7SlYuSeslHZC0S9Lisg/CrKciYtoHMAIszp6/HHgYWABcA6zNytcCf5c9XwZ8HxDwVuDHbewjynocuuKKtsqbLbd6rR9D89gx03c3ItrKgDoeETuz57+mlmv7dGA5cFO22U3A+7Pny4Gbo+Y+4GRJIzPtpwz17lj+b2MXrVmXrV5W767VX5d/feN7Npa3em8bPLMancsS1r8R+DEwJyLGs1VPAXOy56cDT+Redigr64nG85aZzmPyAZQPgjOuv37iMdM+HDzDpe2BBUkvo5bJ51MR8UwtDXdNRMRsR9i6lQF1Nl/oxhao1fu0M6DgQBoebbVEkl5CLYBuiYjbs+Ij9W5a9vdoVn4YODP38jOyskm6lQF1tiNorVqaVq1QEfu0amtndE7AjcDeiMin2dkCfDh7/mHge7nyD2WjdG8FfpXr9vXcdC1Eqy9/43lPO+/pQBoibYycvY3aSMUu4MHssQx4BXAPsB/YBpyabS/g76nl4f45cG4vR+f88CPh0dbonK9YMGvNdw836wYHkVkiB5FZIgeRWSIHkVmifpkKcQz4TfZ3UJzG4BzPIB0LtH88c9t5s74Y4gaQtGOQ7t86SMczSMcCxR+Pu3NmiRxEZon6KYg29LoCBRuk4xmkY4GCj6dvzonMqqqfWiKzSup5EElaKmlflthkba/r0wlJj0v6uaQHJe3IypomculHkjZKOippd66ssoloWhzP1ZIOZ/9GD0pallt3VXY8+yS9Z9Y7bOdS77IewAnUpkycDZwI/AxY0Ms6dXgcjwOnNZQ1TeTSjw/gPGAxsHum+tNBIpo+OZ6rgTVNtl2Qfe9OAuZl38cTZrO/XrdEbwYORMSjEfE74FZqiU4GQatELn0nIu4FftlQ3PeJaFppcTytLAdujYjjEfEYcIDa97JtvQ6ivkpqkiCAH0p6IMsdAa0TuVRFJRLRzNKlWRd0Y657nXw8vQ6iQfG2iFgMvBf4pKTz8iuj1m+o7DBo1eufuQF4DbAIGAeuLeqNex1EbSU16XcRcTj7exS4g1p3oFUil6pISkTTbyLiSEQ8FxHPA9/khS5b8vH0OojuB+ZLmifpROBiaolOKkPSSyW9vP4ceDewm9aJXKqikoloWmk4b7uQ2r8R1I7nYkknSZoHzAd+Mqs374ORlGXUUhM/Any21/XpoP5nUxvd+RnwUP0YaJHIpR8fwGZqXZzfUzsn+Fir+tNBIpo+OZ5/zuq7Kwuckdz2n82OZx/w3tnuz1csmCXqdXfOrPIcRGaJHERmiRxEZokcRGaJHERmiRxEZokcRGaJ/h/gIhFlO89QtQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "up_action = env.unwrapped.get_action_meanings().index('UP')\n",
    "\n",
    "for i in range(50):\n",
    "    state, reward, done, info = env.step(up_action) #presses up 10 times\n",
    "\n",
    "plt.imshow(state);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## reinforcement learning loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode: 1, reward: 240.0\n",
      "episode: 2, reward: 165.0\n",
      "episode: 3, reward: 260.0\n",
      "episode: 4, reward: 210.0\n",
      "episode: 5, reward: 105.0\n",
      "episode: 6, reward: 105.0\n",
      "episode: 7, reward: 155.0\n",
      "episode: 8, reward: 60.0\n",
      "episode: 9, reward: 80.0\n",
      "episode: 10, reward: 125.0\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('SpaceInvadersNoFrameskip-v4')\n",
    "\n",
    "env.seed(1234)\n",
    "\n",
    "n_episodes = 10\n",
    "\n",
    "for episode in range(n_episodes):\n",
    "    \n",
    "    episode_reward = 0\n",
    "    done = False\n",
    "    state = env.reset()\n",
    "    \n",
    "    while not done:\n",
    "        \n",
    "        action = env.action_space.sample()\n",
    "        \n",
    "        state, reward, done, _ = env.step(action)\n",
    "        \n",
    "        episode_reward += reward\n",
    "        \n",
    "    print(f'episode: {episode+1}, reward: {episode_reward}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}