2.actor-critic.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "import torch.autograd as autograd\n",
    "\n",
    "from common.multiprocessing_env import SubprocVecEnv\n",
    "from common.minipacman import MiniPacman"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import clear_output\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h4>USE CUDA</h4>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "USE_CUDA = torch.cuda.is_available()\n",
    "Variable = lambda *args, **kwargs: autograd.Variable(*args, **kwargs).cuda() if USE_CUDA else autograd.Variable(*args, **kwargs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>Actor Critic Algorithm</h1>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Abstract A2C class</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class OnPolicy(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(OnPolicy, self).__init__()\n",
    "        \n",
    "    def forward(self, x):\n",
    "        raise NotImplementedError\n",
    "        \n",
    "    def act(self, x, deterministic=False):\n",
    "        logit, value = self.forward(x)\n",
    "        probs = F.softmax(logit)\n",
    "        \n",
    "        if deterministic:\n",
    "            action = probs.max(1)[1]\n",
    "        else:\n",
    "            action = probs.multinomial()\n",
    "        \n",
    "        return action\n",
    "    \n",
    "    def evaluate_actions(self, x, action):\n",
    "        logit, value = self.forward(x)\n",
    "        \n",
    "        probs     = F.softmax(logit)\n",
    "        log_probs = F.log_softmax(logit)\n",
    "        \n",
    "        action_log_probs = log_probs.gather(1, action)\n",
    "        entropy = -(probs * log_probs).sum(1).mean()\n",
    "        \n",
    "        return logit, action_log_probs, value, entropy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Neural Network architecture for A2C</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ActorCritic(OnPolicy):\n",
    "    def __init__(self, in_shape, num_actions):\n",
    "        super(ActorCritic, self).__init__()\n",
    "        \n",
    "        self.in_shape = in_shape\n",
    "        \n",
    "        self.features = nn.Sequential(\n",
    "            nn.Conv2d(in_shape[0], 16, kernel_size=3, stride=1),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(16, 16, kernel_size=3, stride=2),\n",
    "            nn.ReLU(),\n",
    "        )\n",
    "        \n",
    "        self.fc = nn.Sequential(\n",
    "            nn.Linear(self.feature_size(), 256),\n",
    "            nn.ReLU(),\n",
    "        )\n",
    "        \n",
    "        self.critic  = nn.Linear(256, 1)\n",
    "        self.actor   = nn.Linear(256, num_actions)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = self.features(x)\n",
    "        x = x.view(x.size(0), -1)\n",
    "        x = self.fc(x)\n",
    "        logit = self.actor(x)\n",
    "        value = self.critic(x)\n",
    "        return logit, value\n",
    "    \n",
    "    def feature_size(self):\n",
    "        return self.features(autograd.Variable(torch.zeros(1, *self.in_shape))).view(1, -1).size(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Simple class to save expirience for A2C update</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# @ikostrikov style\n",
    "\n",
    "class RolloutStorage(object):\n",
    "    def __init__(self, num_steps, num_envs, state_shape):\n",
    "        self.num_steps = num_steps\n",
    "        self.num_envs  = num_envs\n",
    "        self.states  = torch.zeros(num_steps + 1, num_envs, *state_shape)\n",
    "        self.rewards = torch.zeros(num_steps,     num_envs, 1)\n",
    "        self.masks   = torch.ones(num_steps  + 1, num_envs, 1)\n",
    "        self.actions = torch.zeros(num_steps,     num_envs, 1).long()\n",
    "        self.use_cuda = False\n",
    "            \n",
    "    def cuda(self):\n",
    "        self.use_cuda  = True\n",
    "        self.states    = self.states.cuda()\n",
    "        self.rewards   = self.rewards.cuda()\n",
    "        self.masks     = self.masks.cuda()\n",
    "        self.actions   = self.actions.cuda()\n",
    "        \n",
    "    def insert(self, step, state, action, reward, mask):\n",
    "        self.states[step + 1].copy_(state)\n",
    "        self.actions[step].copy_(action)\n",
    "        self.rewards[step].copy_(reward)\n",
    "        self.masks[step + 1].copy_(mask)\n",
    "        \n",
    "    def after_update(self):\n",
    "        self.states[0].copy_(self.states[-1])\n",
    "        self.masks[0].copy_(self.masks[-1])\n",
    "        \n",
    "    def compute_returns(self, next_value, gamma):\n",
    "        returns   = torch.zeros(self.num_steps + 1, self.num_envs, 1)\n",
    "        if self.use_cuda:\n",
    "            returns = returns.cuda()\n",
    "        returns[-1] = next_value\n",
    "        for step in reversed(range(self.num_steps)):\n",
    "            returns[step] = returns[step + 1] * gamma * self.masks[step + 1] + self.rewards[step]\n",
    "        return returns[:-1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>Creating environments</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "mode = \"regular\"\n",
    "num_envs = 16\n",
    "\n",
    "def make_env():\n",
    "    def _thunk():\n",
    "        env = MiniPacman(mode, 1000)\n",
    "        return env\n",
    "\n",
    "    return _thunk\n",
    "\n",
    "envs = [make_env() for i in range(num_envs)]\n",
    "envs = SubprocVecEnv(envs)\n",
    "\n",
    "state_shape = envs.observation_space.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Init and Train</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#a2c hyperparams:\n",
    "gamma = 0.99\n",
    "entropy_coef = 0.01\n",
    "value_loss_coef = 0.5\n",
    "max_grad_norm = 0.5\n",
    "num_steps = 5\n",
    "num_frames = int(10e5)\n",
    "\n",
    "#rmsprop hyperparams:\n",
    "lr    = 7e-4\n",
    "eps   = 1e-5\n",
    "alpha = 0.99\n",
    "\n",
    "#Init a2c and rmsprop\n",
    "actor_critic = ActorCritic(envs.observation_space.shape, envs.action_space.n)\n",
    "optimizer = optim.RMSprop(actor_critic.parameters(), lr, eps=eps, alpha=alpha)\n",
    "    \n",
    "if USE_CUDA:\n",
    "    actor_critic = actor_critic.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "rollout = RolloutStorage(num_steps, num_envs, envs.observation_space.shape)\n",
    "rollout.cuda()\n",
    "\n",
    "all_rewards = []\n",
    "all_losses  = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "state = envs.reset()\n",
    "state = torch.FloatTensor(np.float32(state))\n",
    "\n",
    "rollout.states[0].copy_(state)\n",
    "\n",
    "episode_rewards = torch.zeros(num_envs, 1)\n",
    "final_rewards   = torch.zeros(num_envs, 1)\n",
    "\n",
    "for i_update in range(num_frames):\n",
    "\n",
    "    for step in range(num_steps):\n",
    "        action = actor_critic.act(Variable(state))\n",
    "\n",
    "        next_state, reward, done, _ = envs.step(action.squeeze(1).cpu().data.numpy())\n",
    "\n",
    "        reward = torch.FloatTensor(reward).unsqueeze(1)\n",
    "        episode_rewards += reward\n",
    "        masks = torch.FloatTensor(1-np.array(done)).unsqueeze(1)\n",
    "        final_rewards *= masks\n",
    "        final_rewards += (1-masks) * episode_rewards\n",
    "        episode_rewards *= masks\n",
    "\n",
    "        if USE_CUDA:\n",
    "            masks = masks.cuda()\n",
    "\n",
    "        state = torch.FloatTensor(np.float32(next_state))\n",
    "        rollout.insert(step, state, action.data, reward, masks)\n",
    "\n",
    "\n",
    "    _, next_value = actor_critic(Variable(rollout.states[-1], volatile=True))\n",
    "    next_value = next_value.data\n",
    "\n",
    "    returns = rollout.compute_returns(next_value, gamma)\n",
    "\n",
    "    logit, action_log_probs, values, entropy = actor_critic.evaluate_actions(\n",
    "        Variable(rollout.states[:-1]).view(-1, *state_shape),\n",
    "        Variable(rollout.actions).view(-1, 1)\n",
    "    )\n",
    "\n",
    "    values = values.view(num_steps, num_envs, 1)\n",
    "    action_log_probs = action_log_probs.view(num_steps, num_envs, 1)\n",
    "    advantages = Variable(returns) - values\n",
    "\n",
    "    value_loss = advantages.pow(2).mean()\n",
    "    action_loss = -(Variable(advantages.data) * action_log_probs).mean()\n",
    "\n",
    "    optimizer.zero_grad()\n",
    "    loss = value_loss * value_loss_coef + action_loss - entropy * entropy_coef\n",
    "    loss.backward()\n",
    "    nn.utils.clip_grad_norm(actor_critic.parameters(), max_grad_norm)\n",
    "    optimizer.step()\n",
    "    \n",
    "    if i_update % 100 == 0:\n",
    "        all_rewards.append(final_rewards.mean())\n",
    "        all_losses.append(loss.data[0])\n",
    "        \n",
    "        clear_output(True)\n",
    "        plt.figure(figsize=(20,5))\n",
    "        plt.subplot(131)\n",
    "        plt.title('epoch %s. reward: %s' % (i_update, np.mean(all_rewards[-10:])))\n",
    "        plt.plot(all_rewards)\n",
    "        plt.subplot(132)\n",
    "        plt.title('loss %s' % all_losses[-1])\n",
    "        plt.plot(all_losses)\n",
    "        plt.show()\n",
    "        \n",
    "    rollout.after_update()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>Saving the model!</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(actor_critic.state_dict(), \"actor_critic_\" + mode)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>Let's see the game!<h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time \n",
    "\n",
    "def displayImage(image, step, reward):\n",
    "    clear_output(True)\n",
    "    s = \"step: \" + str(step) + \" reward: \" + str(reward)\n",
    "    plt.figure(figsize=(10,3))\n",
    "    plt.title(s)\n",
    "    plt.imshow(image)\n",
    "    plt.show()\n",
    "    time.sleep(0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPQAAADSCAYAAAB92u3VAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAD85JREFUeJzt3X2wXHV9x/H3hzzwEMOTgRQS4CLNMEXaCU5KbWshGNCI\n2kArNbQooXTEGUF8qBKwFTqtnTgVqS2OCoJSQRlKYaAUgRQQBC2QRJ4fIw15IE9QSgWhGPn2j/O7\n081y783uOWf33P3l85q5c8+ec3b3e87Zzzlnz/72t4oIzCwPOzRdgJnVx4E2y4gDbZYRB9osIw60\nWUYcaLOMONA2bkgakhSSJjZdy6ByoEch6TxJl/f5ORdKekzSy5J+Kun3Rpjn8+lFf3Tb+LdJulPS\nS5I2Sjqzf5U3S9IsSa+2bi9JkyVdLWlVWl9z2+5zlKTbJb0oaVW/a+4VB3qckHQM8EXgFGAqcATw\ndNs8BwEnAOvbxk8DbgK+AbwZ+FXglg6ft5GjYc3P+1XgvhHG3wWcBGwYYdrLwKXAZ2qso3kRsV3/\nAWcB64CfAU8A84D5wGvAL4CXgAfSvLsBl1AEah3wN8CENG0RcDdwIfAi8Dgwr4s6fgScuo15bgKO\nBVYBR7eM/1vgOx0+zxAQwKnAauDONP7tqYb/Bh4A5qbxRwEPtdx/KXBfy+0fAsel4cXAT9O6fBQ4\nvmW+4fVzAfD88LoDvgQ8R7Hz+liqbWIX620hcBVwHnD5KPOsHV6eEaYdDaxq+nVY2+u56QIaXXg4\nGFgD7JtuDwEHpeE3vECAaymOglOAvYF7gdPStEXAFuCTwCTggynYe6bpi4EbRqljQtqBLAZWphfg\nhcDOLfOcAFyXhtsDfRvwlRTITcC/AvuP8lzDgf6ntBw7AzNSyI6lOGs7Jt3eK01/FZiWlmsjxc5s\napr2CvDmlhr3TY/xQYqj4D5t6+cMYGK670cpdnz7AXsCt7cGeqx1lqbvCjwJzHSg0/I0XUCjC1+c\nmm5KG3VS27StXiDAdOB/20J2InB7Gl4EPAuoZfq9wIc6qGPf9EJeBuyTwnM38IU0fSrwFDCUbrcH\n+kmKI+tvAjsB/wDcPcpzDQf6LS3jzqLtCA/cDJychn8I/AHFUfwWiiPifIqj94NjLNf9wIKW9bO6\nbfptwEdbbr+LLo7QFDuxs0baXm3zbTeB3q6vJkbESkmfoHgxvFXSzcCnIuLZEWY/gOIItV7S8Lgd\nKI7ww9ZFepUkz1CEdVteSf//MSLWA0j6MvAXwOdSfd+JiFVj3P/aiLgv3fevgOck7RYRL45yn9a6\nDwBOkPT+lnGTKI6YAHcAcymCcQfwAnAkxQ7ujuE7SPow8CmKnQbAmyh2TiM9JxTrpnXcM6PU+gaS\nZlOE8bBO77M92O4vikXEdyPiHRQv6qC4MEUabrWG4gU8LSJ2T3+7RsRbW+aZoZa0A/tTHLW3VcML\nFGFpfc7W4XnAxyVtkLSB4hT1KklnpekPjnHfUZ+2ZXgNxQ5j95a/KRGxJE0fDvQRafgOikAfmYaR\ndABwMXA6xSn47sDDQOv6aK9rfVqWYft3UPewuRQ7jtVpnfw58IeSVnTxGNnZrgMt6WBJ75S0I8X7\nxFeA19PkjcCQpB0A0pHzFuB8SbtK2kHSQZKObHnIvSmCN0nSCcCvATd2WM63gDMk7S1pD4r34jek\nafOAQ4HZ6e9Z4DSKq7vD9z1e0mxJk4C/BO4a4+jc7nLg/ZLeLWmCpJ0kzZU0M03/EcX1hsOBeyPi\nEYod4G8Bd6Z5plAEdjOApFNSzWO5imJ9zUzLvLjDegEuAg7i/9fJ14F/A949PIOkHSXtlG5OTsul\nNG2HNG1ScVM7SZrcxfOPS9t1oIEdgSUUV1k3UATy7DTtn9P/51v2+h8GJlNcwX0BuJriPe+we4BZ\n6fG+AHwgIp4HkHSOpO+PUctfU3z08iTwGPCT9BhExPMRsWH4D/gl8EJEvJSm3wacQ/GC3kRxbeCP\nO10JEbEGWJAeYzPFEfszpNdHRLwMrAAeiYjX0t1+DDwTEZvSPI8C56fxG4Ffp7gOMJaLKd6rP5Ae\n/5rWiWOts4j4eds6eQl4NSI2t8z2BMVOekZ6nlcodkRQnG28QrHD3T8Nd/RR33imrd/yWVmSFgF/\nlk7fzRqxvR+hzbLiQJtlxKfcZhnxEdosIw60WUb62lJs2rRpMTQ01M+nNMvC8uXLn4uIvbY1X18D\nPTQ0xLJly/r5lGZZkNRRs9hKp9yS5kt6QtJKSd208jGzHigdaEkTKJoevgc4BDhR0iF1FWZm3aty\nhD4cWBkRT6fmgFdSNB80s4ZUCfQMtv7q29o0biuSPiJpmaRlmzdvbp9sZjXq+cdWEXFRRMyJiDl7\n7bXNi3RmVkGVQK9j6++yzkzjzKwhVQJ9HzBL0oHpe6QLgevrKcvMyij9OXREbJF0OsX3TCcAl6Yv\nvptZQyo1LImIG+m8R46ubN2TTzM6/eLKeKjVOt9eAONhk/Xie1Fuy22WEQfaLCMOtFlGHGizjDjQ\nZhlxoM0y4kCbZcSBNsuIA22WEQfaLCNZ/Jxsd03+mm3z537Qu9f8Nut83qablPoIbZaRKn2K7Sfp\ndkmPSnpE0pl1FmZm3atyyr0F+HRErJA0FVguaWn6WVEza0DpI3RErI+IFWn4ZxS/afyGPsXMrH9q\neQ8taQg4jOIHz82sIZUDLelNwL8An4iI/xlhunv9NOuTqr+cMYkizFdExDUjzeNeP836p8pVbgGX\nAI9FxJfrK8nMyqpyhP5d4EPAOyXdn/6OrakuMyuhSq+fdwHjoKs1MxuWRdPPXjUN7MXjNt2MMWe9\nWrWDtMnc9NMsIw60WUYcaLOMONBmGXGgzTLiQJtlxIE2y4gDbZYRB9osIw60WUayaPrZq14/e/GD\n7+71s3udrt5e9c7pXj/NrBF19FgyQdJPJN1QR0FmVl4dR+gzKToINLOGVe2CaCbwXuCb9ZRjZlVU\nPUL/PfBZ4PUaajGziqr0KfY+YFNELN/GfO7106xPqvYp9vuSVgFXUvQtdnn7TO7106x/qvxyxtkR\nMTMihoCFwG0RcVJtlZlZ1/w5tFlGamkpFhE/AH5Qx2OZWXlZNP3sFff6OV502gS3x2UMAJ9ym2XE\ngTbLiANtlhEH2iwjDrRZRhxos4w40GYZcaDNMuJAm2XEgTbLyHbX9LPpXjd71dvkoOmup9QeFpIZ\nH6HNMlK1T7HdJV0t6XFJj0n67boKM7PuVT3l/gpwU0R8QNJkYJcaajKzkkoHWtJuwBHAIoCIeA14\nrZ6yzKyMKqfcBwKbgW+ljva/KWlK+0zuJNCsf6oEeiLwNuBrEXEY8DKwuH0mdxJo1j9VAr0WWBsR\n96TbV1ME3MwaUqXXzw3AGkkHp1HzgEdrqcrMSql6lfsM4Ip0hftp4JTqJZlZWZUCHRH3A3NqqsXM\nKsqi6We+PWm6zSN4+3bDTT/NMuJAm2XEgTbLiANtlhEH2iwjDrRZRhxos4w40GYZcaDNMpJFS7Gm\nO/7rRr6tnrrV+TYboM3b+Pb1EdosIw60WUaq9vr5SUmPSHpY0vck7VRXYWbWvdKBljQD+DgwJyIO\nBSYAC+sqzMy6V/WUeyKws6SJFF34Plu9JDMrq0oXROuALwGrgfXAixFxS/t87vXTrH+qnHLvASyg\n6M53X2CKpJPa53Ovn2b9U+WU+2jgPyNic0T8ArgG+J16yjKzMqoEejXwdkm7qPg0fR7wWD1lmVkZ\nVd5D30PRF/cK4KH0WBfVVJeZlVC1189zgXNrqqUvmm6a15Uuah2gpepa05tskJoWu6WYWUYcaLOM\nONBmGXGgzTLiQJtlxIE2y4gDbZYRB9osIw60WUYcaLOMZNHrZ6902uSvm+akvWpGODiNE7vX6ert\nZtUOVBPgLvgIbZaRbQZa0qWSNkl6uGXcnpKWSnoq/d+jt2WaWSc6OUJ/G5jfNm4xcGtEzAJuTbfN\nrGHbDHRE3An8V9voBcBlafgy4Lia6zKzEsq+h54eEevT8AZgek31mFkFlS+KRXHZdtTri+7106x/\nygZ6o6R9ANL/TaPN6F4/zfqnbKCvB05OwycD19VTjplV0cnHVt8DfgwcLGmtpFOBJcAxkp6i6M53\nSW/LNLNObLOlWEScOMqkeTXXYmYVZdH0s1fN+HrxuLk2ORwPerVmB2mbuemnWUYcaLOMONBmGXGg\nzTLiQJtlxIE2y4gDbZYRB9osIw60WUYcaLOMZNH0s5ueNHvRQ+d46PUzZ52u3+ii71N10VC0V6+v\nXvAR2iwjZXv9/DtJj0t6UNK1knbvbZlm1omyvX4uBQ6NiN8AngTOrrkuMyuhVK+fEXFLRGxJN/8D\nmNmD2sysS3W8h/5T4Ps1PI6ZVVQp0JI+B2wBrhhjHvf6adYnpQMtaRHwPuBPYozr+u7106x/Sn0O\nLWk+8FngyIj4eb0lmVlZZXv9vBCYCiyVdL+kr/e4TjPrQNlePy/pQS1mVlEWTT+bbm7XjUGq1QqD\ntM3c9NMsIw60WUYcaLOMONBmGXGgzTLiQJtlxIE2y4gDbZYRB9osIw60WUbGbdPPQeodc5BqtUKu\n28xHaLOMlOr1s2XapyWFpGm9Kc/MulG2108k7Qe8C1hdc01mVlKpXj+TCyh6LcnzzYjZACr1HlrS\nAmBdRDzQwbzuJNCsT7oOtKRdgHOAz3cyvzsJNOufMkfog4ADgQckraLoZH+FpF+pszAz617Xn0NH\nxEPA3sO3U6jnRMRzNdZlZiWU7fXTzMahsr1+tk4fqq0aM6tE/WwCJ2kz8Ezb6GlArqfruS6bl6v/\nDoiIbV5V7mugRyxAWhYRcxotokdyXTYv1/jlttxmGXGgzTIyHgJ9UdMF9FCuy+blGqcafw9tZvUZ\nD0doM6tJo4GWNF/SE5JWSlrcZC11krRK0kPpp3aXNV1PFSN9H17SnpKWSnoq/d+jyRrLGGW5zpO0\nLm23+yUd22SNZTQWaEkTgK8C7wEOAU6UdEhT9fTAURExe9A/BmHk78MvBm6NiFnAren2oPk2I3zP\nH7ggbbfZEXFjn2uqrMkj9OHAyoh4OiJeA64EFjRYj41glO/DLwAuS8OXAcf1tagajPE9/4HWZKBn\nAGtabq9N43IQwL9LWi7pI00X0wPTI2J9Gt4ATG+ymJqdIenBdEo+cG8lfFGsN94REbMp3k58TNIR\nTRfUK1F8TJLLRyVfA94CzAbWA+c3W073mgz0OmC/ltsz07iBFxHr0v9NwLUUby9yslHSPgDp/6aG\n66lFRGyMiF9GxOvAxQzgdmsy0PcBsyQdKGkysBC4vsF6aiFpiqSpw8MUHSm+ocfUAXc9cHIaPhm4\nrsFaajO8k0qOZwC3W2Md7UfEFkmnAzcDE4BLI+KRpuqp0XTgWklQrN/vRsRNzZZUXvo+/FxgmqS1\nwLnAEuCq9N34Z4A/aq7CckZZrrmSZlO8hVgFnNZYgSW5pZhZRnxRzCwjDrRZRhxos4w40GYZcaDN\nMuJAm2XEgTbLiANtlpH/AwCPbOPy3djDAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f77f0e37210>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "env = MiniPacman(mode, 1000)\n",
    "\n",
    "done = False\n",
    "state = env.reset()\n",
    "total_reward = 0\n",
    "step   = 1\n",
    "\n",
    "\n",
    "while not done:\n",
    "    current_state = torch.FloatTensor(state).unsqueeze(0)\n",
    "    if USE_CUDA:\n",
    "        current_state = current_state.cuda()\n",
    "        \n",
    "    action = actor_critic.act(Variable(current_state))\n",
    "    \n",
    "    next_state, reward, done, _ = env.step(action.data[0, 0])\n",
    "    total_reward += reward\n",
    "    state = next_state\n",
    "    \n",
    "    image = torch.FloatTensor(state).permute(1, 2, 0).cpu().numpy()\n",
    "    displayImage(image, step, total_reward)\n",
    "    step += 1"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}