5-Deep-Q-Network.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exercise 5: Deep Q-Network (DQN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from __future__ import division\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import os\n",
    "import random\n",
    "import scipy.misc\n",
    "import tensorflow as tf\n",
    "import tensorflow.contrib.slim as slim\n",
    "\n",
    "from unityenv import UnityEnvironment\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "batch_size = 32 # How many experiences to use for each training step.\n",
    "update_freq = 4 # How often to perform a training step.\n",
    "y = .99 # Discount factor on the target Q-values\n",
    "startE = 1 # Starting chance of random action\n",
    "endE = 0.1 # Final chance of random action\n",
    "anneling_steps = 50000 # How many steps of training to reduce startE to endE.\n",
    "num_episodes = 10000 # How many episodes of game environment to train network with.\n",
    "pre_train_steps = 5000 # How many steps of random actions before training begins.\n",
    "model_path = \"./models/dqn\" # The path to save our model to.\n",
    "summary_path = './summaries/dqn' # The path to save summary statistics to.\n",
    "h_size = 256 # The number of units in the hidden layer.\n",
    "learning_rate = 1e-3 # Agent Learning Rate\n",
    "load_model = False # Whether to load a saved model.\n",
    "train_model = True # Whether to train the model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load the Unity environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "env_config = {'--grid-size': 5, '--num-objects': 1, '--num-goals': 1}\n",
    "env = UnityEnvironment(file_name=\"RandomGridWorld\", train_model=train_model, \n",
    "                       worker_num=4, config=env_config)\n",
    "env.bw_render = True\n",
    "env.resolution = 32\n",
    "print(str(env))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Examine observation space"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "observations, _ = env.reset()\n",
    "plt.imshow(np.reshape(observations[0], [32, 32]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Observation (o) corresponds to a greyscale pixel image of the screen."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Implementing the network itself"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Qnetwork():\n",
    "    def __init__(self ,h_size, num_actions, lr, scope):\n",
    "        with tf.variable_scope(scope):\n",
    "            # The network recieves a frame from the game, flattened into an array.\n",
    "            # It then resizes it and processes it through four convolutional layers.\n",
    "            self.observation_input =  tf.placeholder(shape=[None, 32, 32, 3],dtype=tf.float32)\n",
    "            self.conv1 = slim.conv2d(self.observation_input, 32, \n",
    "                                     kernel_size=[3,3], stride=[2,2], \n",
    "                                     biases_initializer=None,\n",
    "                                     activation_fn=tf.nn.elu)\n",
    "            self.conv2 = slim.conv2d(self.conv1, 64, \n",
    "                                     kernel_size=[3,3], \n",
    "                                     stride=[2,2], \n",
    "                                     biases_initializer=None,\n",
    "                                     activation_fn=tf.nn.elu)\n",
    "\n",
    "            # We take the output from the final convolutional layer \n",
    "            # and split it into separate advantage and value streams.\n",
    "            self.hidden = slim.fully_connected(slim.flatten(self.conv2), \n",
    "                                               h_size, activation_fn=tf.nn.elu)\n",
    "            self.advantage = slim.fully_connected(self.hidden, num_actions, activation_fn=None,\n",
    "                                                  biases_initializer=None)\n",
    "            self.value = slim.fully_connected(self.hidden, 1, activation_fn=None,\n",
    "                                                  biases_initializer=None)\n",
    "\n",
    "            # Then combine them together to get our final Q-values.\n",
    "            self.q_out = self.value + tf.subtract(self.advantage, \n",
    "                                                  tf.reduce_mean(self.advantage,axis=1, keep_dims=True))\n",
    "            self.predict = tf.argmax(self.q_out,1)\n",
    "\n",
    "            # Below we obtain the loss by taking the sum of squares difference \n",
    "            # between the target and prediction Q values.\n",
    "            self.targetQ = tf.placeholder(shape=[None],dtype=tf.float32)\n",
    "            self.actions = tf.placeholder(shape=[None],dtype=tf.int32)\n",
    "            self.actions_onehot = tf.one_hot(self.actions,num_actions,dtype=tf.float32)\n",
    "\n",
    "            self.Q = tf.reduce_sum(tf.multiply(self.q_out, self.actions_onehot), axis=1)\n",
    "\n",
    "            self.td_error = tf.square(self.targetQ - self.Q)\n",
    "            self.loss = tf.reduce_mean(self.td_error)\n",
    "            self.trainer = tf.train.AdamOptimizer(learning_rate=lr)\n",
    "            self.update = self.trainer.minimize(self.loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Experience Replay"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This class allows us to store experies and sample then randomly to train the network."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class experience_buffer():\n",
    "    def __init__(self, buffer_size = 50000):\n",
    "        self.buffer = []\n",
    "        self.buffer_size = buffer_size\n",
    "    \n",
    "    def add(self,experience):\n",
    "        if len(self.buffer) + len(experience) >= self.buffer_size:\n",
    "            self.buffer[0:(len(experience)+len(self.buffer))-self.buffer_size] = []\n",
    "        self.buffer.extend(experience)\n",
    "            \n",
    "    def sample(self,size):\n",
    "        return np.reshape(np.array(random.sample(self.buffer,size)),[size,5])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "These functions allow us to update the parameters of our target network with those of the primary network."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def update_target_graph(from_scope, to_scope):\n",
    "    from_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, from_scope)\n",
    "    to_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, to_scope)\n",
    "\n",
    "    op_holder = []\n",
    "    for from_var,to_var in zip(from_vars,to_vars):\n",
    "        op_holder.append(to_var.assign(from_var))\n",
    "    return op_holder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training the network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "mainQN = Qnetwork(h_size, env.action_space_size, learning_rate, \"main\")\n",
    "targetQN = Qnetwork(h_size, env.action_space_size, learning_rate, \"target\")\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "if not os.path.exists(summary_path):\n",
    "    os.makedirs(summary_path)\n",
    "    \n",
    "if not os.path.exists(model_path):\n",
    "    os.makedirs(model_path)\n",
    "\n",
    "saver = tf.train.Saver()\n",
    "\n",
    "trainables = tf.trainable_variables()\n",
    "\n",
    "update_target_ops = update_target_graph(\"main\", \"target\")\n",
    "\n",
    "myBuffer = experience_buffer()\n",
    "\n",
    "# Set the rate of random action decrease. \n",
    "e = startE\n",
    "stepDrop = (startE - endE)/anneling_steps\n",
    "\n",
    "# Create lists to contain total rewards and steps per episode\n",
    "episode_lengths = []\n",
    "episode_rewards = []\n",
    "losses = []\n",
    "total_steps = 0\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    summary_writer = tf.summary.FileWriter(summary_path)\n",
    "    if load_model == True:\n",
    "        print('Loading Model...')\n",
    "        ckpt = tf.train.get_checkpoint_state(model_path)\n",
    "        saver.restore(sess,ckpt.model_checkpoint_path)\n",
    "    for i in range(num_episodes):\n",
    "        episodeBuffer = experience_buffer()\n",
    "        observations, _ = env.reset()\n",
    "        observation = np.concatenate([observations[0], observations[0], observations[0]], axis=2)\n",
    "        done = False\n",
    "        episode_reward = 0\n",
    "        episode_steps = 0\n",
    "        while not done:\n",
    "            episode_steps+=1\n",
    "            # Choose an action by greedily (with e chance of random action) from the Q-network\n",
    "            if (np.random.rand(1) < e or total_steps < pre_train_steps) and train_model:\n",
    "                action = np.random.randint(0,env.action_space_size)\n",
    "            else:\n",
    "                action = sess.run(mainQN.predict, \n",
    "                                  feed_dict={mainQN.observation_input:[observation]})[0]\n",
    "            if not train_model and np.random.rand(1) < 0.1:\n",
    "                action = np.random.randint(0,env.action_space_size)\n",
    "            observations_1, _, reward, done = env.step(action, np.zeros([49]).tolist())\n",
    "            observation_1 = observation[:, :, 1:]\n",
    "            observation_1 = np.concatenate([observation_1, observations_1[0]], axis=2)\n",
    "            total_steps += 1\n",
    "            \n",
    "            # Save the experience to our episode buffer.\n",
    "            episodeBuffer.add(np.reshape(np.array([observation,action,reward,observation_1,done]),[1,5])) \n",
    "            \n",
    "            if total_steps > pre_train_steps and train_model:\n",
    "                if total_steps % 1000 == 0:\n",
    "                    sess.run(update_target_ops)\n",
    "                \n",
    "                if e > endE:\n",
    "                    e -= stepDrop\n",
    "                \n",
    "                if total_steps % (update_freq) == 0:\n",
    "                    # Get a random batch of experiences.\n",
    "                    trainBatch = myBuffer.sample(batch_size) \n",
    "                    # Below we perform the Double-DQN update to the target Q-values\n",
    "                    Q1 = sess.run(mainQN.predict, \n",
    "                                  feed_dict={mainQN.observation_input:np.stack(trainBatch[:,3], axis=0)})\n",
    "                    Q2 = sess.run(targetQN.q_out, \n",
    "                                  feed_dict={targetQN.observation_input:np.stack(trainBatch[:,3], axis=0)})\n",
    "                    end_multiplier = -(trainBatch[:,4] - 1)\n",
    "                    doubleQ = Q2[range(batch_size),Q1]\n",
    "                    targetQ = trainBatch[:,2] + (y*doubleQ * end_multiplier)\n",
    "                    # Update the network with our target values.\n",
    "                    _, q_loss = sess.run([mainQN.update, mainQN.loss],\n",
    "                        feed_dict={mainQN.observation_input:np.stack(trainBatch[:,0], axis=0),\n",
    "                                   mainQN.targetQ:targetQ, \n",
    "                                   mainQN.actions:trainBatch[:,1]})\n",
    "                    losses.append(q_loss)\n",
    "            episode_reward += reward\n",
    "            observation = observation_1\n",
    "                    \n",
    "        myBuffer.add(episodeBuffer.buffer)\n",
    "        episode_lengths.append(episode_steps)\n",
    "        episode_rewards.append(episode_reward)\n",
    "        # Periodically save the model and summary statistics\n",
    "        if i % 1000 == 0 and i != 0:\n",
    "            saver.save(sess, model_path+'/model-'+str(i)+'.cptk')\n",
    "            print(\"Saved Model\")\n",
    "        if i % 50 == 0 and i != 0:\n",
    "            summary = tf.Summary()\n",
    "            summary.value.add(tag='Info/Reward', simple_value=float(np.mean(episode_rewards[-50:])))\n",
    "            summary.value.add(tag='Info/Q Loss', simple_value=float(np.mean(losses[-50:])))\n",
    "            summary.value.add(tag='Info/Epsilon', simple_value=float(e))\n",
    "            summary.value.add(tag='Info/Episode Length', simple_value=float(np.mean(episode_lengths[-50:])))\n",
    "            summary_writer.add_summary(summary, i)\n",
    "            summary_writer.flush()\n",
    "            print (\"Mean Reward: {}\".format(np.mean(episode_rewards[-50:])))\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "env.close()"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}