DQN Optimizer.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Trial 2\n",
    "\n",
    "We use the basic framework from Q learning instead of what is outlined in the paper.\n",
    "\n",
    "1. Import statements\n",
    "    * The most important part here is the import for the function that outputs the values\n",
    "    \n",
    "2. Delcaring Parameters\n",
    "\n",
    "3. Setting up the convolutional network\n",
    "\n",
    "4. Training the network\n",
    "\n",
    "This would be a Deep Q network to start off with. The problems I forsee with this is the insanely large layers which may impede performance and the modeling of the system. The accuracy and performance of this method, whislt simple, may not be up too par either. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import random\n",
    "import tensorflow as tf\n",
    "#import matplotlib.pyplot as plt\n",
    "#^ Don't have this library yet, but its just for plotting\n",
    "#also need to import simulator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "size = 16\n",
    "#size = sim.getsize() #simulator needs to have a function for getting size of problem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#These lines establish the feed-forward part of the network used to choose actions\n",
    "inputs1 = tf.placeholder(shape=[1,size],dtype=tf.float32) #The 16 in this line needs to get changed to however many\n",
    "                                                        #elements there are to optimize\n",
    "    \n",
    "W = tf.Variable(tf.random_uniform([size,size],0,0.01))   #The 16 corresponds to the number of elements to optimize\n",
    "                                                    #The 4 corresponds to the number of moves at each step, which \n",
    "                                                    #is equivalent to the number of elements in our case\n",
    "Qout = tf.matmul(inputs1,W)\n",
    "predict = tf.argmax(Qout,1)\n",
    "\n",
    "#Below we obtain the loss by taking the sum of squares difference between the target and prediction Q values.\n",
    "nextQ = tf.placeholder(shape=[1,size],dtype=tf.float32)\n",
    "loss = tf.reduce_sum(tf.square(nextQ - Qout))\n",
    "trainer = tf.train.GradientDescentOptimizer(learning_rate=0.1)\n",
    "updateModel = trainer.minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\ichis\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\python\\util\\tf_should_use.py:170: initialize_all_variables (from tensorflow.python.ops.variables) is deprecated and will be removed after 2017-03-02.\n",
      "Instructions for updating:\n",
      "Use `tf.global_variables_initializer` instead.\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'sim' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-9-81310f6f9cd7>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     12\u001b[0m     \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnum_episodes\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     13\u001b[0m         \u001b[1;31m#Reset environment and get first new observation\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 14\u001b[1;33m         \u001b[0ms\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msim\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreset\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;31m#make this as well\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     15\u001b[0m         \u001b[0mrAll\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     16\u001b[0m         \u001b[0md\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'sim' is not defined"
     ]
    }
   ],
   "source": [
    "init = tf.initialize_all_variables()\n",
    "\n",
    "# Set learning parameters\n",
    "y = .99\n",
    "e = 0.1\n",
    "num_episodes = 2000\n",
    "#create lists to contain total rewards and steps per episode\n",
    "jList = []\n",
    "rList = []\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    for i in range(num_episodes):\n",
    "        #Reset environment and get first new observation\n",
    "        s = sim.reset() #make this as well\n",
    "        rAll = 0\n",
    "\n",
    "        j = 0\n",
    "        #The Q-Network\n",
    "        while j < size:\n",
    "            j+=1\n",
    "            #Choose an action by greedily (with e chance of random action) from the Q-network\n",
    "            a,allQ = sess.run([predict,Qout],feed_dict={inputs1:np.identity(size)[s:s+1]}) #replaced\n",
    "            if np.random.rand(1) < e:\n",
    "                a[0] = sim.getRandomSample(); #Need to make this function\n",
    "            #Get new state and reward from environment\n",
    "            s1,r = sim.step(a[0]) #need to make this function, gets reward for doing action A and returns new state as well\n",
    "            #Obtain the Q' values by feeding the new state through our network\n",
    "            Q1 = sess.run(Qout,feed_dict={inputs1:np.identity(size)[s1:s1+1]}) #replaced\n",
    "            #Obtain maxQ' and set our target value for chosen action.\n",
    "            maxQ1 = np.max(Q1) #change here?\n",
    "            targetQ = allQ\n",
    "            targetQ[0,a[0]] = r + y*maxQ1 #change here also for min?\n",
    "            #Train our network using target and predicted Q values\n",
    "            _,W1 = sess.run([updateModel,W],feed_dict={inputs1:np.identity(size)[s:s+1],nextQ:targetQ}) #Replaced\n",
    "            rAll += r\n",
    "            s = s1\n",
    "           \n",
    "        \n",
    "        e = 1./((i/50) + 10) #reduce random chance as training progresses\n",
    "\n",
    "        #keep the best reward\n",
    "        if sim.reward(s) > sim.reward(sBest):\n",
    "            sBest = s\n",
    "        \n",
    "        #Logging?\n",
    "        jList.append(j)\n",
    "        rList.append(rAll)\n",
    "\n",
    "\n",
    "print(sBest)\n",
    "\n",
    "#should memorize score and print out one with the lowest score\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "I am assuming that it optimizes for the highest score possible, therefore, this gets a bit complicated as lower times need a higher score. Could be possible via an inverting function to get the reciprocal of the time as the score\n",
    "\n",
    "Otherwise I need to figure out where to change it to be minimize instead of maximize. I marked a few places, though it may be easier to ignore it for now and just change the input funciton."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}