snake/TestingSnake.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Imports the libraries and makes matplotlib appear in a second window:\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib as mpl\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "import random\n",
    "\n",
    "from collections import deque\n",
    "\n",
    "%matplotlib qt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Snake ():\n",
    "    def __init__ (self, nrows, ncols):\n",
    "        \n",
    "        self.nrows, self.ncols = nrows, ncols\n",
    "        self.board = np.zeros((nrows,ncols)) #0 for empty, 1 for food and 2 for snake body\n",
    "        self.movingDirection = np.zeros((4))\n",
    "        self.headY, self.headX = nrows//2, ncols//2\n",
    "        self.foodY, self.foodX = None, None\n",
    "        self.tailY, self.tailX = self.headY, self.headX\n",
    "        self.movingDirection = np.array(([1,0,0,0]))\n",
    "        self.snakeBody = deque()\n",
    "        self.snakeBody.appendleft((self.headY,self.headX))\n",
    "        self.createFood()\n",
    "    def checkIfBelongsToSnake (self,y, x):\n",
    "        return bool(self.snakeBody.count((y,x)))\n",
    "\n",
    "    def createFood(self):\n",
    "        self.foodY, self.foodX = self.headY, self.headX\n",
    "        while (self.checkIfBelongsToSnake(self.foodY,self.foodX)):\n",
    "            self.foodY, self.foodX = random.randrange(self.nrows), random.randrange(self.ncols)\n",
    "        self.board[self.foodY, self.foodX] = 1\n",
    "        \n",
    "    def move(self, action):\n",
    "        if (action == 0): # Move up\n",
    "            if ((self.headY-1)<0):# Snake dies if it tries to go out of the area\n",
    "                self.__init__(self.nrows, self.ncols)\n",
    "                reward = -1\n",
    "                return self.board, True, reward\n",
    "            else:\n",
    "                self.headY, self.headX = self.headY-1, self.headX\n",
    "        elif (action == 1): # Move right\n",
    "            if ((self.headX+1)==self.ncols):# Snake dies if it tries to go out of the area\n",
    "                self.__init__(self.nrows, self.ncols)\n",
    "                reward = -1\n",
    "                return self.board, True, reward\n",
    "            else:\n",
    "                self.headY, self.headX = self.headY , self.headX+1\n",
    "        elif (action == 2): # Move down\n",
    "            if ((self.headY+1)==self.nrows):# Snake dies if it tries to go out of the area\n",
    "                self.__init__(self.nrows, self.ncols)\n",
    "                reward = -1\n",
    "                return self.board, True, reward\n",
    "            else:\n",
    "                self.headY, self.headX = self.headY+1, self.headX\n",
    "        elif (action == 3): # Move Left\n",
    "            if ((self.headX-1)<0):# Snake dies if it tries to go out of the area\n",
    "                self.__init__(self.nrows, self.ncols)\n",
    "                reward = -1\n",
    "                return self.board, True, reward\n",
    "            else:\n",
    "                self.headY, self.headX = self.headY , self.headX-1\n",
    "        else:\n",
    "            raise Exception (\"Invalid action\")\n",
    "        self.movingDirection = np.zeros((4))\n",
    "        self.movingDirection[action] = 1\n",
    "        \n",
    "        if (self.checkIfBelongsToSnake(self.headY, self.headX) and not (self.headY == self.tailY and self.headX == self.tailX)):\n",
    "            self.__init__(self.nrows, self.ncols) # Snake dies if bumps into a body part when its length is more than 1\n",
    "            reward = -1\n",
    "            return self.board, True, reward\n",
    "        \n",
    "        else:\n",
    "            self.snakeBody.appendleft((self.headY, self.headX))\n",
    "            \n",
    "            if (self.board[self.headY, self.headX] == 1): # If it collects an apple\n",
    "                reward = 1\n",
    "                self.createFood()\n",
    "            else:\n",
    "                reward = 0\n",
    "                self.board[self.tailY, self.tailX] = 0\n",
    "                self.snakeBody.pop()\n",
    "                if (self.snakeBody):\n",
    "                    self.tailY, self.tailX = self.snakeBody[-1]\n",
    "   \n",
    "            self.board[self.headY, self.headX] = 2\n",
    "\n",
    "            return self.board, False, reward\n",
    "    def snakeVision (self):\n",
    "        # This vision will be normalised afterwards in range 0 to 1 (a measurement of proximity)\n",
    "        state = np.zeros((24)) # It indicates, in order, distance to wall, to snake body and to apple starting from up, then diagonally up-right and so on till diagonally up left (clockwise)\n",
    "        \n",
    "        #Direction Up:\n",
    "        state[0] = self.headY + 1 #Distance to wall\n",
    "        for index, i in enumerate(range (self.headY-1, -1, -1)):\n",
    "            if (self.board[i, self.headX] ==2):\n",
    "                state[1] = index + 1 #Distance to snake body\n",
    "                break\n",
    "        for index, i in enumerate(range (self.headY-1, -1, -1)):\n",
    "            if (self.board[i, self.headX] ==1):\n",
    "                state[2] = index + 1 #Distance to apple\n",
    "                break\n",
    "        \n",
    "        #Direction Up-Right:\n",
    "        state[3] = min(self.headY+1, self.ncols-self.headX) \n",
    "        for index, i in enumerate(range (1,int(state[3]))):\n",
    "            if (self.board[self.headY-i, self.headX+i] ==2):\n",
    "                state[4] = index + 1\n",
    "                break\n",
    "        for index, i in enumerate(range (1,int(state[3]))):\n",
    "            if (self.board[self.headY-i, self.headX+i] ==1):\n",
    "                state[5] = index + 1\n",
    "                break\n",
    "        \n",
    "        #Direction RIGTH:\n",
    "        state[6] = self.ncols - self.headX\n",
    "        for index, i in enumerate(range (self.headX+1, self.ncols)):\n",
    "            if (self.board[self.headY, i] ==2):\n",
    "                state[7] = index + 1\n",
    "                break\n",
    "        for index, i in enumerate(range (self.headX+1, self.ncols)):\n",
    "            if (self.board[self.headY, i] ==1):\n",
    "                state[8] = index + 1\n",
    "                break\n",
    "                \n",
    "        #Direction Down-Right:\n",
    "        state[9] = min(self.nrows-self.headY, self.ncols-self.headX) \n",
    "        for index, i in enumerate(range (1,int(state[9]))):\n",
    "            if (self.board[self.headY+i, self.headX+i] ==2):\n",
    "                state[10] = index + 1\n",
    "                break\n",
    "        for index, i in enumerate(range (1,int(state[9]))):\n",
    "            if (self.board[self.headY+i, self.headX+i] ==1):\n",
    "                state[11] = index + 1\n",
    "                break\n",
    "        \n",
    "        #Direction DOWN:\n",
    "        state[12] = self.nrows - self.headY\n",
    "        for index, i in enumerate(range (self.headY+1, self.nrows)):\n",
    "            if (self.board[i, self.headX] ==2):\n",
    "                state[13] = index + 1\n",
    "                break\n",
    "        for index, i in enumerate(range (self.headY+1, self.nrows)):\n",
    "            if (self.board[i, self.headX] ==1):\n",
    "                state[14] = index + 1\n",
    "                break\n",
    "                \n",
    "        #Direction Down-Left:\n",
    "        state[15] = min(self.nrows-self.headY, self.headX+1) \n",
    "        for index, i in enumerate(range (1,int(state[15]))):\n",
    "            if (self.board[self.headY+i, self.headX-i] ==2):\n",
    "                state[16] = index + 1\n",
    "                break\n",
    "        for index, i in enumerate(range (1,int(state[15]))):\n",
    "            if (self.board[self.headY+i, self.headX-i] ==1):\n",
    "                state[17] = index + 1\n",
    "                break\n",
    "        \n",
    "        #Direction LEFT:\n",
    "        state[18] = self.headX + 1 \n",
    "        for index, i in enumerate(range (self.headX-1, -1, -1)):\n",
    "            if (self.board[self.headY, i] ==2):\n",
    "                state[19] = index + 1\n",
    "                break\n",
    "        for index, i in enumerate(range (self.headX-1, -1, -1)):\n",
    "            if (self.board[self.headY, i] ==1):\n",
    "                state[20] = index + 1\n",
    "                break\n",
    "                \n",
    "        #Direction Up-Left:\n",
    "        state[21] = min(self.headY+1, self.headX+1) \n",
    "        for index, i in enumerate(range (1,int(state[21]))):\n",
    "            if (self.board[self.headY-i, self.headX-i] ==2):\n",
    "                state[22] = index + 1\n",
    "                break\n",
    "        for index, i in enumerate(range (1,int(state[21]))):\n",
    "            if (self.board[self.headY-i, self.headX-i] ==1):\n",
    "                state[23] = index + 1\n",
    "                break\n",
    "\n",
    "        return self.normaliseVision(state)\n",
    "    def normaliseVision (self,state):\n",
    "        for i in range (0, len(state)):\n",
    "            if (state[i] !=0): # If a vision input is 0, it means that we didn't find food or body, so 1/inf = 0 while preventing a division-by-0 exception from being raised\n",
    "                state[i] = 1/state[i]\n",
    "            # Instead of measuring distance, we are measuring proximity. 1 if next cell, 0.5 if 2 cells apart, 0.33 if 3 cells apart...\n",
    "        return state\n",
    "    def getFoodDirection(self): # Smelling sense\n",
    "        # Same order than action (0 north, 1 east, 2 south, 3 west). 1 if food is in that direction and 0 if not. Two will be activated at the same time if food is diagonally and 1 if there is a straight line in between head and food\n",
    "        foodDir = np.zeros((4))\n",
    "        if (self.foodY<self.headY):\n",
    "            foodDir[0] = 1 # North\n",
    "        if (self.foodX>self.headX):\n",
    "            foodDir[1] = 1 # East\n",
    "        if (self.foodY>self.headY):\n",
    "            foodDir[2] = 1 # South\n",
    "        if (self.foodX<self.headX):\n",
    "            foodDir[3] = 1 # West\n",
    "        return foodDir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            [(None, 24)]         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 256)          6400        input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 256)          65792       dense[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 128)          32896       dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 8)            1032        dense_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "input_2 (InputLayer)            [(None, 4)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_3 (InputLayer)            [(None, 4)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate (Concatenate)       (None, 16)           0           dense_3[0][0]                    \n",
      "                                                                 input_2[0][0]                    \n",
      "                                                                 input_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_4 (Dense)                 (None, 4)            68          concatenate[0][0]                \n",
      "==================================================================================================\n",
      "Total params: 106,188\n",
      "Trainable params: 106,188\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "stateInput = keras.Input(shape = (24))\n",
    "directionInput = keras.Input(shape=(4))\n",
    "foodInput = keras.Input (shape = (4)) # Specifies in which direction is the next apple (two-hot vector)\n",
    "\n",
    "dense1 = keras.layers.Dense(units = 256, activation = \"relu\") (stateInput)\n",
    "dense2 = keras.layers.Dense(units = 256, activation = \"relu\") (dense1)\n",
    "dense3 = keras.layers.Dense(units = 128, activation = \"relu\") (dense2)\n",
    "dense4 = keras.layers.Dense(units = 8, activation = \"relu\") (dense3)\n",
    "dense5 = keras.layers.Concatenate ()([dense4, directionInput,foodInput])\n",
    "\n",
    "outputLayer = keras.layers.Dense (units = 4) (dense5)\n",
    "\n",
    "model = keras.Model (inputs = [stateInput, directionInput, foodInput], outputs = outputLayer)\n",
    "model.compile(optimizer=keras.optimizers.Adam(lr=0.001),loss =\"mse\")\n",
    "\n",
    "print (model.summary())\n",
    "staticModel = keras.models.clone_model( model, input_tensors=None, clone_function=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0  Episode Reward:  22 .Steps: 223\n",
      "1  Episode Reward:  20 .Steps: 176\n",
      "2  Episode Reward:  23 .Steps: 165\n"
     ]
    }
   ],
   "source": [
    "# MATPLOTLIB is interactive in the new window:\n",
    "\n",
    "plt.ion()\n",
    "plt.show()\n",
    "\n",
    "snake = Snake (9, 9)\n",
    "\n",
    "\n",
    "numberEpisodes = 3\n",
    "\n",
    "discount = 0.99\n",
    "counter = 0\n",
    "\n",
    "rewards = []\n",
    "model.load_weights(\"Snake1.h5\")\n",
    "for episode in range (numberEpisodes):\n",
    "\n",
    "    done = False\n",
    "    episodeReward = 0\n",
    "    prevObs = None\n",
    "    \n",
    "    for step in range(1000):\n",
    "        if (step == 999): # In case it is gonna time out, it has to reset\n",
    "            snake.__init__(9,9)\n",
    "            break\n",
    "        if (done):\n",
    "            break\n",
    "        plt.clf()\n",
    "        \n",
    "        if (prevObs is None):\n",
    "            action = random.choice([0,1,2,3])\n",
    "        else:\n",
    "            previousObs = np.expand_dims(prevObs, axis=0)\n",
    "            previousDirec = np.expand_dims (prevDirections, axis = 0)\n",
    "            previousFoodDir = np.expand_dims(prevFoodDirections, axis = 0)\n",
    "            action = np.argmax(model.predict([previousObs,previousDirec,previousFoodDir]))\n",
    "\n",
    "        state, done, reward = snake.move(action)\n",
    "       \n",
    "        arrayToShow = np.copy(state) # If we do not create a copy, the board object would be changed\n",
    "        arrayToShow[snake.tailY, snake.tailX] = 4\n",
    "        arrayToShow[snake.headY,snake.headX]  = 3\n",
    "        \n",
    "        cmap = mpl.colors.ListedColormap([\"purple\", \"yellow\", \"mediumturquoise\", \"lightseagreen\",\"turquoise\"])\n",
    "        norm = mpl.colors.BoundaryNorm(np.arange(0,6, 1), cmap.N) \n",
    "\n",
    "        plt.imshow(arrayToShow, cmap=cmap, norm=norm)\n",
    "\n",
    "        plt.show()\n",
    "       \n",
    "        plt.gca().set_xticks([x-0.5 for x in np.arange(snake.ncols)], minor='true')\n",
    "        plt.gca().set_yticks([y-0.5 for y in np.arange(snake.nrows)], minor='true')\n",
    "        plt.grid(which=\"minor\")\n",
    "        plt.xticks([], [])\n",
    "        plt.yticks([], [])\n",
    "        \n",
    "        plt.pause(0.02)\n",
    "  \n",
    "        obs = snake.snakeVision()\n",
    "        obsDirections = snake.movingDirection\n",
    "        foodDirections = snake.getFoodDirection()\n",
    "\n",
    "        prevDirections = obsDirections\n",
    "        prevObs = obs\n",
    "        prevFoodDirections = foodDirections\n",
    "        episodeReward += reward\n",
    "        \n",
    "    print (episode, \" Episode Reward: \", episodeReward, \".Steps:\", step)\n",
    "    rewards.append(episodeReward)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.clf()\n",
    "plt.plot (list(range(len(rewards))), rewards)\n",
    "plt.plot (list(range(len(rewards))), [np.average(rewards)]*len(rewards))\n",
    "plt.title(\"Rewards obtained along the learning process\")\n",
    "plt.xlabel(\"Episode number\")\n",
    "plt.ylabel(\"Reward obtained\")\n",
    "plt.legend([\"Rewards\", \"Average: %.2f\" % np.average(rewards)])\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}