UC_MatrixRL.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import random\n",
    "from tqdm.notebook import tqdm\n",
    "import copy\n",
    "from scipy.stats import bernoulli\n",
    "import scipy\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Environment(object):\n",
    "    '''General RL environment'''\n",
    "\n",
    "    def __init__(self):\n",
    "        pass\n",
    "\n",
    "    def reset(self):\n",
    "        pass\n",
    "\n",
    "    def advance(self, action):\n",
    "        '''\n",
    "        Moves one step in the environment.\n",
    "        Args:\n",
    "            action\n",
    "        Returns:\n",
    "            reward - double - reward\n",
    "            newState - int - new state\n",
    "            pContinue - 0/1 - flag for end of the episode\n",
    "        '''\n",
    "        return 0, 0, 0\n",
    "\n",
    "def make_riverSwim(epLen=20, nState=5):\n",
    "    '''\n",
    "    Makes the benchmark RiverSwim MDP.\n",
    "    Args:\n",
    "        NULL - works for default implementation\n",
    "    Returns:\n",
    "        riverSwim - Tabular MDP environment '''\n",
    "    nAction = 2\n",
    "    R_true = {}\n",
    "    P_true = {}\n",
    "    states = {}\n",
    "    for s in range(nState):\n",
    "        states[(s)] = 0.0\n",
    "        for a in range(nAction):\n",
    "            R_true[s, a] = (0, 0)\n",
    "            P_true[s, a] = np.zeros(nState)\n",
    "\n",
    "    # Rewards\n",
    "    R_true[0, 0] = (5/1000, 0)\n",
    "    R_true[nState - 1, 1] = (1, 0)\n",
    "\n",
    "    # Transitions\n",
    "    for s in range(nState):\n",
    "        P_true[s, 0][max(0, s-1)] = 1.\n",
    "\n",
    "    for s in range(1, nState - 1):\n",
    "        P_true[s, 1][min(nState - 1, s + 1)] = 0.3\n",
    "        P_true[s, 1][s] = 0.6\n",
    "        P_true[s, 1][max(0, s-1)] = 0.1\n",
    "\n",
    "    P_true[0, 1][0] = 0.3\n",
    "    P_true[0, 1][1] = 0.7\n",
    "    P_true[nState - 1, 1][nState - 1] = 0.9\n",
    "    P_true[nState - 1, 1][nState - 2] = 0.1\n",
    "\n",
    "    riverSwim = TabularMDP(nState, nAction, epLen)\n",
    "    riverSwim.R = R_true\n",
    "    riverSwim.P = P_true\n",
    "    riverSwim.states = states\n",
    "    riverSwim.reset()\n",
    "\n",
    "    return riverSwim\n",
    "\n",
    "class TabularMDP(Environment):\n",
    "    '''\n",
    "    Tabular MDP\n",
    "    R - dict by (s,a) - each R[s,a] = (meanReward, sdReward)\n",
    "    P - dict by (s,a) - each P[s,a] = transition vector size S\n",
    "    '''\n",
    "\n",
    "    def __init__(self, nState, nAction, epLen):\n",
    "        '''\n",
    "        Initialize a tabular episodic MDP\n",
    "        Args:\n",
    "            nState  - int - number of states\n",
    "            nAction - int - number of actions\n",
    "            epLen   - int - episode length\n",
    "        Returns:\n",
    "            Environment object\n",
    "        '''\n",
    "\n",
    "        self.nState = nState\n",
    "        self.nAction = nAction\n",
    "        self.epLen = epLen\n",
    "\n",
    "        self.timestep = 0\n",
    "        self.state = 0\n",
    "\n",
    "        # Now initialize R and P\n",
    "        self.R = {}\n",
    "        self.P = {}\n",
    "        self.states = {}\n",
    "        for state in range(nState):\n",
    "            for action in range(nAction):\n",
    "                self.R[state, action] = (1, 1)\n",
    "                self.P[state, action] = np.ones(nState) / nState\n",
    "\n",
    "    def reset(self):\n",
    "        \"Resets the Environment\"\n",
    "        self.timestep = 0\n",
    "        self.state = 0\n",
    "\n",
    "    def advance(self,action):\n",
    "        '''\n",
    "        Move one step in the environment\n",
    "        Args:\n",
    "        action - int - chosen action\n",
    "        Returns:\n",
    "        reward - double - reward\n",
    "        newState - int - new state\n",
    "        episodeEnd - 0/1 - flag for end of the episode\n",
    "        '''\n",
    "        if self.R[self.state, action][1] < 1e-9:\n",
    "            # Hack for no noise\n",
    "            reward = self.R[self.state, action][0]\n",
    "        else:\n",
    "            reward = np.random.normal(loc=self.R[self.state, action][0],\n",
    "                                      scale=self.R[self.state, action][1])\n",
    "        #print(self.state, action, self.P[self.state, action])\n",
    "        newState = np.random.choice(self.nState, p=self.P[self.state, action])\n",
    "\n",
    "        # Update the environment\n",
    "        self.state = newState\n",
    "        self.timestep += 1\n",
    "\n",
    "        episodeEnd = 0\n",
    "        if self.timestep == self.epLen:\n",
    "            episodeEnd = 1\n",
    "            #newState = None\n",
    "            self.reset()\n",
    "\n",
    "        return reward, newState, episodeEnd\n",
    "\n",
    "    def argmax(self,b):\n",
    "        #print(b)\n",
    "        return np.random.choice(np.where(b == b.max())[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "class UC_MatrixRL(object):\n",
    "    def __init__(self,env,N):\n",
    "        self.env = env\n",
    "        #See Step 3 of Algorithm 1\n",
    "        self.N = N\n",
    "        #The dimensionality of phi(s,a), see Step 2 or Assumption 1\n",
    "        self.d1 = self.env.nState * self.env.nAction\n",
    "        #The dimensionality of psi(s'), see Step 2 or Assumption 1\n",
    "        self.d2 = self.env.nState\n",
    "        #Step 2\n",
    "        self.features_state_action = {(s,a): np.zeros(self.d1) for s in self.env.states.keys() for a in range(self.env.nAction)}\n",
    "        #Step 3\n",
    "        self.features_next_state = {(s): np.zeros(self.d2) for s in self.env.states.keys()}\n",
    "        # A hack for using numpy's linear algebra functions in Step 4\n",
    "        self.features_next_state_mat = np.identity(self.d2)\n",
    "        # Creates the Identity Matrix for a dictionary\n",
    "        self.createIdentity()\n",
    "        # Initialize our Q matrix\n",
    "        self.Q = {(h,s,a): 0.0 for h in range(self.env.epLen) for s in self.env.states.keys() \\\n",
    "                   for a in range(self.env.nAction)}\n",
    "        #Step 4, this step maybe unnecessary since we are using Sherman-Morrison\n",
    "        self.A = np.identity(self.d1)\n",
    "        #For use in the Sherman-Morrison Update\n",
    "        self.Ainv = np.linalg.inv(self.A)\n",
    "        #Step 4\n",
    "        self.M = np.zeros((self.d1,self.d2))\n",
    "        #See Assumptions 2,2' and Theorem 1, this equals 1 in the tabular case\n",
    "        self.C_phi = 1.0\n",
    "        # See Assumption 2'(Stronger Feature Regularity), and consider the case when v_1 = v_2 = ....\n",
    "        self.C_psi = np.sqrt(env.nState)\n",
    "        # See Theorem 1\n",
    "        self.C_M = 1.0\n",
    "        # See Theorem 1\n",
    "        self.C_psi_ = 1.0\n",
    "        # This value scales our confidence interval, must be > 0\n",
    "        self.c = 1.0\n",
    "        # For use in updating M_n, see Step 13 and Eqn (2)\n",
    "        self.sums = np.zeros((self.d1,self.d2))\n",
    "        #Creates K_psi, see Section 3.1: Estimating the core matrix.\n",
    "        self.createK()\n",
    "        self.lam = 1.0\n",
    "        self.delta = 1/self.N\n",
    "\n",
    "    def createK(self):\n",
    "        '''\n",
    "        A function that creates K_psi and (K_psi)^-1 in computing M_n\n",
    "        See Section 3.1: Estimating the core matrix\n",
    "        '''\n",
    "        self.K = np.zeros((self.d2,self.d2))\n",
    "        for s_ in self.env.states.keys():\n",
    "            self.K = self.K + np.outer(self.features_next_state[s_],self.features_next_state[s_])\n",
    "        self.Kinv = np.linalg.inv(self.K)\n",
    "\n",
    "    def act(self,s,h):\n",
    "        '''\n",
    "        A function that returns the argmax of Q given the state and timestep\n",
    "        '''\n",
    "        return self.env.argmax(np.array([self.Q[(h,s,a)] for a in range(self.env.nAction)]))\n",
    "\n",
    "    def createIdentity(self):\n",
    "        '''\n",
    "            A function that creates the Identity Matrix for a Dictionary\n",
    "        '''\n",
    "        i = 0\n",
    "        for key in self.features_state_action.keys():\n",
    "            self.features_state_action[key][i] = 1\n",
    "            i += 1\n",
    "        j = 0\n",
    "        for key in self.features_next_state.keys():\n",
    "            self.features_next_state[key][j] = 1\n",
    "            j += 1\n",
    "\n",
    "    def proj(self, x, lo, hi):\n",
    "        '''Projects the value of x into the [lo,hi] interval'''\n",
    "        return max(min(x,hi),lo)\n",
    "\n",
    "    def compute_Q(self,n):\n",
    "        '''\n",
    "        A function that computes the Optimisic Q-Values, see step 6 and Equations 4,8.\n",
    "        '''\n",
    "        Q = {(h,s,a): 0.0 for h in range(self.env.epLen) for s in self.env.states.keys() \\\n",
    "                   for a in range(self.env.nAction)}\n",
    "        V = {h: np.zeros(self.env.nState) for h in range(self.env.epLen + 1)}\n",
    "        for h in range(self.env.epLen-1,-1,-1):\n",
    "            for s in self.env.states.keys():\n",
    "                for a in range(self.env.nAction):\n",
    "                    #For use in computing Q, like in UCRL_VTR we have access to the true reward function.\n",
    "                    r = self.env.R[s,a][0]\n",
    "\n",
    "                    value = np.dot(np.matmul(np.dot(self.features_state_action[s,a].T,self.M),\\\n",
    "                            self.features_next_state_mat),V[h+1])\n",
    "\n",
    "                    bonus = 2 * self.C_psi * h * self.Beta(n) * np.dot(\\\n",
    "                           np.dot(self.features_state_action[s,a],self.Ainv),self.features_state_action[s,a])\n",
    "                    # Computing the optimistic Q-values as according to Eqn (8).\n",
    "                    Q[h,s,a] = self.proj(r+value+bonus,0,self.env.epLen)\n",
    "                V[h][s] = max(np.array([self.Q[(h,s,a)] for a in range(self.env.nAction)]))\n",
    "        self.Q = Q.copy()\n",
    "        \n",
    "    '''\n",
    "    def Beta_2(self,n):\n",
    "        \n",
    "        A function that computes Beta under the assumption that Theorem 2 holds, see equation 8\n",
    "        with the multiplicitive H term included this is no longer an optimal bonus.\n",
    "        \n",
    "        first = self.c*(self.C_M * self.C_psi_ ** 2)\n",
    "        second = np.log(self.N*self.env.epLen*self.C_phi)*self.d1\n",
    "        #The line of code below for an 'anytime' version - there are no theoretical garanuntees for this statement.\n",
    "        #second = np.log(n*self.env.epLen*self.C_phi)*self.d1\n",
    "        return first * second\n",
    "    '''\n",
    "        \n",
    "    def Beta(self,n):\n",
    "        '''\n",
    "        A function that return Beta_k according to Theorem 20.5 in Bandits book\n",
    "        Also, if you return np.sqrt(first + second) instead of first + second then\n",
    "        you get higher cumlative reward. However, in Theorem 19.2, Beta_n is already\n",
    "        under the square root.\n",
    "        '''\n",
    "        #Step 3\n",
    "        #Bonus as according to step 3\n",
    "        '''\n",
    "        return np.sqrt(16*pow(self.m_2,2)*pow(env.epLen,2)*self.d*np.log(1+env.epLen*k) \\\n",
    "            *np.log(pow(k+1,2)*env.epLen/self.delta)*np.log(pow(k+1,2)*env.epLen/self.delta))\n",
    "        '''\n",
    "        \n",
    "        #Confidence bound from Chapter 20 of the Bandit Algorithms book, see Theorem 20.5.\n",
    "        first = np.sqrt(self.lam)*np.sqrt(self.C_M*self.d1)\n",
    "        (sign, logdet) = np.linalg.slogdet(scipy.linalg.sqrtm(self.A))\n",
    "        #second = np.sqrt(2*np.log(1/self.delta) + self.d*np.log((self.d*self.lam + k*self.L*self.L)/(self.d*self.lam)))\n",
    "        det = sign * logdet\n",
    "        second = np.sqrt(2*np.log(1/self.delta) + np.log(n) + min(det,pow(10,10)) - np.log(pow(self.lam,self.d1)))\n",
    "        #print(det)\n",
    "        return first + second\n",
    "\n",
    "    def update_core_matrix(self,s,a,s_):\n",
    "        '''\n",
    "        A function that performs step 12 and 13.\n",
    "        '''\n",
    "        #While the line below is in Algorithm 1, when using the Sherman Morrison update it is not needed.\n",
    "        #self.A = self.A + np.outer(self.features_state_action[s,a],self.features_state_action[s,a])\n",
    "        #Sherman Morrison Update (Rich's book Eqn 9.22)\n",
    "        self.Ainv = self.Ainv - np.dot((np.outer(np.dot(self.Ainv,self.features_state_action[s,a]) \\\n",
    "                 ,self.features_state_action[s,a])),self.Ainv) / \\\n",
    "                    (1 + np.dot(np.dot(self.features_state_action[s,a],self.Ainv),self.features_state_action[s,a]))\n",
    "        #The summation step of Eqn (2)\n",
    "        self.sums = self.sums + np.outer(self.features_state_action[s,a],self.features_next_state[s_])\n",
    "        #Here (K_psi)^-1 was pre-computed during the initialization. See Eqn (2)\n",
    "        self.M = np.matmul(np.matmul(self.Ainv,self.sums),self.Kinv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = make_riverSwim(epLen = 20, nState = 4)\n",
    "N = 1000\n",
    "agent = UC_MatrixRL(env,N)\n",
    "R = 0\n",
    "Rvec = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "490a2dd36ca243a5afe6aedc237977ab",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1000.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "for n in tqdm(range(1,N+1)):\n",
    "    env.reset()\n",
    "    done = 0\n",
    "    while not done:\n",
    "        s = env.state\n",
    "        h = env.timestep\n",
    "        a = agent.act(s,h)\n",
    "        r,s_,done = env.advance(a)\n",
    "        R += r\n",
    "        agent.update_core_matrix(s,a,s_)\n",
    "    Rvec.append(R)\n",
    "    agent.compute_Q(n)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The 2-norm of (True Transition Probabiltites - Estimated Probabilities) is: 0.01641594675530863\n"
     ]
    }
   ],
   "source": [
    "true_p = []\n",
    "for values in env.P.values():\n",
    "    for value in values:\n",
    "        true_p.append(value)\n",
    "        \n",
    "estimated_p = []\n",
    "for values in agent.M:\n",
    "    for value in values:\n",
    "        estimated_p.append(value)\n",
    "        \n",
    "print('The 2-norm of (True Transition Probabiltites - Estimated Probabilities) is:',np.linalg.norm(np.subtract(true_p,estimated_p)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1a1f01abe0>]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXxU5dn/8c+VkIR9iSwGwm6UTRAZFrX1UXHBFa2iuFRUlGpttfaxFmtt61OtS7ef1sJT3ABFEHdUUClqrYpAEGQLS9jDkgQRCMHs1++PHPuMGCFAksnMfN+v17zmzD3nTq57knxz5j5nzjF3R0RE4kNCpAsQEZG6o9AXEYkjCn0RkTii0BcRiSMKfRGRONIg0gUcTOvWrb1Lly6RLkNEJKosXLhwh7u32b+93od+ly5dyMzMjHQZIiJRxcw2VtWu6R0RkTii0BcRiSMKfRGROKLQFxGJIwp9EZE4otAXEYkjCn0RkTii0BcRqWfW7yjkkbdXUlFR86e+V+iLiNQjK7fv4aonPuWFBZvZvqeoxr9+vf9ErohIPCgsLmPCh+v4+/vZpDZJ5tnRg2nfslGNfx+FvohIhBWVlnPp+E9Yub2A7x3Tmj9f3o92zRvWyveq1vSOmbU0s5fMbKWZZZnZSWaWamazzWxNcN8qbP27zSzbzFaZ2Tlh7QPMbGnw3GNmZrUxKBGRaPLw2ytZub2Ax67sz7OjB9Va4EP15/QfBd529x5APyALGAvMcfcMYE7wGDPrBYwEegPDgHFmlhh8nfHAGCAjuA2roXGIiEQdd+fRf67hmY83cPXgTlzUrz21vS180NA3s+bAqcBTQZEl7r4LGA5MClabBFwcLA8Hprl7sbuvB7KBQWaWBjR397leeTX2yWF9RETiSt6eIn46dRF//edqzj8+jd9d1LtOvm915vS7AfnAM2bWD1gI3A60c/dtAO6+zczaBut3AD4N658TtJUGy/u3f4uZjaHyHQGdOnWq9mBERKJB7p4ifjDuE/ILirltaAZ3nJlR61v4X6vO9E4D4ERgvLv3BwoJpnK+Q1WV+wHav93oPsHdQ+4eatPmW9cAEBGJWsu27Ob8xz5i174SXrrlJH5+1rF1FvhQvdDPAXLcfV7w+CUq/wnkBlM2BPd5Yet3DOufDmwN2tOraBcRiQs79hYzZnImxWXlPH3dQPqmt6zzGg4a+u6+HdhsZscFTUOBFcAMYFTQNgp4PVieAYw0sxQz60rlDtv5wVRQgZkNCY7auTasj4hITMsvKOaScR/zRWEJU28awuBuR0Wkjuoep/9TYIqZJQPrgOup/Icx3cxGA5uAEQDuvtzMplP5j6EMuNXdy4OvcwswEWgEzApuIiIxLb+gmFueW0junmKev3EwfTq0iFgtVnkgTf0VCoVc18gVkWi1p6iUy8Z/wsYv9vGnEf24sF/7Ovm+ZrbQ3UP7t+sTuSIitWRPUSljJmeyLr+QyTcM4uRjWke6JIW+iEhNW7+jkAUbdvL0R+tZub2ARy7rWy8CHxT6IiI1atbSbdw+bTEl5RU0S2nAX6/oxyX90w/esY4o9EVEaoC7M+6DtfzxnVX069iSP1zSh+5tmtIwKfHgneuQQl9EpAb877/W8cd3VjGoaypPXzeQpin1M17rZ1UiIlFk0aYv+dO7qxjW+2gev6o/DRLr7/Wp6m9lIiJRoKColNunLebo5g15+LK+9TrwQVv6IiKHrbC4jF+8uIScL/cx/Ucn0aJRUqRLOiiFvojIYViTW8BPnl/EqtwCfn1+T0JdUiNdUrUo9EVEDtHsFbnc/coSyiqcZ0cP4vsZ0XM2YIW+iEg1VVQ4f5m9msffz+aoJslMvH4QJ3Ss+zNlHgmFvohINXxVUs7/vLmCqfM3cUWoI/df0oeker7TtioKfRGRavjtjGVMz8xhxIB0Hrr0+Dq98ElNUuiLiBxARYVz/1tZTM/M4eb/6s7Yc3tEuqQjotAXEfkOJWUV3PzcQt5bmcd1J3fhzrOPjXRJR0yhLyJShbLyCn7/5greW5nHmFO7cfe5PaJ2SiecQl9EZD/uzk+nLmLWsu1cPbgTvzqvZ6RLqjEKfRGRgLvz2aZdzF27g1nLtnP9KV249/xekS6rRin0RUSAtfl7+d2M5fx7zQ4AQp1bcc95PUlIiP4pnXAKfRGJa0Wl5Tz+XjaT5m6goKiMMad247IB6XRt3aTenzztcCj0RSRubd9dxC1TFrJo0y6GdEvl/ov7cEzbZpEuq1Yp9EUk7uTuKeKhWSuZvSIXd+dvV/bnwn7tI11WnVDoi0hc+feafO54YTGFxeV8P6M1tw3NoE+HFpEuq85UK/TNbANQAJQDZe4eMrNU4AWgC7ABuNzdvwzWvxsYHax/m7u/E7QPACYCjYCZwO3u7jU3HBGRqpVXOJPnbuAPM7No0zSFiTcPiquw/9qh7KU43d1PcPdQ8HgsMMfdM4A5wWPMrBcwEugNDAPGmdnXVwYeD4wBMoLbsCMfgojIgZVXOHe++Dn3vbGC7m2a8uZt34/LwIcju1zicGBSsDwJuDisfZq7F7v7eiAbGGRmaUBzd58bbN1PDusjIlIrikrL+fVry3h10RbuOPNYZt3+fVKbJEe6rIipbug78K6ZLTSzMUFbO3ffBhDctw3aOwCbw/rmBG0dguX927/FzMaYWaaZZebn51ezRBGRb3J37nppCVPnb+LyUDq3n5kRE6dSOBLV3ZF7irtvNbO2wGwzW3mAdat6Rf0A7d9udJ8ATAAIhUKa8xeRwzLug7XM+HwrPz/rWG4bmhHpcuqFam3pu/vW4D4PeBUYBOQGUzYE93nB6jlAx7Du6cDWoD29inYRkRr3zvLt/PGdVQw/oT0/PeOYSJdTbxw09M2siZk1+3oZOBtYBswARgWrjQJeD5ZnACPNLMXMulK5w3Z+MAVUYGZDrPL91bVhfUREaoS787sZy7l1ymf0S2/Bw5f2jfspnXDVmd5pB7wavGgNgOfd/W0zWwBMN7PRwCZgBIC7Lzez6cAKoAy41d3Lg691C/93yOas4CYiUmPGfbCWiZ9s4JL+Hbj3gl40TEo8eKc4YvX9MPlQKOSZmZmRLkNE6rnS8grufW0Z0xZs5vzj0/jblf1j7mRph8LMFoYdYv8fsXc2IRGJSw+8lcW0BZu5bEA6j8V54B+ITsMgIlGtvMJ5cGYWEz/ZwOjvdeXeC2Lr/Pc1TaEvIlGrsLiM0ZMW8Om6nZx3/NHcHeUXLa8LCn0RiUoVFc4dLyxm/vqdPHzp8YwY0FFTOtWg0BeRqLOzsIT73ljOuyty+c0FvbhiYKdIlxQ1FPoiElU+WJXHQ7NWsiZvL7cNzeD6U7pEuqSootAXkajg7jw3bxP3vraMlo2TGHf1iZzT++hIlxV1FPoiEhUeeCuLJz9az+CuqTxz/UAaJyu+DodeNRGp19ydZz7ewJMfreeHQzrzu4t6k6gdtodNoS8i9Za789Cslfzjw3UM6ZbKby7spcA/Qgp9EamXCovL+P2bK5i2YDM/OLEDD1/al6REnUTgSCn0RaTeKSmr4PpnFjB/w05+fFp3fnHOcTpTZg1R6ItIvbF7XymPzlnDa4u3sLOwhEcu7cvlAzsevKNUm0JfROqFDTsKuWLCXHL3FBPq3IoHLu7DucenRbqsmKPQF5GI27WvhJsmZ1JUWsEz1w3k9B5tD95JDotCX0Qi6q0l27jvjeXs2lfKxBsGcnL31pEuKaZpV7iIRMxzn27k1uc/IykxgeduHKzArwPa0heRiHh+3iZ+/doyTj+uDX+76kSapiiO6oJeZRGpU2XlFUyau5Hfv7mC049rwz9+GCK5gSYd6opCX0TqhLtz/1tZPD9vE1+VlvP9jNaMv2aAAr+OKfRFpNblFxRzz6tLeXdFLmf1asfZvdpxSf8ONNAnbOucQl9Eao2788aSbfz29WUUlpRzz3k9ufH7XfXp2ghS6ItIrSivcH7z+jKmzNtEv44t+fOIvhzTtlmky4p71X5vZWaJZrbIzN4MHqea2WwzWxPctwpb924zyzazVWZ2Tlj7ADNbGjz3mOnfvUhMWpKzixsmLmDKvE386L+68fLNJynw64lDmVC7HcgKezwWmOPuGcCc4DFm1gsYCfQGhgHjzCwx6DMeGANkBLdhR1S9iNQrWdv2cNn4T7jo8Y+Zu/YL7hp2HHef21Nz9/VItX4SZpYOnA88GdY8HJgULE8CLg5rn+buxe6+HsgGBplZGtDc3ee6uwOTw/qISJT7YFUeoycuYPnWPdxx5rHMv2coPz7tmEiXJfup7pz+/wPuAsLfn7Vz920A7r7NzL4+WUYH4NOw9XKCttJgef/2bzGzMVS+I6BTJ13lXqQ+Kyot54aJC/hk7Rc0b9iAF28+iT4dWkS6LPkOB93SN7MLgDx3X1jNr1nVPL0foP3bje4T3D3k7qE2bdpU89uKSF1bnVvAqKfn88naL7j19O58+quhCvx6rjpb+qcAF5nZeUBDoLmZPQfkmllasJWfBuQF6+cA4SfATge2Bu3pVbSLSBT654pcfvbCYszg1+f35Mbvd4t0SVINB93Sd/e73T3d3btQuYP2PXe/BpgBjApWGwW8HizPAEaaWYqZdaVyh+38YCqowMyGBEftXBvWR0SiyPQFm7n5uYV0Pqox795xqgI/ihzJcfoPAdPNbDSwCRgB4O7LzWw6sAIoA2519/Kgzy3ARKARMCu4iUgUeXvZdn75yhKGdD2KcVefSKsmyZEuSQ6BVR5IU3+FQiHPzMyMdBkiAizfupsR/zuXjHbNmHbTEBolJx68k0SEmS1099D+7Tp4VkSqZcGGnYz8x6c0TWnAhB8OUOBHKZ2GQUQOqKColLteWsKsZdvp0LIR08YMoV3zhpEuSw6TQl9EvlPOl/u4cVImK7cXMGJAOr88twetm6ZEuiw5Agp9EanSzsISrnlyHl8UljDx+oGcdpwuVh4LFPoi8i1FpeXcNDmTrbuLmHrTYAZ0To10SVJDtCNXRL5h++4ixjy7kIUbv+Svl5+gwI8x2tIXEQB2f1XKn99dxbOfbiQpMYEHf3A85/dNi3RZUsMU+iLC6twCrn9mAdt2f8UVoY7cdGo3urdpGumypBYo9EXi3OuLt/CLF5fQonESL99yMv07tTp4J4laCn2ROPbqohzufHEJJ3RsyeNX9SetRaNIlyS1TKEvEocKi8v4w8wspszbRP9OLZl0wyCapigO4oF+yiJxJq+giOufWcCKbXu47uQujD23Bw2TdEqFeKHQF4kTJWUVvJC5mQdnZlFSVsGjI/tzUb/2kS5L6phCXyQO7NhbzFVPfMrq3L30TGvOI5f25fh0XeEqHin0RWLcp+u+4EfPLqS4rJzfXdiLywd2pHGy/vTjlX7yIjFs/vqd3DBxAa0aJ/P0dSF9ulYU+iKxyN15Y8k2fjZtEa2bpvDqj0+mrU6HLCj0RWLOuvy9/GFmFv/MyqNnWnOeuHaAAl/+Q6EvEiMKikp56qP1PP3Rehz4yenH8JMzjtHhmPINCn2RGLBsy25unJRJbkERg7qk8qcR/eiY2jjSZUk9pNAXiXKLNn3Jj55dSIME47Ufn0K/ji0jXZLUYwp9kSj2SfYOrn5qHikNEnj5lpPp3V7H3suBKfRFotAXe4uZnpnD395bQ6fUxky9aQjtW+pkaXJwB71ylpk1NLP5Zva5mS03s/uC9lQzm21ma4L7VmF97jazbDNbZWbnhLUPMLOlwXOPmZnVzrBEYtea3AIuevxjHn57Jccd3YzpPzpJgS/VVp3LJRYDZ7h7P+AEYJiZDQHGAnPcPQOYEzzGzHoBI4HewDBgnJl9ffjAeGAMkBHchtXgWERi3vKtu7l+4gL2Fpcx+YZBvHzzybTT4ZhyCA4a+l5pb/AwKbg5MByYFLRPAi4OlocD09y92N3XA9nAIDNLA5q7+1x3d2ByWB8ROYiXF+Zw4d8+oqSsgonXD+TUY9uQkKA3y3JoqnVhdDNLNLPFQB4w293nAe3cfRtAcN82WL0DsDmse07Q1iFY3r+9qu83xswyzSwzPz//UMYjEpPeX5nH2FeWEOqcypu3fU9Xt5LDVq3Qd/dydz8BSKdyq73PAVavatPDD9Be1feb4O4hdw+1adOmOiWKxKSi0nIeeGsFN07OpHubpjxxbYi2zTSdI4fvkI7ecfddZvYBlXPxuWaW5u7bgqmbvGC1HKBjWLd0YGvQnl5Fu4hUobzC+cVLS3jj862c1asdf73iBF3dSo5YdY7eaWNmLYPlRsCZwEpgBjAqWG0U8HqwPAMYaWYpZtaVyh2284MpoAIzGxIctXNtWB8RCVNWXsHPpy/mjc+38sthPXji2pACX2pEdX6L0oBJwRE4CcB0d3/TzOYC081sNLAJGAHg7svNbDqwAigDbnX38uBr3QJMBBoBs4KbiISZv34nY19Zwrr8Qu4adhy3nNY90iVJDLHKA2nqr1Ao5JmZmZEuQ6TWLc3ZzeS5G3hl0RaaJCfy4A/6cn7ftEiXJVHKzBa6e2j/dr1fFKkH3lqyjZ9M/YyGDRI57/g0HrikD80bJkW6LIlBCn2RCMratofH38/mrSXb6NexJc+OHqSwl1ql0BeJgNLyCsZ/sJbH5qzBgVtP785Pz8jQue+l1in0ReqYu3P7tEXMXLqd4Se053cX9qZVk+RIlyVxQqEvUsdmfL6VmUu3c+fZx/KTMzIiXY7EGYW+SB366+zVPP5+Nv07teSW046JdDkSh6p1GgYROXKvLsrh0TlruKBvGk+PGkiiTpYmEaAtfZFa5u5MW7CZ385YzqCuqfx5RD8aJGp7SyJDoS9SS9ydtfmFPDgzizkr8zixU0sev7K/Al8iSqEvUgs279zHHS8sJnPjlyQlGrec1p27zjkOXSxOIk2hL1LD/r0mn9umLqKswrnz7GMZ1udojmnbLNJliQAKfZEa81VJOX9/P5txH2RzTNum/OOHIbq2bhLpskS+QaEvUgP+tTqfO15YzM7CEi7om8bDl/aliU6FLPWQfitFjoC7M3Ppdm6btoj2LRvy0A8GcHbvoyNdlsh3UuiLHKaSsgp+OvUz3lmeS9/0Fky9aYi27qXe02+oyCHasbeYNz7fytMfr2fzzq+47uQujD23h06WJlFBoS9STWXlFcxZmccDb2Wxaec+WjdN5rEr+3NRv/aRLk2k2hT6ItXw4ep8xr68hK27izi6eUNeuvkk+nVsSZI+aCVRRqEvcgDuzu9mLGfS3I00a9iAR0eewLA+R5PSQFM5Ep0U+iLfIa+giPvfzGLG51u5clBHxg7rSYvGuqqVRDeFvkgVPs7ewU+nLmJvcRm3Dc3gjjMzdAoFiQkKfZH9bNhRyC3PLaRNsxQm3zCIPh1aRLokkRqj0BcJsyRnFzdNziQhwZh4/SA6pjaOdEkiNeqghx6YWUcze9/MssxsuZndHrSnmtlsM1sT3LcK63O3mWWb2SozOyesfYCZLQ2ee8z0flnqifIKZ9wH2Vz0+MeUVzhPjRqowJeYVJ3jzcqA/3b3nsAQ4FYz6wWMBea4ewYwJ3hM8NxIoDcwDBhnZl8f6jAeGANkBLdhNTgWkcOSV1DEyAlzeeTtVZzZsx2v3XoKAzq3OnhHkSh00Okdd98GbAuWC8wsC+gADAdOC1abBHwA/DJon+buxcB6M8sGBpnZBqC5u88FMLPJwMXArBocj8gh+feafO588XMKisr4y+X9uKR/B+2wlZh2SJ8sMbMuQH9gHtAu+Ifw9T+GtsFqHYDNYd1ygrYOwfL+7VV9nzFmlmlmmfn5+YdSoki1uDvTF2zmumcWYBiTbhjED05MV+BLzKv2jlwzawq8DPzM3fcc4I+jqif8AO3fbnSfAEwACIVCVa4jcrgKi8v48ZTP+NfqfAZ1SeXJ60I0b6jj7yU+VCv0zSyJysCf4u6vBM25Zpbm7tvMLA3IC9pzgI5h3dOBrUF7ehXtInVi8859vPLZFt5aupXsvL3cPjSDH5/eXZ+ulbhSnaN3DHgKyHL3v4Q9NQMYFSyPAl4Pax9pZilm1pXKHbbzgymgAjMbEnzNa8P6iNSqFVv3cOHjH/HXf65mb1EZE68fxB1nHavAl7hTnS39U4AfAkvNbHHQ9ivgIWC6mY0GNgEjANx9uZlNB1ZQeeTPre5eHvS7BZgINKJyB6524kqtcncmfrKBP8zMolXjZD648zS66BKGEsfMvX5PmYdCIc/MzIx0GRKF3J0HZ61kwofrOOWYo3h0ZH9aN02JdFkidcLMFrp7aP92fSJXYlJJWQW/fm0p0zNzuHpwJ+67qDcNdBpkEYW+xJZ9JWW8/NkWHv3nanbsLeHG73XlnvN76lBMkYBCX2LGrn0ljJzwKSu3F9AzrTl3ndODESEdey8STqEvMaGotJwbJ2WyLr+Qp0aFOKNHW4W9SBUU+hL1CovLuPm5hWRu/JK/X3UiQ3u2i3RJIvWW9mxJVCuvcO54YTEfZ+/ggUv6cH7ftEiXJFKvaUtfotamL/Zxx/TFLNz4Jb+9sBdXD+4c6ZJE6j2FvkSlJ/+9jvvfyqJJciL3X9yHa4Yo8EWqQ6EvUef9lXk8MDOL045rw28u6EW3Nk0jXZJI1FDoS9Qor3CmzNvI/7yxgp5HN2f81QNolKxz54gcCoW+1HsFRaUsydnNlHkbmbl0Oxltm/KPHyrwRQ6HQl/qrQ07Cvnju6v4cHU+BUVlAFwzpBO/H95Hx+CLHCaFvtQ7m3fu497Xl/HBqnyaJCcytGc7zul9NAM6t+LoFg0jXZ5IVFPoS70yb90X3DBxAaXlzphTu3HDKV0V9CI1SKEv9ULWtj386tWlLNq0i/YtGjJ59GCOaaujckRqmkJfIqq4rJyxLy/l1UVbSGmQwKiTOvOj/+pO+5aNIl2aSExS6EvEbNn1FXdO/5y5677gykEdueOsY2nbTFM5IrVJoS8R8c7y7fz39M9xdx65tC+XD+wY6ZJE4oJCX+pUcVk5D85cycRPNpDeqhETrx+kuXuROqTQlzrh7izevIsHZ65k/oadXDmoE2OH9aBF46RIlyYSVxT6Uuuy8wr47xeX8PnmXbRolMQfLjmeqwZ3inRZInFJoS+1prisnL+/l834f62lWcMkfnVeD64c1IlmDbV1LxIpCn2pFau2F3DDxAVs2fUVpx3Xhl+f30tz9yL1wEGvnGVmT5tZnpktC2tLNbPZZrYmuG8V9tzdZpZtZqvM7Jyw9gFmtjR47jHTyVNiUll5Bfe8upRhj35I/t5i/jSiH89cN1CBL1JPVOdyiROBYfu1jQXmuHsGMCd4jJn1AkYCvYM+48zs61MhjgfGABnBbf+vKVFu4cYvuex/5zJl3iaG9mjL7DtO5bIB6To5mkg9ctDpHXf/0My67Nc8HDgtWJ4EfAD8Mmif5u7FwHozywYGmdkGoLm7zwUws8nAxcCsIx6B1AuzV+Qy5tlMGiXpSlYi9dnhzum3c/dtAO6+zczaBu0dgE/D1ssJ2kqD5f3bq2RmY6h8V0CnTjrKoz4rKavgyY/W8Zd3V9OnfQumjhlC0xTtKhKpr2r6r7Oq9/F+gPYqufsEYAJAKBT6zvUksgqLy7jyiU9ZkrObs3u1448j+inwReq5w/0LzTWztGArPw3IC9pzgPDP06cDW4P29CraJUoVlZZz6/OfsWzLbn5/cR+uGdxJc/ciUaA6O3KrMgMYFSyPAl4Pax9pZilm1pXKHbbzg6mgAjMbEhy1c21YH4kyu78q5eon5/HBqnweuOR4fjikswJfJEocdEvfzKZSudO2tZnlAL8FHgKmm9loYBMwAsDdl5vZdGAFUAbc6u7lwZe6hcojgRpRuQNXO3Gj0Podhfzk+c9YnVvA41f154K+7SNdkogcAnOv31PmoVDIMzMzI11G3HN3Zny+ld+8vpySsgrGXX0ip/doe/COIhIRZrbQ3UP7t2uvmxxUdl4Bf5m9mplLt3Ncu2Y8OSpEx9TGkS5LRA6DQl++0/odhfzp3VW8tWQbAKO/15Wx5/YgKfFwdwWJSKQp9OVb9hSV8uDMLF7MzKHcncsGpHPbGRl0Okpb9yLRTqEv37B8625+8eISVmzbw/l90/jZ0Awy2jWLdFkiUkMU+gLArn0lvLs8l9/MWEbTlCSeuDbEWb3aRbosEalhCn3hk+wd/OyFxeQVFJPRtilPjgrR+agmkS5LRGqBQj+O5Xy5j/EfrGXKvE10Sm3Ms6MHMaTbUdpRKxLDFPpxKndPET8Y9wl5BcVcPbgTY8/toStaicQBhX4cyvlyH5eNn8vur0p56eaTCHVJjXRJIlJH9D4+znycvYMr/vEp+0rKmHLjYAW+SJzRln6ccHf+580VPPPxBjqlNua5GwfTN71lpMsSkTqm0I8DS3N289DbWXyc/QWXnpjOfcN767z3InFKf/kx7r2Vudz87GckJMDtQzP42ZkZOg2ySBxT6MewT9bu4MdTPuO4o5vxxLUhjm7RMNIliUiEKfRj1EdrdnDj5AV0Sm3MM9cPpHXTlEiXJCL1gEI/hhSXlTNv3U5mLt3Gy5/l0L1NU6bcOJijFPgiElDox4CvL3Dy+zez2LG3mIZJCYwIdeTOs48jtUlypMsTkXpEoR/lNuwo5N7Xl/HvNTvol96C+y/uzUndW9OikT5dKyLfptCPUiVlFUz4cC2PvZdNcmIC913Um2uGdCYxQUfmiMh3U+hHmbw9RcxZmceT/17H2vxCzj8+jd9c2It2zXVkjogcnEK/nistr2Dhxi95ffEWZq/IZcfeEgC6HNWYZ64bqIuTi8ghUejXQ3uKSvls45dkbSvghQWb2PDFPhomJfC9Y1ozoHMqg7ul0rdDCxroFMgicogU+hGw+6tSlm/ZTc6ur8jatoe1+YV8VVLGuvxCCorKKCmv+M+6fdNb8MhlfTnv+DSdOkFEjlidp4iZDQMeBRKBJ939obquoTa5Ozv2lrAufy+rcwsoKC5j594SSsor2FtcxqYv9rF0y26KyyqDPSnR6JnWnEZJiQzt2ZbUJik0SkpkQOdWHNuuKW2apei0CSJSY+o09M0sEfg7cBaQAywwsxnuvqIu69ifu1NcVkFpeQUlZRUUFrjUjWsAAAWDSURBVJezr7SMigqocKe8wil3x90pr4DC4jJ2f1VKcVk5m3d+xd7iMvYFW+qrcgsoKCr7xtdvlJRISlICjZIS6ZjamCsGduSsXu3onNqENs1SaJScGKGRi0i8qest/UFAtruvAzCzacBwoMZDf/TEBaz/ohB3KK9wKty/sVx5g7LyCvbsF9KHIsGgaUoDGiYl0uWoJgw/oT3d2zSlS+smHNuuGS0bJdFE0zIiUk/UdRp1ADaHPc4BBu+/kpmNAcYAdOrU6bC+UZfWTWiUnEiCGQkGCQlGghmJZiQkELQbiQlG84YNSElKJDkxgeQGCTROTqRJSoP/9E1MMBISgr5mNEpOoGXjZFIaJJDaJJnGyQp1EYkOdZ1WVU1O+7ca3CcAEwBCodC3nq+Oey/odTjdRERiWl0f85cDdAx7nA5sreMaRETiVl2H/gIgw8y6mlkyMBKYUcc1iIjErTqd3nH3MjP7CfAOlYdsPu3uy+uyBhGReFbneyDdfSYws66/r4iI1P30joiIRJBCX0Qkjij0RUTiiEJfRCSOmPthffapzphZPrDxMLu3BnbUYDnRQGOODxpzfDiSMXd29zb7N9b70D8SZpbp7qFI11GXNOb4oDHHh9oYs6Z3RETiiEJfRCSOxHroT4h0ARGgMccHjTk+1PiYY3pOX0REvinWt/RFRCSMQl9EJI7EZOib2TAzW2Vm2WY2NtL11BQz62hm75tZlpktN7Pbg/ZUM5ttZmuC+1Zhfe4OXodVZnZO5Ko/MmaWaGaLzOzN4HFMj9nMWprZS2a2Mvh5nxQHY74j+L1eZmZTzaxhrI3ZzJ42szwzWxbWdshjNLMBZrY0eO4xM6vqAlVV8+CC37Fyo/KUzWuBbkAy8DnQK9J11dDY0oATg+VmwGqgF/AIMDZoHws8HCz3CsafAnQNXpfESI/jMMf+c+B54M3gcUyPGZgE3BgsJwMtY3nMVF5KdT3QKHg8Hbgu1sYMnAqcCCwLazvkMQLzgZOovBrhLODc6tYQi1v6/7n4uruXAF9ffD3qufs2d/8sWC4Asqj8YxlOZUgQ3F8cLA8Hprl7sbuvB7KpfH2iipmlA+cDT4Y1x+yYzaw5leHwFIC7l7j7LmJ4zIEGQCMzawA0pvKqejE1Znf/ENi5X/MhjdHM0oDm7j7XK/8DTA7rc1CxGPpVXXy9Q4RqqTVm1gXoD8wD2rn7Nqj8xwC0DVaLldfi/wF3ARVhbbE85m5APvBMMKX1pJk1IYbH7O5bgD8Bm4BtwG53f5cYHnOYQx1jh2B5//ZqicXQr9bF16OZmTUFXgZ+5u57DrRqFW1R9VqY2QVAnrsvrG6XKtqiasxUbvGeCIx39/5AIZVv+79L1I85mMceTuU0RnugiZldc6AuVbRF1Zir4bvGeERjj8XQj+mLr5tZEpWBP8XdXwmac4O3fAT3eUF7LLwWpwAXmdkGKqfqzjCz54jtMecAOe4+L3j8EpX/BGJ5zGcC6909391LgVeAk4ntMX/tUMeYEyzv314tsRj6MXvx9WAP/VNAlrv/JeypGcCoYHkU8HpY+0gzSzGzrkAGlTuAooa73+3u6e7ehcqf5Xvufg2xPebtwGYzOy5oGgqsIIbHTOW0zhAzaxz8ng+lcp9VLI/5a4c0xmAKqMDMhgSv1bVhfQ4u0nuza2kP+XlUHtmyFrgn0vXU4Li+R+XbuCXA4uB2HnAUMAdYE9ynhvW5J3gdVnEIe/jr4w04jf87eiemxwycAGQGP+vXgFZxMOb7gJXAMuBZKo9aiakxA1Op3GdRSuUW++jDGSMQCl6ntcDjBGdXqM5Np2EQEYkjsTi9IyIi30GhLyISRxT6IiJxRKEvIhJHFPoiInFEoS8iEkcU+iIiceT/AwzG4mbZXPGoAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(Rvec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}