LSVI_PHE_old.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 363,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import random\n",
    "from tqdm.notebook import tqdm\n",
    "import copy\n",
    "from scipy.stats import bernoulli\n",
    "import matplotlib.pyplot as plt\n",
    "import scipy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 364,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Environment(object):\n",
    "    '''General RL environment'''\n",
    "\n",
    "    def __init__(self):\n",
    "        pass\n",
    "\n",
    "    def reset(self):\n",
    "        pass\n",
    "\n",
    "    def advance(self, action):\n",
    "        '''\n",
    "        Moves one step in the environment.\n",
    "        Args:\n",
    "            action\n",
    "        Returns:\n",
    "            reward - double - reward\n",
    "            newState - int - new state\n",
    "            pContinue - 0/1 - flag for end of the episode\n",
    "        '''\n",
    "        return 0, 0, 0\n",
    "\n",
    "\n",
    "def make_riverSwim(epLen=20, nState=5):\n",
    "    '''\n",
    "    Makes the benchmark RiverSwim MDP code by Ian Osband.\n",
    "    Args:\n",
    "        NULL - works for default implementation\n",
    "    Returns:\n",
    "        riverSwim - Tabular MDP environment '''\n",
    "    nAction = 2\n",
    "    R_true = {}\n",
    "    P_true = {}\n",
    "    states = {}\n",
    "    for s in range(nState):\n",
    "        states[(s)] = 0.0\n",
    "        for a in range(nAction):\n",
    "            R_true[s, a] = (0, 0)\n",
    "            P_true[s, a] = np.zeros(nState)\n",
    "\n",
    "    # Rewards\n",
    "    R_true[0, 0] = (5/1000, 0)\n",
    "    R_true[nState - 1, 1] = (1, 0)\n",
    "\n",
    "    # Transitions\n",
    "    for s in range(nState):\n",
    "        P_true[s, 0][max(0, s-1)] = 1.\n",
    "\n",
    "    for s in range(1, nState - 1):\n",
    "        P_true[s, 1][min(nState - 1, s + 1)] = 0.3\n",
    "        P_true[s, 1][s] = 0.6\n",
    "        P_true[s, 1][max(0, s-1)] = 0.1\n",
    "\n",
    "    P_true[0, 1][0] = 0.3\n",
    "    P_true[0, 1][1] = 0.7\n",
    "    P_true[nState - 1, 1][nState - 1] = 0.9\n",
    "    P_true[nState - 1, 1][nState - 2] = 0.1\n",
    "\n",
    "    riverSwim = TabularMDP(nState, nAction, epLen)\n",
    "    riverSwim.R = R_true\n",
    "    riverSwim.P = P_true\n",
    "    riverSwim.states = states\n",
    "    riverSwim.reset()\n",
    "\n",
    "    return riverSwim\n",
    "\n",
    "\n",
    "\n",
    "class TabularMDP(Environment):\n",
    "    '''\n",
    "    Tabular MDP\n",
    "    R - dict by (s,a) - each R[s,a] = (meanReward, sdReward)\n",
    "    P - dict by (s,a) - each P[s,a] = transition vector size S\n",
    "    '''\n",
    "\n",
    "    def __init__(self, nState, nAction, epLen):\n",
    "        '''\n",
    "        Initialize a tabular episodic MDP\n",
    "        Args:\n",
    "            nState  - int - number of states\n",
    "            nAction - int - number of actions\n",
    "            epLen   - int - episode length\n",
    "        Returns:\n",
    "            Environment object\n",
    "        '''\n",
    "\n",
    "        self.nState = nState\n",
    "        self.nAction = nAction\n",
    "        self.epLen = epLen\n",
    "\n",
    "        self.timestep = 0\n",
    "        self.state = 0\n",
    "\n",
    "        # Now initialize R and P\n",
    "        self.R = {}\n",
    "        self.P = {}\n",
    "        self.states = {}\n",
    "        for state in range(nState):\n",
    "            for action in range(nAction):\n",
    "                self.R[state, action] = (1, 1)\n",
    "                self.P[state, action] = np.ones(nState) / nState\n",
    "\n",
    "    def reset(self):\n",
    "        \"Resets the Environment\"\n",
    "        self.timestep = 0\n",
    "        self.state = 0\n",
    "\n",
    "    def advance(self,action):\n",
    "        '''\n",
    "        Move one step in the environment\n",
    "        Args:\n",
    "        action - int - chosen action\n",
    "        Returns:\n",
    "        reward - double - reward\n",
    "        newState - int - new state\n",
    "        episodeEnd - 0/1 - flag for end of the episode\n",
    "        '''\n",
    "        if self.R[self.state, action][1] < 1e-9:\n",
    "            # Hack for no noise\n",
    "            reward = self.R[self.state, action][0]\n",
    "        else:\n",
    "            reward = np.random.normal(loc=self.R[self.state, action][0],\n",
    "                                      scale=self.R[self.state, action][1])\n",
    "        #print(self.state, action, self.P[self.state, action])\n",
    "        newState = np.random.choice(self.nState, p=self.P[self.state, action])\n",
    "\n",
    "        # Update the environment\n",
    "        self.state = newState\n",
    "        self.timestep += 1\n",
    "\n",
    "        episodeEnd = 0\n",
    "        if self.timestep == self.epLen:\n",
    "            episodeEnd = 1\n",
    "            #newState = None\n",
    "            self.reset()\n",
    "\n",
    "        return reward, newState, episodeEnd\n",
    "\n",
    "    def argmax(self,b):\n",
    "        #print(b)\n",
    "        return np.random.choice(np.where(b == b.max())[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RLSVI(object):\n",
    "    def __init__(self,env,K,sigma):\n",
    "        self.env = env\n",
    "        self.K = K\n",
    "        self.sigma = sigma\n",
    "        self.S = self.env.nState\n",
    "        self.A = self.env.nAction\n",
    "        self.H = self.env.epLen\n",
    "        self.d = self.S * self.A\n",
    "        self.Q = np.zeros((self.H+1,self.S,self.A))\n",
    "        self.theta = np.zeros((self.H,self.d))\n",
    "        self.lam = 0.01\n",
    "        self.G = np.zeros((self.H,self.d,self.d))\n",
    "        self.phi = np.identity(self.d)\n",
    "        self.b = np.zeros((self.K,self.H,4))\n",
    "        self.I = np.identity(self.d)\n",
    "        \n",
    "    \n",
    "    def act(self,s,h):\n",
    "        return self.env.argmax(self.Q[h,s,:])\n",
    "    \n",
    "    def update_buffer(self,s,a,r,s_,k,h):\n",
    "        self.b[k,h,0] = s\n",
    "        self.b[k,h,1] = a\n",
    "        self.b[k,h,2] = r\n",
    "        self.b[k,h,3] = s_\n",
    "    \n",
    "    def update(self,k):\n",
    "        for h in range(self.H-1,-1,-1):\n",
    "            \n",
    "            self.sums_1 = np.zeros((self.d,self.d))\n",
    "            self.sums_2 = np.zeros(self.d)\n",
    "            for l in range(k):\n",
    "                #Z = np.random.binomial(m_h,0.5)\n",
    "                s,a,r,s_ = self.b[l,h,0],self.b[l,h,1],self.b[l,h,2],self.b[l,h,3]\n",
    "                row = int(s*self.A + a)\n",
    "                self.sums_1 = self.sums_1 + np.outer(self.phi[row,:],self.phi[row,:])\n",
    "                self.sums_2 = self.sums_2 + self.phi[row,:]*(r + max(self.Q[h+1,int(s_),:]))\n",
    "            self.G[h] = 1/pow(self.sigma,2)*self.sums_1 + self.lam*self.I\n",
    "            Ginv = np.linalg.inv(self.G[h])\n",
    "            temp = 1/pow(self.sigma,2)*np.dot(Ginv,self.sums_2)\n",
    "            self.theta[h] = np.random.multivariate_normal(temp,Ginv)\n",
    "            for ss in range(self.S):\n",
    "                for aa in range(self.A):\n",
    "                    row = int(ss*self.A + aa)\n",
    "                    self.Q[h,ss,aa] = max(0,min(np.inner(self.theta[h],self.phi[row,:]),self.H))\n",
    "    \n",
    "    def run(self):\n",
    "        Rvec = []\n",
    "        R = 0\n",
    "        for k in tqdm(range(self.K)):\n",
    "            self.env.reset()\n",
    "            done = 0\n",
    "            while not done:\n",
    "                s = self.env.state\n",
    "                h = self.env.timestep\n",
    "                a = self.act(s,h)\n",
    "                r,s_,done = self.env.advance(a)\n",
    "                self.update_buffer(s,a,r,s_,k,h)\n",
    "                R += r\n",
    "            Rvec.append(R)\n",
    "            self.update(k)\n",
    "        return Rvec\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 536,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LSVI_PHE(object):\n",
    "    def __init__(self,env,K,v):\n",
    "        self.env = env\n",
    "        self.K = K\n",
    "        self.v = v\n",
    "        self.S = self.env.nState\n",
    "        self.A = self.env.nAction\n",
    "        self.H = self.env.epLen\n",
    "        self.d = self.S * self.A\n",
    "        self.Q = np.zeros((self.H+1,self.S,self.A)) \n",
    "        self.Q[self.H] = 0.0\n",
    "        self.theta = np.zeros((self.H,self.d))\n",
    "        self.lam = 0.01 \n",
    "        self.G = np.zeros((self.H,self.d,self.d))\n",
    "        self.phi = np.identity(self.d)\n",
    "        self.b = np.zeros((self.K,self.H,4))\n",
    "        self.I = np.identity(self.d)\n",
    "        \n",
    "    \n",
    "    def act(self,s,h):\n",
    "        c = np.random.uniform()\n",
    "        if c < 1.0:\n",
    "            return self.env.argmax(self.Q[h,s,:])\n",
    "        else:\n",
    "            return 1\n",
    "    \n",
    "    def update_buffer(self,s,a,r,s_,k,h):\n",
    "        self.b[k,h,0] = s\n",
    "        self.b[k,h,1] = a\n",
    "        self.b[k,h,2] = r\n",
    "        self.b[k,h,3] = s_\n",
    "    \n",
    "    def update(self,k):\n",
    "        #print(self.Q)\n",
    "        for h in range(self.H-1,-1,-1):\n",
    "            #m_h = int(np.ceil(self.v*(self.H - h)))\n",
    "            m_h = int(self.v)\n",
    "            \n",
    "            self.sums_1 = np.zeros((self.d,self.d))\n",
    "            self.sums_2 = np.zeros(self.d)\n",
    "            for l in range(k):\n",
    "                Z = np.random.binomial(m_h,0.5) - m_h\n",
    "                #print(Z)\n",
    "                s,a,r,s_ = self.b[l,h,0],self.b[l,h,1],self.b[l,h,2],self.b[l,h,3]\n",
    "                row = int(s*self.A + a)\n",
    "                self.sums_1 = self.sums_1 + np.outer(self.phi[row,:],self.phi[row,:])\n",
    "                self.sums_2 = self.sums_2 + self.phi[row,:]*(r + max(self.Q[h+1,int(s_),:]) + Z)\n",
    "            self.G[h] = self.sums_1 + self.lam*self.I\n",
    "            self.theta[h] = np.dot(np.linalg.inv(self.G[h]),self.sums_2)\n",
    "            for ss in range(self.S):\n",
    "                for aa in range(self.A):\n",
    "                    #Z = np.random.normal(0.0,self.v)\n",
    "                    row = int(ss*self.A + aa)\n",
    "                    self.Q[h,ss,aa] = max(-self.H*m_h,min(np.inner(self.theta[h],self.phi[row,:]),0.0))\n",
    "    \n",
    "    def run(self):\n",
    "        Rvec = []\n",
    "        R = 0\n",
    "        for k in tqdm(range(self.K)):\n",
    "            self.env.reset()\n",
    "            done = 0\n",
    "            while not done:\n",
    "                s = self.env.state\n",
    "                h = self.env.timestep\n",
    "                a = self.act(s,h)\n",
    "                r,s_,done = self.env.advance(a)\n",
    "                self.update_buffer(s,a,r,s_,k,h)\n",
    "                R += r\n",
    "            Rvec.append(R)\n",
    "            if k > max(self.H,self.d):\n",
    "                self.update(k)\n",
    "        return Rvec\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 537,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "37d3387fb65d4c19883fdcd3eb231637",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1000.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "K = 1000\n",
    "env = make_riverSwim(epLen = 20,nState = 6)\n",
    "\n",
    "'''\n",
    "agent1 = RLSVI(env,K,1.0)\n",
    "R1 = agent1.run()\n",
    "'''\n",
    "agent = LSVI_PHE(env,K,1.0)\n",
    "R = agent.run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 538,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi40LCBodHRwOi8vbWF0cGxvdGxpYi5vcmcv7US4rQAAIABJREFUeJzt3Xd4FcX6wPHvm0YgQEKogRASINIhaKQpAoI0RawUGyKXYhe7V6/lWq5X/YkNQbwgokAAQSmiSFNE6b1L6IFACCEkEFLP/P7YBQME0k7KOXk/z3Oec3Z2dnY2C/Oeszs7I8YYlFJKlT0eJV0BpZRSJUMDgFJKlVEaAJRSqozSAKCUUmWUBgCllCqjNAAopVQZpQFAKaXKKA0ASilVRuUaAETEV0RWi8gmEdkmIm/Y6WEiskpEdovINBHxsdPL2cvR9vrQbGW9ZKfvEpEeRXVQSimlcie5PQksIgL4GWNOi4g3sBx4EngamGWMiRKRscAmY8wYEXkEaGmMGSEiA4DbjTH9RaQpMBVoA9QGFgFXGWOyLrfvatWqmdDQUCccplJKlR3r1q2LN8ZUzy2fV24ZjBUhTtuL3vbLADcC99jpXwOvA2OAvvZngO+Az+wg0heIMsakAftEJBorGKy43L5DQ0NZu3ZtblVUSimVjYgcyEu+PN0DEBFPEdkIxAELgT1AojEm084SA9SxP9cBDgHY608BVbOn57CNUkqpYpanAGCMyTLGRADBWN/am+SUzX6Xy6y7XPoFRGSYiKwVkbXHjx/PS/WUUkoVQL56ARljEoFfgXZAgIicu4QUDByxP8cAdQHs9f5AQvb0HLbJvo9xxphIY0xk9eq5XsJSSilVQLneAxCR6kCGMSZRRMoD3YD/AkuBu4AoYBAw295kjr28wl6/xBhjRGQOMEVEPsS6CRwOrM5vhTMyMoiJiSE1NTW/m7oFX19fgoOD8fb2LumqKKVcXK4BAAgCvhYRT6xfDNONMfNEZDsQJSJvARuA8Xb+8cA39k3eBGAAgDFmm4hMB7YDmcCjV+oBdDkxMTFUqlSJ0NBQrHvLZYcxhhMnThATE0NYWFhJV0cp5eLy0gtoM9A6h/S9WPcDLk5PBe6+TFlvA2/nv5p/S01NLZONP4CIULVqVfTeiFLKGVzySeCy2PifU5aPXSnlXC4ZAJRSym0ZA5unw9qvinxXGgAKwNPTk4iICJo3b87dd99NSkoKABUrVrwg38SJE3nssccAeP3116lTpw4RERHnX4mJicVed6VUKWYMLPsAZg2FlWOKfHcaAAqgfPnybNy4ka1bt+Lj48PYsWPztN3IkSPZuHHj+VdAQEAR11Qp5RISD8KU/vB+Q1j6FtS5Bh7+o8h3m5deQOoKOnbsyObNm0u6GkopV3R0CywfBdt+AJMFDW6E8B7QZih4eBb57l06ALwxdxvbjyQ5tcymtSvzWp9mecqbmZnJTz/9RM+ePQE4e/YsERER59cnJCRw6623nl8eNWoU3377LQBVqlRh6dKlTqy5UsqlxKyD8d3AOKB2a7jlI6gdkft2TuTSAaCkZG/oO3bsyJAhQ4C/Lw2dM3HixAsGsxs5ciTPPvts8VZWKVW6OLJg42T48Vnw8oUHZkPdS3rUFwuXDgB5/abubBc39EoplSdZmfDLK7BqDJQPhGFLoUpoiVXHpQOAUkq5BGMgerF1g/fIBgjtCPdMB58KJVotDQDFKPs9AIAffvgBnfBGKTeXkgALXoZNU0A8oetrcP1IKAUPdeY6I1hJioyMNBdPCLNjxw6aNMlpNOqyQ/8GSrmQGQ/Ctu+h1UDo/jb4Vc11k9SMLNKzHFT2LdigjyKyzhgTmVs+fQ5AKaWKSsxaq/G/dijcPjZPjf9fx5Jp8/YiHhif78GS800vASmllLOdOQF/fgJ/fGzd7O36rzxv+uEvf5Ge5WDwdaFFVz+bBgCllHK2eU/Cjrng7QcPLQBf/1w3STiTzts/7uDnbUcZfkN9+kYU/Yy5GgCUUsqZtnxnNf4R90LP/+Ta+CelZjBh+T4+WrQbgDuvDubp7lcVR001ACillFNkpsOqsbDwX1CxJvT+INdunj9vPcrjU9eTkWW4OiSAZ7o3okODqsU27LsGAKWUKoyUBNg+23rAK/00+NeFO/+Xa+O/fHc8j01ZT5Ogygy9oT69mtfC27N4++VoL6ACyD4cdJ8+fc4P67x//36aN29+Sf6VK1fStm1bIiIiaNKkCa+//jr79+8nODgYh8NxQd6IiAhWr17N66+/zgcffFAsx6OUKgBjYNUX8GETmPcUVKwBfT6Gp7ZASLsrbGaYu+kI941fRQUfT8Y/GMmtrWoXe+MPGgAKJPtw0IGBgYwePfqK+QcNGsS4cePOb9OvXz9CQ0OpW7cuv//++/l8O3fuJDk5mTZtSmZcEKVUHqUlw/QH4KfnoXIdeGAOPLoarnnwig94HUk8S8+PfufxqRsAmDGiAzUq+RZTpS+lAaCQ2rdvz+HDh6+YJy4ujqCgIMD69dC0aVMABg4cSFRU1Pl8UVFRDBw4sOgqq5QqPGNg/nOwYw60ugceWQn1O4Fn7g9tjVr4FwcTUhjZ7Sp+fbYzjWpVKoYKX55r3wP46UVrPG1nqtUCer2bp6xZWVksXrz4/GiglzNy5EgaNWpE586d6dmzJ4MGDcLX15d+/frRunVrPv30U7y8vJg2bRozZsxwxlEopZwtPQV2L4DD62DTVGg7Anr9N8+brz94khnrYhjYJoQnu4UXYUXzTn8BFMC54aCrVq1KQkICN9100xXzv/rqq6xdu5bu3bszZcqU8/MH1KpVi2bNmrF48WI2btyIt7d3jvcQlFIl7GwijL3OGtbhz0/hql7Q4508b/7XsWQemriGahV9GFlKGn9w9V8Aefym7mzn7gGcOnWKW265hdGjR/PEE09ccZsGDRrw8MMPM3ToUKpXr86JEyeoWrXq+ctANWvW1Ms/SpVG6WdgzuOQsBduehOa3wmVa+d5MLfElHT6f7GCjEwH7/ZrRY3KJXfN/2L6C6AQ/P39+eSTT/jggw/IyMi4bL4ff/yRc4Pu7d69G09Pz/PzAd95553Mnz+fadOmMWDAgGKpt1Iqj47/BZ+3t673t3sUrnsC/OvkayTPH7fEcjIlg6nD2tGzeVARVjb/XPsXQCnQunVrWrVqRVRUFB07dmTXrl0EBwefXz9q1ChmzpzJyJEjqVChAl5eXkyePBlPT2u+z4CAANq1a8exY8cICwsrqcNQSl0sbgd8bnfn7PQidHkp30U4HIbv1sVQv5ofLerkPhxEcct1OGgRqQtMAmoBDmCcMeZjEXkdGAoct7P+0xgz397mJWAIkAU8YYxZYKf3BD4GPIH/GWOueA1Hh4POmf4NlCoCx7ZD0hGI3QD7foeDKyErzXqit83QfBVljOF0Wib3jV/NpkOJvH17c+5tW6+IKn6pvA4HnZdfAJnAM8aY9SJSCVgnIgvtdaOMMRc8rSQiTYEBQDOgNrBIRM4NbDEauAmIAdaIyBxjzPa8HZJSShWBPUvgt/fh4J9/p/lVh6t6QKfnrZ6B+bDzaBL3frmKE2fSAbilZRD3tAlxZo2dJtcAYIyJBWLtz8kisgO40jB1fYEoY0wasE9EooFzTzZFG2P2AohIlJ1XA4BSqvhlpsH3w63x+j3LwQ3PQ8Nu1hAONZqCh2e+istyGPafOMPIaZswwMhuV9E6JICO4dWKbWyf/MrXPQARCQVaA6uA64DHROQBYC3Wr4STWMFhZbbNYvg7YBy6KL1tQSptjCm1f9CiVppncFPKZZw6DCs/txr/xrfAzR9CpZoFKio908G/521j1vrDpKRnAfDpwNb0aVXbmTUuEnkOACJSEZgJPGWMSRKRMcCbgLHf/w94CMipZTbk3OPoktZMRIYBwwBCQi792eTr63u+C2VZCwLGGE6cOIGvb+npRqaUyzixB1Z8BvuWwYloK61hN+j/baHm531n/g6+XXmQ+tX9GHhtCO0bVKV5Kbzhm5M8BQAR8cZq/CcbY2YBGGOOZVv/JTDPXowB6mbbPBg4Yn++XPp5xphxwDiwbgJfvD44OJiYmBiOHz9+8aoywdfX94JeRkqpPNjyHcy0n9gP6wQNb4L6na1XARr/+NNpLNkRx8/bjrJkZxzXhlZh+vD2LvelNNcAINYRjQd2GGM+zJYeZN8fALgd2Gp/ngNMEZEPsW4ChwOrsX4ZhItIGHAY60bxPfmtsLe3t3aXVErlXVYGLHodAuvD3RMhqFWhitsRm0T/L1aQlJpJeW9PbmkZxKj+ES7X+EPefgFcB9wPbBGRjXbaP4GBIhKBdRlnPzAcwBizTUSmY93czQQeNcZkAYjIY8ACrG6gE4wx25x4LEopdan1X8OpQ3DPjEI3/mmZWTw0cQ3JaZl8dk9rejUPwtPD9Rr+c/LSC2g5OV/Xn3+Fbd4G3s4hff6VtlNKKadKioUfn4GazSH8ymN25SbLYXht9jZiT6Xy1eBr6dKohpMqWXL0SWCllHsyBpa+ZX3u9kahbvTuPpbM0Elr2X8ihYi6AXS+qrqTKlmyNAAopdyPwwFf9YRDq+C6JyG8W4GLiktKZfDENZw4nc7rfZpyT9t6Lnm9PycaAJRS7iUrwxq2+dAqaDXQ+vZfABlZDr7+cz/vL9hFWqaDD+5uxV3XuFcPPA0ASin3YQx8fas1rEPL/tB3dIEv/XywYBdfLNtLcJXyvHN7C25wk8s+2WkAUEq5jz1LrMb/2n9Yg7gVsPFfsecEXyzby9UhAcwY0cGle/pciQYApZR7OJsIc5+CysHWbF35bPwPnDjD3vgzjPttLyv2nqB6pXKM6h/hto0/aABQSrmDjFQY29Hq7z9oLniVy/OmxhiGf7OOX7Zbgxt4eggPdgjl4c4NqFmKZu8qChoAlFKu7dg2+H4EnDoIvd6DsI752vzTJdH8sv0YLYP9eaFnYxpUr0gtf/du+M/RAKCUck2Jh2DvUmss/1MH4bYxEJH30WUcDsPYZXv4cOFf1KtagenD2+Prnb8hoF2dBgCllGtJP2Nd698y3Vr2LJfvxn/2xsP83y9/cTAhheAq5ZlRBht/0ACglHIVKQmwKcoa0jnpsNXHv1HvvydxyYPpaw7x0vdbyHIYKpXz4q3bmjPg2rp4eeY0Wr370wCglCr9YtZZT/ZmWdMs0ukF6PxSvnr6zN10hOdnbqZOQHkGXxfKfe3qlclv/dlpAFBKlV4ZqbD8Q/jtv9by3ROtGbw8vfNcxPLd8Uxbe4i5m45Qv5ofY+67hka1KhVNfV2MBgClVOljDMT/BdPus94D6sEdX0JI/maR3XU0mcETV+Mw0L1pTf5zRwuqVsx7F1F3pwFAKVW6JMXClH5wdLO1HHFvgYd0mPjnPrw9PVj+wo0E+vk4uaKuTwOAUqp0MAYSD8D85yF+N3R5BRr1glrNC1TcvvgzTF8bw62tamvjfxkaAJRSJc8YmPsErJ9kLXd/Czo8XuDidh1NpsdHyxCBx25s6KRKuh8NAEqpknXyAETdA8e2wtUPQPO7IOyGAhdnjOHV2VupUsGbt25rQYPqFZ1YWfeiAUApVXISD8H856zGv8Pj1tj9HoXrmjnhj/2s2pfAW7c15+aWQU6qqHvSAKCUKn5nTsAfo+DPT63l1vdZl30K6XhyGh8t+ouO4dW4t21IoctzdxoAlFLFwxiI2w6/vgs754FxgHcFaxiHpn0LXbzDYbh//CqSUzN56Powt5m2sShpAFBKFb3UJJh2L+xbZi036QNthkFox0JN1p7dG3O3sfNoMs/1aESXRjWcUqa70wCglCp6S9+2Gv+W/eH6kVCjidOKznIY3pi7jUkrDlC/mh8jOjVwWtnuTgOAUqpoxW6G1V/CNYOhz0dOL/6/P+9k0ooDtKobwNj7rnbrGbycTQOAUqpopJ6CPUvht/fAtzJ0e82pxR9LSmXEt+vYcDCR1iEBzBzRAQ9t/PMl1zFQRaSuiCwVkR0isk1EnrTTA0VkoYjstt+r2OkiIp+ISLSIbBaRq7OVNcjOv1tEBhXdYSmlSsyxbTDmOng3BGYMgvhdVg+f8lWcupvPlkSz4WAiA9vUZerQdtr4F0BefgFkAs8YY9aLSCVgnYgsBB4EFhtj3hWRF4EXgReAXkC4/WoLjAHaikgg8BoQCRi7nDnGmJPOPiilVAlJPgpjOlifrx4EDbpAWCeoEOjU3fwRHc+U1QcZ2CaE/9zRwqlllyW5BgBjTCwQa39OFpEdQB2gL9DZzvY18CtWAOgLTDLGGGCliASISJCdd6ExJgHADiI9galOPB6lVEnJyoBvbrc+3zYWIgY6fRebYxIZ8+seftp6lPLenjzRVYd5KIx83QMQkVCgNbAKqGkHB4wxsSJyrt9VHeBQts1i7LTLpSulXFn0Ytj2Pez6CVLirclaiqDxj45L5qGJa0jNcNAmLJD37mxJkH95p++nLMlzABCRisBM4CljTNIVHrLIaYW5QvrF+xkGDAMICdEn+ZQq1Q78Cd/eYc3LG1gfbnzZ6u3jJGfTs0hISSc67jSDJqwGYN7j19O8jr/T9lGW5SkAiIg3VuM/2Rgzy04+JiJB9rf/ICDOTo8B6mbbPBg4Yqd3vij914v3ZYwZB4wDiIyMvCRAKKVKiejF8J3d2Pf72hq62YnSMx30+ngZ+0+knE/7750ttPF3olwDgFhf9ccDO4wxH2ZbNQcYBLxrv8/Olv6YiERh3QQ+ZQeJBcA753oLAd2Bl5xzGEqpYrXte5jxIPj6w6OroXojpxaflJrBW/O2s/9ECk/c2JB6Vf3o1Kg61XQ2L6fKyy+A64D7gS0istFO+ydWwz9dRIYAB4G77XXzgd5ANJACDAYwxiSIyJvAGjvfv8/dEFZKuQhHlnWtf/GbUO0q+Mdiq4+/Ey3YdpQXZ27mZEoGd1xdh5E3XaXj+hSRvPQCWk7O1+8BuuaQ3wCPXqasCcCE/FRQKVWKrJsIPz4NXr4wYIrTG/998Wf41w9bOZuRxTdD2tAxvLpTy1cX0ieBlVJ5c3gdLHwNqjeGh352+oNdu44m03f0cjKyDLMe7kCrugFOLV9dSgOAUuryHA44vhMOrYQlb1nf+O+d4fTGf8/x0+encIwa2k4b/2KiAUAplbP43TB9EMRts5arNoR+kyDA+d2zX5y5GYBPBrSmbf2qTi9f5UwDgFLqUqfjYMZgq/Fv9yiE32QN6eCR6/Bh+bZw+zHW7D/J8Bvq06dVbaeXry5PA4BS6kIx6+Db263RPG8bAxH3FNmu4k+n8VTUBiqV8+LxruFFth+VMw0ASilrjt51X8Gh1bB7gZV2+zho1b9Idzty2kbOZmQRNaw9Fctpc1Tc9C+uVFmXlQGT74QjG8DbD/xD4I4voF6HIt3tl8v28vvueJ7v2Yg2Yc4dLVTljQYApcqqtGRrspad8yBhL9w5HlrcVSy73hd/hlGL/sK/vDcPdggtln2qS2kAUKosSjsNk/pa3/qrNoQe7xRb4x+XnMqwSWvxEOH7RzpQwUeboZKif3mlygqHA9Z/Ddt/sK71Z6RA539C5xeKrQrxp9MY8MVKYk6eZdwD11C/esVi27e6lAYApcqK396F3/4L5SpDw67Q9DZofmeR7/Zw4lkmrzzA4h1xHEk8y+n0TD7s14rOjWrkvrEqUhoAlCoLDqyAZe9Dq3vgts+hmAZXy8xycMfnf3AsKY1qFX3o1aIWA9qEcHWIc58kVgWjAUApd5aZZn3r3xQF/nWh93vF0vhnOQxHEs8yZfVBjiWl8WKvxgztWB9Pnbi9VNEAoJS7Or7LmqM36TDUaAp9PoFylYp8tw6HYdCE1SyPjgegVbA/Q64P08a/FNIAoJQ7ykiFWcOsxr/Xe9B2eLHs1hjD0ElrWR4dz4MdQmkbFkiXxjXw9nT+EBKq8DQAKOVOUpOsyz1bpkPsRuj8UrE1/lkOw5TVB1m8M47729XjtT5NdSKXUk4DgFLuIukITLwFEvaAV3m47ino/GKR7zbmZApf/LaXuZuPkJiSQaOalbTxdxEaAJRyBwtehhWfWZ9vehM6PF6kN3szsxz8uCWWyasOsnqfNbNrtYo+PNejEfe1rYeXXvJxCRoAlHJ1W2dZjX9wG+j6Lwi7och25XAY/jFpLX9Ex5OW6cDPx5MH2tejXf2q3Ni4Br7enkW2b+V8GgCUclVJsfDdYDi4wnq4697pTp+pK7vpaw/x0qwtZDkM1zesRvdmNbm3bT3t3ePCNAAo5YpOH4dp90LsZuj4DLR7pEga/7PpWXy3PoYdsUlMWXWQBtX9uC2iDsM61aecl37bd3UaAJRyNQdWwLd3QsYZ61p/11edVvS5bpybYk4BkJqRRXJqJn4+noTXqMh7d7WktT7F6zY0ACjlSvb9Dl/fYn0ugv79q/clsGhHHF0aVaeWf3kAujauQZfGNfRSjxvSAKCUK1nypnW9/9HVUDnIacVmZjkY8+sevlsfg395bz6/9xrK++glHnenAUApV2AMbJ0Jh1ZBr/ed2vgDPPfdZr7fcJgqFbx5tkcjbfzLCA0ASrmCtePhx2egYi1ofa9Tix772x6+33CYuoHlWfZcF32AqwzJ9WkNEZkgInEisjVb2usiclhENtqv3tnWvSQi0SKyS0R6ZEvvaadFi0jRP56olLv442Or8ff1h6FLwMfPaUV//ed+3v1pJxF1A/jpyRu08S9j8vILYCLwGTDpovRRxpgPsieISFNgANAMqA0sEpGr7NWjgZuAGGCNiMwxxmwvRN2Vcn9bZ8HC16B6ExgwGfzrFLpIYwx7jp9mzsYjfLIkmsa1KvHdiPb69G4ZlGsAMMYsE5HQPJbXF4gyxqQB+0QkGmhjr4s2xuwFEJEoO68GAKVycjYRFr5qTeHo6QMPzIZKNQtV5NFTqUxbc4hV+07w554TALQJDeSze1pr419GFeYewGMi8gCwFnjGGHMSqAOszJYnxk4DOHRRetucChWRYcAwgJCQkEJUTykX5HDAwT9h7QTrpq+XLzyyotCNv8NhGP6N1b+/SgVvHroujFtaBdGyjr82/mVYQQPAGOBNwNjv/wc8BOR0AdGQ870Gk1PBxphxwDiAyMjIHPMo5ZbOnoTpD8C+ZdZy5ENWjx/PwvfVmL3pMJtiTvHB3a2465rgQpen3EOB/mUZY46d+ywiXwLz7MUYoG62rMHAEfvz5dKVKrt2zIXoxZB6yvrsyICwTtDtdajZ3CmN/9JdcYyctolWdQO4o3Xh7yEo91Ggf10iEmSMibUXbwfO9RCaA0wRkQ+xbgKHA6uxfhmEi0gYcBjrRvE9ham4Ui4tKwN+et661APWfL0Nu1nf+sNvKvRQzumZDt6Zv4MNhxLZdCgRgNf6NMVDn+ZV2eQaAERkKtAZqCYiMcBrQGcRicC6jLMfGA5gjNkmItOxbu5mAo8aY7Lsch4DFgCewARjzDanH41SriArA+Y/B+u+gma3w62fQbmKTiv+xOk07v3fKnYeTebqkAAe7dKAYR0b4F/B22n7UO5BjCm9l9kjIyPN2rVrS7oaSjnX0nfgt/9aI3j2/I/Tis1yGGatj+HlH7aSnungtT5NGXxdmNPKV65DRNYZYyJzy6dPAitVXNJTYOG/YM3/oN710OMdpxb/ZNQG5m2OpXVIAC/1akKbsECnlq/cjwYApYqDwwETusPRLXBVT7jp306ZstHhMMzdfITRS6P569hp+kUG8/btLfDWrp0qDzQAKFXUjmyA2Y/Bsa1Ov+wz5rc9vL9gFyIw/Ib6PNujkTb+Ks80AChVlLZ8B98PB5+KcMNz0PklpxS7dFccP26OZcHWo3RpVJ1R/SMIqODjlLJV2aEBQKmicnQLzBwCtVpAv0kQWN8pxa7el8Dgr9YgAi3q+PPKLU218VcFogFAqaLy52fWe9/RTmv8DyWkMOLbdQT6+bD02c74l9eunargNAAo5UyppyAjFX55BbZMh6sHQVArpxS9L/4MI6dtJDElna8faqONvyo0DQBKOYMjC358GtZN/DutepNCd/U8dTaDZX8d54tle9h6OAmATwa2pmN49UKVqxRoAFDKObZ9bzX+9btAk1ugfBVo3Ae8CnZtPik1gwnL9/HpkmiyHAZPD+HBDqHcdU0wzev4O7fuqszSAKBUYWWkwqLXoUoo3DcLPAreDTM1I4tV+xJ4/rtNHEtKw8/Hk/fuakXHq6pR2Vcv+Sjn0gCgVGFsnAqzHwHjgN4fFKrxP3oqlZs+/I3ktEz8fDz5eEAE7etXpUZlXydWWKm/aQBQqqB+ew+Wvg1Vwqz+/S37Faq4SSv2k5yWyZu3NadH05ra8KsipwFAqYI4uMpq/Gu1hH5fF7ibZ0p6Jgln0vkjOp7Pf91D18Y1uL9dPSdXVqmcaQBQKr8WvQHLPwSfSnDHlwVq/DOzHHyyeDejf91DlsMakbeclwdPdA13dm2VuiwNAErlx9wnrd4+letYN3xrNM53EWfTsxj01WpW70ugY3g1ercIIrSqHy2C/alYTv9LquKj/9qUyqvD663Gv1ZLGPxTgSZx+XblAf49bzvpmQ6e6BrOk13D8dRZulQJ0QCgVF6kJMBXvaF8IAyen+/G/3DiWf71w1aW7IwD4MN+rbi9dR3ECUNCK1VQGgCUupKsTGvqxnUTIfMs9PkYylXKVxExJ1O4+ZPlnDqbQcfwaozqH0G1iuWKpr5K5YMGAKWuZPoDsOtHqFgLbngeWvXP1+bbjpzi/vGrOZ2WycyH23NNPZ2lS5UeGgCUupyYdVbj3/EZ6PpqnjdLz3Sw9kACK/acYOxve8jIMrx5W3Nt/FWpowFAqYsZYw3stnYCiCdcMzhPm63dn8CiHXFM+GMf6ZkOAGr7+zL2/mtoGRxQlDVWqkA0ACh1sV3zrca/RlO49VMIqHvF7IcSUpjwxz6++mM/AGHV/HikcwPahAVSr6pfMVRYqYLRAKBUdtvnwLynoGZzGLo019E8J63Yz+tztuEwEF6jIqP6R9CsdmXt3aNcggYApcC67PPLK7DiM6hQFW4fm2vjv/NoEm/M3U6bsEDevr0F9av5acOvXIoGAKUOr4NlH1iXfspXgcfXQ/nLX7PCcwLwAAAUwklEQVR3OAxr9ifwwszNVPL1Ysy911DFT+fkVa4n17FrRWSCiMSJyNZsaYEislBEdtvvVex0EZFPRCRaRDaLyNXZthlk598tIoOK5nCUyqe4HfDNHRC9CJrfCU/vvGLjD/DanG30H7eSAwkp/LN3E238lcvKyy+AicBnwKRsaS8Ci40x74rIi/byC0AvINx+tQXGAG1FJBB4DYgEDLBOROYYY04660CUypezJ2HF57DhGxAPeGyNNaHLZWw8lMgrP2wh4XQ6R06l0rNZLf59WzNqVNIhm5XryjUAGGOWiUjoRcl9gc7256+BX7ECQF9gkjHGACtFJEBEguy8C40xCQAishDoCUwt9BEoVRDzRlrTOFZtCLeMumzjn5aZxYTl+/nvzzvx8/GkR/NaBJT34cmu4fhX0Bm6lGsr6D2AmsaYWABjTKyI1LDT6wCHsuWLsdMul34JERkGDAMICQkpYPWUuoLko7BjLrQZBr3fzzFLUmoGz8/YzM/bjgLg7Sl8OSiSDg2qFWdNlSpSzr4JnFMXCHOF9EsTjRkHjAOIjIzMMY9SBZZ8zJrJy5EJbYZfsjojy0HU6oOM+XUPR06l0i8ymA4NqnFb6xy/ryjl0goaAI6JSJD97T8IiLPTY4DsT80EA0fs9M4Xpf9awH0rVTCrvrC6emalQ/3OUK3hBat/3hrLl7/vY92Bk4RWrcC0Ye1oW79qiVRVqeJQ0AAwBxgEvGu/z86W/piIRGHdBD5lB4kFwDvnegsB3YGXCl5tpfIg7TQkHrRG8tz3GxzfaaV3egHaP3pB1ui4ZEZ8ux6AN29rzn1tQ7RPv3J7uQYAEZmK9e29mojEYPXmeReYLiJDgIPA3Xb2+UBvIBpIAQYDGGMSRORNYI2d79/nbggr5XQx62DRa3DgTzBZgEDtCLjuSWtEz4vG8k/PdPDP77dSzsuDZc93oaZOxq7KiLz0Ahp4mVVdc8hrgEdzyIsxZgIwIV+1Uyq/MtNg2r3WBC5N+kDo9dDgRqjaIMfs0XHJDJ20jn3xZxjRqYE2/qpM0SeBlftwZMFXvSA5FgZMhca9r5h92V/HGf7NOs5mZPHKzU146LqwYqqoUqWDBgDlPn54xBrWof1j0KjXZbMlpWZw8EQKL83aQmpmFvMev57mdfyLsaJKlQ4aAJR7WD4KNkdBxH3Q/S3I4QZuwpl0np2x6fy8vAATB1+rjb8qszQAKNcXvRgWvQ4Nu1kPdl3U+G86lMjopdH8sv0YAH0jatMxvDrNalemSVDlEqiwUqWDBgDl2jLT4KfnIbABDJgCXhdOtr7pUCJ9R/8BQPemNXmgfSjXh+vTvEqBBgDlypJiYf6zcCIa7p15SeOfmeXgg1924evtwcKRnagbWKGEKqpU6aQBQLme5KPw/XDY+6u13PZhCO92QZbouNO8NGsza/af5IWejbXxVyoHGgCUa4nbAZP6wulj1mBure+DoFbnV59Oy+SZ6RtZsO0YInDH1XV4uHPOzwAoVdZpAFCuY/dCmHw3YKDHf6D9Ixesnr3xMM9/t5m0TAfdmtTglZubElpNJ2VX6nI0ACjXkHwMfn4JKtaAQfOg+lXnV331xz4m/LGPQwlnCa1agae6XUXfiNo6lo9SudAAoEq/zHSYOQRO7IY7x59v/I8npxG1+iD/t/AvKvl6MfyG+jx2Y0Mq+epELUrlhQYAVboZA1H3wP7f4bax0OIuwLrc8/T0TWQ5DOE1KjLzkQ5U1oZfqXzRAKBKt5WfQ/RC6PQiRAxkf/wZRi36i9kbj9C4ViWe69GIzo1q4Omhl3uUyi8NAKp0MsYa0fO39yDsBuj0AtPWHOSFmVsAaBXsz8cDWutNXqUKQQOAKn32LIUZD0JqIgAZnV/l2xUHeGPudiqV82LSkDa0Dqly5TKUUrnSAKBKjzPx1pSNm6ZiqoSyrs79rE4L5n+TEkk4E4d/eW++G9Ge8JqVSrqmSrkFDQCq5BkDh1bB0rdh3zJoNZBXTt3K5G2GCj6eNKvtx62tavNS78aU8/Is6doq5TY0AKiSZQys/xrmPmkt3vopO4NuY/LHv1O/mh+Lnu6Eh97gVapIaABQxc/hgLXj4cAf1tAOx3dCrZZk3T2JB3+I4/fpv+PlIUwf0V4bf6WKkAYAVfy2zbJG8fT2g8Aw6PA4XP80H/4ex++743mgfT1uCK9OtYrlci9LKVVgGgBU8Yrfbd3orRIKj68HD+ua/uil0YxeuocmQZV549ZmOoyDUsVAA4AqelkZ1oTtS96EFZ9ZaYN/JtMIP248zIJtR1mw7Rj1q/sRNbSdNv5KFRMNAKpo7frJGsrBOKzl8lWgz8dk1W3HU1EbmLc5loAK3vSNqM2LvRrjX0GHc1CquGgAUEVn3zKYNRwqBcG1/wDv8mS2HsT7Sw7w1ZSfSc900DeiNu/e0ZLyPtq9U6nipgFAOV9mmnWdf83/rG/+3V6Fa/+BMYbXftjK5FUHaRMWSPemNRlyfZhe8lGqhBQqAIjIfiAZyAIyjTGRIhIITANCgf1AP2PMSbH+l38M9AZSgAeNMesLs39VCp2Jh49aQsYZqNkC7p8FFWuQmJLO09M3sWRnHBF1A5g+vH1J11SpMs/DCWV0McZEGGMi7eUXgcXGmHBgsb0M0AsIt1/DgDFO2LcqbX54xGr8O70II36HijWYvyWWyLcWsWRnHE2CKjNlaNuSrqVSiqK5BNQX6Gx//hr4FXjBTp9kjDHAShEJEJEgY0xsEdRBlYQ9S2D3Aky3fzPqbC9+H/Mnp85msPf4GRpU9+Phzg3p0awmFXz0yqNSpUFh/yca4BcRMcAXxphxQM1zjboxJlZEath56wCHsm0bY6dpAHB1xsCK0fDLyxBQj2cPtmPm5t34+XgSGRpIj2a1GHJ9mD7YpVQpU9gAcJ0x5ojdyC8UkZ1XyJvTnT5zSSaRYViXiAgJCSlk9VSRSkmAnT9aQzms+AzK+bO/25fMmhxP65AAJgy6lip+PiVdS6XUZRQqABhjjtjvcSLyPdAGOHbu0o6IBAFxdvYYoG62zYOBIzmUOQ4YBxAZGXlJgFClwKYo2DkPdv0MjgwrLaQ9KffMZsjoFVT08eKrB68loII2/kqVZgUOACLiB3gYY5Ltz92BfwNzgEHAu/b7bHuTOcBjIhIFtAVO6fV/F3NsO6wcDRu+Bb/qUL8TRA6BkHake1Wi3xcr2XP8DM/3bKSNv1IuoDC/AGoC39t9uL2AKcaYn0VkDTBdRIYAB4G77fzzsbqARmN1Ax1ciH2r4rZyLPz8gvW5WiMYuhjK/T0xS9SK/Ww9nMRD14XxSOeGJVNHpVS+FDgAGGP2Aq1ySD8BdM0h3QCPFnR/qgQYY13uWfMlHF4HvgHQ/1uoczX4/D0X7/wtsbw6exutQwJ4+eYmJVhhpVR+aH88dXnnevYANLkVurwMNRpfku2bFQeo7e/L1KHt8NTx+5VyGRoA1KUcDvjrJ6vxr1gLHl1pDeJ2EWMMn/+6hxV7T/B8z0b4eut4Pkq5Eg0A6kLRi63pGU8dgnKV4dFVUD7ggiwOh2HDoUTWHUjg/QW76NOqNsNvaFBCFVZKFZQGAGXJyoQfHoYt063lDk9A6/svafxjT53l5e+3smSn1bu3ZbA/H/WP0Es/SrkgDQAKkmJhzmMQvQia9IHO/4SaTS/JtuHgSYZOWkf86TSqVSzHpwNb06quvzb+SrkoDQBlXfxu+PJGSEuyGv9+33Do5Fm++GEL248kAX8/rr3hYCJ+Pp681qcp97Wrh7enM8YSVEqVFA0AZdmZEzDxFqvxHzCVw9Xas2FLLB8s2MX+Eyk0q12ZwGxDOXRpVJ1nujeieR3/Eqy0UspZNACUFcZYN3Y3RVnf+jNTYc9SSE/mZO+xvLc9mKmr/wSgnJcHM0a059rQwBKutFKqKGkAKAuObYdJt8KZ4wA4/GqS4eNPVkBD9tbqRb95VUhJP0TToMoMbFOXm1vWvuCbv1LKPWkAcGeZ6fDHx7D0LYyHN9vCR/BpTDgLTgT9necgVK/kxTPdGzHk+rCSq6tSqthpAHA3ZxMhdhOcPQlL34H4XZjyVfiq0gj+vaUF5b09ebRLKI1qVcbH04OK5bxoExaIj5fe0FWqrNEA4E6O/wVf3wKnj51Piqt/B7329ePESQdh1fyYMrQtQf7lS7CSSqnSQgOAOzAGfn4R1k4A8STp6of58EADFhwuR+z2qpT3Ft65vQV3XlOHcl46XINSyqIBwNUZAwtfhVVjIbQjC+o9w/AFKQDc3DKIm1sE0aKOP3UDK5RwRZVSpY0GAFcWuxmmDoSkGKjfhbhbJzP83V+pVdmXTwa2pk2YduNUSl2eBgBXdfIAjOsExsHRiCfY2GAE741fA6CNv1IqTzQAuKJTh2HBP8E4WBz+CkNWNoWVG/HyED4eEKGNv1IqTzQAuJAshyF+2f+o+euzAKys0Z8hW5rSrn4gz/VoRM3KvgRX0Wv9Sqm80QDgIj5bspuYZV/zlhkNAq9kDGbGwU40qlmJ8YOuxa+cnkqlVP5oq1FKLd8dz8ZDJzmWlMaRxLN02vM+73ouwCEe7LrjF0bWj+BpEQLKe+OhwzErpQpAA0Ap9NOWWB6evB4w3Oy5hjsrbKe35y9k+ofg9fByGvnqaJxKqcLTAFAKGGNISc9i7qYj7D+Rwpe/76V1SACT2x2hwpyPIANofAtet4+FcpVKurpKKTehAaCEHUpI4b7xqzhwIsVOMXRrUJHPmm/Dd/F7UKMZPPQT6Ld+pZSTaQAoQXM2HeGJqRsA6NqoOg8086Tj+qfwOLwZDgM+FaHPR9r4K6WKhAaAErLl99l4/vIZP/kep37FDModS4YDp6CcP3R5GSrVglYDwdO7pKuqlHJTxR4ARKQn8DHgCfzPGPNucdehJJgz8Zxc+ikpR3ZwKvkMVyWtIsjTj4CQ5ngFBFsNfZV60OwOqNqgpKurlCoDijUAiIgnMBq4CYgB1ojIHGPM9uKsR3EyZ0+y+885eP35IfWz9pNiqiHGj42+1xD84AS8atcp6Soqpcqo4v4F0AaINsbsBRCRKKAvUOIBwBhDepYD43BAxlnr3RiMybQ/OziblsG+40n8svUIp8+m4Z9xHA9HOsY4yMoyGBz4phzFIz0Zb5NOC7ObTqzlKiDVeDP3qreoe8P91KrsS1N/35I+ZKVUGVfcAaAOcCjbcgzQ1tk7SYw/yqnPu+FhshAMggMP40AweOA4/zq/bBx42J+9ycRLHDmW6wdUA67NYz0yxIedla5jf2h/Ijr2pk/16s46RKWUKrTiDgA5PbJqLsggMgwYBhASElKgnXh6+3CiQn2MeGDEE5Dzn0UEh3iCeFz48vDE28sLh4cP6V5+IJ4gVogw2fIF+JWjemU/avqXt27Uevudq7h1eOUDrHQvX7w9fWgsQuMCHYVSShWt4g4AMUDdbMvBwJHsGYwx44BxAJGRkRcEh7yq5B/I1c/OKWgdlVKqTCjumcDXAOEiEiYiPsAAQFtqpZQqAcX6C8AYkykijwELsLqBTjDGbCvOOiillLIU+3MAxpj5wPzi3q9SSqkLFfclIKWUUqWEBgCllCqjNAAopVQZpQFAKaXKKA0ASilVRokxBXrWqliIyHHgQCGKqAbEO6k6rkKP2f2VteMFPeb8qmeMyXXsmVIdAApLRNYaYyJLuh7FSY/Z/ZW14wU95qKil4CUUqqM0gCglFJllLsHgHElXYESoMfs/sra8YIec5Fw63sASimlLs/dfwEopZS6DLcMACLSU0R2iUi0iLxY0vVxFhGpKyJLRWSHiGwTkSft9EARWSgiu+33Kna6iMgn9t9hs4hcXbJHUHAi4ikiG0Rknr0cJiKr7GOeZg8vjoiUs5ej7fWhJVnvghKRABH5TkR22ue7vbufZxEZaf+73ioiU0XE193Os4hMEJE4EdmaLS3f51VEBtn5d4vIoILWx+0CQLaJ53sBTYGBItK0ZGvlNJnAM8aYJkA74FH72F4EFhtjwoHF9jJYf4Nw+zUMGFP8VXaaJ4Ed2Zb/C4yyj/kkMMROHwKcNMY0BEbZ+VzRx8DPxpjGQCusY3fb8ywidYAngEhjTHOs4eIH4H7neSLQ86K0fJ1XEQkEXsOaTrcN8Nq5oJFvxhi3egHtgQXZll8CXirpehXRsc4GbgJ2AUF2WhCwy/78BTAwW/7z+VzphTVz3GLgRmAe1tSi8YDXxecca66J9vZnLzuflPQx5PN4KwP7Lq63O59n/p4vPNA+b/OAHu54noFQYGtBzyswEPgiW/oF+fLzcrtfAOQ88XydEqpLkbF/8rYGVgE1jTGxAPZ7DTubu/wtPgKeBxz2clUg0RiTaS9nP67zx2yvP2XndyX1gePAV/Zlr/+JiB9ufJ6NMYeBD4CDQCzWeVuHe5/nc/J7Xp12vt0xAOQ68byrE5GKwEzgKWNM0pWy5pDmUn8LEbkFiDPGrMuenENWk4d1rsILuBoYY4xpDZzh78sCOXH5Y7YvYfQFwoDagB/WJZCLudN5zs3ljtFpx+6OASDXieddmYh4YzX+k40xs+zkYyISZK8PAuLsdHf4W1wH3Coi+4EorMtAHwEBInJuRrvsx3X+mO31/kBCcVbYCWKAGGPMKnv5O6yA4M7nuRuwzxhz3BiTAcwCOuDe5/mc/J5Xp51vdwwAbjvxvIgIMB7YYYz5MNuqOcC5ngCDsO4NnEt/wO5N0A44de6npqswxrxkjAk2xoRincslxph7gaXAXXa2i4/53N/iLju/S30zNMYcBQ6JSCM7qSuwHTc+z1iXftqJSAX73/m5Y3bb85xNfs/rAqC7iFSxfzl1t9Pyr6RviBTRTZbewF/AHuDlkq6PE4/reqyfepuBjfarN9a1z8XAbvs90M4vWD2i9gBbsHpYlPhxFOL4OwPz7M/1gdVANDADKGen+9rL0fb6+iVd7wIeawSw1j7XPwBV3P08A28AO4GtwDdAOXc7z8BUrHscGVjf5IcU5LwCD9nHHg0MLmh99ElgpZQqo9zxEpBSSqk80ACglFJllAYApZQqozQAKKVUGaUBQCmlyigNAEopVUZpAFBKqTJKA4BSSpVR/w8U+7/vBZy+wAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(R,label = 'PHE')\n",
    "plt.plot(R1[0:K],label = 'RLSVI')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 539,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.27076498837593466"
      ]
     },
     "execution_count": 539,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agent.Q[0,0,0] - agent.Q[0,0,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 540,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.26335815902965365"
      ]
     },
     "execution_count": 540,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agent1.Q[0,0,0] - agent1.Q[0,0,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 541,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[-8.25613278, -7.98536779],\n",
       "        [ 0.        ,  0.        ],\n",
       "        [ 0.        ,  0.        ],\n",
       "        [ 0.        ,  0.        ],\n",
       "        [ 0.        ,  0.        ],\n",
       "        [ 0.        ,  0.        ]],\n",
       "\n",
       "       [[-8.03808898, -7.73819681],\n",
       "        [-8.1070902 , -7.39753138],\n",
       "        [ 0.        ,  0.        ],\n",
       "        [ 0.        ,  0.        ],\n",
       "        [ 0.        ,  0.        ],\n",
       "        [ 0.        ,  0.        ]],\n",
       "\n",
       "       [[-7.7254559 , -7.57486489],\n",
       "        [-7.62235148, -7.15365958],\n",
       "        [-6.91877538, -6.08552639],\n",
       "        [ 0.        ,  0.        ],\n",
       "        [ 0.        ,  0.        ],\n",
       "        [ 0.        ,  0.        ]],\n",
       "\n",
       "       [[-7.55299317, -7.2953701 ],\n",
       "        [-7.6848104 , -6.95336925],\n",
       "        [-7.14190934, -5.87632782],\n",
       "        [-6.20953011, -4.05922222],\n",
       "        [ 0.        ,  0.        ],\n",
       "        [ 0.        ,  0.        ]],\n",
       "\n",
       "       [[-7.17425592, -7.04227205],\n",
       "        [-7.17601816, -6.67761888],\n",
       "        [-6.86325366, -5.74057776],\n",
       "        [-5.51756589, -4.42990414],\n",
       "        [-4.79319352, -1.38429342],\n",
       "        [ 0.        ,  0.        ]],\n",
       "\n",
       "       [[-6.87936022, -6.68114389],\n",
       "        [-6.98701383, -6.38041179],\n",
       "        [-6.24674088, -5.54515372],\n",
       "        [-6.26042889, -4.20277991],\n",
       "        [-4.3157653 , -2.32699863],\n",
       "        [-2.42779775,  0.        ]],\n",
       "\n",
       "       [[-6.44579343, -6.37129336],\n",
       "        [-6.42006745, -6.05923436],\n",
       "        [-6.3141772 , -5.29173104],\n",
       "        [-5.38187763, -3.99681785],\n",
       "        [-3.91125234, -2.45207573],\n",
       "        [-2.24377301,  0.        ]],\n",
       "\n",
       "       [[-5.9385313 , -5.92327749],\n",
       "        [-6.0723069 , -5.82996265],\n",
       "        [-5.67974626, -4.99264138],\n",
       "        [-5.35848427, -3.92428984],\n",
       "        [-4.68246017, -2.26621074],\n",
       "        [-2.16366675,  0.        ]],\n",
       "\n",
       "       [[-5.66363416, -5.55473582],\n",
       "        [-5.70072269, -5.44394562],\n",
       "        [-5.55008253, -4.70074841],\n",
       "        [-4.54952796, -3.69416632],\n",
       "        [-3.48687762, -2.18530342],\n",
       "        [-2.07784232,  0.        ]],\n",
       "\n",
       "       [[-5.26602076, -5.21760169],\n",
       "        [-5.31373151, -5.06395773],\n",
       "        [-5.07856719, -4.5722756 ],\n",
       "        [-4.98495883, -3.5217464 ],\n",
       "        [-3.37816811, -2.09862074],\n",
       "        [-1.87872618,  0.        ]],\n",
       "\n",
       "       [[-4.73357191, -4.79200793],\n",
       "        [-4.94749757, -4.65725086],\n",
       "        [-4.69753795, -4.33490869],\n",
       "        [-4.63553057, -3.41194979],\n",
       "        [-4.20363398, -1.89751344],\n",
       "        [-2.83063568,  0.        ]],\n",
       "\n",
       "       [[-4.35343745, -4.40833999],\n",
       "        [-4.32774934, -4.20223549],\n",
       "        [-4.28862065, -3.98431567],\n",
       "        [-4.00968033, -3.24567032],\n",
       "        [-3.94303984, -1.85894204],\n",
       "        [-1.79852604,  0.        ]],\n",
       "\n",
       "       [[-3.85295181, -3.85042928],\n",
       "        [-3.86262614, -3.83038113],\n",
       "        [-3.87581757, -3.61769969],\n",
       "        [-3.24869858, -2.98247024],\n",
       "        [-2.77589927, -1.8165113 ],\n",
       "        [-1.71926896,  0.        ]],\n",
       "\n",
       "       [[-3.40015361, -3.41104993],\n",
       "        [-3.32241524, -3.3778259 ],\n",
       "        [-3.30882954, -3.25952757],\n",
       "        [-3.10668342, -2.80365827],\n",
       "        [-2.49207047, -1.73646165],\n",
       "        [-2.06987375,  0.        ]],\n",
       "\n",
       "       [[-2.87858606, -3.0120345 ],\n",
       "        [-2.90264763, -2.9520686 ],\n",
       "        [-2.77680195, -2.82066182],\n",
       "        [-3.00341463, -2.51699117],\n",
       "        [-2.20521019, -1.58022312],\n",
       "        [-1.42117445,  0.        ]],\n",
       "\n",
       "       [[-2.38397506, -2.45928759],\n",
       "        [-2.53722961, -2.48350101],\n",
       "        [-2.60035093, -2.3392508 ],\n",
       "        [-2.22726229, -2.25655308],\n",
       "        [-2.2420583 , -1.43538619],\n",
       "        [-1.31792217,  0.        ]],\n",
       "\n",
       "       [[-2.04922812, -1.96620377],\n",
       "        [-1.9153796 , -1.99004657],\n",
       "        [-1.87837593, -1.91892026],\n",
       "        [-1.98853035, -1.9161985 ],\n",
       "        [-1.82282228, -1.33110139],\n",
       "        [-2.12218585,  0.        ]],\n",
       "\n",
       "       [[-1.50389657, -1.53107257],\n",
       "        [-1.45018784, -1.45835935],\n",
       "        [-1.48177735, -1.41337536],\n",
       "        [-1.39685489, -1.43018183],\n",
       "        [-1.51339462, -1.14340771],\n",
       "        [-0.94027025,  0.        ]],\n",
       "\n",
       "       [[-0.96255981, -1.00920841],\n",
       "        [-1.00996673, -1.06373904],\n",
       "        [-0.92854141, -0.99331956],\n",
       "        [-0.8784068 , -0.93669295],\n",
       "        [-1.12029039, -0.94967296],\n",
       "        [-0.54717249,  0.        ]],\n",
       "\n",
       "       [[-0.46270462, -0.55550808],\n",
       "        [-0.53005662, -0.51213267],\n",
       "        [-0.45933531, -0.53475936],\n",
       "        [-0.46860356, -0.48557555],\n",
       "        [-0.62208398, -0.54990835],\n",
       "        [-0.62421973,  0.        ]],\n",
       "\n",
       "       [[ 0.        ,  0.        ],\n",
       "        [ 0.        ,  0.        ],\n",
       "        [ 0.        ,  0.        ],\n",
       "        [ 0.        ,  0.        ],\n",
       "        [ 0.        ,  0.        ],\n",
       "        [ 0.        ,  0.        ]]])"
      ]
     },
     "execution_count": 541,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agent.Q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}