main.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Import modules & prepare data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "original data.shape: (1259300, 10, 10)\n",
      "original target.shape: (1259300,)\n",
      "\n",
      "electron.shape: (3150, 10, 10), (3150,)\n",
      "muon.shape: (700, 10, 10), (700,)\n",
      "pion.shape: (981050, 10, 10), (981050,)\n",
      "kaon.shape: (160300, 10, 10), (160300,)\n",
      "proton.shape: (114100, 10, 10), (114100,)\n"
     ]
    }
   ],
   "source": [
    "# Import sklearn/tensorflow modules.\n",
    "import tensorflow as tf\n",
    "import tensorflow_datasets as tfds\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Conv2D, MaxPooling2D, Dense, Flatten\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "# Import other modules.\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import pickle\n",
    "from os import walk\n",
    "from time import sleep\n",
    "from IPython.display import clear_output\n",
    "%matplotlib inline\n",
    "\n",
    "# Data source: https://www.kaggle.com/stephenmugisha/particle-collisions\n",
    "\n",
    "_, _, files = next(walk('data'))\n",
    "\n",
    "target_names = {\n",
    "    '0': 'electron',    # Negatively charged particle that is a lepton  (doesn't take part in strong force).\n",
    "    '1': 'muon',    # Electron with 200 times more mass and makes up lots of cosmic radiation.\n",
    "    '2': 'pion',    # Meson (connects with strong force) that can be positive, negative, or neutral\n",
    "    '3': 'kaon',    # Pion with more mass.\n",
    "    '4': 'proton'    # Positively charged particle with 2 up quarks and 1 down quark.\n",
    "}\n",
    "\n",
    "# Check how the data is formatted/stored.\n",
    "file_test = open(f'data/{files[0]}', 'rb')\n",
    "file = pickle.load(file_test)\n",
    "\n",
    "print(file[0].shape)    # Group of 3000 images.\n",
    "print(file[0][0].shape)    # Check first image.\n",
    "print(file[1].shape)    # Group of 3000 targets.\n",
    "print(file[1][0])    # Classified as 'pion.'\n",
    "\n",
    "# Collect all the data.\n",
    "data = []\n",
    "target = []\n",
    "\n",
    "for file in files:\n",
    "    file = open(f'data/{files[0]}', 'rb')\n",
    "    file =  pickle.load(file)\n",
    "\n",
    "    for sample, sample_target in zip(file[0], file[1]):\n",
    "        data.append(sample)\n",
    "        target.append(sample_target)\n",
    "\n",
    "data = np.array(data)\n",
    "target = np.array(target)\n",
    "\n",
    "sleep(7)\n",
    "clear_output()\n",
    "\n",
    "print(f'original data.shape: {data.shape}')\n",
    "print(f'original target.shape: {target.shape}\\n')\n",
    "\n",
    "# Edit target values to 0, 1, 2...\n",
    "new_target = []\n",
    "\n",
    "for tar in target:\n",
    "    if tar == 11:\n",
    "        new_target.append(0)\n",
    "    elif tar == 13:\n",
    "        new_target.append(1)\n",
    "    elif tar == 211:\n",
    "        new_target.append(2)\n",
    "    elif tar == 321:\n",
    "        new_target.append(3)\n",
    "    else:\n",
    "        new_target.append(4)\n",
    "    \n",
    "target = np.array(new_target)\n",
    "\n",
    "for i in range(5):\n",
    "    particle_indexes = np.where(target == i)[0]\n",
    "\n",
    "    data_modified = data[particle_indexes]\n",
    "    target_modified = target[particle_indexes]\n",
    "    \n",
    "    print(f'{target_names[str(i)]}.shape: {data_modified.shape}, {target_modified.shape}')\n",
    "\n",
    "# NOTE: through the above code, we find there are inconsistencies in the data (less samples for one particle but lots of samples in another)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Reshaping data (to make even)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "even data.shape: (3500, 10, 10)\n",
      "even target.shape: (3500,)\n",
      "even data.shape (sklearn version): (3500, 100)\n"
     ]
    }
   ],
   "source": [
    "muon_indexes = np.where(target == 1)[0]    # since this has least # of samples... use this as limiter\n",
    "\n",
    "def set_data_samples(number_of_samples):    # limit to <= 700 samples for each particle. (to keep data even)\n",
    "    \n",
    "    data_new, target_new = ((data[muon_indexes])[:number_of_samples], (target[muon_indexes])[:number_of_samples])\n",
    "    \n",
    "    for i in [0, 2, 3, 4]:\n",
    "        particle_indexes = np.where(target == i)[0]\n",
    "        data_modified = (data[particle_indexes])[:number_of_samples]    \n",
    "        target_modified = (target[particle_indexes])[:number_of_samples]\n",
    "    \n",
    "        data_new = np.append(data_new, data_modified, axis=0)\n",
    "        target_new = np.append(target_new, target_modified)\n",
    "\n",
    "    return (np.array(data_new), np.array(target_new))\n",
    "\n",
    "\n",
    "data_50_samples, target_50_samples = set_data_samples(50)\n",
    "data_100_samples, target_100_samples = set_data_samples(100)\n",
    "data_150_samples, target_150_samples = set_data_samples(150)\n",
    "data_200_samples, target_200_samples = set_data_samples(200)\n",
    "data_250_samples, target_250_samples = set_data_samples(250)\n",
    "data_300_samples, target_300_samples = set_data_samples(300)\n",
    "data_350_samples, target_350_samples = set_data_samples(350)\n",
    "data_400_samples, target_400_samples = set_data_samples(400)\n",
    "data_450_samples, target_450_samples = set_data_samples(450)\n",
    "data_500_samples, target_500_samples = set_data_samples(500)\n",
    "data_550_samples, target_550_samples = set_data_samples(550)\n",
    "data_600_samples, target_600_samples = set_data_samples(600)\n",
    "data_650_samples, target_650_samples = set_data_samples(650)\n",
    "data_700_samples, target_700_samples = set_data_samples(700)\n",
    "\n",
    "\n",
    "# data_140_samples, target_140_samples = set_data_samples(140)    # NOTE: These will be used for later tests.\n",
    "# data_280_samples, target_280_samples = set_data_samples(280)\n",
    "# data_420_samples, target_420_samples = set_data_samples(420)\n",
    "# data_560_samples, target_560_samples = set_data_samples(560)\n",
    "# data_700_samples, target_700_samples = set_data_samples(700)\n",
    "\n",
    "data = data_700_samples    # Regular number of samples that should be used.\n",
    "target = target_700_samples\n",
    "\n",
    "data_sklearn = data.reshape((3500, -1))\n",
    "\n",
    "print(f'even data.shape: {data.shape}')\n",
    "print(f'even target.shape: {target.shape}')\n",
    "\n",
    "print(f'even data.shape (sklearn version): {data_sklearn.shape}')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Data analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train.shape: (2625, 10, 10, 1)\n",
      "y_train.shape: (2625, 5)\n",
      "X_test.shape: (875, 10, 10, 1)\n",
      "y_test.shape: (875, 5)\n",
      "\n",
      "sklearn versions\n",
      "X_train.shape: (2625, 100)\n",
      "y_train.shape: (2625,)\n",
      "X_test.shape: (875, 100)\n",
      "y_test.shape: (875,)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtIAAAH4CAYAAABntQpnAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAgnElEQVR4nO3dfZCV5Xn48evAwi4rCwEUFdhK1EDUCiYTmmI1LlJDXREBszQiIkozttE6RZMmtlGWzLRG0owljR1rE/EF7ARkxbfImBHWxpF00tHIaCPRNr51dSSIsOElw8rz+6M/tj2CCpdn3WX5fGb84zz7PDf3Wediv/ucs0upKIoiAACAg9KnuzcAAACHIiENAAAJQhoAABKENAAAJAhpAABIENIAAJAgpAEAIEFIAwBAgpAGAIAEId1NXnrppSiVSnHHHXd0yfo7duyI5ubmaG1t7ZL1AYAD09DQEA0NDd29DbpAVXdvgK6xY8eOWLRoUUSE4QWAbvSP//iP3b0FuoiQJiL+J7xra2u7exsA0OucfPLJ3b0Fuoi3dnSRF154IWbPnh3Dhw+P6urqOOmkk+KWW26p2HVvv/12XHvttXH88cdHdXV1DB8+PBobG+P555+Pl156KY466qiIiFi0aFGUSqUolUoxb968iIhobm6OUqkUTz31VHzhC1+IIUOGxAknnBAREbt27YrrrrsuPv7xj0f//v1j5MiRceWVV8bbb79d9uePHj06pk6dGmvWrIlPf/rTMWDAgPjkJz8Zt99++4f7xMEhbu98bdiwIZqammLw4MExdOjQuOaaa6KjoyM2btwYf/RHfxR1dXUxevToWLx4cee1d9xxR5RKpXjppZfK1mxtbY1SqbTPW7Vuv/32GD9+fNTU1MTQoUNjxowZ8Ytf/KLsnHnz5sXAgQPjxRdfjMbGxhg4cGDU19fHtddeG7/97W+76tMAh4W98/7000/HzJkzY9CgQTF48OCYM2dObNq0qfO8/b2146233oovf/nLMXLkyOjfv38cf/zx8dd//df7zGWpVIqrrroq7r777jjppJOitrY2xo8fHw899NBH8RT5AEK6C/zHf/xHTJgwIZ599tn4zne+Ew899FCcd955cfXVV3e+3eLDXNfe3h5nnHFG/NM//VNcdtll8eCDD8att94aY8aMiddffz2OPfbYWLNmTUREzJ8/P9avXx/r16+P66+/vuzPmzlzZpx44omxcuXKuPXWW6Moipg+fXr83d/9XVxyySXx8MMPxzXXXBN33nlnnH322fsM9zPPPBPXXnttLFiwIO6///4YN25czJ8/P/71X/+1gp9NODTNmjUrxo8fH6tWrYovfelLcfPNN8eCBQti+vTpcd5558V9990XZ599dnzta1+LlpaWg17/xhtvjPnz58cpp5wSLS0tsWTJktiwYUNMnDgxXnjhhbJzd+/eHdOmTYvJkyfH/fffH5dffnncfPPNcdNNN1Xq6cJhbcaMGXHiiSfGvffeG83NzbF69eqYMmVK7N69e7/n79q1KyZNmhR33XVXXHPNNfHwww/HnDlzYvHixTFz5sx9zn/44Yfje9/7Xnzzm9+MVatWdX7j/F//9V9d/dT4IAUVN2XKlGLUqFHF1q1by45fddVVRU1NTfHWW28Vv/rVr4qIKJYuXXpQ1xVFUXzzm98sIqL48Y9//J572LRpUxERxcKFC/f52MKFC4uIKG644Yay42vWrCkioli8eHHZ8R/+8IdFRBS33XZb57HjjjuuqKmpKV5++eXOYzt37iyGDh1aXHHFFe+5L+jt9s7Xd77znbLjp512WhERRUtLS+ex3bt3F0cddVQxc+bMoiiKYunSpUVEFL/61a/Krl23bl0REcW6deuKoiiKLVu2FAMGDCgaGxvLznvllVeK6urqYvbs2Z3HLr300iIiihUrVpSd29jYWIwdO/bDPl04rO2d9wULFpQdX758eRERxbJly4qiKIqzzjqrOOusszo/fuutt+53Lm+66aYiIopHH32081hEFEcffXSxbdu2zmNvvPFG0adPn+LGG2/sgmfFwXBHusJ27doVjz32WMyYMSNqa2ujo6Oj87/GxsbYtWtX/PSnP/1Q1z3yyCMxZsyY+MM//MMPtdcLL7yw7PHatWsjIjrfArJXU1NTHHHEEfHYY4+VHT/ttNPid37ndzof19TUxJgxY+Lll1/+UPuC3mDq1Kllj0866aQolUpx7rnndh6rqqqKE0888aBnZv369bFz5859ZrW+vj7OPvvsfWa1VCrF+eefX3Zs3LhxZhUq5OKLLy57PGvWrKiqqop169bt9/y1a9fGEUccEV/4whfKju+d6XfP8KRJk6Kurq7z8dFHHx3Dhw83wz2AkK6wzZs3R0dHR/zDP/xD9OvXr+y/xsbGiIj49a9//aGu27RpU4waNepD7/XYY4/dZw9VVVWd76/eq1QqxTHHHBObN28uOz5s2LB91qyuro6dO3d+6L3BoW7o0KFlj/v37x+1tbVRU1Ozz/Fdu3Yd1Np7Z/HdMxwRMWLEiH1mdX9/bnV19UH/ucD+HXPMMWWPq6qqYtiwYfvM4l6bN2+OY445JkqlUtnx4cOHR1VVla+3hxC/taPChgwZEn379o1LLrkkrrzyyv2e8/GPfzza29tT10VEHHXUUfHaa6996L2+e4CHDRsWHR0dsWnTprKYLooi3njjjZgwYcKH/jOB97Y3dt/98wjv/uZ77xfV119/fZ812tra4sgjj+yiHQL788Ybb8TIkSM7H3d0dMTmzZv3G8AR/zPD//Zv/xZFUZR9LX7zzTejo6PDDB9C3JGusNra2pg0aVI8/fTTMW7cuPjMZz6zz3/7G6yDue7cc8+NX/7yl51vxdif6urqiIiD+m518uTJERGxbNmysuOrVq2K7du3d34c6BqjR4+OiIgNGzaUHX/ggQfKHk+cODEGDBiwz6y+9tprsXbtWrMKH7Hly5eXPV6xYkV0dHS857/jMHny5PjNb34Tq1evLjt+1113dX6cQ4M70l1gyZIlccYZZ8SZZ54Zf/ZnfxajR4+O9vb2ePHFF+PBBx98zwA+0Ov+4i/+In74wx/GBRdcEF//+tfj937v92Lnzp3x+OOPx9SpUzvfS3XcccfF/fffH5MnT46hQ4fGkUce2fmFen/OOeecmDJlSnzta1+Lbdu2xR/8wR/Ehg0bYuHChfGpT30qLrnkkq74dAH/34QJE2Ls2LHxla98JTo6OmLIkCFx3333xRNPPFF23sc+9rG4/vrr46/+6q9i7ty5cdFFF8XmzZtj0aJFUVNTEwsXLuymZwCHp5aWlqiqqopzzjknnnvuubj++utj/PjxMWvWrP2eP3fu3Ljlllvi0ksvjZdeeilOPfXUeOKJJ+Jv//Zvo7Gx8UP/DBQfHXeku8DJJ58cTz31VPzu7/5ufOMb34jPf/7zMX/+/Lj33nvf97vMA72urq4unnjiiZg/f37cdtttcd5558WXvvSl2LhxY4wYMaLzvB/84AdRW1sb06ZNiwkTJkRzc/P77rtUKsXq1avjmmuuiaVLl0ZjY2Pnr8Jbu3Zt511uoGv07ds3HnzwwfjkJz8Zf/qnfxpz586N6urq+N73vrfPudddd118//vfj2eeeSamT58eV111VZxyyinx5JNPxic+8Ylu2D0cvlpaWuL555+PmTNnxg033BDnn39+PProo9G/f//9nl9TUxPr1q2Liy++OL797W/HueeeG3fccUd85StfSf06TLpPqSiKors3AQBwqGlubo5FixbFpk2bvK/5MOWONAAAJAhpAABI8NYOAABIcEcaAAAShDQAACQIaQAASEj/gyx79uyJtra2qKur2+efmgb2VRRFtLe3x4gRI6JPn575Pay5hoNjrqH3OZi5Tod0W1tb1NfXZy+Hw9arr74ao0aN6u5t7Je5hhxzDb3Pgcx1OqTr6uqyl76vrVu3dsm60N22bdsW9fX1XTY7lbB3b+edd17069evYutOnz69YmtFRMyYMaOi60HWoTTXlebrNb3Vwcx1OqS76uWhQYMGdcm60FP05JdW9+6tX79+FQ3p2traiq0V4e8Jep5DYa4rzRzS2x3I7PTMN3QBAEAPJ6QBACBBSAMAQIKQBgCABCENAAAJQhoAABKENAAAJAhpAABIENIAAJAgpAEAIEFIAwBAgpAGAIAEIQ0AAAlCGgAAEoQ0AAAkCGkAAEgQ0gAAkFD1YRfYunVrDBo0qBJ7iYiI1tbWiq0VEdHQ0FDR9eBwcOedd1Z0rmfNmlWxtSIimpqaKroeHA4q/fW6VCpVbK2IiKIoKroefBTckQYAgAQhDQAACUIaAAAShDQAACQIaQAASBDSAACQIKQBACBBSAMAQIKQBgCABCENAAAJQhoAABKENAAAJAhpAABIENIAAJAgpAEAIEFIAwBAgpAGAIAEIQ0AAAlCGgAAEqq6ewPv1tDQUNH1WltbK7peROX3CL3dihUrKrreypUrK7peRERTU1PF14TerCiKiq5nrjkUuSMNAAAJQhoAABKENAAAJAhpAABIENIAAJAgpAEAIEFIAwBAgpAGAIAEIQ0AAAlCGgAAEoQ0AAAkCGkAAEgQ0gAAkCCkAQAgQUgDAECCkAYAgAQhDQAACUIaAAAShDQAACQIaQAASKjq7g10tYaGhoqv2dzc3KPXg96uqamp4muuXLmyout1xR6hNzMzHIrckQYAgAQhDQAACUIaAAAShDQAACQIaQAASBDSAACQIKQBACBBSAMAQIKQBgCABCENAAAJQhoAABKENAAAJAhpAABIENIAAJAgpAEAIEFIAwBAgpAGAIAEIQ0AAAlCGgAAEqq6ewOHoubm5u7eAlBhTU1N3b0FoIdbuXJlxdf0d8+hzR1pAABIENIAAJAgpAEAIEFIAwBAgpAGAIAEIQ0AAAlCGgAAEoQ0AAAkCGkAAEgQ0gAAkCCkAQAgQUgDAECCkAYAgAQhDQAACUIaAAAShDQAACQIaQAASBDSAACQIKQBACBBSAMAQEJVd2+ArtHc3HxIrAkcuJUrV1Z8zaampoqvCb1VV8yLuT60uSMNAAAJQhoAABKENAAAJAhpAABIENIAAJAgpAEAIEFIAwBAgpAGAIAEIQ0AAAlCGgAAEoQ0AAAkCGkAAEgQ0gAAkCCkAQAgQUgDAECCkAYAgAQhDQAACUIaAAAShDQAACQIaQAASKjq7g3QNZqbm7t7Cx+otbW14ms2NDRUfE3oKZqamrp7C0CFmetDmzvSAACQIKQBACBBSAMAQIKQBgCABCENAAAJQhoAABKENAAAJAhpAABIENIAAJAgpAEAIEFIAwBAgpAGAIAEIQ0AAAlCGgAAEoQ0AAAkCGkAAEgQ0gAAkCCkAQAgQUgDAEBCVXdvgMNXQ0NDxddsbW2t6HpdsUcAOJSsXLmy4ms2NTVVfM3u4I40AAAkCGkAAEgQ0gAAkCCkAQAgQUgDAECCkAYAgAQhDQAACUIaAAAShDQAACQIaQAASBDSAACQIKQBACBBSAMAQIKQBgCABCENAAAJQhoAABKENAAAJAhpAABIENIAAJAgpAEAIKGquzcAAEDP1dTUVPE1V65cWdH1umKPB8IdaQAASBDSAACQIKQBACBBSAMAQIKQBgCABCENAAAJQhoAABKENAAAJAhpAABIENIAAJAgpAEAIEFIAwBAgpAGAIAEIQ0AAAlCGgAAEoQ0AAAkCGkAAEgQ0gAAkCCkAQAgoaq7NwCV1NDQUNH1WltbK7bW9u3bK7YWkFcqlbp7C3DYa2pqquh6K1eurNhaO3bsOOBz3ZEGAIAEIQ0AAAlCGgAAEoQ0AAAkCGkAAEgQ0gAAkCCkAQAgQUgDAECCkAYAgAQhDQAACUIaAAAShDQAACQIaQAASBDSAACQIKQBACBBSAMAQIKQBgCABCENAAAJQhoAABKqshcWRREREdu2bavYZqCn2b59e8XW2rFjR0T87+z0ROYacsw1dK+9X2MrYefOnRFxYHOdDun29vaIiKivr88uAYel9vb2GDx4cHdvY7/MNeSYa+h9DmSuS0Xy2+g9e/ZEW1tb1NXVRalUSm0QDidFUUR7e3uMGDEi+vTpme+qMtdwcMw19D4HM9fpkAYAgMNZz/z2GQAAejghDQAACUIaAAAShDQAACQIaQAASBDSAACQIKQBACBBSAMAQIKQBgCABCENAAAJQhoAABKENAAAJAhpAABIENIAAJAgpAEAIEFIAwBAgpAGAIAEIQ0AAAlCGgAAEoQ0AAAkCGkAAEgQ0gAAkCCkAQAgQUgDAECCkO5Bmpubo1Qqxa9//evu3grQA7W1tUVzc3P8/Oc/7+6tABBCGuCQ0dbWFosWLRLSAD2EkAboRjt37uzuLQCQJKR7uOeffz6OP/74+OxnPxtvvvlm3HLLLfG5z30uhg8fHkcccUSceuqpsXjx4ti9e/c+195+++0xfvz4qKmpiaFDh8aMGTPiF7/4Rdk58+bNi4EDB8aLL74YjY2NMXDgwKivr49rr702fvvb335UTxMOaXvflvX000/HzJkzY9CgQTF48OCYM2dObNq0qfO80aNHx9SpU6OlpSU+9alPRU1NTSxatCgiIp599tm44IILYsiQIVFTUxOnnXZa3HnnnZ3Xtra2xoQJEyIi4rLLLotSqRSlUimam5s7z3nggQdi4sSJUVtbG3V1dXHOOefE+vXr97vX5557Li666KIYPHhwHH300XH55ZfH1q1bu/CzBND7COke7PHHH4/TTz89xo0bF+vWrYvhw4fHf/7nf8bs2bPj7rvvjoceeijmz58f3/72t+OKK64ou/bGG2+M+fPnxymnnBItLS2xZMmS2LBhQ0ycODFeeOGFsnN3794d06ZNi8mTJ8f9998fl19+edx8881x0003fZRPFw55M2bMiBNPPDHuvffeaG5ujtWrV8eUKVPKvtF96qmn4qtf/WpcffXVsWbNmrjwwgtj48aNcfrpp8dzzz0X3/3ud6OlpSVOPvnkmDdvXixevDgiIj796U/H0qVLIyLiG9/4Rqxfvz7Wr18ff/InfxIREffcc09ccMEFMWjQoPiXf/mX+MEPfhBbtmyJhoaGeOKJJ/bZ64UXXhhjxoyJVatWxde//vW45557YsGCBR/BZwmgFynoMRYuXFhERLFp06bi7rvvLvr3719cffXVxTvvvLPf8995551i9+7dxV133VX07du3eOutt4qiKIotW7YUAwYMKBobG8vOf+WVV4rq6upi9uzZnccuvfTSIiKKFStWlJ3b2NhYjB07tsLPEHqnvbO7YMGCsuPLly8vIqJYtmxZURRFcdxxxxV9+/YtNm7cWHbeF7/4xaK6urp45ZVXyo6fe+65RW1tbfH2228XRVEUP/vZz4qIKJYuXVp23jvvvFOMGDGiOPXUU8v+vmhvby+GDx9enH766fvsdfHixWVrfPnLXy5qamqKPXv25D4JAIchd6R7oL/5m7+JefPmxbe+9a1YsmRJ9Onzv/+bnn766Zg2bVoMGzYs+vbtG/369Yu5c+fGO++8E7/85S8jImL9+vWxc+fOmDdvXtm69fX1cfbZZ8djjz1WdrxUKsX5559fdmzcuHHx8ssvd80ThF7q4osvLns8a9asqKqqinXr1nUeGzduXIwZM6bsvLVr18bkyZOjvr6+7Pi8efNix44d+7w94902btwYbW1tcckll5T9fTFw4MC48MIL46c//Wns2LGj7Jpp06aVPR43blzs2rUr3nzzzQ9+ogBEhLd29EjLli2LkSNHxhe/+MWy46+88kqceeaZ8d///d+xZMmS+MlPfhI/+9nP4pZbbomI//2hpc2bN0dExLHHHrvP2iNGjOj8+F61tbVRU1NTdqy6ujp27dpVsecEh4Njjjmm7HFVVVUMGzasbOb2N5ebN29+z3nd+/H380Ezv2fPntiyZUvZ8WHDhpU9rq6ujgg//AhwMIR0D7RmzZro169fnHnmmWV3hVevXh3bt2+PlpaWmDNnTpxxxhnxmc98Jvr37192/d4vkK+//vo+a7e1tcWRRx7ZtU8ADlNvvPFG2eOOjo7YvHlzWbSWSqV9rhs2bNh7zmtEfODMftDM9+nTJ4YMGfLBTwCAgyKke6DjjjsufvKTn0R1dXWceeaZnT8cuPcL8N47RxERRVHEP//zP5ddP3HixBgwYEAsW7as7Phrr73W+RIyUHnLly8ve7xixYro6OiIhoaG971u8uTJsXbt2s5w3uuuu+6K2tra+P3f//2IeO+7xmPHjo2RI0fGPffcE0VRdB7fvn17rFq1qvM3eQBQWUK6hzr22GPj8ccfj6OOOio+97nPxbPPPhvnnHNO9O/fPy666KJ45JFH4r777ospU6bs85Ltxz72sbj++uvjgQceiLlz58YjjzwSy5Yti0mTJkVNTU0sXLiwm54V9G4tLS3xl3/5l/HjH/84/v7v/z6uuOKKGD9+fMyaNet9r1u4cGH069cvJk2aFMuXL49HHnkk5syZEw8//HA0NzfH4MGDIyLihBNOiAEDBsTy5cujtbU1/v3f/73zjvPixYvj5z//eUydOjUeeOCBWLlyZUyaNCnefvvt+Na3vvVRPH2Aw46Q7sGOPPLIWLt2bZxwwglx1llnxW9+85tYtWpVbNmyJWbOnBl//ud/Hqeddlp897vf3efa6667Lr7//e/HM888E9OnT4+rrroqTjnllHjyySfjE5/4RDc8G+j9Wlpa4vnnn4+ZM2fGDTfcEOeff348+uij+7z96t3Gjh0bTz75ZIwdOzauvPLKmD59ejz77LOxdOnS+OpXv9p5Xm1tbdx+++2xefPm+PznPx8TJkyI2267LSIiZs+eHatXr47NmzfHH//xH8dll10WgwYNinXr1sUZZ5zRpc8b4HBVKv7v64AAHLTm5uZYtGhRbNq0yc8gABxG3JEGAIAEIQ0AAAne2gEAAAlV3b0BAKDr7NmzJ9ra2qKurm6/v8ccKFcURbS3t8eIESPK/rXY/RHSANCLtbW17fPPzwMf7NVXX41Ro0a97zlCGgB6sbq6uoj4nygYNGhQN+8Ger5t27ZFfX195+y8n3RIe6kIDs7BvFTUXcw1HJxDYa73zvKgQYOENByEA/k6mA5pLxVBzoG8VNRdzDXk9OS5BrpOOqQP5HZ3xtatW7tkXehuB/NSUXfZu7fPfvazUVVVuXd+/ehHP6rYWtCTHApzDXSd9FfKrnrZ18tO9HY9+S0Te/dWVVVV0ZA21/R2PXmuga7TM9/QBQAAPZyQBgCABCENAAAJQhoAABKENAAAJAhpAABIENIAAJAgpAEAIEFIAwBAgpAGAIAEIQ0AAAlCGgAAEoQ0AAAkCGkAAEgQ0gAAkCCkAQAgQUgDAEBC1YddYOvWrTFo0KBK7CUiIlpbWyu2VkREQ0NDRdeDw8GPfvQjcw0AH8AdaQAASBDSAACQIKQBACBBSAMAQIKQBgCABCENAAAJQhoAABKENAAAJAhpAABIENIAAJAgpAEAIEFIAwBAgpAGAIAEIQ0AAAlCGgAAEoQ0AAAkCGkAAEgQ0gAAkCCkAQAgoaq7N/BuDQ0NFV2vubm5out11ZrQm5lrAHojd6QBACBBSAMAQIKQBgCABCENAAAJQhoAABKENAAAJAhpAABIENIAAJAgpAEAIEFIAwBAgpAGAIAEIQ0AAAlCGgAAEoQ0AAAkCGkAAEgQ0gAAkCCkAQAgQUgDAECCkAYAgAQhDQAACVXdvYGu1tzcXPE1W1tbK7peQ0NDRdeD3q4r5rpUKlV0vaIoKroeAD2PO9IAAJAgpAEAIEFIAwBAgpAGAIAEIQ0AAAlCGgAAEoQ0AAAkCGkAAEgQ0gAAkCCkAQAgQUgDAECCkAYAgAQhDQAACUIaAAAShDQAACQIaQAASBDSAACQIKQBACBBSAMAQEJVd2/gUNTQ0FDR9VpbWyu6XkTl9wi9XVEUFV3PXAP0fu5IAwBAgpAGAIAEIQ0AAAlCGgAAEoQ0AAAkCGkAAEgQ0gAAkCCkAQAgQUgDAECCkAYAgAQhDQAACUIaAAAShDQAACQIaQAASBDSAACQIKQBACBBSAMAQIKQBgCABCENAAAJQhoAABKqunsDRDQ0NHT3FoAKM9cAvZ870gAAkCCkAQAgQUgDAECCkAYAgAQhDQAACUIaAAAShDQAACQIaQAASBDSAACQIKQBACBBSAMAQIKQBgCABCENAAAJQhoAABKENAAAJAhpAABIENIAAJAgpAEAIEFIAwBAgpAGAICEqu7eAIeOUqlU0fWKoqjoesDBa21treh6DQ0NFV0PoCdzRxoAABKENAAAJAhpAABIENIAAJAgpAEAIEFIAwBAgpAGAIAEIQ0AAAlCGgAAEoQ0AAAkCGkAAEgQ0gAAkCCkAQAgQUgDAECCkAYAgAQhDQAACUIaAAAShDQAACQIaQAASKjq7g1w6CiKoqLrtba2VnS9iIiGhoaKrwm9mZkByHNHGgAAEoQ0AAAkCGkAAEgQ0gAAkCCkAQAgQUgDAECCkAYAgAQhDQAACUIaAAAShDQAACQIaQAASBDSAACQIKQBACBBSAMAQIKQBgCABCENAAAJQhoAABKENAAAJAhpAABIENIAAJBQ1d0b4PDV0NDQ3VsAKqy1tbXia/q7Auip3JEGAIAEIQ0AAAlCGgAAEoQ0AAAkCGkAAEgQ0gAAkCCkAQAgQUgDAECCkAYAgAQhDQAACUIaAAAShDQAACQIaQAASBDSAACQIKQBACBBSAMAQIKQBgCABCENAAAJQhoAABKqunsD0JO1trZWbK3t27dXbC3oqRoaGrp7Cx/IXAOV4o40AAAkCGkAAEgQ0gAAkCCkAQAgQUgDAECCkAYAgAQhDQAACUIaAAAShDQAACQIaQAASBDSAACQIKQBACBBSAMAQIKQBgCABCENAAAJQhoAABKENAAAJAhpAABIENIAAJBQlb2wKIqIiNi2bVvFNgM9zfbt2yu21o4dOyLif2enJzLXHA4Ot7kGuk46pNvb2yMior6+vmKbgcNBe3t7DB48uLu3sV/mGnJ68lwDXScd0iNGjIhXX3016urqolQqVXJP0CsVRRHt7e0xYsSI7t7KezLXcHAOhbn2ShMcnL2zciCvNJUKr0cBQK/12muveZUJEl599dUYNWrU+54jpAGgF9uzZ0+0tbV5pQkO0P99palPn/f/vRxCGgAAEvz6OwAASBDSAACQIKQBACBBSAMAQIKQBgCABCENAAAJQhoAABKENAAAJAhpAABIENIAAJAgpAEAIEFIAwBAwv8DJoS0Viis63sAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 900x600 with 6 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Check for NaN values.\n",
    "print(np.isnan(np.sum(data)))    # -> False\n",
    "\n",
    "# Check range of values.\n",
    "print(np.max(data), np.min(data))    # -> 8, 0\n",
    "\n",
    "sleep(4)\n",
    "clear_output()\n",
    "\n",
    "# Get indexes for each different target possibility.\n",
    "indexes = [\n",
    "    np.where(target == 0)[0][0],\n",
    "    np.where(target == 1)[0][0],\n",
    "    np.where(target == 2)[0][0],\n",
    "    np.where(target == 3)[0][0],\n",
    "    np.where(target == 4)[0][0],\n",
    "]\n",
    "\n",
    "samples = [data[index] for index in indexes]\n",
    "samples_target = [target_names[str(target[index])] for index in indexes]\n",
    "\n",
    "fig, axs = plt.subplots(2, 3, figsize=(9, 6), subplot_kw={\n",
    "    'yticks': (),\n",
    "    'xticks': ()\n",
    "})\n",
    "\n",
    "axs = [ax for ax in axs.flatten()]\n",
    "\n",
    "for sample, sample_target, ax in zip(samples, samples_target, axs):\n",
    "    ax.imshow(sample, cmap='binary')\n",
    "    ax.set_title(str(sample_target))\n",
    "\n",
    "# Split the data\n",
    "X_train, X_test, y_train, y_test = train_test_split(data, target, stratify=target)\n",
    "\n",
    "# Change the train and test datasets.\n",
    "scaler = StandardScaler()\n",
    "\n",
    "# Reshape + change data\n",
    "X_train = X_train.reshape(X_train.shape[0], -1)\n",
    "X_test = X_test.reshape(X_test.shape[0], -1)\n",
    "\n",
    "# Scale and reshape data back to og form.\n",
    "X_train = (scaler.fit_transform(X_train)).reshape((X_train.shape[0], 10, 10, 1))\n",
    "X_test = (scaler.transform(X_test)).reshape((X_test.shape[0], 10, 10, 1))\n",
    "\n",
    "y_train = to_categorical(y_train)\n",
    "y_test = to_categorical(y_test)\n",
    "\n",
    "X_train_sklearn, X_test_sklearn, y_train_sklearn, y_test_sklearn = train_test_split(data_sklearn, target, stratify=target)\n",
    "\n",
    "y_train_sklearn, y_test_sklearn = ([], [])\n",
    "\n",
    "for label in list(y_train):\n",
    "    y_train_sklearn.append(list(label).index(1))    # changing sample from [0, 1, 0, 0, 0] to [2, .....]\n",
    "\n",
    "for label in list(y_test):\n",
    "    y_test_sklearn.append(list(label).index(1))\n",
    "\n",
    "y_train_sklearn = np.array(y_train_sklearn)\n",
    "y_test_sklearn = np.array(y_test_sklearn)\n",
    "\n",
    "print(f'X_train.shape: {X_train.shape}')\n",
    "print(f'y_train.shape: {y_train.shape}')\n",
    "print(f'X_test.shape: {X_test.shape}')\n",
    "print(f'y_test.shape: {y_test.shape}')\n",
    "\n",
    "print('\\nsklearn versions')\n",
    "print(f'X_train.shape: {X_train_sklearn.shape}')\n",
    "print(f'y_train.shape: {y_train_sklearn.shape}')\n",
    "print(f'X_test.shape: {X_test_sklearn.shape}')\n",
    "print(f'y_test.shape: {y_test_sklearn.shape}')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create TensorFlow Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/3\n",
      "42/42 [==============================] - 1s 3ms/step - loss: 2.0945 - accuracy: 0.1859\n",
      "Epoch 2/3\n",
      "42/42 [==============================] - 0s 3ms/step - loss: 2.0066 - accuracy: 0.2118\n",
      "Epoch 3/3\n",
      "42/42 [==============================] - 0s 2ms/step - loss: 1.9595 - accuracy: 0.2179\n",
      "Train accuracy: 21.8%\n",
      "Test accuracy: 22.4%\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA0MAAAHBCAYAAACrJ2AVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAA9hAAAPYQGoP6dpAACESElEQVR4nOzdd1xWdf/H8dfFRgQUZTgQcKC4N+K2222mZYn2S7Nxl4W7pWl3VhZNs6U2HKlpVJparqycuZLEiVvEASoOQFDWdX5/UBS5QMXDeD8fj+vxkHN9zznvc93d17k+1/len2MxDMNARERERESkhLExO4CIiIiIiIgZVAyJiIiIiEiJpGJIRERERERKJBVDIiIiIiJSIqkYEhERERGREknFkIiIiIiIlEgqhkREREREpERSMSQiIiIiIiWSiiERERERESmRVAyJ3AYzZ87EYrGwdetWs6OIiEgx8OGHH2KxWKhbt67ZUUSKNRVDIiIiIoXM9OnTAdi9ezebN282OY1I8aViSERERKQQ2bp1K9u3b6dHjx4ATJs2zeREV5eammp2BJFbpmJI5A5Zv349//nPf3B1daVUqVK0bNmSJUuW5BqTmprKs88+S0BAAE5OTnh4eNC0aVPmzZuXM+bw4cP069ePihUr4ujoiLe3N//5z3+Iioq6w0ckIiIF4a/i580336Rly5Z8/fXXVxQeJ06c4IknnsDX1xcHBwcqVqzI/fffz6lTp3LGXLhwgWeeeYaqVavi6OiIl5cX3bt3Z+/evQCsXr0ai8XC6tWrc207JiYGi8XCzJkzc5YNGjSI0qVLs3PnTjp37oyrqyv/+c9/AFi5ciW9evWicuXKODk5Ub16dZ588kkSEhKuOLa9e/fSv39/vL29cXR0pEqVKgwcOJC0tDRiYmKws7MjPDz8ivXWrl2LxWLh22+/vanXVORa7MwOIFISrFmzhk6dOlG/fn2mTZuGo6MjkydPpmfPnsybN4/Q0FAARo0axezZs5kwYQKNGjUiJSWFXbt2cfbs2Zxtde/enaysLN5++22qVKlCQkICGzZs4MKFCyYdnYiI3C6XLl1i3rx5NGvWjLp16/Loo4/y+OOP8+233/Lwww8D2YVQs2bNyMjI4MUXX6R+/fqcPXuWFStWcP78eby9vUlOTqZ169bExMTwwgsvEBwczMWLF1m7di1xcXHUqlUr39nS09O55557ePLJJxk9ejSZmZkAHDp0iJCQEB5//HHc3d2JiYlh4sSJtG7dmp07d2Jvbw/A9u3bad26NeXLl+fVV1+lRo0axMXFsXjxYtLT0/H39+eee+5h6tSpPP/889ja2ubs++OPP6ZixYrce++9t+FVFvkHQ0Ru2YwZMwzA+P3336/6fIsWLQwvLy8jOTk5Z1lmZqZRt25do3LlyobVajUMwzDq1q1r9O7d+5r7SUhIMABj0qRJt/cARESkUJg1a5YBGFOnTjUMwzCSk5ON0qVLG23atMkZ8+ijjxr29vbGnj17rrmdV1991QCMlStXXnPMqlWrDMBYtWpVruVHjhwxAGPGjBk5yx5++GEDMKZPn37d/Far1cjIyDCOHj1qAMaiRYtynrvrrruMMmXKGKdPn75hpu+//z5n2YkTJww7OzvjlVdeue6+RW6GpsmJFLCUlBQ2b97M/fffT+nSpXOW29raMmDAAI4fP86+ffsAaN68OcuWLWP06NGsXr2aS5cu5dqWh4cH1apV45133mHixIls27YNq9V6R49HREQKzrRp03B2dqZfv34AlC5dmgceeIB169Zx4MABAJYtW0aHDh0ICgq65naWLVtGYGAgHTt2vK35+vTpc8Wy06dPM3jwYHx9fbGzs8Pe3h4/Pz8AoqOjgexp4GvWrKFv3754enpec/vt27enQYMGfPLJJznLpk6disVi4YknnritxyIC+s2QSIE7f/48hmFQoUKFK56rWLEiQM40uA8//JAXXniBhQsX0qFDBzw8POjdu3fOCdBisfDLL7/QpUsX3n77bRo3boynpyfDhg0jOTn5zh2UiIjcdgcPHmTt2rX06NEDwzC4cOECFy5c4P777wf+7jB35swZKleufN1t5WVMfpUqVQo3N7dcy6xWK507d2bBggU8//zz/PLLL2zZsoVNmzYB5Hypd/78ebKysvKUadiwYfzyyy/s27ePjIwMPv/8c+6//358fHxu6/GIgIohkQJXtmxZbGxsiIuLu+K5kydPAlC+fHkAXFxceOWVV9i7dy/x8fFMmTKFTZs20bNnz5x1/Pz8mDZtGvHx8ezbt4+RI0cyefJknnvuuTtzQCIiUiCmT5+OYRh89913lC1bNufxV1e5L7/8kqysLDw9PTl+/Ph1t5WXMU5OTgCkpaXlWn61xgeQ/YXcv+3atYvt27fzzjvvMHToUNq3b0+zZs0oV65crnEeHh7Y2treMBPAgw8+SLly5fjkk0/49ttviY+PJyws7IbridwMFUMiBczFxYXg4GAWLFiQa9qb1Wplzpw5VK5cmcDAwCvW8/b2ZtCgQfTv3599+/ZdtYVpYGAg48aNo169evzxxx8FehwiIlJwsrKy+PLLL6lWrRqrVq264vHMM88QFxfHsmXL6NatG6tWrcqZYn013bp1Y//+/fz666/XHOPv7w/Ajh07ci1fvHhxnnP/VSA5OjrmWv7pp5/m+tvZ2Zl27drx7bffXrPY+ouTkxNPPPEEX375JRMnTqRhw4a0atUqz5lE8kPd5ERuo19//ZWYmJgrloeHh9OpUyc6dOjAs88+i4ODA5MnT2bXrl3Mmzcv52QSHBzM3XffTf369SlbtizR0dHMnj2bkJAQSpUqxY4dOxgyZAgPPPAANWrUwMHBgV9//ZUdO3YwevToO3y0IiJyuyxbtoyTJ0/y1ltv0b59+yuer1u3Lh9//DHTpk3j448/ZtmyZbRt25YXX3yRevXqceHCBZYvX86oUaOoVasWI0aMICIigl69ejF69GiaN2/OpUuXWLNmDXfffTcdOnTAx8eHjh07Eh4eTtmyZfHz8+OXX35hwYIFec5dq1YtqlWrxujRozEMAw8PD3744QdWrlx5xdi/OswFBwczevRoqlevzqlTp1i8eDGffvoprq6uOWOffvpp3n77bSIjI/niiy9u6jUVyRNz+zeIFA9/dZO71uPIkSPGunXrjLvuustwcXExnJ2djRYtWhg//PBDru2MHj3aaNq0qVG2bFnD0dHRqFq1qjFy5EgjISHBMAzDOHXqlDFo0CCjVq1ahouLi1G6dGmjfv36xvvvv29kZmaacegiInIb9O7d23BwcLhup7V+/foZdnZ2Rnx8vHHs2DHj0UcfNXx8fAx7e3ujYsWKRt++fY1Tp07ljD9//rwxfPhwo0qVKoa9vb3h5eVl9OjRw9i7d2/OmLi4OOP+++83PDw8DHd3d+Ohhx4ytm7detVuci4uLlfNtWfPHqNTp06Gq6urUbZsWeOBBx4wYmNjDcB4+eWXrxj7wAMPGOXKlTMcHByMKlWqGIMGDTIuX758xXbbt29veHh4GKmpqXl8FUXyz2IYhmFaJSYiIiIi8i+nT5/Gz8+PoUOH8vbbb5sdR4oxTZMTERERkULh+PHjHD58mHfeeQcbGxuGDx9udiQp5tRAQUREREQKhS+++IL27duze/duvvrqKypVqmR2JCnmNE1ORERERERKJF0ZEhERERGREknFkIiIiIiIlEgqhkREREREpETKVze58PBwFixYwN69e3F2dqZly5a89dZb1KxZ85rrxMXF8cwzzxAZGcmBAwcYNmwYkyZNumLc/Pnzeemllzh06BDVqlXj9ddf5957781zNqvVysmTJ3F1dc25gaWIiBQ8wzBITk6mYsWK2NjoO7a/6LwkImKevJ6b8lUMrVmzhrCwMJo1a0ZmZiZjx46lc+fO7NmzBxcXl6uuk5aWhqenJ2PHjuX999+/6piNGzcSGhrKa6+9xr333sv3339P3759Wb9+PcHBwXnKdvLkSXx9ffNzOCIichsdO3aMypUrmx2j0NB5SUTEfDc6N91SN7kzZ87g5eXFmjVraNu27Q3Ht2/fnoYNG15xZSg0NJSkpCSWLVuWs6xr166ULVuWefPm5SlLYmIiZcqU4dixY7i5ueXrOERE5OYlJSXh6+vLhQsXcHd3NztOoaHzkoiIefJ6brqlm64mJiYC4OHhcSubYePGjYwcOTLXsi5dulx1Ot21/DUFwc3NTScdERETaCpYbjoviYiY70bnppsuhgzDYNSoUbRu3Zq6deve7GYAiI+Px9vbO9cyb29v4uPjr7lOWloaaWlpOX8nJSXdUgYRERERESlZbvqXrkOGDGHHjh15nsZ2I/+u2gzDuG4lFx4ejru7e85D87JFRERERCQ/bqoYGjp0KIsXL2bVqlW35ceyPj4+V1wFOn369BVXi/5pzJgxJCYm5jyOHTt2yzlERERERKTkyFcxZBgGQ4YMYcGCBfz6668EBATclhAhISGsXLky17KffvqJli1bXnMdR0fHnHnYmo8tIiIiIiL5la/fDIWFhTF37lwWLVqEq6trztUcd3d3nJ2dgewrNidOnGDWrFk560VFRQFw8eJFzpw5Q1RUFA4ODtSuXRuA4cOH07ZtW9566y169erFokWL+Pnnn1m/fv3tOEYREREREZEr5Ku19rV+wzNjxgwGDRoEwKBBg4iJiWH16tXXXc/Pz4+YmJicv7/77jvGjRvH4cOHc266et999+U1GklJSbi7u5OYmKirRCIid5Def69Or4uIiHny+h58S/cZKkx00hERMYfef69Or4uIiHny+h58093kREREREREijIVQyIiIiIiUiKpGBIRERERkRJJxZCIiIiIiJRIKoZERERERKREUjEkIiIiIiIlkoqhPy3ZEcehMxfNjiEiIiIiUqJZrQbrDpxhW+z5At+XXYHvoQj4Yt1hJiyJpn5ld+Y/1RJ7W9WIIiIiIiJ3UuKlDL6LPM5Xm45yOCGFtoGezHq0eYHuU8UQcHf9inz060F2HE/kw18O8EznmmZHEhEREREpEXafTGT2xqMsjDrB5QwrAKUd7aha3gWr1cDGxlJg+1YxBPi4OxF+Xz2e/uoPPll1kHaBnjT19zA7loiIiIhIsZSWmcWynfHM3nSUyKN/T4cL9C7NgBB/7m1UidKOBV+qqBj6U/d6FejTuDLz/zjOyG+iWDqsDa5O9mbHEhEREREpNk5cuMRXm44S8fsxzqakA2BnY6FrXR8GtPCjeYAHFkvBXQn6NxVD/zD+ntpsPnKWY+cu8coPe3j3gQZmRxIRERERKdKsVoP1BxOYvekov0SfwmpkL/dxc+LB4Cr0a+aLl5uTKdlUDP2Dq5M9E/s2pN9nG/ku8jj/qeVFt3oVzI4lIiIiIlLkJKZm8G3kMb7aHMuRhJSc5S2rlWNACz861vY2vXGZiqF/aR7gwVPtq/HJqkOM+X4njf3K4m1SpSoiIiIiUtTsOpHdEGHR9r8bIrg62tGnSWUealGF6l6uJif8m4qhqxj+n0DW7k9g54lEnv12O18+0rxAu1iIiIiIiBRlaZlZLN0Zx6yNR9kWeyFneS0fVwaE+NG7YSVc7kBDhPwqfIkKAQc7G94PbcjdH61j3YEEvtwYwyOtAsyOJSIiIiJSqBw/n8pXm2OJ+P0Y5/7REKFbvQoMaOFHM/+yd7QhQn6pGLqG6l6lGdujNi8t3EX4sr20ql6eQO/Cc0lPRERERMQMVqvBuoMJzN54lF/3XqUhQnNfvFyLxs9MVAxdx0PBVfg1+hSr9p1h+NdRLAxriaOdrdmxRERERETuuL8aIszZdJSYs6k5y1tV/7MhQpA3diY3RMgvFUPXYbFYeOv++nSdtI7ouCQmrtzPmG5BZscSEREREbljrt8QwY/qXqVNTnjzVAzdgJerE2/eV48nZkfy2drDtA/0IqRaObNjiYiIiIgUmMsZ2Q0RZm8qWg0R8qvoH8Ed0LmOD/2b+zJvyzGe+SaKZSPa4u5sb3YsEREREZHb6ti57IYI32z9uyGCva2FbnUrMCDEj6Z+hbshQn6pGMqjcT1qs/HQWWLOpvK/Rbv4oF8jsyOJiIiIiNwyq9Vg7YEzzNl0lF/2nsb4syFCBXcn/i+4Cn2bFZ2GCPmlYiiPXBzteD+0IfdP3ciiqJPcVcuLXg0rmR1LREREROSmXEhN57vI41c0RGhdvTwPtfCjY5BXkWuIkF8qhvKhUZWyDL2rOpN+PsC4hbto6u9BpTLOZscSEREREcmznccTmb0phkVRJ0nLzN0QYUCIH9U8i25DhPxSMZRPQzpUZ/W+M0Qdu8Az30Qx9/EW2NgUn3mTIiIiIlL8XM7IYsmO7IYIUccu5Cyv5ePKwBB/ejeqSCmHklcalLwjvkV2tjZMCm1I9w/XsenwOb5Yf5gn2lYzO5aIiIiIyBWu1RChe70KDGjhR5Ni1hAhv1QM3QT/8i683LM2L8zfyTsr9tG6uie1K7qZHUtEREREBKvVYM2BM8zZeJRf9/3dEKGiuxMPBlchtFkVPF0dzQ1ZSKgYukl9m/ryS/RpftpzihER21g8pDVO9rZmxxIRERGREupCajrfbD3GnE2xxJ77uyFCmxrZDRH+U6v4N0TILxVDN8lisRB+Xz3+iL3A/lMXeWv5Xl7uWcfsWCIiIiJSwuw4foHZG4+yePs/GiI42fFAE1/+r0WVEtUQIb9UDN2CcqUdeeeB+jwy43dm/BbDXbW8aFPD0+xYIiIiIlLM/dUQYdamo2z/R0OEoApuDAzxo1fDktkQIb/0Ct2iDjW9GBjix6yNR3n22+0sH96Wsi4OZscSERERkWLo2LlU5mw+yje/H+N8agYADrY2dK/nw4AQPxpXKdkNEfJLxdBtMKZbEL8dTODQmRRe/H4nk/+vsf4jFBEREZHbwmo1WLP/DLM3HWXVvxoi/F8LP0Kb+VK+tBoi3AwVQ7eBs4MtH/RrRO9PfmPZrnjm/3GC+5tUNjuWiIiIiBRh51OyGyJ8tfnKhggDWvhxlxoi3DIVQ7dJ3UrujOocyNvL9/Hyol009/egSrlSZscSERERkSJm+7ELzN50lB+u0hDhoRZVqKqGCLeNiqHb6Mm21Vi99wxbYs4x6psoIp4MwdZG0+VERERE5PouZ2Txw/aTzNl0lO3HE3OW1/6zIcI9aohQIHRd7TaytbHwXt8GlHa0Y+vR80xdc8jsSCIixcLkyZMJCAjAycmJJk2asG7dumuOXbBgAZ06dcLT0xM3NzdCQkJYsWJFrjG7d++mT58++Pv7Y7FYmDRp0hXbGT9+PBaLJdfDx8fndh+aiJRwsWdTCV8aTYvwX3juux1sP56Ig60N9zaqxPynWrJkWGv6Na+iQqiAqBi6zXw9SvFqr+z7Db2/cj87jl8wN5CISBEXERHBiBEjGDt2LNu2baNNmzZ069aN2NjYq45fu3YtnTp1YunSpURGRtKhQwd69uzJtm3bcsakpqZStWpV3nzzzesWOHXq1CEuLi7nsXPnztt+fCJS8mRZDVbtPc0jM7bQ7t1VfLr2MBdSM6hUxpnnu9Zkw5i7eD+0IU381BmuoFkM469+FEVbUlIS7u7uJCYm4ubmZmoWwzAYMm8bS3bEUbW8Cz8Oa61qXkSKrYJ+/w0ODqZx48ZMmTIlZ1lQUBC9e/cmPDw8T9uoU6cOoaGh/O9//7viOX9/f0aMGMGIESNyLR8/fjwLFy4kKirqpnIXpvOSiBQOfzVEmLP5KMfOXcpZ3jbQM6chgn5icXvk9T1Yn9ALgMVi4fXedYmMOc/hhBTeWBrNhN71zI4lIlLkpKenExkZyejRo3Mt79y5Mxs2bMjTNqxWK8nJyXh4eOR7/wcOHKBixYo4OjoSHBzMG2+8QdWqVfO9HREp2aKOXWD2xqP8sOMk6X82RHBzsuOBpr481MKPgPIuJicsuVQMFZAypRx4r28D/u+LzczZFMtdtby4q5a32bFERIqUhIQEsrKy8PbO/f7p7e1NfHx8nrbx3nvvkZKSQt++ffO17+DgYGbNmkVgYCCnTp1iwoQJtGzZkt27d1OuXLkrxqelpZGWlpbzd1JSUr72JyLFy+WMLBb/2RBhxz8aItSp+GdDhAaVcHawNTGhgIqhAtWqenkeax3AtPVHeP67HSwf0VY3xBIRuQn/njNvGEae5tHPmzeP8ePHs2jRIry8vPK1z27duuX8u169eoSEhFCtWjW+/PJLRo0adcX48PBwXnnllXztQ0SKn6NnU/hqcyzfbD3GhdQMABxsbehRvwIDQvxo5FtGvwMqRPLVQCE8PJxmzZrh6uqKl5cXvXv3Zt++fTdcb82aNTRp0gQnJyeqVq3K1KlTrxgzadIkatasibOzM76+vowcOZLLly/nJ16h9FyXmtT0diXhYjqj5++kmPxES0Tkjihfvjy2trZXXAU6ffr0FVeL/i0iIoLHHnuMb775ho4dO95yFhcXF+rVq8eBAweu+vyYMWNITEzMeRw7duyW9ykiRUOW1eDXvacYNGML7d9dzWf/aoiw8c+GCI2rqCFCYZOvYmjNmjWEhYWxadMmVq5cSWZmJp07dyYlJeWa6xw5coTu3bvTpk0btm3bxosvvsiwYcOYP39+zpivvvqK0aNH8/LLLxMdHc20adOIiIhgzJgxN39khYSTvS2T+jXEwdaGn6NP8fXvOjmKiOSVg4MDTZo0YeXKlbmWr1y5kpYtW15zvXnz5jFo0CDmzp1Ljx49bkuWtLQ0oqOjqVChwlWfd3R0xM3NLddDRIq3cynpTF1ziPbvruLRmVtZve8MhgHtAj35YmBT1j7fgafbV6ecZgYVWvmaJrd8+fJcf8+YMQMvLy8iIyNp27btVdeZOnUqVapUybmHQ1BQEFu3buXdd9+lT58+AGzcuJFWrVrx4IMPAtmdffr378+WLVvyezyFUlAFN57vWpMJS6J59Yc9tKhaTj+UExHJo1GjRjFgwACaNm1KSEgIn332GbGxsQwePBjIviJz4sQJZs2aBWQXQgMHDuSDDz6gRYsWOVeVnJ2dcXd3B7IbM+zZsyfn3ydOnCAqKorSpUtTvXp1AJ599ll69uxJlSpVOH36NBMmTCApKYmHH374Tr8EIlKIGIbB9uOJzNoYw4874nIaIrg729O3aWX+L9gPf33OKzJu6TdDiYnZPwa7XoeejRs30rlz51zLunTpwrRp08jIyMDe3p7WrVszZ84ctmzZQvPmzTl8+DBLly697gmnqP1Q9dFWAfy69zQbDp1lREQU3w0Owd5Wt3kSEbmR0NBQzp49y6uvvkpcXBx169Zl6dKl+Pn5ARAXF5frnkOffvopmZmZhIWFERYWlrP84YcfZubMmQCcPHmSRo0a5Tz37rvv8u6779KuXTtWr14NwPHjx+nfvz8JCQl4enrSokULNm3alLNfESlZ/mqIMHvjUXae+LshQt1Kbgxs4U/PBhXVEKEIuun7DBmGQa9evTh//vx17wQeGBjIoEGDePHFF3OWbdiwgVatWnHy5Mmc6QYfffQRzzzzDIZhkJmZyVNPPcXkyZOvud3x48df9Yeqhfl+DnGJl+jy/lqSLmcy7D81GNUp0OxIIiK3TPfTuTq9LiLFQ0xCCl9tPso3W4+TeOnPhgh2NtxdvwIDWvjRUA0RCqUCv8/QkCFD2LFjB+vXr7/h2Kt1Afrn8tWrV/P6668zefJkgoODOXjwIMOHD6dChQq89NJLV93mmDFjcnXzSUpKwtfX92YP546o4O7M6/fWY+i8bXyy6iDtAj1p4lfW7FgiIiIi8g9ZVoNVe08ze9NR1uw/k7O8UhlnHmrhR2gzXzxcHExMKLfLTRVDQ4cOZfHixaxdu5bKlStfd6yPj89VuwDZ2dnl3KfhpZdeYsCAATz++ONAdgvTlJQUnnjiCcaOHYuNzZXTyRwdHXF0LHo/RuvZoCK/7j3N99tOMOqbKJYMa0NpR3U4FxERETHb2YtpfLP1OF9tPsrx85dylrcL9GRgiB/ta3pha6OrQMVJvj6FG4bB0KFD+f7771m9ejUBAQE3XCckJIQffvgh17KffvqJpk2bYm9vD0BqauoVBY+trS2GYRTLVtSv9KrDliPnOHo2ldd+2MNb99c3O5KIiIhIiWQYBlHHLjB749HshghZuRsiPNTCD79yaohQXOWrGAoLC2Pu3LksWrQIV1fXnCs+7u7uODs7A1d29Rk8eDAff/wxo0aN4r///S8bN25k2rRpzJs3L2e7PXv2ZOLEiTRq1ChnmtxLL73EPffcg61t8fshmpuTPRP7NqDf55uI2HqMDrW86FrXx+xYIiIiIiXGpfQsfth+klmbYth14u9GXPUquTMgxI97GlTEyb74fQ6V3PJVDE2ZMgWA9u3b51o+Y8YMBg0aBFzZ1ScgIIClS5cycuRIPvnkEypWrMiHH36Y01YbYNy4cVgsFsaNG8eJEyfw9PSkZ8+evP766zd5WIVfcNVyDG5XjSmrDzFmwQ4aVymDl5uT2bFEREREirWYhBTmbDrKt5G5GyL0rF+RASHZDRGk5LjpbnKFTVHs2pOeaeXeyb+x+2QS7QI9mflIM3UjEZEipyi+/94Jel1ECo+/GiLM2nSUtf9oiFC5bHZDhL5N1RChuCnwbnJy6xzsbJgU2pC7P1rPmv1nmL3pKAND/M2OJSIiIlIsnL2YRsTWY3y1KZYTF7IbIlgs0D7QkwEhfrQLVEOEkk7FkMlqeLvyYvcgXl68m9eXRNOyWjmqe7maHUtERESkSDIMgz9iLzBn01GW/KMhQplS9oQ29eXB4CpqiCA5VAwVAgND/Ph172nW7D/D8K+j+P7pVjjYXdlOXERERESu7lJ6Fou3n2DWxqPsPvl3Q4T6ld0Z0MKPnmqIIFehYqgQsFgsvHN/fbpMWsvuk0m8//N+Xuhay+xYIiIiIoXekb8aImw9RtLlTODvhggDQ/xooIYIch0qhgoJLzcnwu+rz+A5kUxdc4j2gZ4EVy1ndiwRERGRQifLavDr3tPM2hjDugMJOct9PZx5KDi7IUJZNUSQPFAxVIh0retD36aV+WbrcUZ9s51lI9rg5mRvdiwRERGRQiHhYhoRvx9j7ubcDRE61PRiQAs/2gV6YqOGCJIPKoYKmf/1rMOmw+eIPZfK+EW7mRja0OxIIiIiIqb5qyHC7I0xLN0Zf0VDhP8L9qNKuVImp5SiSsVQIVPa0Y73QxvywNQNLNh2gg61vOjZoKLZsURERETuqNT0TBZHnWTWxqPsifu7IUID3zIMaOHH3fUrqCGC3DIVQ4VQE7+yDLmrBh/+coCx3++kqX9ZKrg7mx1LREREpMAdPnOROZti+TbyGMl/NkRwtLPhngYVGRDiR/3KZcwNKMWKiqFCauhd1Vmz/wzbj13gmW+2M+exYM2BFRERkWIpM8vKr3tPM3vT0VwNEap4lOKhFlV4oIkaIkjBUDFUSNnb2jAptCHdP1jHhkNnmf7bER5vU9XsWCIiIiK3zXUbIoT40a6GGiJIwVIxVIgFlHfhpbtr8+L3O3l7+T5a1yhPLR83s2OJiIiI3LTshgjnmbXxKEt3xpGRZQBQtpQ9fZv58lCwH74eaoggd4aKoUKuf3Nfft17ip+jTzPi6ygWhrXSjwVFRESkyElNz2RR1Elm/6shQsM/GyL0UEMEMYGKoULOYrHwZp/6dJ20lr3xyby7Yh/j7q5tdiwRERGRPDl85iKzNx3lu8jjVzREGBjiT73K7iYnlJJMxVARUL60I2/fX59HZ27li/VH6FDLi1bVy5sdS0REROSqMrOs/LL3NLM3HmX9wb8bIviVK8VDwX480LQyZUqpIYKYT8VQEXFXLW8ealGFOZtieeab7Swf0UZvIiIiIlKonElOI+L3WOZujuVk4mUguyHCf2p58VALP9qqIYIUMiqGipCx3Wuz4eBZDiekMHbhLj7u3wiLRW8oIiIiYh7DMIg8mt0QYdmuvxsieLg4ENrMlwebV1FDBCm0VAwVIc4Otkzq15D7Jm9gyY44OgZ5cW+jymbHEhERkRIoNT2ThdtOMnvTUaL/1RBhYIgf3eupIYIUfiqGipj6lcswomMN3v1pP/9buJumfh76tkVERETumENnLjJ741HmRx4nOe3vhgi9GlZkQAs1RJCiRcVQEfRU++qs3neGrUfP88w325n3RAtsNf9WRERECkhmlpWfo08ze1MMvx08m7Pcv1wpHmrhx/1N1BBBiiYVQ0WQrY2F90Mb0u2DdWyJOcenaw/xdPvqZscSERGRYuZMchpfb4ll7pZY4v7VEGFAiD9tqpdXQwQp0lQMFVG+HqUYf08dnv12OxN/2k/bGp7UraTL0iIiInLrLqSm8/Li3SzdqYYIUrypGCrC+jSuxC/Rp1i2K57hX2/jx6FtcHbQDxVFRETk1ry8eDeLok4C0LhKGQb82RDB0U6fM6R4sTE7gNw8i8XCG/fWw8vVkUNnUnhzWbTZkURERKSI2xZ7nkVRJ7FY4KvHg1nwdCvubVRZhZAUSyqGiriyLg68+0ADAL7ceJRV+06bnEhERESKKsMwmLAk+8vVPo0r06p6eZMTiRQsFUPFQNtATx5p5Q/A89/t4OzFNHMDiYiISJH04444Io+ep5SDLc91qWl2HJECp2KomHihay0CvUtzJjmNMQt2YhiG2ZFERESkCLmckcWby/YCMLhdNbzdnExOJFLwVAwVE072tkwKbYS9rYWf9pzim63HzI4kIiIiRcj0345w4sIlKrg78d82Vc2OI3JHqBgqRmpXdOPZztmXtF/5YQ9Hz6aYnEhERESKgjPJaUxedQiA57vWVHdaKTFUDBUzj7epSouqHqSmZzEiIorMLKvZkURERKSQm7hyHxfTMmlQ2Z1eDSqZHUfkjlExVMzY2lh4r29DXJ3s2BZ7gU/+/JZHRERE5Gqi45KI+D17ev24u2tjY2MxOZHInaNiqBiqVMaZCb3rAvDhrwfYFnve5EQiIiJSGBmGwetLorEa0KNeBZr5e5gdSeSOUjFUTPVqWIleDSuSZTUYGRFFSlqm2ZFERESkkPl172nWH0zAwdaG0d1qmR1H5I5TMVSMvdqrLhXdnYg5m5pzAzURERERgIwsK68vzf588Ehrf3w9SpmcSOTOUzFUjLk72/Nu3wZYLDBvSywr95wyO5KIiIgUEl9tOsrhMymUc3FgSIfqZscRMYWKoWKuZbXyPPHnvQJemL+D08mXTU4kIiIiZktMzWDSLwcAGNU5EFcne5MTiZhDxVAJMKpzIEEV3DiXks4L3+3AMAyzI4mIiIiJPvz1ABdSMwj0Lk1oU1+z44iYRsVQCeBoZ8sH/RriYGfDqn1nmLM51uxIIiIiYpIjCSnM2hgDwLgetbGz1cdBKbn0X38JEejtyuiu2V1iXl+yh0NnLpqcSEQk7yZPnkxAQABOTk40adKEdevWXXPsggUL6NSpE56enri5uRESEsKKFStyjdm9ezd9+vTB398fi8XCpEmTbnm/IkXFG0ujycgy6FDTk7aBnmbHETGViqESZFBLf9rUKM/lDCsjvo4iI8tqdiQRkRuKiIhgxIgRjB07lm3bttGmTRu6detGbOzVr3KvXbuWTp06sXTpUiIjI+nQoQM9e/Zk27ZtOWNSU1OpWrUqb775Jj4+PrdlvyJFwYZDCazccwpbGwtjewSZHUfEdBajmPyAJCkpCXd3dxITE3FzczM7TqF1KukyXSat5UJqBkM6VOfZLjXNjiQiRVxBv/8GBwfTuHFjpkyZkrMsKCiI3r17Ex4enqdt1KlTh9DQUP73v/9d8Zy/vz8jRoxgxIgRt3W/Oi9JYZNlNej50Xr2xCUxMMSPV3vVNTuSSIHJ63twvq4MhYeH06xZM1xdXfHy8qJ3797s27fvhuutWbOGJk2a4OTkRNWqVZk6deoVYy5cuEBYWBgVKlTAycmJoKAgli5dmp94kgfebk68cW89ACavPsjvMedMTiQicm3p6elERkbSuXPnXMs7d+7Mhg0b8rQNq9VKcnIyHh4eBbrftLQ0kpKScj1ECpP5kcfZE5eEq5MdIzoGmh1HpFDIVzG0Zs0awsLC2LRpEytXriQzM5POnTuTkpJyzXWOHDlC9+7dadOmDdu2bePFF19k2LBhzJ8/P2dMeno6nTp1IiYmhu+++459+/bx+eefU6lSpZs/Mrmm7vUqcH+TylgNGBkRRfLlDLMjiYhcVUJCAllZWXh7e+da7u3tTXx8fJ628d5775GSkkLfvn0LdL/h4eG4u7vnPHx91aFLCo+UtEze+Sn7C+xhd9XAw8XB5EQihYNdfgYvX748198zZszAy8uLyMhI2rZte9V1pk6dSpUqVXJ+nBoUFMTWrVt599136dOnDwDTp0/n3LlzbNiwAXv77D73fn5++T0WyYeXe9Zm85GzHDt3ifGL9/Be3wZmRxIRuSaLxZLrb8Mwrlh2NfPmzWP8+PEsWrQILy+vAt3vmDFjGDVqVM7fSUlJKoik0Ji65hBnktPwK1eKgS31GUvkL7fUQCExMRHgulMPNm7ceMU0gy5durB161YyMrKvSCxevJiQkBDCwsLw9vambt26vPHGG2RlZV1zu5qOcGtcnex5v29DbCww/4/jLN0ZZ3YkEZErlC9fHltb2yuuxpw+ffqKqzb/FhERwWOPPcY333xDx44dC3y/jo6OuLm55XqIFAYnLlzis7WHARjTLQhHO1uTE4kUHjddDBmGwahRo2jdujV16177B3jx8fFXnWaQmZlJQkICAIcPH+a7774jKyuLpUuXMm7cON577z1ef/31a25X0xFuXVN/D55uXx2AF7/fSXziZZMTiYjk5uDgQJMmTVi5cmWu5StXrqRly5bXXG/evHkMGjSIuXPn0qNHjzu2X5HC6J3le0nLtBIc4EGXOtf/EkGkpMnXNLl/GjJkCDt27GD9+vU3HHu1aQb/XG61WvHy8uKzzz7D1taWJk2acPLkSd55552rdv4BTUe4XYZ3rMHaA2fYcTyR577bzpePNMfG5sZTT0RE7pRRo0YxYMAAmjZtSkhICJ999hmxsbEMHjwYyD4fnDhxglmzZgHZhdDAgQP54IMPaNGiRc7VHWdnZ9zd3YHs36ru2bMn598nTpwgKiqK0qVLU7169TztV6QoiDp2gYVRJ7FY4KW7a+dpeqlISXJTxdDQoUNZvHgxa9eupXLlytcd6+Pjc9VpBnZ2dpQrVw6AChUqYG9vj63t35dtg4KCiI+PJz09HQeHK3/k5+joiKOj483El3+wt7Xh/dCG9PhwHesOJDBzQwyPtg4wO5aISI7Q0FDOnj3Lq6++SlxcHHXr1mXp0qU5vy2Ni4vLde+fTz/9lMzMTMLCwggLC8tZ/vDDDzNz5kwATp48SaNGjXKee/fdd3n33Xdp164dq1evztN+RQo7wzB47cfsor9P48rUreRuciKRwidfxZBhGAwdOpTvv/+e1atXExBw4w/NISEh/PDDD7mW/fTTTzRt2jSnWUKrVq2YO3cuVqsVG5vsmXv79++nQoUKVy2E5Paq5lmacT1qM27hLt5cvpdW1ctT08fV7FgiIjmefvppnn766as+91eB85e/ipnr8ff3Jy+32bvefkUKuyU744g8eh5ne1ue030FRa4qX78ZCgsLY86cOcydOxdXV1fi4+OJj4/n0qVLOWPGjBnDwIEDc/4ePHgwR48eZdSoUURHRzN9+nSmTZvGs88+mzPmqaee4uzZswwfPpz9+/ezZMkS3njjjVzf6EnB+r/gKtxVy4v0TCvDv95GWua1m1eIiIhI4XY5I4s3l+0FYHC7ani7OZmcSKRwylcxNGXKFBITE2nfvj0VKlTIeUREROSM+fd0hYCAAJYuXcrq1atp2LAhr732Gh9++GFOW20AX19ffvrpJ37//Xfq16/PsGHDGD58OKNHj74Nhyh5YbFYeKtPfcq5OLA3PpmJP+03O5KIiIjcpOm/HeH4+UtUcHfiibZVzY4jUmhZjLzMEygCkpKScHd3JzExUe1Mb8HKPaf476ytWCzw1ePBtKxW3uxIIlLI6f336vS6iFnOJKfR4d3VXEzL5P3QBtzb6Pq/7xYpjvL6HnxL9xmS4qdTbW/6N6+CYcAz32wnMTXD7EgiIiKSDxNX7udiWib1K7vTq0Els+OIFGoqhuQKL90dREB5F+ISL/PSol1mxxEREZE82hufRMTv2T9XeOnu2rpdhsgNqBiSK5RysOP90IbY2lhYvP0ki6JOmB1JREREbsAwDCb8GI3VgB71KtDM38PsSCKFnoohuaqGvmUYdlcNAMYt3MWJC5dusIaIiIiYadW+06w/mICDrQ0vdK1ldhyRIkHFkFxTWIdqNKpShuTLmYyKiCLLWix6bYiIiBQ7GVlWJiyJBuCR1v5UKVfK5EQiRYOKIbkmO1sbJoU2pJSDLZuPnOOLdYfNjiQiIiJX8dWmoxw+k0I5FwfCOlQ3O45IkaFiSK7Lr5wL43vWAeDdn/ax+2SiyYlERETknxJTM5j0ywEARnYKxM3J3uREIkWHiiG5oQeaVqZLHW8ysgxGfB3F5YwssyOJiIjInz789QAXUjMI9C5Nv2a+ZscRKVJUDMkNWSwWwu+rj6erIwdOX+TNZXvNjiQiIiLAkYQUZm2MAWBcj9rY2eqjnUh+6P8xkiceLg68c399AGZuiGHt/jMmJxIREZHwpdFkZBm0r+lJ20BPs+OIFDkqhiTP2tf04uEQPwCe/XY751PSTU4kIiJScm04lMBPe05ha2NhXI8gs+OIFEkqhiRfRncLorpXaU4npzFmwU4MQ+22RURE7rQsa/YNVgH+L7gK1b1cTU4kUjSpGJJ8cXawZVJoQ+xtLSzfHc93kcfNjiQiIlLizP/jOHviknB1smNEx0Cz44gUWSqGJN/qVnJnVKeaAIxfvJvYs6kmJxIRESk5UtIyeWfFPgCG3VUDDxcHkxOJFF0qhuSmPNG2Ks0DPEhJz2LkN1FkZlnNjiQiIlIiTF1ziDPJafiVK8XAln5mxxEp0lQMyU2xtbEwsW8DXB3tiDx6nqlrDpkdSUREpNg7eeESn609DMCYbrVwtLM1OZFI0aZiSG5a5bKleLV3HQAm/XyA7ccumBtIRESkmHt7+V7SMq0EB3jQpY6P2XFEijwVQ3JLejesxN31K5BpNRgZEUVqeqbZkURERIqlqGMXWBh1EosFXrq7NhaLxexIIkWeiiG5JRaLhdd716OCuxOHE1J4fUm02ZFERESKHcMweO3HPQDc16gydSu5m5xIpHhQMSS3zL2UPe890ACArzbH8kv0KZMTiYiIFC9LdsYRefQ8zva2PN+1ptlxRIoNFUNyW7SsXp7HWwcA8ML8HSRcTDM5kYiISPFwOSOLN5ftBWBwu2p4uzmZnEik+FAxJLfNs11qUsvHlYSL6YyevwPDMMyOJCIiUuTN+C2G4+cv4ePmxH/bBpgdR6RYUTEkt42TvS2T+jXEwdaGn6NPM2/LMbMjiYiIFGlnktP4ZNVBAJ7vWpNSDnYmJxIpXlQMyW1Vy8ctZy7zaz/u4fCZiyYnEhERKbomrtzPxbRM6ld2p3fDSmbHESl2VAzJbfdoqwBaVS/HpYwsRkZEkZFlNTuSiIhIkbM3PomI32MBGNejNjY2aqUtcrupGJLbzsbGwrsPNMDd2Z7txxP56JcDZkcSEREpUgzD4PUl0VgN6F7Ph+YBHmZHEimWVAxJgajg7szr99YF4ONVB4k8et7kRCIiIkXHqn2nWXcgAQdbG0Z3DTI7jkixpWJICszd9StyX6NKWA0YGRHFxbRMsyOJiIgUehlZ1pybmD/S2p8q5UqZnEik+FIxJAVqfK86VCrjTOy5VF79YbfZcURERAq9uZtjOXQmhXIuDoR1qG52HJFiTcWQFCg3J3veD22IxQLfbD3O8l1xZkcSEREptBJTM3j/5/0AjOwUiJuTvcmJRIo3FUNS4JoHePBUu2oAjF6wk1NJl01OJCIiUjh99OsBLqRmEOhdmn7NfM2OI1LsqRiSO2JEx0DqVnLjQmoGz323A8MwzI4kIiJSqBxJSOHLjTEAjO1RGztbfUwTKWj6f5ncEQ52NkwKbYijnQ1r959h1sajZkcSEREpVMKXRpORZdC+piftAj3NjiNSIqgYkjumupcrY3tktwd9Y2k0B04lm5xIRESkcNh46Cw/7TmFrY2Fsd3VSlvkTlExJHfUgBZ+tAv0JC3TyvCvo0jPtJodSURExFRZVoMJS/YA8GDzKtTwdjU5kUjJoWJI7iiLxcI7D9THw8WBPXFJTFy53+xIIiIippr/x3F2n0zC1cmOkZ0CzY4jUqKoGJI7zsvVifD76gHw6dpDbDp81uREIiIi5khJy+SdFfsAGHZXDTxcHExOJFKyqBgSU3Sp40NoU18MA575ZjtJlzPMjiQiInLHfbrmEGeS0/ArV4qBLf3MjiNS4qgYEtP8r2dt/MqV4sSFS7y8aLfZcURERO6okxcu8dm6wwCM6VYLRztbkxOJlDwqhsQ0Lo52vB/aEFsbC99vO8Hi7SfNjiQiInLHvL18L5czrDQP8KBLHR+z44iUSCqGxFSNq5RlSIfqAIz7ficnL1wyOZGIFEaTJ08mICAAJycnmjRpwrp16645dsGCBXTq1AlPT0/c3NwICQlhxYoVV4ybP38+tWvXxtHRkdq1a/P999/nen78+PFYLJZcDx8ffWCV2yPq2AUWRp3EYoGXetTGYrGYHUmkRMpXMRQeHk6zZs1wdXXFy8uL3r17s2/fvhuut2bNGpo0aYKTkxNVq1Zl6tSp1xz79ddfY7FY6N27d36iSRE25K7qNPQtQ9LlTJ75ZjtWq2F2JBEpRCIiIhgxYgRjx45l27ZttGnThm7duhEbG3vV8WvXrqVTp04sXbqUyMhIOnToQM+ePdm2bVvOmI0bNxIaGsqAAQPYvn07AwYMoG/fvmzevDnXturUqUNcXFzOY+fOnQV6rFIyGIbBhB+zW2nf16gy9Sq7m5xIpOSyGIaR50+eXbt2pV+/fjRr1ozMzEzGjh3Lzp072bNnDy4uLldd58iRI9StW5f//ve/PPnkk/z22288/fTTzJs3jz59+uQae/ToUVq1akXVqlXx8PBg4cKFeT6QpKQk3N3dSUxMxM3NLc/rSeFwJCGFHh+uIzU9i7Hdg/hv26pmRxKRPCro99/g4GAaN27MlClTcpYFBQXRu3dvwsPD87SNOnXqEBoayv/+9z8AQkNDSUpKYtmyZTljunbtStmyZZk3bx6QfWVo4cKFREVF3VRunZfkWn7ccZIhc7fhbG/Lqmfb4+PuZHYkkWInr+/B+boytHz5cgYNGkSdOnVo0KABM2bMIDY2lsjIyGuuM3XqVKpUqcKkSZMICgri8ccf59FHH+Xdd9/NNS4rK4v/+7//45VXXqFqVX0QLmkCyrvw0t21AXhnxT6i45JMTiQihUF6ejqRkZF07tw51/LOnTuzYcOGPG3DarWSnJyMh4dHzrKNGzdesc0uXbpcsc0DBw5QsWJFAgIC6NevH4cPH77JIxHJdjkjizeX7QXgyXZVVQiJmOyWfjOUmJgIkOsE82/XOuFs3bqVjIy/2ym/+uqreHp68thjj+Vp32lpaSQlJeV6SNHWr5kvHYO8Sc+yMuLrKC5nZJkdSURMlpCQQFZWFt7e3rmWe3t7Ex8fn6dtvPfee6SkpNC3b9+cZfHx8TfcZnBwMLNmzWLFihV8/vnnxMfH07JlS86evfq90XRekryY8VsMx89fwsfNiSc0C0LEdDddDBmGwahRo2jdujV169a95rhrnXAyMzNJSEgA4LfffmPatGl8/vnned5/eHg47u7uOQ9fX9+bOxApNCwWC2/1qUf50o7sO5WccxM6EZF//7jcMIw8/eB83rx5jB8/noiICLy8vPK1zW7dutGnTx/q1atHx44dWbJkCQBffvnlVfel85LcyJnkND5ZdRCA57vWpJSDncmJROSmi6EhQ4awY8eOnLnV13O1E85fy5OTk3nooYf4/PPPKV++fJ73P2bMGBITE3Mex44dy98BSKFUrrQj79xfH4Bp64+w/kCCyYlExEzly5fH1tb2iqtAp0+fvuKLtn+LiIjgscce45tvvqFjx465nvPx8cn3Nl1cXKhXrx4HDhy46vM6L8mNvP/zfi6mZVK/sju9G1YyO46IcJPF0NChQ1m8eDGrVq2icuXK1x17rROOnZ0d5cqV49ChQ8TExNCzZ0/s7Oyws7Nj1qxZLF68GDs7Ow4dOnTV7To6OuLm5pbrIcVDh1peDGiRfRfuZ76N4kJqusmJRMQsDg4ONGnShJUrV+ZavnLlSlq2bHnN9ebNm8egQYOYO3cuPXr0uOL5kJCQK7b5008/XXebaWlpREdHU6FChas+r/OSXM/e+CS+3pLdAXFcj9rY2KiVtkhhkK/rs4ZhMHToUL7//ntWr15NQEDADdcJCQnhhx9+yLXsp59+omnTptjb21OrVq0rWpWOGzeO5ORkPvjgA00zKKFe7B7Eb4cSOHwmhbHf7+LjBxvpHgwiJdSoUaMYMGAATZs2JSQkhM8++4zY2FgGDx4MZF+ROXHiBLNmzQKyC6GBAwfywQcf0KJFi5wv5JydnXF3z25hPHz4cNq2bctbb71Fr169WLRoET///DPr16/P2e+zzz5Lz549qVKlCqdPn2bChAkkJSXx8MMP3+FXQIo6wzB4fUk0VgO61/OhecC1f2stIndWvq4MhYWFMWfOHObOnYurqyvx8fHEx8dz6dLfN8ocM2YMAwcOzPl78ODBHD16lFGjRhEdHc306dOZNm0azz77LABOTk7UrVs316NMmTK4urpSt25dHBwcbtOhSlHi7GDLB6GNsLOxsGRnHN9vO2F2JBExSWhoKJMmTeLVV1+lYcOGrF27lqVLl+Lnl30FOS4uLtc9hz799FMyMzMJCwujQoUKOY/hw4fnjGnZsiVff/01M2bMoH79+sycOZOIiAiCg4Nzxhw/fpz+/ftTs2ZN7rvvPhwcHNi0aVPOfkXyavW+M6w7kICDrQ2juwaZHUdE/iFf9xm61jfzM2bMYNCgQQAMGjSImJgYVq9enfP8mjVrGDlyJLt376ZixYq88MILOd/oXc2gQYO4cOGC7jMkfLLqIO+s2EdpRzuWDW+Dr0cpsyOJyL/o/ffq9LoIQEaWla6T1nLoTApPtq3KmO4qhkTuhLy+B+erGCrMdNIpnrKsBqGfbmTr0fM08y/L10+EYKt51iKFit5/r06viwB8uSGGlxfvxsPFgdXPtcfNyd7sSCIlQoHcdFXkTrO1sfB+aENKO9rxe8x5pq65ekMNERGRwiYxNYNJP+8HYGSnQBVCIoWQiiEp9Hw9SvHKPXUAeH/lfnYeTzQ5kYiIyI199OsBzqdmEOhdmv7N1BBKpDBSMSRFwn2NK9G9ng+ZVoMREdu4lJ5ldiQREZFrOpKQwpcbYwAY26M2drb6yCVSGOn/mVIkWCwWXu9dD283Rw6dSSF8WbTZkURERK7pzWXRZGQZtK/pSbtAT7PjiMg1qBiSIqOsiwPvPtAAgFkbj7Jq72mTE4mIiFxp46GzrNh9ClsbC2PVPU6kUFMxJEVKmxqePNoq+2a/z323g7MX00xOJCIi8rcsq8GEJXsAeLB5FWp4u5qcSESuR8WQFDnPd61JoHdpEi6mMXrBTopJd3gRESkGFvxxnN0nk3B1smNExxpmxxGRG1AxJEWOk70tk0Ib4WBrw8o9p4j4/ZjZkUREREhJy+SdFfsAGHpXdcqVdjQ5kYjciIohKZJqV3Tj2S6BALz64x5iElJMTiQiIiXdp2sOcTo5Db9ypXi4pb/ZcUQkD1QMSZH1eOuqhFQtR2p6FiMiosjMspodSURESqiTFy7x2brDAIzpVgtHO1uTE4lIXqgYkiLLxsbCe30b4OpkR9SxC3y86qDZkUREpIR6Z8U+LmdYaR7gQZc6PmbHEZE8UjEkRVrFMs68fm89AD769SB/xJ43OZGIiJQ0Uccu8P22E1gs8FKP2lgsFrMjiUgeqRiSIu+eBhXp3bAiWVaDkRFRpKRlmh1JRERKCMMwmPBjdivt+xpVpl5ld5MTiUh+qBiSYuGVXnWp6O7E0bOpvPbnSUlERKSgLd0Zz9aj53G2t+W5LjXNjiMi+aRiSIoFd2d73uvbEIsFvv79GD/tjjc7koiIFHOXM7IIXxYNwJPtquLj7mRyIhHJLxVDUmyEVCvHE22rAjB6wU5OJ182OZGIiBRnM36L4fj5S/i4OeWcf0SkaFExJMXKqE6B1K7gxrmUdJ7/bgeGYZgdSUREiqGEi2l88mcX0+e61KSUg53JiUTkZqgYkmLF0c6WD/o1xNHOhtX7zjBn01GzI4mISDE0ceV+LqZlUr+yO/c2qmR2HBG5SSqGpNip4e3KmG61AJiwJJqDpy+anEhERIqTvfFJfL0lFoBxPWpjY6NW2iJFlYohKZYGhvjTpkZ50jKtjIjYRnqm1exIIiJSDBiGwetLorEa0K2uD80DPMyOJCK3QMWQFEs2NhbefaABZUrZs+tEEh/8st/sSCIiUgys3neGdQcScLC1YUy3ILPjiMgtUjEkxZa3mxNv3lcPgMmrD7HlyDmTE4mISFGWkWVlwpLse9k90sqfKuVKmZxIRG6ViiEp1rrWrcADTSpjGDAyIoqkyxlmRxIRkSJq3pZYDp1JwcPFgbC7qpsdR0RuAxVDUuy9fE8dqniU4sSFS4xfvNvsOCIiUgQlpmbw/srsKdcjOwXi5mRvciIRuR1UDEmxV9rRjvdDG2BjgQV/nGDJjjizI4mISBHz0a8HOJ+aQQ2v0vRv5mt2HBG5TVQMSYnQxM+DsA7ZUxpe/H4n8YmXTU4kIiJFRUxCCl9ujAFgbI8g7Gz18UmkuND/m6XEGPafGtSv7E7ipQye/XY7VqthdiQRESkCwpdFk5Fl0C7Qk/Y1vcyOIyK3kYohKTHsbW2YFNoQZ3tb1h9MYMaGGLMjiYhIIbfx0FlW7D6FrY2FcT3USlukuFExJCVKVc/SjLs7+2T21vK97I1PMjmRiIgUVlarkdNKu39zX2p4u5qcSERuNxVDUuI82LwK/6nlRXqmlRFfR3E5I8vsSCIiUgjN/+M4u08m4epkx8iOgWbHEZECoGJIShyLxcKbfepTzsWBvfHJvPfTPrMjiYhIIZOSlsk7K7LPD0Pvqk650o4mJxKRgqBiSEokT1dH3upTH4Av1h9hw8EEkxOJiEhh8unaw5xOTqOKRykebulvdhwRKSAqhqTE6ljbmweDq2AY8My320lMzTA7koiIFAJxiZf4bO0hAMZ0q4Wjna3JiUSkoKgYkhJtXI8gAsq7EJd4mbELd2IYarctIlLSvb18H5czrDQP8KBrXR+z44hIAVIxJCVaKQc7JoU2xNbGwo874lgUddLsSCIiYqLtxy7w/bYTALzUozYWi8XkRCJSkFQMSYnXwLcMI/5TA4CXFu7i+PlUkxOJiIgZDMPgtR+zW2nf17gS9Sq7m5xIRAqaiiER4Kn21WhcpQzJaZk88812sqyaLiciUtIs3RnP1qPncba35fkutcyOIyJ3gIohEcDO1ob3Qxvi4mDL5iPn+HzdYbMjiYjIHXQ5I4s3l0cD8GS7qvi4O5mcSETuBBVDIn/yK+fCy/fUAeC9n/ax60SiyYlEROROmbkhhmPnLuHt5sgTbauaHUdE7hAVQyL/8ECTynSt40NGlsGIiCguZ2SZHUlERApYwsU0Pv71IADPd6lFKQc7kxOJyJ2iYkjkHywWC2/cVw8vV0cOnr7Im8v2mh1JREQK2Psr93MxLZN6ldy5t1Els+OIyB2Ur2IoPDycZs2a4erqipeXF71792bfvn03XG/NmjU0adIEJycnqlatytSpU3M9//nnn9OmTRvKli1L2bJl6dixI1u2bMnfkYjcJh4uDrzzQAMge9rEmv1nTE4kIiIFZV98MvO2xALw0t21sbFRK22RkiRfxdCaNWsICwtj06ZNrFy5kszMTDp37kxKSso11zly5Ajdu3enTZs2bNu2jRdffJFhw4Yxf/78nDGrV6+mf//+rFq1io0bN1KlShU6d+7MiRMnbv7IRG5Bu0BPBrX0B+DZb7dzLiXd3EAiInLbGYbBhCV7sBrQra4PzQM8zI4kIndYvoqh5cuXM2jQIOrUqUODBg2YMWMGsbGxREZGXnOdqVOnUqVKFSZNmkRQUBCPP/44jz76KO+++27OmK+++oqnn36ahg0bUqtWLT7//HOsViu//PLLzR+ZyC0a3a0W1b1KcyY5jRcX7MQw1G5bxCyTJ08mICAAJycnmjRpwrp16645dsGCBXTq1AlPT0/c3NwICQlhxYoVV4ybP38+tWvXxtHRkdq1a/P999/f0n6l6Fm97wzrDiTgYGvD6G5qpS1SEt3Sb4YSE7O7bXl4XPublI0bN9K5c+dcy7p06cLWrVvJyMi46jqpqalkZGRcd7tpaWkkJSXleojcTk72tkwKbYi9rYXlu+P5NvK42ZFESqSIiAhGjBjB2LFj2bZtG23atKFbt27ExsZedfzatWvp1KkTS5cuJTIykg4dOtCzZ0+2bduWM2bjxo2EhoYyYMAAtm/fzoABA+jbty+bN2++6f1K0ZKRZWXCkuwbrA5q5Y9fOReTE4mIGSzGTX7dbRgGvXr14vz589f9piwwMJBBgwbx4osv5izbsGEDrVq14uTJk1SoUOGKdcLCwlixYgW7du3Cyenqff7Hjx/PK6+8csXyxMRE3NzcbuKIRK5u6ppDvLlsLy4Otiwd3kYnTJF/SUpKwt3dvcDef4ODg2ncuDFTpkzJWRYUFETv3r0JDw/P0zbq1KlDaGgo//vf/wAIDQ0lKSmJZcuW5Yzp2rUrZcuWZd68ebdlvwX9usitmbUxhv8t2o2HiwOrn2uPm5O92ZFE5DbK63vwTV8ZGjJkCDt27Mg5aVyPxZL7x4h/1V//Xg7w9ttvM2/ePBYsWHDNQghgzJgxJCYm5jyOHTuWzyMQyZv/tqlKcIAHKelZjIyIIjPLanYkkRIjPT2dyMjIK2YYdO7cmQ0bNuRpG1arleTk5FyzDa41a+Gvbd6O/UrhlZiawfsr9wMwslOgCiGREuymiqGhQ4eyePFiVq1aReXKla871sfHh/j4+FzLTp8+jZ2dHeXKlcu1/N133+WNN97gp59+on79+tfdrqOjI25ubrkeIgXB1sbCe30b4Opoxx+xF5iy+pDZkURKjISEBLKysvD29s613Nvb+4pzy7W89957pKSk0Ldv35xl8fHx193mzexX07eLjo9XHeB8agY1vErTv5mv2XFExET5KoYMw2DIkCEsWLCAX3/9lYCAgBuuExISwsqVK3Mt++mnn2jatCn29n9/E/POO+/w2muvsXz5cpo2bZqfWCIFrnLZUrzWuy4Ak345QNSxC+YGEilhrjbD4GqzC/5t3rx5jB8/noiICLy8vPK9zfzsNzw8HHd395yHr68+ZBdGMQkpzNwQA8DYHkHY2eqWiyIlWb7eAcLCwpgzZw5z587F1dWV+Ph44uPjuXTpUs6YMWPGMHDgwJy/Bw8ezNGjRxk1ahTR0dFMnz6dadOm8eyzz+aMefvttxk3bhzTp0/H398/Z7sXL168DYcocnv0aliRng0qkmU1GBkRRWp6ptmRRIq98uXLY2tre9UZBv++avNvERERPPbYY3zzzTd07Ngx13PXmrXw1zZvZr+avl00hC+LJiPLoF2gJ+1ret14BREp1vJVDE2ZMoXExETat29PhQoVch4RERE5Y+Li4nJ12gkICGDp0qWsXr2ahg0b8tprr/Hhhx/Sp0+fnDGTJ08mPT2d+++/P9d2/9l+W8RsFouFCb3qUsHdiSMJKUxYEm12JJFiz8HBgSZNmlwxw2DlypW0bNnymuvNmzePQYMGMXfuXHr06HHF89eatfDXNm9mv5q+XfhtOnyWFbtPYWtjYWyPILPjiEghYJefwXlpPDdz5swrlrVr144//vjjmuvExMTkJ4aIadxL2fNe3wb83xebmbs5lrtqetGx9vW/nRaRWzNq1CgGDBhA06ZNCQkJ4bPPPiM2NpbBgwcD2VdkTpw4waxZs4DsQmjgwIF88MEHtGjRIufqjrOzM+7u7gAMHz6ctm3b8tZbb9GrVy8WLVrEzz//zPr16/O8XylarFYjp5V2/+a+BHq7mpxIRAoDTZQVyaeW1crz3zZVAXhh/g7OJKeZnEikeAsNDWXSpEm8+uqrNGzYkLVr17J06VL8/PyAK2ckfPrpp2RmZhIWFpZrtsHw4cNzxrRs2ZKvv/6aGTNmUL9+fWbOnElERATBwcF53q8ULfP/OM6uE0m4OtoxsmOg2XFEpJC46fsMFTa6n4PcSWmZWfT6+Df2xifzn1pefPFw0zz9mFukONL779XpdSk8UtMzaf/Oak4npzGmWy2ebFfN7EgiUsAK/D5DIiWZo50tk/o1xMHOhl/2nmbuFt2RXkSksJq65jCnk9Oo4lGKQa38zY4jIoWIiiGRm1TLx40XutYC4LUf93DojLofiogUNnGJl/hsbfb94cZ0q4Wjna3JiUSkMFExJHILHmnpT+vq5bmcYWVkRBQZWVazI4mIyD+8s3wflzOsNPf3oGtdH7PjiEgho2JI5BbY2Fh494EGuDvbs+N4Ih/+csDsSCIi8qftxy6wYNsJAMbdHaTfdorIFVQMidwiH3cnwu+rB8Anqw4SefScyYlERMQwDF77MbuV9n2NK1G/chlzA4lIoaRiSOQ26F6vAvc1roTVgBERUVxMyzQ7kohIibZsVzxbj57Hyd6G57vUMjuOiBRSKoZEbpNX7qlD5bLOHDt3iVcW7zY7johIiXU5I4vwZdEAPNm2Gj7uTiYnEpHCSsWQyG3i6mTPxL4NsbHAt5HHWbYzzuxIIiIl0swNMRw7dwlvN0eebFfV7DgiUoipGBK5jZoHePBU++yb+Y35fienki6bnEhEpGRJuJjGJ78eBOD5LrUo5WBnciIRKcxUDIncZsP/E0i9Su5cSM3g2W+3Y7UaZkcSESkx3l+5n+S0TOpVcufeRpXMjiMihZyKIZHbzMHOhvdDG+Jkb8O6AwnM2hhjdiQRkRJhX3wy87bEAvDS3bWxsVErbRG5PhVDIgWguldpxnYPAiB82V72n0o2OZGISPH3+tJorAZ0q+tD8wAPs+OISBGgYkikgDzUwo/2NT1Jy7Qy4uso0jKzzI4kIlJsrdp3mrX7z+Bga8PobmqlLSJ5o2JIpIBYLBbevr8+Hi4O7IlLYuLK/WZHEhEpljKyrLy+JLuV9qBW/viVczE5kYgUFSqGRAqQl6sTb95XD4DP1h5m46GzJicSESl+vt4Sy8HTF/FwcSCsQ3Wz44hIEaJiSKSAda7jQ//mvhgGPPNNFImXMsyOJCJSbCReysi58j6yYw3cne1NTiQiRYmKIZE7YFyP2viXK8XJxMu8vGiX2XFERIqNj389wPnUDGp4laZ/8ypmxxGRIkbFkMgd4OJox8TQhtjaWFgYdZJFUSfMjiQiUuTFJKQwc0MMAGN7BGFnq481IpI/etcQuUMaVynL0Luy57KPW7iLExcumZxIRKRoe3PZXjKyDNoGetK+ppfZcUSkCFIxJHIHDelQnYa+ZUi+nMkz30RhtRpmRxIRKZI2HT7L8t3x2NpYGNcjyOw4IlJEqRgSuYPsbG2YFNqQUg62bDp8ji/WHzY7kohIkWO1GkxYsgeA/s19CfR2NTmRiBRVKoZE7jD/8i683LM2AO+s2Meek0kmJxIRKVoWbDvBrhNJuDraMbJjoNlxRKQIUzEkYoK+TX3pXNubjCyDERHbuJyRZXYkEZEiITU9k3dW7AVgyF3VKVfa0eREIlKUqRgSMYHFYiH8vnqUL+3I/lMXeXv5PrMjiYgUCVPXHOZUUhq+Hs4MauVvdhwRKeJUDImYpFxpR955oD4A0387wroDZ0xOJCJSuMUlXuKztYcAGNMtCEc7W5MTiUhRp2JIxEQdanoxMMQPgKfm/MHSnXEmJxIRKbzeWb6PyxlWmvt70K2uj9lxRKQYUDEkYrIx3YIIDvDgYlomT3/1B6/8sJv0TKvZsURECpXtxy6wYFv2DavH3R2ExWIxOZGIFAcqhkRM5uxgy1ePB/Nku6oAzPgthtDPNuqmrCIifzKMv1tp39eoEvUrlzE3kIgUGyqGRAoBO1sbxnQL4vOBTXFzsmNb7AV6fLiOVftOmx1NRMR0y3bF83vMeZzsbXiua02z44hIMaJiSKQQ6VTbmyXD2lCvkjsXUjN4ZMbvvLtiH1lWw+xoIiKmuJyRRfiyaACebFuNCu7OJicSkeJExZBIIePrUYrvngphQIvsxgofrzrIgGmbOZOcZnIyEZE778sNMRw7dwlvN8ec6cQiIreLiiGRQsjRzpbXetflg34NKeVgy4ZDZ+nx4To2Hz5rdjQRkTsm4WIaH/96EIDnutSilIOdyYlEpLhRMSRSiPVqWInFQ1oT6F2a08lpPPjFZqasPoRV0+ZEpAR4f+V+ktMyqVfJnfsaVTI7jogUQyqGRAq56l6lWRjWivsaVSLLavDW8r38d9ZWLqSmmx1NRKTA7D+VzLwtsQCM6xGEjY1aaYvI7adiSKQIKOVgx3t9GxB+Xz0c7Gz4Ze9peny4nu3HLpgdTUSkQExYEo3VgK51fAiuWs7sOCJSTKkYEikiLBYL/ZtXYcFTLfErV4oTFy7xwNSNzNoYg2Fo2pyIFB+r9p1m7f4zONjaMKZ7LbPjiEgxpmJIpIipW8mdH4a2pksdb9KzrPxv0W6GfR3FxbRMs6OJiNyyzCwrry/JbqU9qJU/fuVcTE4kIsWZiiGRIsjNyZ6pDzVhXI8g7Gws/LD9JPd8vJ598clmRxMRuSXztsRy8PRFPFwcCOtQ3ew4IlLM5asYCg8Pp1mzZri6uuLl5UXv3r3Zt2/fDddbs2YNTZo0wcnJiapVqzJ16tQrxsyfP5/atWvj6OhI7dq1+f777/MTTaTEsVgsPN6mKhFPtsDHzYnDZ1Lo9cl65kceNzuaiMhNSbyUwcSV+wEY2bEG7s72JicSkeIuX8XQmjVrCAsLY9OmTaxcuZLMzEw6d+5MSkrKNdc5cuQI3bt3p02bNmzbto0XX3yRYcOGMX/+/JwxGzduJDQ0lAEDBrB9+3YGDBhA37592bx5880fmUgJ0cTPgyXDWtOmRnkuZ1h55tvtjJ6/g8sZWWZHExHJl09WHeR8agbVvUrTv3kVs+OISAlgMW7hl9dnzpzBy8uLNWvW0LZt26uOeeGFF1i8eDHR0dE5ywYPHsz27dvZuHEjAKGhoSQlJbFs2bKcMV27dqVs2bLMmzcvT1mSkpJwd3cnMTERNze3mz0kkSIry2rw8a8HmfTLfgwDaldwY/L/Nca/vObbS8HS++/V6XXJn6NnU+g4cQ0ZWQYzHmlGh5peZkcSkSIsr+/Bt/SbocTERAA8PDyuOWbjxo107tw517IuXbqwdetWMjIyrjtmw4YNtxJPpESxtbEwvGMNZj8aTDkXB/bEJdHzo/Us3xVndjQRkRsKX7qXjCyDtoGeKoRE5I656WLIMAxGjRpF69atqVu37jXHxcfH4+3tnWuZt7c3mZmZJCQkXHdMfHz8NbeblpZGUlJSroeIQOsa5VkyrA3N/MuSnJbJ4Dl/8OoPe0jPtJodTUTkqjYfPsvy3fHYWLJvsCoicqfcdDE0ZMgQduzYkadpbBZL7rtG/zUz75/Lrzbm38v+KTw8HHd395yHr69vfuKLFGs+7k7M/W8LnmhbFYDpvx2h32cbOXnhksnJRERys1oNJvzZSrt/8yoEeruanEhESpKbKoaGDh3K4sWLWbVqFZUrV77uWB8fnyuu8Jw+fRo7OzvKlSt33TH/vlr0T2PGjCExMTHncezYsZs5FJFiy97Whhe7B/HZgCa4OtnxR+wFeny4jtX7TpsdTSTfJk+eTEBAAE5OTjRp0oR169Zdc2xcXBwPPvggNWvWxMbGhhEjRlwxJiMjg1dffZVq1arh5OREgwYNWL58ea4x48ePx2Kx5Hr4+Pjc7kMr8RZsO8HOE4m4OtoxqlOg2XFEpITJVzFkGAZDhgxhwYIF/PrrrwQEBNxwnZCQEFauXJlr2U8//UTTpk2xt7e/7piWLVtec7uOjo64ubnleojIlTrX8WHJ0DbUreTG+dQMHpn5OxN/2keW9aZ7p4jcUREREYwYMYKxY8eybds22rRpQ7du3YiNjb3q+LS0NDw9PRk7diwNGjS46phx48bx6aef8tFHH7Fnzx4GDx7Mvffey7Zt23KNq1OnDnFxcTmPnTt33vbjK8lS0zN5Z8VeAIbcVZ1ypR1NTiQiJU2+iqGwsDDmzJnD3LlzcXV1JT4+nvj4eC5d+nvqzZgxYxg4cGDO34MHD+bo0aOMGjWK6Ohopk+fzrRp03j22WdzxgwfPpyffvqJt956i7179/LWW2/x888/X/XbPBHJvyrlSvHd4Jb8X3AVDAM+/PUgA6dvJuFimtnRRG5o4sSJPPbYYzz++OMEBQUxadIkfH19mTJlylXH+/v788EHHzBw4EDc3d2vOmb27Nm8+OKLdO/enapVq/LUU0/RpUsX3nvvvVzj7Ozs8PHxyXl4enre9uMryT5dc5hTSWn4ejgzqJW/2XFEpATKVzE0ZcoUEhMTad++PRUqVMh5RERE5IyJi4vL9W1dQEAAS5cuZfXq1TRs2JDXXnuNDz/8kD59+uSMadmyJV9//TUzZsygfv36zJw5k4iICIKDg2/DIYoIgJO9La/fW48P+jWklIMtvx08S/cP1rHlyDmzo4lcU3p6OpGRkVd0HO3cufMtdRxNS0vDyckp1zJnZ2fWr1+fa9mBAweoWLEiAQEB9OvXj8OHD193m2rsk3dxiZf4dO0hAMZ0C8LRztbkRCJSEtnlZ3Bebkk0c+bMK5a1a9eOP/7447rr3X///dx///35iSMiN6FXw0rUqejGU3P+4MDpi/T/fBPPdanJk22rXrdpiYgZEhISyMrKynfH0Rvp0qULEydOpG3btlSrVo1ffvmFRYsWkZX1982Kg4ODmTVrFoGBgZw6dYoJEybQsmVLdu/enfOb138KDw/nlVdeuelMJc07y/dxOcNKM/+ydKur32KJiDlu6T5DIlI0VfdyZdGQVtzbqBJZVoM3l+3lv7MiSUzNMDuayFXlt+PojXzwwQfUqFGDWrVq4eDgwJAhQ3jkkUewtf376kS3bt3o06cP9erVo2PHjixZsgSAL7/88qrbVGOfvNtx/AILtp0AYFyP2voiRkRMo2JIpIQq5WDHxL4NeOPeejjY2vBz9Cl6fLSOnccTzY4mkqN8+fLY2trmu+PojXh6erJw4UJSUlI4evQoe/fupXTp0tdtDOTi4kK9evU4cODAVZ9XY5+8MQyD137cA8B9jSrRwLeMuYFEpERTMSRSglksFh4MrsKCp1vi6+HM8fOX6DNlA7M3Hc3TtFiRgubg4ECTJk2u6Di6cuXK63YczSsnJycqVapEZmYm8+fPp1evXtccm5aWRnR0NBUqVLjl/ZZky3bF83vMeZzsbXiua02z44hICadiSESoW8mdH4e2oXNtb9KzrLy0cBfDv44iJS3T7GgijBo1ii+++ILp06cTHR3NyJEjiY2NZfDgwcCVXUwBoqKiiIqK4uLFi5w5c4aoqCj27NmT8/zmzZtZsGABhw8fZt26dXTt2hWr1crzzz+fM+bZZ59lzZo1HDlyhM2bN3P//feTlJTEww8/fGcOvBhKy8wifFn2DVafaFuNCu7OJicSkZIuXw0URKT4cne259MBTfhi3RHeXL6XxdtPsvtkIlMeaqI7woupQkNDOXv2LK+++ipxcXHUrVuXpUuX4ufnB1zZxRSgUaNGOf+OjIxk7ty5+Pn5ERMTA8Dly5cZN24chw8fpnTp0nTv3p3Zs2dTpkyZnPWOHz9O//79SUhIwNPTkxYtWrBp06ac/Ur+zfwthmPnLuHt5sjgdlXNjiMigsUoJnNhkpKScHd3JzExUfO0RW7R1phzDJm7jfikyzjb2/L6vXW5r3Fls2NJIaX336vT65JbwsU0OryzmuS0TN59oAH3N9F7iogUnLy+B2uanIhcoam/B0uGtaZNjfJcyshi1DfbGbNgJ5czsm68sojIVUz6eT/JaZnUreTGfY0qmR1HRARQMSQi11CutCMzH2nOiI41sFhg3pZY+kzZwNGzKWZHE5EiZv+pZOZuzp7K+FKP2tjYqJW2iBQOKoZE5JpsbSyM6BjIrEeb4+HiwO6TSdz90XqW77r5m12KSMkzYUk0VgO61vEhuOqVN6wVETGLiiERuaE2NTxZOqwNTf3Kknw5k8FzIpnw4x4ysqxmRxORQm71vtOs3X8Ge1sLo7vVMjuOiEguKoZEJE983J2Y90QL/tsm+6aUX6w/Qr/PNhGXeMnkZCJSWGVmWXl9SXYr7UEt/fEv72JyIhGR3FQMiUie2dvaMLZHbT4d0ARXJzsij56nx4frWbv/jNnRRKQQmrcllgOnL+Lh4sCQu2qYHUdE5AoqhkQk37rU8eHHoa2pU9GNcynpPDxjCxNX7ifLWiw69YvIbZB4KYP3fz4AwMiONXB3tjc5kYjIlVQMichN8SvnwvynWvJgcBUMAz785QAPT99CwsU0s6OJSCHwyaqDnEtJp7pXafo3r2J2HBGRq1IxJCI3zcneljfurcf7oQ1wtrdl/cEEeny4jt9jzpkdTURMdPRsCjN+OwLA2B5B2Nnq44aIFE56dxKRW3Zvo8osHtKK6l6lOZWURr/PNvHZ2kMYhqbNiZREby7bS0aWQdtATzrU9DI7jojINakYEpHbooa3K4vCWtGrYUWyrAZvLN3LE7MjSbyUYXY0EbmDNh8+y7Jd8dhYYGz3ILPjiIhcl4ohEbltXBztmBTakAm96+Jga8PKPae4+6N17DyeaHY0EbkDrFaDCX+20u7fvAo1fVxNTiQicn0qhkTktrJYLDzUwo/5T7XE18OZY+cu0WfKBuZsOqppcyLF3PfbTrDzRCKujnaM7BRodhwRkRtSMSQiBaJeZXd+HNKGjkHepGdZGbdwFyMiokhJyzQ7mogUgNT0TN5esReAsLuqU760o8mJRERuTMWQiBQY91L2fD6wCWO61cLWxsKiqJP0+uQ3DpxKNjuaiNxmn645zKmkNHw9nHmklb/ZcURE8kTFkIgUKIvFwpPtqjHvvy3wdnPk4OmL3PPxbyzcdsLsaCJym8QnXubTtYcAGNMtCEc7W5MTiYjkjYohEbkjmgd4sGRYG1pVL8eljCxGRETx4vc7uZyRZXY0EblFb6/Yy+UMK838y9Ktro/ZcURE8kzFkIjcMeVLOzLr0WCG/acGFgvM3RzL/VM3EHs21exoInKTdhy/wII/sq/0jutRG4vFYnIiEZG8UzEkIneUrY2FUZ0C+fKR5ni4OLDrRBI9PlrHit3xZkcTkXwyDIMJP2a30r6vUSUa+JYxN5CISD6pGBIRU7QN9GTJsNY08StL8uVMnpwdyetL9pCRZTU7mojk0fJd8WyJOYeTvQ3Pda1pdhwRkXxTMSQipqng7szXT7Tg8dYBAHy+7gj9P9tEfOJlk5OJyI2kZWYRviy7lfYTbatRwd3Z5EQiIvmnYkhETGVva8O4u2sz9aHGuDrasfXoebp/uI51B86YHU1ErmPmbzHEnkvFy9WRwe2qmh1HROSmqBgSkUKha90K/DC0NbUruHEuJZ2B07cw6ef9ZFkNs6OJyL+cvZjGx78eBOC5LjUp5WBnciIRkZujYkhECg3/8i4seLol/Zv7Yhgw6ecDDJqxhbMX08yOJiL/8P7P+0lOy6RuJTf6NK5sdhwRkZumYkhEChUne1vC76vPxL4NcLa3Zd2BBHp8uJ6tMefMjiYiwP5TyczdHAtkt9K2sVErbREpulQMiUihdF/jyiwa0opqni7EJ12m32eb+GLdYQxD0+ZEzPT6kmisBnSp402LquXMjiMicktUDIlIoRXo7criIa25p0FFMq0GE5ZEM3hOJImXMsyOJlIird53mjX7z2Bva2FMtyCz44iI3DIVQyJSqLk42vFBv4a81rsuDrY2rNh9ip4frWfXiUSzo4mUKJlZVl5fkn2D1UEt/fEv72JyIhGRW6diSEQKPYvFwoAWfnz3VAiVyzoTey6V+6ZsYO7mWE2bE7lD5v1+jAOnL1K2lD1D7qphdhwRkdtCxZCIFBn1K5dhydA2dAzyIj3Tyovf72TUN9tJTc80O5pIsZZ4KYP3V+4HYGSnQNyd7U1OJCJye6gYEpEixb2UPZ8NaMrobrWwtbHw/bYT9Pr4Nw6eTjY7mkix9cmqg5xLSae6V2kebF7F7DgiIreNiiERKXJsbCwMbleNuY8H4+XqyIHTF7nn499YFHXC7Ggixc7RsynM/C0GgLHdg7Cz1UcHESk+9I4mIkVWcNVyLBnWhpbVypGansXwr6MYt3AnaZlZZkcTKTbeXLaX9CwrbWqUp31NT7PjiIjcViqGRKRI83R1ZPZjwQy7qzoWC8zZFMv9UzZy7Fyq2dFEirzNh8+ybFc8NpbsG6xaLLrBqogULyqGRKTIs7WxMKpzTWYMakbZUvbsPJFIjw/XsXLPKbOjiRRZ1j/v7QXQv3kVavq4mpxIROT2y3cxtHbtWnr27EnFihWxWCwsXLjwhut88sknBAUF4ezsTM2aNZk1a9YVYyZNmkTNmjVxdnbG19eXkSNHcvny5fzGE5ESrH1NL5YMa0PjKmVIupzJf2dtJXxpNBlZVrOjiRQ53287wc4Tibg62jGyU6DZcURECkS+i6GUlBQaNGjAxx9/nKfxU6ZMYcyYMYwfP57du3fzyiuvEBYWxg8//JAz5quvvmL06NG8/PLLREdHM23aNCIiIhgzZkx+44lICVexjDNfPxHCo60CAPh07WEe/HwT8Yn6ckUkr1LTM3l7xV4Awu6qTvnSjiYnEhEpGHb5XaFbt25069Ytz+Nnz57Nk08+SWhoKABVq1Zl06ZNvPXWW/Ts2ROAjRs30qpVKx588EEA/P396d+/P1u2bMlvPBERHOxs+F/P2jTzL8vz3+3g95jz9PhwHR/0a0TrGuXNjidS6H229jCnktLw9XBmUEt/s+OIiBSYAv/NUFpaGk5OTrmWOTs7s2XLFjIyMgBo3bo1kZGROcXP4cOHWbp0KT169CjoeCJSjHWrV4EfhrYmqIIbZ1PSGTB9Mx/8fACr1TA7mkihFZ94mU/XHAZgdNcgnOxtTU4kIlJwCrwY6tKlC1988QWRkZEYhsHWrVuZPn06GRkZJCQkANCvXz9ee+01Wrdujb29PdWqVaNDhw6MHj36mttNS0sjKSkp10NE5N/8y7vw/dMt6dfMF8OA93/ez6CZv3MuJd3saCKF0tsr9nIpI4tm/mXpXs/H7DgiIgWqwIuhl156iW7dutGiRQvs7e3p1asXgwYNAsDWNvvbptWrV/P6668zefJk/vjjDxYsWMCPP/7Ia6+9ds3thoeH4+7unvPw9fUt6EMRkSLKyd6WN/vU590HGuBkb8Pa/Wfo8eE6Io+eNzuaSKGy4/gFFvyRffNitdIWkZKgwIshZ2dnpk+fTmpqKjExMcTGxuLv74+rqyvly2fP3X/ppZcYMGAAjz/+OPXq1ePee+/ljTfeIDw8HKv16l2gxowZQ2JiYs7j2LFjBX0oIlLE3d+kMgvDWlHV04W4xMuEfrqRL9YdxjA0bU7EMAwm/JjdSvveRpVo4FvG3EAiInfAHbvPkL29PZUrV8bW1pavv/6au+++Gxub7N2npqbm/Psvtra2GIZxzQ8pjo6OuLm55XqIiNxILR83Fg9pzd31K5D5531UnprzB0mXM8yOJmKq5bvi2RJzDid7G57vWtPsOCIid0S+u8ldvHiRgwcP5vx95MgRoqKi8PDwoEqVKowZM4YTJ07k3Eto//79bNmyheDgYM6fP8/EiRPZtWsXX375Zc42evbsycSJE2nUqBHBwcEcPHiQl156iXvuuSdnKp2IyO1S2tGOj/o3onmAB6/9uIflu+OJjk9i8v81pk5Fd7PjidxxaZlZhC/LbqX9RNtqVHB3NjmRiMidke8rQ1u3bqVRo0Y0atQIgFGjRtGoUSP+97//ARAXF0dsbGzO+KysLN577z0aNGhAp06duHz5Mhs2bMDf3z9nzLhx43jmmWcYN24ctWvX5rHHHqNLly58+umnt3h4IiJXZ7FYGBjiz3eDW1KpjDNHz6Zy7+QNzNsSq2lzhdDkyZMJCAjAycmJJk2asG7dumuOjYuL48EHH6RmzZrY2NgwYsSIK8ZkZGTw6quvUq1aNZycnGjQoAHLly+/pf0WZV9uiCH2XCpero482baq2XFERO4Yi1FMzvpJSUm4u7uTmJioKXMiki8XUtMZ9c12ft17GoD7GlViwr11KeWQ74vnJVJBv/9GREQwYMAAJk+eTKtWrfj000/54osv2LNnD1WqVLlifExMDO+//z5NmjTh/fffp127dkyaNCnXmBdeeIE5c+bw+eefU6tWLVasWMGoUaPYsGFDzpd9+d3vvxWV89LZi2m0f2c1yWmZvHN/fR5oqoZEIlL05fU9WMWQiAhgtRpMXXuId1fsw2pAoHdpJv9fE6p7lTY7WqFX0O+/wcHBNG7cmClTpuQsCwoKonfv3oSHh1933fbt29OwYcMriqGKFSsyduxYwsLCcpb17t2b0qVLM2fOnFveLxSd89JLC3cxe9NR6lZyY3FYa2xs1EFORIq+vL4H37EGCiIihZmNjYWn21dn7n9b4OnqyP5TF7nn4/Us3n7S7GglWnp6OpGRkXTu3DnX8s6dO7Nhw4ab3u61bgi+fv36m95vUbz/3YFTyczdkj21fVyP2iqERKTEUTEkIvIPLaqWY8mw1oRULUdqehbD5m3jpYW7SMvMMjtaiZSQkEBWVhbe3t65lnt7exMfH3/T2+3SpQsTJ07kwIEDWK1WVq5cyaJFi4iLi7vp/RbF+99NWBJNltWgSx1vWlQtZ3YcEZE7TsWQiMi/eLk6MefxYIZ0qA7A7E1HeWDqRo6dSzU5Wcn175t/GoZxSzcE/eCDD6hRowa1atXCwcGBIUOG8Mgjj1zRwTQ/+y1q979bve80a/afwd7WwphuQWbHERExhYohEZGrsLWx8GyXmsx4pBllStmz43giPT5cx897TpkdrUQpX748tra2V1yNOX369BVXbfLD09OThQsXkpKSwtGjR9m7dy+lS5cmICDgpvdblO5/l5ll5fUl2TdYfTjEH//yLiYnEhExh4ohEZHr6FDTiyXD2tDQtwxJlzN5fNZWwpdFk5llNTtaieDg4ECTJk1YuXJlruUrV66kZcuWt7x9JycnKlWqRGZmJvPnz6dXr153ZL9mm/f7MQ6cvkjZUvYM/U8Ns+OIiJhGfWNFRG6gUhlnvnkyhPBl0cz4LYZP1xxm29ELfPRgI7zdnG68Abklo0aNYsCAATRt2pSQkBA+++wzYmNjGTx4MMAVN/sGiIqKArJvFH7mzBmioqJwcHCgdu3aAGzevJkTJ07QsGFDTpw4wfjx47FarTz//PN53m9RlXgpg/dX7gdgZKdA3J3tTU4kImIeFUMiInngYGfDyz3r0Mzfg+e/28GWmHP0+HAdH/ZrRMvq5c2OV6yFhoZy9uxZXn31VeLi4qhbty5Lly7Fz88PuPJm30DOvYIAIiMjmTt3Ln5+fsTExABw+fJlxo0bx+HDhyldujTdu3dn9uzZlClTJs/7LaomrzrIuZR0qnm60L/5je+XJCJSnOk+QyIi+XT4zEWe/uoP9sYnY2OBkR0DCetQvcS2Jdb779UVxtcl9mwqHSeuIT3LyoxBzehQy8vsSCIiBUL3GRIRKSBVPUuzMKwVfZtWxmrAeyv388jM3zmXkm52NJHrCl8WTXqWlTY1ytO+pqfZcURETKdiSETkJjjZ2/L2/Q145/76ONnbsGb/Ge7+cB1/xJ43O5rIVW05co5lu+KxsWTfYPVWWpOLiBQXKoZERG7BA019WRjWiqrlXTiZeJm+Uzcyff0RiskMZCkmrFaD137cA0C/5lWo6eNqciIRkcJBxZCIyC2q5ePGoiGt6FG/AplWg1d/3MPTX/1B0uUMs6OJAPD9thPsPJFIaUc7RnUKNDuOiEihoWJIROQ2cHWy5+P+jXjlnjrY21pYtiueez5az56TSWZHkxIuNT2Td1bsAyCsQ3XKl3Y0OZGISOGhYkhE5DaxWCw83NKfbwe3pFIZZ2LOpnLv5N+I+D1W0+bENJ+tPUx80mUql3XmkVb+ZscRESlUVAyJiNxmDX3L8OPQ1nSo6UlappUX5u/k2W93cCk9y+xoUsLEJ17m0zWHARjTLQgne1uTE4mIFC4qhkRECkBZFwemPdyM57rUxMYC8/84Tu9PfuPQmYtmR5MS5J0V+7iUkUVTv7J0r+djdhwRkUJHxZCISAGxsbEQ1qE6Xz3egvKlHdl3Kpl7PlrPD9tPmh1NSoCdxxOZ/8dxAF66W620RUSuRsWQiEgBC6lWjqXDW9Oiqgcp6VkMnbeNlxftIi1T0+akYBjG3620721UiQa+ZcwNJCJSSKkYEhG5A7xcnZjzWDBhHaoB8OXGo/SdupHj51NNTibF0Yrd8WyJOYeTvQ3PdalpdhwRkUJLxZCIyB1iZ2vDc11qMWNQM8qUsmf78UR6fLieX/eeMjuaFCNpmVm8sXQvAE+0qUrFMs4mJxIRKbxUDImI3GEdannx49DWNPAtQ+KlDB6duZW3lu8lM8tqdjQpBr7cEEPsuVS8XB15sl01s+OIiBRqKoZERExQuWwpvn0yhEEt/QGYsvoQ//fFZk4nXTY3mBRpZy+m8dEvBwF4rktNXBztTE4kIlK4qRgSETGJg50N4++pwycPNsbFwZbNR87R/cP1bDiUYHY0KaIm/XyA5LRM6lR0o0/jymbHEREp9FQMiYiYrEf9CvwwtDW1fFxJuJjGQ19s5uNfD2C1GmZHkyLkwKlk5m6JBbJbadvYqJW2iMiNqBgSESkEqnqW5vunW/FAk8pYDXj3p/08+uXvnE9JNzuaFBGvL40my2rQpY43LaqWMzuOiEiRoGJIRKSQcHaw5Z0HGvB2n/o42tmwet8Z7v5oPdtiz5sdTQq5NfvPsHrfGextLYzpFmR2HBGRIkPFkIhIIdO3mS/fP92KgPIunLhwib6fbmTGb0cwDE2bkytlZlmZ8OcNVh8O8ce/vIvJiUREig4VQyIihVDtim4sHtKK7vV8yMgyeOWHPQyZu43kyxlmR5NC5uvfj3Hg9EXKlrJn6H9qmB1HRKRIUTEkIlJIuTrZ88mDjXm5Z23sbS0s2RnHPR//RnRcktnRpJBIupzB+yv3AzCiYyDuzvYmJxIRKVpUDImIFGIWi4VHWgXwzZMhVHR34khCCr0/+Y1vfj9mdjQpBD759SBnU9Kp5unCg8FVzI4jIlLkqBgSESkCGlUpy5JhbWhf05O0TCvPz9/Bs99u51J6ltnRxCSxZ1OZ8VsMAON61MbeVqd0EZH80juniEgRUdbFgekPN+PZzoHYWOC7yOPcO/k3Dp+5aHY0McGby6NJz7LSpkZ52tf0NDuOiEiRpGJIRKQIsbGxMOSuGsx5PJjypR3ZG59Mz4/W8+OOk2ZHkztoy5FzLN0Zj40l+6qQxaIbrIqI3AwVQyIiRVDLauVZOqw1zQM8SEnPYsjcbYxfvJv0TKvZ0aSAWa0GE5Zkt9Lu17wKNX1cTU4kIlJ0qRgSESmivNycmPt4ME+1rwbAzA0xPPDpRo6fTzU5mRSkhVEn2HE8kdKOdozsGGh2HBGRIk3FkIhIEWZna8MLXWsx7eGmuDvbs/3YBe7+aD2r9p42O5oUgNT0TN5evg+AsA7V8XR1NDmRiEjRpmJIRKQY+E+QNz8ObU2Dyu5cSM3gkZm/8/byvWRmadpccfLZ2sPEJ12mcllnHmnlb3YcEZEiT8WQiEgx4etRim8Gh/BwiB8Ak1cf4qFpmzmdfNnkZHI7xCde5tM1hwEY3a0WTva2JicSESn6VAyJiBQjjna2vNKrLh/1b4SLgy2bDp+jx4fr2XjorNnR5Ba9s2IflzKyaOpXlh71KpgdR0SkWFAxJCJSDPVsUJHFQ1tT09uVM8lp/N8Xm/hk1UGsVsPsaHITdh5PZP4fxwEYd7daaYuI3C75LobWrl1Lz549qVixIhaLhYULF95wnU8++YSgoCCcnZ2pWbMms2bNumLMhQsXCAsLo0KFCjg5OREUFMTSpUvzG09ERP5UzbM0C8Na0adxZaxG9pWFx778nfMp6WZHk3wwDIPX/myl3bthRRr6ljE3kIhIMWKX3xVSUlJo0KABjzzyCH369Lnh+ClTpjBmzBg+//xzmjVrxpYtW/jvf/9L2bJl6dmzJwDp6el06tQJLy8vvvvuOypXrsyxY8dwddW9E0REboWzgy3vPlCf5gFl+d+i3azad4a7P1rPJ//XWB+qi4gVu+PZcuQcTvY2PN+1ltlxRESKlXwXQ926daNbt255Hj979myefPJJQkNDAahatSqbNm3irbfeyimGpk+fzrlz59iwYQP29vYA+Pn55TeaiIhchcViIbRZFepVKsPTX0USczaVB6ZuYFyP2gwM8dOUq0IsLTOLN5buBeCJNlWpWMbZ5EQiIsVLgf9mKC0tDScnp1zLnJ2d2bJlCxkZGQAsXryYkJAQwsLC8Pb2pm7durzxxhtkZWUVdDwRkRKjdkU3Fg9tTbe6PmRkGby8eDdD5m3jYlqm2dHkGmZtOErsuVS8XB15sl01s+OIiBQ7BV4MdenShS+++ILIyEgMw2Dr1q1Mnz6djIwMEhISADh8+DDfffcdWVlZLF26lHHjxvHee+/x+uuvX3O7aWlpJCUl5XqIiMj1uTnZM/n/GvO/u2tjZ2NhyY447vloPXvj9R5a2Jy9mMaHvx4A4NkuNXFxzPdkDhERuYECL4ZeeuklunXrRosWLbC3t6dXr14MGjQIAFvb7HskWK1WvLy8+Oyzz2jSpAn9+vVj7NixTJky5ZrbDQ8Px93dPefh6+tb0IciIlIsWCwWHm0dwDeDQ6jo7sThhBR6f/Ibu04kmh1N/mHSzwdIvpxJnYpu3N+4stlxRESKpQIvhpydnZk+fTqpqanExMQQGxuLv78/rq6ulC9fHoAKFSoQGBiYUxwBBAUFER8fT3r61bsejRkzhsTExJzHsWPHCvpQRESKlcZVyvLjsDa0DfSkqZ8HQRXczI4k/1DDuzRlStkzrkdtbGz0uy4RkYJwx66529vbU7ly9jdbX3/9NXfffTc2Ntm1WKtWrZg7dy5WqzVn2f79+6lQoQIODg5X3Z6joyOOjo53JryISDHl4eLAzEHNSEnPxFYfuAuVgSH+9GlcWdPjREQKUL6vDF28eJGoqCiioqIAOHLkCFFRUcTGxgLZV2wGDhyYM37//v3MmTOHAwcOsGXLFvr168euXbt44403csY89dRTnD17luHDh7N//36WLFnCG2+8QVhY2C0enoiI3IiNjQVXJ3uzY8hVqBASESlY+X6X3bp1Kx06dMj5e9SoUQA8/PDDzJw5k7i4uJzCCCArK4v33nuPffv2YW9vT4cOHdiwYQP+/v45Y3x9ffnpp58YOXIk9evXp1KlSgwfPpwXXnjhFg5NRERERETk2iyGYRhmh7gdkpKScHd3JzExETc3zXsXEblT9P57dXpdRETMk9f34AJvoCAiInKrJk+eTEBAAE5OTjRp0oR169Zdc2xcXBwPPvggNWvWxMbGhhEjRlx13KRJk6hZsybOzs74+voycuRILl++nPP8+PHjsVgsuR4+Pj63+9BERMREKoZERKRQi4iIYMSIEYwdO5Zt27bRpk0bunXrlmtK9j+lpaXh6enJ2LFjadCgwVXHfPXVV4wePZqXX36Z6Ohopk2bRkREBGPGjMk1rk6dOsTFxeU8du7ceduPT0REzKNfZoqISKE2ceJEHnvsMR5//HEg+4rOihUrmDJlCuHh4VeM9/f354MPPgBg+vTpV93mxo0badWqFQ8++GDOOv3792fLli25xtnZ2elqkIhIMaYrQyIiUmilp6cTGRlJ586dcy3v3LkzGzZsuOnttm7dmsjIyJzi5/DhwyxdupQePXrkGnfgwAEqVqxIQEAA/fr14/Dhwze9TxERKXx0ZUhERAqthIQEsrKy8Pb2zrXc29ub+Pj4m95uv379OHPmDK1bt8YwDDIzM3nqqacYPXp0zpjg4GBmzZpFYGAgp06dYsKECbRs2ZLdu3dTrly5K7aZlpZGWlpazt9JSUk3nU9ERO4MXRkSEZFCz2LJfUNYwzCuWJYfq1ev5vXXX2fy5Mn88ccfLFiwgB9//JHXXnstZ0y3bt3o06cP9erVo2PHjixZsgSAL7/88qrbDA8Px93dPefh6+t70/lEROTO0JUhEREptMqXL4+tre0VV4FOnz59xdWi/HjppZcYMGBAzu+Q6tWrR0pKCk888QRjx47FxubK7wpdXFyoV68eBw4cuOo2x4wZk3PvPci+MqSCSESkcNOVIRERKbQcHBxo0qQJK1euzLV85cqVtGzZ8qa3m5qaekXBY2tri2EYXOv2e2lpaURHR1OhQoWrPu/o6Iibm1uuh4iIFG66MiQiIoXaqFGjGDBgAE2bNiUkJITPPvuM2NhYBg8eDGRfkTlx4gSzZs3KWScqKgqAixcvcubMGaKionBwcKB27doA9OzZk4kTJ9KoUSOCg4M5ePAgL730Evfccw+2trYAPPvss/Ts2ZMqVapw+vRpJkyYQFJSEg8//PCdfQFERKTAqBgSEZFCLTQ0lLNnz/Lqq68SFxdH3bp1Wbp0KX5+fkD2TVb/fc+hRo0a5fw7MjKSuXPn4ufnR0xMDADjxo3DYrEwbtw4Tpw4gaenJz179uT111/PWe/48eP079+fhIQEPD09adGiBZs2bcrZr4iIFH0W41rzAYqYpKQk3N3dSUxM1NQEEZE7SO+/V6fXRUTEPHl9Dy42V4b+qunUylRE5M766323mHy3dtvovCQiYp68npuKTTGUnJwMoM49IiImSU5Oxt3d3ewYhYbOSyIi5rvRuanYTJOzWq2cPHkSV1fXm7r3xF8tUI8dO6bpDDdBr9+t0et3a/T63Zpbff0MwyA5OZmKFStetSV1SaXzkvn0Gt4avX63Rq/frblT56Zic2XIxsaGypUr3/J21A711uj1uzV6/W6NXr9bcyuvn64IXUnnpcJDr+Gt0et3a/T63ZqCPjfpKzwRERERESmRVAyJiIiIiEiJpGLoT46Ojrz88ss4OjqaHaVI0ut3a/T63Rq9frdGr1/hpP9dbp1ew1uj1+/W6PW7NXfq9Ss2DRRERERERETyQ1eGRERERESkRFIxJCIiIiIiJZKKIRERERERKZFUDImIiIiISIlUooqhyZMnExAQgJOTE02aNGHdunXXHb9mzRqaNGmCk5MTVatWZerUqXcoaeGUn9dv9erVWCyWKx579+69g4kLh7Vr19KzZ08qVqyIxWJh4cKFN1xH/+39Lb+vn/7byy08PJxmzZrh6uqKl5cXvXv3Zt++fTdcT/8N3hk6L90anZduns5Nt0bnpltTmM5NJaYYioiIYMSIEYwdO5Zt27bR5v/buZuWNtoojONHNEHsQmoLNiC0IiUuBPEFiRLbhZBSv4NkqeIL1YWEfgNB6sYiCMGtCxNBcNNAo6W0XbQMSAlUQW1dGEQQKbqoyunieRRSJ7XJJJPR+f8gC4czcLy54OKuNN3d8vz5c/nx44fp/Pb2tvT29kp3d7cYhiEvX76U0dFRicViNm/uDLme34Vv377J3t7e5efx48c2bewcx8fH0tzcLDMzM/80T/Yy5Xp+F8jef9bW1mRoaEg+ffokiURCzs7OJBQKyfHxcdZ3yKA96CVr6CVr6CZr6CZrHNVN6hIdHR06MDCQ8ayxsVEjkYjp/MTEhDY2NmY86+/v10AgULQdnSzX80smkyoienh4aMN2N4eI6NLS0l9nyF52/3J+ZO/v9vf3VUR0bW0t6wwZtAe9ZA29VDh0kzV0k3Wl7CZX/GXo169f8uXLFwmFQhnPQ6GQfPjwwfSdjx8/Xpl/9uyZfP78WU5PT4u2qxPlc34XWlpaxOfzSU9PjySTyWKueWuQvcIge+aOjo5ERKSmpibrDBksPnrJGnrJfuSvMMifuVJ2kysuQwcHB3J+fi61tbUZz2trayWdTpu+k06nTefPzs7k4OCgaLs6UT7n5/P5ZG5uTmKxmMTjcfH7/dLT0yPv3r2zY+UbjexZQ/ayU1UZHx+XYDAoTU1NWefIYPHRS9bQS/Yjf9aQv+xK3U0Veb95A5WVlWX8rKpXnl03b/bcLXI5P7/fL36///Lnzs5O2d3dlampKXny5ElR97wNyF7+yF52w8PDsr6+Lu/fv792lgzag16yhl6yF/nLH/nLrtTd5Iq/DN2/f1/Ky8uv/GvR/v7+lRvmhQcPHpjOV1RUyL1794q2qxPlc35mAoGAbG5uFnq9W4fsFR7ZExkZGZHl5WVJJpNSV1f311kyWHz0kjX0kv3IX+GRP2d0kysuQ16vV9ra2iSRSGQ8TyQS0tXVZfpOZ2fnlfk3b95Ie3u7eDyeou3qRPmcnxnDMMTn8xV6vVuH7BWem7OnqjI8PCzxeFzevn0r9fX1175DBouPXrKGXrIf+Ss8N+fPUd1k6esXbpCFhQX1eDwajUY1lUrpixcv9M6dO7qzs6OqqpFIRPv6+i7nt7a2tKqqSsfGxjSVSmk0GlWPx6OLi4ul+hVKKtfzm56e1qWlJd3Y2NCvX79qJBJREdFYLFaqX6Fkfv78qYZhqGEYKiL66tUrNQxDv3//rqpk7zq5nh/ZyzQ4OKjV1dW6urqqe3t7l5+Tk5PLGTJYGvSSNfSSNXSTNXSTNU7qJtdchlRVX79+rQ8fPlSv16utra0ZX98XDof16dOnGfOrq6va0tKiXq9XHz16pLOzszZv7Cy5nN/k5KQ2NDRoZWWl3r17V4PBoK6srJRg69K7+DrNPz/hcFhVyd51cj0/spfJ7OxEROfn5y9nyGDp0EvW0Ev5o5usoZuscVI3lf2/EAAAAAC4iiv+zxAAAAAA/InLEAAAAABX4jIEAAAAwJW4DAEAAABwJS5DAAAAAFyJyxAAAAAAV+IyBAAAAMCVuAwBAAAAcCUuQwAAAABcicsQAAAAAFfiMgQAAADAlbgMAQAAAHCl3+Dr7lWB1NlEAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 1000x500 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "class Model(tf.keras.Model):\n",
    "    def __init__(self):\n",
    "        super(Model, self).__init__()\n",
    "\n",
    "        self.input_layers = [\n",
    "            MaxPooling2D((2, 2)),\n",
    "            Flatten()\n",
    "        ]\n",
    "\n",
    "        self.hidden_layers = [\n",
    "            Dense(19, activation='relu', kernel_regularizer='l2'),\n",
    "            Dense(19, activation='relu', kernel_regularizer='l2')\n",
    "        ]\n",
    "\n",
    "        self.output_layer = Dense(5, activation='softmax')\n",
    "    \n",
    "    def call(self, inputs):\n",
    "        h = self.input_layers[0](inputs)\n",
    "        h = self.input_layers[1](h)\n",
    "\n",
    "        for hidden_layer in self.hidden_layers:\n",
    "            h = hidden_layer(h)\n",
    "        \n",
    "        return self.output_layer(h)\n",
    "\n",
    "tf_model = Model()\n",
    "\n",
    "tf_model.build(input_shape=(1, 10, 10, 1))\n",
    "tf_model.compile(optimizer='sgd', loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "history = tf_model.fit(X_train, y_train, epochs=3, batch_size=64).history\n",
    "\n",
    "sleep(4)\n",
    "# clear_output()\n",
    "\n",
    "print(f'Train accuracy: {tf_model.evaluate(X_train, y_train, verbose=False)[1]*100:.1f}%')\n",
    "print(f'Test accuracy: {tf_model.evaluate(X_test, y_test, verbose=False)[1]*100:.1f}%')\n",
    "\n",
    "fig, (ax_1, ax_2) = plt.subplots(1, 2, figsize=(10, 5))\n",
    "\n",
    "ax_1.plot(range(3), history['loss'])\n",
    "ax_2.plot(range(3), history['accuracy'])\n",
    "\n",
    "ax_1.set_title('Loss')\n",
    "ax_2.set_title('Accuracy');\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Re-iter Train and Test Accuracies 50 Times"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = [[], []]\n",
    "\n",
    "for i in range(50):\n",
    "    tf_model = Model()\n",
    "\n",
    "    tf_model.build(input_shape=(1, 10, 10, 1))\n",
    "    tf_model.compile(optimizer='sgd', loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "    history = tf_model.fit(X_train, y_train, epochs=3, batch_size=64,verbose=False).history\n",
    "    train = tf_model.evaluate(X_train, y_train, verbose=False)[1]\n",
    "    test = tf_model.evaluate(X_test, y_test, verbose=False)[1]\n",
    "\n",
    "    results[0].append(train)\n",
    "    results[1].append(test)\n",
    "\n",
    "results = np.array(results)\n",
    "results = np.transpose(results)\n",
    "df = pd.DataFrame(results, columns=['train accuracy', 'test accuracy'])\n",
    "df.to_csv('R Data Analysis/data/initial_accuracy.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create `sklearn` Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Accuracy: 25.90%\n",
      "Test Accuracy: 20.34%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\rohan\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:444: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    }
   ],
   "source": [
    "logreg = LogisticRegression()\n",
    "\n",
    "logreg.fit(X_train_sklearn, y_train_sklearn)\n",
    "\n",
    "print(f'Train Accuracy: {logreg.score(X_train_sklearn, y_train_sklearn)*100:.2f}%')\n",
    "print(f'Test Accuracy: {logreg.score(X_test_sklearn, y_test_sklearn)*100:.2f}%')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### More data analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "electron accuracy: 42.9%\n",
      "muon accuracy: 46.3%\n",
      "pion accuracy: 34.9%\n",
      "kaon accuracy: 27.4%\n",
      "proton accuracy: 5.1%\n"
     ]
    }
   ],
   "source": [
    "def individual_particle_test(model):\n",
    "    for i in range(5):\n",
    "        int_y_test = np.array([np.argmax(y, axis=None, out=None) for y in y_test])    # convert back to integer for comparison.\n",
    "        particle_indexes = np.where(int_y_test == i)    # gives indexes for all electron, muon, etc testcases...\n",
    "\n",
    "        X_test_modified = X_test[particle_indexes]\n",
    "        y_test_modified = y_test[particle_indexes]\n",
    "\n",
    "        print(f'{target_names[str(i)]} accuracy: {model.evaluate(X_test_modified, y_test_modified, verbose=False)[1]*100:.1f}%')\n",
    "\n",
    "individual_particle_test(tf_model)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test Results\n",
    "<img src='images/test_1_(19).png'>\n",
    "<img src='images/test 2 (19).png'></br>\n",
    "difference between one test and another (inconsistencies)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Conclusion:\n",
    "* The algorithm shows some promise for some particles, but little for others.\n",
    "    * muon could have 100% accuracy because of its mass... has less quantum effect compared to others.\n",
    "    * electron could be low because of quantum effects\n",
    "    * proton is actually pretty consistent.\n",
    "    * out of mesons, kaon and proton lowest while pion higher (because of higher mass than other mesons)\n",
    "    * next steps, find reason for fluctuations in percentages and relationships between particles considering their properties."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Graph relationship between sample size and accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Train accuracy: 18.6% (50 samples)\n",
      "\t50.0 samples test: 27.2%\n",
      "\t100.0 samples test: 29.0%\n",
      "\t150.0 samples test: 29.3%\n",
      "\t200.0 samples test: 32.3%\n",
      "\t250.0 samples test: 30.0%\n",
      "\t300.0 samples test: 32.5%\n",
      "\t350.0 samples test: 33.3%\n",
      "\t400.0 samples test: 32.8%\n",
      "\t450.0 samples test: 32.9%\n",
      "\t500.0 samples test: 33.0%\n",
      "\t550.0 samples test: 33.1%\n",
      "\t600.0 samples test: 30.9%\n",
      "\t650.0 samples test: 33.2%\n",
      "\t700.0 samples test: 33.2%\n",
      "\n",
      "Train accuracy: 17.9% (100 samples)\n",
      "\t50.0 samples test: 27.2%\n",
      "\t100.0 samples test: 29.0%\n",
      "\t150.0 samples test: 29.3%\n",
      "\t200.0 samples test: 32.3%\n",
      "\t250.0 samples test: 30.0%\n",
      "\t300.0 samples test: 32.5%\n",
      "\t350.0 samples test: 33.3%\n",
      "\t400.0 samples test: 32.8%\n",
      "\t450.0 samples test: 32.9%\n",
      "\t500.0 samples test: 33.0%\n",
      "\t550.0 samples test: 33.1%\n",
      "\t600.0 samples test: 30.9%\n",
      "\t650.0 samples test: 33.2%\n",
      "\t700.0 samples test: 33.2%\n",
      "\n",
      "Train accuracy: 33.7% (150 samples)\n",
      "\t50.0 samples test: 27.2%\n",
      "\t100.0 samples test: 29.0%\n",
      "\t150.0 samples test: 29.3%\n",
      "\t200.0 samples test: 32.3%\n",
      "\t250.0 samples test: 30.0%\n",
      "\t300.0 samples test: 32.5%\n",
      "\t350.0 samples test: 33.3%\n",
      "\t400.0 samples test: 32.8%\n",
      "\t450.0 samples test: 32.9%\n",
      "\t500.0 samples test: 33.0%\n",
      "\t550.0 samples test: 33.1%\n",
      "\t600.0 samples test: 30.9%\n",
      "\t650.0 samples test: 33.2%\n",
      "\t700.0 samples test: 33.2%\n",
      "\n",
      "Train accuracy: 34.4% (200 samples)\n",
      "\t50.0 samples test: 27.2%\n",
      "\t100.0 samples test: 29.0%\n",
      "\t150.0 samples test: 29.3%\n",
      "\t200.0 samples test: 32.3%\n",
      "\t250.0 samples test: 30.0%\n",
      "\t300.0 samples test: 32.5%\n",
      "\t350.0 samples test: 33.3%\n",
      "\t400.0 samples test: 32.8%\n",
      "\t450.0 samples test: 32.9%\n",
      "\t500.0 samples test: 33.0%\n",
      "\t550.0 samples test: 33.1%\n",
      "\t600.0 samples test: 30.9%\n",
      "\t650.0 samples test: 33.2%\n",
      "\t700.0 samples test: 33.2%\n",
      "\n",
      "Train accuracy: 22.9% (250 samples)\n",
      "\t50.0 samples test: 27.2%\n",
      "\t100.0 samples test: 29.0%\n",
      "\t150.0 samples test: 29.3%\n",
      "\t200.0 samples test: 32.3%\n",
      "\t250.0 samples test: 30.0%\n",
      "\t300.0 samples test: 32.5%\n",
      "\t350.0 samples test: 33.3%\n",
      "\t400.0 samples test: 32.8%\n",
      "\t450.0 samples test: 32.9%\n",
      "\t500.0 samples test: 33.0%\n",
      "\t550.0 samples test: 33.1%\n",
      "\t600.0 samples test: 30.9%\n",
      "\t650.0 samples test: 33.2%\n",
      "\t700.0 samples test: 33.2%\n",
      "\n",
      "Train accuracy: 24.7% (300 samples)\n",
      "\t50.0 samples test: 27.2%\n",
      "\t100.0 samples test: 29.0%\n",
      "\t150.0 samples test: 29.3%\n",
      "\t200.0 samples test: 32.3%\n",
      "\t250.0 samples test: 30.0%\n",
      "\t300.0 samples test: 32.5%\n",
      "\t350.0 samples test: 33.3%\n",
      "\t400.0 samples test: 32.8%\n",
      "\t450.0 samples test: 32.9%\n",
      "\t500.0 samples test: 33.0%\n",
      "\t550.0 samples test: 33.1%\n",
      "\t600.0 samples test: 30.9%\n",
      "\t650.0 samples test: 33.2%\n",
      "\t700.0 samples test: 33.2%\n",
      "\n",
      "Train accuracy: 27.2% (350 samples)\n",
      "\t50.0 samples test: 27.2%\n",
      "\t100.0 samples test: 29.0%\n",
      "\t150.0 samples test: 29.3%\n",
      "\t200.0 samples test: 32.3%\n",
      "\t250.0 samples test: 30.0%\n",
      "\t300.0 samples test: 32.5%\n",
      "\t350.0 samples test: 33.3%\n",
      "\t400.0 samples test: 32.8%\n",
      "\t450.0 samples test: 32.9%\n",
      "\t500.0 samples test: 33.0%\n",
      "\t550.0 samples test: 33.1%\n",
      "\t600.0 samples test: 30.9%\n",
      "\t650.0 samples test: 33.2%\n",
      "\t700.0 samples test: 33.2%\n",
      "\n",
      "Train accuracy: 18.4% (400 samples)\n",
      "\t50.0 samples test: 27.2%\n",
      "\t100.0 samples test: 29.0%\n",
      "\t150.0 samples test: 29.3%\n",
      "\t200.0 samples test: 32.3%\n",
      "\t250.0 samples test: 30.0%\n",
      "\t300.0 samples test: 32.5%\n",
      "\t350.0 samples test: 33.3%\n",
      "\t400.0 samples test: 32.8%\n",
      "\t450.0 samples test: 32.9%\n",
      "\t500.0 samples test: 33.0%\n",
      "\t550.0 samples test: 33.1%\n",
      "\t600.0 samples test: 30.9%\n",
      "\t650.0 samples test: 33.2%\n",
      "\t700.0 samples test: 33.2%\n",
      "\n",
      "Train accuracy: 15.2% (450 samples)\n",
      "\t50.0 samples test: 27.2%\n",
      "\t100.0 samples test: 29.0%\n",
      "\t150.0 samples test: 29.3%\n",
      "\t200.0 samples test: 32.3%\n",
      "\t250.0 samples test: 30.0%\n",
      "\t300.0 samples test: 32.5%\n",
      "\t350.0 samples test: 33.3%\n",
      "\t400.0 samples test: 32.8%\n",
      "\t450.0 samples test: 32.9%\n",
      "\t500.0 samples test: 33.0%\n",
      "\t550.0 samples test: 33.1%\n",
      "\t600.0 samples test: 30.9%\n",
      "\t650.0 samples test: 33.2%\n",
      "\t700.0 samples test: 33.2%\n",
      "\n",
      "Train accuracy: 25.2% (500 samples)\n",
      "\t50.0 samples test: 27.2%\n",
      "\t100.0 samples test: 29.0%\n",
      "\t150.0 samples test: 29.3%\n",
      "\t200.0 samples test: 32.3%\n",
      "\t250.0 samples test: 30.0%\n",
      "\t300.0 samples test: 32.5%\n",
      "\t350.0 samples test: 33.3%\n",
      "\t400.0 samples test: 32.8%\n",
      "\t450.0 samples test: 32.9%\n",
      "\t500.0 samples test: 33.0%\n",
      "\t550.0 samples test: 33.1%\n",
      "\t600.0 samples test: 30.9%\n",
      "\t650.0 samples test: 33.2%\n",
      "\t700.0 samples test: 33.2%\n",
      "\n",
      "Train accuracy: 31.6% (550 samples)\n",
      "\t50.0 samples test: 27.2%\n",
      "\t100.0 samples test: 29.0%\n",
      "\t150.0 samples test: 29.3%\n",
      "\t200.0 samples test: 32.3%\n",
      "\t250.0 samples test: 30.0%\n",
      "\t300.0 samples test: 32.5%\n",
      "\t350.0 samples test: 33.3%\n",
      "\t400.0 samples test: 32.8%\n",
      "\t450.0 samples test: 32.9%\n",
      "\t500.0 samples test: 33.0%\n",
      "\t550.0 samples test: 33.1%\n",
      "\t600.0 samples test: 30.9%\n",
      "\t650.0 samples test: 33.2%\n",
      "\t700.0 samples test: 33.2%\n",
      "\n",
      "Train accuracy: 21.1% (600 samples)\n",
      "\t50.0 samples test: 27.2%\n",
      "\t100.0 samples test: 29.0%\n",
      "\t150.0 samples test: 29.3%\n",
      "\t200.0 samples test: 32.3%\n",
      "\t250.0 samples test: 30.0%\n",
      "\t300.0 samples test: 32.5%\n",
      "\t350.0 samples test: 33.3%\n",
      "\t400.0 samples test: 32.8%\n",
      "\t450.0 samples test: 32.9%\n",
      "\t500.0 samples test: 33.0%\n",
      "\t550.0 samples test: 33.1%\n",
      "\t600.0 samples test: 30.9%\n",
      "\t650.0 samples test: 33.2%\n",
      "\t700.0 samples test: 33.2%\n",
      "\n",
      "Train accuracy: 33.1% (650 samples)\n",
      "\t50.0 samples test: 27.2%\n",
      "\t100.0 samples test: 29.0%\n",
      "\t150.0 samples test: 29.3%\n",
      "\t200.0 samples test: 32.3%\n",
      "\t250.0 samples test: 30.0%\n",
      "\t300.0 samples test: 32.5%\n",
      "\t350.0 samples test: 33.3%\n",
      "\t400.0 samples test: 32.8%\n",
      "\t450.0 samples test: 32.9%\n",
      "\t500.0 samples test: 33.0%\n",
      "\t550.0 samples test: 33.1%\n",
      "\t600.0 samples test: 30.9%\n",
      "\t650.0 samples test: 33.2%\n",
      "\t700.0 samples test: 33.2%\n",
      "\n",
      "Train accuracy: 42.0% (700 samples)\n",
      "\t50.0 samples test: 27.2%\n",
      "\t100.0 samples test: 29.0%\n",
      "\t150.0 samples test: 29.3%\n",
      "\t200.0 samples test: 32.3%\n",
      "\t250.0 samples test: 30.0%\n",
      "\t300.0 samples test: 32.5%\n",
      "\t350.0 samples test: 33.3%\n",
      "\t400.0 samples test: 32.8%\n",
      "\t450.0 samples test: 32.9%\n",
      "\t500.0 samples test: 33.0%\n",
      "\t550.0 samples test: 33.1%\n",
      "\t600.0 samples test: 30.9%\n",
      "\t650.0 samples test: 33.2%\n",
      "\t700.0 samples test: 33.2%\n"
     ]
    }
   ],
   "source": [
    "info_x_samples = [\n",
    "    (data_50_samples, target_50_samples), \n",
    "    (data_100_samples, target_100_samples),\n",
    "    (data_150_samples, target_150_samples), \n",
    "    (data_200_samples, target_200_samples), \n",
    "    (data_250_samples, target_250_samples), \n",
    "    (data_300_samples, target_300_samples), \n",
    "    (data_350_samples, target_350_samples), \n",
    "    (data_400_samples, target_400_samples), \n",
    "    (data_450_samples, target_450_samples),\n",
    "    (data_500_samples, target_500_samples), \n",
    "    (data_550_samples, target_550_samples), \n",
    "    (data_600_samples, target_600_samples), \n",
    "    (data_650_samples, target_650_samples), \n",
    "    (data_700_samples, target_700_samples)\n",
    "]\n",
    "sample_numbers = (list(range(50, 701, 50)))\n",
    "results = []\n",
    "\n",
    "def samples_test(verbose=True):\n",
    "    \n",
    "    results.append([])\n",
    "\n",
    "    for sample_number, i in zip(sample_numbers, range(14)):\n",
    "        data_x_samples = info_x_samples[i][0]\n",
    "        target_x_samples = info_x_samples[i][1]\n",
    "    \n",
    "        scaler = StandardScaler()\n",
    "    \n",
    "        data_x_samples = data_x_samples.reshape(data_x_samples.shape[0], -1)\n",
    "        data_x_samples = (scaler.fit_transform(data_x_samples)).reshape((data_x_samples.shape[0], 10, 10, 1))\n",
    "        \n",
    "        target_x_samples = to_categorical(target_x_samples)\n",
    "    \n",
    "        temp_tf_model = Model()\n",
    "    \n",
    "        temp_tf_model.build(input_shape=(1, 10, 10, 1))\n",
    "        temp_tf_model.compile(optimizer='sgd', loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "        \n",
    "        temp_tf_model.fit(data_x_samples, target_x_samples, epochs=3, batch_size=64, verbose=False)\n",
    "    \n",
    "        initial_accuracy = temp_tf_model.evaluate(X_train, y_train, verbose=False)[1]\n",
    "        \n",
    "        if verbose:\n",
    "            print(f'\\nTrain accuracy: {initial_accuracy*100:.1f}% ({sample_number} samples)')\n",
    "        \n",
    "        if verbose:\n",
    "            for re_iter in info_x_samples:\n",
    "                data_new = re_iter[0]\n",
    "                target_new = re_iter[1]\n",
    "        \n",
    "                scaler = StandardScaler()\n",
    "        \n",
    "                data_new = data_new.reshape(data_new.shape[0], -1)\n",
    "                data_new = (scaler.fit_transform(data_new)).reshape((data_new.shape[0], 10, 10, 1))\n",
    "        \n",
    "                target_new = to_categorical(target_new)\n",
    "        \n",
    "                print(f'\\t{(data_new.shape[0])/5} samples test: {tf_model.evaluate(data_new, target_new, verbose=False)[1]*100:.1f}%')\n",
    "        \n",
    "        results[-1].append(initial_accuracy)\n",
    "    \n",
    "\n",
    "samples_test()\n",
    "for i in range(49):\n",
    "    samples_test(verbose=False)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Prepare results from tests for R data analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = np.array(results)\n",
    "df = pd.DataFrame(results, columns=[f'{num} samples' for num in range(50, 701, 50)])\n",
    "df.to_csv('R Data Analysis/data/samples_test_data.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 5 Test Runs (when trained on 700 samples):\n",
    "<img src='images/accuracy_sample_test1.png'>\n",
    "<img src='images/accuracy_sample_test2.png'>\n",
    "<img src='images/accuracy_sample_test3.png'>\n",
    "<img src='images/accuracy_sample_test4.png'>\n",
    "<img src='images/accuracy_sample_test5.png'>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Additional test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Train accuracy: 25.3% (50 samples)\n",
      "\t50.0 samples test: 25.6%\n",
      "\t100.0 samples test: 25.6%\n",
      "\t150.0 samples test: 25.9%\n",
      "\t200.0 samples test: 25.6%\n",
      "\t250.0 samples test: 25.1%\n",
      "\t300.0 samples test: 25.3%\n",
      "\t350.0 samples test: 25.1%\n",
      "\t400.0 samples test: 25.5%\n",
      "\t450.0 samples test: 25.5%\n",
      "\t500.0 samples test: 25.3%\n",
      "\t550.0 samples test: 25.3%\n",
      "\t600.0 samples test: 23.2%\n",
      "\t650.0 samples test: 23.1%\n",
      "\t700.0 samples test: 23.0%\n",
      "\n",
      "Train accuracy: 18.9% (100 samples)\n",
      "\t50.0 samples test: 19.2%\n",
      "\t100.0 samples test: 19.4%\n",
      "\t150.0 samples test: 19.6%\n",
      "\t200.0 samples test: 19.2%\n",
      "\t250.0 samples test: 19.0%\n",
      "\t300.0 samples test: 18.8%\n",
      "\t350.0 samples test: 18.9%\n",
      "\t400.0 samples test: 19.0%\n",
      "\t450.0 samples test: 18.8%\n",
      "\t500.0 samples test: 18.5%\n",
      "\t550.0 samples test: 18.8%\n",
      "\t600.0 samples test: 18.8%\n",
      "\t650.0 samples test: 18.7%\n",
      "\t700.0 samples test: 18.8%\n",
      "\n",
      "Train accuracy: 20.8% (150 samples)\n",
      "\t50.0 samples test: 24.8%\n",
      "\t100.0 samples test: 34.2%\n",
      "\t150.0 samples test: 23.2%\n",
      "\t200.0 samples test: 22.8%\n",
      "\t250.0 samples test: 22.0%\n",
      "\t300.0 samples test: 21.9%\n",
      "\t350.0 samples test: 21.5%\n",
      "\t400.0 samples test: 21.5%\n",
      "\t450.0 samples test: 21.2%\n",
      "\t500.0 samples test: 21.2%\n",
      "\t550.0 samples test: 21.3%\n",
      "\t600.0 samples test: 21.1%\n",
      "\t650.0 samples test: 20.9%\n",
      "\t700.0 samples test: 21.0%\n",
      "\n",
      "Train accuracy: 25.0% (200 samples)\n",
      "\t50.0 samples test: 21.2%\n",
      "\t100.0 samples test: 23.4%\n",
      "\t150.0 samples test: 24.9%\n",
      "\t200.0 samples test: 25.0%\n",
      "\t250.0 samples test: 24.9%\n",
      "\t300.0 samples test: 24.2%\n",
      "\t350.0 samples test: 24.5%\n",
      "\t400.0 samples test: 24.3%\n",
      "\t450.0 samples test: 24.9%\n",
      "\t500.0 samples test: 24.9%\n",
      "\t550.0 samples test: 24.8%\n",
      "\t600.0 samples test: 24.7%\n",
      "\t650.0 samples test: 24.7%\n",
      "\t700.0 samples test: 24.9%\n",
      "\n",
      "Train accuracy: 41.2% (250 samples)\n",
      "\t50.0 samples test: 37.6%\n",
      "\t100.0 samples test: 38.8%\n",
      "\t150.0 samples test: 40.5%\n",
      "\t200.0 samples test: 40.6%\n",
      "\t250.0 samples test: 40.7%\n",
      "\t300.0 samples test: 40.4%\n",
      "\t350.0 samples test: 40.4%\n",
      "\t400.0 samples test: 40.5%\n",
      "\t450.0 samples test: 40.2%\n",
      "\t500.0 samples test: 40.7%\n",
      "\t550.0 samples test: 40.5%\n",
      "\t600.0 samples test: 40.6%\n",
      "\t650.0 samples test: 40.4%\n",
      "\t700.0 samples test: 40.6%\n",
      "\n",
      "Train accuracy: 30.5% (300 samples)\n",
      "\t50.0 samples test: 30.8%\n",
      "\t100.0 samples test: 31.6%\n",
      "\t150.0 samples test: 31.5%\n",
      "\t200.0 samples test: 31.4%\n",
      "\t250.0 samples test: 30.8%\n",
      "\t300.0 samples test: 30.7%\n",
      "\t350.0 samples test: 30.4%\n",
      "\t400.0 samples test: 30.2%\n",
      "\t450.0 samples test: 30.6%\n",
      "\t500.0 samples test: 30.5%\n",
      "\t550.0 samples test: 30.5%\n",
      "\t600.0 samples test: 30.7%\n",
      "\t650.0 samples test: 30.7%\n",
      "\t700.0 samples test: 30.5%\n",
      "\n",
      "Train accuracy: 18.9% (350 samples)\n",
      "\t50.0 samples test: 28.0%\n",
      "\t100.0 samples test: 30.0%\n",
      "\t150.0 samples test: 19.9%\n",
      "\t200.0 samples test: 19.4%\n",
      "\t250.0 samples test: 29.4%\n",
      "\t300.0 samples test: 29.5%\n",
      "\t350.0 samples test: 29.4%\n",
      "\t400.0 samples test: 18.9%\n",
      "\t450.0 samples test: 19.1%\n",
      "\t500.0 samples test: 18.9%\n",
      "\t550.0 samples test: 18.8%\n",
      "\t600.0 samples test: 18.9%\n",
      "\t650.0 samples test: 18.7%\n",
      "\t700.0 samples test: 18.8%\n",
      "\n",
      "Train accuracy: 28.9% (400 samples)\n",
      "\t50.0 samples test: 20.4%\n",
      "\t100.0 samples test: 22.8%\n",
      "\t150.0 samples test: 25.7%\n",
      "\t200.0 samples test: 26.9%\n",
      "\t250.0 samples test: 27.7%\n",
      "\t300.0 samples test: 28.1%\n",
      "\t350.0 samples test: 27.6%\n",
      "\t400.0 samples test: 28.1%\n",
      "\t450.0 samples test: 28.3%\n",
      "\t500.0 samples test: 28.4%\n",
      "\t550.0 samples test: 28.4%\n",
      "\t600.0 samples test: 28.6%\n",
      "\t650.0 samples test: 28.5%\n",
      "\t700.0 samples test: 28.3%\n",
      "\n",
      "Train accuracy: 41.0% (450 samples)\n",
      "\t50.0 samples test: 26.8%\n",
      "\t100.0 samples test: 30.2%\n",
      "\t150.0 samples test: 41.6%\n",
      "\t200.0 samples test: 41.8%\n",
      "\t250.0 samples test: 41.8%\n",
      "\t300.0 samples test: 42.0%\n",
      "\t350.0 samples test: 41.5%\n",
      "\t400.0 samples test: 40.9%\n",
      "\t450.0 samples test: 40.9%\n",
      "\t500.0 samples test: 40.8%\n",
      "\t550.0 samples test: 41.2%\n",
      "\t600.0 samples test: 41.2%\n",
      "\t650.0 samples test: 41.3%\n",
      "\t700.0 samples test: 41.3%\n",
      "\n",
      "Train accuracy: 37.5% (500 samples)\n",
      "\t50.0 samples test: 35.6%\n",
      "\t100.0 samples test: 34.6%\n",
      "\t150.0 samples test: 37.3%\n",
      "\t200.0 samples test: 37.4%\n",
      "\t250.0 samples test: 38.3%\n",
      "\t300.0 samples test: 38.6%\n",
      "\t350.0 samples test: 38.8%\n",
      "\t400.0 samples test: 38.2%\n",
      "\t450.0 samples test: 38.5%\n",
      "\t500.0 samples test: 38.2%\n",
      "\t550.0 samples test: 38.0%\n",
      "\t600.0 samples test: 38.1%\n",
      "\t650.0 samples test: 38.2%\n",
      "\t700.0 samples test: 38.3%\n",
      "\n",
      "Train accuracy: 22.5% (550 samples)\n",
      "\t50.0 samples test: 23.2%\n",
      "\t100.0 samples test: 23.0%\n",
      "\t150.0 samples test: 23.3%\n",
      "\t200.0 samples test: 23.4%\n",
      "\t250.0 samples test: 22.6%\n",
      "\t300.0 samples test: 22.9%\n",
      "\t350.0 samples test: 22.3%\n",
      "\t400.0 samples test: 22.0%\n",
      "\t450.0 samples test: 22.1%\n",
      "\t500.0 samples test: 22.3%\n",
      "\t550.0 samples test: 22.2%\n",
      "\t600.0 samples test: 22.8%\n",
      "\t650.0 samples test: 22.6%\n",
      "\t700.0 samples test: 22.6%\n",
      "\n",
      "Train accuracy: 32.8% (600 samples)\n",
      "\t50.0 samples test: 34.0%\n",
      "\t100.0 samples test: 33.8%\n",
      "\t150.0 samples test: 33.1%\n",
      "\t200.0 samples test: 32.8%\n",
      "\t250.0 samples test: 32.2%\n",
      "\t300.0 samples test: 32.6%\n",
      "\t350.0 samples test: 32.6%\n",
      "\t400.0 samples test: 33.3%\n",
      "\t450.0 samples test: 33.3%\n",
      "\t500.0 samples test: 33.0%\n",
      "\t550.0 samples test: 33.2%\n",
      "\t600.0 samples test: 33.2%\n",
      "\t650.0 samples test: 33.0%\n",
      "\t700.0 samples test: 32.7%\n",
      "\n",
      "Train accuracy: 35.2% (650 samples)\n",
      "\t50.0 samples test: 34.8%\n",
      "\t100.0 samples test: 34.8%\n",
      "\t150.0 samples test: 34.1%\n",
      "\t200.0 samples test: 32.7%\n",
      "\t250.0 samples test: 35.7%\n",
      "\t300.0 samples test: 35.1%\n",
      "\t350.0 samples test: 34.9%\n",
      "\t400.0 samples test: 35.1%\n",
      "\t450.0 samples test: 35.2%\n",
      "\t500.0 samples test: 35.1%\n",
      "\t550.0 samples test: 35.1%\n",
      "\t600.0 samples test: 34.9%\n",
      "\t650.0 samples test: 34.8%\n",
      "\t700.0 samples test: 34.9%\n",
      "\n",
      "Train accuracy: 26.9% (700 samples)\n",
      "\t50.0 samples test: 25.6%\n",
      "\t100.0 samples test: 26.0%\n",
      "\t150.0 samples test: 26.1%\n",
      "\t200.0 samples test: 26.5%\n",
      "\t250.0 samples test: 26.8%\n",
      "\t300.0 samples test: 27.2%\n",
      "\t350.0 samples test: 26.9%\n",
      "\t400.0 samples test: 26.8%\n",
      "\t450.0 samples test: 27.0%\n",
      "\t500.0 samples test: 26.6%\n",
      "\t550.0 samples test: 26.6%\n",
      "\t600.0 samples test: 26.5%\n",
      "\t650.0 samples test: 26.7%\n",
      "\t700.0 samples test: 26.9%\n"
     ]
    }
   ],
   "source": [
    "info_x_samples = [\n",
    "    (data_50_samples, target_50_samples), \n",
    "    (data_100_samples, target_100_samples),\n",
    "    (data_150_samples, target_150_samples), \n",
    "    (data_200_samples, target_200_samples), \n",
    "    (data_250_samples, target_250_samples), \n",
    "    (data_300_samples, target_300_samples), \n",
    "    (data_350_samples, target_350_samples), \n",
    "    (data_400_samples, target_400_samples), \n",
    "    (data_450_samples, target_450_samples),\n",
    "    (data_500_samples, target_500_samples), \n",
    "    (data_550_samples, target_550_samples), \n",
    "    (data_600_samples, target_600_samples), \n",
    "    (data_650_samples, target_650_samples), \n",
    "    (data_700_samples, target_700_samples)\n",
    "]\n",
    "sample_numbers = (list(range(50, 701, 50)))\n",
    "results = []\n",
    "\n",
    "def samples_test(verbose=True):\n",
    "    \n",
    "    results.append([])\n",
    "\n",
    "    for sample_number, i in zip(sample_numbers, range(14)):\n",
    "        data_x_samples = info_x_samples[i][0]\n",
    "        target_x_samples = info_x_samples[i][1]\n",
    "    \n",
    "        scaler = StandardScaler()\n",
    "    \n",
    "        data_x_samples = data_x_samples.reshape(data_x_samples.shape[0], -1)\n",
    "        data_x_samples = (scaler.fit_transform(data_x_samples)).reshape((data_x_samples.shape[0], 10, 10, 1))\n",
    "        \n",
    "        target_x_samples = to_categorical(target_x_samples)\n",
    "    \n",
    "        model = Model()\n",
    "    \n",
    "        model.build(input_shape=(1, 10, 10, 1))\n",
    "        model.compile(optimizer='sgd', loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "        \n",
    "        model.fit(data_x_samples, target_x_samples, epochs=3, batch_size=64, verbose=False)\n",
    "    \n",
    "        initial_accuracy = model.evaluate(X_train, y_train, verbose=False)[1]\n",
    "        \n",
    "        if verbose:\n",
    "            print(f'\\nTrain accuracy: {initial_accuracy*100:.1f}% ({sample_number} samples)')\n",
    "        \n",
    "        if verbose:\n",
    "            for re_iter in info_x_samples:\n",
    "                data_new = re_iter[0]\n",
    "                target_new = re_iter[1]\n",
    "        \n",
    "                scaler = StandardScaler()\n",
    "        \n",
    "                data_new = data_new.reshape(data_new.shape[0], -1)\n",
    "                data_new = (scaler.fit_transform(data_new)).reshape((data_new.shape[0], 10, 10, 1))\n",
    "        \n",
    "                target_new = to_categorical(target_new)\n",
    "        \n",
    "                print(f'\\t{(data_new.shape[0])/5} samples test: {model.evaluate(data_new, target_new, verbose=False)[1]*100:.1f}%')\n",
    "        \n",
    "        results[-1].append(initial_accuracy)\n",
    "\n",
    "\n",
    "samples_test()\n",
    "for i in range(49):\n",
    "    samples_test(verbose=False)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### New Test with Constant Test Samples (175 samples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run 1\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 17.6% test: 18.4%\n",
      "\t100.0 samples train: 23.7% test: 22.6%\n",
      "\t150.0 samples train: 23.5% test: 16.7%\n",
      "\t200.0 samples train: 20.8% test: 14.4%\n",
      "\t250.0 samples train: 29.6% test: 18.3%\n",
      "\t300.0 samples train: 23.4% test: 19.9%\n",
      "\t350.0 samples train: 29.9% test: 22.1%\n",
      "\t400.0 samples train: 29.8% test: 21.8%\n",
      "\t450.0 samples train: 33.4% test: 20.6%\n",
      "\t500.0 samples train: 22.7% test: 20.1%\n",
      "\t550.0 samples train: 17.8% test: 21.4%\n",
      "\t600.0 samples train: 15.9% test: 19.2%\n",
      "\t650.0 samples train: 22.2% test: 21.9%\n",
      "\t700.0 samples train: 46.4% test: 19.1%\n",
      "\n",
      "\n",
      "Run 2\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 31.6% test: 26.9%\n",
      "\t100.0 samples train: 20.8% test: 18.6%\n",
      "\t150.0 samples train: 20.6% test: 23.0%\n",
      "\t200.0 samples train: 26.1% test: 16.5%\n",
      "\t250.0 samples train: 30.4% test: 20.3%\n",
      "\t300.0 samples train: 37.2% test: 20.2%\n",
      "\t350.0 samples train: 19.9% test: 19.0%\n",
      "\t400.0 samples train: 25.7% test: 21.6%\n",
      "\t450.0 samples train: 28.0% test: 20.2%\n",
      "\t500.0 samples train: 43.5% test: 19.7%\n",
      "\t550.0 samples train: 42.3% test: 19.5%\n",
      "\t600.0 samples train: 34.4% test: 20.2%\n",
      "\t650.0 samples train: 18.8% test: 19.8%\n",
      "\t700.0 samples train: 32.5% test: 19.4%\n",
      "\n",
      "\n",
      "Run 3\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 20.9% test: 21.9%\n",
      "\t100.0 samples train: 24.8% test: 38.6%\n",
      "\t150.0 samples train: 19.6% test: 24.7%\n",
      "\t200.0 samples train: 35.7% test: 16.2%\n",
      "\t250.0 samples train: 17.5% test: 19.3%\n",
      "\t300.0 samples train: 25.0% test: 22.6%\n",
      "\t350.0 samples train: 29.2% test: 19.7%\n",
      "\t400.0 samples train: 30.1% test: 19.4%\n",
      "\t450.0 samples train: 34.6% test: 21.6%\n",
      "\t500.0 samples train: 30.8% test: 20.5%\n",
      "\t550.0 samples train: 34.1% test: 21.3%\n",
      "\t600.0 samples train: 21.8% test: 19.7%\n",
      "\t650.0 samples train: 21.5% test: 19.7%\n",
      "\t700.0 samples train: 23.4% test: 19.0%\n",
      "\n",
      "\n",
      "Run 4\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 20.9% test: 19.1%\n",
      "\t100.0 samples train: 16.8% test: 17.8%\n",
      "\t150.0 samples train: 28.8% test: 24.5%\n",
      "\t200.0 samples train: 17.2% test: 20.8%\n",
      "\t250.0 samples train: 28.0% test: 24.8%\n",
      "\t300.0 samples train: 27.1% test: 19.5%\n",
      "\t350.0 samples train: 32.2% test: 24.0%\n",
      "\t400.0 samples train: 33.3% test: 20.2%\n",
      "\t450.0 samples train: 26.6% test: 21.9%\n",
      "\t500.0 samples train: 32.2% test: 21.0%\n",
      "\t550.0 samples train: 35.8% test: 20.2%\n",
      "\t600.0 samples train: 29.7% test: 18.9%\n",
      "\t650.0 samples train: 32.4% test: 21.6%\n",
      "\t700.0 samples train: 21.9% test: 19.8%\n",
      "\n",
      "\n",
      "Run 5\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 7.5% test: 6.9%\n",
      "\t100.0 samples train: 20.5% test: 19.3%\n",
      "\t150.0 samples train: 21.5% test: 22.4%\n",
      "\t200.0 samples train: 15.6% test: 16.5%\n",
      "\t250.0 samples train: 24.5% test: 19.7%\n",
      "\t300.0 samples train: 20.4% test: 18.5%\n",
      "\t350.0 samples train: 22.3% test: 17.8%\n",
      "\t400.0 samples train: 17.5% test: 22.7%\n",
      "\t450.0 samples train: 33.8% test: 21.8%\n",
      "\t500.0 samples train: 32.0% test: 21.6%\n",
      "\t550.0 samples train: 25.7% test: 19.9%\n",
      "\t600.0 samples train: 38.4% test: 20.2%\n",
      "\t650.0 samples train: 33.1% test: 19.7%\n",
      "\t700.0 samples train: 40.4% test: 19.8%\n",
      "\n",
      "\n",
      "Run 6\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 5.9% test: 11.7%\n",
      "\t100.0 samples train: 18.7% test: 14.2%\n",
      "\t150.0 samples train: 36.8% test: 15.3%\n",
      "\t200.0 samples train: 19.9% test: 14.2%\n",
      "\t250.0 samples train: 28.5% test: 25.7%\n",
      "\t300.0 samples train: 27.5% test: 20.5%\n",
      "\t350.0 samples train: 26.9% test: 18.9%\n",
      "\t400.0 samples train: 19.3% test: 19.3%\n",
      "\t450.0 samples train: 20.2% test: 19.2%\n",
      "\t500.0 samples train: 35.5% test: 19.1%\n",
      "\t550.0 samples train: 22.3% test: 21.3%\n",
      "\t600.0 samples train: 39.1% test: 20.8%\n",
      "\t650.0 samples train: 27.1% test: 19.5%\n",
      "\t700.0 samples train: 33.9% test: 19.3%\n",
      "\n",
      "\n",
      "Run 7\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 19.3% test: 24.3%\n",
      "\t100.0 samples train: 18.4% test: 23.9%\n",
      "\t150.0 samples train: 19.2% test: 19.7%\n",
      "\t200.0 samples train: 26.4% test: 18.9%\n",
      "\t250.0 samples train: 32.2% test: 17.1%\n",
      "\t300.0 samples train: 26.5% test: 18.6%\n",
      "\t350.0 samples train: 25.2% test: 20.1%\n",
      "\t400.0 samples train: 34.5% test: 21.3%\n",
      "\t450.0 samples train: 27.1% test: 22.7%\n",
      "\t500.0 samples train: 27.8% test: 18.6%\n",
      "\t550.0 samples train: 21.9% test: 19.9%\n",
      "\t600.0 samples train: 30.5% test: 20.2%\n",
      "\t650.0 samples train: 40.2% test: 19.1%\n",
      "\t700.0 samples train: 33.3% test: 19.2%\n",
      "\n",
      "\n",
      "Run 8\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 24.6% test: 26.2%\n",
      "\t100.0 samples train: 29.3% test: 33.8%\n",
      "\t150.0 samples train: 23.7% test: 21.3%\n",
      "\t200.0 samples train: 24.7% test: 18.6%\n",
      "\t250.0 samples train: 33.2% test: 20.6%\n",
      "\t300.0 samples train: 22.0% test: 22.2%\n",
      "\t350.0 samples train: 22.6% test: 26.1%\n",
      "\t400.0 samples train: 20.8% test: 19.5%\n",
      "\t450.0 samples train: 24.1% test: 19.0%\n",
      "\t500.0 samples train: 31.2% test: 20.0%\n",
      "\t550.0 samples train: 23.5% test: 21.1%\n",
      "\t600.0 samples train: 28.6% test: 21.8%\n",
      "\t650.0 samples train: 32.4% test: 19.5%\n",
      "\t700.0 samples train: 34.6% test: 19.8%\n",
      "\n",
      "\n",
      "Run 9\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 22.5% test: 20.8%\n",
      "\t100.0 samples train: 15.5% test: 23.9%\n",
      "\t150.0 samples train: 20.8% test: 19.4%\n",
      "\t200.0 samples train: 24.5% test: 29.5%\n",
      "\t250.0 samples train: 33.3% test: 17.9%\n",
      "\t300.0 samples train: 31.0% test: 20.1%\n",
      "\t350.0 samples train: 22.9% test: 18.7%\n",
      "\t400.0 samples train: 23.6% test: 20.1%\n",
      "\t450.0 samples train: 16.7% test: 21.6%\n",
      "\t500.0 samples train: 41.7% test: 19.8%\n",
      "\t550.0 samples train: 37.9% test: 19.4%\n",
      "\t600.0 samples train: 29.5% test: 19.7%\n",
      "\t650.0 samples train: 19.2% test: 21.8%\n",
      "\t700.0 samples train: 20.8% test: 19.7%\n",
      "\n",
      "\n",
      "Run 10\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 24.1% test: 24.7%\n",
      "\t100.0 samples train: 25.6% test: 24.6%\n",
      "\t150.0 samples train: 23.0% test: 25.5%\n",
      "\t200.0 samples train: 26.8% test: 19.2%\n",
      "\t250.0 samples train: 19.2% test: 20.1%\n",
      "\t300.0 samples train: 28.9% test: 17.3%\n",
      "\t350.0 samples train: 31.1% test: 19.0%\n",
      "\t400.0 samples train: 33.7% test: 19.3%\n",
      "\t450.0 samples train: 28.5% test: 19.2%\n",
      "\t500.0 samples train: 27.5% test: 20.3%\n",
      "\t550.0 samples train: 26.2% test: 19.7%\n",
      "\t600.0 samples train: 25.6% test: 20.9%\n",
      "\t650.0 samples train: 23.5% test: 21.7%\n",
      "\t700.0 samples train: 44.9% test: 19.0%\n",
      "\n",
      "\n",
      "Run 11\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 16.6% test: 17.1%\n",
      "\t100.0 samples train: 18.7% test: 20.8%\n",
      "\t150.0 samples train: 29.5% test: 23.3%\n",
      "\t200.0 samples train: 26.0% test: 20.3%\n",
      "\t250.0 samples train: 21.6% test: 24.3%\n",
      "\t300.0 samples train: 25.2% test: 17.9%\n",
      "\t350.0 samples train: 32.2% test: 19.1%\n",
      "\t400.0 samples train: 25.7% test: 19.3%\n",
      "\t450.0 samples train: 27.1% test: 19.1%\n",
      "\t500.0 samples train: 19.3% test: 19.7%\n",
      "\t550.0 samples train: 33.1% test: 21.8%\n",
      "\t600.0 samples train: 31.9% test: 19.2%\n",
      "\t650.0 samples train: 31.6% test: 19.7%\n",
      "\t700.0 samples train: 37.3% test: 20.3%\n",
      "\n",
      "\n",
      "Run 12\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 16.0% test: 16.5%\n",
      "\t100.0 samples train: 32.5% test: 21.6%\n",
      "\t150.0 samples train: 22.8% test: 25.1%\n",
      "\t200.0 samples train: 23.7% test: 20.0%\n",
      "\t250.0 samples train: 39.9% test: 19.9%\n",
      "\t300.0 samples train: 24.7% test: 19.8%\n",
      "\t350.0 samples train: 33.8% test: 19.9%\n",
      "\t400.0 samples train: 18.0% test: 18.5%\n",
      "\t450.0 samples train: 18.3% test: 21.7%\n",
      "\t500.0 samples train: 26.6% test: 18.9%\n",
      "\t550.0 samples train: 40.9% test: 20.7%\n",
      "\t600.0 samples train: 24.2% test: 18.7%\n",
      "\t650.0 samples train: 18.3% test: 21.4%\n",
      "\t700.0 samples train: 30.0% test: 19.7%\n",
      "\n",
      "\n",
      "Run 13\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 16.6% test: 18.1%\n",
      "\t100.0 samples train: 22.9% test: 11.3%\n",
      "\t150.0 samples train: 18.3% test: 18.9%\n",
      "\t200.0 samples train: 19.9% test: 20.7%\n",
      "\t250.0 samples train: 36.3% test: 19.5%\n",
      "\t300.0 samples train: 25.7% test: 20.1%\n",
      "\t350.0 samples train: 19.8% test: 20.6%\n",
      "\t400.0 samples train: 23.9% test: 20.5%\n",
      "\t450.0 samples train: 28.1% test: 19.4%\n",
      "\t500.0 samples train: 17.6% test: 21.7%\n",
      "\t550.0 samples train: 23.9% test: 21.0%\n",
      "\t600.0 samples train: 24.6% test: 19.7%\n",
      "\t650.0 samples train: 24.5% test: 21.4%\n",
      "\t700.0 samples train: 36.8% test: 20.1%\n",
      "\n",
      "\n",
      "Run 14\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 20.9% test: 20.6%\n",
      "\t100.0 samples train: 8.0% test: 10.5%\n",
      "\t150.0 samples train: 27.2% test: 34.9%\n",
      "\t200.0 samples train: 20.3% test: 19.0%\n",
      "\t250.0 samples train: 26.3% test: 24.0%\n",
      "\t300.0 samples train: 17.0% test: 25.6%\n",
      "\t350.0 samples train: 16.6% test: 19.8%\n",
      "\t400.0 samples train: 24.7% test: 18.7%\n",
      "\t450.0 samples train: 42.4% test: 19.9%\n",
      "\t500.0 samples train: 20.8% test: 19.2%\n",
      "\t550.0 samples train: 35.9% test: 20.6%\n",
      "\t600.0 samples train: 28.8% test: 21.0%\n",
      "\t650.0 samples train: 21.3% test: 19.4%\n",
      "\t700.0 samples train: 31.6% test: 19.8%\n",
      "\n",
      "\n",
      "Run 15\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 21.4% test: 19.4%\n",
      "\t100.0 samples train: 25.3% test: 22.1%\n",
      "\t150.0 samples train: 16.4% test: 18.3%\n",
      "\t200.0 samples train: 17.3% test: 19.7%\n",
      "\t250.0 samples train: 23.8% test: 20.5%\n",
      "\t300.0 samples train: 26.3% test: 19.8%\n",
      "\t350.0 samples train: 27.7% test: 22.7%\n",
      "\t400.0 samples train: 20.5% test: 20.0%\n",
      "\t450.0 samples train: 32.5% test: 19.3%\n",
      "\t500.0 samples train: 24.5% test: 21.3%\n",
      "\t550.0 samples train: 25.2% test: 19.8%\n",
      "\t600.0 samples train: 32.0% test: 20.2%\n",
      "\t650.0 samples train: 26.2% test: 20.8%\n",
      "\t700.0 samples train: 41.7% test: 19.8%\n",
      "\n",
      "\n",
      "Run 16\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 24.6% test: 18.9%\n",
      "\t100.0 samples train: 24.0% test: 25.5%\n",
      "\t150.0 samples train: 26.3% test: 18.9%\n",
      "\t200.0 samples train: 17.1% test: 24.5%\n",
      "\t250.0 samples train: 25.5% test: 18.9%\n",
      "\t300.0 samples train: 26.3% test: 25.3%\n",
      "\t350.0 samples train: 26.5% test: 22.9%\n",
      "\t400.0 samples train: 29.1% test: 20.9%\n",
      "\t450.0 samples train: 25.0% test: 24.8%\n",
      "\t500.0 samples train: 29.0% test: 21.3%\n",
      "\t550.0 samples train: 32.8% test: 19.7%\n",
      "\t600.0 samples train: 37.6% test: 20.8%\n",
      "\t650.0 samples train: 30.8% test: 19.9%\n",
      "\t700.0 samples train: 27.7% test: 20.0%\n",
      "\n",
      "\n",
      "Run 17\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 19.8% test: 22.1%\n",
      "\t100.0 samples train: 20.8% test: 18.4%\n",
      "\t150.0 samples train: 24.0% test: 23.0%\n",
      "\t200.0 samples train: 24.7% test: 23.5%\n",
      "\t250.0 samples train: 20.9% test: 20.3%\n",
      "\t300.0 samples train: 21.2% test: 20.8%\n",
      "\t350.0 samples train: 38.4% test: 18.4%\n",
      "\t400.0 samples train: 38.8% test: 22.5%\n",
      "\t450.0 samples train: 30.8% test: 21.7%\n",
      "\t500.0 samples train: 22.7% test: 19.5%\n",
      "\t550.0 samples train: 19.1% test: 19.2%\n",
      "\t600.0 samples train: 36.5% test: 21.3%\n",
      "\t650.0 samples train: 35.9% test: 19.1%\n",
      "\t700.0 samples train: 18.9% test: 21.3%\n",
      "\n",
      "\n",
      "Run 18\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 20.9% test: 18.5%\n",
      "\t100.0 samples train: 21.6% test: 23.9%\n",
      "\t150.0 samples train: 17.4% test: 20.6%\n",
      "\t200.0 samples train: 13.2% test: 18.5%\n",
      "\t250.0 samples train: 28.3% test: 26.6%\n",
      "\t300.0 samples train: 12.6% test: 21.6%\n",
      "\t350.0 samples train: 18.9% test: 19.7%\n",
      "\t400.0 samples train: 21.1% test: 23.9%\n",
      "\t450.0 samples train: 29.8% test: 20.9%\n",
      "\t500.0 samples train: 34.6% test: 19.0%\n",
      "\t550.0 samples train: 26.7% test: 20.2%\n",
      "\t600.0 samples train: 25.4% test: 21.1%\n",
      "\t650.0 samples train: 36.6% test: 20.7%\n",
      "\t700.0 samples train: 33.0% test: 19.4%\n",
      "\n",
      "\n",
      "Run 19\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 16.6% test: 17.8%\n",
      "\t100.0 samples train: 17.6% test: 12.8%\n",
      "\t150.0 samples train: 19.8% test: 18.1%\n",
      "\t200.0 samples train: 22.4% test: 18.7%\n",
      "\t250.0 samples train: 20.4% test: 18.7%\n",
      "\t300.0 samples train: 28.0% test: 19.2%\n",
      "\t350.0 samples train: 14.3% test: 20.3%\n",
      "\t400.0 samples train: 25.1% test: 20.3%\n",
      "\t450.0 samples train: 19.4% test: 19.1%\n",
      "\t500.0 samples train: 36.1% test: 19.3%\n",
      "\t550.0 samples train: 18.9% test: 19.4%\n",
      "\t600.0 samples train: 34.6% test: 20.3%\n",
      "\t650.0 samples train: 35.0% test: 20.6%\n",
      "\t700.0 samples train: 35.3% test: 21.5%\n",
      "\n",
      "\n",
      "Run 20\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 24.1% test: 22.5%\n",
      "\t100.0 samples train: 20.0% test: 19.3%\n",
      "\t150.0 samples train: 10.5% test: 22.7%\n",
      "\t200.0 samples train: 18.9% test: 20.6%\n",
      "\t250.0 samples train: 19.7% test: 18.9%\n",
      "\t300.0 samples train: 18.5% test: 23.0%\n",
      "\t350.0 samples train: 28.8% test: 24.8%\n",
      "\t400.0 samples train: 27.9% test: 21.9%\n",
      "\t450.0 samples train: 25.7% test: 19.3%\n",
      "\t500.0 samples train: 22.2% test: 19.8%\n",
      "\t550.0 samples train: 27.2% test: 19.3%\n",
      "\t600.0 samples train: 38.3% test: 20.0%\n",
      "\t650.0 samples train: 22.3% test: 19.9%\n",
      "\t700.0 samples train: 18.4% test: 20.7%\n",
      "\n",
      "\n",
      "Run 21\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 20.3% test: 18.3%\n",
      "\t100.0 samples train: 17.1% test: 25.4%\n",
      "\t150.0 samples train: 36.8% test: 20.0%\n",
      "\t200.0 samples train: 25.1% test: 19.7%\n",
      "\t250.0 samples train: 33.3% test: 20.0%\n",
      "\t300.0 samples train: 27.9% test: 20.3%\n",
      "\t350.0 samples train: 24.2% test: 20.7%\n",
      "\t400.0 samples train: 35.9% test: 19.8%\n",
      "\t450.0 samples train: 22.2% test: 20.6%\n",
      "\t500.0 samples train: 27.8% test: 19.7%\n",
      "\t550.0 samples train: 21.8% test: 19.7%\n",
      "\t600.0 samples train: 31.9% test: 21.0%\n",
      "\t650.0 samples train: 21.5% test: 22.3%\n",
      "\t700.0 samples train: 35.2% test: 19.8%\n",
      "\n",
      "\n",
      "Run 22\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 28.9% test: 27.5%\n",
      "\t100.0 samples train: 14.7% test: 21.0%\n",
      "\t150.0 samples train: 30.4% test: 21.5%\n",
      "\t200.0 samples train: 25.9% test: 27.5%\n",
      "\t250.0 samples train: 17.0% test: 19.7%\n",
      "\t300.0 samples train: 22.7% test: 21.5%\n",
      "\t350.0 samples train: 26.5% test: 23.4%\n",
      "\t400.0 samples train: 25.1% test: 20.3%\n",
      "\t450.0 samples train: 26.8% test: 21.7%\n",
      "\t500.0 samples train: 30.5% test: 19.9%\n",
      "\t550.0 samples train: 19.5% test: 21.1%\n",
      "\t600.0 samples train: 34.0% test: 21.3%\n",
      "\t650.0 samples train: 22.0% test: 19.9%\n",
      "\t700.0 samples train: 33.6% test: 20.3%\n",
      "\n",
      "\n",
      "Run 23\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 19.3% test: 20.2%\n",
      "\t100.0 samples train: 19.7% test: 17.8%\n",
      "\t150.0 samples train: 17.8% test: 26.7%\n",
      "\t200.0 samples train: 27.3% test: 24.5%\n",
      "\t250.0 samples train: 17.5% test: 25.6%\n",
      "\t300.0 samples train: 19.5% test: 18.7%\n",
      "\t350.0 samples train: 19.1% test: 18.1%\n",
      "\t400.0 samples train: 22.5% test: 19.2%\n",
      "\t450.0 samples train: 22.6% test: 18.6%\n",
      "\t500.0 samples train: 36.4% test: 18.6%\n",
      "\t550.0 samples train: 28.9% test: 19.8%\n",
      "\t600.0 samples train: 25.8% test: 19.2%\n",
      "\t650.0 samples train: 33.2% test: 19.5%\n",
      "\t700.0 samples train: 31.3% test: 19.3%\n",
      "\n",
      "\n",
      "Run 24\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 33.2% test: 32.6%\n",
      "\t100.0 samples train: 27.2% test: 29.1%\n",
      "\t150.0 samples train: 20.5% test: 20.8%\n",
      "\t200.0 samples train: 16.7% test: 18.7%\n",
      "\t250.0 samples train: 25.4% test: 20.6%\n",
      "\t300.0 samples train: 14.9% test: 19.3%\n",
      "\t350.0 samples train: 25.1% test: 19.3%\n",
      "\t400.0 samples train: 36.7% test: 17.9%\n",
      "\t450.0 samples train: 34.6% test: 21.8%\n",
      "\t500.0 samples train: 30.5% test: 22.4%\n",
      "\t550.0 samples train: 23.0% test: 20.3%\n",
      "\t600.0 samples train: 39.0% test: 17.9%\n",
      "\t650.0 samples train: 21.8% test: 19.0%\n",
      "\t700.0 samples train: 36.0% test: 20.2%\n",
      "\n",
      "\n",
      "Run 25\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 12.8% test: 20.6%\n",
      "\t100.0 samples train: 21.9% test: 20.8%\n",
      "\t150.0 samples train: 35.2% test: 17.3%\n",
      "\t200.0 samples train: 14.7% test: 25.7%\n",
      "\t250.0 samples train: 20.3% test: 23.4%\n",
      "\t300.0 samples train: 19.7% test: 25.0%\n",
      "\t350.0 samples train: 19.2% test: 21.1%\n",
      "\t400.0 samples train: 24.4% test: 20.0%\n",
      "\t450.0 samples train: 28.7% test: 19.5%\n",
      "\t500.0 samples train: 21.2% test: 20.6%\n",
      "\t550.0 samples train: 24.4% test: 19.8%\n",
      "\t600.0 samples train: 27.2% test: 21.0%\n",
      "\t650.0 samples train: 17.2% test: 20.0%\n",
      "\t700.0 samples train: 33.1% test: 20.0%\n",
      "\n",
      "\n",
      "Run 26\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 20.3% test: 20.9%\n",
      "\t100.0 samples train: 21.6% test: 20.2%\n",
      "\t150.0 samples train: 16.7% test: 25.5%\n",
      "\t200.0 samples train: 18.5% test: 18.1%\n",
      "\t250.0 samples train: 20.8% test: 16.9%\n",
      "\t300.0 samples train: 19.5% test: 19.2%\n",
      "\t350.0 samples train: 36.4% test: 19.7%\n",
      "\t400.0 samples train: 25.3% test: 17.5%\n",
      "\t450.0 samples train: 27.1% test: 20.0%\n",
      "\t500.0 samples train: 32.0% test: 20.0%\n",
      "\t550.0 samples train: 27.2% test: 19.8%\n",
      "\t600.0 samples train: 37.7% test: 19.5%\n",
      "\t650.0 samples train: 37.1% test: 20.5%\n",
      "\t700.0 samples train: 20.9% test: 19.4%\n",
      "\n",
      "\n",
      "Run 27\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 20.3% test: 17.3%\n",
      "\t100.0 samples train: 21.3% test: 19.7%\n",
      "\t150.0 samples train: 21.4% test: 25.3%\n",
      "\t200.0 samples train: 21.5% test: 18.1%\n",
      "\t250.0 samples train: 18.1% test: 22.3%\n",
      "\t300.0 samples train: 35.1% test: 19.1%\n",
      "\t350.0 samples train: 31.2% test: 19.5%\n",
      "\t400.0 samples train: 25.5% test: 22.7%\n",
      "\t450.0 samples train: 28.5% test: 21.9%\n",
      "\t500.0 samples train: 25.3% test: 20.1%\n",
      "\t550.0 samples train: 31.0% test: 19.9%\n",
      "\t600.0 samples train: 34.2% test: 19.9%\n",
      "\t650.0 samples train: 20.4% test: 21.6%\n",
      "\t700.0 samples train: 24.0% test: 21.4%\n",
      "\n",
      "\n",
      "Run 28\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 18.2% test: 15.7%\n",
      "\t100.0 samples train: 17.6% test: 20.0%\n",
      "\t150.0 samples train: 21.9% test: 21.1%\n",
      "\t200.0 samples train: 20.9% test: 22.5%\n",
      "\t250.0 samples train: 19.3% test: 20.5%\n",
      "\t300.0 samples train: 14.5% test: 20.2%\n",
      "\t350.0 samples train: 18.8% test: 17.5%\n",
      "\t400.0 samples train: 21.1% test: 19.5%\n",
      "\t450.0 samples train: 26.7% test: 19.4%\n",
      "\t500.0 samples train: 31.4% test: 19.4%\n",
      "\t550.0 samples train: 22.7% test: 20.8%\n",
      "\t600.0 samples train: 43.9% test: 19.9%\n",
      "\t650.0 samples train: 26.0% test: 21.3%\n",
      "\t700.0 samples train: 21.0% test: 19.2%\n",
      "\n",
      "\n",
      "Run 29\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 14.4% test: 15.9%\n",
      "\t100.0 samples train: 22.7% test: 22.5%\n",
      "\t150.0 samples train: 20.3% test: 21.4%\n",
      "\t200.0 samples train: 26.4% test: 20.5%\n",
      "\t250.0 samples train: 23.3% test: 21.9%\n",
      "\t300.0 samples train: 28.4% test: 22.2%\n",
      "\t350.0 samples train: 20.7% test: 17.9%\n",
      "\t400.0 samples train: 21.7% test: 21.4%\n",
      "\t450.0 samples train: 31.7% test: 21.3%\n",
      "\t500.0 samples train: 23.5% test: 20.5%\n",
      "\t550.0 samples train: 20.2% test: 20.0%\n",
      "\t600.0 samples train: 31.1% test: 21.6%\n",
      "\t650.0 samples train: 34.8% test: 19.9%\n",
      "\t700.0 samples train: 26.6% test: 19.5%\n",
      "\n",
      "\n",
      "Run 30\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 18.7% test: 18.3%\n",
      "\t100.0 samples train: 18.9% test: 19.4%\n",
      "\t150.0 samples train: 21.2% test: 23.8%\n",
      "\t200.0 samples train: 33.1% test: 16.0%\n",
      "\t250.0 samples train: 28.2% test: 24.3%\n",
      "\t300.0 samples train: 21.7% test: 18.1%\n",
      "\t350.0 samples train: 29.1% test: 18.6%\n",
      "\t400.0 samples train: 21.1% test: 17.3%\n",
      "\t450.0 samples train: 31.2% test: 19.5%\n",
      "\t500.0 samples train: 21.9% test: 20.3%\n",
      "\t550.0 samples train: 17.6% test: 21.0%\n",
      "\t600.0 samples train: 34.5% test: 19.7%\n",
      "\t650.0 samples train: 39.7% test: 20.2%\n",
      "\t700.0 samples train: 29.4% test: 19.7%\n",
      "\n",
      "\n",
      "Run 31\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 17.1% test: 17.5%\n",
      "\t100.0 samples train: 16.0% test: 22.4%\n",
      "\t150.0 samples train: 20.6% test: 15.7%\n",
      "\t200.0 samples train: 16.1% test: 22.5%\n",
      "\t250.0 samples train: 22.7% test: 23.5%\n",
      "\t300.0 samples train: 25.5% test: 22.1%\n",
      "\t350.0 samples train: 21.8% test: 20.9%\n",
      "\t400.0 samples train: 17.7% test: 19.7%\n",
      "\t450.0 samples train: 26.7% test: 20.7%\n",
      "\t500.0 samples train: 28.9% test: 21.8%\n",
      "\t550.0 samples train: 24.1% test: 19.4%\n",
      "\t600.0 samples train: 42.5% test: 20.2%\n",
      "\t650.0 samples train: 31.4% test: 20.6%\n",
      "\t700.0 samples train: 38.5% test: 21.6%\n",
      "\n",
      "\n",
      "Run 32\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 26.7% test: 23.7%\n",
      "\t100.0 samples train: 26.4% test: 19.2%\n",
      "\t150.0 samples train: 26.0% test: 25.4%\n",
      "\t200.0 samples train: 36.8% test: 19.7%\n",
      "\t250.0 samples train: 17.7% test: 21.0%\n",
      "\t300.0 samples train: 31.6% test: 18.3%\n",
      "\t350.0 samples train: 32.2% test: 21.8%\n",
      "\t400.0 samples train: 34.9% test: 20.8%\n",
      "\t450.0 samples train: 33.9% test: 19.2%\n",
      "\t500.0 samples train: 24.3% test: 19.9%\n",
      "\t550.0 samples train: 24.6% test: 19.5%\n",
      "\t600.0 samples train: 24.0% test: 19.7%\n",
      "\t650.0 samples train: 32.0% test: 20.3%\n",
      "\t700.0 samples train: 26.1% test: 19.5%\n",
      "\n",
      "\n",
      "Run 33\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 17.6% test: 14.3%\n",
      "\t100.0 samples train: 30.1% test: 22.4%\n",
      "\t150.0 samples train: 18.3% test: 17.7%\n",
      "\t200.0 samples train: 23.3% test: 21.0%\n",
      "\t250.0 samples train: 32.7% test: 19.3%\n",
      "\t300.0 samples train: 28.1% test: 19.5%\n",
      "\t350.0 samples train: 24.4% test: 19.7%\n",
      "\t400.0 samples train: 29.0% test: 20.0%\n",
      "\t450.0 samples train: 25.7% test: 22.3%\n",
      "\t500.0 samples train: 38.4% test: 19.4%\n",
      "\t550.0 samples train: 34.1% test: 19.3%\n",
      "\t600.0 samples train: 22.2% test: 20.6%\n",
      "\t650.0 samples train: 31.1% test: 19.4%\n",
      "\t700.0 samples train: 23.8% test: 19.5%\n",
      "\n",
      "\n",
      "Run 34\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 18.7% test: 20.8%\n",
      "\t100.0 samples train: 15.5% test: 20.1%\n",
      "\t150.0 samples train: 26.2% test: 24.7%\n",
      "\t200.0 samples train: 20.4% test: 23.5%\n",
      "\t250.0 samples train: 20.0% test: 25.6%\n",
      "\t300.0 samples train: 23.0% test: 24.6%\n",
      "\t350.0 samples train: 31.7% test: 18.3%\n",
      "\t400.0 samples train: 19.7% test: 19.8%\n",
      "\t450.0 samples train: 27.5% test: 20.9%\n",
      "\t500.0 samples train: 30.3% test: 20.3%\n",
      "\t550.0 samples train: 33.9% test: 19.7%\n",
      "\t600.0 samples train: 30.6% test: 19.7%\n",
      "\t650.0 samples train: 25.8% test: 20.5%\n",
      "\t700.0 samples train: 34.7% test: 19.8%\n",
      "\n",
      "\n",
      "Run 35\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 15.5% test: 18.2%\n",
      "\t100.0 samples train: 21.6% test: 15.9%\n",
      "\t150.0 samples train: 20.1% test: 25.4%\n",
      "\t200.0 samples train: 17.5% test: 18.9%\n",
      "\t250.0 samples train: 29.9% test: 19.5%\n",
      "\t300.0 samples train: 25.0% test: 18.9%\n",
      "\t350.0 samples train: 31.0% test: 21.6%\n",
      "\t400.0 samples train: 30.7% test: 21.5%\n",
      "\t450.0 samples train: 28.0% test: 20.8%\n",
      "\t500.0 samples train: 26.8% test: 21.7%\n",
      "\t550.0 samples train: 19.7% test: 18.7%\n",
      "\t600.0 samples train: 25.6% test: 19.4%\n",
      "\t650.0 samples train: 28.5% test: 19.0%\n",
      "\t700.0 samples train: 31.3% test: 20.0%\n",
      "\n",
      "\n",
      "Run 36\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 19.8% test: 18.2%\n",
      "\t100.0 samples train: 20.5% test: 22.3%\n",
      "\t150.0 samples train: 27.8% test: 25.0%\n",
      "\t200.0 samples train: 28.0% test: 27.5%\n",
      "\t250.0 samples train: 25.6% test: 20.1%\n",
      "\t300.0 samples train: 17.0% test: 23.9%\n",
      "\t350.0 samples train: 40.9% test: 24.3%\n",
      "\t400.0 samples train: 27.9% test: 22.2%\n",
      "\t450.0 samples train: 29.8% test: 20.6%\n",
      "\t500.0 samples train: 25.1% test: 19.5%\n",
      "\t550.0 samples train: 28.4% test: 19.8%\n",
      "\t600.0 samples train: 37.6% test: 19.8%\n",
      "\t650.0 samples train: 31.3% test: 21.0%\n",
      "\t700.0 samples train: 30.8% test: 19.5%\n",
      "\n",
      "\n",
      "Run 37\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 19.3% test: 21.0%\n",
      "\t100.0 samples train: 25.9% test: 23.2%\n",
      "\t150.0 samples train: 22.4% test: 18.1%\n",
      "\t200.0 samples train: 28.8% test: 19.7%\n",
      "\t250.0 samples train: 23.3% test: 22.6%\n",
      "\t300.0 samples train: 42.0% test: 25.8%\n",
      "\t350.0 samples train: 21.3% test: 16.0%\n",
      "\t400.0 samples train: 30.1% test: 22.9%\n",
      "\t450.0 samples train: 23.0% test: 19.9%\n",
      "\t500.0 samples train: 27.3% test: 22.1%\n",
      "\t550.0 samples train: 31.3% test: 20.8%\n",
      "\t600.0 samples train: 36.4% test: 19.4%\n",
      "\t650.0 samples train: 20.5% test: 20.5%\n",
      "\t700.0 samples train: 20.9% test: 20.1%\n",
      "\n",
      "\n",
      "Run 38\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 19.8% test: 20.7%\n",
      "\t100.0 samples train: 14.4% test: 17.0%\n",
      "\t150.0 samples train: 28.5% test: 30.4%\n",
      "\t200.0 samples train: 20.4% test: 19.7%\n",
      "\t250.0 samples train: 30.0% test: 20.6%\n",
      "\t300.0 samples train: 31.8% test: 24.9%\n",
      "\t350.0 samples train: 21.0% test: 20.1%\n",
      "\t400.0 samples train: 41.0% test: 22.2%\n",
      "\t450.0 samples train: 19.4% test: 22.1%\n",
      "\t500.0 samples train: 37.9% test: 19.7%\n",
      "\t550.0 samples train: 29.7% test: 21.9%\n",
      "\t600.0 samples train: 21.1% test: 21.3%\n",
      "\t650.0 samples train: 20.9% test: 21.4%\n",
      "\t700.0 samples train: 26.1% test: 19.4%\n",
      "\n",
      "\n",
      "Run 39\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 21.9% test: 22.7%\n",
      "\t100.0 samples train: 19.5% test: 22.7%\n",
      "\t150.0 samples train: 31.9% test: 19.9%\n",
      "\t200.0 samples train: 26.3% test: 22.1%\n",
      "\t250.0 samples train: 18.6% test: 18.6%\n",
      "\t300.0 samples train: 34.1% test: 21.3%\n",
      "\t350.0 samples train: 21.7% test: 21.8%\n",
      "\t400.0 samples train: 25.3% test: 22.5%\n",
      "\t450.0 samples train: 33.6% test: 20.2%\n",
      "\t500.0 samples train: 29.4% test: 18.4%\n",
      "\t550.0 samples train: 37.5% test: 19.5%\n",
      "\t600.0 samples train: 26.5% test: 21.0%\n",
      "\t650.0 samples train: 32.8% test: 20.0%\n",
      "\t700.0 samples train: 34.8% test: 19.4%\n",
      "\n",
      "\n",
      "Run 40\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 17.1% test: 16.0%\n",
      "\t100.0 samples train: 15.2% test: 14.1%\n",
      "\t150.0 samples train: 22.4% test: 19.5%\n",
      "\t200.0 samples train: 28.3% test: 20.9%\n",
      "\t250.0 samples train: 16.9% test: 21.3%\n",
      "\t300.0 samples train: 19.0% test: 22.9%\n",
      "\t350.0 samples train: 18.9% test: 22.7%\n",
      "\t400.0 samples train: 18.3% test: 19.1%\n",
      "\t450.0 samples train: 36.2% test: 20.0%\n",
      "\t500.0 samples train: 19.8% test: 20.1%\n",
      "\t550.0 samples train: 27.0% test: 19.8%\n",
      "\t600.0 samples train: 27.9% test: 19.7%\n",
      "\t650.0 samples train: 31.1% test: 21.1%\n",
      "\t700.0 samples train: 19.1% test: 19.7%\n",
      "\n",
      "\n",
      "Run 41\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 22.5% test: 18.7%\n",
      "\t100.0 samples train: 18.4% test: 28.7%\n",
      "\t150.0 samples train: 24.6% test: 20.8%\n",
      "\t200.0 samples train: 35.6% test: 19.7%\n",
      "\t250.0 samples train: 23.1% test: 19.1%\n",
      "\t300.0 samples train: 18.6% test: 18.6%\n",
      "\t350.0 samples train: 31.9% test: 20.8%\n",
      "\t400.0 samples train: 19.5% test: 19.5%\n",
      "\t450.0 samples train: 25.0% test: 20.1%\n",
      "\t500.0 samples train: 33.3% test: 20.2%\n",
      "\t550.0 samples train: 22.1% test: 20.9%\n",
      "\t600.0 samples train: 29.5% test: 20.0%\n",
      "\t650.0 samples train: 38.2% test: 20.7%\n",
      "\t700.0 samples train: 29.7% test: 19.0%\n",
      "\n",
      "\n",
      "Run 42\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 18.7% test: 17.7%\n",
      "\t100.0 samples train: 14.4% test: 16.7%\n",
      "\t150.0 samples train: 23.8% test: 20.3%\n",
      "\t200.0 samples train: 26.4% test: 20.8%\n",
      "\t250.0 samples train: 17.2% test: 16.2%\n",
      "\t300.0 samples train: 27.8% test: 27.0%\n",
      "\t350.0 samples train: 30.3% test: 17.7%\n",
      "\t400.0 samples train: 31.6% test: 19.2%\n",
      "\t450.0 samples train: 18.2% test: 20.3%\n",
      "\t500.0 samples train: 26.3% test: 19.8%\n",
      "\t550.0 samples train: 31.7% test: 20.2%\n",
      "\t600.0 samples train: 37.2% test: 21.3%\n",
      "\t650.0 samples train: 33.5% test: 19.2%\n",
      "\t700.0 samples train: 27.9% test: 19.5%\n",
      "\n",
      "\n",
      "Run 43\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 28.9% test: 28.7%\n",
      "\t100.0 samples train: 11.2% test: 30.7%\n",
      "\t150.0 samples train: 27.8% test: 22.6%\n",
      "\t200.0 samples train: 17.1% test: 27.8%\n",
      "\t250.0 samples train: 16.5% test: 17.8%\n",
      "\t300.0 samples train: 21.3% test: 22.6%\n",
      "\t350.0 samples train: 29.6% test: 20.6%\n",
      "\t400.0 samples train: 28.1% test: 19.9%\n",
      "\t450.0 samples train: 22.5% test: 21.0%\n",
      "\t500.0 samples train: 37.5% test: 20.2%\n",
      "\t550.0 samples train: 28.6% test: 20.7%\n",
      "\t600.0 samples train: 31.2% test: 19.7%\n",
      "\t650.0 samples train: 31.4% test: 18.7%\n",
      "\t700.0 samples train: 30.7% test: 20.2%\n",
      "\n",
      "\n",
      "Run 44\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 16.0% test: 19.2%\n",
      "\t100.0 samples train: 20.0% test: 17.8%\n",
      "\t150.0 samples train: 19.9% test: 19.3%\n",
      "\t200.0 samples train: 15.7% test: 25.3%\n",
      "\t250.0 samples train: 22.7% test: 23.0%\n",
      "\t300.0 samples train: 20.0% test: 23.0%\n",
      "\t350.0 samples train: 19.4% test: 19.9%\n",
      "\t400.0 samples train: 29.1% test: 20.6%\n",
      "\t450.0 samples train: 31.1% test: 22.4%\n",
      "\t500.0 samples train: 24.5% test: 19.2%\n",
      "\t550.0 samples train: 42.1% test: 19.2%\n",
      "\t600.0 samples train: 22.6% test: 20.8%\n",
      "\t650.0 samples train: 36.6% test: 20.5%\n",
      "\t700.0 samples train: 21.5% test: 21.0%\n",
      "\n",
      "\n",
      "Run 45\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 18.7% test: 18.5%\n",
      "\t100.0 samples train: 27.2% test: 24.7%\n",
      "\t150.0 samples train: 22.4% test: 21.8%\n",
      "\t200.0 samples train: 32.5% test: 21.6%\n",
      "\t250.0 samples train: 24.0% test: 26.3%\n",
      "\t300.0 samples train: 27.4% test: 21.5%\n",
      "\t350.0 samples train: 20.7% test: 22.9%\n",
      "\t400.0 samples train: 24.9% test: 21.5%\n",
      "\t450.0 samples train: 18.9% test: 21.7%\n",
      "\t500.0 samples train: 23.5% test: 17.7%\n",
      "\t550.0 samples train: 26.2% test: 20.2%\n",
      "\t600.0 samples train: 27.6% test: 20.0%\n",
      "\t650.0 samples train: 28.5% test: 20.7%\n",
      "\t700.0 samples train: 25.1% test: 19.9%\n",
      "\n",
      "\n",
      "Run 46\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 26.2% test: 26.4%\n",
      "\t100.0 samples train: 18.7% test: 16.6%\n",
      "\t150.0 samples train: 20.1% test: 25.8%\n",
      "\t200.0 samples train: 23.7% test: 21.0%\n",
      "\t250.0 samples train: 25.4% test: 22.5%\n",
      "\t300.0 samples train: 42.8% test: 20.7%\n",
      "\t350.0 samples train: 38.3% test: 22.9%\n",
      "\t400.0 samples train: 36.9% test: 19.4%\n",
      "\t450.0 samples train: 19.9% test: 19.2%\n",
      "\t500.0 samples train: 39.6% test: 19.2%\n",
      "\t550.0 samples train: 37.1% test: 20.8%\n",
      "\t600.0 samples train: 28.8% test: 19.3%\n",
      "\t650.0 samples train: 19.7% test: 20.8%\n",
      "\t700.0 samples train: 30.5% test: 20.8%\n",
      "\n",
      "\n",
      "Run 47\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 23.5% test: 20.1%\n",
      "\t100.0 samples train: 15.7% test: 12.6%\n",
      "\t150.0 samples train: 18.9% test: 16.6%\n",
      "\t200.0 samples train: 22.4% test: 22.6%\n",
      "\t250.0 samples train: 20.4% test: 27.4%\n",
      "\t300.0 samples train: 19.4% test: 18.9%\n",
      "\t350.0 samples train: 23.9% test: 20.6%\n",
      "\t400.0 samples train: 14.7% test: 20.5%\n",
      "\t450.0 samples train: 28.5% test: 22.1%\n",
      "\t500.0 samples train: 21.1% test: 18.5%\n",
      "\t550.0 samples train: 35.9% test: 21.7%\n",
      "\t600.0 samples train: 32.3% test: 20.6%\n",
      "\t650.0 samples train: 38.1% test: 21.6%\n",
      "\t700.0 samples train: 31.4% test: 19.0%\n",
      "\n",
      "\n",
      "Run 48\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 34.8% test: 32.3%\n",
      "\t100.0 samples train: 16.3% test: 13.5%\n",
      "\t150.0 samples train: 35.8% test: 17.4%\n",
      "\t200.0 samples train: 25.1% test: 21.8%\n",
      "\t250.0 samples train: 26.8% test: 22.2%\n",
      "\t300.0 samples train: 23.9% test: 17.3%\n",
      "\t350.0 samples train: 30.6% test: 25.4%\n",
      "\t400.0 samples train: 34.2% test: 22.7%\n",
      "\t450.0 samples train: 19.3% test: 21.5%\n",
      "\t500.0 samples train: 21.4% test: 21.8%\n",
      "\t550.0 samples train: 25.6% test: 20.3%\n",
      "\t600.0 samples train: 25.8% test: 19.8%\n",
      "\t650.0 samples train: 27.6% test: 21.7%\n",
      "\t700.0 samples train: 32.3% test: 19.4%\n",
      "\n",
      "\n",
      "Run 49\n",
      "(875, 10, 10)\n",
      "\t50.0 samples train: 25.1% test: 23.9%\n",
      "\t100.0 samples train: 18.1% test: 27.2%\n",
      "\t150.0 samples train: 26.9% test: 15.7%\n",
      "\t200.0 samples train: 23.6% test: 21.6%\n",
      "\t250.0 samples train: 31.7% test: 16.7%\n",
      "\t300.0 samples train: 27.1% test: 19.7%\n",
      "\t350.0 samples train: 29.7% test: 19.8%\n",
      "\t400.0 samples train: 23.5% test: 20.9%\n",
      "\t450.0 samples train: 21.7% test: 21.4%\n",
      "\t500.0 samples train: 29.3% test: 19.7%\n",
      "\t550.0 samples train: 39.4% test: 21.0%\n",
      "\t600.0 samples train: 29.3% test: 18.9%\n",
      "\t650.0 samples train: 24.4% test: 21.7%\n",
      "\t700.0 samples train: 36.2% test: 18.6%\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "info_x_samples = [\n",
    "    (data_50_samples, target_50_samples), \n",
    "    (data_100_samples, target_100_samples),\n",
    "    (data_150_samples, target_150_samples), \n",
    "    (data_200_samples, target_200_samples), \n",
    "    (data_250_samples, target_250_samples), \n",
    "    (data_300_samples, target_300_samples), \n",
    "    (data_350_samples, target_350_samples), \n",
    "    (data_400_samples, target_400_samples), \n",
    "    (data_450_samples, target_450_samples),\n",
    "    (data_500_samples, target_500_samples), \n",
    "    (data_550_samples, target_550_samples), \n",
    "    (data_600_samples, target_600_samples), \n",
    "    (data_650_samples, target_650_samples), \n",
    "    (data_700_samples, target_700_samples)\n",
    "]\n",
    "sample_numbers = (list(range(50, 701, 50)))\n",
    "results = []\n",
    "\n",
    "def samples_test(verbose=True):\n",
    "\n",
    "    dummy, static_X_test, dummy, static_y_test = train_test_split(data_700_samples, target_700_samples, stratify=target_700_samples)\n",
    "    results.append([])\n",
    "\n",
    "    for sample_number, i in zip(sample_numbers, range(14)):\n",
    "        data_x_samples = info_x_samples[i][0]\n",
    "        target_x_samples = info_x_samples[i][1]\n",
    "    \n",
    "        scaler = StandardScaler()\n",
    "    \n",
    "        data_x_samples = data_x_samples.reshape(data_x_samples.shape[0], -1)\n",
    "        data_x_samples = (scaler.fit_transform(data_x_samples)).reshape((data_x_samples.shape[0], 10, 10, 1))\n",
    "        \n",
    "        target_x_samples = to_categorical(target_x_samples)\n",
    "\n",
    "        split_X_train, split_X_test, split_y_train, split_y_test = train_test_split(data_x_samples, target_x_samples, stratify=target_x_samples)\n",
    "# TODO: why is the test accuracy different from the initial accuracy test?\n",
    "        model = Model()\n",
    "    \n",
    "        model.build(input_shape=(1, 10, 10, 1))\n",
    "        model.compile(optimizer='sgd', loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "        \n",
    "        model.fit(split_X_train, split_y_train, epochs=3, batch_size=64, verbose=False)\n",
    "    \n",
    "        train_accuracy = model.evaluate(split_X_train, split_y_train, verbose=False)[1]\n",
    "\n",
    "        static_X_test = static_X_test.reshape(static_X_test.shape[0], -1)\n",
    "        static_X_test = (scaler.transform(static_X_test)).reshape((static_X_test.shape[0], 10, 10, 1))\n",
    "\n",
    "        test_accuracy = model.evaluate(static_X_test, to_categorical(static_y_test), verbose=False)[1]\n",
    "        \n",
    "        print(f'\\t{(data_x_samples.shape[0])/5} samples train: {train_accuracy*100:.1f}% test: {test_accuracy*100:.1f}%')\n",
    "\n",
    "        results[-1].append(test_accuracy)\n",
    "\n",
    "for i in range(49):\n",
    "    print(f\"Run {i+1}\")\n",
    "    samples_test(verbose=False)\n",
    "    print(\"\\n\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Prepare CSV for R Data Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = np.array(results)\n",
    "df = pd.DataFrame(results, columns=[f'{num} samples' for num in range(50, 701, 50)])\n",
    "df.to_csv('R Data Analysis/data/samples_test_175_data.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Total dataset test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Setup data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "original data.shape: (1259300, 10, 10)\n",
      "original target.shape: (1259300,)\n",
      "\n",
      "electron.shape: (3150, 10, 10), (3150,)\n",
      "muon.shape: (700, 10, 10), (700,)\n",
      "pion.shape: (981050, 10, 10), (981050,)\n",
      "kaon.shape: (160300, 10, 10), (160300,)\n",
      "proton.shape: (114100, 10, 10), (114100,)\n",
      "\n",
      "X_train.shape: (944475, 10, 10, 1)\n",
      "y_train.shape: (944475, 5)\n",
      "X_test.shape: (314825, 10, 10, 1)\n",
      "y_test.shape: (314825, 5)\n"
     ]
    }
   ],
   "source": [
    "# In this test, we evaluate the models accuracies on the dataset without limit of 700 per class.\n",
    "# However, first we must setup the data, as shown in the top of the notebook.\n",
    "\n",
    "data_total = []\n",
    "target_total = []\n",
    "\n",
    "for file in files:\n",
    "    file = open(f'data/{files[0]}', 'rb')\n",
    "    file = pickle.load(file)\n",
    "\n",
    "    for sample, sample_target in zip(file[0], file[1]):\n",
    "        data_total.append(sample)\n",
    "        target_total.append(sample_target)\n",
    "\n",
    "data_total = np.array(data_total)\n",
    "target_total = np.array(target_total)\n",
    "\n",
    "sleep(7)\n",
    "clear_output()\n",
    "\n",
    "print(f'original data.shape: {data_total.shape}')\n",
    "print(f'original target.shape: {target_total.shape}\\n')\n",
    "\n",
    "# Edit target values to 0, 1, 2...\n",
    "new_target = []\n",
    "\n",
    "for tar in target_total:\n",
    "    if tar == 11:\n",
    "        new_target.append(0)\n",
    "    elif tar == 13:\n",
    "        new_target.append(1)\n",
    "    elif tar == 211:\n",
    "        new_target.append(2)\n",
    "    elif tar == 321:\n",
    "        new_target.append(3)\n",
    "    else:\n",
    "        new_target.append(4)\n",
    "    \n",
    "target_total = np.array(new_target)\n",
    "\n",
    "for i in range(5):\n",
    "    particle_indexes = np.where(target_total == i)[0]\n",
    "\n",
    "    data_modified = data_total[particle_indexes]\n",
    "    target_modified = target_total[particle_indexes]\n",
    "    \n",
    "    print(f'{target_names[str(i)]}.shape: {data_modified.shape}, {target_modified.shape}')\n",
    "\n",
    "# Split the data\n",
    "X_train, X_test, y_train, y_test = train_test_split(data_total, target_total, stratify=target_total)\n",
    "\n",
    "# Change the train and test datasets.\n",
    "scaler = StandardScaler()\n",
    "\n",
    "# Reshape + change data\n",
    "X_train = X_train.reshape(X_train.shape[0], -1)\n",
    "X_test = X_test.reshape(X_test.shape[0], -1)\n",
    "\n",
    "# Scale and reshape data back to og form.\n",
    "X_train = (scaler.fit_transform(X_train)).reshape((X_train.shape[0], 10, 10, 1))\n",
    "X_test = (scaler.transform(X_test)).reshape((X_test.shape[0], 10, 10, 1))\n",
    "\n",
    "y_train = to_categorical(y_train)\n",
    "y_test = to_categorical(y_test)\n",
    "\n",
    "print(f'\\nX_train.shape: {X_train.shape}')\n",
    "print(f'y_train.shape: {y_train.shape}')\n",
    "print(f'X_test.shape: {X_test.shape}')\n",
    "print(f'y_test.shape: {y_test.shape}')\n",
    "\n",
    "# NOTE: We are using the unbalanced dataset in order to see if there is correlation to high accuracy with particles that have high # of samples.\n",
    "#       Based on the dataset partitions below, we would expect pion, kaon, and proton to have a higher accuracy."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Prepare test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 0.7%\n",
      "proton accuracy: 0.6%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 1.5%\n",
      "proton accuracy: 0.3%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 1.3%\n",
      "proton accuracy: 0.9%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 1.3%\n",
      "proton accuracy: 0.9%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 1.1%\n",
      "proton accuracy: 0.9%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 1.3%\n",
      "proton accuracy: 0.6%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 0.9%\n",
      "proton accuracy: 0.6%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 1.1%\n",
      "proton accuracy: 0.9%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 1.3%\n",
      "proton accuracy: 0.6%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 1.5%\n",
      "proton accuracy: 1.2%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 1.1%\n",
      "proton accuracy: 0.9%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 1.1%\n",
      "proton accuracy: 1.2%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 1.1%\n",
      "proton accuracy: 0.9%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 0.9%\n",
      "proton accuracy: 0.9%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 1.3%\n",
      "proton accuracy: 0.9%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 1.3%\n",
      "proton accuracy: 0.6%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 1.5%\n",
      "proton accuracy: 1.2%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 1.1%\n",
      "proton accuracy: 0.6%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 1.5%\n",
      "proton accuracy: 1.2%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 1.1%\n",
      "proton accuracy: 0.6%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 1.1%\n",
      "proton accuracy: 0.3%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 1.5%\n",
      "proton accuracy: 0.6%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 1.1%\n",
      "proton accuracy: 0.6%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 0.9%\n",
      "proton accuracy: 0.9%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 1.3%\n",
      "proton accuracy: 0.6%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 1.3%\n",
      "proton accuracy: 0.6%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 1.5%\n",
      "proton accuracy: 0.9%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 1.5%\n",
      "proton accuracy: 0.6%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 0.7%\n",
      "proton accuracy: 0.9%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 1.1%\n",
      "proton accuracy: 0.9%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 1.3%\n",
      "proton accuracy: 0.6%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 1.1%\n",
      "proton accuracy: 0.3%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 1.1%\n",
      "proton accuracy: 0.6%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 1.5%\n",
      "proton accuracy: 0.6%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 0.9%\n",
      "proton accuracy: 0.3%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 1.3%\n",
      "proton accuracy: 0.6%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 1.1%\n",
      "proton accuracy: 0.9%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 1.1%\n",
      "proton accuracy: 1.2%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 1.1%\n",
      "proton accuracy: 0.9%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 1.3%\n",
      "proton accuracy: 1.2%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 1.1%\n",
      "proton accuracy: 0.3%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 1.5%\n",
      "proton accuracy: 0.9%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 1.3%\n",
      "proton accuracy: 0.6%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 1.1%\n",
      "proton accuracy: 0.9%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 1.4%\n",
      "proton accuracy: 0.6%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 1.3%\n",
      "proton accuracy: 0.9%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 1.3%\n",
      "proton accuracy: 0.6%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 1.3%\n",
      "proton accuracy: 0.9%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 1.5%\n",
      "proton accuracy: 0.9%\n",
      "electron accuracy: 0.0%\n",
      "muon accuracy: 0.0%\n",
      "pion accuracy: 100.0%\n",
      "kaon accuracy: 1.3%\n",
      "proton accuracy: 0.6%\n"
     ]
    }
   ],
   "source": [
    "results = []\n",
    "\n",
    "def individual_particle_test_modified(model):    # The only difference with this one is we record the data\n",
    "\n",
    "    results.append([])\n",
    "    model_recreate = Model()\n",
    "\n",
    "    model_recreate.build(input_shape=(1, 10, 10, 1))\n",
    "    model_recreate.compile(optimizer='sgd', loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "    history = model_recreate.fit(X_train, y_train, epochs=3, batch_size=64, verbose=False).history    # We want to recreate the model every run so that we get different results\n",
    "\n",
    "    for i in range(5):\n",
    "        int_y_test = np.array([np.argmax(y, axis=None, out=None) for y in y_test])    # convert back to integer for comparison.\n",
    "        particle_indexes = np.where(int_y_test == i)    # gives indexes for all electron, muon, etc testcases...\n",
    "\n",
    "        X_test_modified = X_test[particle_indexes]\n",
    "        y_test_modified = y_test[particle_indexes]\n",
    "\n",
    "        particle_accuracy = model_recreate.evaluate(X_test_modified, y_test_modified, verbose=False)[1]\n",
    "\n",
    "        print(f'{target_names[str(i)]} accuracy: {particle_accuracy*100:.1f}%')\n",
    "\n",
    "        results[-1].append(particle_accuracy)\n",
    "\n",
    "for i in range(50):\n",
    "    individual_particle_test_modified(tf_model)\n",
    "\n",
    "# Basically, in this test, train on ENTIRE dataset;\n",
    "# test on specific particle (recorded as result)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Prepare results for R data analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = np.array(results)\n",
    "df = pd.DataFrame(results, columns=['electron accuracy', 'muon accuracy', 'pion accuracy', 'kaon accuracy', 'proton accuracy'])\n",
    "df.to_csv('R Data Analysis/data/total_dataset_test_data.csv')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Muon vs All Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1400, 10, 10)\n",
      "(1400, 2)\n"
     ]
    }
   ],
   "source": [
    "# In this test, we change the target where the muon is 1 and all else is changed to 0.\n",
    "\n",
    "muon_data, muon_target = (data_700_samples, target_700_samples)\n",
    "\n",
    "muon_target = list(muon_target)\n",
    "\n",
    "# muon before is label 1 (already), everything else needs to be changed to a 0\n",
    "for i in range(len(muon_target)):\n",
    "    if muon_target[i] != 1:\n",
    "        muon_target[i] = 0\n",
    "\n",
    "muon_target = np.array(muon_target)\n",
    "\n",
    "# change dataset to 700 muon and 700 other\n",
    "muon_indexes = np.where(muon_target == 1)\n",
    "other_indexes = np.where(muon_target == 0)\n",
    "\n",
    "muon_target = np.append(muon_target[muon_indexes], muon_target[other_indexes][:700])\n",
    "muon_data = np.append(muon_data[muon_indexes], muon_data[other_indexes][:700], axis=0)\n",
    "\n",
    "# muon_data = muon_data.reshape(muon_data.shape[0], -1)\n",
    "# muon_data = (scaler.fit_transform(muon_data)).reshape(muon_data.shape[0], 10, 10, 1)\n",
    "\n",
    "print(muon_data.shape)\n",
    "print(moun_target.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train accuracy: 1.0\n",
      "Test accuracy: 1.0\n",
      "\n"
     ]
    }
   ],
   "source": [
    "muon_X_train, muon_X_test, muon_y_train, muon_y_test = train_test_split(muon_data, muon_target, test_size=100, stratify=muon_target)\n",
    "\n",
    "scaler = StandardScaler()\n",
    "\n",
    "muon_X_train = muon_X_train.reshape(muon_X_train.shape[0], -1)\n",
    "muon_X_test = muon_X_test.reshape(muon_X_test.shape[0], -1)\n",
    "\n",
    "muon_X_train = (scaler.fit_transform(muon_X_train)).reshape((muon_X_train.shape[0], 10, 10, 1))\n",
    "muon_X_test = (scaler.transform(muon_X_test)).reshape((muon_X_test.shape[0], 10, 10, 1))\n",
    "\n",
    "muon_y_test = to_categorical(muon_y_test)\n",
    "muon_y_train = to_categorical(muon_y_train)\n",
    "\n",
    "# muon_X_train, muon_y_train = (np.append(muon_data[:350], muon_data[700:1050], axis=0), np.append(muon_target[:350], muon_target[700:1050]))\n",
    "# muon_X_test, muon_y_test = (np.append(muon_data[350:700], muon_data[1050:1400], axis=0), np.append(muon_target[350:700], muon_target[1050:1400]))\n",
    "\n",
    "# moun_target = to_categorical(muon_target)\n",
    "\n",
    "class Muon_Model(tf.keras.Model):\n",
    "    def __init__(self):\n",
    "        super(Muon_Model, self).__init__()\n",
    "\n",
    "        self.input_layers = [\n",
    "            MaxPooling2D((2, 2)),\n",
    "            Flatten()\n",
    "        ]\n",
    "\n",
    "        self.hidden_layers = [\n",
    "            Dense(19, activation='relu', kernel_regularizer='l2'),\n",
    "            Dense(19, activation='relu', kernel_regularizer='l2')\n",
    "        ]\n",
    "\n",
    "        self.output_layer = Dense(2, activation='softmax')\n",
    "    \n",
    "    def call(self, inputs):\n",
    "        h = self.input_layers[0](inputs)\n",
    "        h = self.input_layers[1](h)\n",
    "\n",
    "        for hidden_layer in self.hidden_layers:\n",
    "            h = hidden_layer(h)\n",
    "        \n",
    "        return self.output_layer(h)\n",
    "\n",
    "\n",
    "muon_model = Muon_Model()\n",
    "\n",
    "muon_model.build(input_shape=(1, 10, 10, 1))\n",
    "muon_model.compile(optimizer='sgd', loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "history = muon_model.fit(muon_X_train, muon_y_train, epochs=7, batch_size=64, verbose=False)\n",
    "\n",
    "train_accuracy_muon = muon_model.evaluate(muon_X_train, muon_y_train, verbose=False)[1]\n",
    "test_accuracy_muon = muon_model.evaluate(muon_X_test, muon_y_test, verbose=False)[1]\n",
    "\n",
    "print(f'Train accuracy: {train_accuracy_muon}')\n",
    "print(f'Test accuracy: {test_accuracy_muon}\\n')\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Muon Test: Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[50  0]\n",
      " [ 0 50]]\n",
      "TP Rate 100%\n",
      "FP Rate 100%\n"
     ]
    }
   ],
   "source": [
    "predictions = muon_model.predict(muon_X_test)\n",
    "predictions_stable = []\n",
    "\n",
    "for prediction in predictions:\n",
    "    predictions_stable.append(np.argmax(prediction))\n",
    "\n",
    "answers = muon_y_test\n",
    "answers_stable = []\n",
    "\n",
    "for answer in answers:\n",
    "    answers_stable.append(np.argmax(answer))\n",
    "\n",
    "print(confusion_matrix(np.array(answers_stable), np.array(predictions_stable)))\n",
    "\n",
    "# Remember, 0 is everything else, 1 is muon\n",
    "\n",
    "print(f'TP Rate 100%')    # 100% on predicting samples labeled 1\n",
    "print(f'FP Rate 100%')    # 100% on predicting samples labeled 0"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Electron vs Non-Electron"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1400, 10, 10)\n",
      "(1400,)\n"
     ]
    }
   ],
   "source": [
    "# In this test, we change the target where the electron is 1 and all else is changed to 0.\n",
    "\n",
    "electron_data, electron_target = (data_700_samples, target_700_samples)\n",
    "\n",
    "electron_target = list(electron_target)\n",
    "\n",
    "# electron is label 0 (change to 1), everything else becomes 0. \n",
    "for i in range(len(electron_target)):\n",
    "    if electron_target[i] != 0:\n",
    "        electron_target[i] = 0\n",
    "    else:\n",
    "        electron_target[i] = 1 \n",
    "\n",
    "electron_target = np.array(electron_target)        \n",
    "\n",
    "# change dataset to 700 electron and 700 other\n",
    "electron_indexes = np.where(electron_target == 1)\n",
    "other_indexes = np.where(electron_target == 0)\n",
    "\n",
    "electron_target = np.append(electron_target[electron_indexes], electron_target[other_indexes][:700])\n",
    "electron_data = np.append(electron_data[electron_indexes], electron_data[other_indexes][:700], axis=0)\n",
    "\n",
    "print(electron_data.shape)\n",
    "print(electron_target.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train.shape: (1300, 10, 10, 1)\n",
      "y_train.shape: (1300, 2)\n",
      "X_test.shape: (100, 10, 10, 1)\n",
      "y_test.shape: (100, 2)\n"
     ]
    }
   ],
   "source": [
    "electron_X_train, electron_X_test, electron_y_train, electron_y_test = train_test_split(electron_data, electron_target, test_size = 100, stratify = electron_target)\n",
    "\n",
    "# Change the train and test datasets.\n",
    "scaler = StandardScaler()\n",
    "\n",
    "# Reshape + change data\n",
    "electron_X_train = electron_X_train.reshape(electron_X_train.shape[0], -1)\n",
    "electron_X_test = electron_X_test.reshape(electron_X_test.shape[0], -1)\n",
    "\n",
    "# Scale and reshape data back to og form.\n",
    "electron_X_train = (scaler.fit_transform(electron_X_train)).reshape((electron_X_train.shape[0], 10, 10, 1))\n",
    "electron_X_test = (scaler.transform(electron_X_test)).reshape((electron_X_test.shape[0], 10, 10, 1))\n",
    "\n",
    "electron_y_test = to_categorical(electron_y_test)\n",
    "electron_y_train = to_categorical(electron_y_train)\n",
    "\n",
    "print(f'X_train.shape: {electron_X_train.shape}')\n",
    "print(f'y_train.shape: {electron_y_train.shape}')\n",
    "print(f'X_test.shape: {electron_X_test.shape}')\n",
    "print(f'y_test.shape: {electron_y_test.shape}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train accuracy: 100.0%\n",
      "Test accuracy: 100.0%\n"
     ]
    }
   ],
   "source": [
    "\n",
    "class Electron_Model(tf.keras.Model):\n",
    "    def __init__(self):\n",
    "        super(Electron_Model, self).__init__()\n",
    "\n",
    "        self.input_layers = [\n",
    "            MaxPooling2D((2, 2)),\n",
    "            Flatten()\n",
    "        ]\n",
    "\n",
    "        self.hidden_layers = [\n",
    "            Dense(19, activation='relu', kernel_regularizer='l2'),\n",
    "            Dense(19, activation='relu', kernel_regularizer='l2')\n",
    "        ]\n",
    "\n",
    "        self.output_layer = Dense(2, activation='softmax')\n",
    "    \n",
    "    def call(self, inputs):\n",
    "        h = self.input_layers[0](inputs)\n",
    "        h = self.input_layers[1](h)\n",
    "\n",
    "        for hidden_layer in self.hidden_layers:\n",
    "            h = hidden_layer(h)\n",
    "        \n",
    "        return self.output_layer(h)\n",
    "\n",
    "electron_model = Electron_Model()\n",
    "\n",
    "electron_model.build(input_shape=(1, 10, 10, 1))\n",
    "electron_model.compile(optimizer='sgd', loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "history = electron_model.fit(electron_X_train, electron_y_train, epochs=7, batch_size=64, verbose=False)\n",
    "\n",
    "sleep(4)\n",
    "\n",
    "print(f'Train accuracy: {electron_model.evaluate(electron_X_train, electron_y_train, verbose=False)[1]*100:.1f}%')\n",
    "print(f'Test accuracy: {electron_model.evaluate(electron_X_test, electron_y_test, verbose=False)[1]*100:.1f}%')\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Electron Test: Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[50  0]\n",
      " [ 0 50]]\n",
      "TP Rate 100%\n",
      "FP Rate 100%\n"
     ]
    }
   ],
   "source": [
    "predictions = electron_model.predict(electron_X_test)\n",
    "predictions_stable = []\n",
    "\n",
    "for prediction in predictions:\n",
    "    predictions_stable.append(np.argmax(prediction))\n",
    "\n",
    "answers = electron_y_test\n",
    "answers_stable = []\n",
    "\n",
    "for answer in answers:\n",
    "    answers_stable.append(np.argmax(answer))\n",
    "\n",
    "print(confusion_matrix(np.array(answers_stable), np.array(predictions_stable)))\n",
    "\n",
    "# Remember, 0 is everything else, 1 is electron\n",
    "\n",
    "# Remember, 0 is electron, 1 is everything else\n",
    "\n",
    "print(f'TP Rate 100%')    # 50% on predicting samples labeled 1\n",
    "print(f'FP Rate 100%')    # 100% on predicting samples labeled 0\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pion vs Non-Pion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1400, 10, 10)\n",
      "(1400,)\n"
     ]
    }
   ],
   "source": [
    "# In this test, we change the target where the pion is 0 and all else is changed to 1.\n",
    "\n",
    "pion_data, pion_target = (data_700_samples, target_700_samples)\n",
    "\n",
    "pion_target = list(pion_target)\n",
    "\n",
    "# pion is label 2 (change to 1), everything else needs to be changed to a 0\n",
    "for i in range(len(pion_target)):\n",
    "    if pion_target[i] != 2:\n",
    "        pion_target[i] = 0\n",
    "    else:\n",
    "        pion_target[i] = 1\n",
    "\n",
    "pion_target = np.array(pion_target)        \n",
    "\n",
    "# change dataset to 700 pion and 700 other\n",
    "pion_indexes = np.where(pion_target == 1)\n",
    "other_indexes = np.where(pion_target == 0)\n",
    "\n",
    "pion_target = np.append(pion_target[pion_indexes], pion_target[other_indexes][:700])\n",
    "pion_data = np.append(pion_data[pion_indexes], pion_data[other_indexes][:700], axis=0)\n",
    "\n",
    "print(pion_data.shape)\n",
    "print(pion_target.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train.shape: (1300, 10, 10, 1)\n",
      "y_train.shape: (1300, 2)\n",
      "X_test.shape: (100, 10, 10, 1)\n",
      "y_test.shape: (100, 2)\n"
     ]
    }
   ],
   "source": [
    "pion_X_train, pion_X_test, pion_y_train, pion_y_test = train_test_split(pion_data, pion_target, test_size = 100, stratify = pion_target)\n",
    "\n",
    "# Change the train and test datasets.\n",
    "scaler = StandardScaler()\n",
    "\n",
    "# Reshape + change data\n",
    "pion_X_train = pion_X_train.reshape(pion_X_train.shape[0], -1)\n",
    "pion_X_test = pion_X_test.reshape(pion_X_test.shape[0], -1)\n",
    "\n",
    "# Scale and reshape data back to og form.\n",
    "pion_X_train = (scaler.fit_transform(pion_X_train)).reshape((pion_X_train.shape[0], 10, 10, 1))\n",
    "pion_X_test = (scaler.transform(pion_X_test)).reshape((pion_X_test.shape[0], 10, 10, 1))\n",
    "\n",
    "pion_y_test = to_categorical(pion_y_test)\n",
    "pion_y_train = to_categorical(pion_y_train)\n",
    "\n",
    "print(f'X_train.shape: {pion_X_train.shape}')\n",
    "print(f'y_train.shape: {pion_y_train.shape}')\n",
    "print(f'X_test.shape: {pion_X_test.shape}')\n",
    "print(f'y_test.shape: {pion_y_test.shape}')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train accuracy: 70.8%\n",
      "Test accuracy: 72.0%\n"
     ]
    }
   ],
   "source": [
    "class Pion_Model(tf.keras.Model):\n",
    "    def __init__(self):\n",
    "        super(Pion_Model, self).__init__()\n",
    "\n",
    "        self.input_layers = [\n",
    "            MaxPooling2D((2, 2)),\n",
    "            Flatten()\n",
    "        ]\n",
    "\n",
    "        self.hidden_layers = [\n",
    "            Dense(19, activation='relu', kernel_regularizer='l2'),\n",
    "            Dense(19, activation='relu', kernel_regularizer='l2')\n",
    "        ]\n",
    "\n",
    "        self.output_layer = Dense(2, activation='softmax')\n",
    "    \n",
    "    def call(self, inputs):\n",
    "        h = self.input_layers[0](inputs)\n",
    "        h = self.input_layers[1](h)\n",
    "\n",
    "        for hidden_layer in self.hidden_layers:\n",
    "            h = hidden_layer(h)\n",
    "        \n",
    "        return self.output_layer(h)\n",
    "\n",
    "pion_model = Pion_Model()\n",
    "\n",
    "pion_model.build(input_shape=(1, 10, 10, 1))\n",
    "pion_model.compile(optimizer='sgd', loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "history = pion_model.fit(pion_X_train, pion_y_train, epochs=7, batch_size=64, verbose=False)\n",
    "\n",
    "sleep(4)\n",
    "\n",
    "print(f'Train accuracy: {pion_model.evaluate(pion_X_train, pion_y_train, verbose=False)[1]*100:.1f}%')\n",
    "print(f'Test accuracy: {pion_model.evaluate(pion_X_test, pion_y_test, verbose=False)[1]*100:.1f}%')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Pion Test: Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[23 27]\n",
      " [ 1 49]]\n",
      "TP Rate 64.5%\n",
      "FP Rate 96%\n"
     ]
    }
   ],
   "source": [
    "predictions = pion_model.predict(pion_X_test)\n",
    "predictions_stable = []\n",
    "\n",
    "for prediction in predictions:\n",
    "    predictions_stable.append(np.argmax(prediction))\n",
    "\n",
    "answers = pion_y_test\n",
    "answers_stable = []\n",
    "\n",
    "for answer in answers:\n",
    "    answers_stable.append(np.argmax(answer))\n",
    "\n",
    "print(confusion_matrix(np.array(answers_stable), np.array(predictions_stable)))\n",
    "\n",
    "# Remember, 1 is pion, 0 is everything else.\n",
    "\n",
    "print(f'TP Rate 64.5%')    # 50% on predicting samples labeled 1 TODO: fix these\n",
    "print(f'FP Rate 96%')    # 0% on predicting smaples labeled 0\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Kaon vs Non-Kaon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1400, 10, 10)\n",
      "(1400,)\n"
     ]
    }
   ],
   "source": [
    "# In this test, we change the target where the kaon is 1 and all else is changed to 0.\n",
    "\n",
    "kaon_data, kaon_target = (data_700_samples, target_700_samples)\n",
    "\n",
    "kaon_target = list(kaon_target)\n",
    "\n",
    "# kaon is label 3 (make 1), everything else needs to be changed to a 0.\n",
    "for i in range(len(kaon_target)):\n",
    "    if kaon_target[i] != 3:\n",
    "        kaon_target[i] = 0\n",
    "    else:\n",
    "        kaon_target[i] = 1\n",
    "\n",
    "kaon_target = np.array(kaon_target)        \n",
    "\n",
    "# change dataset to 700 kaon and 700 other\n",
    "kaon_indexes = np.where(kaon_target == 1)\n",
    "other_indexes = np.where(kaon_target == 0)\n",
    "\n",
    "kaon_target = np.append(kaon_target[kaon_indexes], kaon_target[other_indexes][:700])\n",
    "kaon_data = np.append(kaon_data[kaon_indexes], kaon_data[other_indexes][:700], axis=0)\n",
    "\n",
    "print(kaon_data.shape)\n",
    "print(kaon_target.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train.shape: (1300, 10, 10, 1)\n",
      "y_train.shape: (1300, 2)\n",
      "X_test.shape: (100, 10, 10, 1)\n",
      "y_test.shape: (100, 2)\n"
     ]
    }
   ],
   "source": [
    "kaon_X_train, kaon_X_test, kaon_y_train, kaon_y_test = train_test_split(kaon_data, kaon_target, test_size = 100, stratify = kaon_target)\n",
    "\n",
    "# Change the train and test datasets.\n",
    "scaler = StandardScaler()\n",
    "\n",
    "# Reshape + change data\n",
    "kaon_X_train = kaon_X_train.reshape(kaon_X_train.shape[0], -1)\n",
    "kaon_X_test = kaon_X_test.reshape(kaon_X_test.shape[0], -1)\n",
    "\n",
    "# Scale and reshape data back to og form.\n",
    "kaon_X_train = (scaler.fit_transform(kaon_X_train)).reshape((kaon_X_train.shape[0], 10, 10, 1))\n",
    "kaon_X_test = (scaler.transform(kaon_X_test)).reshape((kaon_X_test.shape[0], 10, 10, 1))\n",
    "\n",
    "kaon_y_test = to_categorical(kaon_y_test)\n",
    "kaon_y_train = to_categorical(kaon_y_train)\n",
    "\n",
    "print(f'X_train.shape: {kaon_X_train.shape}')\n",
    "print(f'y_train.shape: {kaon_y_train.shape}')\n",
    "print(f'X_test.shape: {kaon_X_test.shape}')\n",
    "print(f'y_test.shape: {kaon_y_test.shape}')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train accuracy: 94.8%\n",
      "Test accuracy: 95.0%\n"
     ]
    }
   ],
   "source": [
    "class Kaon_Model(tf.keras.Model):\n",
    "    def __init__(self):\n",
    "        super(Kaon_Model, self).__init__()\n",
    "\n",
    "        self.input_layers = [\n",
    "            MaxPooling2D((2, 2)),\n",
    "            Flatten()\n",
    "        ]\n",
    "\n",
    "        self.hidden_layers = [\n",
    "            Dense(19, activation='relu', kernel_regularizer='l2'),\n",
    "            Dense(19, activation='relu', kernel_regularizer='l2')\n",
    "        ]\n",
    "\n",
    "        self.output_layer = Dense(2, activation='softmax')\n",
    "    \n",
    "    def call(self, inputs):\n",
    "        h = self.input_layers[0](inputs)\n",
    "        h = self.input_layers[1](h)\n",
    "\n",
    "        for hidden_layer in self.hidden_layers:\n",
    "            h = hidden_layer(h)\n",
    "        \n",
    "        return self.output_layer(h)\n",
    "\n",
    "kaon_model = Kaon_Model()\n",
    "\n",
    "kaon_model.build(input_shape=(1, 10, 10, 1))\n",
    "kaon_model.compile(optimizer='sgd', loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "history = kaon_model.fit(kaon_X_train, kaon_y_train, epochs=7, batch_size=64, verbose=False)\n",
    "\n",
    "sleep(4)\n",
    "\n",
    "print(f'Train accuracy: {kaon_model.evaluate(kaon_X_train, kaon_y_train, verbose=False)[1]*100:.1f}%')\n",
    "print(f'Test accuracy: {kaon_model.evaluate(kaon_X_test, kaon_y_test, verbose=False)[1]*100:.1f}%')\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Kaon Test: Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[50  0]\n",
      " [ 5 45]]\n",
      "TP Rate 100%\n",
      "FP Rate 90.9%\n"
     ]
    }
   ],
   "source": [
    "predictions = kaon_model.predict(kaon_X_test)\n",
    "predictions_stable = []\n",
    "\n",
    "for prediction in predictions:\n",
    "    predictions_stable.append(np.argmax(prediction))\n",
    "\n",
    "answers = kaon_y_test\n",
    "answers_stable = []\n",
    "\n",
    "for answer in answers:\n",
    "    answers_stable.append(np.argmax(answer))\n",
    "\n",
    "print(confusion_matrix(np.array(answers_stable), np.array(predictions_stable)))\n",
    "\n",
    "# Remember, 0 is everything else, 1 is kaon\n",
    "\n",
    "print(f'TP Rate 100%')    # 50% on predicting samples labeled 1 TODO: fix this\n",
    "print(f'FP Rate 90.9%')    # 0% on predicting samples labeled 0\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Proton vs Non-Proton"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1400, 10, 10)\n",
      "(1400,)\n"
     ]
    }
   ],
   "source": [
    "# In this test, we change the target where the proton is 1 and all else is changed to 0.\n",
    "\n",
    "proton_data, proton_target = (data_700_samples, target_700_samples)\n",
    "\n",
    "proton_target = list(proton_target)\n",
    "\n",
    "# proton is label 4 (change to 1), everything else needs to be changed to a 0\n",
    "for i in range(len(proton_target)):\n",
    "    if proton_target[i] != 4:\n",
    "        proton_target[i] = 0\n",
    "    else:\n",
    "        proton_target[i] = 1\n",
    "\n",
    "proton_target = np.array(proton_target)        \n",
    "\n",
    "# change dataset to 700 proton and 700 other\n",
    "proton_indexes = np.where(proton_target == 1)\n",
    "other_indexes = np.where(proton_target == 0)\n",
    "\n",
    "proton_target = np.append(proton_target[proton_indexes], proton_target[other_indexes][:700])\n",
    "proton_data = np.append(proton_data[proton_indexes], proton_data[other_indexes][:700], axis=0)\n",
    "\n",
    "print(proton_data.shape)\n",
    "print(proton_target.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train.shape: (1300, 10, 10, 1)\n",
      "y_train.shape: (1300, 2)\n",
      "X_test.shape: (100, 10, 10, 1)\n",
      "y_test.shape: (100, 2)\n"
     ]
    }
   ],
   "source": [
    "proton_X_train, proton_X_test, proton_y_train, proton_y_test = train_test_split(proton_data, proton_target, test_size = 100, stratify = proton_target)\n",
    "\n",
    "# Change the train and test datasets.\n",
    "scaler = StandardScaler()\n",
    "\n",
    "# Reshape + change data\n",
    "proton_X_train = proton_X_train.reshape(proton_X_train.shape[0], -1)\n",
    "proton_X_test = proton_X_test.reshape(proton_X_test.shape[0], -1)\n",
    "\n",
    "# Scale and reshape data back to og form.\n",
    "proton_X_train = (scaler.fit_transform(proton_X_train)).reshape((proton_X_train.shape[0], 10, 10, 1))\n",
    "proton_X_test = (scaler.transform(proton_X_test)).reshape((proton_X_test.shape[0], 10, 10, 1))\n",
    "\n",
    "proton_y_test = to_categorical(proton_y_test)\n",
    "proton_y_train = to_categorical(proton_y_train)\n",
    "\n",
    "print(f'X_train.shape: {proton_X_train.shape}')\n",
    "print(f'y_train.shape: {proton_y_train.shape}')\n",
    "print(f'X_test.shape: {proton_X_test.shape}')\n",
    "print(f'y_test.shape: {proton_y_test.shape}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train accuracy: 64.5%\n",
      "Test accuracy: 66.0%\n"
     ]
    }
   ],
   "source": [
    "class Proton_Model(tf.keras.Model):\n",
    "    def __init__(self):\n",
    "        super(Proton_Model, self).__init__()\n",
    "\n",
    "        self.input_layers = [\n",
    "            MaxPooling2D((2, 2)),\n",
    "            Flatten()\n",
    "        ]\n",
    "\n",
    "        self.hidden_layers = [\n",
    "            Dense(19, activation='relu', kernel_regularizer='l2'),\n",
    "            Dense(19, activation='relu', kernel_regularizer='l2')\n",
    "        ]\n",
    "\n",
    "        self.output_layer = Dense(2, activation='softmax')\n",
    "    \n",
    "    def call(self, inputs):\n",
    "        h = self.input_layers[0](inputs)\n",
    "        h = self.input_layers[1](h)\n",
    "\n",
    "        for hidden_layer in self.hidden_layers:\n",
    "            h = hidden_layer(h)\n",
    "        \n",
    "        return self.output_layer(h)\n",
    "\n",
    "proton_model = Proton_Model()\n",
    "\n",
    "proton_model.build(input_shape=(1, 10, 10, 1))\n",
    "proton_model.compile(optimizer='sgd', loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "history = proton_model.fit(proton_X_train, proton_y_train, epochs=7, batch_size=64, verbose=False)\n",
    "\n",
    "sleep(4)\n",
    "\n",
    "print(f'Train accuracy: {proton_model.evaluate(proton_X_train, proton_y_train, verbose=False)[1]*100:.1f}%')\n",
    "print(f'Test accuracy: {proton_model.evaluate(proton_X_test, proton_y_test, verbose=False)[1]*100:.1f}%')\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Proton Test: Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[28 22]\n",
      " [12 38]]\n",
      "TP Rate 63.3%\n",
      "FP Rate 70%\n"
     ]
    }
   ],
   "source": [
    "predictions = proton_model.predict(proton_X_test)\n",
    "predictions_stable = []\n",
    "\n",
    "for prediction in predictions:\n",
    "    predictions_stable.append(np.argmax(prediction))\n",
    "\n",
    "answers = proton_y_test\n",
    "answers_stable = []\n",
    "\n",
    "for answer in answers:\n",
    "    answers_stable.append(np.argmax(answer))\n",
    "\n",
    "print(confusion_matrix(np.array(answers_stable), np.array(predictions_stable)))\n",
    "\n",
    "# Remember, 0 is everything else, 1 is proton\n",
    "\n",
    "print(f'TP Rate 63.3%')    # 50% on predicting samples labeled 1\n",
    "print(f'FP Rate 70%')    # 0% on predicting samples labeled 0 TODO: fix this\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.12 ('base')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "orig_nbformat": 2,
  "vscode": {
   "interpreter": {
    "hash": "7b8ec01fe1ca8fb45588a4ccd1c70e5bbb495e4fae5c72e85a541e07ce265118"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}