imbalance.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn import tree\n",
    "from sklearn.model_selection import cross_val_score, train_test_split\n",
    "from sklearn.metrics import accuracy_score as acc\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import accuracy_score\n",
    "import xgboost as xgb\n",
    "from imblearn.over_sampling import SMOTE\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import confusion_matrix, classification_report\n",
    "from sklearn import svm\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras.datasets import mnist\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense, Dropout\n",
    "from tensorflow.keras.optimizers import RMSprop\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(X_train, y_train, X_test, y_test):\n",
    "    y_train_pred = clf.predict(X_train)\n",
    "    print(\"Training accuracy on all features: {:.3f}\".format(acc(y_train, y_train_pred)))\n",
    "\n",
    "    y_test_pred = clf.predict(X_test)\n",
    "    print(\"Testing accuracy on all features: {:.3f}\".format(acc(y_test, y_test_pred)))\n",
    "\n",
    "def cross_val_res(clf, X, y, fold):\n",
    "    scores = cross_val_score(clf, X.values, y.values, cv=fold)\n",
    "    print(scores)\n",
    "    print(scores.mean())\n",
    "\n",
    "def xgboo(X_train, y_train, X_test, y_test):\n",
    "    train = xgb.DMatrix(X_train, label=y_train)\n",
    "    test = xgb.DMatrix(X_test, label=y_test)\n",
    "    prev = 0\n",
    "    for depth in range(8):\n",
    "        for eta in range(30):\n",
    "            for epoch in range(20):\n",
    "                param = {\n",
    "                    'max_depth': depth+1,\n",
    "                    'eta': (eta+1)/10,\n",
    "                    'objective': 'multi:softmax',\n",
    "                    'num_class': 3} \n",
    "                epochs = epoch \n",
    "                model = xgb.train(param, train, epochs)\n",
    "                predictions = model.predict(test)\n",
    "                acc = accuracy_score(y_test, predictions)*100\n",
    "                #if epoch%5 == 0: print('.',end='')\n",
    "                if prev<acc:\n",
    "                    prev = acc\n",
    "                    #print()\n",
    "                    print(\"depth={}, eta={}, epochs={}, acc={}\".format(depth+1,(eta+1)/100,epoch,acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training dataset shape: (3591, 6), (3591,)\n",
      "Testing dataset shape: (1197, 6), (1197,)\n"
     ]
    }
   ],
   "source": [
    "#input_file = \"data_3824.csv\"\n",
    "input_file = \"data.csv\"\n",
    "data = pd.read_csv(input_file, header = 0)\n",
    "\n",
    "df = data[data.columns[1:]]\n",
    "df = df[df['total_orders']<100]\n",
    "\n",
    "features = list(df.columns[:-1])\n",
    "\n",
    "y = df[\"label\"]\n",
    "X = df[features]\n",
    "\n",
    "cols = ['days_last_order', 'total_orders', 'total_value', 'unique_days_visited',\n",
    "       'total_sessions', 'avg_duration_btw_login', \n",
    "       #'avg_session_time',\n",
    "       #'avg_page_visits'\n",
    "       ]\n",
    "from sklearn.model_selection import cross_val_score, train_test_split\n",
    "from sklearn.metrics import accuracy_score as acc\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(df[cols].values, df['label'].values, test_size = 0.25, random_state = 123)\n",
    "\n",
    "y_train = y_train.ravel()\n",
    "y_test = y_test.ravel()\n",
    "\n",
    "print(\"Training dataset shape: {}, {}\".format(X_train.shape, y_train.shape))\n",
    "print(\"Testing dataset shape: {}, {}\".format(X_test.shape, y_test.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Before OverSampling, counts of label '1': 704\n",
      "Before OverSampling, counts of label '0': 2887 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"Before OverSampling, counts of label '1': {}\".format(sum(y_train == 1)))\n",
    "print(\"Before OverSampling, counts of label '0': {} \\n\".format(sum(y_train == 0)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols = ['days_last_order', \n",
    "        'total_orders', \n",
    "        'total_value', \n",
    "        'unique_days_visited',\n",
    "        'total_sessions', \n",
    "        'avg_duration_btw_login', \n",
    "        #'avg_session_time',\n",
    "        #'avg_page_visits'\n",
    "        ]\n",
    "\n",
    "def split(df, cols):\n",
    "    X_train, X_test, y_train, y_test = train_test_split(df[cols].values, df['label'].values, test_size = 0.25, random_state = 123)\n",
    "\n",
    "    y_train = y_train.ravel()\n",
    "    y_test = y_test.ravel()\n",
    "\n",
    "    #print(\"Training dataset shape: {}, {}\".format(X_train.shape, y_train.shape))\n",
    "    #print(\"Testing dataset shape: {}, {}\".format(X_test.shape, y_test.shape))\n",
    "\n",
    "    return X_train, X_test, y_train, y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = split(df, cols)\n",
    "scaler = StandardScaler()\n",
    "X_train = scaler.fit_transform(X_train)\n",
    "X_test = scaler.fit_transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = LogisticRegression()\n",
    "lr.fit(X_train, y_train.ravel())\n",
    "predictions = lr.predict(X_test)\n",
    "#print(classification_report(y_test, predictions))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Before OverSampling, counts of label '1': 704\n",
      "Before OverSampling, counts of label '0': 2887 \n",
      "\n",
      "After OverSampling, the shape of train_X: (5774, 6)\n",
      "After OverSampling, the shape of train_y: (5774,) \n",
      "\n",
      "After OverSampling, counts of label '1': 2887\n",
      "After OverSampling, counts of label '0': 2887\n"
     ]
    }
   ],
   "source": [
    "print(\"Before OverSampling, counts of label '1': {}\".format(sum(y_train == 1)))\n",
    "print(\"Before OverSampling, counts of label '0': {} \\n\".format(sum(y_train == 0)))\n",
    "\n",
    "# import SMOTE module from imblearn library\n",
    "# pip install imblearn (if you don't have imblearn in your system)\n",
    "sm = SMOTE(random_state = 2)\n",
    "X_train_res, y_train_res = sm.fit_resample(X_train, y_train.ravel())\n",
    "\n",
    "print('After OverSampling, the shape of train_X: {}'.format(X_train_res.shape))\n",
    "print('After OverSampling, the shape of train_y: {} \\n'.format(y_train_res.shape))\n",
    "\n",
    "print(\"After OverSampling, counts of label '1': {}\".format(sum(y_train_res == 1)))\n",
    "print(\"After OverSampling, counts of label '0': {}\".format(sum(y_train_res == 0)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "conf_mat = pd.DataFrame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TP: 16.5% FP: 37.7%, FN: 4.5% , TN: 41.4%\n"
     ]
    }
   ],
   "source": [
    "clf = LogisticRegression()\n",
    "clf = clf.fit(X_train_res, y_train_res.ravel())\n",
    "predictions = clf.predict(X_test)\n",
    "\n",
    "# print classification report\n",
    "#print(classification_report(y_test, predictions))\n",
    "\n",
    "tn, fp, fn, tp = (confusion_matrix(y_test, predictions)/len((y_test))*100).ravel()\n",
    "print(\"TP: {:.1f}% FP: {:.1f}%, FN: {:.1f}% , TN: {:.1f}%\".format(tp, fp, fn, tn))\n",
    "conf_mat = conf_mat.append([['Logistic Regression', tp, fp, fn, tn]])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TP: 18.3% FP: 53.2%, FN: 2.7% , TN: 25.8%\n"
     ]
    }
   ],
   "source": [
    "clf = tree.DecisionTreeClassifier(max_depth=8)\n",
    "clf = clf.fit(X_train_res,y_train_res)\n",
    "\n",
    "predictions = clf.predict(X_test)\n",
    "tn, fp, fn, tp = (confusion_matrix(y_test, predictions)/len((y_test))*100).ravel()\n",
    "print(\"TP: {:.1f}% FP: {:.1f}%, FN: {:.1f}% , TN: {:.1f}%\".format(tp, fp, fn, tn))\n",
    "conf_mat = conf_mat.append([['Desicion Tree', tp, fp, fn, tn]])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TP: 13.5% FP: 34.0%, FN: 7.4% , TN: 45.0%\n"
     ]
    }
   ],
   "source": [
    "clf = RandomForestClassifier(n_estimators=100, max_depth=8)\n",
    "clf = clf.fit(X_train_res, y_train_res)\n",
    "\n",
    "predictions = clf.predict(X_test)\n",
    "tn, fp, fn, tp = (confusion_matrix(y_test, predictions)/len((y_test))*100).ravel()\n",
    "print(\"TP: {:.1f}% FP: {:.1f}%, FN: {:.1f}% , TN: {:.1f}%\".format(tp, fp, fn, tn))\n",
    "conf_mat = conf_mat.append([['Random Forest', tp, fp, fn, tn]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TP: 18.0% FP: 45.8%, FN: 2.9% , TN: 33.2%\n"
     ]
    }
   ],
   "source": [
    "clf = svm.SVC(kernel='linear', C=1).fit(X_train_res, y_train_res)\n",
    "\n",
    "predictions = clf.predict(X_test)\n",
    "tn, fp, fn, tp = (confusion_matrix(y_test, predictions)/len((y_test))*100).ravel()\n",
    "print(\"TP: {:.1f}% FP: {:.1f}%, FN: {:.1f}% , TN: {:.1f}%\".format(tp, fp, fn, tn))\n",
    "conf_mat = conf_mat.append([['SVC', tp, fp, fn, tn]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TP: 18.0% FP: 48.9%, FN: 2.9% , TN: 30.2%\n"
     ]
    }
   ],
   "source": [
    "train = xgb.DMatrix(X_train_res, label=y_train_res)\n",
    "test = xgb.DMatrix(X_test, label=y_test)\n",
    "depth=4\n",
    "eta=0.03\n",
    "epochs=15\n",
    "\n",
    "param = {\n",
    "    'max_depth': depth+1,\n",
    "    'eta': (eta+1)/10,\n",
    "    'objective': 'multi:softmax',\n",
    "    'num_class': 3} \n",
    " \n",
    "model = xgb.train(param, train, epochs)\n",
    "predictions = model.predict(test)\n",
    "\n",
    "tn, fp, fn, tp = (confusion_matrix(y_test, predictions)/len((y_test))*100).ravel()\n",
    "print(\"TP: {:.1f}% FP: {:.1f}%, FN: {:.1f}% , TN: {:.1f}%\".format(tp, fp, fn, tn))\n",
    "conf_mat = conf_mat.append([['XGBoost', tp, fp, fn, tn]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_images = X_train_res.astype('float32')\n",
    "test_images = X_test.astype('float32')\n",
    "train_labels = keras.utils.to_categorical(y_train_res, 2)\n",
    "test_labels = keras.utils.to_categorical(y_test, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "WARNING:tensorflow:From D:\\Personal\\Online_learning\\Udemy\\MachineLearning\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\tracking\\tracking.py:111: Model.state_updates (from tensorflow.python.keras.engine.training) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This property should not be used in TensorFlow 2.0, as updates are applied automatically.\n",
      "WARNING:tensorflow:From D:\\Personal\\Online_learning\\Udemy\\MachineLearning\\Anaconda\\lib\\site-packages\\tensorflow\\python\\training\\tracking\\tracking.py:111: Layer.updates (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This property should not be used in TensorFlow 2.0, as updates are applied automatically.\n",
      "INFO:tensorflow:Assets written to: .\\checkpoints\\assets\n",
      "578/578 - 4s - loss: 0.6590 - accuracy: 0.6056 - val_loss: 0.5923 - val_accuracy: 0.6124\n",
      "Epoch 2/10\n",
      "578/578 - 1s - loss: 0.6542 - accuracy: 0.6200 - val_loss: 0.6681 - val_accuracy: 0.5213\n",
      "Epoch 3/10\n",
      "INFO:tensorflow:Assets written to: .\\checkpoints\\assets\n",
      "578/578 - 4s - loss: 0.6483 - accuracy: 0.6321 - val_loss: 0.6188 - val_accuracy: 0.6149\n",
      "Epoch 4/10\n",
      "578/578 - 1s - loss: 0.6454 - accuracy: 0.6315 - val_loss: 0.6697 - val_accuracy: 0.5865\n",
      "Epoch 5/10\n",
      "INFO:tensorflow:Assets written to: .\\checkpoints\\assets\n",
      "578/578 - 3s - loss: 0.6432 - accuracy: 0.6347 - val_loss: 0.6120 - val_accuracy: 0.6600\n",
      "Epoch 6/10\n",
      "578/578 - 1s - loss: 0.6422 - accuracy: 0.6363 - val_loss: 0.6137 - val_accuracy: 0.6516\n",
      "Epoch 7/10\n",
      "578/578 - 1s - loss: 0.6395 - accuracy: 0.6411 - val_loss: 0.6703 - val_accuracy: 0.5906\n",
      "Epoch 8/10\n",
      "578/578 - 1s - loss: 0.6435 - accuracy: 0.6411 - val_loss: 0.6449 - val_accuracy: 0.5856\n",
      "Epoch 9/10\n",
      "INFO:tensorflow:Assets written to: .\\checkpoints\\assets\n",
      "578/578 - 4s - loss: 0.6396 - accuracy: 0.6463 - val_loss: 0.5827 - val_accuracy: 0.6642\n",
      "Epoch 10/10\n",
      "578/578 - 1s - loss: 0.6403 - accuracy: 0.6420 - val_loss: 0.6509 - val_accuracy: 0.5606\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(Dense(128, activation='relu', input_shape=(len(cols),)))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(64, activation='relu'))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(32, activation='relu'))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(16, activation='relu'))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(2, activation='softmax'))\n",
    "\n",
    "callback = tf.keras.callbacks.ModelCheckpoint(\n",
    "    filepath='./checkpoints',\n",
    "    save_weights_only=False,\n",
    "    monitor='val_accuracy',\n",
    "    mode='max',\n",
    "    save_best_only=True)\n",
    "\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer=RMSprop(),\n",
    "              metrics=['accuracy']\n",
    "              )\n",
    "\n",
    "history = model.fit(train_images, train_labels,\n",
    "            batch_size=10,\n",
    "            epochs=10,\n",
    "            verbose=2,\n",
    "            validation_data=(test_images, test_labels),\n",
    "            callbacks = [callback]\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-17-bdbf5bddded6>:2: Sequential.predict_classes (from tensorflow.python.keras.engine.sequential) is deprecated and will be removed after 2021-01-01.\n",
      "Instructions for updating:\n",
      "Please use instead:* `np.argmax(model.predict(x), axis=-1)`,   if your model does multi-class classification   (e.g. if it uses a `softmax` last-layer activation).* `(model.predict(x) > 0.5).astype(\"int32\")`,   if your model does binary classification   (e.g. if it uses a `sigmoid` last-layer activation).\n",
      "TP: 11.9% FP: 24.5%, FN: 9.1% , TN: 54.6%\n"
     ]
    }
   ],
   "source": [
    "model = keras.models.load_model(\"checkpoints\")\n",
    "predictions = model.predict_classes(test_images)\n",
    "tn, fp, fn, tp = (confusion_matrix(y_test, predictions)/len((y_test))*100).ravel()\n",
    "print(\"TP: {:.1f}% FP: {:.1f}%, FN: {:.1f}% , TN: {:.1f}%\".format(tp, fp, fn, tn))\n",
    "conf_mat = conf_mat.append([['Neural Network', tp, fp, fn, tn]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: models\\assets\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0, 1, 0, ..., 0, 0, 0], dtype=int64)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.save('models')\n",
    "reconstructed_model = keras.models.load_model(\"models\")\n",
    "reconstructed_model.predict_classes(test_images)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th>TP</th>\n",
       "      <th>FP</th>\n",
       "      <th>FN</th>\n",
       "      <th>TN</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Desicion Tree</td>\n",
       "      <td>18.295739</td>\n",
       "      <td>53.216374</td>\n",
       "      <td>2.673350</td>\n",
       "      <td>25.814536</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>SVC</td>\n",
       "      <td>18.045113</td>\n",
       "      <td>45.781119</td>\n",
       "      <td>2.923977</td>\n",
       "      <td>33.249791</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>XGBoost</td>\n",
       "      <td>18.045113</td>\n",
       "      <td>48.872180</td>\n",
       "      <td>2.923977</td>\n",
       "      <td>30.158730</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Logistic Regression</td>\n",
       "      <td>16.457811</td>\n",
       "      <td>37.677527</td>\n",
       "      <td>4.511278</td>\n",
       "      <td>41.353383</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Random Forest</td>\n",
       "      <td>13.533835</td>\n",
       "      <td>34.001671</td>\n",
       "      <td>7.435255</td>\n",
       "      <td>45.029240</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Neural Network</td>\n",
       "      <td>11.862991</td>\n",
       "      <td>24.477861</td>\n",
       "      <td>9.106099</td>\n",
       "      <td>54.553049</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 Model         TP         FP        FN         TN\n",
       "0        Desicion Tree  18.295739  53.216374  2.673350  25.814536\n",
       "0                  SVC  18.045113  45.781119  2.923977  33.249791\n",
       "0              XGBoost  18.045113  48.872180  2.923977  30.158730\n",
       "0  Logistic Regression  16.457811  37.677527  4.511278  41.353383\n",
       "0        Random Forest  13.533835  34.001671  7.435255  45.029240\n",
       "0       Neural Network  11.862991  24.477861  9.106099  54.553049"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conf_mat.columns=['Model','TP','FP','FN','TN']\n",
    "conf_mat.sort_values(['FN','FP'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "conf_mat.to_csv('conf_mat.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(69.0, 0.5, 'truth')"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjQAAAGpCAYAAACam6wDAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAAAe9ElEQVR4nO3debiWddXo8e8SUJyVEETAqXDCt2w4Nlg2WA6VmpVHfHuLzKLUyjplYae3mbJsPg2+ZANpaqSZZKUZaYWVikMoIkFOEKOalVYq7HX+2I/0ROwt6jPd9+/78bqv537u8be9rg2LtX5DZCaSJElVtkm3GyBJkvR4GdBIkqTKM6CRJEmVZ0AjSZIqz4BGkiRV3tBuN2AgD911q8OvpC749NM/0O0mSMU69Y6zo5Pva+XftcNG7t7Rtq/PDI0kSaq8ns3QSJKkNutb2+0WtIwZGkmSVHlmaCRJKlX2dbsFLWNAI0lSqfrqE9BYcpIkSZVnhkaSpEKlJSdJklR5lpwkSZJ6hxkaSZJKZclJkiRVnhPrSZIk9Q4zNJIklcqSkyRJqjxHOUmSJPUOMzSSJBXKifUkSVL1WXKSJEnqHWZoJEkqlSUnSZJUeU6sJ0mS1DvM0EiSVCpLTpIkqfIc5SRJktQ7zNBIklQqS06SJKnyLDlJkiT1DjM0kiQVKrM+89AY0EiSVKoa9aGx5CRJkirPDI0kSaWqUadgAxpJkkpVo5KTAY0kSaVycUpJkqSNFxHbRcT5EXFLRCyIiGdHxIiIuCwiFjU+t2+6/tSIWBwRCyPikEd6vgGNJEmlyr7WbY/sC8AlmbkX8BRgATAVmJ2ZE4DZje9ExD7AJGAicCjwlYgYMtjDDWgkSSpVX1/rtkFExDbAgcDXATLzwcy8FzgSmNG4bAbwisb+kcB5mflAZt4GLAb2H+wdBjSSJOlxi4gpETG3aZvSdHp3YDXwzYi4PiLOjIgtgdGZuRyg8Tmqcf1YYEnT/UsbxwZkp2BJkkrVwlFOmTkdmD7A6aHA04C3ZeZVEfEFGuWlAcSGXjHY+83QSJJUqg6VnOjPsCzNzKsa38+nP8BZGRFjABqfq5quH990/zhg2WAvMKCRJEltlZkrgCURsWfj0EHAzcAsYHLj2GTgosb+LGBSRGwWEbsBE4CrB3uHJSdJkkrV2ZmC3wZ8JyI2BW4FjqM/sTIzIo4H7gSOBsjM+RExk/6gZw1wUj7CSpoGNJIkFaqTq21n5g3AMzZw6qABrp8GTNvY51tykiRJlWeGRpKkUrk4pSRJqrwaLU5pyUmSJFWeGRpJkkplyUmSJFWeJSdJkqTeYYZGkqRSWXKSJEmVZ8lJkiSpd5ihkSSpVJacJElS5dUooLHkJEmSKs8MjSRJpapRp2ADGkmSSmXJSZIkqXeYoZEkqVSWnCRJUuVZcpIkSeodZmgkSSqVJSdJklR5lpwkSZJ6hxkaSZJKVaMMjQGNJEmlyux2C1rGkpMkSao8MzSSJJXKkpMkSaq8GgU0lpwkSVLlmaGRJKlUTqwnSZIqz5KTJElS7zBDI0lSqWo0D40BjSRJpbLkJEmS1DvM0EiSVKoaZWgMaCRJKlWNhm1bcpIkSZVnhkaSpEJln6OcJElS1dWoD40lJ0mSVHlmaCRJKlWNOgUb0EiSVKoa9aGx5CRJkirPDI0kSaWqUadgAxpJkkplQCNJkiqvRqtt24dGkiRVnhkaSZJKZclJpfvLX+/jg6d9nsW33gERfPR97+TKq67lglmXsP122wJw8psnc+Bz9ufGmxfyoU9+EYAkOfENr+HFzz+gm82XKmnrMSM4/HNvYcsdtiX7khvOuZy537yUUXvvzKEfP45hWwznz0tXM+vkr/LgfX9nzFN257BPHN9/c8Ccz1/I7y+d290fQr2lRsO2DWj0mJz2+TM44JnP4HPT3s9DDz3E3//xAFdedS2vPeYVHPefr/6Xa5+0+y589+tfZOjQIay+6x5eNflEXnDAsxg6dEiXWi9VU9/aPmZ/7BxW3nQ7m245nOMu/ii3zbmRl37yjcyedg5LrrqFJ//vA3nWm1/GLz9zPqsXLuWbh/83ubaPLUdtx/E/mcain11Hrq3Pv8qlh9mHRo/affffz7W/u4lXHX4IAMOGDWObrbca8PrNhw9fF7w88OCDENGRdkp1c/+qe1l50+0APHj/P7hr8TK2Hj2CEbuPYclVtwBw269uYs/D/hcAa/7x4LrgZehmw6A+/xhXq2Rf67Yua1uGJiL2Ao4ExtL/a7QMmJWZC9r1TnXG0j+uYPvttuX90z7LwsW3ss+eE5j6jrcAcO4FP2TWJbOZuNcETnnrm9h2m60BmDf/Fv77459j2cpVfOK/3212Rnqcth03ktETd2HZDX9g9e+XMOElT2PRZdex18ueydZjRqy7bqf9nshLT38T244dyQ/feYbZGf2rGpWc2pKhiYj3AucBAVwNXNPYPzcipg5y35SImBsRc8/89rntaJpaYM3atSz4/WKOOeplnP+tL7P55sP5+lkzOeaol/GTmd/ggm99mR2eMILTv/S1dfc8eeJeXPSd/+G8M7/AmWfN5IEHHuziTyBV27AtNuOoM07mZx85mwfv+zs/OuVrPP11L+H1F3+UzbYcTt9Da9Zdu+yGP3DmS6byrSM+wLNPPJwhmw3rYsul9mlXhuZ4YGJmPtR8MCI+C8wHTtvQTZk5HZgO8NBdt9YnbKyZHUeNZPQOI3nyxL0AOPgFz+XMs2cycsT266559RGHcdIpH/y3e5+4685sPnw4i269nX333qNjbZbqYpOhQ3jlGScz/we/5veX9HfwvecPyznvtZ8EYMRuO/LEF+33b/fdvXgZD/39AXbYYxwrbrytk01WD8sajXJqVx+aPmCnDRwf0zinChv5hBHsOGoHbrtjKQC/vfYGnrjrzqy+655118z+xa950u67ALB02QrWrFkLwLIVK7n9zqWMHTO68w2XauCln3ojdy9exjVn/mTdsS2esE3/TgTPeduRXP+d2QBsO34HYkj/H/PbjH0CI3Yfw5+Xru54m9XD+rJ1W5e1K0PzDmB2RCwCljSO7Qw8CXhrm96pDnrfO0/gvR/+FA+teYjxO43ho+97J5/4/BksXHQrBIzdcTQffM/bAbhu3ny+ftZMhg4dyiabBO9/90nrhnZL2njjnrEH//Gq57FqwZ284cfTAPjF6TPZftcdefrrXgzAwkvmMm/mLwEY/4w9eNaJh9P30Foyk0vf/y3+/qf7utZ+qZ0i2zTtcURsAuxPf6fgAJYC12Tm2o2535KT1B2ffvoHut0EqVin3nF2R4eB3v+x/2rZ37Vbvr+zbV9f20Y5ZWYf8Nt2PV+SJD1OPVAqahXnoZEkSZXnTMGSJJWqRqOcDGgkSSqVJSdJkqSNFxG3R8SNEXFDRMxtHBsREZdFxKLG5/ZN158aEYsjYmFEHPJIzzegkSSpVJ1fy+mFmblfZj6j8X0qMDszJwCzG9+JiH2AScBE4FDgKxEx6Jo5BjSSJJWq+xPrHQnMaOzPAF7RdPy8zHwgM28DFtM/FcyADGgkSdLj1rweY2Obst4lCfw0Iq5tOjc6M5cDND5HNY6P5Z8T80L/XHZjB3u/nYIlSSpUK9dyal6PcQAHZOayiBgFXBYRtwxy7YYm6Rs0DWRAI0lSqTo4yikzlzU+V0XEhfSXkFZGxJjMXB4RY4BVjcuXAuObbh8HLBvs+ZacJElSW0XElhGx9cP7wMHATcAsYHLjssnARY39WcCkiNgsInYDJgBXD/YOMzSSJJWqcxma0cCFEQH9scc5mXlJRFwDzIyI44E7gaMBMnN+RMwEbgbWACc90lqQBjSSJJVq44dbP77XZN4KPGUDx+8GDhrgnmnAtI19hyUnSZJUeWZoJEkqVY2WPjCgkSSpUFmjgMaSkyRJqjwzNJIklapGGRoDGkmSStXCmYK7zZKTJEmqPDM0kiSVypKTJEmqvBoFNJacJElS5ZmhkSSpUJn1ydAY0EiSVCpLTpIkSb3DDI0kSaWqUYbGgEaSpEK5lpMkSVIPMUMjSVKpapShMaCRJKlU9VnKyZKTJEmqPjM0kiQVqk6dgg1oJEkqVY0CGktOkiSp8szQSJJUqhp1CjagkSSpUHXqQ2PJSZIkVZ4ZGkmSSmXJSZIkVZ0lJ0mSpB5ihkaSpFJZcpIkSVWXBjSSJKnyahTQ2IdGkiRVnhkaSZIKZclJkiRVX40CGktOkiSp8szQSJJUKEtOkiSp8uoU0FhykiRJlWeGRpKkQtUpQ2NAI0lSqTK63YKWseQkSZIqzwyNJEmFsuQkSZIqL/ssOUmSJPUMMzSSJBXKkpMkSaq8dJSTJElS7zBDI0lSoSw5SZKkynOUkyRJUg8xQyNJUqEyu92C1jGgkSSpUJacJEmSeogZGkmSClWnDI0BjSRJhapTHxpLTpIkqfLM0EiSVChLTpIkqfJcy0mSJKmHmKGRJKlQruUkSZIqr8+SkyRJ0qMTEUMi4vqIuLjxfUREXBYRixqf2zdde2pELI6IhRFxyCM924BGkqRCZUbLto10MrCg6ftUYHZmTgBmN74TEfsAk4CJwKHAVyJiyGAPNqCRJKlQ2Rct2x5JRIwDXgac2XT4SGBGY38G8Iqm4+dl5gOZeRuwGNh/sOcb0EiSpMctIqZExNymbcp6l3weeA/Q3BV5dGYuB2h8jmocHwssabpuaePYgOwULElSoVq59EFmTgemb+hcRLwcWJWZ10bECzbicRtK+QzaWgMaSZIK1cGZgg8AjoiIlwLDgW0i4mxgZUSMyczlETEGWNW4fikwvun+ccCywV5gyUmSJLVVZp6ameMyc1f6O/v+PDP/C5gFTG5cNhm4qLE/C5gUEZtFxG7ABODqwd5hhkaSpEL1wDw0pwEzI+J44E7gaIDMnB8RM4GbgTXASZm5drAHPWJAExF7AKcAuzRfn5kveszNlyRJXdeNtZwy8wrgisb+3cBBA1w3DZi2sc/dmAzN94AzgK8Bg0ZHkiRJ3bAxAc2azPxq21siSZI6qpWjnLptwIAmIkY0dn8YEScCFwIPPHw+M+9pc9skSVIb9UAfmpYZLENzLf1jvh/+aU9pOpfA7u1qlCRJ0qMxYECTmbsBRMTwzPxH87mIGN7uhkmSpPbqRqfgdtmYeWh+vZHHJElShWS2buu2wfrQ7Ej/ugmbR8RT+WfpaRtgiw60TZIkaaMM1ofmEOD19E83/Nmm438F3tfGNgGw+U7Pa/crJG3AyC226XYTpGKd2uH3FdEpODNnADMi4lWZeUEH2yRJkjqgTn1oNmYemn0jYuL6BzPzI21ojyRJ0qO2MQHNfU37w4GXAwva0xxJktQpRZScHpaZn2n+HhGfpn8VTEmSVGE9MDipZR7Lattb4KR6kiRVXlEZmoi4kX8GcUOAHQD7z0iSpJ6xMRmalzftrwFWZuaaNrVHkiR1SDGjnCJiE+BHmblvh9ojSZI6pK/bDWihQZc+yMw+4HcRsXOH2iNJkvSobUzJaQwwPyKuBu5/+GBmHtG2VkmSpLZLCik5NWzFv/ajCeCT7WmOJEnqlL4ajdvemIBmaGb+ovlARGzepvZIkiQ9aoOttn0CcCKwe0TMazq1NXBluxsmSZLaq6+QktM5wE+ATwBTm47/NTPvaWurJElS2xXRhyYz/wz8GTi2c82RJEl69B7L0geSJKkG6jQPjQGNJEmFqlPJadCJ9SRJkqrADI0kSYWy5CRJkiqvTgGNJSdJklR5ZmgkSSpUnToFG9BIklSovvrEM5acJElS9ZmhkSSpUKWs5SRJkmosu92AFrLkJEmSKs8MjSRJharTPDQGNJIkFaov6tOHxpKTJEmqPDM0kiQVqk6dgg1oJEkqVJ360FhykiRJlWeGRpKkQtVp6QMDGkmSClWnmYItOUmSpMozQyNJUqEc5SRJkiqvTn1oLDlJkqTKM0MjSVKh6jQPjQGNJEmFqlMfGktOkiSp8szQSJJUqDp1CjagkSSpUHXqQ2PJSZIkVZ4ZGkmSClWnDI0BjSRJhcoa9aGx5CRJkirPDI0kSYWy5CRJkiqvTgGNJSdJklR5ZmgkSSpUnZY+MKCRJKlQdZop2JKTJElqq4gYHhFXR8TvImJ+RHy4cXxERFwWEYsan9s33XNqRCyOiIURccgjvcOARpKkQvW1cHsEDwAvysynAPsBh0bEs4CpwOzMnADMbnwnIvYBJgETgUOBr0TEkMFeYEAjSVKhOhXQZL/7Gl+HNbYEjgRmNI7PAF7R2D8SOC8zH8jM24DFwP6DvcOARpIkPW4RMSUi5jZtU9Y7PyQibgBWAZdl5lXA6MxcDtD4HNW4fCywpOn2pY1jA7JTsCRJhWrlKKfMnA5MH+T8WmC/iNgOuDAi9h3kcRvqrjxocw1oJEkqVDdGOWXmvRFxBf19Y1ZGxJjMXB4RY+jP3kB/RmZ8023jgGWDPdeSkyRJhepUH5qI2KGRmSEiNgdeDNwCzAImNy6bDFzU2J8FTIqIzSJiN2ACcPVg7zBDI0mS2m0MMKMxUmkTYGZmXhwRvwFmRsTxwJ3A0QCZOT8iZgI3A2uAkxolqwEZ0EiSVKhOzRScmfOAp27g+N3AQQPcMw2YtrHvMKCRJKlQfTVa/MA+NJIkqfLM0EiSVKiNmOG3MgxoJEkqVH0KTpacJElSDZihkSSpUJacJElS5XVjpuB2seQkSZIqzwyNJEmFqtM8NAY0kiQVqj7hjCUnSZJUA2ZoJEkqlKOcJElS5dWpD40lJ0mSVHlmaCRJKlR98jMGNJIkFatOfWgsOUmSpMozQyNJUqHq1CnYgEaSpELVJ5yx5CRJkmrADI0kSYWqU6dgAxpJkgqVNSo6WXKSJEmVZ4ZGkqRCWXKSJEmVV6dh25acJElS5ZmhkSSpUPXJzxjQSJJULEtOkiRJPcSARo/a16Z/hmVLf8cN189ed2z77bfjkh+fy4L5c7jkx+ey3XbbAjBs2DDO/Npnuf66n3Ht3Mt4/oHP7lazpVr43Jc+xk2L5nDFr2f927kT3nocK+5dwIgR2wFw4Auew6VXnM/lV17EpVeczwEHPrPDrVWv62vh1m0GNHrUvv3tmbzs5a/5l2Pvfc9J/PzyOew98bn8/PI5vPc9JwHwxuP/E4CnPu3FHHrYJD71qQ8QER1vs1QX3z3nBxz76in/dnynsTty4Aufw9Ily9Ydu+eeP/G6SSfwwgOO5OQTTuVLZ3yyk01VBWQL/+s2Axo9ar+acxX3/Onefzl2+OGH8O2zvgfAt8/6HkcccSgAe++9Bz+/fA4Aq1ffzZ/v/QvPePpTOtpeqU5+++u53Lve7x/ARz4+lY9+8NNk/vMvlpvmLWDlitUA3LJgEZsN34xNNx3WqaZKHdXxgCYijuv0O9V+o0eNZMWKVQCsWLGKUTs8AYB5827miMMPYciQIey663ie9rT/YNz4nbrZVKl2Dj7shSxfvpKbb1o44DUvP+Jgbpq3gAcffKiDLVOvq1PJqRujnD4MfHNDJyJiCjAFIIZsyyabbNnJdqkNvvmt89h7rwlc9dufcOedS/nNb+ayZs2abjdLqo3NNx/OO971Zo555RsHvGbPvZ7E+z/8Lo45auBrVKZeKBW1SlsCmoiYN9ApYPRA92XmdGA6wNBNx9bn/3IBVq66ix13HMWKFavYccdRrFp9NwBr167lXad8aN11v/rFRSxefFuXWinVzy67jWfnXcbx8zk/AGDMTqP56S8u4LCDjmH1qrsYs9NovnH2/+Ntb5nKHbcv6W5jpTZqV4ZmNHAI8Kf1jgfw6za9U1108Q9/yuteezSfOv3LvO61R/PDH14K9P/rMSL429/+zosPeh5r1qxhwYJFXW6tVB+33LyIfSc8d933a+b9jENe8Gruuedettl2a86eeQYf/8hnueaq67vYSvWqXigVtUq7ApqLga0y84b1T0TEFW16pzrk7LO+zPMPfDYjR47g9lvn8uGPfJpPnv5lzjvnDI57/bEsWfJHjjn2zQCMGjWSH//oHPr6+lj2xxVMPu7tXW69VG1fPfPTPOe5+zPiCdtx3fzLOf20L3HuWRds8No3vOk17LbbzrzzlBN45yknADDpqDdy1133dLLJ6mF9WZ9iSGSP/jCWnKTuGLnFNt1uglSsFfcu6Oi8Fq/d5ZUt+7v2rDu+39U5OVz6QJKkQtUpc2BAI0lSoVzLSZIkqYeYoZEkqVDOQyNJkiqvTsO2LTlJkqTKM0MjSVKh6tQp2IBGkqRC1akPjSUnSZJUeWZoJEkqVJ06BRvQSJJUqF5d/uixsOQkSZIqzwyNJEmFcpSTJEmqPPvQSJKkynPYtiRJUg8xQyNJUqHsQyNJkirPYduSJEk9xAyNJEmFcpSTJEmqPEc5SZIk9RAzNJIkFapOo5zM0EiSVKjMbNk2mIgYHxGXR8SCiJgfESc3jo+IiMsiYlHjc/ume06NiMURsTAiDnmkn8WARpIktdsa4F2ZuTfwLOCkiNgHmArMzswJwOzGdxrnJgETgUOBr0TEkMFeYEAjSVKh+siWbYPJzOWZeV1j/6/AAmAscCQwo3HZDOAVjf0jgfMy84HMvA1YDOw/2DsMaCRJKlS28L+ImBIRc5u2KRt6Z0TsCjwVuAoYnZnLoT/oAUY1LhsLLGm6bWnj2IDsFCxJkh63zJwOTB/smojYCrgAeEdm/iUiBrx0Q68Y7NkGNJIkFaqvg0sfRMQw+oOZ72Tm9xuHV0bEmMxcHhFjgFWN40uB8U23jwOWDfZ8S06SJBUqW7gNJvpTMV8HFmTmZ5tOzQImN/YnAxc1HZ8UEZtFxG7ABODqwd5hhkaSJLXbAcBrgRsj4obGsfcBpwEzI+J44E7gaIDMnB8RM4Gb6R8hdVJmrh3sBQY0kiQVqlMT62XmHDbcLwbgoAHumQZM29h3GNBIklQoZwqWJEnqIWZoJEkq1CMtWVAlBjSSJBXKkpMkSVIPMUMjSVKhskYZGgMaSZIKVac+NJacJElS5ZmhkSSpUHXqFGxAI0lSoSw5SZIk9RAzNJIkFcqSkyRJqrw6Ddu25CRJkirPDI0kSYXqq1GnYAMaSZIKZclJkiSph5ihkSSpUJacJElS5VlykiRJ6iFmaCRJKpQlJ0mSVHmWnCRJknqIGRpJkgplyUmSJFWeJSdJkqQeYoZGkqRCZfZ1uwktY0AjSVKh+iw5SZIk9Q4zNJIkFSod5SRJkqrOkpMkSVIPMUMjSVKhLDlJkqTKq9NMwZacJElS5ZmhkSSpUHVa+sCARpKkQtmHRpIkVZ7DtiVJknqIGRpJkgplyUmSJFWew7YlSZJ6iBkaSZIKZclJkiRVnqOcJEmSeogZGkmSCmXJSZIkVZ6jnCRJknqIGRpJkgrl4pSSJKnyLDlJkiT1EDM0kiQVylFOkiSp8urUh8aSkyRJqjwzNJIkFcqSkyRJqrw6BTSWnCRJUuWZoZEkqVD1yc9A1CndpN4REVMyc3q32yGVxt89lcqSk9plSrcbIBXK3z0VyYBGkiRVngGNJEmqPAMatYs1fKk7/N1TkewULEmSKs8MjSRJqjwDGkmSVHkGNGqpiDg0IhZGxOKImNrt9kiliIhvRMSqiLip222RusGARi0TEUOALwOHAfsAx0bEPt1tlVSMbwGHdrsRUrcY0KiV9gcWZ+atmfkgcB5wZJfbJBUhM38J3NPtdkjdYkCjVhoLLGn6vrRxTJKktjKgUSvFBo45L4Akqe0MaNRKS4HxTd/HAcu61BZJUkEMaNRK1wATImK3iNgUmATM6nKbJEkFMKBRy2TmGuCtwKXAAmBmZs7vbqukMkTEucBvgD0jYmlEHN/tNkmd5NIHkiSp8szQSJKkyjOgkSRJlWdAI0mSKs+ARpIkVZ4BjSRJqjwDGklExAsi4uLG/hGDrZQeEdtFxImP4R0fioh3P552StJADGikGmusgP6oZOaszDxtkEu2Ax51QCNJ7WRAI1VUROwaEbdExIyImBcR50fEFhFxe0R8ICLmAEdHxMER8ZuIuC4ivhcRWzXuP7Rx/xzglU3PfX1EfKmxPzoiLoyI3zW25wCnAU+MiBsi4vTGdadExDWNdny46Vn/NyIWRsTPgD07+L9HUmGGdrsBkh6XPYHjM/PKiPgG/8yc/CMznxsRI4HvAy/OzPsj4r3A/4mITwFfA14ELAa+O8Dzvwj8IjOPamR7tgKmAvtm5n4AEXEwMAHYn/4FSmdFxIHA/fQvf/FU+v+suQ64trU/viT1M6CRqm1JZl7Z2D8beHtj/+EA5VnAPsCVEQGwKf3T4+8F3JaZiwAi4mxgygae/yLgdQCZuRb4c0Rsv941Bze26xvft6I/wNkauDAz/9Z4h+t6SWobAxqp2tZfu+Th7/c3PgO4LDOPbb4oIvbbwL2PVQCfyMz/We8d72jhOyRpUPahkapt54h4dmP/WGDOeud/CxwQEU8CaPSx2QO4BdgtIp7YdO+GzAZOaNw7JCK2Af5Kf/blYZcCb2jqmzM2IkYBvwSOiojNI2Jr4PDH84NK0mAMaKRqWwBMjoh5wAjgq80nM3M18Hrg3MY1vwX2ysx/0F9i+lGjU/AdAzz/ZOCFEXEj/f1fJmbm3fSXsG6KiNMz86fAOcBvGtedD2ydmdfRX/q6AbgA+FULf25J+heuti1VVETsClycmft2uy2S1G1maCRJUuWZoZEkSZVnhkaSJFWeAY0kSao8AxpJklR5BjSSJKnyDGgkSVLl/X+00ID+RrmTwwAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 720x504 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "predictions = model.predict_classes(test_images)\n",
    "\n",
    "import seaborn as sn\n",
    "import matplotlib.pyplot as plt\n",
    "cm = tf.math.confusion_matrix(labels=y_test, predictions=predictions)\n",
    "plt.figure(figsize=(10, 7))\n",
    "sn.heatmap(cm, annot=True, fmt='d')\n",
    "plt.xlabel('predicted')\n",
    "plt.ylabel('truth')"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "63fd5069d213b44bf678585dea6b12cceca9941eaf7f819626cde1f2670de90d"
  },
  "kernelspec": {
   "display_name": "Python 3.9.0 64-bit",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}