OAMR.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rAvYMYCb2NkS",
        "outputId": "c2463211-e7b5-41ed-f028-60051d65347c"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive/\n",
            "/content/drive/My Drive/modrec\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive/')\n",
        "%cd /content/drive/My Drive/modrec"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Import all the things we need ---\n",
        "#   by setting env variables before Keras import you can set up which backend and which GPU it uses\n",
        "\n",
        "import os,random\n",
        "os.environ[\"KERAS_BACKEND\"] = \"tensorflow\"\n",
        "# os.environ[\"THEANO_FLAGS\"]  = \"device=gpu%d\"%(0)\n",
        "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n",
        "import numpy as np\n",
        "import matplotlib\n",
        "import matplotlib.pyplot as plt\n",
        "from matplotlib.collections import LineCollection\n",
        "from matplotlib.colors import ListedColormap, BoundaryNorm\n",
        "#from matplotlib import pyplot as plt\n",
        "import pickle, random, sys\n",
        "import keras\n",
        "import keras.backend as K\n",
        "from keras.callbacks import LearningRateScheduler,TensorBoard\n",
        "from tensorflow.keras.optimizers import Adam\n",
        "import csv\n",
        "import mltools,rmldataset2016\n",
        "#import rmlmodels.CNN2Model as cnn2"
      ],
      "metadata": {
        "id": "FZho1h462RTX"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import numpy as np\n",
        "import keras.models as models\n",
        "from keras.layers import Reshape,Dense,Dropout,Activation,Flatten, Input\n",
        "from keras.layers import Convolution2D,Conv2D, MaxPooling2D, ZeroPadding2D\n",
        "from keras.layers import Activation, BatchNormalization\n",
        "from keras.models import Model\n",
        "from keras import regularizers\n",
        "\n",
        "def CNN2Model(weights=None,\n",
        "             input_shape=[2,128],\n",
        "             classes=11,\n",
        "             **kwargs):\n",
        "    if weights is not None and not (os.path.exists(weights)):\n",
        "        raise ValueError('The `weights` argument should be either '\n",
        "                         '`None` (random initialization), '\n",
        "                         'or the path to the weights file to be loaded.')\n",
        "\n",
        "    dr = 0.5 # dropout rate (%)\n",
        "\n",
        "    input = Input(shape=input_shape + [1], name='input')\n",
        "\n",
        "\n",
        "    x = Reshape(target_shape=input_shape + [1])(input)\n",
        "\n",
        "\n",
        "    x = Conv2D(50, (1, 8), padding='same', activation=\"leaky_relu\", name=\"conv1\", kernel_initializer='glorot_uniform', kernel_regularizer=\"l2\")(x)\n",
        "    x = BatchNormalization()(x) # Add batch normalization\n",
        "    x = Dropout(dr)(x)\n",
        "\n",
        "    x = Conv2D(50, (2, 8), padding=\"valid\", activation=\"leaky_relu\", name=\"conv2\", kernel_initializer='glorot_uniform')(x)\n",
        "    x = BatchNormalization()(x) # Add batch normalization\n",
        "    x = Dropout(dr)(x)\n",
        "\n",
        "    x = Flatten()(x)\n",
        "\n",
        "    x = Dense(256, activation='relu', kernel_initializer='he_normal', name=\"dense1\")(x)\n",
        "   # x = BatchNormalization()(x) # Add batch normalization\n",
        "    x = Dropout(dr)(x)\n",
        "\n",
        "    x = Dense(classes, kernel_initializer='he_normal', name=\"dense2\")(x)\n",
        "    output = Activation('softmax')(x)\n",
        "\n",
        "    model = Model(inputs=input, outputs=output)\n",
        "\n",
        "    # Load weights.\n",
        "    if weights is not None:\n",
        "        model.load_weights(weights)\n",
        "\n",
        "    return model\n",
        "\n",
        "if __name__ == '__main__':\n",
        "    print(CNN2Model().summary())"
      ],
      "metadata": {
        "id": "X7lRaGQSAIO7",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ea2df360-560a-45cb-9bb0-108157bab0df"
      },
      "execution_count": 66,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model_23\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input (InputLayer)          [(None, 2, 128, 1)]       0         \n",
            "                                                                 \n",
            " reshape_29 (Reshape)        (None, 2, 128, 1)         0         \n",
            "                                                                 \n",
            " conv1 (Conv2D)              (None, 2, 128, 50)        450       \n",
            "                                                                 \n",
            " batch_normalization_33 (Ba  (None, 2, 128, 50)        200       \n",
            " tchNormalization)                                               \n",
            "                                                                 \n",
            " dropout_69 (Dropout)        (None, 2, 128, 50)        0         \n",
            "                                                                 \n",
            " conv2 (Conv2D)              (None, 1, 121, 50)        40050     \n",
            "                                                                 \n",
            " batch_normalization_34 (Ba  (None, 1, 121, 50)        200       \n",
            " tchNormalization)                                               \n",
            "                                                                 \n",
            " dropout_70 (Dropout)        (None, 1, 121, 50)        0         \n",
            "                                                                 \n",
            " flatten_23 (Flatten)        (None, 6050)              0         \n",
            "                                                                 \n",
            " dense1 (Dense)              (None, 256)               1549056   \n",
            "                                                                 \n",
            " dropout_71 (Dropout)        (None, 256)               0         \n",
            "                                                                 \n",
            " dense2 (Dense)              (None, 11)                2827      \n",
            "                                                                 \n",
            " activation_23 (Activation)  (None, 11)                0         \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 1592783 (6.08 MB)\n",
            "Trainable params: 1592583 (6.08 MB)\n",
            "Non-trainable params: 200 (800.00 Byte)\n",
            "_________________________________________________________________\n",
            "None\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "#set Keras data format as channels_first\n",
        "K.set_image_data_format('channels_last')\n",
        "print(K.image_data_format())\n",
        "\n",
        "(mods,snrs,lbl),(X_train,Y_train),(X_val,Y_val),(X_test,Y_test),(train_idx,val_idx,test_idx) = \\\n",
        "    rmldataset2016.load_data()\n",
        "\n",
        "in_shp = list(X_train.shape[1:])\n",
        "print(X_train.shape, in_shp)\n",
        "classes = mods\n",
        "print(classes)\n",
        "\n",
        "\n",
        "# Set up some params\n",
        "nb_epoch = 100    # number of epochs to train on\n",
        "batch_size = 500  # training batch size\n",
        "\n",
        "model = CNN2Model(None, input_shape=in_shp,classes=len(classes))\n",
        "\n",
        "model.compile(loss='categorical_crossentropy',metrics=['accuracy'],optimizer='adam')\n",
        "model.summary()\n",
        "\n",
        "filepath = 'weights/weights2.h5'\n",
        "history = model.fit(X_train,\n",
        "    Y_train,\n",
        "    batch_size=batch_size,\n",
        "    epochs=nb_epoch,\n",
        "    verbose=2,\n",
        "    validation_data=[X_val,Y_val],\n",
        "    callbacks = [\n",
        "                keras.callbacks.ModelCheckpoint(filepath, monitor='val_loss', verbose=1, save_best_only=True, mode='auto'),\n",
        "                keras.callbacks.ReduceLROnPlateau(monitor='val_loss',factor=0.5,verbose=1,patince=5,min_lr=0.000001),\n",
        "                keras.callbacks.EarlyStopping(monitor='val_loss', patience=50, verbose=1, mode='auto')\n",
        "                #keras.callbacks.TensorBoard(log_dir='./logs/',histogram_freq=1,write_graph=False,write_grads=1,write_images=False,update_freq='epoch')\n",
        "                ]\n",
        "                    )\n",
        "\n",
        "mltools.show_history(history)\n",
        "\n",
        "#Show simple version of performance\n",
        "score = model.evaluate(X_test, Y_test, verbose=1, batch_size=batch_size)\n",
        "print(score)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "7nPEmhmE3Iiw",
        "outputId": "8e783e38-839e-480c-f027-c737e7d0e96e"
      },
      "execution_count": 67,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "channels_last\n",
            "220000\n",
            "132000\n",
            "44000\n",
            "44000\n",
            "(132000, 2, 128)\n",
            "(44000, 2, 128)\n",
            "(44000, 2, 128)\n",
            "(132000, 11)\n",
            "(44000, 11)\n",
            "(44000, 11)\n",
            "(132000, 2, 128) [2, 128]\n",
            "['8PSK', 'AM-DSB', 'AM-SSB', 'BPSK', 'CPFSK', 'GFSK', 'PAM4', 'QAM16', 'QAM64', 'QPSK', 'WBFM']\n",
            "Model: \"model_24\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input (InputLayer)          [(None, 2, 128, 1)]       0         \n",
            "                                                                 \n",
            " reshape_30 (Reshape)        (None, 2, 128, 1)         0         \n",
            "                                                                 \n",
            " conv1 (Conv2D)              (None, 2, 128, 50)        450       \n",
            "                                                                 \n",
            " batch_normalization_35 (Ba  (None, 2, 128, 50)        200       \n",
            " tchNormalization)                                               \n",
            "                                                                 \n",
            " dropout_72 (Dropout)        (None, 2, 128, 50)        0         \n",
            "                                                                 \n",
            " conv2 (Conv2D)              (None, 1, 121, 50)        40050     \n",
            "                                                                 \n",
            " batch_normalization_36 (Ba  (None, 1, 121, 50)        200       \n",
            " tchNormalization)                                               \n",
            "                                                                 \n",
            " dropout_73 (Dropout)        (None, 1, 121, 50)        0         \n",
            "                                                                 \n",
            " flatten_24 (Flatten)        (None, 6050)              0         \n",
            "                                                                 \n",
            " dense1 (Dense)              (None, 256)               1549056   \n",
            "                                                                 \n",
            " dropout_74 (Dropout)        (None, 256)               0         \n",
            "                                                                 \n",
            " dense2 (Dense)              (None, 11)                2827      \n",
            "                                                                 \n",
            " activation_24 (Activation)  (None, 11)                0         \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 1592783 (6.08 MB)\n",
            "Trainable params: 1592583 (6.08 MB)\n",
            "Non-trainable params: 200 (800.00 Byte)\n",
            "_________________________________________________________________\n",
            "Epoch 1/100\n",
            "\n",
            "Epoch 1: val_loss improved from inf to 2.47153, saving model to weights/weights2.h5\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/keras/src/engine/training.py:3103: UserWarning: You are saving your model as an HDF5 file via `model.save()`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')`.\n",
            "  saving_api.save_model(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "264/264 - 8s - loss: 1.9572 - accuracy: 0.2793 - val_loss: 2.4715 - val_accuracy: 0.0909 - lr: 0.0010 - 8s/epoch - 31ms/step\n",
            "Epoch 2/100\n",
            "\n",
            "Epoch 2: val_loss did not improve from 2.47153\n",
            "264/264 - 6s - loss: 1.6723 - accuracy: 0.3712 - val_loss: 2.8429 - val_accuracy: 0.1151 - lr: 0.0010 - 6s/epoch - 21ms/step\n",
            "Epoch 3/100\n",
            "\n",
            "Epoch 3: val_loss did not improve from 2.47153\n",
            "264/264 - 6s - loss: 1.5870 - accuracy: 0.4013 - val_loss: 3.1887 - val_accuracy: 0.2314 - lr: 0.0010 - 6s/epoch - 23ms/step\n",
            "Epoch 4/100\n",
            "\n",
            "Epoch 4: val_loss improved from 2.47153 to 1.70219, saving model to weights/weights2.h5\n",
            "264/264 - 7s - loss: 1.5464 - accuracy: 0.4127 - val_loss: 1.7022 - val_accuracy: 0.3593 - lr: 0.0010 - 7s/epoch - 27ms/step\n",
            "Epoch 5/100\n",
            "\n",
            "Epoch 5: val_loss improved from 1.70219 to 1.46967, saving model to weights/weights2.h5\n",
            "264/264 - 6s - loss: 1.5062 - accuracy: 0.4264 - val_loss: 1.4697 - val_accuracy: 0.4270 - lr: 0.0010 - 6s/epoch - 23ms/step\n",
            "Epoch 6/100\n",
            "\n",
            "Epoch 6: val_loss improved from 1.46967 to 1.39320, saving model to weights/weights2.h5\n",
            "264/264 - 6s - loss: 1.4800 - accuracy: 0.4342 - val_loss: 1.3932 - val_accuracy: 0.4580 - lr: 0.0010 - 6s/epoch - 22ms/step\n",
            "Epoch 7/100\n",
            "\n",
            "Epoch 7: val_loss did not improve from 1.39320\n",
            "264/264 - 6s - loss: 1.4652 - accuracy: 0.4374 - val_loss: 1.4044 - val_accuracy: 0.4633 - lr: 0.0010 - 6s/epoch - 22ms/step\n",
            "Epoch 8/100\n",
            "\n",
            "Epoch 8: val_loss improved from 1.39320 to 1.37936, saving model to weights/weights2.h5\n",
            "264/264 - 6s - loss: 1.4532 - accuracy: 0.4414 - val_loss: 1.3794 - val_accuracy: 0.4700 - lr: 0.0010 - 6s/epoch - 22ms/step\n",
            "Epoch 9/100\n",
            "\n",
            "Epoch 9: val_loss improved from 1.37936 to 1.36896, saving model to weights/weights2.h5\n",
            "264/264 - 6s - loss: 1.4366 - accuracy: 0.4494 - val_loss: 1.3690 - val_accuracy: 0.4768 - lr: 0.0010 - 6s/epoch - 22ms/step\n",
            "Epoch 10/100\n",
            "\n",
            "Epoch 10: val_loss did not improve from 1.36896\n",
            "264/264 - 6s - loss: 1.4338 - accuracy: 0.4482 - val_loss: 1.3881 - val_accuracy: 0.4598 - lr: 0.0010 - 6s/epoch - 21ms/step\n",
            "Epoch 11/100\n",
            "\n",
            "Epoch 11: val_loss did not improve from 1.36896\n",
            "264/264 - 6s - loss: 1.4288 - accuracy: 0.4505 - val_loss: 1.3941 - val_accuracy: 0.4618 - lr: 0.0010 - 6s/epoch - 22ms/step\n",
            "Epoch 12/100\n",
            "\n",
            "Epoch 12: val_loss did not improve from 1.36896\n",
            "264/264 - 6s - loss: 1.4215 - accuracy: 0.4531 - val_loss: 1.4703 - val_accuracy: 0.4320 - lr: 0.0010 - 6s/epoch - 22ms/step\n",
            "Epoch 13/100\n",
            "\n",
            "Epoch 13: val_loss improved from 1.36896 to 1.34323, saving model to weights/weights2.h5\n",
            "264/264 - 6s - loss: 1.4124 - accuracy: 0.4565 - val_loss: 1.3432 - val_accuracy: 0.4825 - lr: 0.0010 - 6s/epoch - 22ms/step\n",
            "Epoch 14/100\n",
            "\n",
            "Epoch 14: val_loss did not improve from 1.34323\n",
            "264/264 - 6s - loss: 1.4096 - accuracy: 0.4574 - val_loss: 1.3948 - val_accuracy: 0.4720 - lr: 0.0010 - 6s/epoch - 21ms/step\n",
            "Epoch 15/100\n",
            "\n",
            "Epoch 15: val_loss did not improve from 1.34323\n",
            "264/264 - 5s - loss: 1.4060 - accuracy: 0.4572 - val_loss: 1.3776 - val_accuracy: 0.4727 - lr: 0.0010 - 5s/epoch - 21ms/step\n",
            "Epoch 16/100\n",
            "\n",
            "Epoch 16: val_loss did not improve from 1.34323\n",
            "264/264 - 6s - loss: 1.4042 - accuracy: 0.4583 - val_loss: 1.3466 - val_accuracy: 0.4768 - lr: 0.0010 - 6s/epoch - 21ms/step\n",
            "Epoch 17/100\n",
            "\n",
            "Epoch 17: val_loss did not improve from 1.34323\n",
            "264/264 - 6s - loss: 1.4001 - accuracy: 0.4598 - val_loss: 1.4153 - val_accuracy: 0.4506 - lr: 0.0010 - 6s/epoch - 22ms/step\n",
            "Epoch 18/100\n",
            "\n",
            "Epoch 18: val_loss did not improve from 1.34323\n",
            "264/264 - 6s - loss: 1.3928 - accuracy: 0.4616 - val_loss: 1.4031 - val_accuracy: 0.4518 - lr: 0.0010 - 6s/epoch - 22ms/step\n",
            "Epoch 19/100\n",
            "\n",
            "Epoch 19: val_loss did not improve from 1.34323\n",
            "264/264 - 6s - loss: 1.3947 - accuracy: 0.4601 - val_loss: 8.3449 - val_accuracy: 0.2091 - lr: 0.0010 - 6s/epoch - 21ms/step\n",
            "Epoch 20/100\n",
            "\n",
            "Epoch 20: val_loss did not improve from 1.34323\n",
            "264/264 - 6s - loss: 1.3898 - accuracy: 0.4634 - val_loss: 1.3677 - val_accuracy: 0.4696 - lr: 0.0010 - 6s/epoch - 22ms/step\n",
            "Epoch 21/100\n",
            "\n",
            "Epoch 21: val_loss did not improve from 1.34323\n",
            "264/264 - 6s - loss: 1.3862 - accuracy: 0.4635 - val_loss: 1.3642 - val_accuracy: 0.4753 - lr: 0.0010 - 6s/epoch - 21ms/step\n",
            "Epoch 22/100\n",
            "\n",
            "Epoch 22: val_loss did not improve from 1.34323\n",
            "264/264 - 6s - loss: 1.3935 - accuracy: 0.4614 - val_loss: 280.3001 - val_accuracy: 0.0820 - lr: 0.0010 - 6s/epoch - 22ms/step\n",
            "Epoch 23/100\n",
            "\n",
            "Epoch 23: val_loss did not improve from 1.34323\n",
            "\n",
            "Epoch 23: ReduceLROnPlateau reducing learning rate to 0.0005000000237487257.\n",
            "264/264 - 6s - loss: 1.3804 - accuracy: 0.4666 - val_loss: 1.4609 - val_accuracy: 0.4402 - lr: 0.0010 - 6s/epoch - 21ms/step\n",
            "Epoch 24/100\n",
            "\n",
            "Epoch 24: val_loss improved from 1.34323 to 1.32272, saving model to weights/weights2.h5\n",
            "264/264 - 6s - loss: 1.3537 - accuracy: 0.4765 - val_loss: 1.3227 - val_accuracy: 0.4901 - lr: 5.0000e-04 - 6s/epoch - 23ms/step\n",
            "Epoch 25/100\n",
            "\n",
            "Epoch 25: val_loss did not improve from 1.32272\n",
            "264/264 - 6s - loss: 1.3460 - accuracy: 0.4776 - val_loss: 1.3586 - val_accuracy: 0.4722 - lr: 5.0000e-04 - 6s/epoch - 22ms/step\n",
            "Epoch 26/100\n",
            "\n",
            "Epoch 26: val_loss improved from 1.32272 to 1.30410, saving model to weights/weights2.h5\n",
            "264/264 - 6s - loss: 1.3399 - accuracy: 0.4795 - val_loss: 1.3041 - val_accuracy: 0.4956 - lr: 5.0000e-04 - 6s/epoch - 23ms/step\n",
            "Epoch 27/100\n",
            "\n",
            "Epoch 27: val_loss did not improve from 1.30410\n",
            "264/264 - 6s - loss: 1.3399 - accuracy: 0.4819 - val_loss: 1.3097 - val_accuracy: 0.4934 - lr: 5.0000e-04 - 6s/epoch - 21ms/step\n",
            "Epoch 28/100\n",
            "\n",
            "Epoch 28: val_loss did not improve from 1.30410\n",
            "264/264 - 6s - loss: 1.3392 - accuracy: 0.4806 - val_loss: 1.3079 - val_accuracy: 0.4983 - lr: 5.0000e-04 - 6s/epoch - 22ms/step\n",
            "Epoch 29/100\n",
            "\n",
            "Epoch 29: val_loss did not improve from 1.30410\n",
            "264/264 - 6s - loss: 1.3334 - accuracy: 0.4823 - val_loss: 1.3421 - val_accuracy: 0.4810 - lr: 5.0000e-04 - 6s/epoch - 21ms/step\n",
            "Epoch 30/100\n",
            "\n",
            "Epoch 30: val_loss improved from 1.30410 to 1.30003, saving model to weights/weights2.h5\n",
            "264/264 - 6s - loss: 1.3320 - accuracy: 0.4842 - val_loss: 1.3000 - val_accuracy: 0.5005 - lr: 5.0000e-04 - 6s/epoch - 24ms/step\n",
            "Epoch 31/100\n",
            "\n",
            "Epoch 31: val_loss did not improve from 1.30003\n",
            "264/264 - 6s - loss: 1.3292 - accuracy: 0.4837 - val_loss: 1.3650 - val_accuracy: 0.4702 - lr: 5.0000e-04 - 6s/epoch - 22ms/step\n",
            "Epoch 32/100\n",
            "\n",
            "Epoch 32: val_loss did not improve from 1.30003\n",
            "264/264 - 6s - loss: 1.3267 - accuracy: 0.4849 - val_loss: 1.3359 - val_accuracy: 0.4821 - lr: 5.0000e-04 - 6s/epoch - 21ms/step\n",
            "Epoch 33/100\n",
            "\n",
            "Epoch 33: val_loss did not improve from 1.30003\n",
            "264/264 - 5s - loss: 1.3259 - accuracy: 0.4859 - val_loss: 1.3308 - val_accuracy: 0.4917 - lr: 5.0000e-04 - 5s/epoch - 21ms/step\n",
            "Epoch 34/100\n",
            "\n",
            "Epoch 34: val_loss did not improve from 1.30003\n",
            "264/264 - 6s - loss: 1.3267 - accuracy: 0.4837 - val_loss: 1.4278 - val_accuracy: 0.4550 - lr: 5.0000e-04 - 6s/epoch - 21ms/step\n",
            "Epoch 35/100\n",
            "\n",
            "Epoch 35: val_loss did not improve from 1.30003\n",
            "264/264 - 6s - loss: 1.3231 - accuracy: 0.4869 - val_loss: 1.4550 - val_accuracy: 0.4435 - lr: 5.0000e-04 - 6s/epoch - 22ms/step\n",
            "Epoch 36/100\n",
            "\n",
            "Epoch 36: val_loss did not improve from 1.30003\n",
            "264/264 - 6s - loss: 1.3235 - accuracy: 0.4844 - val_loss: 1.3214 - val_accuracy: 0.4903 - lr: 5.0000e-04 - 6s/epoch - 22ms/step\n",
            "Epoch 37/100\n",
            "\n",
            "Epoch 37: val_loss did not improve from 1.30003\n",
            "264/264 - 6s - loss: 1.3214 - accuracy: 0.4874 - val_loss: 1.3862 - val_accuracy: 0.4625 - lr: 5.0000e-04 - 6s/epoch - 21ms/step\n",
            "Epoch 38/100\n",
            "\n",
            "Epoch 38: val_loss did not improve from 1.30003\n",
            "264/264 - 6s - loss: 1.3169 - accuracy: 0.4865 - val_loss: 1.3179 - val_accuracy: 0.4900 - lr: 5.0000e-04 - 6s/epoch - 21ms/step\n",
            "Epoch 39/100\n",
            "\n",
            "Epoch 39: val_loss did not improve from 1.30003\n",
            "264/264 - 6s - loss: 1.3167 - accuracy: 0.4883 - val_loss: 1.6348 - val_accuracy: 0.4092 - lr: 5.0000e-04 - 6s/epoch - 22ms/step\n",
            "Epoch 40/100\n",
            "\n",
            "Epoch 40: val_loss did not improve from 1.30003\n",
            "\n",
            "Epoch 40: ReduceLROnPlateau reducing learning rate to 0.0002500000118743628.\n",
            "264/264 - 5s - loss: 1.3181 - accuracy: 0.4886 - val_loss: 1.3447 - val_accuracy: 0.4850 - lr: 5.0000e-04 - 5s/epoch - 21ms/step\n",
            "Epoch 41/100\n",
            "\n",
            "Epoch 41: val_loss improved from 1.30003 to 1.29459, saving model to weights/weights2.h5\n",
            "264/264 - 6s - loss: 1.3007 - accuracy: 0.4936 - val_loss: 1.2946 - val_accuracy: 0.5027 - lr: 2.5000e-04 - 6s/epoch - 24ms/step\n",
            "Epoch 42/100\n",
            "\n",
            "Epoch 42: val_loss did not improve from 1.29459\n",
            "264/264 - 5s - loss: 1.2960 - accuracy: 0.4969 - val_loss: 1.3288 - val_accuracy: 0.4884 - lr: 2.5000e-04 - 5s/epoch - 21ms/step\n",
            "Epoch 43/100\n",
            "\n",
            "Epoch 43: val_loss did not improve from 1.29459\n",
            "264/264 - 6s - loss: 1.2942 - accuracy: 0.4986 - val_loss: 1.4014 - val_accuracy: 0.4618 - lr: 2.5000e-04 - 6s/epoch - 21ms/step\n",
            "Epoch 44/100\n",
            "\n",
            "Epoch 44: val_loss did not improve from 1.29459\n",
            "264/264 - 6s - loss: 1.2928 - accuracy: 0.4972 - val_loss: 5.4859 - val_accuracy: 0.2494 - lr: 2.5000e-04 - 6s/epoch - 22ms/step\n",
            "Epoch 45/100\n",
            "\n",
            "Epoch 45: val_loss did not improve from 1.29459\n",
            "264/264 - 5s - loss: 1.2942 - accuracy: 0.4972 - val_loss: 1.4357 - val_accuracy: 0.4493 - lr: 2.5000e-04 - 5s/epoch - 21ms/step\n",
            "Epoch 46/100\n",
            "\n",
            "Epoch 46: val_loss did not improve from 1.29459\n",
            "264/264 - 6s - loss: 1.2910 - accuracy: 0.4989 - val_loss: 5.0782 - val_accuracy: 0.2690 - lr: 2.5000e-04 - 6s/epoch - 22ms/step\n",
            "Epoch 47/100\n",
            "\n",
            "Epoch 47: val_loss did not improve from 1.29459\n",
            "264/264 - 5s - loss: 1.2912 - accuracy: 0.4965 - val_loss: 1.3321 - val_accuracy: 0.4835 - lr: 2.5000e-04 - 5s/epoch - 21ms/step\n",
            "Epoch 48/100\n",
            "\n",
            "Epoch 48: val_loss did not improve from 1.29459\n",
            "264/264 - 6s - loss: 1.2897 - accuracy: 0.4987 - val_loss: 2.0749 - val_accuracy: 0.3455 - lr: 2.5000e-04 - 6s/epoch - 21ms/step\n",
            "Epoch 49/100\n",
            "\n",
            "Epoch 49: val_loss did not improve from 1.29459\n",
            "264/264 - 6s - loss: 1.2893 - accuracy: 0.4979 - val_loss: 1.4322 - val_accuracy: 0.4473 - lr: 2.5000e-04 - 6s/epoch - 22ms/step\n",
            "Epoch 50/100\n",
            "\n",
            "Epoch 50: val_loss did not improve from 1.29459\n",
            "264/264 - 6s - loss: 1.2857 - accuracy: 0.4992 - val_loss: 9.1827 - val_accuracy: 0.2473 - lr: 2.5000e-04 - 6s/epoch - 21ms/step\n",
            "Epoch 51/100\n",
            "\n",
            "Epoch 51: val_loss did not improve from 1.29459\n",
            "\n",
            "Epoch 51: ReduceLROnPlateau reducing learning rate to 0.0001250000059371814.\n",
            "264/264 - 6s - loss: 1.2848 - accuracy: 0.5007 - val_loss: 1.6386 - val_accuracy: 0.3951 - lr: 2.5000e-04 - 6s/epoch - 22ms/step\n",
            "Epoch 52/100\n",
            "\n",
            "Epoch 52: val_loss did not improve from 1.29459\n",
            "264/264 - 6s - loss: 1.2782 - accuracy: 0.5035 - val_loss: 1.3862 - val_accuracy: 0.4688 - lr: 1.2500e-04 - 6s/epoch - 22ms/step\n",
            "Epoch 53/100\n",
            "\n",
            "Epoch 53: val_loss did not improve from 1.29459\n",
            "264/264 - 6s - loss: 1.2777 - accuracy: 0.5033 - val_loss: 1.7115 - val_accuracy: 0.3845 - lr: 1.2500e-04 - 6s/epoch - 21ms/step\n",
            "Epoch 54/100\n",
            "\n",
            "Epoch 54: val_loss did not improve from 1.29459\n",
            "264/264 - 5s - loss: 1.2776 - accuracy: 0.5035 - val_loss: 2.9098 - val_accuracy: 0.3071 - lr: 1.2500e-04 - 5s/epoch - 21ms/step\n",
            "Epoch 55/100\n",
            "\n",
            "Epoch 55: val_loss did not improve from 1.29459\n",
            "264/264 - 6s - loss: 1.2754 - accuracy: 0.5053 - val_loss: 1.2972 - val_accuracy: 0.5029 - lr: 1.2500e-04 - 6s/epoch - 21ms/step\n",
            "Epoch 56/100\n",
            "\n",
            "Epoch 56: val_loss did not improve from 1.29459\n",
            "264/264 - 5s - loss: 1.2753 - accuracy: 0.5033 - val_loss: 1.9781 - val_accuracy: 0.3500 - lr: 1.2500e-04 - 5s/epoch - 21ms/step\n",
            "Epoch 57/100\n",
            "\n",
            "Epoch 57: val_loss did not improve from 1.29459\n",
            "264/264 - 6s - loss: 1.2732 - accuracy: 0.5051 - val_loss: 1.5185 - val_accuracy: 0.4331 - lr: 1.2500e-04 - 6s/epoch - 22ms/step\n",
            "Epoch 58/100\n",
            "\n",
            "Epoch 58: val_loss did not improve from 1.29459\n",
            "264/264 - 5s - loss: 1.2699 - accuracy: 0.5061 - val_loss: 1.6577 - val_accuracy: 0.4095 - lr: 1.2500e-04 - 5s/epoch - 21ms/step\n",
            "Epoch 59/100\n",
            "\n",
            "Epoch 59: val_loss did not improve from 1.29459\n",
            "264/264 - 6s - loss: 1.2724 - accuracy: 0.5062 - val_loss: 1.6105 - val_accuracy: 0.4103 - lr: 1.2500e-04 - 6s/epoch - 21ms/step\n",
            "Epoch 60/100\n",
            "\n",
            "Epoch 60: val_loss did not improve from 1.29459\n",
            "264/264 - 6s - loss: 1.2724 - accuracy: 0.5069 - val_loss: 1.3066 - val_accuracy: 0.4952 - lr: 1.2500e-04 - 6s/epoch - 22ms/step\n",
            "Epoch 61/100\n",
            "\n",
            "Epoch 61: val_loss did not improve from 1.29459\n",
            "\n",
            "Epoch 61: ReduceLROnPlateau reducing learning rate to 6.25000029685907e-05.\n",
            "264/264 - 5s - loss: 1.2725 - accuracy: 0.5033 - val_loss: 2.1507 - val_accuracy: 0.3274 - lr: 1.2500e-04 - 5s/epoch - 21ms/step\n",
            "Epoch 62/100\n",
            "\n",
            "Epoch 62: val_loss did not improve from 1.29459\n",
            "264/264 - 6s - loss: 1.2687 - accuracy: 0.5069 - val_loss: 1.3751 - val_accuracy: 0.4673 - lr: 6.2500e-05 - 6s/epoch - 21ms/step\n",
            "Epoch 63/100\n",
            "\n",
            "Epoch 63: val_loss did not improve from 1.29459\n",
            "264/264 - 5s - loss: 1.2692 - accuracy: 0.5076 - val_loss: 1.2981 - val_accuracy: 0.4989 - lr: 6.2500e-05 - 5s/epoch - 21ms/step\n",
            "Epoch 64/100\n",
            "\n",
            "Epoch 64: val_loss improved from 1.29459 to 1.29212, saving model to weights/weights2.h5\n",
            "264/264 - 6s - loss: 1.2674 - accuracy: 0.5081 - val_loss: 1.2921 - val_accuracy: 0.5031 - lr: 6.2500e-05 - 6s/epoch - 24ms/step\n",
            "Epoch 65/100\n",
            "\n",
            "Epoch 65: val_loss did not improve from 1.29212\n",
            "264/264 - 5s - loss: 1.2661 - accuracy: 0.5091 - val_loss: 1.4617 - val_accuracy: 0.4457 - lr: 6.2500e-05 - 5s/epoch - 21ms/step\n",
            "Epoch 66/100\n",
            "\n",
            "Epoch 66: val_loss did not improve from 1.29212\n",
            "264/264 - 6s - loss: 1.2642 - accuracy: 0.5103 - val_loss: 1.3404 - val_accuracy: 0.4815 - lr: 6.2500e-05 - 6s/epoch - 21ms/step\n",
            "Epoch 67/100\n",
            "\n",
            "Epoch 67: val_loss did not improve from 1.29212\n",
            "264/264 - 6s - loss: 1.2631 - accuracy: 0.5079 - val_loss: 1.3125 - val_accuracy: 0.4918 - lr: 6.2500e-05 - 6s/epoch - 21ms/step\n",
            "Epoch 68/100\n",
            "\n",
            "Epoch 68: val_loss did not improve from 1.29212\n",
            "264/264 - 6s - loss: 1.2631 - accuracy: 0.5081 - val_loss: 1.3390 - val_accuracy: 0.4845 - lr: 6.2500e-05 - 6s/epoch - 22ms/step\n",
            "Epoch 69/100\n",
            "\n",
            "Epoch 69: val_loss improved from 1.29212 to 1.29068, saving model to weights/weights2.h5\n",
            "264/264 - 6s - loss: 1.2650 - accuracy: 0.5091 - val_loss: 1.2907 - val_accuracy: 0.5053 - lr: 6.2500e-05 - 6s/epoch - 23ms/step\n",
            "Epoch 70/100\n",
            "\n",
            "Epoch 70: val_loss did not improve from 1.29068\n",
            "264/264 - 5s - loss: 1.2648 - accuracy: 0.5091 - val_loss: 1.3365 - val_accuracy: 0.4834 - lr: 6.2500e-05 - 5s/epoch - 21ms/step\n",
            "Epoch 71/100\n",
            "\n",
            "Epoch 71: val_loss did not improve from 1.29068\n",
            "264/264 - 6s - loss: 1.2638 - accuracy: 0.5091 - val_loss: 1.2967 - val_accuracy: 0.5060 - lr: 6.2500e-05 - 6s/epoch - 22ms/step\n",
            "Epoch 72/100\n",
            "\n",
            "Epoch 72: val_loss did not improve from 1.29068\n",
            "264/264 - 5s - loss: 1.2646 - accuracy: 0.5085 - val_loss: 1.4424 - val_accuracy: 0.4548 - lr: 6.2500e-05 - 5s/epoch - 21ms/step\n",
            "Epoch 73/100\n",
            "\n",
            "Epoch 73: val_loss did not improve from 1.29068\n",
            "264/264 - 6s - loss: 1.2622 - accuracy: 0.5101 - val_loss: 1.3172 - val_accuracy: 0.4947 - lr: 6.2500e-05 - 6s/epoch - 21ms/step\n",
            "Epoch 74/100\n",
            "\n",
            "Epoch 74: val_loss did not improve from 1.29068\n",
            "264/264 - 6s - loss: 1.2611 - accuracy: 0.5104 - val_loss: 1.3167 - val_accuracy: 0.5000 - lr: 6.2500e-05 - 6s/epoch - 22ms/step\n",
            "Epoch 75/100\n",
            "\n",
            "Epoch 75: val_loss did not improve from 1.29068\n",
            "264/264 - 6s - loss: 1.2655 - accuracy: 0.5080 - val_loss: 1.3281 - val_accuracy: 0.4918 - lr: 6.2500e-05 - 6s/epoch - 22ms/step\n",
            "Epoch 76/100\n",
            "\n",
            "Epoch 76: val_loss did not improve from 1.29068\n",
            "264/264 - 6s - loss: 1.2635 - accuracy: 0.5087 - val_loss: 1.4957 - val_accuracy: 0.4365 - lr: 6.2500e-05 - 6s/epoch - 23ms/step\n",
            "Epoch 77/100\n",
            "\n",
            "Epoch 77: val_loss did not improve from 1.29068\n",
            "264/264 - 5s - loss: 1.2637 - accuracy: 0.5089 - val_loss: 1.3932 - val_accuracy: 0.4692 - lr: 6.2500e-05 - 5s/epoch - 21ms/step\n",
            "Epoch 78/100\n",
            "\n",
            "Epoch 78: val_loss did not improve from 1.29068\n",
            "264/264 - 6s - loss: 1.2606 - accuracy: 0.5103 - val_loss: 1.3006 - val_accuracy: 0.5053 - lr: 6.2500e-05 - 6s/epoch - 22ms/step\n",
            "Epoch 79/100\n",
            "\n",
            "Epoch 79: val_loss did not improve from 1.29068\n",
            "\n",
            "Epoch 79: ReduceLROnPlateau reducing learning rate to 3.125000148429535e-05.\n",
            "264/264 - 6s - loss: 1.2600 - accuracy: 0.5098 - val_loss: 1.2945 - val_accuracy: 0.5021 - lr: 6.2500e-05 - 6s/epoch - 22ms/step\n",
            "Epoch 80/100\n",
            "\n",
            "Epoch 80: val_loss did not improve from 1.29068\n",
            "264/264 - 6s - loss: 1.2621 - accuracy: 0.5093 - val_loss: 1.3127 - val_accuracy: 0.4971 - lr: 3.1250e-05 - 6s/epoch - 22ms/step\n",
            "Epoch 81/100\n",
            "\n",
            "Epoch 81: val_loss improved from 1.29068 to 1.28972, saving model to weights/weights2.h5\n",
            "264/264 - 6s - loss: 1.2593 - accuracy: 0.5110 - val_loss: 1.2897 - val_accuracy: 0.5064 - lr: 3.1250e-05 - 6s/epoch - 23ms/step\n",
            "Epoch 82/100\n",
            "\n",
            "Epoch 82: val_loss did not improve from 1.28972\n",
            "264/264 - 5s - loss: 1.2606 - accuracy: 0.5112 - val_loss: 1.3111 - val_accuracy: 0.4929 - lr: 3.1250e-05 - 5s/epoch - 21ms/step\n",
            "Epoch 83/100\n",
            "\n",
            "Epoch 83: val_loss did not improve from 1.28972\n",
            "264/264 - 6s - loss: 1.2597 - accuracy: 0.5121 - val_loss: 1.3367 - val_accuracy: 0.4808 - lr: 3.1250e-05 - 6s/epoch - 21ms/step\n",
            "Epoch 84/100\n",
            "\n",
            "Epoch 84: val_loss did not improve from 1.28972\n",
            "264/264 - 6s - loss: 1.2602 - accuracy: 0.5115 - val_loss: 1.3128 - val_accuracy: 0.4956 - lr: 3.1250e-05 - 6s/epoch - 22ms/step\n",
            "Epoch 85/100\n",
            "\n",
            "Epoch 85: val_loss improved from 1.28972 to 1.28762, saving model to weights/weights2.h5\n",
            "264/264 - 6s - loss: 1.2613 - accuracy: 0.5107 - val_loss: 1.2876 - val_accuracy: 0.5086 - lr: 3.1250e-05 - 6s/epoch - 23ms/step\n",
            "Epoch 86/100\n",
            "\n",
            "Epoch 86: val_loss did not improve from 1.28762\n",
            "264/264 - 5s - loss: 1.2577 - accuracy: 0.5118 - val_loss: 1.3174 - val_accuracy: 0.4916 - lr: 3.1250e-05 - 5s/epoch - 21ms/step\n",
            "Epoch 87/100\n",
            "\n",
            "Epoch 87: val_loss did not improve from 1.28762\n",
            "264/264 - 6s - loss: 1.2580 - accuracy: 0.5119 - val_loss: 1.3199 - val_accuracy: 0.4937 - lr: 3.1250e-05 - 6s/epoch - 22ms/step\n",
            "Epoch 88/100\n",
            "\n",
            "Epoch 88: val_loss did not improve from 1.28762\n",
            "264/264 - 6s - loss: 1.2591 - accuracy: 0.5107 - val_loss: 1.2926 - val_accuracy: 0.5027 - lr: 3.1250e-05 - 6s/epoch - 22ms/step\n",
            "Epoch 89/100\n",
            "\n",
            "Epoch 89: val_loss did not improve from 1.28762\n",
            "264/264 - 5s - loss: 1.2596 - accuracy: 0.5116 - val_loss: 1.3028 - val_accuracy: 0.4999 - lr: 3.1250e-05 - 5s/epoch - 21ms/step\n",
            "Epoch 90/100\n",
            "\n",
            "Epoch 90: val_loss did not improve from 1.28762\n",
            "264/264 - 6s - loss: 1.2591 - accuracy: 0.5114 - val_loss: 1.3051 - val_accuracy: 0.5026 - lr: 3.1250e-05 - 6s/epoch - 21ms/step\n",
            "Epoch 91/100\n",
            "\n",
            "Epoch 91: val_loss did not improve from 1.28762\n",
            "264/264 - 5s - loss: 1.2595 - accuracy: 0.5130 - val_loss: 1.3495 - val_accuracy: 0.4803 - lr: 3.1250e-05 - 5s/epoch - 21ms/step\n",
            "Epoch 92/100\n",
            "\n",
            "Epoch 92: val_loss did not improve from 1.28762\n",
            "264/264 - 6s - loss: 1.2566 - accuracy: 0.5125 - val_loss: 1.3539 - val_accuracy: 0.4803 - lr: 3.1250e-05 - 6s/epoch - 22ms/step\n",
            "Epoch 93/100\n",
            "\n",
            "Epoch 93: val_loss did not improve from 1.28762\n",
            "264/264 - 5s - loss: 1.2570 - accuracy: 0.5116 - val_loss: 1.3169 - val_accuracy: 0.4883 - lr: 3.1250e-05 - 5s/epoch - 21ms/step\n",
            "Epoch 94/100\n",
            "\n",
            "Epoch 94: val_loss did not improve from 1.28762\n",
            "264/264 - 6s - loss: 1.2592 - accuracy: 0.5102 - val_loss: 1.2923 - val_accuracy: 0.5079 - lr: 3.1250e-05 - 6s/epoch - 22ms/step\n",
            "Epoch 95/100\n",
            "\n",
            "Epoch 95: val_loss did not improve from 1.28762\n",
            "\n",
            "Epoch 95: ReduceLROnPlateau reducing learning rate to 1.5625000742147677e-05.\n",
            "264/264 - 5s - loss: 1.2598 - accuracy: 0.5114 - val_loss: 1.2962 - val_accuracy: 0.5022 - lr: 3.1250e-05 - 5s/epoch - 21ms/step\n",
            "Epoch 96/100\n",
            "\n",
            "Epoch 96: val_loss did not improve from 1.28762\n",
            "264/264 - 6s - loss: 1.2550 - accuracy: 0.5121 - val_loss: 1.3049 - val_accuracy: 0.4949 - lr: 1.5625e-05 - 6s/epoch - 21ms/step\n",
            "Epoch 97/100\n",
            "\n",
            "Epoch 97: val_loss did not improve from 1.28762\n",
            "264/264 - 6s - loss: 1.2562 - accuracy: 0.5121 - val_loss: 1.2955 - val_accuracy: 0.5008 - lr: 1.5625e-05 - 6s/epoch - 22ms/step\n",
            "Epoch 98/100\n",
            "\n",
            "Epoch 98: val_loss did not improve from 1.28762\n",
            "264/264 - 5s - loss: 1.2532 - accuracy: 0.5133 - val_loss: 1.3016 - val_accuracy: 0.4971 - lr: 1.5625e-05 - 5s/epoch - 21ms/step\n",
            "Epoch 99/100\n",
            "\n",
            "Epoch 99: val_loss did not improve from 1.28762\n",
            "264/264 - 6s - loss: 1.2552 - accuracy: 0.5125 - val_loss: 1.3006 - val_accuracy: 0.4978 - lr: 1.5625e-05 - 6s/epoch - 21ms/step\n",
            "Epoch 100/100\n",
            "\n",
            "Epoch 100: val_loss did not improve from 1.28762\n",
            "264/264 - 5s - loss: 1.2566 - accuracy: 0.5113 - val_loss: 1.2889 - val_accuracy: 0.5058 - lr: 1.5625e-05 - 5s/epoch - 21ms/step\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyError",
          "evalue": "'acc'",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-67-16508dd32031>\u001b[0m in \u001b[0;36m<cell line: 38>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     36\u001b[0m                     )\n\u001b[1;32m     37\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 38\u001b[0;31m \u001b[0mmltools\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshow_history\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     39\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     40\u001b[0m \u001b[0;31m#Show simple version of performance\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/content/drive/My Drive/modrec/mltools.py\u001b[0m in \u001b[0;36mshow_history\u001b[0;34m(history)\u001b[0m\n\u001b[1;32m     19\u001b[0m     \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfigure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m     \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtitle\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Training accuracy performance'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 21\u001b[0;31m     \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mepoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhistory\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'acc'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'train_acc'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     22\u001b[0m     \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mepoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhistory\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'val_acc'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'val_acc'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     23\u001b[0m     \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlegend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyError\u001b[0m: 'acc'"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAi4AAAGzCAYAAAAIWpzfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAuc0lEQVR4nO3dfVyUVf7/8TcgDJqCbAjehJJYWlpqqIRmbC1JZbp+d9ssW0HX0kotpU0lTVJLKs1s03K11LavpemaaRKmpN+y2K+l0o1l3uBdJigVYFAQzPn98f0xNQLKkIIHXs/HYx6P5sw51/W55lw0b6+bGS9jjBEAAIAFvOu6AAAAgOoiuAAAAGsQXAAAgDUILgAAwBoEFwAAYA2CCwAAsAbBBQAAWIPgAgAArEFwAQAA1iC4oMEYNmyYwsPDazT20UcflZeX19ktCPVSaWmpJkyYoLCwMHl7e2vQoEF1XRJQrzSq6wKA6gaCzZs36/e///25LQb4jRYvXqxZs2Zp3Lhxuuqqq9S2bdu6LgmoV7z4rSLUtf/+7/92e/6vf/1LGzdu1CuvvOLWfsMNNyg0NLTG6/n555/ldDrlcDg8HltaWqrS0lL5+/vXeP1oGG6//XZt3bpVX3/9dV2XAtRLBBecd8aMGaP58+frTLtmUVGRmjRpUktVoTqMMfrpp5/UuHHjui6lVv16u6+//nodP35cn3/++VlZttPpVElJCaEZ+P+4xgVW+P3vf68uXbpo+/btuvbaa9WkSRM9/PDDkqQ333xT/fv3V+vWreVwOBQREaEZM2aorKzMbRmnXuNy8OBBeXl5afbs2Vq4cKEiIiLkcDjUs2dPffTRR25jK7vGxcvLS2PGjNGaNWvUpUsXORwOde7cWWlpaRXq37Jli3r06CF/f39FRETon//8Z7Wvm3n//ff1l7/8RW3btpXD4VBYWJjGjx+vH3/8sULf3bt367bbblOLFi3UuHFjdezYUZMnT3brc/ToUY0YMcL1fl188cW69957VVJSUuW2StLSpUvl5eWlgwcPutrCw8N1yy23aMOGDerRo4caN26sf/7zn5KkJUuW6Prrr1dISIgcDocuv/xyvfDCC5Vu49tvv62YmBg1a9ZMAQEB6tmzp1599VVJUnJysnx9fXXixIkK40aOHKnmzZvrp59+qvL9GzZsmJo2baqsrCzFxcXpggsuUOvWrTV9+vQK4djpdGru3Lnq3Lmz/P39FRoaqlGjRun7779361fVdnt5eWnz5s3atWuXvLy85OXlpS1btkiSCgsL9eCDDyosLEwOh0MdO3bU7NmzK9RQvl8tW7ZMnTt3lsPhUFpamuv937p1q+6//361aNFCzZs316hRo1RSUqK8vDzFx8crKChIQUFBmjBhQoVlz549W71799aFF16oxo0bKzIyUqtWrarwnnmyb59pf5KkvLw8jRs3zrXtHTp00JNPPimn01nlvAFV4RoXWOPbb7/VTTfdpNtvv11//etfXaeNli5dqqZNmyoxMVFNmzbVu+++q6lTp6qgoECzZs0643JfffVVnTx5UqNGjZKXl5eeeuop/elPf1JWVpZ8fX1PO3br1q1avXq17rvvPjVr1kz/+Mc/9Oc//1mHDx/WhRdeKEnauXOnbrzxRrVq1UrTpk1TWVmZpk+frhYtWlRru1euXKmioiLde++9uvDCC7Vt2zY999xz+vrrr7Vy5UpXv08//VR9+/aVr6+vRo4cqfDwcO3fv1/r1q3T448/Lkn65ptv1KtXL+Xl5WnkyJHq1KmTjh49qlWrVqmoqEh+fn7VqunXvvrqK91xxx0aNWqU7r77bnXs2FGS9MILL6hz584aOHCgGjVqpHXr1um+++6T0+nU6NGjXeOXLl2qv/3tb+rcubOSkpLUvHlz7dy5U2lpaRoyZIiGDh2q6dOna8WKFRozZoxrXElJiVatWqU///nPZzwaUVZWphtvvFFXX321nnrqKaWlpSk5OVmlpaWaPn26q9+oUaO0dOlSDR8+XPfff78OHDigefPmaefOnfrggw/c9odTt/uiiy7SK6+8oscff1w//PCDUlJSJEmXXXaZjDEaOHCgNm/erBEjRqhbt27asGGDHnroIR09elTPPPOMW73vvvuuXn/9dY0ZM0bBwcEKDw9XZmamJGns2LFq2bKlpk2bpv/85z9auHChmjdvrg8//FBt27bVzJkzlZqaqlmzZqlLly6Kj493LffZZ5/VwIEDdeedd6qkpETLly/XX/7yF7311lvq37+/Ww3V2bersz8VFRUpJiZGR48e1ahRo9S2bVt9+OGHSkpK0rFjxzR37twz7WKAOwOcZ0aPHm1O3TVjYmKMJLNgwYIK/YuKiiq0jRo1yjRp0sT89NNPrraEhATTrl071/MDBw4YSebCCy803333nav9zTffNJLMunXrXG3JyckVapJk/Pz8zL59+1xtn3zyiZFknnvuOVfbgAEDTJMmTczRo0ddbXv37jWNGjWqsMzKVLZ9KSkpxsvLyxw6dMjVdu2115pmzZq5tRljjNPpdP13fHy88fb2Nh999FGFZZb3q2xbjTFmyZIlRpI5cOCAq61du3ZGkklLS6tW3XFxcaZ9+/au53l5eaZZs2YmKirK/Pjjj1XWHR0dbaKiotxeX716tZFkNm/eXGE9v5aQkGAkmbFjx7otu3///sbPz8+cOHHCGGPM+++/bySZZcuWuY1PS0ur0H667Y6JiTGdO3d2a1uzZo2RZB577DG39ltvvdV4eXm57UOSjLe3t9m1a5db3/L3Py4ursJ74+XlZe655x5XW2lpqbnoootMTEyM2zJOnZOSkhLTpUsXc/3117u1V3ffrs7+NGPGDHPBBReYPXv2uL0+adIk4+PjYw4fPlxhLHA6nCqCNRwOh4YPH16h/dfXU5w8eVK5ubnq27evioqKtHv37jMud/DgwQoKCnI979u3ryQpKyvrjGNjY2MVERHhen7llVcqICDANbasrEybNm3SoEGD1Lp1a1e/Dh066Kabbjrj8iX37SssLFRubq569+4tY4x27twpSTpx4oTee+89/e1vf6twF0v5aR+n06k1a9ZowIAB6tGjR4X11PR274svvlhxcXGnrTs/P1+5ubmKiYlRVlaW8vPzJUkbN27UyZMnNWnSpApHTX5dT3x8vP73f/9X+/fvd7UtW7ZMYWFhiomJqVadvz5aU34qpKSkRJs2bZL0f0e2AgMDdcMNNyg3N9f1iIyMVNOmTbV58+ZqbXdlUlNT5ePjo/vvv9+t/cEHH5QxRm+//bZbe0xMjC6//PJKlzVixAi39yYqKkrGGI0YMcLV5uPjox49elTYh389J99//73y8/PVt29f7dixo8J6zrRvV3d/Wrlypfr27augoCC39zU2NlZlZWV67733Kt1OoCqcKoI12rRpU+mpjF27dmnKlCl69913VVBQ4PZa+Qfk6Zz6QV8eYk69rqE6Y8vHl489fvy4fvzxR3Xo0KFCv8raKnP48GFNnTpVa9eurVBT+faVf5h06dKlyuWcOHFCBQUFp+1TExdffHGl7R988IGSk5OVkZGhoqIit9fy8/MVGBjoCiJnqmnw4MEaN26cli1bpqlTpyo/P19vvfWWxo8fX63A5e3trfbt27u1XXrppZLkumZn7969ys/PV0hISKXLOH78uNvzqra7MocOHVLr1q3VrFkzt/bLLrvM9Xp1l33qPhcYGChJCgsLq9B+6v7y1ltv6bHHHlNmZqaKi4td7ZW9h2fat6u7P+3du1effvppladGT31fgTMhuMAald2pkpeXp5iYGAUEBGj69OmKiIiQv7+/duzYoYkTJ1br4j8fH59K2001brj7LWOro6ysTDfccIO+++47TZw4UZ06ddIFF1ygo0ePatiwYefk4saqgsCpFzuXq2xe9u/frz/84Q/q1KmT5syZo7CwMPn5+Sk1NVXPPPOMx3UHBQXplltucQWXVatWqbi4WH/96189Ws7pOJ1OhYSEaNmyZZW+fuoH77m8c+p0y65qn6us/df74fvvv6+BAwfq2muv1fPPP69WrVrJ19dXS5YscV0IXZ31eLpvO51O3XDDDZowYUKlr5cHSKC6CC6w2pYtW/Ttt99q9erVuvbaa13tBw4cqMOqfhESEiJ/f3/t27evwmuVtZ3qs88+0549e/Tyyy+7XWS5ceNGt37lRxNOdwtuixYtFBAQcMbbdMuPOOXl5al58+au9lOPCpzOunXrVFxcrLVr17r9y/3U0y3lpyI+//zzMx6Bio+P1x//+Ed99NFHWrZsmbp3767OnTtXqx6n06msrCy3D8k9e/ZIkutOs4iICG3atEl9+vQ566GkXbt22rRpk06ePOl21KX8VGa7du3O6voq8+9//1v+/v7asGGD23cZLVmypEbLq+7+FBERoR9++EGxsbE1Wg9wKq5xgdXK/1X4638FlpSU6Pnnn6+rktz4+PgoNjZWa9as0TfffONq37dvX4XrGqoaL7lvnzFGzz77rFu/Fi1a6Nprr9XixYt1+PBht9fKx5Z//fy6dev08ccfV1hXeb/yMPHraw8KCwv18ssvn7He09Wdn59f4UOyX79+atasmVJSUirc0nzqv+xvuukmBQcH68knn9T//M//eHy0Zd68eW7Lnjdvnnx9ffWHP/xBknTbbbeprKxMM2bMqDC2tLRUeXl5Hq3v126++WaVlZW51SBJzzzzjLy8vKp9vdNv4ePjIy8vL7cjZwcPHtSaNWtqtLzq7k+33XabMjIytGHDhgp98vLyVFpaWqP1o+HiiAus1rt3bwUFBSkhIUH333+/vLy89Morr5y1UzVnw6OPPqp33nlHffr00b333uv6AOvSpYvrFteqdOrUSREREfr73/+uo0ePKiAgQP/+978rvf7mH//4h6655hpdddVVGjlypC6++GIdPHhQ69evd61n5syZeueddxQTE6ORI0fqsssu07Fjx7Ry5Upt3bpVzZs3V79+/dS2bVuNGDFCDz30kHx8fLR48WK1aNGiQiiqSr9+/eTn56cBAwZo1KhR+uGHH7Ro0SKFhITo2LFjrn4BAQF65plndNddd6lnz54aMmSIgoKC9Mknn6ioqMgtLPn6+ur222/XvHnz5OPjozvuuKNatUiSv7+/0tLSlJCQoKioKL399ttav369Hn74YdcpoJiYGI0aNUopKSnKzMxUv3795Ovrq71792rlypV69tlndeutt1Z7nb82YMAAXXfddZo8ebIOHjyorl276p133tGbb76pcePGuV0Ee670799fc+bM0Y033qghQ4bo+PHjmj9/vjp06KBPP/20Rsuszv700EMPae3atbrllls0bNgwRUZGqrCwUJ999plWrVqlgwcPKjg4+CxvLeq1Wr+PCTiDqm6HPvUW03IffPCBufrqq03jxo1N69atzYQJE8yGDRsq3Cpb1e3Qs2bNqrBMSSY5Odn1vKrboUePHl1hbLt27UxCQoJbW3p6uunevbvx8/MzERER5sUXXzQPPvig8ff3r+Jd+MUXX3xhYmNjTdOmTU1wcLC5++67XbemLlmyxK3v559/bv7rv/7LNG/e3Pj7+5uOHTuaRx55xK3PoUOHTHx8vGnRooVxOBymffv2ZvTo0aa4uNjVZ/v27SYqKsr4+fmZtm3bmjlz5lR5O3T//v0rrXvt2rXmyiuvNP7+/iY8PNw8+eSTZvHixRWWUd63d+/epnHjxiYgIMD06tXLvPbaaxWWuW3bNiPJ9OvX74zvW7mEhARzwQUXmP3795t+/fqZJk2amNDQUJOcnGzKysoq9F+4cKGJjIw0jRs3Ns2aNTNXXHGFmTBhgvnmm2+qtd1V7asnT54048ePN61btza+vr7mkksuMbNmzXK7tdmYqver8vf/1FuPy/fN8tu6T93uX3vppZfMJZdcYhwOh+nUqZNZsmTJb963q7M/nTx50iQlJZkOHToYPz8/ExwcbHr37m1mz55tSkpKKqwHOB2+8h+oI4MGDdKuXbu0d+/eui7FGp988om6deumf/3rXxo6dGi1xgwbNkyrVq3SDz/8cI6rA1AbuMYFqAWnfj3/3r17lZqayq9de2jRokVq2rSp/vSnP9V1KQDqCNe4ALWgffv2GjZsmNq3b69Dhw7phRdekJ+fX5W3iMLdunXr9MUXX2jhwoUaM2aMLrjggrouCUAdIbgAteDGG2/Ua6+9puzsbDkcDkVHR2vmzJm65JJL6ro0K4wdO1Y5OTm6+eabNW3atLouB0Ad8vgal/fee0+zZs3S9u3bdezYMb3xxhsaNGjQacds2bJFiYmJ2rVrl8LCwjRlyhQNGzbsN5QNAAAaIo+vcSksLFTXrl01f/78avU/cOCA+vfvr+uuu06ZmZkaN26c7rrrrkrv6QcAADid33RXkZeX1xmPuEycOFHr1693+3bF22+/XXl5eUpLS6vpqgEAQAN0zq9xycjIqPBVz3FxcRo3blyVY4qLi91+AMzpdOq7777ThRdeWONfsAUAALXLGKOTJ0+qdevW8vY+Ozcyn/Pgkp2drdDQULe20NBQFRQU6Mcff6z0N0FSUlK4AA8AgHriyJEjuuiii87Kss7Lu4qSkpKUmJjoep6fn6+2bdvqyJEjCggIqMPKAABAdRUUFCgsLMztx0V/q3MeXFq2bKmcnBy3tpycHAUEBFT5C6wOh8Pt10vLBQQEEFwAALDM2bzM45x/c250dLTS09Pd2jZu3Kjo6OhzvWoAAFDPeBxcfvjhB2VmZrp+bfbAgQPKzMx0/WpsUlKS4uPjXf3vueceZWVlacKECdq9e7eef/55vf766xo/fvzZ2QIAANBgeBxcPv74Y3Xv3l3du3eXJCUmJqp79+6aOnWqJOnYsWOuECNJF198sdavX6+NGzeqa9euevrpp/Xiiy8qLi7uLG0CAABoKKz4deiCggIFBgYqPz+fa1wAALDEufj85tehAQCANQguAADAGgQXAABgDYILAACwBsEFAABYg+ACAACsQXABAADWILgAAABrEFwAAIA1CC4AAMAaBBcAAGANggsAALAGwQUAAFiD4AIAAKxBcAEAANYguAAAAGsQXAAAgDUILgAAwBoEFwAAYA2CCwAAsAbBBQAAWIPgAgAArEFwAQAA1iC4AAAAaxBcAACANQguAADAGgQXAABgDYILAACwBsEFAABYg+ACAACsQXABAADWILgAAABrEFwAAIA1CC4AAMAaBBcAAGANggsAALAGwQUAAFiD4AIAAKxBcAEAANYguAAAAGsQXAAAgDUILgAAwBoEFwAAYA2CCwAAsAbBBQAAWIPgAgAArEFwAQAA1iC4AAAAaxBcAACANQguAADAGgQXAABgDYILAACwBsEFAABYg+ACAACsQXABAADWILgAAABrEFwAAIA1CC4AAMAaBBcAAGANggsAALAGwQUAAFiD4AIAAKxBcAEAANaoUXCZP3++wsPD5e/vr6ioKG3btu20/efOnauOHTuqcePGCgsL0/jx4/XTTz/VqGAAANBweRxcVqxYocTERCUnJ2vHjh3q2rWr4uLidPz48Ur7v/rqq5o0aZKSk5P15Zdf6qWXXtKKFSv08MMP/+biAQBAw+JxcJkzZ47uvvtuDR8+XJdffrkWLFigJk2aaPHixZX2//DDD9WnTx8NGTJE4eHh6tevn+64444zHqUBAAA4lUfBpaSkRNu3b1dsbOwvC/D2VmxsrDIyMiod07t3b23fvt0VVLKyspSamqqbb765yvUUFxeroKDA7QEAANDIk865ubkqKytTaGioW3toaKh2795d6ZghQ4YoNzdX11xzjYwxKi0t1T333HPaU0UpKSmaNm2aJ6UBAIAG4JzfVbRlyxbNnDlTzz//vHbs2KHVq1dr/fr1mjFjRpVjkpKSlJ+f73ocOXLkXJcJAAAs4NERl+DgYPn4+CgnJ8etPScnRy1btqx0zCOPPKKhQ4fqrrvukiRdccUVKiws1MiRIzV58mR5e1fMTg6HQw6Hw5PSAABAA+DRERc/Pz9FRkYqPT3d1eZ0OpWenq7o6OhKxxQVFVUIJz4+PpIkY4yn9QIAgAbMoyMukpSYmKiEhAT16NFDvXr10ty5c1VYWKjhw4dLkuLj49WmTRulpKRIkgYMGKA5c+aoe/fuioqK0r59+/TII49owIABrgADAABQHR4Hl8GDB+vEiROaOnWqsrOz1a1bN6Wlpbku2D18+LDbEZYpU6bIy8tLU6ZM0dGjR9WiRQsNGDBAjz/++NnbCgAA0CB4GQvO1xQUFCgwMFD5+fkKCAio63IAAEA1nIvPb36rCAAAWIPgAgAArEFwAQAA1iC4AAAAaxBcAACANQguAADAGgQXAABgDYILAACwBsEFAABYg+ACAACsQXABAADWILgAAABrEFwAAIA1CC4AAMAaBBcAAGANggsAALAGwQUAAFiD4AIAAKxBcAEAANYguAAAAGsQXAAAgDUILgAAwBoEFwAAYA2CCwAAsAbBBQAAWIPgAgAArEFwAQAA1iC4AAAAaxBcAACANQguAADAGgQXAABgDYILAACwBsEFAABYg+ACAACsQXABAADWILgAAABrEFwAAIA1CC4AAMAaBBcAAGANggsAALAGwQUAAFiD4AIAAKxBcAEAANYguAAAAGsQXAAAgDUILgAAwBoEFwAAYA2CCwAAsAbBBQAAWIPgAgAArEFwAQAA1iC4AAAAaxBcAACANQguAADAGgQXAABgDYILAACwBsEFAABYg+ACAACsQXABAADWILgAAABrEFwAAIA1CC4AAMAaBBcAAGCNGgWX+fPnKzw8XP7+/oqKitK2bdtO2z8vL0+jR49Wq1at5HA4dOmllyo1NbVGBQMAgIarkacDVqxYocTERC1YsEBRUVGaO3eu4uLi9NVXXykkJKRC/5KSEt1www0KCQnRqlWr1KZNGx06dEjNmzc/G/UDAIAGxMsYYzwZEBUVpZ49e2revHmSJKfTqbCwMI0dO1aTJk2q0H/BggWaNWuWdu/eLV9f3xoVWVBQoMDAQOXn5ysgIKBGywAAALXrXHx+e3SqqKSkRNu3b1dsbOwvC/D2VmxsrDIyMiods3btWkVHR2v06NEKDQ1Vly5dNHPmTJWVlVW5nuLiYhUUFLg9AAAAPAouubm5KisrU2hoqFt7aGiosrOzKx2TlZWlVatWqaysTKmpqXrkkUf09NNP67HHHqtyPSkpKQoMDHQ9wsLCPCkTAADUU+f8riKn06mQkBAtXLhQkZGRGjx4sCZPnqwFCxZUOSYpKUn5+fmux5EjR851mQAAwAIeXZwbHBwsHx8f5eTkuLXn5OSoZcuWlY5p1aqVfH195ePj42q77LLLlJ2drZKSEvn5+VUY43A45HA4PCkNAAA0AB4dcfHz81NkZKTS09NdbU6nU+np6YqOjq50TJ8+fbRv3z45nU5X2549e9SqVatKQwsAAEBVPD5VlJiYqEWLFunll1/Wl19+qXvvvVeFhYUaPny4JCk+Pl5JSUmu/vfee6++++47PfDAA9qzZ4/Wr1+vmTNnavTo0WdvKwAAQIPg8fe4DB48WCdOnNDUqVOVnZ2tbt26KS0tzXXB7uHDh+Xt/UseCgsL04YNGzR+/HhdeeWVatOmjR544AFNnDjx7G0FAABoEDz+Hpe6wPe4AABgnzr/HhcAAIC6RHABAADWILgAAABrEFwAAIA1CC4AAMAaBBcAAGANggsAALAGwQUAAFiD4AIAAKxBcAEAANYguAAAAGsQXAAAgDUILgAAwBoEFwAAYA2CCwAAsAbBBQAAWIPgAgAArEFwAQAA1iC4AAAAaxBcAACANQguAADAGgQXAABgDYILAACwBsEFAABYg+ACAACsQXABAADWILgAAABrEFwAAIA1CC4AAMAaBBcAAGANggsAALAGwQUAAFiD4AIAAKxBcAEAANYguAAAAGsQXAAAgDUILgAAwBoEFwAAYA2CCwAAsAbBBQAAWIPgAgAArEFwAQAA1iC4AAAAaxBcAACANQguAADAGgQXAABgDYILAACwBsEFAABYg+ACAACsQXABAADWILgAAABrEFwAAIA1CC4AAMAaBBcAAGANggsAALAGwQUAAFiD4AIAAKxBcAEAANYguAAAAGsQXAAAgDUILgAAwBoEFwAAYA2CCwAAsEaNgsv8+fMVHh4uf39/RUVFadu2bdUat3z5cnl5eWnQoEE1WS0AAGjgPA4uK1asUGJiopKTk7Vjxw517dpVcXFxOn78+GnHHTx4UH//+9/Vt2/fGhcLAAAaNo+Dy5w5c3T33Xdr+PDhuvzyy7VgwQI1adJEixcvrnJMWVmZ7rzzTk2bNk3t27c/4zqKi4tVUFDg9gAAAPAouJSUlGj79u2KjY39ZQHe3oqNjVVGRkaV46ZPn66QkBCNGDGiWutJSUlRYGCg6xEWFuZJmQAAoJ7yKLjk5uaqrKxMoaGhbu2hoaHKzs6udMzWrVv10ksvadGiRdVeT1JSkvLz812PI0eOeFImAACopxqdy4WfPHlSQ4cO1aJFixQcHFztcQ6HQw6H4xxWBgAAbORRcAkODpaPj49ycnLc2nNyctSyZcsK/ffv36+DBw9qwIABrjan0/l/K27USF999ZUiIiJqUjcAAGiAPDpV5Ofnp8jISKWnp7vanE6n0tPTFR0dXaF/p06d9NlnnykzM9P1GDhwoK677jplZmZy7QoAAPCIx6eKEhMTlZCQoB49eqhXr16aO3euCgsLNXz4cElSfHy82rRpo5SUFPn7+6tLly5u45s3by5JFdoBAADOxOPgMnjwYJ04cUJTp05Vdna2unXrprS0NNcFu4cPH5a3N1/ICwAAzj4vY4yp6yLOpKCgQIGBgcrPz1dAQEBdlwMAAKrhXHx+c2gEAABYg+ACAACsQXABAADWILgAAABrEFwAAIA1CC4AAMAaBBcAAGANggsAALAGwQUAAFiD4AIAAKxBcAEAANYguAAAAGsQXAAAgDUILgAAwBoEFwAAYA2CCwAAsAbBBQAAWIPgAgAArEFwAQAA1iC4AAAAaxBcAACANQguAADAGgQXAABgDYILAACwBsEFAABYg+ACAACsQXABAADWILgAAABrEFwAAIA1CC4AAMAaBBcAAGANggsAALAGwQUAAFiD4AIAAKxBcAEAANYguAAAAGsQXAAAgDUILgAAwBoEFwAAYA2CCwAAsAbBBQAAWIPgAgAArEFwAQAA1iC4AAAAaxBcAACANQguAADAGgQXAABgDYILAACwBsEFAABYg+ACAACsQXABAADWILgAAABrEFwAAIA1CC4AAMAaBBcAAGANggsAALAGwQUAAFiD4AIAAKxBcAEAANYguAAAAGsQXAAAgDUILgAAwBoEFwAAYI0aBZf58+crPDxc/v7+ioqK0rZt26rsu2jRIvXt21dBQUEKCgpSbGzsafsDAABUxePgsmLFCiUmJio5OVk7duxQ165dFRcXp+PHj1faf8uWLbrjjju0efNmZWRkKCwsTP369dPRo0d/c/EAAKBh8TLGGE8GREVFqWfPnpo3b54kyel0KiwsTGPHjtWkSZPOOL6srExBQUGaN2+e4uPjK+1TXFys4uJi1/OCggKFhYUpPz9fAQEBnpQLAADqSEFBgQIDA8/q57dHR1xKSkq0fft2xcbG/rIAb2/FxsYqIyOjWssoKirSzz//rN/97ndV9klJSVFgYKDrERYW5kmZAACgnvIouOTm5qqsrEyhoaFu7aGhocrOzq7WMiZOnKjWrVu7hZ9TJSUlKT8/3/U4cuSIJ2UCAIB6qlFtruyJJ57Q8uXLtWXLFvn7+1fZz+FwyOFw1GJlAADABh4Fl+DgYPn4+CgnJ8etPScnRy1btjzt2NmzZ+uJJ57Qpk2bdOWVV3peKQAAaPA8OlXk5+enyMhIpaenu9qcTqfS09MVHR1d5binnnpKM2bMUFpamnr06FHzagEAQIPm8amixMREJSQkqEePHurVq5fmzp2rwsJCDR8+XJIUHx+vNm3aKCUlRZL05JNPaurUqXr11VcVHh7uuhamadOmatq06VncFAAAUN95HFwGDx6sEydOaOrUqcrOzla3bt2UlpbmumD38OHD8vb+5UDOCy+8oJKSEt16661uy0lOTtajjz7626oHAAANisff41IXzsV94AAA4Nyq8+9xAQAAqEsEFwAAYA2CCwAAsAbBBQAAWIPgAgAArEFwAQAA1iC4AAAAaxBcAACANQguAADAGgQXAABgDYILAACwBsEFAABYg+ACAACsQXABAADWILgAAABrEFwAAIA1CC4AAMAaBBcAAGANggsAALAGwQUAAFiD4AIAAKxBcAEAANYguAAAAGsQXAAAgDUILgAAwBoEFwAAYA2CCwAAsAbBBQAAWIPgAgAArEFwAQAA1iC4AAAAaxBcAACANQguAADAGgQXAABgDYILAACwBsEFAABYg+ACAACsQXABAADWILgAAABrEFwAAIA1CC4AAMAaBBcAAGANggsAALAGwQUAAFiD4AIAAKxBcAEAANYguAAAAGsQXAAAgDUILgAAwBoEFwAAYA2CCwAAsAbBBQAAWIPgAgAArEFwAQAA1iC4AAAAaxBcAACANQguAADAGgQXAABgDYILAACwBsEFAABYg+ACAACsQXABAADWqFFwmT9/vsLDw+Xv76+oqCht27bttP1XrlypTp06yd/fX1dccYVSU1NrVCwAAGjYPA4uK1asUGJiopKTk7Vjxw517dpVcXFxOn78eKX9P/zwQ91xxx0aMWKEdu7cqUGDBmnQoEH6/PPPf3PxAACgYfEyxhhPBkRFRalnz56aN2+eJMnpdCosLExjx47VpEmTKvQfPHiwCgsL9dZbb7narr76anXr1k0LFiyo1joLCgoUGBio/Px8BQQEeFIuAACoI+fi87uRJ51LSkq0fft2JSUludq8vb0VGxurjIyMSsdkZGQoMTHRrS0uLk5r1qypcj3FxcUqLi52Pc/Pz5f0f28AAACwQ/nntofHSE7Lo+CSm5ursrIyhYaGurWHhoZq9+7dlY7Jzs6utH92dnaV60lJSdG0adMqtIeFhXlSLgAAOA98++23CgwMPCvL8ii41JakpCS3ozR5eXlq166dDh8+fNY2HDVTUFCgsLAwHTlyhNN2dYy5OH8wF+cX5uP8kZ+fr7Zt2+p3v/vdWVumR8ElODhYPj4+ysnJcWvPyclRy5YtKx3TsmVLj/pLksPhkMPhqNAeGBjITnieCAgIYC7OE8zF+YO5OL8wH+cPb++z9+0rHi3Jz89PkZGRSk9Pd7U5nU6lp6crOjq60jHR0dFu/SVp48aNVfYHAACoisenihITE5WQkKAePXqoV69emjt3rgoLCzV8+HBJUnx8vNq0aaOUlBRJ0gMPPKCYmBg9/fTT6t+/v5YvX66PP/5YCxcuPLtbAgAA6j2Pg8vgwYN14sQJTZ06VdnZ2erWrZvS0tJcF+AePnzY7ZBQ79699eqrr2rKlCl6+OGHdckll2jNmjXq0qVLtdfpcDiUnJxc6ekj1C7m4vzBXJw/mIvzC/Nx/jgXc+Hx97gAAADUFX6rCAAAWIPgAgAArEFwAQAA1iC4AAAAaxBcAACANc6b4DJ//nyFh4fL399fUVFR2rZt22n7r1y5Up06dZK/v7+uuOIKpaam1lKl9Z8nc7Fo0SL17dtXQUFBCgoKUmxs7BnnDtXn6d9FueXLl8vLy0uDBg06twU2IJ7ORV5enkaPHq1WrVrJ4XDo0ksv5f9TZ4mnczF37lx17NhRjRs3VlhYmMaPH6+ffvqplqqtv9577z0NGDBArVu3lpeX12l/PLncli1bdNVVV8nhcKhDhw5aunSp5ys254Hly5cbPz8/s3jxYrNr1y5z9913m+bNm5ucnJxK+3/wwQfGx8fHPPXUU+aLL74wU6ZMMb6+vuazzz6r5crrH0/nYsiQIWb+/Plm586d5ssvvzTDhg0zgYGB5uuvv67lyusfT+ei3IEDB0ybNm1M3759zR//+MfaKbae83QuiouLTY8ePczNN99stm7dag4cOGC2bNliMjMza7ny+sfTuVi2bJlxOBxm2bJl5sCBA2bDhg2mVatWZvz48bVcef2TmppqJk+ebFavXm0kmTfeeOO0/bOyskyTJk1MYmKi+eKLL8xzzz1nfHx8TFpamkfrPS+CS69evczo0aNdz8vKykzr1q1NSkpKpf1vu+02079/f7e2qKgoM2rUqHNaZ0Pg6VycqrS01DRr1sy8/PLL56rEBqMmc1FaWmp69+5tXnzxRZOQkEBwOUs8nYsXXnjBtG/f3pSUlNRWiQ2Gp3MxevRoc/3117u1JSYmmj59+pzTOhua6gSXCRMmmM6dO7u1DR482MTFxXm0rjo/VVRSUqLt27crNjbW1ebt7a3Y2FhlZGRUOiYjI8OtvyTFxcVV2R/VU5O5OFVRUZF+/vnns/pLoA1RTedi+vTpCgkJ0YgRI2qjzAahJnOxdu1aRUdHa/To0QoNDVWXLl00c+ZMlZWV1VbZ9VJN5qJ3797avn2763RSVlaWUlNTdfPNN9dKzfjF2frs9vgr/8+23NxclZWVuX4yoFxoaKh2795d6Zjs7OxK+2dnZ5+zOhuCmszFqSZOnKjWrVtX2DnhmZrMxdatW/XSSy8pMzOzFipsOGoyF1lZWXr33Xd15513KjU1Vfv27dN9992nn3/+WcnJybVRdr1Uk7kYMmSIcnNzdc0118gYo9LSUt1zzz16+OGHa6Nk/EpVn90FBQX68ccf1bhx42otp86PuKD+eOKJJ7R8+XK98cYb8vf3r+tyGpSTJ09q6NChWrRokYKDg+u6nAbP6XQqJCRECxcuVGRkpAYPHqzJkydrwYIFdV1ag7NlyxbNnDlTzz//vHbs2KHVq1dr/fr1mjFjRl2Xhhqq8yMuwcHB8vHxUU5Ojlt7Tk6OWrZsWemYli1betQf1VOTuSg3e/ZsPfHEE9q0aZOuvPLKc1lmg+DpXOzfv18HDx7UgAEDXG1Op1OS1KhRI3311VeKiIg4t0XXUzX5u2jVqpV8fX3l4+PjarvsssuUnZ2tkpIS+fn5ndOa66uazMUjjzyioUOH6q677pIkXXHFFSosLNTIkSM1efJktx8FxrlV1Wd3QEBAtY+2SOfBERc/Pz9FRkYqPT3d1eZ0OpWenq7o6OhKx0RHR7v1l6SNGzdW2R/VU5O5kKSnnnpKM2bMUFpamnr06FEbpdZ7ns5Fp06d9NlnnykzM9P1GDhwoK677jplZmYqLCysNsuvV2ryd9GnTx/t27fPFR4lac+ePWrVqhWh5TeoyVwUFRVVCCflgdLwG8O16qx9dnt23fC5sXz5cuNwOMzSpUvNF198YUaOHGmaN29usrOzjTHGDB061EyaNMnV/4MPPjCNGjUys2fPNl9++aVJTk7mduizxNO5eOKJJ4yfn59ZtWqVOXbsmOtx8uTJutqEesPTuTgVdxWdPZ7OxeHDh02zZs3MmDFjzFdffWXeeustExISYh577LG62oR6w9O5SE5ONs2aNTOvvfaaycrKMu+8846JiIgwt912W11tQr1x8uRJs3PnTrNz504jycyZM8fs3LnTHDp0yBhjzKRJk8zQoUNd/ctvh37ooYfMl19+aebPn2/v7dDGGPPcc8+Ztm3bGj8/P9OrVy/zn//8x/VaTEyMSUhIcOv/+uuvm0svvdT4+fmZzp07m/Xr19dyxfWXJ3PRrl07I6nCIzk5ufYLr4c8/bv4NYLL2eXpXHz44YcmKirKOBwO0759e/P444+b0tLSWq66fvJkLn7++Wfz6KOPmoiICOPv72/CwsLMfffdZ77//vvaL7ye2bx5c6X//y9//xMSEkxMTEyFMd26dTN+fn6mffv2ZsmSJR6v18sYjpUBAAA71Pk1LgAAANVFcAEAANYguAAAAGsQXAAAgDUILgAAwBoEFwAAYA2CCwAAsAbBBQAAWIPgAgAArEFwAQAA1iC4AAAAa/w/ftiVzvrzq/kAAAAASUVORK5CYII=\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "score = model.evaluate(X_test, Y_test, verbose=1, batch_size=batch_size)\n",
        "print(score)"
      ],
      "metadata": {
        "id": "h5LN7Xg_3h-q",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2dd00a88-c358-4fc3-9f92-42c90d9e79c4"
      },
      "execution_count": 68,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "88/88 [==============================] - 1s 6ms/step - loss: 1.2925 - accuracy: 0.5018\n",
            "[1.2924607992172241, 0.5018181800842285]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "acc = {}\n",
        "acc_mod_snr = np.zeros( (len(classes),len(snrs)) )\n",
        "i = 0\n",
        "!mkdir figure2\n",
        "!mkdir predictresult2\n",
        "def srn_plot():#save acc for mod per SNR\n",
        "    def predict(model):\n",
        "    # Plot confusion matrix\n",
        "      model.load_weights(filepath)\n",
        "      test_Y_hat = model.predict(X_test, batch_size=batch_size)\n",
        "      confnorm,_,_ = mltools.calculate_confusion_matrix(Y_test,test_Y_hat,classes)\n",
        "      mltools.plot_confusion_matrix(confnorm, labels=['8PSK','AM-DSB','AM-SSB','BPSK','CPFSK','GFSK','4-PAM','16-QAM','64-QAM','QPSK','WBFM'],save_filename='figure2/cnn2_total_confusion')\n",
        "\n",
        "    # Plot confusion matrix\n",
        "    acc = {}\n",
        "    acc_mod_snr = np.zeros( (len(classes),len(snrs)) )\n",
        "    i = 0\n",
        "    for snr in snrs:\n",
        "\n",
        "        # extract classes @ SNR\n",
        "        # test_SNRs = map(lambda x: lbl[x][1], test_idx)\n",
        "        test_SNRs = [lbl[x][1] for x in test_idx]\n",
        "\n",
        "        test_X_i = X_test[np.where(np.array(test_SNRs) == snr)]\n",
        "        test_Y_i = Y_test[np.where(np.array(test_SNRs) == snr)]\n",
        "\n",
        "        # estimate classes\n",
        "        test_Y_i_hat = model.predict(test_X_i)\n",
        "        confnorm_i,cor,ncor = mltools.calculate_confusion_matrix(test_Y_i,test_Y_i_hat,classes)\n",
        "        acc[snr] = 1.0 * cor / (cor + ncor)\n",
        "        result = cor / (cor + ncor)\n",
        "        with open('acc222.csv', 'a', newline='') as f0:\n",
        "            write0 = csv.writer(f0)\n",
        "            write0.writerow([result])\n",
        "        mltools.plot_confusion_matrix(confnorm_i, labels=['8PSK','AM-DSB','AM-SSB','BPSK','CPFSK','GFSK','4-PAM','16-QAM','64-QAM','QPSK','WBFM'], title=\"Confusion Matrix\",save_filename=\"figure2/Confusion(SNR=%d)(ACC=%2f).png\" % (snr,100.0*acc[snr]))\n",
        "\n",
        "        acc_mod_snr[:,i] = np.round(np.diag(confnorm_i)/np.sum(confnorm_i,axis=1),3)\n",
        "        i = i +1\n",
        "\n",
        "    #plot acc of each mod in one picture\n",
        "    dis_num=11\n",
        "    for g in range(int(np.ceil(acc_mod_snr.shape[0]/dis_num))):\n",
        "        assert (0 <= dis_num <= acc_mod_snr.shape[0])\n",
        "        beg_index = g*dis_num\n",
        "        end_index = np.min([(g+1)*dis_num,acc_mod_snr.shape[0]])\n",
        "\n",
        "        plt.figure(figsize=(12, 10))\n",
        "        plt.xlabel(\"Signal to Noise Ratio\")\n",
        "        plt.ylabel(\"Classification Accuracy\")\n",
        "        plt.title(\"Classification Accuracy for Each Mod\")\n",
        "\n",
        "        for i in range(beg_index,end_index):\n",
        "            plt.plot(snrs, acc_mod_snr[i], label=classes[i])\n",
        "            # 设置数字标签\n",
        "            for x, y in zip(snrs, acc_mod_snr[i]):\n",
        "                plt.text(x, y, y, ha='center', va='bottom', fontsize=8)\n",
        "\n",
        "        plt.legend()\n",
        "        plt.grid()\n",
        "        plt.savefig('figure2/acc_with_mod_{}.png'.format(g+1))\n",
        "        plt.close()\n",
        "    #save acc for mod per SNR\n",
        "    fd = open('predictresult2/acc_for_mod_on_cnn2.dat', 'wb')\n",
        "    pickle.dump(('128','cnn2', acc_mod_snr), fd)\n",
        "    fd.close()\n",
        "\n",
        "    # Save results to a pickle file for plotting later\n",
        "    print(acc)\n",
        "    fd = open('predictresult2/cnn2_d0.5.dat','wb')\n",
        "    pickle.dump( (\"CNN2\", 0.5, acc) , fd )\n",
        "\n",
        "    # Plot accuracy curve\n",
        "    plt.plot(snrs, list(map(lambda x: acc[x], snrs)))\n",
        "    plt.xlabel(\"Signal to Noise Ratio\")\n",
        "    plt.ylabel(\"Classification Accuracy\")\n",
        "    plt.title(\"Classification Accuracy on RadioML 2016.10 Alpha\")\n",
        "    plt.tight_layout()\n",
        "    plt.savefig('figure2/each_acc.png')\n",
        "    plt.close()\n",
        "srn_plot()"
      ],
      "metadata": {
        "id": "E-imgUtIIBbm",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c95ad901-3563-4e9d-9e60-53dd753d63a0"
      },
      "execution_count": 69,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "mkdir: cannot create directory ‘figure2’: File exists\n",
            "mkdir: cannot create directory ‘predictresult2’: File exists\n",
            "69/69 [==============================] - 0s 2ms/step\n",
            "69/69 [==============================] - 0s 2ms/step\n",
            "69/69 [==============================] - 0s 3ms/step\n",
            "69/69 [==============================] - 0s 2ms/step\n",
            "69/69 [==============================] - 0s 2ms/step\n",
            "69/69 [==============================] - 0s 2ms/step\n",
            "69/69 [==============================] - 0s 2ms/step\n",
            "69/69 [==============================] - 0s 3ms/step\n",
            "69/69 [==============================] - 0s 2ms/step\n",
            "69/69 [==============================] - 0s 2ms/step\n",
            "69/69 [==============================] - 0s 3ms/step\n",
            "69/69 [==============================] - 0s 2ms/step\n",
            "69/69 [==============================] - 0s 2ms/step\n",
            "69/69 [==============================] - 0s 2ms/step\n",
            "69/69 [==============================] - 0s 2ms/step\n",
            "69/69 [==============================] - 0s 2ms/step\n",
            "69/69 [==============================] - 0s 2ms/step\n",
            "69/69 [==============================] - 0s 2ms/step\n",
            "69/69 [==============================] - 0s 2ms/step\n",
            "69/69 [==============================] - 0s 3ms/step\n",
            "{-20: 0.09, -18: 0.09272727272727273, -16: 0.09727272727272727, -14: 0.11090909090909092, -12: 0.12590909090909091, -10: 0.1918181818181818, -8: 0.3145454545454546, -6: 0.4727272727272727, -4: 0.5718181818181818, -2: 0.6695454545454546, 0: 0.7109090909090909, 2: 0.7263636363636363, 4: 0.73, 6: 0.7227272727272728, 8: 0.7381818181818182, 10: 0.7545454545454545, 12: 0.7268181818181818, 14: 0.7331818181818182, 16: 0.73, 18: 0.7263636363636363}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "mZaQeRu7l6WN"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}