Tune-model-comparison.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from cv2 import cv2\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from pathlib import Path\n",
    "import os.path\n",
    "\n",
    "from tensorflow.keras.preprocessing import image\n",
    "from tensorflow.keras.applications import MobileNet\n",
    "from tensorflow.keras.layers import Dense\n",
    "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras import layers, models\n",
    "\n",
    "from pprint import pprint as pp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num GPUs Available:  1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-11-23 18:58:04.772601: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:939] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2021-11-23 18:58:04.779660: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:939] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2021-11-23 18:58:04.780069: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:939] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n"
     ]
    }
   ],
   "source": [
    "print(\"Num GPUs Available: \", len(tf.config.list_physical_devices('GPU')))\n",
    "physical_devices = tf.config.list_physical_devices(\"GPU\")\n",
    "tf.config.experimental.set_memory_growth(physical_devices[0], True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create Data df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "TRAIN_DIR = 'DATASET_A_AP/train'\n",
    "TEST_DIR = 'DATASET_A_AP/test'\n",
    "\n",
    "# lr = 0.001\n",
    "lr = 0.001\n",
    "epoch = 10\n",
    "batch_size = 128\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert folder to dataframe of images' paths & labels\n",
    "def get_paths_labels(path, allowed_extension=\"jpg\"):\n",
    "        global Path\n",
    "        images_dir = Path(path)\n",
    "        \n",
    "        filepaths = pd.Series((images_dir.glob(fr'**/*.{allowed_extension}'))).astype(str)\n",
    "        filepaths.name = \"path\"\n",
    "        \n",
    "        labels = filepaths.str.split(\"/\")[:].str[-2]\n",
    "        labels.name = \"label\"\n",
    "\n",
    "        # Concatenate filepaths and labels\n",
    "        df = pd.concat([filepaths, labels], axis=1)\n",
    "\n",
    "        # Shuffle the DataFrame and reset index\n",
    "        df = df.sample(frac=1).reset_index(drop = True)\n",
    "        return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = get_paths_labels(TRAIN_DIR)\n",
    "test_df = get_paths_labels(TEST_DIR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>path</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DATASET_A_AP/train/X/TRAIN_X_828.jpg</td>\n",
       "      <td>X</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DATASET_A_AP/train/X/TRAIN_X_559.jpg</td>\n",
       "      <td>X</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>DATASET_A_AP/train/S/TRAIN_S_2067.jpg</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>DATASET_A_AP/train/I/TRAIN_I_90.jpg</td>\n",
       "      <td>I</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>DATASET_A_AP/train/U/TRAIN_U_830.jpg</td>\n",
       "      <td>U</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57712</th>\n",
       "      <td>DATASET_A_AP/train/A/TRAIN_A_1352.jpg</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57713</th>\n",
       "      <td>DATASET_A_AP/train/C/TRAIN_C_1108.jpg</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57714</th>\n",
       "      <td>DATASET_A_AP/train/Z/TRAIN_Z_1429.jpg</td>\n",
       "      <td>Z</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57715</th>\n",
       "      <td>DATASET_A_AP/train/Y/TRAIN_Y_1775.jpg</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57716</th>\n",
       "      <td>DATASET_A_AP/train/D/TRAIN_D_2402.jpg</td>\n",
       "      <td>D</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>57717 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        path label\n",
       "0       DATASET_A_AP/train/X/TRAIN_X_828.jpg     X\n",
       "1       DATASET_A_AP/train/X/TRAIN_X_559.jpg     X\n",
       "2      DATASET_A_AP/train/S/TRAIN_S_2067.jpg     S\n",
       "3        DATASET_A_AP/train/I/TRAIN_I_90.jpg     I\n",
       "4       DATASET_A_AP/train/U/TRAIN_U_830.jpg     U\n",
       "...                                      ...   ...\n",
       "57712  DATASET_A_AP/train/A/TRAIN_A_1352.jpg     A\n",
       "57713  DATASET_A_AP/train/C/TRAIN_C_1108.jpg     C\n",
       "57714  DATASET_A_AP/train/Z/TRAIN_Z_1429.jpg     Z\n",
       "57715  DATASET_A_AP/train/Y/TRAIN_Y_1775.jpg     Y\n",
       "57716  DATASET_A_AP/train/D/TRAIN_D_2402.jpg     D\n",
       "\n",
       "[57717 rows x 2 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create data generator and use flow_from_dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 46174 validated image filenames belonging to 26 classes.\n",
      "Found 11543 validated image filenames belonging to 26 classes.\n",
      "Found 14430 validated image filenames belonging to 26 classes.\n"
     ]
    }
   ],
   "source": [
    "data_generator = ImageDataGenerator(validation_split=0.2, rescale=1./255.,\n",
    "                                   zoom_range=0.1,\n",
    "                                    width_shift_range=0.1,height_shift_range=0.1,\n",
    "                                    shear_range=0.1,\n",
    "                                   )\n",
    "test_generator = ImageDataGenerator(rescale=1./255.)\n",
    "\n",
    "train_images = data_generator.flow_from_dataframe(\n",
    "    dataframe=train_df,\n",
    "    x_col='path',\n",
    "    y_col='label',\n",
    "    target_size=(28, 28),\n",
    "    color_mode='grayscale',\n",
    "    class_mode='categorical',\n",
    "    batch_size=batch_size,\n",
    "    shuffle=True,\n",
    "    subset='training'\n",
    ")\n",
    "\n",
    "val_images = data_generator.flow_from_dataframe(\n",
    "    dataframe=train_df,\n",
    "    x_col='path',\n",
    "    y_col='label',\n",
    "    target_size=(28, 28),\n",
    "    color_mode='grayscale',\n",
    "    class_mode='categorical',\n",
    "    batch_size=batch_size,\n",
    "    shuffle=True,\n",
    "    subset='validation'\n",
    ")\n",
    "\n",
    "\n",
    "test_images = test_generator.flow_from_dataframe(\n",
    "    dataframe=test_df,\n",
    "    x_col='path',\n",
    "    y_col='label',\n",
    "    target_size=(28, 28),\n",
    "    color_mode='grayscale',\n",
    "    class_mode='categorical',\n",
    "    # batch_size=1,\n",
    "    shuffle=False,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Compare Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model candidate = 12\n",
      "\n",
      "model_func_name : get_model_1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-11-23 00:42:07.665616: I tensorflow/core/platform/cpu_feature_guard.cc:151] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2021-11-23 00:42:07.667011: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:939] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2021-11-23 00:42:07.668050: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:939] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2021-11-23 00:42:07.668966: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:939] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2021-11-23 00:42:08.073917: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:939] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2021-11-23 00:42:08.074362: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:939] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2021-11-23 00:42:08.074760: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:939] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2021-11-23 00:42:08.075149: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1525] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 5592 MB memory:  -> device: 0, name: NVIDIA GeForce RTX 2080 Ti, pci bus id: 0000:09:00.0, compute capability: 7.5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Training...\n",
      "Epoch 1/10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-11-23 00:42:09.214791: I tensorflow/stream_executor/cuda/cuda_dnn.cc:366] Loaded cuDNN version 8204\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "361/361 [==============================] - 23s 57ms/step - loss: 0.5710 - accuracy: 0.8293 - val_loss: 2.8451 - val_accuracy: 0.2148\n",
      "Epoch 2/10\n",
      "361/361 [==============================] - 21s 57ms/step - loss: 0.1574 - accuracy: 0.9524 - val_loss: 0.1902 - val_accuracy: 0.9415\n",
      "Epoch 3/10\n",
      "361/361 [==============================] - 21s 57ms/step - loss: 0.1032 - accuracy: 0.9676 - val_loss: 0.1089 - val_accuracy: 0.9674\n",
      "Epoch 4/10\n",
      "361/361 [==============================] - 20s 57ms/step - loss: 0.0823 - accuracy: 0.9742 - val_loss: 0.1194 - val_accuracy: 0.9626\n",
      "Epoch 5/10\n",
      "361/361 [==============================] - 20s 57ms/step - loss: 0.0686 - accuracy: 0.9782 - val_loss: 0.0632 - val_accuracy: 0.9800\n",
      "Epoch 6/10\n",
      "361/361 [==============================] - 21s 57ms/step - loss: 0.0605 - accuracy: 0.9806 - val_loss: 0.0813 - val_accuracy: 0.9736\n",
      "Epoch 7/10\n",
      "361/361 [==============================] - 21s 57ms/step - loss: 0.0521 - accuracy: 0.9834 - val_loss: 0.0558 - val_accuracy: 0.9835\n",
      "Epoch 8/10\n",
      "361/361 [==============================] - 21s 57ms/step - loss: 0.0527 - accuracy: 0.9838 - val_loss: 0.1578 - val_accuracy: 0.9524\n",
      "Epoch 9/10\n",
      "361/361 [==============================] - 21s 57ms/step - loss: 0.0486 - accuracy: 0.9844 - val_loss: 0.1432 - val_accuracy: 0.9584\n",
      "Epoch 10/10\n",
      "361/361 [==============================] - 20s 57ms/step - loss: 0.0437 - accuracy: 0.9859 - val_loss: 0.0820 - val_accuracy: 0.9757\n",
      "\n",
      "Evaluating...\n",
      "451/451 [==============================] - 4s 9ms/step - loss: 0.4994 - accuracy: 0.9002\n",
      "get_model_1 : test loss = 0.5, test acc= 0.9\n",
      "\n",
      "model_func_name : get_model_2\n",
      "\n",
      "Training...\n",
      "Epoch 1/10\n",
      "361/361 [==============================] - 21s 57ms/step - loss: 0.7623 - accuracy: 0.7724 - val_loss: 2.6301 - val_accuracy: 0.2248\n",
      "Epoch 2/10\n",
      "361/361 [==============================] - 21s 57ms/step - loss: 0.2227 - accuracy: 0.9329 - val_loss: 0.1671 - val_accuracy: 0.9508\n",
      "Epoch 3/10\n",
      "361/361 [==============================] - 21s 57ms/step - loss: 0.1489 - accuracy: 0.9540 - val_loss: 0.3640 - val_accuracy: 0.8941\n",
      "Epoch 4/10\n",
      "361/361 [==============================] - 20s 57ms/step - loss: 0.1161 - accuracy: 0.9649 - val_loss: 0.1085 - val_accuracy: 0.9666\n",
      "Epoch 5/10\n",
      "361/361 [==============================] - 20s 57ms/step - loss: 0.1019 - accuracy: 0.9693 - val_loss: 0.1193 - val_accuracy: 0.9627\n",
      "Epoch 6/10\n",
      "361/361 [==============================] - 21s 57ms/step - loss: 0.0849 - accuracy: 0.9741 - val_loss: 0.0949 - val_accuracy: 0.9705\n",
      "Epoch 7/10\n",
      "361/361 [==============================] - 21s 57ms/step - loss: 0.0775 - accuracy: 0.9765 - val_loss: 0.2691 - val_accuracy: 0.9264\n",
      "Epoch 8/10\n",
      "361/361 [==============================] - 21s 57ms/step - loss: 0.0729 - accuracy: 0.9777 - val_loss: 0.0595 - val_accuracy: 0.9822\n",
      "Epoch 9/10\n",
      "361/361 [==============================] - 21s 57ms/step - loss: 0.0671 - accuracy: 0.9805 - val_loss: 0.0961 - val_accuracy: 0.9711\n",
      "Epoch 10/10\n",
      "361/361 [==============================] - 20s 57ms/step - loss: 0.0601 - accuracy: 0.9817 - val_loss: 0.0899 - val_accuracy: 0.9723\n",
      "\n",
      "Evaluating...\n",
      "451/451 [==============================] - 4s 9ms/step - loss: 0.5791 - accuracy: 0.8926\n",
      "get_model_2 : test loss = 0.58, test acc= 0.89\n",
      "\n",
      "model_func_name : get_model_3\n",
      "\n",
      "Training...\n",
      "Epoch 1/10\n",
      "361/361 [==============================] - 21s 57ms/step - loss: 1.6225 - accuracy: 0.6394 - val_loss: 3.8195 - val_accuracy: 0.1835\n",
      "Epoch 2/10\n",
      "361/361 [==============================] - 20s 56ms/step - loss: 0.6076 - accuracy: 0.8457 - val_loss: 0.3045 - val_accuracy: 0.9276\n",
      "Epoch 3/10\n",
      "361/361 [==============================] - 20s 56ms/step - loss: 0.3394 - accuracy: 0.9079 - val_loss: 0.1612 - val_accuracy: 0.9601\n",
      "Epoch 4/10\n",
      "361/361 [==============================] - 21s 57ms/step - loss: 0.2456 - accuracy: 0.9331 - val_loss: 0.1167 - val_accuracy: 0.9704\n",
      "Epoch 5/10\n",
      "361/361 [==============================] - 21s 57ms/step - loss: 0.1975 - accuracy: 0.9451 - val_loss: 0.1540 - val_accuracy: 0.9674\n",
      "Epoch 6/10\n",
      "361/361 [==============================] - 20s 57ms/step - loss: 0.1619 - accuracy: 0.9544 - val_loss: 0.0944 - val_accuracy: 0.9776\n",
      "Epoch 7/10\n",
      "361/361 [==============================] - 20s 56ms/step - loss: 0.1490 - accuracy: 0.9589 - val_loss: 0.0770 - val_accuracy: 0.9796\n",
      "Epoch 8/10\n",
      "361/361 [==============================] - 20s 57ms/step - loss: 0.1353 - accuracy: 0.9622 - val_loss: 0.0966 - val_accuracy: 0.9726\n",
      "Epoch 9/10\n",
      "361/361 [==============================] - 21s 57ms/step - loss: 0.1346 - accuracy: 0.9628 - val_loss: 0.0720 - val_accuracy: 0.9811\n",
      "Epoch 10/10\n",
      "361/361 [==============================] - 21s 58ms/step - loss: 0.1216 - accuracy: 0.9672 - val_loss: 0.0676 - val_accuracy: 0.9829\n",
      "\n",
      "Evaluating...\n",
      "451/451 [==============================] - 4s 9ms/step - loss: 0.3563 - accuracy: 0.9314\n",
      "get_model_3 : test loss = 0.36, test acc= 0.93\n",
      "\n",
      "model_func_name : get_stn_a_model_1\n",
      "\n",
      "Training...\n",
      "Epoch 1/10\n",
      "361/361 [==============================] - 23s 62ms/step - loss: 1.0861 - accuracy: 0.6865 - val_loss: 0.3071 - val_accuracy: 0.9123\n",
      "Epoch 2/10\n",
      "361/361 [==============================] - 23s 64ms/step - loss: 0.3209 - accuracy: 0.9075 - val_loss: 0.1487 - val_accuracy: 0.9582\n",
      "Epoch 3/10\n",
      "361/361 [==============================] - 25s 68ms/step - loss: 0.2136 - accuracy: 0.9361 - val_loss: 0.1035 - val_accuracy: 0.9720\n",
      "Epoch 4/10\n",
      "361/361 [==============================] - 25s 68ms/step - loss: 0.1731 - accuracy: 0.9486 - val_loss: 0.0789 - val_accuracy: 0.9790\n",
      "Epoch 5/10\n",
      "361/361 [==============================] - 24s 65ms/step - loss: 0.1462 - accuracy: 0.9567 - val_loss: 0.0696 - val_accuracy: 0.9799\n",
      "Epoch 6/10\n",
      "361/361 [==============================] - 24s 67ms/step - loss: 0.1275 - accuracy: 0.9615 - val_loss: 0.0623 - val_accuracy: 0.9832\n",
      "Epoch 7/10\n",
      "361/361 [==============================] - 22s 61ms/step - loss: 0.1139 - accuracy: 0.9660 - val_loss: 0.0543 - val_accuracy: 0.9837\n",
      "Epoch 8/10\n",
      "361/361 [==============================] - 22s 61ms/step - loss: 0.1044 - accuracy: 0.9677 - val_loss: 0.0563 - val_accuracy: 0.9859\n",
      "Epoch 9/10\n",
      "361/361 [==============================] - 22s 61ms/step - loss: 0.0911 - accuracy: 0.9712 - val_loss: 0.0597 - val_accuracy: 0.9828\n",
      "Epoch 10/10\n",
      "361/361 [==============================] - 22s 61ms/step - loss: 0.0936 - accuracy: 0.9715 - val_loss: 0.0447 - val_accuracy: 0.9883\n",
      "\n",
      "Evaluating...\n",
      "451/451 [==============================] - 5s 11ms/step - loss: 0.5410 - accuracy: 0.8800\n",
      "get_stn_a_model_1 : test loss = 0.54, test acc= 0.88\n",
      "\n",
      "model_func_name : get_stn_a_model_2\n",
      "\n",
      "Training...\n",
      "Epoch 1/10\n",
      "361/361 [==============================] - 23s 62ms/step - loss: 0.9169 - accuracy: 0.7334 - val_loss: 0.2053 - val_accuracy: 0.9440\n",
      "Epoch 2/10\n",
      "361/361 [==============================] - 22s 60ms/step - loss: 0.2609 - accuracy: 0.9254 - val_loss: 0.1136 - val_accuracy: 0.9680\n",
      "Epoch 3/10\n",
      "361/361 [==============================] - 23s 63ms/step - loss: 0.1743 - accuracy: 0.9489 - val_loss: 0.0882 - val_accuracy: 0.9753\n",
      "Epoch 4/10\n",
      "361/361 [==============================] - 22s 60ms/step - loss: 0.1358 - accuracy: 0.9600 - val_loss: 0.0670 - val_accuracy: 0.9811\n",
      "Epoch 5/10\n",
      "361/361 [==============================] - 22s 61ms/step - loss: 0.1170 - accuracy: 0.9660 - val_loss: 0.0584 - val_accuracy: 0.9844\n",
      "Epoch 6/10\n",
      "361/361 [==============================] - 22s 61ms/step - loss: 0.0992 - accuracy: 0.9706 - val_loss: 0.0495 - val_accuracy: 0.9863\n",
      "Epoch 7/10\n",
      "361/361 [==============================] - 22s 60ms/step - loss: 0.0880 - accuracy: 0.9734 - val_loss: 0.0462 - val_accuracy: 0.9868\n",
      "Epoch 8/10\n",
      "361/361 [==============================] - 22s 61ms/step - loss: 0.0831 - accuracy: 0.9751 - val_loss: 0.0402 - val_accuracy: 0.9890\n",
      "Epoch 9/10\n",
      "361/361 [==============================] - 22s 61ms/step - loss: 0.0745 - accuracy: 0.9780 - val_loss: 0.0408 - val_accuracy: 0.9893\n",
      "Epoch 10/10\n",
      "361/361 [==============================] - 22s 60ms/step - loss: 0.0758 - accuracy: 0.9772 - val_loss: 0.0364 - val_accuracy: 0.9912\n",
      "\n",
      "Evaluating...\n",
      "451/451 [==============================] - 5s 11ms/step - loss: 0.6142 - accuracy: 0.8671\n",
      "get_stn_a_model_2 : test loss = 0.61, test acc= 0.87\n",
      "\n",
      "model_func_name : get_stn_a_model_3\n",
      "\n",
      "Training...\n",
      "Epoch 1/10\n",
      "361/361 [==============================] - 23s 61ms/step - loss: 0.7314 - accuracy: 0.7901 - val_loss: 0.1595 - val_accuracy: 0.9556\n",
      "Epoch 2/10\n",
      "361/361 [==============================] - 22s 60ms/step - loss: 0.2060 - accuracy: 0.9397 - val_loss: 0.0852 - val_accuracy: 0.9781\n",
      "Epoch 3/10\n",
      "361/361 [==============================] - 22s 61ms/step - loss: 0.1356 - accuracy: 0.9596 - val_loss: 0.0647 - val_accuracy: 0.9835\n",
      "Epoch 4/10\n",
      "361/361 [==============================] - 22s 61ms/step - loss: 0.1045 - accuracy: 0.9681 - val_loss: 0.0487 - val_accuracy: 0.9867\n",
      "Epoch 5/10\n",
      "361/361 [==============================] - 22s 61ms/step - loss: 0.0894 - accuracy: 0.9732 - val_loss: 0.0411 - val_accuracy: 0.9893\n",
      "Epoch 6/10\n",
      "361/361 [==============================] - 22s 61ms/step - loss: 0.0774 - accuracy: 0.9768 - val_loss: 0.0355 - val_accuracy: 0.9900\n",
      "Epoch 7/10\n",
      "361/361 [==============================] - 22s 61ms/step - loss: 0.0684 - accuracy: 0.9792 - val_loss: 0.0320 - val_accuracy: 0.9918\n",
      "Epoch 8/10\n",
      "361/361 [==============================] - 22s 61ms/step - loss: 0.0610 - accuracy: 0.9821 - val_loss: 0.0288 - val_accuracy: 0.9916\n",
      "Epoch 9/10\n",
      "361/361 [==============================] - 22s 62ms/step - loss: 0.0583 - accuracy: 0.9819 - val_loss: 0.0278 - val_accuracy: 0.9929\n",
      "Epoch 10/10\n",
      "361/361 [==============================] - 22s 61ms/step - loss: 0.0536 - accuracy: 0.9836 - val_loss: 0.0311 - val_accuracy: 0.9910\n",
      "\n",
      "Evaluating...\n",
      "451/451 [==============================] - 5s 11ms/step - loss: 0.5837 - accuracy: 0.8801\n",
      "get_stn_a_model_3 : test loss = 0.58, test acc= 0.88\n",
      "\n",
      "model_func_name : get_stn_a_model_4\n",
      "\n",
      "Training...\n",
      "Epoch 1/10\n",
      "361/361 [==============================] - 24s 63ms/step - loss: 0.5714 - accuracy: 0.8374 - val_loss: 0.1224 - val_accuracy: 0.9679\n",
      "Epoch 2/10\n",
      "361/361 [==============================] - 22s 62ms/step - loss: 0.1458 - accuracy: 0.9572 - val_loss: 0.0654 - val_accuracy: 0.9822\n",
      "Epoch 3/10\n",
      "361/361 [==============================] - 22s 62ms/step - loss: 0.0878 - accuracy: 0.9743 - val_loss: 0.0422 - val_accuracy: 0.9889\n",
      "Epoch 4/10\n",
      "361/361 [==============================] - 23s 62ms/step - loss: 0.0682 - accuracy: 0.9796 - val_loss: 0.0460 - val_accuracy: 0.9877\n",
      "Epoch 5/10\n",
      "361/361 [==============================] - 22s 61ms/step - loss: 0.0606 - accuracy: 0.9817 - val_loss: 0.0344 - val_accuracy: 0.9905\n",
      "Epoch 6/10\n",
      "361/361 [==============================] - 22s 62ms/step - loss: 0.0526 - accuracy: 0.9832 - val_loss: 0.0266 - val_accuracy: 0.9919\n",
      "Epoch 7/10\n",
      "361/361 [==============================] - 22s 61ms/step - loss: 0.0429 - accuracy: 0.9871 - val_loss: 0.0321 - val_accuracy: 0.9914\n",
      "Epoch 8/10\n",
      "361/361 [==============================] - 22s 61ms/step - loss: 0.0411 - accuracy: 0.9869 - val_loss: 0.0270 - val_accuracy: 0.9926\n",
      "Epoch 9/10\n",
      "361/361 [==============================] - 22s 61ms/step - loss: 0.0348 - accuracy: 0.9891 - val_loss: 0.0264 - val_accuracy: 0.9921\n",
      "Epoch 10/10\n",
      "361/361 [==============================] - 22s 62ms/step - loss: 0.0304 - accuracy: 0.9909 - val_loss: 0.0268 - val_accuracy: 0.9925\n",
      "\n",
      "Evaluating...\n",
      "451/451 [==============================] - 5s 11ms/step - loss: 0.5958 - accuracy: 0.8911\n",
      "get_stn_a_model_4 : test loss = 0.6, test acc= 0.89\n",
      "\n",
      "model_func_name : get_stn_a_model_5\n",
      "\n",
      "Training...\n",
      "Epoch 1/10\n",
      "361/361 [==============================] - 25s 66ms/step - loss: 0.6951 - accuracy: 0.7989 - val_loss: 0.1699 - val_accuracy: 0.9518\n",
      "Epoch 2/10\n",
      "361/361 [==============================] - 24s 68ms/step - loss: 0.1911 - accuracy: 0.9436 - val_loss: 0.0783 - val_accuracy: 0.9787\n",
      "Epoch 3/10\n",
      "361/361 [==============================] - 24s 67ms/step - loss: 0.1245 - accuracy: 0.9634 - val_loss: 0.0578 - val_accuracy: 0.9846\n",
      "Epoch 4/10\n",
      "361/361 [==============================] - 24s 68ms/step - loss: 0.0942 - accuracy: 0.9721 - val_loss: 0.0435 - val_accuracy: 0.9894\n",
      "Epoch 5/10\n",
      "361/361 [==============================] - 23s 64ms/step - loss: 0.0786 - accuracy: 0.9762 - val_loss: 0.0370 - val_accuracy: 0.9906\n",
      "Epoch 6/10\n",
      "361/361 [==============================] - 22s 62ms/step - loss: 0.0638 - accuracy: 0.9801 - val_loss: 0.0334 - val_accuracy: 0.9902\n",
      "Epoch 7/10\n",
      "361/361 [==============================] - 22s 62ms/step - loss: 0.0606 - accuracy: 0.9816 - val_loss: 0.0366 - val_accuracy: 0.9893\n",
      "Epoch 8/10\n",
      "361/361 [==============================] - 22s 62ms/step - loss: 0.0515 - accuracy: 0.9844 - val_loss: 0.0348 - val_accuracy: 0.9900\n",
      "Epoch 9/10\n",
      "361/361 [==============================] - 22s 61ms/step - loss: 0.0485 - accuracy: 0.9851 - val_loss: 0.0281 - val_accuracy: 0.9919\n",
      "Epoch 10/10\n",
      "361/361 [==============================] - 22s 62ms/step - loss: 0.0419 - accuracy: 0.9875 - val_loss: 0.0274 - val_accuracy: 0.9923\n",
      "\n",
      "Evaluating...\n",
      "451/451 [==============================] - 5s 11ms/step - loss: 0.5059 - accuracy: 0.9014\n",
      "get_stn_a_model_5 : test loss = 0.51, test acc= 0.9\n",
      "\n",
      "model_func_name : get_stn_a_model_6\n",
      "\n",
      "Training...\n",
      "Epoch 1/10\n",
      "361/361 [==============================] - 23s 62ms/step - loss: 0.5144 - accuracy: 0.8518 - val_loss: 0.1111 - val_accuracy: 0.9681\n",
      "Epoch 2/10\n",
      "361/361 [==============================] - 22s 62ms/step - loss: 0.1193 - accuracy: 0.9648 - val_loss: 0.0658 - val_accuracy: 0.9817\n",
      "Epoch 3/10\n",
      "361/361 [==============================] - 22s 62ms/step - loss: 0.0775 - accuracy: 0.9766 - val_loss: 0.0445 - val_accuracy: 0.9875\n",
      "Epoch 4/10\n",
      "361/361 [==============================] - 22s 62ms/step - loss: 0.0595 - accuracy: 0.9819 - val_loss: 0.0401 - val_accuracy: 0.9885\n",
      "Epoch 5/10\n",
      "361/361 [==============================] - 22s 62ms/step - loss: 0.0472 - accuracy: 0.9859 - val_loss: 0.0402 - val_accuracy: 0.9887\n",
      "Epoch 6/10\n",
      "361/361 [==============================] - 22s 62ms/step - loss: 0.0387 - accuracy: 0.9875 - val_loss: 0.0306 - val_accuracy: 0.9922\n",
      "Epoch 7/10\n",
      "361/361 [==============================] - 22s 62ms/step - loss: 0.0364 - accuracy: 0.9883 - val_loss: 0.0285 - val_accuracy: 0.9923\n",
      "Epoch 8/10\n",
      "361/361 [==============================] - 23s 65ms/step - loss: 0.0317 - accuracy: 0.9899 - val_loss: 0.0355 - val_accuracy: 0.9906\n",
      "Epoch 9/10\n",
      "361/361 [==============================] - 24s 68ms/step - loss: 0.0295 - accuracy: 0.9911 - val_loss: 0.0300 - val_accuracy: 0.9925\n",
      "Epoch 10/10\n",
      "361/361 [==============================] - 24s 67ms/step - loss: 0.0246 - accuracy: 0.9923 - val_loss: 0.0234 - val_accuracy: 0.9929\n",
      "\n",
      "Evaluating...\n",
      "451/451 [==============================] - 5s 11ms/step - loss: 0.5374 - accuracy: 0.9059\n",
      "get_stn_a_model_6 : test loss = 0.54, test acc= 0.91\n",
      "\n",
      "model_func_name : get_stn_a_model_7\n",
      "\n",
      "Training...\n",
      "Epoch 1/10\n",
      "361/361 [==============================] - 25s 64ms/step - loss: 1.2056 - accuracy: 0.6374 - val_loss: 0.2728 - val_accuracy: 0.9236\n",
      "Epoch 2/10\n",
      "361/361 [==============================] - 23s 62ms/step - loss: 0.3527 - accuracy: 0.8968 - val_loss: 0.1435 - val_accuracy: 0.9578\n",
      "Epoch 3/10\n",
      "361/361 [==============================] - 22s 61ms/step - loss: 0.2355 - accuracy: 0.9319 - val_loss: 0.0951 - val_accuracy: 0.9742\n",
      "Epoch 4/10\n",
      "361/361 [==============================] - 23s 63ms/step - loss: 0.1787 - accuracy: 0.9474 - val_loss: 0.0736 - val_accuracy: 0.9789\n",
      "Epoch 5/10\n",
      "361/361 [==============================] - 24s 68ms/step - loss: 0.1466 - accuracy: 0.9581 - val_loss: 0.0661 - val_accuracy: 0.9806\n",
      "Epoch 6/10\n",
      "361/361 [==============================] - 24s 68ms/step - loss: 0.1224 - accuracy: 0.9647 - val_loss: 0.0549 - val_accuracy: 0.9853\n",
      "Epoch 7/10\n",
      "361/361 [==============================] - 24s 67ms/step - loss: 0.1075 - accuracy: 0.9689 - val_loss: 0.0527 - val_accuracy: 0.9859\n",
      "Epoch 8/10\n",
      "361/361 [==============================] - 24s 67ms/step - loss: 0.1040 - accuracy: 0.9701 - val_loss: 0.0498 - val_accuracy: 0.9862\n",
      "Epoch 9/10\n",
      "361/361 [==============================] - 25s 68ms/step - loss: 0.0923 - accuracy: 0.9734 - val_loss: 0.0404 - val_accuracy: 0.9891\n",
      "Epoch 10/10\n",
      "361/361 [==============================] - 24s 67ms/step - loss: 0.0823 - accuracy: 0.9759 - val_loss: 0.0468 - val_accuracy: 0.9877\n",
      "\n",
      "Evaluating...\n",
      "451/451 [==============================] - 6s 12ms/step - loss: 0.5580 - accuracy: 0.8807\n",
      "get_stn_a_model_7 : test loss = 0.56, test acc= 0.88\n",
      "\n",
      "model_func_name : get_stn_a_model_8\n",
      "\n",
      "Training...\n",
      "Epoch 1/10\n",
      "361/361 [==============================] - 26s 68ms/step - loss: 1.7254 - accuracy: 0.5891 - val_loss: 4.2880 - val_accuracy: 0.0877\n",
      "Epoch 2/10\n",
      "361/361 [==============================] - 24s 67ms/step - loss: 0.6860 - accuracy: 0.8233 - val_loss: 0.2992 - val_accuracy: 0.9284\n",
      "Epoch 3/10\n",
      "361/361 [==============================] - 24s 67ms/step - loss: 0.4125 - accuracy: 0.8912 - val_loss: 0.1593 - val_accuracy: 0.9592\n",
      "Epoch 4/10\n",
      "361/361 [==============================] - 25s 68ms/step - loss: 0.2753 - accuracy: 0.9255 - val_loss: 0.1278 - val_accuracy: 0.9697\n",
      "Epoch 5/10\n",
      "361/361 [==============================] - 24s 67ms/step - loss: 0.1973 - accuracy: 0.9463 - val_loss: 0.0879 - val_accuracy: 0.9769\n",
      "Epoch 6/10\n",
      "361/361 [==============================] - 24s 68ms/step - loss: 0.1692 - accuracy: 0.9545 - val_loss: 0.0867 - val_accuracy: 0.9795\n",
      "Epoch 7/10\n",
      "361/361 [==============================] - 24s 67ms/step - loss: 0.1449 - accuracy: 0.9614 - val_loss: 0.0746 - val_accuracy: 0.9814\n",
      "Epoch 8/10\n",
      "361/361 [==============================] - 24s 68ms/step - loss: 0.1207 - accuracy: 0.9676 - val_loss: 0.0693 - val_accuracy: 0.9840\n",
      "Epoch 9/10\n",
      "361/361 [==============================] - 25s 69ms/step - loss: 0.1080 - accuracy: 0.9703 - val_loss: 0.0845 - val_accuracy: 0.9762\n",
      "Epoch 10/10\n",
      "361/361 [==============================] - 23s 64ms/step - loss: 0.1036 - accuracy: 0.9727 - val_loss: 0.0519 - val_accuracy: 0.9879\n",
      "\n",
      "Evaluating...\n",
      "451/451 [==============================] - 6s 13ms/step - loss: 0.4316 - accuracy: 0.9095\n",
      "get_stn_a_model_8 : test loss = 0.43, test acc= 0.91\n",
      "\n",
      "model_func_name : get_stn_a_model_9\n",
      "\n",
      "Training...\n",
      "Epoch 1/10\n",
      "361/361 [==============================] - 26s 68ms/step - loss: 0.8079 - accuracy: 0.7605 - val_loss: 0.1694 - val_accuracy: 0.9527\n",
      "Epoch 2/10\n",
      "361/361 [==============================] - 24s 67ms/step - loss: 0.2221 - accuracy: 0.9340 - val_loss: 0.0856 - val_accuracy: 0.9743\n",
      "Epoch 3/10\n",
      "361/361 [==============================] - 24s 67ms/step - loss: 0.1564 - accuracy: 0.9534 - val_loss: 0.0603 - val_accuracy: 0.9831\n",
      "Epoch 4/10\n",
      "361/361 [==============================] - 24s 67ms/step - loss: 0.1155 - accuracy: 0.9653 - val_loss: 0.0578 - val_accuracy: 0.9848\n",
      "Epoch 5/10\n",
      "361/361 [==============================] - 24s 67ms/step - loss: 0.0951 - accuracy: 0.9709 - val_loss: 0.0449 - val_accuracy: 0.9868\n",
      "Epoch 6/10\n",
      "361/361 [==============================] - 24s 65ms/step - loss: 0.0830 - accuracy: 0.9749 - val_loss: 0.0391 - val_accuracy: 0.9892\n",
      "Epoch 7/10\n",
      "361/361 [==============================] - 24s 68ms/step - loss: 0.0781 - accuracy: 0.9764 - val_loss: 0.0464 - val_accuracy: 0.9856\n",
      "Epoch 8/10\n",
      "361/361 [==============================] - 25s 68ms/step - loss: 0.0655 - accuracy: 0.9791 - val_loss: 0.0377 - val_accuracy: 0.9882\n",
      "Epoch 9/10\n",
      "361/361 [==============================] - 24s 68ms/step - loss: 0.0609 - accuracy: 0.9817 - val_loss: 0.0370 - val_accuracy: 0.9892\n",
      "Epoch 10/10\n",
      "361/361 [==============================] - 24s 67ms/step - loss: 0.0586 - accuracy: 0.9822 - val_loss: 0.0313 - val_accuracy: 0.9909\n",
      "\n",
      "Evaluating...\n",
      "451/451 [==============================] - 5s 11ms/step - loss: 0.4430 - accuracy: 0.9091\n",
      "get_stn_a_model_9 : test loss = 0.44, test acc= 0.91\n"
     ]
    }
   ],
   "source": [
    "from inspect import getmembers, isfunction\n",
    "from asl_model import tool\n",
    "\n",
    "model_dict = dict()\n",
    "\n",
    "print(f\"model candidate = {len(tool.all_models.items())}\")\n",
    "\n",
    "for model_func_name, model_func_obj in tool.all_models.items():\n",
    "    # if 'stn_a_model_8' not in model_func_name:\n",
    "    #     continue\n",
    "    \n",
    "    print(f\"\\nmodel_func_name : {model_func_name}\")\n",
    "    model = model_func_obj()\n",
    "    model.compile(\n",
    "        optimizer=tf.keras.optimizers.Adam(learning_rate = lr),\n",
    "        loss='categorical_crossentropy',\n",
    "        metrics=['accuracy']\n",
    "    )\n",
    "    \n",
    "    print('\\nTraining...')\n",
    "    history = model.fit(\n",
    "        train_images,\n",
    "        validation_data=val_images,\n",
    "        epochs=epoch,\n",
    "    )\n",
    "    \n",
    "    print('\\nEvaluating...')\n",
    "    result = model.evaluate(test_images)\n",
    "    print(f\"{model_func_name} : test loss = {round(result[0], 2)}, test acc= {round(result[1], 2)}\")\n",
    "    \n",
    "    acc = result[1]\n",
    "    \n",
    "    record = {\n",
    "        'acc': acc,\n",
    "        'history': history,\n",
    "    }\n",
    "    \n",
    "    model_dict[model_func_name] = record"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "get_model_1 = 0.9002078771591187\n",
      "get_model_2 = 0.8925849199295044\n",
      "get_model_3 = 0.9001929080963135\n",
      "get_stn_a_model_1 = 0.8800415992736816\n",
      "get_stn_a_model_2 = 0.8670824766159058\n",
      "get_stn_a_model_3 = 0.8801108598709106\n",
      "get_stn_a_model_4 = 0.8910602927207947\n",
      "get_stn_a_model_5 = 0.901386022567749\n",
      "get_stn_a_model_6 = 0.9058905243873596\n",
      "get_stn_a_model_7 = 0.8807345628738403\n",
      "get_stn_a_model_8 = 0.9094941020011902\n",
      "get_stn_a_model_9 = 0.9091476202011108\n"
     ]
    }
   ],
   "source": [
    "for model_name, r in model_dict.items():\n",
    "    print(f\"{model_name} = {r['acc']}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "CS 4210",
   "language": "python",
   "name": "cs4210"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}