04_1_Model_Hacking.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# imports\n",
    "from pyimagesearch.learningratefinder import LearningRateFinder\n",
    "from pyimagesearch.clr_callback import CyclicLR\n",
    "from tensorflow.keras.layers import GlobalAveragePooling2D\n",
    "from tensorflow.keras.layers import Input\n",
    "from tensorflow.keras.layers import Dense\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.applications import VGG16\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from wandb.keras import WandbCallback\n",
    "from pyimagesearch import config\n",
    "from utils import data_utils\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import wandb\n",
    "import time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The `pyimagesearch` module comes from our tutorial [Keras Learning Rate Finder](https://www.pyimagesearch.com/2019/08/05/keras-learning-rate-finder/). "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fix random seed for better reproducibility\n",
    "tf.random.set_seed(666)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# don't change this\n",
    "CLASSES = [b'daisy', b'dandelion', b'roses', b'sunflowers', b'tulips']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define the constants\n",
    "BATCH_SIZE = 64\n",
    "EPOCHS = 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# let's load up the tfrecord filenames\n",
    "tfr_pattern_train = \"train_tfr/*.tfrec\"\n",
    "train_filenames = tf.io.gfile.glob(tfr_pattern_train)\n",
    "tfr_pattern_test = \"test_tfr/*.tfrec\"\n",
    "test_filenames = tf.io.gfile.glob(tfr_pattern_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create the train and test dataset\n",
    "training_dataset, steps_per_epoch = data_utils.batch_dataset(train_filenames, BATCH_SIZE, True)\n",
    "validation_dataset, validation_steps = data_utils.batch_dataset(test_filenames, BATCH_SIZE, False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create a utility function to define our model\n",
    "def create_model(img_size=(224,224), num_class=5, train_base=True):\n",
    "    input_layer = Input(shape=(img_size[0],img_size[1],3))\n",
    "    base = VGG16(input_tensor=input_layer,\n",
    "                    include_top=False,\n",
    "                    weights=\"imagenet\")\n",
    "    base.trainable = train_base\n",
    "    x = base.output\n",
    "    x = GlobalAveragePooling2D()(x)\n",
    "    \n",
    "    preds = Dense(num_class, activation=\"softmax\")(x)\n",
    "    return Model(inputs=input_layer, outputs=preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# instantiate the model, supply the loss scaled optimizer,\n",
    "# and compile it\n",
    "model = create_model()\n",
    "opt = Adam(learning_rate=config.MIN_LR)\n",
    "model.compile(loss=\"sparse_categorical_crossentropy\",\n",
    "              optimizer=opt,\n",
    "              metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Brief steps to find a learning rate range**:\n",
    "\n",
    "![](https://i.ibb.co/J3LjtNS/Screen-Shot-2019-12-08-at-9-55-29-AM.png)\n",
    "\n",
    "<center><a href=\"https://www.pyimagesearch.com/2019/08/05/keras-learning-rate-finder/\">Credits</a></center>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/43\n",
      "48/48 [==============================] - 77s 2s/step - loss: 1.9289 - accuracy: 0.2122\n",
      "Epoch 2/43\n",
      "48/48 [==============================] - 62s 1s/step - loss: 1.9036 - accuracy: 0.2214\n",
      "Epoch 3/43\n",
      "48/48 [==============================] - 62s 1s/step - loss: 1.9275 - accuracy: 0.2116\n",
      "Epoch 4/43\n",
      "48/48 [==============================] - 62s 1s/step - loss: 1.9264 - accuracy: 0.2135\n",
      "Epoch 5/43\n",
      "48/48 [==============================] - 62s 1s/step - loss: 1.9091 - accuracy: 0.2194\n",
      "Epoch 6/43\n",
      "48/48 [==============================] - 62s 1s/step - loss: 1.9201 - accuracy: 0.2152\n",
      "Epoch 7/43\n",
      "48/48 [==============================] - 62s 1s/step - loss: 1.9227 - accuracy: 0.2109\n",
      "Epoch 8/43\n",
      "48/48 [==============================] - 62s 1s/step - loss: 1.9090 - accuracy: 0.2210\n",
      "Epoch 9/43\n",
      "48/48 [==============================] - 62s 1s/step - loss: 1.9116 - accuracy: 0.2129\n",
      "Epoch 10/43\n",
      "48/48 [==============================] - 62s 1s/step - loss: 1.9018 - accuracy: 0.2178\n",
      "Epoch 11/43\n",
      "48/48 [==============================] - 62s 1s/step - loss: 1.8922 - accuracy: 0.2132\n",
      "Epoch 12/43\n",
      "48/48 [==============================] - 62s 1s/step - loss: 1.8609 - accuracy: 0.2178\n",
      "Epoch 13/43\n",
      "48/48 [==============================] - 62s 1s/step - loss: 1.8337 - accuracy: 0.2116\n",
      "Epoch 14/43\n",
      "48/48 [==============================] - 62s 1s/step - loss: 1.7655 - accuracy: 0.2165\n",
      "Epoch 15/43\n",
      "48/48 [==============================] - 62s 1s/step - loss: 1.6759 - accuracy: 0.2288\n",
      "Epoch 16/43\n",
      "48/48 [==============================] - 62s 1s/step - loss: 1.5383 - accuracy: 0.3164\n",
      "Epoch 17/43\n",
      "48/48 [==============================] - 62s 1s/step - loss: 1.2837 - accuracy: 0.5264\n",
      "Epoch 18/43\n",
      "48/48 [==============================] - 62s 1s/step - loss: 0.8584 - accuracy: 0.6842\n",
      "Epoch 19/43\n",
      "48/48 [==============================] - 62s 1s/step - loss: 0.5707 - accuracy: 0.7946\n",
      "Epoch 20/43\n",
      "48/48 [==============================] - 62s 1s/step - loss: 0.4775 - accuracy: 0.8311\n",
      "Epoch 21/43\n",
      "48/48 [==============================] - 62s 1s/step - loss: 0.4184 - accuracy: 0.8555\n",
      "Epoch 22/43\n",
      "48/48 [==============================] - 62s 1s/step - loss: 0.3560 - accuracy: 0.8721\n",
      "Epoch 23/43\n",
      "48/48 [==============================] - 62s 1s/step - loss: 0.3753 - accuracy: 0.8626\n",
      "Epoch 24/43\n",
      "48/48 [==============================] - 62s 1s/step - loss: 0.4289 - accuracy: 0.8385\n",
      "Epoch 25/43\n",
      "48/48 [==============================] - 61s 1s/step - loss: 0.4242 - accuracy: 0.8431\n",
      "Epoch 26/43\n",
      "48/48 [==============================] - 61s 1s/step - loss: 0.5258 - accuracy: 0.8037\n",
      "Epoch 27/43\n",
      "48/48 [==============================] - 60s 1s/step - loss: 1.2424 - accuracy: 0.5228\n",
      "Epoch 28/43\n",
      "48/48 [==============================] - 57s 1s/step - loss: 1.6774 - accuracy: 0.2334\n",
      "Epoch 29/43\n",
      "25/48 [==============>...............] - ETA: 26s - loss: 1.6063 - accuracy: 0.2506"
     ]
    }
   ],
   "source": [
    "# use the learning rate finder to find a suitable range \n",
    "# to train our network\n",
    "lrf = LearningRateFinder(model)\n",
    "lrf.find(\n",
    "    training_dataset,\n",
    "    1e-10, 1e+1,\n",
    "    stepsPerEpoch=steps_per_epoch,\n",
    "    batchSize=config.BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEOCAYAAABmVAtTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl8VOXZ//HPlZ0sJCEJawiEHUTWCCqgWDdcilZtC61t3Wt9qrWta1dbu9v2aautlvZxaX8uVVDrjoqiKKCEHcIW9kDIQiArWSZz/f6YQwwhGzCTM5O53q/XvJw5c8853xniXHPOfc59i6pijDHGAES4HcAYY0zwsKJgjDGmiRUFY4wxTawoGGOMaWJFwRhjTBMrCsYYY5pYUTDGGNMkYEVBRAaKyPsikiciG0XkO620ERH5i4jki8g6EZkUqDzGGGM6FhXAdXuA76vqKhFJAlaKyDuqmteszSXAcOc2FXjU+a8xxhgXBGxPQVULVXWVc78S2AQMaNHsCuBf6rMcSBGRfoHKZIwxpn2B3FNoIiKDgYnAJy2eGgDsbfa4wFlW2Na60tPTdfDgwf4NaIwx3dzKlStLVTWjo3YBLwoikggsAO5U1YqTXMctwC0AWVlZ5Obm+jGhMcZ0fyKyuzPtAnr2kYhE4ysIT6vqi6002QcMbPY401l2DFWdp6o5qpqTkdFhoTPGGHOSAnn2kQD/B2xS1T+20ewV4OvOWUhnAuWq2uahI2OMMYEVyMNH04CvAetFZI2z7AdAFoCqPga8AVwK5AM1wPUBzGOMMaYDASsKqvoRIB20UeB/ApXBGGPMibErmo0xxjSxomCMMaZJWBaF8iMNFByqOWZZo1cpqaxzKZExxgSHsCsKC1YWMP5nbzP9t+9z1wtrOTpH9SPv5XPGL99l0aaiTq9r9Z5DFFfWtvl8o1dZsq2EvWU15O4qa7WN16t4vcp7m4t4aXUBK3e33u5UdHYe7g37yqmp97CrtJql+aV4Gr1+z2KMCW5dckVzMPnzom1N9+evLCBvfwV5hZ9dU/fNf6/k6ZumMnVIWrvr+WBrCd94/FMAfnTZaKZmp9E3OY5Gr/Le5mKG90nkw60lPPxeftNrEmOjmDW2L98+bxhx0ZF8uquM3y/cwp6ymuPWf/6o3uwuq+Hui0cycWAKBypqGZKRSHFFLSWVdQxKSyAlPpqYyAgOH2kgv7iKnEGpRER81rff0OhlwcoCnv10D2sLyvnJ5WO4YXr2Mdtp9CqPfbCddzcVsXrP4WOei4mK4KqJA8hKi+fSsf3YsL+ci8b0JSbqs98SB8pr+cXrefTpGccPLx19zPaNMaFHOvsrMljk5OToyV7RvP/wES784wdU1zce99zMkRn07RnHcyv2IgIf3HUeawsO858Ve2lo9HLzjCFMyErh3bwihvdJ4upHlwKQEh/N4ZqGdrd7w7Rs3tl0gL1lR1p9fsLAFKZk92LsgGR+8VoexSdwGCsyQmj0HvtvOH1YOpsKKzhYXX9c+6xe8QzNSCA1IYYIEeavLGh67uyhaVTVeUjuEc1H+aX0io9pdR1jB/RkUFoCJRV1HKioPaaoDUjpwekDkklNiGb2+AHkF1fy6OLtDE5P4NdXnc6gtIRj1qWqbC+pIjM1nrjoSKrrPCTEht1vFWMCTkRWqmpOh+3CpSjsLathxu/eB+APXxzP1ZMz+XRnGS+t3sfPZp9GTFQEB6vquO3pVXyys3OHcF6/Yzoj+yTxwKsbeSeviKIK35f5t88bxsi+SSzZVsLdF48iIym26TXbS6qY98EOdh2sZkJWClOze/G5UX2OWa+n0UvBoSNERQp//2AHxZW1rCsoZ2hGIr17xpK3v4LM1B6AsGRbCT1iIumdFMvWoirioiMYnJZAQmwURRW1ZKcn8Oc5E0nuEc1P/ruB19cXHlfEeifF8trt0+ndM+6491hR28D9C9YTFSlEivDiat8F5z2iIznS4Cuud5w/nJ2l1by6dn+7n9eQjASeun4KA3vFA7A0v5TrnlxBvcd3mGp8ZjJrC8oZkp5AUo9oDlXX89A141rda2to9PKfFXs5Y3AvoiOFIRmJ7W7bmHBnRaGFF3L3cvf8dQC8f9dMstMTWm2nqnzz3yt5O6+IyYNSufXcoQiwbl85r67dT2Wth/Ij9dw8Ywj3zBp13OsbvUpkEB9CaWj0sq2oii1FFZRVN3DVxAGkxEfjuwC9Y16vUlJVR0ZiLLsOVrPv8BGmD0tHRKip9/DJzjJq6xsREe57cR2VtR6e/+ZZ7C2r4c7/rGlaT1JsFJV1nk5t88nrzyAxNopNByr5cs5ANh+o4LanV1Fw6LM9rwevOI2vnTX4hD4LY8KJFYUWVJVl2w+yv7yWayZnnlKG2oZG4qIjT2kd4ej+F9fzcX5p0+Gmqydl8t0Lh5OZGk95TUNTsVm24yCD0+MpKDvCT/67gf3lx3fmx0VHMLZ/Mlm94pv2Xq6c0J/vXzSyaU/EGPMZKwomqDU0eomO7Pjkt7Lqen71xiZKKutIjI3i9fWFTMxK4c9fnkhWmu/Lf2dpNXPmLaOooo7eSbHcOD2b9MRYPj++PxEClbUekuKiiIqMoPxIA8k9ogP99owJOlYUTFjxepX/rt3HT/+7kYra1g9LDUlPYEdpNVOye3HN5Ey+lDOw1XbGdEdWFExY2ltWw5sbCjlYXU/BoSO8vq6QPj1jKaqoO+5MrbOGpHHXxSOYPKiXi4mN6RpWFIzB1/EfISAiTRfjrS0o58HX8liz13ddxqNfncSssX073dluTCiyomBMO1SVF1YW8PuFW5quC0mKi2LyoFRmjsjg0tP7tXqKrjGhyoqCMZ1QVefh3gXrWLK15Li+iMeuncSssf1cSmaMf3W2KNiloyasJcZG8devTEJVaWhU1u8r5431hfzfRzv59ZubmZSVansMJqyE3YB4xrRGRIiJimDyoFR+fPkY/n3jFPYfPsKUXy3invlrqaxtfygTY7qLQM7R/LiIFIvIhjaeTxaRV0VkrYhsFBGbitMEjRnDM3jnu+dy3sgMns8t4Pw/fMDzK/YeN86UMd1NIPcUngRmtfP8/wB5qjoemAn8QURiApjHmBMyOD2BJ66fwvxbzyI6MoJ7FqzjHmeoFGO6q4AVBVX9EGhvZDkFksR3HmCi07Zzg+EY04VyBvfirTtncPm4fryydh8vr96H1/YYTDflZp/CI8BoYD+wHviOqtqsLiYoJcVF86PLxjC8dxJ3/mcN33jiUz7OL6W24fhh2I0JZW4WhYuBNUB/YALwiIj0bK2hiNwiIrkikltSUtKVGY1p0jc5jtdun86PLhvNR/mlfPWfn3D7s6vdjmWMX7lZFK4HXlSffGAncPxY1ICqzlPVHFXNycjI6NKQxjQXESHcNGMIS+45jysm9OedvCLmzFsWkGlUjXGDm0VhD3A+gIj0AUYCO1zMY0ynZabG89A14zljcCrLd5Rx9aPL+OPbW9yOZcwpC9jFayLyLL6zitJFpAD4KRANoKqPAQ8CT4rIekCAe1W1NFB5jPG3mKgI/t9NU3k3r5gXVu7lL+/ls624iofnTiSqE8OCGxOMbJgLY/ygodHLlX/9mI37K7j4tD78/WsdjiZgTJfq7DAX9nPGGD+Ijozgtdunc/20wSzcWMQb6wvdjmTMSbGiYIyfiAjfvXAE/ZPjuO3pVVz7z0+sA9qEHCsKxvhRz7ho3rzzHO743DA+yi/lln+tZN/hI27HMqbTrCgY42fJPaL53kUjefd751Ln8fU1FJZbYTChwYqCMQEyrHciz9w8lapaD3c8u9qufjYhwYqCMQE0LjOF314zjhW7DnHzv3Kp81hhMMHNioIxATZ7fH9+c9XpLNlWyp3PrbHht01Qs6JgTBeYMyWLH102mjc3HODuF9YSatcHmfBh03Ea00VunJ5NWXU9f1u8nUmDUrn2zEFuRzLmOLanYEwXERHuumgk54zI4Oev5rFhX7nbkYw5jhUFY7pQRITwpy9PoFdCDLc9vYoKm/vZBBkrCsZ0sV4JMfz1qxPZf/gI97ywzvoXTFCxomCMCyYP6sW9s0bx1sYDPLV0l9txjGliRcEYl9w0I5vPjerNb9/awt6yGrfjGANYUTDGNSLCL64ciwg8tNAm6DHBwYqCMS7qn9KDr0zJ4vX1hWwtqnQ7jjGBKwoi8riIFIvIhnbazBSRNSKyUUQ+CFQWY4LZt2YOJSkuih+9tME6nY3rArmn8CQwq60nRSQF+BswW1VPA74YwCzGBK20xFjumzWKT3eV8cra/W7HMWEuYEVBVT8E2pth5CvAi6q6x2lfHKgsxgS7L+UMZFxmMr96YxPVdR6345gw5mafwgggVUQWi8hKEfm6i1mMcVVEhPDA7NMoqqjjkffz3Y5jwpibRSEKmAxcBlwM/FhERrTWUERuEZFcEcktKSnpyozGdJlJWalcOaE/T3y8k5LKOrfjmDDlZlEoABaqarWqlgIfAuNba6iq81Q1R1VzMjIyujSkMV3pOxeMoN7j5e8fbHc7iglTbhaF/wLTRSRKROKBqcAmF/MY47rs9ASumpTJv5btZs9Bu6DNdL1AnpL6LLAMGCkiBSJyo4jcKiK3AqjqJuAtYB3wKfBPVW3z9FVjwsXdF48E4O8f2t6C6XoBm09BVed2os1DwEOBymBMKOrTM46rJg3ghZUFfOf84fTuGed2JBNG7IpmY4LQrecOxdPo5R9LdrgdxYQZKwrGBKHB6QnMHt+fpz/Zw6HqerfjmDBiRcGYIHXbecOoqW/kiY93uh3FhBErCsYEqRF9krj4tD48sXSXzdBmuowVBWOC2C3nDKWy1sOClQVuRzFhwoqCMUFsUlYKOYNS+fsHO2ho9Lodx4QBKwrGBDER4Vszh3KgopY31he6HceEASsKxgS580b2Jjs9gcc/2mnzLYSxJz/eya7S6oBvx4qCMUEuIkK4Ydpg1haUs3L3IbfjGBe8m1fEA6/m8e/luwO+LSsKxoSAqydnktwj2i5mC0MllXXcu2AdY/r15N5ZowK+PSsKxoSA+JgovpSTyaJNxVTa6alhQ1W5d8E6Kus8/GnOBGKiAv+VbUXBmBBx/ug+eLzKx/kH3Y5iusgra/fz3uZi7p01ihF9krpkm1YUjAkRkwelkhQXxXy7ZiEsVNd5+NUbmzh9QDLXnT24y7ZrRcGYEBEdGcG1Zw7i/S3FHKyymdm6u4ffy6eooo6fXXEakRHSZdu1omBMCJk9vj+NXuXNDQfcjmICaN/hI/zfRzu4elImk7JSu3TbVhSMCSGj+iYxJCOB19btdzuKCaBH3stHEL5/UavT1gdUIGdee1xEikWk3dnUROQMEfGIyDWBymJMdyEizDqtLyt2HaKqzuN2HBMAe8tqeCF3L3OmDKR/So8u334g9xSeBGa110BEIoHfAm8HMIcx3crZQ9Np9Cqf7rSzkLqjxz7YTkSEcNvMYa5sP2BFQVU/BMo6aHY7sAAoDlQOY7qbnMGppMZH8/wKOwupuzlUXc+CVQVcNXEAfZPdmYbVtT4FERkAfAF41K0MxoSiuOhILj29Hx/ll9LotbGQupNnPt1DbYOXG6Znu5bBzY7mPwH3qmqH4wGLyC0ikisiuSUlJV0QzZjgNjErlao6D/nFVW5HMX5S7/Hy1NJdzBie3mUXqrXGzaKQAzwnIruAa4C/iciVrTVU1XmqmqOqORkZGV2Z0ZigdPbQNCIjxM5C6kYWbSqiuLKOG6a5t5cALhYFVc1W1cGqOhiYD9ymqi+7lceYUNI/pQcTBqbw4Vbbc+4u5q8soE/PWM4Z4e4P30CekvossAwYKSIFInKjiNwqIrcGapvGhJPpw9JZt6+cwzX1bkcxp6i4spbFW0u4alJml1693JqoQK1YVeeeQNvrApXDmO7qnBHp/HnRNpZuP8ilp/dzO445BS+t2kejV/ni5Ey3o9gVzcaEqvGZKSTGRvFxfqnbUcwpennNfiZmpTAkI9HtKFYUjAlVUZERTM3uxdLtdhFbKNtZWs2mwgouH9ff7SiAFQVjQtrZw9LZWVrNvsNH3I5iTtKbGwoBmDW2r8tJfKwoGBPCpg1LA7BDSCHsjfWFTBiYwgAXxjlqjRUFY0LYyD5JpCfGsNSKQkjac7CGDfsquCyIThSwomBMCBMRzhqazsfbD6JqQ16EmkWbiwC4+LTgOHQEVhSMCXnTh6VRUllnQ16EoPe3lDAkI4GstHi3ozSxomBMiDt7aDpg/Qqh5kh9I8t3HORcl69gbsmKgjEhbmCveLJ6xfNRvp2aGkqW7zhIvcfLzJG93Y5yDCsKxnQD04al8cmOg3gaOxx02ASJxVuKiYv2XWsSTKwoGNMNnD00nco6D+v2lbsdxXTSkm2lnDUkjbjoSLejHMOKgjHdwIzh6URFCO/kFbkdxXRCSWUdO0qrOWtomttRjmNFwZhuICU+htP692T1nkNuRzGdsHK3b6binMHBdegIrCgY021MGJjC+oJym6IzBKzYdYjYqAjG9k92O8pxrCgY002MH5hCdX0j24or3Y5iOpC7q4zxA1OIiQq+r+DgS2SMOSkTBqYAsGbPYZeTmPbU1HvYsL+CMwanuh2lVVYUjOkmstMTSO4RzWorCkFtzZ7DNHo1KPsToJNFQUSGikisc3+miNwhIikdvOZxESkWkQ1tPP9VEVknIutFZKmIjD/x+MaYo0SEKdm9WL7TLmILZit2HUIEJmWF9p7CAqBRRIYB84CBwDMdvOZJYFY7z+8EzlXV04EHnfUaY07BpKxUdh+ssXmbg1ju7jJG9kkiuUe021Fa1dmi4FVVD/AF4GFVvRtod6xXVf0QKGvn+aWqevT8ueWA+5OTGhPixg7oCcDG/RUuJzFt2V5cxZh+Pd2O0abOFoUGEZkLfAN4zVnmzzJ3I/CmH9dnTFg6eorjeruyOWjVNypxMcF1FXNznS0K1wNnAb9U1Z0ikg382x8BROQ8fEXh3nba3CIiuSKSW1JS4o/NGtMtpSbEMCClhxWFIFbvaSQmMnjP8YnqTCNVzQPuABCRVCBJVX97qhsXkXHAP4FLVLXN3jFVnYfT55CTk2NX5hjTjolZKazYVYaqIiJuxzEtNDQq0ZHB++/S2bOPFotITxHpBawC/iEifzyVDYtIFvAi8DVV3Xoq6zLGfGbG8HSKKurYWmST7gSjhkYv0UG8p9DZZMmqWgFcBfxLVacCF7T3AhF5FlgGjBSRAhG5UURuFZFbnSY/AdKAv4nIGhHJPcn3YIxp5hxn0pYl2+xQa7DxehWPV4O6KHTq8BEQJSL9gC8BP+zMC1R1bgfP3wTc1MntG2M6qV9yD4ZkJPBxfik3zRjidhzTTIPXN99FMA5vcVRnk/0cWAhsV9UVIjIE2Ba4WMaYU3HO8AyWbj9I+ZEGt6OYZhoafV2iwdzR3KlkqvqCqo5T1W85j3eo6tWBjWaMOVmzxvalzuPl051tXipkXFDv8e0pdIeO5kwReckZtqJYRBaIiF1sZkyQmuCMwPnJDhvyIpjU1HsAiA2y2daa6+w+zBPAK0B/5/aqs8wYE4TioiOZODDFxkEKMtuKfWeEDUlPcDlJ2zpbFDJU9QlV9Ti3J4GMAOYyxpyiM4ekkbe/wvoVgsjmQt9cF6P6hv4wFwdF5FoRiXRu1wL2E8SYIDZ1SC+86pvQxQSHzQcq6J8cR3J8cA6GB50vCjfgOx31AFAIXANcF6BMxhg/mJSVSkxkBJ9YZ3PQ2HKgkpF9k9yO0a7Onn20W1Vnq2qGqvZW1SsBO/vImCAWFx3JhIEpLLfO5qDgafSyo7SaEX26QVFow/f8lsIYExA5g1PZuL+C2oZGt6OEvb2HjlDv8TK0d6LbUdp1KkUheE+0NcYAMC4zhUavsqnQ5ldw27YiXyfz8G5cFGy0UmOC3LhMm18hWOSX+E5HHRbkRaHdsY9EpJLWv/wF6BGQRMYYv+mXHEd6YgzrCqwouC2/qIq+PeNIigveM4+gg6KgqsHdI2KMaZeIMHZAMuutKLguv6SK4X2Cey8BTu3wkTEmBIzLTGFbcSXlNXYRm1u8XiW/uCroDx2BFQVjur0LRvfGq/DWxkK3o4St/eVHqKlvZHjv4D/4YkXBmG7u9AHJ9IyLYsM+OwPJLfnFodHJDAEsCiLyuDOi6oY2nhcR+YuI5IvIOhGZFKgsxoQzEWFY70Q+scHxXLOztBqAIRnBOxDeUYHcU3gSmNXO85cAw53bLcCjAcxiTFibPb4/W4uqmr6cTNfafbCGpNgo0hJi3I7SoYAVBVX9EGhv0JUr8M33rKq6HEhxpvw0xvjZzJG9Afgov9TlJOFp18FqBqXHIxL81/y62acwANjb7HGBs8wY42eD0uJJT4xh9Z5DbkcJS7sP1jAoLfgPHUGIdDSLyC0ikisiuSUlJW7HMSbkiAgTBqawZu9ht6OEHU+jl71lNQxOi3c7Sqe4WRT2AQObPc50lh1HVeepao6q5mRk2Nw+xpyMiVmp7CiptusVutj+w7V4vGp7Cp3wCvB15yykM4FyVbUTqY0JkAkDUwBYU2B7C11p10Ff5/7gECkK7Q5zcSpE5FlgJpAuIgXAT4FoAFV9DHgDuBTIB2qA6wOVxRjjGxxPBNbsOcy5I2yPu6t8VhRC4/BRwIqCqs7t4HkF/idQ2zfGHCspLprhvRNZvdc6m7vSrtIaekRHkpEU63aUTgmJjmZjjH9MHJjK6j2H8TR63Y4SNnYfrGZQWmicjgpWFIwJK58b3ZvyIw18vN2ubu4quw5Wh0x/AlhRMCaszByZQUJMJO/kHXA7StgoqqijX0qc2zE6zYqCMWEkNiqSacPSeX9zCb5uPRNI9R4vVXUeUuODf3iLo6woGBNmzh2Zwb7DR9hh4yAF3OEj9QCkxgf3bGvNWVEwJsxMzU4D4NOd7Q1NZvzhsHOhYIrtKRhjgtXQjATSE2P4ZId1NgfaoeqjewpWFIwxQUpEmJiVypJtpdavEGCHjxzdU7DDR8aYIJadnsDB6nqbjS3ADtf49hSsKBhjgtrXzhwEwBq7ujmgDjl9Cnb4yBgT1DJTe9ArIYZ1BeVuR+nWDtXUExMZQXxMpNtROs2KgjFhSESYlJVC7m7bUwikQ9X1pMRHh8wQF2BFwZiwdeaQNHaWVnOgvNbtKN1WaVV9yAyEd5QVBWPC1JlDfNcrLN1u8zYHSkllnRUFY0xoGNOvJ70SYliyzYpCoJRU1pGRaEXBGBMCIiKE6cPSWbKtBK/XrlfwN69XKa2yPYVjiMgsEdkiIvkicl8rz2eJyPsislpE1onIpYHMY4w51rkjMiitqiev0K5X8LfDRxrweNWKwlEiEgn8FbgEGAPMFZExLZr9CHheVScCc4C/BSqPMeZ454zIIDJCmL+ywO0o3U5JZR2AFYVmpgD5qrpDVeuB54ArWrRRoKdzPxnYH8A8xpgWMpJimTW2Ly/k7qXcudDK+EdTUbA+hSYDgL3NHhc4y5p7ALhWRAqAN4DbA5jHGNOKG6YNprq+kUWbi9yO0q2UVPlO9bU9hRMzF3hSVTOBS4F/i8hxmUTkFhHJFZHckpKSLg9pTHc2cWAqvZNieXeTFQV/ssNHx9sHDGz2ONNZ1tyNwPMAqroMiAPSW65IVeepao6q5mRkZAQorjHhKSJCOH90bz7cWkq9x+t2nG6jpLKO2KgIEmOj3I5yQgJZFFYAw0UkW0Ri8HUkv9KizR7gfAARGY2vKNiugDFd7MIxfaiq8/DCyr0dNzadcvTCtVAa4gICWBRU1QN8G1gIbMJ3ltFGEfm5iMx2mn0fuFlE1gLPAtepDfBuTJc7b2Rvxg7oyfMrrCj4S0kIXqMAEND9GlV9A18HcvNlP2l2Pw+YFsgMxpiOiQgXju7L/767lQ+3lnDOCDtMe6pKK+sZlBbvdowT5nZHszEmSMyd6usCfMGuWfCL0qo60kNwT8GKgjEGgN5JcXxlahaLNhVxpL7R7Tghr6a+kYQQmkfhKCsKxpgml4/rR019I6+utetIT4WqUudpJDbKioIxJoRNzU5jcFo8/16+2+0oIc3jVbwKsVGh9xUbeomNMQETGSHMmZLF+n3lvLm+0O04IavOud4jNjr0vmJDL7ExJqBumJbN0IwE/vDOVjyNdjHbyahr8PXJ2OEjY0zIi4mK4K6LRpJfXMWCVXYm0smod4qpHT4yxnQLs8b2ZWSfJB5auIU6j52JdKLqGnxFIcaKgjGmOxARvn/RCEqr6nl7ow2Ud6Ka+hTs8JExpru4YHQfsnrF88h7+TZd5wk6undlh4+MMd1GRITwvQtHsKWokkWbi92OE1Lq7ewjY0x3dNm4fgxOi+cPb2/BxqrsPDt8ZIzplqIjI7ht5jA2H6hk+Y4yt+OEDDt8ZIzptmZP6E9qfDRPLd3ldpSQYWcfGWO6rbjoSOZMyeLtvAPkF1e6HSckfHb4KPS+YkMvsTGmy33jrMFERUTwmze3uB0lJHzW0Wx9CsaYbqhvchzXnjmIdzcV8Ye3t9gpqh2wPoU2iMgsEdkiIvkicl8bbb4kInkislFEnglkHmPMyfvuhcPpER3Jw+/ls3zHQbfjBDU7fNQKEYkE/gpcAowB5orImBZthgP3A9NU9TTgzkDlMcacmqS4aFb++AISY6NsdrYO2CmprZsC5KvqDlWtB54DrmjR5mbgr6p6CEBV7QoZY4JYfEwUsyf0580NhZRU1rkdJ2gdHSU1OlJcTnLiAlkUBgB7mz0ucJY1NwIYISIfi8hyEZnV2opE5BYRyRWR3JKSkgDFNcZ0xs0zhtDQqMz7cLvbUYJWXaOX2KgIRKwonKgoYDgwE5gL/ENEUlo2UtV5qpqjqjkZGRldHNEY01x2egKXjO3Lcyv2UttgI6i2pq7BG5L9CRDYorAPGNjscaazrLkC4BVVbVDVncBWfEXCGBPE5pyRRWWth3fybATV1tR5GkPydFQIbFFYAQwXkWwRiQHmAK+0aPMyvr0ERCQd3+GkHQHMZIzxg7OGptEvOY7nc/d23DgMlVXXkxof7XaMkxKwoqCqHuDbwEJgE/C8qm4UkZ+LyGyn2ULgoIjkAe8Dd6uqnetmTJCLjBC+MHEAS7aV8renWynzAAAQHklEQVTF+W7HCTrFlXX0TopzO8ZJCehBL1V9Q1VHqOpQVf2ls+wnqvqKc19V9XuqOkZVT1fV5wKZxxjjP3OnZAHwu7e2cKTe+haaK6msIyMp1u0YJyU0e0KMMa4b2Cueh+dOBLDDSM2oqrOnYEXBGBNmLh/Xj0lZKcz7cAcNzmT14W7VnsPUe7xk9op3O8pJsaJgjDlpIsLtnxvOvsNHuOHJFW7HcV2dp5FvP7OKqAjhygn93Y5zUqwoGGNOycyRGfTtGceSbaXc+OSKsJ2hrd7jZfzP3qawvJZZY/uSFGdnHxljwpCI8M73ziEpNopFm4t5aumusBtFde3ew4z40ZvUNngZ3a8nj3xlktuRTpoVBWPMKUuKi2bZD85nZJ8kHng1j+dWhE/Hs6fRy/0vrgfgtplDeeOO6S4nOjVWFIwxfpEYG8UL3zqLqAjh5TX7KK6opbGb7zF4vcp3n19LXmEFP7x0NPfMGhWS4x01Z0XBGOM3PeOiue+SUXy6s4wpv1rE0B+8wa7SardjBURFbQOPvJ/Pq2v3c9dFI7j5nCFuR/ILKwrGGL+6cXo2l4zt2/R45u8Xs2JXmYuJ/Ke6zsPG/eWoKr99czN/fGcrADfN6B4FAXyjlBpjjN+ICH/76iS8Cq+u3c+d/1nDFx9bxs0zsvnhZWM6XkGQqW1o5OP8UsZlpnD1o0vZU1ZzzPO//+J44kJ08LvWWFEwxvidiBApcOXEAUwfns4lf17CP5bs5Oyh6Zw3qrfb8TpFVamub2TmQ4sprfpsQqFJWSms2nOYkX2SeObmqaQlhuaVy22RUDunOCcnR3Nzc92OYYw5AeVHGjjr14uIEOG9u84N+sHivF5l7AMLqWkxptPcKQP59VXjmq7FCKVOZRFZqao5HbWzPgVjTMAl94jmL3MmUlXn4Scvbwz6C9xW7CprKgh3nD+cVT++kPsvGcXtn/NN9yIiIVUQToQdPjLGdIkLxvThvktG8Zs3N/Pymn18YWKm25Ha9K9lu4mMEFb9+EKSe/iuTP7muUNdTtU1bE/BGNNlbp4xhElZKfzs1Ty2FVW6HadVRRW1vLXxADdNz24qCOHEioIxpstERgj/++UJREUIX3/8U4oqav26/sraBg46ncJ7y2rY2+JMoc64Z/46VJU5znwR4SagRUFEZonIFhHJF5H72ml3tYioiHTYCWKMCW2D0hKY9/UcCstrmfqrRXyywz+TLTZ6lc8//BGTf/EuU375LjN+9z4zfvc+j32wvdPrOFhVxwdbS7hhWjbZ6Ql+yRVqAlYURCQS+CtwCTAGmCsix52kLCJJwHeATwKVxRgTXCZlpXL754YB8OV5y7n7hbU88MpG3tpQeNKd0M+t2MOug749g6o6D2cO6UWfnrH85s3N5PziXXZ24srq5Tt8F9ldOq7fSWXoDgLZ0TwFyFfVHQAi8hxwBZDXot2DwG+BuwOYxRgTZL5/0UguPb0ff3xnKy+sLADgyaW7APjx5WO4cHQfstKOn6im0atERhx75o+q8vTyPYzok8jCO89pOjOous7DD15az3/X7OeGJ1fwn1vOpHfPtk+HfWl1Ack9ohk3INlP7zL0BLIoDACaD5VYAExt3kBEJgEDVfV1EbGiYEyYGd2vJ//4eg6Hqut5atku/vTuNgAefC2PB1/L4+pJmVwzOZMfvLSekso6quo8APx5zgSumDCgaT3rCsrJK6zgwStOO+ZU0YTYKP48ZyJzp2Rx/RMrmPKrRfROiqW4so47LxjOHZ8bToRTYJ5auot3NxUze3x/oiLDt7vVtVNSRSQC+CNwXSfa3gLcApCVFZ6dP8Z0Z6kJMdx5wQjmnJHFmxsKSYiN4vGPdrJgVQELVhUc1/47z63h7bwiPj+uHxeN6ctjH2wnLjqC2c0KRXNnDknjZ1ecxj3z11Fc6euI/tO72/hkRxk/u+I0Kmsb+OXrm3zrvmB44N5oCAjYFc0ichbwgKpe7Dy+H0BVf+08Tga2A1XOS/oCZcBsVW3zkmW7otmY8OBp9HL3/HW8tm4/3zxnKF+YNID4mEiS4qK55tGlbD7gO6X1ojF9eH9LMbPG9uPhuRPbXeeh6npSE2LwNHq56V+5LN5ScszzL912NhOzUgP2ntzU2SuaA1kUooCtwPnAPmAF8BVV3dhG+8XAXe0VBLCiYIzxDVL3v+9uJW9/BUu2lQLwxHVnnNC4SnWeRp79ZA9Pf7KHbcVVxx2S6m46WxQCdvhIVT0i8m1gIRAJPK6qG0Xk50Cuqr4SqG0bY7q3uOhI7r9kNKrKg69t4vCR+hMeaC82KpLrpmVz3bTsAKUMTTYgnjHGhAEbEM8YY8wJs6JgjDGmiRUFY4wxTawoGGOMaWJFwRhjTBMrCsYYY5pYUTDGGNPEioIxxpgmIXfxmoiUAIeBcmdRciv3W/43HSg9wU01X29nn2u5vLVsljPwOY8ui/ZzzraeP9WcJ/OZdkXO1jJbztDNOUhVMzrcgqqG3A2Y1979Vv6beyrb6OxzLZdbTndyHr3v75xtPX+qOU/mM+2KnP74t7ecwZezo1uoHj56tYP7Lf97qtvo7HMtl1vOjrO099zJ5uxoe+3p6HWtPd9dcza/bzk7XhYqOdsVcoePToaI5Gonxvxwm+X0r1DJCaGT1XL6VzDmDNU9hRM1z+0AnWQ5/StUckLoZLWc/hV0OcNiT8EYY0znhMuegjHGmE6womCMMaaJFQVjjDFNwrYoiMgQEfk/EZnfbFmCiDwlIv8Qka+6ma8lERkjIs+LyKMico3bedoiIlki8rKIPC4i97mdpy0iMkNEHhORf4rIUrfztEVEIkTklyLysIh8w+08bRGRmSKyxPlMZ7qdpz3O/+e5InK521naIiKjnc9yvoh8qyu3HZJFwfnCKRaRDS2WzxKRLSKS39EXkqruUNUbWyy+CpivqjcDs4MpL3AJ8LCqfgv4ur+yBSDn6fg+wxuAicGaU1WXqOqtwGvAU8GaE7gCyAQagIIgzqlAFRAX5DkB7gWeD0RGJ48//j43OX+fXwKmBSprWxsPuRtwDjAJ2NBsWSSwHRgCxABrgTH4vqRea3Hr3ex185vdvx+Y4Nx/JpjyOre/Ag8BHwfr5wqkAe8D7wHXB2vOZq97HkgK1pzAfcA3W/6tBmHOCOd1fYCngzjnhcAc4Drg8mDN6bxmNvAm8JVA5Gwzf1duzM8f/OAWH/pZwMJmj+8H7u/EepoXha8d/UMBngvSvJHAf4P1cwXuAs5p+dkGW06nTRbwj0Bl9NPneS3wJef+f4I1Z7N2McH87w78EvgT8DbwX5xiFmw5W6zr9UB9nq3doug+BgB7mz0uAKa21VhE0vD9gUwUkftV9dfAi8AjInIZpzakQ2ecaN7BwA+ABHx7C13lhHICbwEPiMhXgF0BzNXSieYEuBF4ImCJWneiOV8EHhaRGcCHgQzWwon+fV4FXAykAI8ENtoxTiinqv4QQESuA0pV1RvQdJ850c9zJr7D2bHAGwFN1kJ3KgonRFUPAre2WFYNXO9Oovap6i7gFrdzdERVNwBB2xHenKr+1O0MHVHVGnzFK6ip6ov4ClhIUNUn3c7QHlVdDCx2Y9sh2dHchn3AwGaPM51lwSpU8lpO/7Kc/mU5/aw7FYUVwHARyRaRGHydSa+4nKk9oZLXcvqX5fQvy+lvXdmB4cdOnGeBQj47Te9GZ/mlwFZ8vfw/dDtnqOW1nJbTcoZPzrZuNiCeMcaYJt3p8JExxphTZEXBGGNMEysKxhhjmlhRMMYY08SKgjHGmCZWFIwxxjSxomD8SkSqunh7/xSRMX5aV6OIrBGRDSLyqoikdNA+RURuO4ntiIi8JyI9ncd+/8xE5IcislFE1jnvqaNxoFpbx+CWwz+30iZDRN46+aQm2FhRMEFNRNodn0tVb1LVPD9t7oiqTlDVsUAZ8D8dtE8BTrgo4LuIaa2qVpzEazskImcBlwOTVHUccAHHDsbmN6paAhSKSNeO+W8CxoqCCTjn1+QCEVnh3KY5y6eIyDIRWS0iS0VkpLP8OhF5RUTeAxaJb1avxeKbhWqziDwtIuK0XSwiOc79KvHNUrZWRJaLSB9n+VDn8XoR+UUnf5kvwzeyJSKSKCKLRGSVs44rnDa/AYY6v8Qfctre7bzHdSLyszbW/VV8wza395kNdvYm1jnbzjqB99IP3wigdQCqWqqq+53Xn+F81mtF5FMRSXK2tcR5f6tE5OxW8kSKyEPN3ts3mz39svOeTHfg9iXVduteN6CqlWXPANOd+1nAJud+TyDKuX8BsMC5fx2+4QF6OY9nAuX4BhGLwPeFfXR9i4Ec574Cn3fu/w74kXP/NWCuc//W1jI2z45vzooXgFnO4yigp3M/HcgHhOPHzL8ImOc8F+Fs95xWtrObZhP7tPGZvQp8w7l/A/ByZ98LkAiswTekwt+Ac53lMcAO4Izmnz8QD8Q5y4YDuc79pveHb4Teo59nLJALZDuPBwDr3f7bs5t/bmE7dLbpUhcAY5wf9wA9RSQRSAaeEpHh+L7Qo5u95h1VLWv2+FNVLQAQkTX4vrA+arGdenxfmgAr8c2yBb4JTq507j8D/L6NnD2cdQ8ANgHvOMsF+JWInAN4nef7tPL6i5zbaudxIr4v2ZbzIPRS1co2Mhx1Fr7x9AH+ja/Ideq9qGqViEwGZgDnAf8R3/SPK4FCVV3htKsA35zF+OYRmQA0AiPaeG/j5LP5wZOd97YTKAb6d/B+TIiwomC6QgRwpqrWNl8oIo8A76vqF8Q3idDiZk9Xt1hHXbP7jbT+t9ugzk/Xdtq054iqThCReGAhvj6Fv+A7NJIBTFbVBhHZhW8u4pYE+LWq/r2D7XhEJEIDOMGLqjbi+zwXi8h64Bv4ikJrvgsUAePx/VvVttJGgNtVdWErz8UBR041swkO1qdgusLbwO1HHzi/SMH3a/PomPLXBXD7y4GrnftzOmqsvolt7gC+73R0JwPFTkE4DxjkNK0Ekpq9dCFwg7MXhIgMEJHerWxiC765etuztFnWrwJLOvteRGSks/d11AR8h6y2AP1E5AynXVKz91foFKmv4Tt81tJC4FsiEu28doSzhwG+PYt2z1IyocOKgvG3eBEpaHb7Hr4v2ByngzKPz2a8+x3waxFZTWD3Wu8Evici64Bh+Pon2qWqq4F1wFzgaXz51wNfBzY7bQ4CH4vvFNaHVPVtfId0ljlt53Ns0TjqdXz9JEe19pndDlzvZP4a8J0TeC+J+A7L5TntxgAPqGo98GV803uuxXd4LA5fv8M3nGWjOH4vDeCfQB6wSnynqf6dz/7NznPek+kGbOhs0+05h4OOqKqKyBx8HbVXdPS6AObpB/xLVS/ssPHxrw2q9+Jk+hC4QlUPuZnD+If1KZhwMBlfR6oAh/GdzeMaVS0UkX+ISE898WsVguq9iEgG8EcrCN2H7SkYY4xpYn0KxhhjmlhRMMYY08SKgjHGmCZWFIwxxjSxomCMMaaJFQVjjDFN/j/DSs4c0dHtVAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# let's plot the learning rates\n",
    "lrf.plot_loss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "                Logging results to <a href=\"https://wandb.com\" target=\"_blank\">Weights & Biases</a> <a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">(Documentation)</a>.<br/>\n",
       "                Project page: <a href=\"https://app.wandb.ai/sayakpaul/ML-Bootcamp-Launchpad\" target=\"_blank\">https://app.wandb.ai/sayakpaul/ML-Bootcamp-Launchpad</a><br/>\n",
       "                Run page: <a href=\"https://app.wandb.ai/sayakpaul/ML-Bootcamp-Launchpad/runs/1pejp8xz\" target=\"_blank\">https://app.wandb.ai/sayakpaul/ML-Bootcamp-Launchpad/runs/1pejp8xz</a><br/>\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Error generating diff: Command '['git', 'diff', '--submodule=diff', 'HEAD']' timed out after 5 seconds\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "W&B Run: https://app.wandb.ai/sayakpaul/ML-Bootcamp-Launchpad/runs/1pejp8xz"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# initialize wandb\n",
    "wandb.init(\"ml-bootcamp\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# instantiate the model, supply the loss scaled optimizer,\n",
    "# and compile it\n",
    "model = create_model()\n",
    "opt = Adam(learning_rate=config.MIN_LR)\n",
    "model.compile(loss=\"sparse_categorical_crossentropy\",\n",
    "              optimizer=opt,\n",
    "              metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# compute the step size and initialize the cyclic learning\n",
    "# rate method\n",
    "step_size = config.STEP_SIZE * steps_per_epoch\n",
    "clr = CyclicLR(\n",
    "\tmode=config.CLR_METHOD,\n",
    "\tbase_lr=1e-6,\n",
    "\tmax_lr=1e-5,\n",
    "\tstep_size=step_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Error generating diff: Command '['git', 'diff', '--submodule=diff', 'HEAD']' timed out after 5 seconds\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "48/48 [==============================] - 73s 2s/step - loss: 1.6296 - accuracy: 0.2933 - val_loss: 1.4477 - val_accuracy: 0.4434\n",
      "Epoch 2/20\n",
      "48/48 [==============================] - 67s 1s/step - loss: 1.3129 - accuracy: 0.5085 - val_loss: 1.0919 - val_accuracy: 0.6719\n",
      "Epoch 3/20\n",
      "48/48 [==============================] - 68s 1s/step - loss: 0.9036 - accuracy: 0.6742 - val_loss: 0.5805 - val_accuracy: 0.7812\n",
      "Epoch 4/20\n",
      "48/48 [==============================] - 68s 1s/step - loss: 0.6019 - accuracy: 0.7835 - val_loss: 0.4196 - val_accuracy: 0.8555\n",
      "Epoch 5/20\n",
      "48/48 [==============================] - 68s 1s/step - loss: 0.4844 - accuracy: 0.8301 - val_loss: 0.3418 - val_accuracy: 0.8887\n",
      "Epoch 6/20\n",
      "48/48 [==============================] - 68s 1s/step - loss: 0.4226 - accuracy: 0.8512 - val_loss: 0.2808 - val_accuracy: 0.9043\n",
      "Epoch 7/20\n",
      "48/48 [==============================] - 68s 1s/step - loss: 0.3542 - accuracy: 0.8714 - val_loss: 0.2663 - val_accuracy: 0.9023\n",
      "Epoch 8/20\n",
      "48/48 [==============================] - 68s 1s/step - loss: 0.3399 - accuracy: 0.8783 - val_loss: 0.1989 - val_accuracy: 0.9414\n",
      "Epoch 9/20\n",
      "48/48 [==============================] - 67s 1s/step - loss: 0.2576 - accuracy: 0.9134 - val_loss: 0.1850 - val_accuracy: 0.9355\n",
      "Epoch 10/20\n",
      "48/48 [==============================] - 67s 1s/step - loss: 0.2357 - accuracy: 0.9215 - val_loss: 0.1874 - val_accuracy: 0.9375\n",
      "Epoch 11/20\n",
      "48/48 [==============================] - 67s 1s/step - loss: 0.1988 - accuracy: 0.9320 - val_loss: 0.1164 - val_accuracy: 0.9668\n",
      "Epoch 12/20\n",
      "48/48 [==============================] - 67s 1s/step - loss: 0.1973 - accuracy: 0.9326 - val_loss: 0.1200 - val_accuracy: 0.9590\n",
      "Epoch 13/20\n",
      "48/48 [==============================] - 67s 1s/step - loss: 0.1419 - accuracy: 0.9564 - val_loss: 0.1232 - val_accuracy: 0.9590\n",
      "Epoch 14/20\n",
      "48/48 [==============================] - 67s 1s/step - loss: 0.1502 - accuracy: 0.9466 - val_loss: 0.0927 - val_accuracy: 0.9707\n",
      "Epoch 15/20\n",
      "48/48 [==============================] - 67s 1s/step - loss: 0.1205 - accuracy: 0.9639 - val_loss: 0.0844 - val_accuracy: 0.9844\n",
      "Epoch 16/20\n",
      "48/48 [==============================] - 67s 1s/step - loss: 0.1092 - accuracy: 0.9655 - val_loss: 0.0770 - val_accuracy: 0.9746\n",
      "Epoch 17/20\n",
      "48/48 [==============================] - 67s 1s/step - loss: 0.1012 - accuracy: 0.9720 - val_loss: 0.0747 - val_accuracy: 0.9805\n",
      "Epoch 18/20\n",
      "48/48 [==============================] - 67s 1s/step - loss: 0.1028 - accuracy: 0.9652 - val_loss: 0.0719 - val_accuracy: 0.9863\n",
      "Epoch 19/20\n",
      "48/48 [==============================] - 67s 1s/step - loss: 0.1112 - accuracy: 0.9619 - val_loss: 0.0788 - val_accuracy: 0.9785\n",
      "Epoch 20/20\n",
      "48/48 [==============================] - 67s 1s/step - loss: 0.0981 - accuracy: 0.9694 - val_loss: 0.0696 - val_accuracy: 0.9824\n"
     ]
    }
   ],
   "source": [
    "# train the network with CLR\n",
    "start = time.time()\n",
    "model.fit_generator(training_dataset, \n",
    "    steps_per_epoch=steps_per_epoch,\n",
    "    validation_data=validation_dataset,\n",
    "    validation_steps=validation_steps,\n",
    "    epochs=EPOCHS,\n",
    "    callbacks=[clr, WandbCallback(data_type=\"image\", labels=CLASSES)])\n",
    "wandb.log({\"training_time\": time.time() - start})"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}