12_custom_models_and_training_with_tensorflow.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Chapter 12 – Custom Models and Training with TensorFlow**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_This notebook contains all the sample code in chapter 12._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<table align=\"left\">\n",
    "  <td>\n",
    "    <a target=\"_blank\" href=\"https://colab.research.google.com/github/ageron/handson-ml2/blob/master/12_custom_models_and_training_with_tensorflow.ipynb\"><img src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" />Run in Google Colab</a>\n",
    "  </td>\n",
    "</table>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First, let's import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures. We also check that Python 3.5 or later is installed (although Python 2.x may work, it is deprecated so we strongly recommend you use Python 3 instead), as well as Scikit-Learn ≥0.20 and TensorFlow ≥2.0."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Python ≥3.5 is required\n",
    "import sys\n",
    "assert sys.version_info >= (3, 5)\n",
    "\n",
    "# Scikit-Learn ≥0.20 is required\n",
    "import sklearn\n",
    "assert sklearn.__version__ >= \"0.20\"\n",
    "\n",
    "try:\n",
    "    # %tensorflow_version only exists in Colab.\n",
    "    %tensorflow_version 2.x\n",
    "    !pip install -U tqdm\n",
    "except Exception:\n",
    "    pass\n",
    "\n",
    "# TensorFlow ≥2.0 is required\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "assert tf.__version__ >= \"2.0\"\n",
    "\n",
    "# Common imports\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "# to make this notebook's output stable across runs\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)\n",
    "\n",
    "# To plot pretty figures\n",
    "%matplotlib inline\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "mpl.rc('axes', labelsize=14)\n",
    "mpl.rc('xtick', labelsize=12)\n",
    "mpl.rc('ytick', labelsize=12)\n",
    "\n",
    "# Where to save the figures\n",
    "PROJECT_ROOT_DIR = \".\"\n",
    "CHAPTER_ID = \"deep\"\n",
    "IMAGES_PATH = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID)\n",
    "os.makedirs(IMAGES_PATH, exist_ok=True)\n",
    "\n",
    "def save_fig(fig_id, tight_layout=True, fig_extension=\"png\", resolution=300):\n",
    "    path = os.path.join(IMAGES_PATH, fig_id + \".\" + fig_extension)\n",
    "    print(\"Saving figure\", fig_id)\n",
    "    if tight_layout:\n",
    "        plt.tight_layout()\n",
    "    plt.savefig(path, format=fig_extension, dpi=resolution)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tensors and operations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 3), dtype=float32, numpy=\n",
       "array([[1., 2., 3.],\n",
       "       [4., 5., 6.]], dtype=float32)>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.constant([[1., 2., 3.], [4., 5., 6.]]) # matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=int32, numpy=42>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.constant(42) # scalar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 3), dtype=float32, numpy=\n",
       "array([[1., 2., 3.],\n",
       "       [4., 5., 6.]], dtype=float32)>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t = tf.constant([[1., 2., 3.], [4., 5., 6.]])\n",
    "t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([2, 3])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tf.float32"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t.dtype"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Indexing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 2), dtype=float32, numpy=\n",
       "array([[2., 3.],\n",
       "       [5., 6.]], dtype=float32)>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t[:, 1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 1), dtype=float32, numpy=\n",
       "array([[2.],\n",
       "       [5.]], dtype=float32)>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t[..., 1, tf.newaxis]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ops"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 3), dtype=float32, numpy=\n",
       "array([[11., 12., 13.],\n",
       "       [14., 15., 16.]], dtype=float32)>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t + 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 3), dtype=float32, numpy=\n",
       "array([[ 1.,  4.,  9.],\n",
       "       [16., 25., 36.]], dtype=float32)>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.square(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 2), dtype=float32, numpy=\n",
       "array([[14., 32.],\n",
       "       [32., 77.]], dtype=float32)>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t @ tf.transpose(t)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Using `keras.backend`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3, 2), dtype=float32, numpy=\n",
       "array([[11., 26.],\n",
       "       [14., 35.],\n",
       "       [19., 46.]], dtype=float32)>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tensorflow import keras\n",
    "K = keras.backend\n",
    "K.square(K.transpose(t)) + 10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### From/To NumPy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3,), dtype=float64, numpy=array([2., 4., 5.])>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = np.array([2., 4., 5.])\n",
    "tf.constant(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1., 2., 3.],\n",
       "       [4., 5., 6.]], dtype=float32)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t.numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1., 2., 3.],\n",
       "       [4., 5., 6.]], dtype=float32)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3,), dtype=float64, numpy=array([ 4., 16., 25.])>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.square(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.,  4.,  9.],\n",
       "       [16., 25., 36.]], dtype=float32)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.square(t)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Conflicting Types"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cannot compute AddV2 as input #1(zero-based) was expected to be a float tensor but is a int32 tensor [Op:AddV2] name: add/\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    tf.constant(2.0) + tf.constant(40)\n",
    "except tf.errors.InvalidArgumentError as ex:\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cannot compute AddV2 as input #1(zero-based) was expected to be a float tensor but is a double tensor [Op:AddV2] name: add/\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    tf.constant(2.0) + tf.constant(40., dtype=tf.float64)\n",
    "except tf.errors.InvalidArgumentError as ex:\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=42.0>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t2 = tf.constant(40., dtype=tf.float64)\n",
    "tf.constant(2.0) + tf.cast(t2, tf.float32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Strings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=string, numpy=b'hello world'>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.constant(b\"hello world\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=string, numpy=b'caf\\xc3\\xa9'>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.constant(\"café\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(4,), dtype=int32, numpy=array([ 99,  97, 102, 233], dtype=int32)>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "u = tf.constant([ord(c) for c in \"café\"])\n",
    "u"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=int32, numpy=4>"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b = tf.strings.unicode_encode(u, \"UTF-8\")\n",
    "tf.strings.length(b, unit=\"UTF8_CHAR\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(4,), dtype=int32, numpy=array([ 99,  97, 102, 233], dtype=int32)>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.strings.unicode_decode(b, \"UTF-8\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### String arrays"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "p = tf.constant([\"Café\", \"Coffee\", \"caffè\", \"咖啡\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(4,), dtype=int32, numpy=array([4, 6, 5, 2], dtype=int32)>"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.strings.length(p, unit=\"UTF8_CHAR\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.RaggedTensor [[67, 97, 102, 233], [67, 111, 102, 102, 101, 101], [99, 97, 102, 102, 232], [21654, 21857]]>"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r = tf.strings.unicode_decode(p, \"UTF8\")\n",
    "r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<tf.RaggedTensor [[67, 97, 102, 233], [67, 111, 102, 102, 101, 101], [99, 97, 102, 102, 232], [21654, 21857]]>\n"
     ]
    }
   ],
   "source": [
    "print(r)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ragged tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor([ 67 111 102 102 101 101], shape=(6,), dtype=int32)\n"
     ]
    }
   ],
   "source": [
    "print(r[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<tf.RaggedTensor [[67, 111, 102, 102, 101, 101], [99, 97, 102, 102, 232]]>\n"
     ]
    }
   ],
   "source": [
    "print(r[1:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<tf.RaggedTensor [[67, 97, 102, 233], [67, 111, 102, 102, 101, 101], [99, 97, 102, 102, 232], [21654, 21857], [65, 66], [], [67]]>\n"
     ]
    }
   ],
   "source": [
    "r2 = tf.ragged.constant([[65, 66], [], [67]])\n",
    "print(tf.concat([r, r2], axis=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<tf.RaggedTensor [[67, 97, 102, 233, 68, 69, 70], [67, 111, 102, 102, 101, 101, 71], [99, 97, 102, 102, 232], [21654, 21857, 72, 73]]>\n"
     ]
    }
   ],
   "source": [
    "r3 = tf.ragged.constant([[68, 69, 70], [71], [], [72, 73]])\n",
    "print(tf.concat([r, r3], axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(4,), dtype=string, numpy=array([b'DEF', b'G', b'', b'HI'], dtype=object)>"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.strings.unicode_encode(r3, \"UTF-8\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(4, 6), dtype=int32, numpy=\n",
       "array([[   67,    97,   102,   233,     0,     0],\n",
       "       [   67,   111,   102,   102,   101,   101],\n",
       "       [   99,    97,   102,   102,   232,     0],\n",
       "       [21654, 21857,     0,     0,     0,     0]], dtype=int32)>"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.to_tensor()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sparse tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = tf.SparseTensor(indices=[[0, 1], [1, 0], [2, 3]],\n",
    "                    values=[1., 2., 3.],\n",
    "                    dense_shape=[3, 4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SparseTensor(indices=tf.Tensor(\n",
      "[[0 1]\n",
      " [1 0]\n",
      " [2 3]], shape=(3, 2), dtype=int64), values=tf.Tensor([1. 2. 3.], shape=(3,), dtype=float32), dense_shape=tf.Tensor([3 4], shape=(2,), dtype=int64))\n"
     ]
    }
   ],
   "source": [
    "print(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3, 4), dtype=float32, numpy=\n",
       "array([[0., 1., 0., 0.],\n",
       "       [2., 0., 0., 0.],\n",
       "       [0., 0., 0., 3.]], dtype=float32)>"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.sparse.to_dense(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "s2 = s * 2.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "unsupported operand type(s) for +: 'SparseTensor' and 'float'\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    s3 = s + 1.\n",
    "except TypeError as ex:\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3, 2), dtype=float32, numpy=\n",
       "array([[ 30.,  40.],\n",
       "       [ 20.,  40.],\n",
       "       [210., 240.]], dtype=float32)>"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s4 = tf.constant([[10., 20.], [30., 40.], [50., 60.], [70., 80.]])\n",
    "tf.sparse.sparse_dense_matmul(s, s4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SparseTensor(indices=tf.Tensor(\n",
      "[[0 2]\n",
      " [0 1]], shape=(2, 2), dtype=int64), values=tf.Tensor([1. 2.], shape=(2,), dtype=float32), dense_shape=tf.Tensor([3 4], shape=(2,), dtype=int64))\n"
     ]
    }
   ],
   "source": [
    "s5 = tf.SparseTensor(indices=[[0, 2], [0, 1]],\n",
    "                     values=[1., 2.],\n",
    "                     dense_shape=[3, 4])\n",
    "print(s5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "indices[1] = [0,1] is out of order. Many sparse ops require sorted indices.\n",
      "    Use `tf.sparse.reorder` to create a correctly ordered copy.\n",
      "\n",
      " [Op:SparseToDense]\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    tf.sparse.to_dense(s5)\n",
    "except tf.errors.InvalidArgumentError as ex:\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3, 4), dtype=float32, numpy=\n",
       "array([[0., 2., 1., 0.],\n",
       "       [0., 0., 0., 0.],\n",
       "       [0., 0., 0., 0.]], dtype=float32)>"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s6 = tf.sparse.reorder(s5)\n",
    "tf.sparse.to_dense(s6)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 6), dtype=int32, numpy=\n",
       "array([[ 2,  3,  4,  5,  6,  7],\n",
       "       [ 0,  7,  9, 10,  0,  0]], dtype=int32)>"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set1 = tf.constant([[2, 3, 5, 7], [7, 9, 0, 0]])\n",
    "set2 = tf.constant([[4, 5, 6], [9, 10, 0]])\n",
    "tf.sparse.to_dense(tf.sets.union(set1, set2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 3), dtype=int32, numpy=\n",
       "array([[2, 3, 7],\n",
       "       [7, 0, 0]], dtype=int32)>"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.sparse.to_dense(tf.sets.difference(set1, set2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 2), dtype=int32, numpy=\n",
       "array([[5, 0],\n",
       "       [0, 9]], dtype=int32)>"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.sparse.to_dense(tf.sets.intersection(set1, set2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "v = tf.Variable([[1., 2., 3.], [4., 5., 6.]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Variable 'UnreadVariable' shape=(2, 3) dtype=float32, numpy=\n",
       "array([[ 2.,  4.,  6.],\n",
       "       [ 8., 10., 12.]], dtype=float32)>"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v.assign(2 * v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Variable 'UnreadVariable' shape=(2, 3) dtype=float32, numpy=\n",
       "array([[ 2., 42.,  6.],\n",
       "       [ 8., 10., 12.]], dtype=float32)>"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v[0, 1].assign(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Variable 'UnreadVariable' shape=(2, 3) dtype=float32, numpy=\n",
       "array([[ 2., 42.,  0.],\n",
       "       [ 8., 10.,  1.]], dtype=float32)>"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v[:, 2].assign([0., 1.])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'ResourceVariable' object does not support item assignment\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    v[1] = [7., 8., 9.]\n",
    "except TypeError as ex:\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Variable 'UnreadVariable' shape=(2, 3) dtype=float32, numpy=\n",
       "array([[100.,  42.,   0.],\n",
       "       [  8.,  10., 200.]], dtype=float32)>"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v.scatter_nd_update(indices=[[0, 0], [1, 2]],\n",
    "                    updates=[100., 200.])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Variable 'UnreadVariable' shape=(2, 3) dtype=float32, numpy=\n",
       "array([[4., 5., 6.],\n",
       "       [1., 2., 3.]], dtype=float32)>"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sparse_delta = tf.IndexedSlices(values=[[1., 2., 3.], [4., 5., 6.]],\n",
    "                                indices=[1, 0])\n",
    "v.scatter_update(sparse_delta)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tensor Arrays"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "array = tf.TensorArray(dtype=tf.float32, size=3)\n",
    "array = array.write(0, tf.constant([1., 2.]))\n",
    "array = array.write(1, tf.constant([3., 10.]))\n",
    "array = array.write(2, tf.constant([5., 7.]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2,), dtype=float32, numpy=array([ 3., 10.], dtype=float32)>"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "array.read(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3, 2), dtype=float32, numpy=\n",
       "array([[1., 2.],\n",
       "       [0., 0.],\n",
       "       [5., 7.]], dtype=float32)>"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "array.stack()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2,), dtype=float32, numpy=array([2., 3.], dtype=float32)>"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mean, variance = tf.nn.moments(array.stack(), axes=0)\n",
    "mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2,), dtype=float32, numpy=array([4.6666665, 8.666667 ], dtype=float32)>"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "variance"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Custom loss function"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's start by loading and preparing the California housing dataset. We first load it, then split it into a training set, a validation set and a test set, and finally we scale it:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import fetch_california_housing\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "housing = fetch_california_housing()\n",
    "X_train_full, X_test, y_train_full, y_test = train_test_split(\n",
    "    housing.data, housing.target.reshape(-1, 1), random_state=42)\n",
    "X_train, X_valid, y_train, y_valid = train_test_split(\n",
    "    X_train_full, y_train_full, random_state=42)\n",
    "\n",
    "scaler = StandardScaler()\n",
    "X_train_scaled = scaler.fit_transform(X_train)\n",
    "X_valid_scaled = scaler.transform(X_valid)\n",
    "X_test_scaled = scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "def huber_fn(y_true, y_pred):\n",
    "    error = y_true - y_pred\n",
    "    is_small_error = tf.abs(error) < 1\n",
    "    squared_loss = tf.square(error) / 2\n",
    "    linear_loss  = tf.abs(error) - 0.5\n",
    "    return tf.where(is_small_error, squared_loss, linear_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAd8AAAEDCAYAAAB0/A4MAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deZyN5fvA8c89Y2KGsWXfIhkiWypJMomoJNVXe6hEql+WVJTWLxVSIkQbUpZKElkiI2QpxVcpsu+7MTNmMcv9++OaYSxjzsycc56zXO/X67xm5swz57meOTPnOs9zX/d1G2stSimllPKeEKcDUEoppYKNJl+llFLKyzT5KqWUUl6myVcppZTyMk2+SimllJdp8lVKKaW8TJOvUj7EGBNtjLHGmDJe2l9XY0yCN/allDpNk69SBWSMmWCMmX2e+6/KTKTVvR+VUsqXafJVKggYYy5yOgal1GmafJXykvNdUjbGVM+876qzNr/WGLPWGJNsjFljjGly1mNdZ4xZYoxJNMbsMcaMNcYUz/b9mMz73jHGHAKW5yHOHsaYzcaYk5kfHz/P9zdlxnbYGDPfGFMo83v1jTGLjDFxxpgEY8w6Y8yNefk9KRUMNPkq5ZveAV4ArgK2ArONMREgCQ5YAMwCGgJ3AY2AT896jIcAA7QAOruyU2PMncAHwAjgCuB9YIwx5vbM718FjAZeB2oDNwHzsj3El8A+4JrMmF4Dkl0+aqWCRCGnA1AqQLQ7T+FSQd7c/tdaOx/AGPMIsBt4APgYeA6YZq0dnrWxMaYn8Icxppy19mDm3dustc/mcb/9gM+ttR9kfr0p86z7BeB7oBpwAphlrY0HdgDrsv38JcA71tp/Mr/enMf9KxUU9MxXKff4GTnTy357oACPtyLrE2ttArAeqJt5VxPgoczLugmZST/rsnLNbI+xJh/7vZxzL1Evy7bvH5GEu80Y84UxposxJjLbtu8CHxtjfjLGvGSMqZOPGJQKeJp8lXKPRGvt5uw35Gw1u4zMjybbfWH52FcIcgacPdE3BGoBa7NtdyIfj50TC5B5tnslcA+wExgA/GOMqZT5/deQRD0TuA74nzHmUTfGoVRA0OSrlPccyvxYMdt9jXLY9tqsT4wxRZHx178z7/odqHd2ss+8JRUwxr+B5mfddz2wIesLa22atfYna+0AoAFQFGif7fv/WmtHWmtvAz4BuhUwJqUCjo75KuU9m4FdwGvGmP5AdWBgDtsOzKxS3gu8ApxEipkAhgArjTEfAuOAeKAOcLu1tkcBYxwGfGWMWYMUdbUDHkSKujDGtEcubf8MHAVuBCKBv40x4Uih2FfAdqA8krhXFTAmpQKOJl+lvMRam2qMuQ8YgxQprQVeBM5p0AH0B4YjFcV/Ae2ttScyH+d/xpgbgEHAEiAUqYj+1g0xzjTG/B9SeDUCGd990lr7feYmsUBH5A1BBLAF6GatXZo5l7gUMAE5uz+SeWz9ChqXUoHGWGudjkEppZQKKjrmq5RSSnlZnpKvMaZWZlebyZ4KSCmllAp0eT3zHQ386olAlFJKqWDhcvLNLBSJBRZ5LhyllFIq8LmUfDMbtr8B9PVsOEoppVTgc3Wq0X+BT6y1u40xOW5kjOkOdAcoUqRIk2rVqhU8Qh+VkZFBSEjO712shSNHClOmTIoXo3KP3I7N3wXy8e3atQtrLcH8v+fv/PH4kpNDKVw4A2MuPHvGH48tLzZt2nTYWlvWlW1zTb7GmEZAa6Bxbttaa8cD4wFq165tN27c6EoMfikmJobo6Ohct0tMhIgIz8fjTq4em78K5OOLjo4mNjaWtWvX5r6xnwrk5w/87/hWroR69SAyMvdt/e3Y8soYs8PVbV15CxKNdOLZaYzZj0yYv9sY83u+ogsix45BgwaQluZ0JEop5RkTJ8LevU5H4X9cuew8Hpia7et+SDLu6YmAAkmpUvDnn1BI+4gppQLU2LFOR+Cfcj3ztdYmWmv3Z92ABCDZWnsot59VkJoKL74oY8BKKRVI7rgD/vrL6Sj8U57PyTKXDFMuKlYMqlaVS89h+Vk8TimlfNR770EA1/Z5VOCWnfkIY6BnT9i/3+lIlFLKfb75BkqW1GG1/NLk6wVJSXDbbfJRKaUCwapVOpxWEPqexQvCw2HdOjkLVkopf5eaCkOHOh2Ff9MzXy9JT4d775V5v0op5a+shcaNYc8epyPxb3rm6yWFCkG3bjo+opTyb8bAL79A8eJOR+Lf9MzXi1q3htWrdZxEKeW/3npLX8PcQZOvl733HhzSGdJKKT+Ung4XXQRFizodif/Ti6BeZIyU5yullD86fBiefdbpKAKDnvl6mbXQpg3s3Ol0JEop5bqUFGjZEhISnI4kMOiZr5cZA6NHQ5UqTkeilFKuK1wYNmyAAF4R0Kv01+iAqCj46iuddqSU8g8nT0KXLjK/V7mHJl+HbNgABw86HYVSSrnmrrvk7Fe5h152dsjrr8tiC9Zq5yullG9bvx46dHA6isCiZ74Ouu02WLPG6SiUUipnR4/KsqgZGU5HElj0zNdB06bJqiBKKeWrSpeG+fOdjiLw6Jmvg0qWhI8/ho0bnY5EKaXOtWsX3HKLdrTyBI8l39hYXTneFaVKaem+Uso3VaoE77yjdSmu+OyzvG3vsZf9gweL0KuXtCNTObv7bqhQAeLinI5EKaVOO34cZs2CevWcjsS3ZWTAgAHw6KN5+zmPnnONHAl33AHx8Z7ci/8bOBDmzXM6CqWUOu3AAZkSqXKWmChLxb79NoSG5u1nPZZ8q1ZNpHRpmDMHWrSQsQN1fiNGwD33OB2FUkqJ9HSoWRNeesnpSHzX/v1w443w9deyvOIPP+Tt5z2WfMPD01m1Sro5rVsHTZvqtJqcGAOTJsEXXzgdiVJKwYIF0Lmz01H4rj//lJy2ejVUry7rG998c94ew6OXnS+7DFasgOho2LdPzoBnzvTkHv3X1VfDddc5HYVSSkmF89ixTkfhm+bNk9fqnTvh2mth5cr8jYt7vM42a45Y166QlCQtyt55R0vXz3b55acblyullFNWrZIeBMWLOx2J7xkzRpojxcfLUOFPP0H58vl7LK9McrnoIvj0U3jzTUm6zz0HPXpok+6zLV8OMTFOR6GUCmYRETIFUp2Wng59+sBTT0l188CBMGUKhIfn/zG91uHKGCnHvuwyGUv46CPYulUGq7XLk+jUST5qv2ellBMOH5ZCq/r1nY7EdyQkwP33w+zZEBYmuatLl4I/rtfbO3TqJGd35crBokXQrJkkYSV++AF69nQ6CqVUMPrqK3jvPaej8B27d0ut0uzZcjXgxx/dk3jBod7OTZvKuEL79vDXX/L1d99pwRHIE33ttU5HoZQKRj17aj1Olt9/h9tvh717oVYtScBRUe57fMcaG1avLmOcbdvKpY5WreQaerCLjJTLHN9843QkSqlgMnKknATpkJf8Hlq0kMR7ww0ya8ediRccXlihRAl5N9GzJ6SkwAMPwBtv6Duv9HS53KGUUt7Srh00auR0FM6yFt59F+68U7pXde4sc54vvtj9+3K8pX+hQjB6tIwzGAOvvioHnJLidGTOqVEDevWS3qpKKeVpq1fLmOYllzgdiXNSU+VE8NlnJQkPGgQTJsgUUE9wPPmCJN3eveVUv2hRmDwZWreWy9HBats2uRQf7FcBlFKeN38+/Puv01E45/hxqUEaN06S7dSp0lrTk5fgfSL5Zrn9dli6FCpXhmXLpPAoWNe6rVFDxhl0/EUp5UkZGfDyy8Fb8Lp9uxz7ggVQtiwsXiyLJXiaTyVfgMaNpRK6cWPYskUS8OLFTkfljJMn4fHHdVlGpZTn3HSTzDoJRitXymybDRugbl3JPc2aeWffPpd8Qc58ly6FDh0gNlYaVud1oeJAULSoFEFkZDgdiVIqUE2bJokn2EyfLqsSHTwIbdrI7JsaNby3f59MviCJZ8YMGfxOS5OFigcMCK5EZIz0wl6zRsd+lVLuN3SojHEG0/CWtdLq+N57ITkZuneXpW+93WnRZ5MvyOLE77wDH34on7/9tvzCEhOdjsy7hg6VtSOVUspd0tIkERUt6nQk3nPyJDzyyOliquHDJb+EhXk/Fp9Ovll69IC5c2WVja+/lksFwZKMjJErABUqOB2JUiqQHDwIL7wg0z2DwdGjMoQ5caIsHjFjBvTt69xZv18kX5Br8itWSGes1atlkHz9eqej8p577oHffnM6CqVUIDh6VNbsTUtzOhLv+PdfKd5dsgQqVoSff4aOHZ2NyaXka4yZbIzZZ4yJM8ZsMsZ083Rg51O3rlSnXXutLGTcvLksbBwMhg2DK690OgqlVCAoXRrWrg2Os96ff5ac8e+/0LChnLw1aeJ0VK6f+b4FVLfWFgc6AIOMMY6EX768LGB8772yoPFtt8kCx4GuenVZ8ShY5z0rpdxj61bo1i04iqwmTZKGTUePShONZcugShWnoxIuJV9r7V/W2qyGjzbzVtNjUeUiPBy+/FIWNM7IkAWOe/cO/Pmwx4/LGw6llMqvChXgscecjsKzshqHdOkibSN79YKZM6FYMacjO81YF+ewGGPGAF2BcOAP4AZrbcJZ23QHugOULVu2yfTp090a7PnMn1+ed96pTVpaCM2aHWbgwL+JiPB8Fk5ISKCYA89kerohJSXEo8fo1LF5SyAfX+/evUlPT2fUqFFOh+Ixgfz8gWeP79ixMPbtC6du3TiPPH5uvPHcnTwZwttv12Hx4nKEhFiefvpf7rxzr0f3meXGG29cY629yqWNrbUu34BQ4HpgIBB2oW2joqKstyxZYm3p0taCtQ0bWrtrl+f3uXjxYs/v5DwGDbJ2+HDP7sOpY/OWQD6+li1b2oYNGzodhkcF8vNnrWePb8UKeQ1xiqefuwMHrG3WTHJBZKS1c+d6dHfnAH6zLubTPFU7W2vTrbXLgCpAzzy9JfCgG26QQqxatWDdOrjmGmlMEYief17K45VSKi/S0qTw6KWXnI7EMzZskFkwK1ZAtWrSsapdO6ejyll+pxoVwsEx3/OpVUsScMuWsG+fJOSZM52Oyv3CwuSPasgQpyNRSvmT99+XRkWB6McfpSfz9u1w9dXSo7l+faejurBck68xppwx5j5jTDFjTKgxpi1wP7DI8+HlTenSsjJFly7SBeuuu6SDSaC1Zrz0UoiOdjoKpZQ/6dVL1qsNNB99JHOW4+Lg7rshJsY/mhK5cuZrkUvMu4FjwDtAb2vtLE8Gll8XXSSLMAweLEm3Xz944gmpeAsUFSvKu7qlS52ORCnlDyZOhP/9D0qUcDoS90lPh+eek97M6enQv78slhAR4XRkrsl1irW19hDQ0guxuI0x8OKLcNllchY8frzMbfvqK+83z/aUI0fg00+hRQunI1FK+brSpQMr8Z44AQ89JEOLhQpJf2Z/mz7lN+0l8+Oee2Qt4HLlYOFCWTB52zano3KPqlXlDD/Q5zYrpQpm40ZpRlTTp6p08m/vXqntmTlTTqbmz/e/xAsBnnxBqvtWrYJ69eDvv09XwwWCpCS44orgW+VJKeW6Z5+FLVucjsI91q2T1/A1a6T2ZcUKaNXK6ajyJ+CTL0hrxuXLZUWLQ4dkVaSpU52OquDCw2Xc11/GOJRS3jd7tswG8Xdz5kg//9275eOqVVCnjtNR5V9QJF+Q8Y45c6T4KiUF7r8fBg3y/0roMmXkOI4edToSpZQvSUyE668PjCtjI0dChw4y1vvAAzKMWKaM01EVTNAkX5CB+TFj4L33pCgrq/dnSkruP+vLqlULnqXBlFKuiYiQYlN/vjKWlgZPPy3TpDIy4LXXYPJkKFLE6cgKLqiSL0jS7d1bBuuLFoXPP5e1go8ccTqy/OvcWf4wjx93OhKllC84cQI++USWYfVXcXFytjt6tEwhnTwZXn01cFZjCrrkm6VDBxkvrVRJPl57rX8v1/fmm/DLL05HoZTyBUePwuHDTkeRfzt3yiXzuXPl8vKiRfDgg05H5V5Bm3wBGjeWhZUbN4bNm6U9WUyM01Hlz8iR0uVFKRXckpPh4ovhhRecjiR/fv1V+vOvXw+1a0vb4Ouvdzoq9wvq5AtQuTL8/LOcCR87JhXRn33mdFT5M2kSDB3qdBRKKSctWCDjpP7om29kDu+BAzIrZcWKwJmffLagT74gCyzPmCGrBaWmwqOPSoesjAynI8ub1q0ldqVU8OrQAcaNczqKvLFWThz+8x/pX/DYYzBvHpQq5XRknqPJN1NoqCzCMHasfP7WW3DvvfKH4C8qVZLK7a++cjoSpZQTRo6EWbNk9TN/cfIkPP746cvkQ4bIYgkXXeRsXJ6Wa2/nYPPEE9I5pVMn+PprGfifNQvKl3c6MtdkZMiyWkqp4NO2rX9Nwzl2TM52f/pJ4p48WVYmCgZ65nseN98slcOXXCIFWU2bwp9/Oh2Va6pWlZU+DhxwOhKllDfNny8nCZdc4nQkrtmyRYpcf/pJ4l6yJHgSL2jyzVG9etK+rGlT2LFDFmWYP9/pqFyTkAA33SRVj0qp4LBggcyN9QfLl5+e3lm/vpzkXHON01F5lybfCyhfXlZFuuceiI+XlUHGjnU6qtwVKyZrd/rT5SelVP6dOCE1K9WqOR1J7r78UhZDOHwY2rWDZcv8I2530+Sbi/BwmDIFXnpJlu978kno08f3l/ILCYGuXf27cYhSKnfx8dCoke9f6bIWJk68hAcflCKrp56C77+H4sWdjswZmnxdEBIiixdMmCBVhCNGwCuvXEFCgtORXdjTT8uKTkqpwBUZKUvt+fKVrpQUePhhmDChBiEh8P778MEH0m8/WGnyzYMuXeDHH2Xu2S+/lKFFC1neyldddZWse/n3305HopTyhE2boH9/31484fBh6UHwxRdQpEg6330HzzzjdFTO0+SbRy1bSruzKlUSWbtWCrJ+/93pqHK2dSvs3et0FEopTyhTRmZn+Kp//pHXyGXLoEoVGDXqD9q3dzoq36DJNx+iouCDD37nhhsksbVoAd9953RU5/fQQ1LcoCseKRVYNm+W1dhatXI6kvP76SeZSrR1K1x5pcweuewyHx+r8yJNvvlUokQaCxbIpejERLjzTnj3XSkq8DUzZ8KzzzodhVLKndau9d2FYD79VBp+xMbCHXdI//xKlZyOyrcE8XB3wRUuLIsw1KoFAwdKgtu0CUaN8q32bh06yE0pFRhOnpTOUL4mI0P64g8ZIl/36wdvvy0te9WZ9My3gIyRaUjTpkkyHjdO5gP70mXe0FApenjgAf9bLEIpda777/e9s97EROmJMGSIvOaMGwfDhmnizYkmXze55x75ZyhbViqir7sOtm1zOqrTypWDbt3kzYJSyr9NnCi1Jr5i/36IjpYlAYsXh7lzoXt3p6PybZp83ejaa6WooG5d2LBBqvxWrHA6KmGMFGZMn+5fKzUppU5LS4MePeRzXzmjXL9eXut+/VX6CqxYAW3aOB2V79Pk62Y1asiiDG3awKFDsiD0tGlOR3Xan3/qogtK+bM2baBoUaejEPPmQfPmsvpb9pMPlTtNvh5QogTMmSPvUFNS4L77pEOWL1RC//e/UnUYH+90JEqpvDhxQt7Y/+c/vjF8NGaM1LfEx8va5z/9JMNbyjWafD0kLEwWYRg+XP5RXn5Zei2npDgdmSTgyZOdjkIplRdbt0ovZKelp0Pv3tKbOSNDZnp8+aX0wVeu06lGHmQM9O0LNWtKpfGkSbLQ/YwZcPHFzsX16qvB3VNVKX+TnAxXXCHVw05KSJBK69mz5QTj44+hc2dnY/JXeubrBXfcAUuXyuXen3+WsZFNm5yLp1AhaYn51FPOxaCUct0778iCLk7avVsqrGfPhtKlYeFCTbwFocnXS7LaqzVqJG3hmjWDJUuci6dOHZl6pJTyfQMGnK5ydsKaNbLY/dq10lRo5Uq44Qbn4gkEmny9qEoVOQO+/XY4elSqFidOdCaWiAioV08uG/lCIZhS6vyeeQZ27HBu5aLvvpNEu2+ffFyxQhKwKhhNvl5WrBh8+y306QOpqVKENXCgM52nQkPl8ndiovf3rZRyza23yht3b7NWCkbvvFNeI7KWVHWyXiWQaPJ1QGioLMIwZox8PniwFDF4u/lFaCgMHSr7PXnSu/tWSl1YWpo0xWnbFi66yLv7Tk2Fnj2lN7O1MlXys8+8H0cg0+TroJ49ZT5wZKT8k914ozMNMHr3huXLvb9fpVTODh2SsVVvO35c5u+OGyf96qdNk/71vjC3OJBo8nVY27Yycf6SS6Qgq2lT+Osv78YwaZIkfqWUb0hIgFKl5AqZN5Petm3Sl/7HH6VPfUyM9K1X7qfJ1wdcccXpxLtjh/zxz5/vvf2HhMgZeN++3tunUipn06fLfHxvWrFCXoM2bJAWkatWybRI5Rm5Jl9jTGFjzCfGmB3GmHhjzFpjzC3eCC6YlC8PixdDp04QFyeXfT780Hv7v/56Gd9RSjnv0UdlnNVbpk2Tq1+HDsksjF9+kT71ynNcOfMtBOwCWgIlgIHAdGNMdc+FFZzCw2HqVFmMOj1dxoT79pXPPa1ECRl7fvNNnXqklJOGDq3N2rXSQcrTrJWCz/vuk9a3PXrIVbASJTy/72CXa/K11p6w1r5mrd1urc2w1s4GtgFNPB9e8AkJkX+Gzz6Tf7733oO77pIxIE+LiJACi7Q0raxQyil3373bKysDpaScnupojEwrGjvWO0lf5WPM1xhTHogCvFwWFFy6doUFC6ToYtYsaeu2e7dn9xkaCs8+C/v2hevcX6W8LCVFEmD16ic8PqXnyBG4+WYptoyIkN4DfftqRbM35am9vjEmDPgCmGit/ec83+8OdAcoW7YsMTEx7ojRJyUkJHjl+N5/P5wBA+qzdm0EjRun8Oab66lVy7OnwVOm1CAhYQ116wbmuoPeeu6cEBsbS3p6esAeHwTu8xcXV4gtWypQu7Znj2/3bnlN2b07gjJlUhg8eD0lSiTgjV9poD53+WKtdemGnCVPBX4AwnLbPioqygayxYsXe21fhw5Z26KFtWBtRIS1333n2f1lHVtKimf34xRvPnfe1rJlS9uwYUOnw/CoQHz+9u+39sAB+dyTxxcTY23p0vJa0rChtbt2eWxX5xWIz112wG/WxZzq0mVnY4wBPgHKA3dba1M99F5AnUeZMjLv7uGHpc1bx44yFuzJwqiZM+HJJz33+Eqp0xYtgo8+8uw+Jk2SSuajR6F9e1i2zJm2lUq4etl5LHA50Npa6+UmiAqkEGriRIiKgpdflvGZTZtg1CjPrM17660yJqSU8qy0NFnv21MyMmTOcNbUpd69ZYnC0FDP7VPlzpV5vpcAPYBGwH5jTELm7UGPR6fOYIxUJk6ZIsn4ww9lPvDx4+7fV1bBx4MPer/ntFLBpG1bWL/eM4+dlCSJfdAgmUkxerRcNdPE67xcz5mstTsArYHzIffdJ+0o77hDKqKbN5cFrqtXd+9+IiLkUrc2U1fKc6ZOlaEldzt4UF4jVq483T++XTv370flj7aX9FPNmkn7t8svl17QTZt6pgl7u3bS33XrVvc/tlLBbNcueOopSbzunuKzYcPp14Rq1WThFE28vkWTrx+rUUPawLVpI+9yb7xR3t262/btzqy2pFQgK11a1sp1d+L98Ud5c759O1x9tbxJr1/fvftQBafJ18+VLCnt4Lp3h+RkuPde97eIfOwxeRe9bZv7HlOpYLZsGezcCa1bu/dxx4+HW26R/vD/+Y9ctapQwb37UO6hyTcAhIVJ8dXw4fIu+qWX4JFH4ORJ9+1j7VpdeEEpd9m5E/btc9/jpafL/2ePHvJ5//6yWEJEhPv2odzLA5NUlBOMkelHl14qFcoTJ8plp2++gYsvLvjjX3klfP21/GNrpaRS+bdtm3unFp04If/z330n0w7HjZNVkZRv0zPfANOxIyxdCpUqwZIlMvbz77/ue/xWrWR+sVIq706elMXpY2Pd83h798INN0jiLVlSZj9o4vUPmnwD0JVXSpFFw4aSeK+9VhJxQRkjBV1RUQV/LKWCTUaGnJmuXi2JsqDWroVrroHff4eaNWHFCim6VP5Bk2+AqlJFijrat5d2cm3aSHu5gipfHubNk8n6SinXTZ8uq4a5o7p59my4/nrYs0fm+a9cCXXqFPxxlfdo8g1gxYpJj+bevSE1Fbp0kQ5ZGRkFe9w6deQfXynluk6dpBiyIKyF99+X5hlZY72LFnmmSYfyLE2+AS40VNrJjR4tnw8eLMUeBWkZWb26NPf4+GPPLu6gVCCwVpppbNtWsCSZlgb/93/yZjojA15/HT7/XFrNKv+jyTdIPPmkXKqKjJQpCK1aSWOO/AoJgc2bZZUlpVTOjJFmGtWq5f8x4uLg9tvlTfRFF8EXX8Arr7i/QYfyHk2+QaRdO2kzV62ajBE1bSqtKfOjUCF4+21ISID9+90bp1KB4sgRmfbXunX+e6Tv2CHjuvPmyZnzTz95dhUk5R2afINM/fpSCX3NNTIP+LrrpB1dfk2cWLCfVyqQxcbKWWt+/fqrvEn+80+ptVi5UhKx8n+afINQhQqweLG0n4uLk3Z048bl77Gef15WPjpxwr0xKuXv1q+X+fb/93/5+/lvvoGWLaWveqtW0se9Zk33xqico8k3SEVEyNjvgAHSteqJJ2QaRHp63h/r4EGZS5yW5v44lfJXH30Ef/yR95+zFoYMkTfHSUnSW33ePChVyv0xKudoe8kgFhIiizDUqiULM7z7LmzZAj165O09WblycjmskP41KQXI5eaRI/P+cydPwjvv1OaHH+TrIUPguee0sCoQ6Zmv4pFHpC1dqVLSpq5Xr8bs2ZO3xyhaVKovZ8zwTIxK+YuNG6W5TV6n4R07JkWRP/xQkfBwuez8/POaeAOVJl8FSFu6FStkTOnffyNp2jTvl8y6dIG2bT0Tn1L+wFqoXVsaX+QlaW7ZIn3YFy+G0qVTWLIE7rrLc3Eq52nyVafUri2Xjxs0iGXPHmjRAr7/3vWfr1lTph49/7w231DB6cknYf78vDW+WLZMKpo3bpTZCGPG/M7VV3suRuUbNPmqM5QpA8OGreOhh6SC+Y47YMQI15Np6X1LvrcAAB2FSURBVNLQqJFnY1TKV738sqwy5KovvoCbbpL5wLfcIom4fPkUzwWofIYmX3WOiy6yTJoEb7whSbdPH2mP50o1c1iYNABYuFA6YCkVDHbvhp49oWJFCA/PfXtr4bXX4KGHpMjq6adh1iwoXtzjoSofoclXnZcx8i7+yy/lEtrYsVJEcvy4az+/ezccOuTZGJXyFRdfLC0kXRnnTU6WpPv66zLjYORIGDVKZwsEG02+6oLuv1/a2ZUtK2NZzZtLZ6zcPPKIjGP9+qvHQ1TKUV9+Cbt2wc03577toUPSavLLL2XVsVmz8t+EQ/k3x95rxcXFcfDgQVJTU50KoUBKlCjB33//7XQYHnH2sZUrF8bSpeW4887i/PWXJNVZs+TjhRw8CIMGyfSj0FAPB62UQ5KTZbglN//8A7fdBlu3ynrbs2dDw4aej0/5JkeSb1xcHAcOHKBy5cqEh4dj/HAiW3x8PJGRkU6H4RHZj81aS1JSEnv27GHhQujSpTgLF0J0NEyaJGuU5qRCBZk3nJAgY1wB+utSQSo5WRYqefTR3Lf96Se4+25pvtGkicwiqFjR8zEq3+XIZeeDBw9SuXJlIiIi/DLxBhNjDBEREVSuXJnExIP88AM8/ri88NxzD7z1Vu6V0EOGwPTp3olXKW/ZsUPOXnPzyScy/z02Fjp2hCVLNPEqh5Jvamoq4a6UBCqfER4eTmpqKmFhsgjDO+9IccmLL8o7/5Mnc/7Z116T/rQX2kYpf7Jpk7Rlfe+9nLfJyID+/aFbN5kp8Nxz0rWqaFHvxal8l2MFV3rG61+yP1/GyCIMM2bIAg0TJkixydGj5//Z0FC59Ny4sa5+pALDSy/JMn85SUyUK0NDhsjf//jxMHSoVDcrBVrtrAqgY0f4+We5hLZkibTH+/ff829brJg0EChaVLtfKf+VlibLcE6fDg0anH+b/fulJuKbb6BECVmR6PHHvRqm8gOafFWBNGkCq1dL1eamTbK04NKl59+2VCmZYvHGG96NUSl3+eEH6Ns35/m869efnmJXo4aswdu6tXdjVP5Bk68qsCpVJOHedptcer7pJvj88/Nv266drB2slL9JS4MOHWDMmPN/f+5cmQe/c6dcBVq5EurW9W6Myn9o8s2j6Ohonn76accfIzfHjh2jfPnybNmyJddtO3XqxPDhwwu0v8jIrOUIITUVOneWJQbPvsRcurSs//vII1ItqpQ/SEuDq6+Gw4fhoovO/f7o0dIBLj4e7rtPphaVK+f9OJX/0OQboN58801uvfVWatasmeu2r7zyCoMHD+a4q70jcxAaKoswfPCBFJb897/S5zk5+cztjJHkW6lSgXanlFdYK60f58+XhUeyS0+H3r2lN3NGhrRk/eILKFLEmViV/9DkG0BOZs7lSUxM5OOPP+axxx5z6efq16/PpZdeyuTJk90Sx1NPyfzHyEiYOhVatZJuV9ndcIOc+b75plt2qZTHDB4sFf1nn8nGx8uqX++/Lx2ushYj0Ypm5Qr9M8mHjIwMXn/9dcqUKUO5cuXo168fGRkZwPkvKXft2pX27dufcV9aWhq9evWiVKlSlCpViueee+7UY4B0lho6dCg1a9YkPDyc+vXrn5Mco6Oj6dmzJ/369aNs2bI0b94cgB9++AFjzKmvAYYOHYox5pzbK6+8AkCHDh2YMmWK235Ht9wi3X+qVYMVK6QIZcOGM7cpW1bmSirly554QsZ6s9u9W9a7njNHhlIWLoSHH3YmPuWffCL5GuPMLb+++OILQkND+eWXX/jggw8YMWIE06ZNy/NjZGRksGLFCsaNG8f48eMZMWLEqe8PHDiQTz75hNGjR7NhwwYGDBhAjx49mDNnzhmPM3nyZKy1LF26lEmTJgGwdOlSmjRpcsbc3J49e7Jv375Tt2effZYKFSrQuXNnAK655hpWr15NUlJSfn8t56hfH1atkrGy7dulCOXHH09/v0QJaU85YwasXeu23SrlFlu3woMPyopFpUufvn/NGrjmGli3DqKipLAqL2v4KgUOLqzgz+rWrcvAgQOJjIwkKiqKjz76iEWLFnH//fe7/BgVK1Zk5MiRGGOoU6cOmzZt4t1336Vv376cOHGCd999lwULFtCiRQsAatSowerVqxk9ejS33XbbqcepUaPGOcVSO3bsoNJZA6qRkZGn+jUPGTKEKVOmEBMTw2WXXQZApUqVSE1NZe/evZRzY6VIhQoQEyMFWN98I2fEY8ZA9+6ntwkN1bm/yvdUqyYFhNnfqM+cKQk5MfH0XN7siVkpV/nEma+1ztzyq8FZs+srVarEwbMHNXNx7bXXnnFm2qxZM/bs2UNcXBwbNmwgOTmZdu3aUaxYsVO3sWPHnlO93KRJk3MeOykpiSI5VHy89dZbjBo1isWLF1O7du1T92e1+3TnmW+WiAhpStC/vxSo9OgB/frJ5yDjZg0awEcfSVWpUk6yVuby7tghZ7hZ9w0fDnfdJYm3SxcpwNLEq/LLpTNfY8zTQFegPjDFWtvVgzH5vLCz1g8zxpwarw0JCcGeldnzumxi1mN9//33VKtW7YL7LnqeRrFlypTh2LFj59w/aNAgPvzwwzPOeLMczewNWbZs2TzF6qqQEFmEoVYtSb7Dh8OWLTB58umuV9u3S/vJEiU8EoJSLjEG2rSBypXl69RUqWYeP16+fvNNeSOpHXJVQbh65rsXGAR86sFYAkLZsmXZt2/fGfetW7funO1WrVp1RpJeuXIllSpVonjx4tStW5fChQuzY8cOLrvssjNul1xySa4xNG7cmA1nVTe98cYbjB8/niVLlpyTeAH+/PNPKleuTPny5V091Hx59FFYsABKlpRLeDfcAHv3ylSOwYPlzHfRIo+GoFSOZs6UGoRbbpHpQrGxcOutkniLFJErOAMGaOJVBedS8rXWzrDWzgSOeDgev9eqVSvmzp3LrFmz2LhxI3379mXXrl3nbLd371569+7Nxo0b+frrrxk2bBh9+vQBZHy2X79+9OvXj08//ZTNmzezdu1aPvzwQ8Znvf2+gLZt2/L3339z5Ig8XYMGDWLkyJFMnTqVokWLsn//fvbv309ytgm4S5cupW3btm76LVzYjTdKkUrNmvD773JpL6vgas8e6QGtlBMuvRSqV5fPt22TjlULF8o0o5iYC69frVReuLXgyhjTHegOcgYYExNz3u1KlChBfHy8O3ftNenp6Zw8eZL09PRTx5CamkpaWhrx8fF06tSJ3377jUceeQSAxx9/nPbt23PkyJFT26enp3PPPfeQlJRE06ZNMcbw8MMP061bt1PbPP/885QoUYKhQ4fSs2dPIiMjadCgAb169TrjcU6ePHnO77J69eo0adKECRMm8PjjjzNs2DDi4uLOmHoEMGvWLKKjo0lOTubbb79lxowZxMfHn3Fs2SUnJ+f4nObH8OFhvPxyPdavL0mzZum88soGmjU7QsuW8PHHxShePJVy5VLctr8sCQkJbj0OXxIbG0t6enrAHh945vmLjQ3ju+8q0bnzDoyB0aOLM3DgFcTGXkT16id46631JCUl441fayD/fQbyseWZtdblG3LpeYIr20ZFRdmcbNiwIcfv+Yu4uDinQ7iguXPn2qioKJuWlpbrth988IFt06bNqa9zOjZPPG/JydY+9JCUwIWEWDtihLUZGdaOHGnt3Llu35211trFixd75oF9QMuWLW3Dhg2dDsOjPPH8HT1q7cSJ8vnUqdYWLix/kzffbG1srNt3d0GB/PcZyMdmrbXAb9bFfOoT1c7K/dq1a8dTTz3F7t27c902LCyMUaNGeSGqcxUuLJ2BXn9d2vNlterr2VMWYVi06HRVtFLuZq0UAlorTTIGDZLezCkp0lxjzhwtAFSeofN8A9gzzzzj0nbds0+6dYAxsghDrVrQtavMA966FaZMkWroOnVOV54q5U7WShtUY+Rvb9Ik+fzdd8+d46uUO7k61ahQ5rahQKgxpgiQZq3VWZnKbe6/XxobdOwoC5C3aCE9osuXl2KX6GinI1SB5Isv4PLL5e+uY0f4+WeZkz5lyrntJJVyN1cvOw8EkoD+wEOZnw/0VFAqeDVvLi0p69SBP/+UntBz50pje+2CpdypWDFZ8OPaayXxVqok61Jr4lXe4OpUo9esteas22sejk0FqUsvlcUYbroJDhyAe+6RtVIPHYJff3U6OuXv/vc/ubxcsqS0ity8GRo3htWr4cornY5OBQstuFI+qWRJOePt1k3WA+7UCV56CRYvdjoy5e+KFJHFEdq0gaNH4fbb5cxX6wqUN2nyVT4rLEw6Cw0dKoUvH38M//wjTThcKOJW6gz798OLL8LEiTBypLSN7NMHvv1WLkEr5U2afJVPMwaee05WjwkPh88+g0cegd9+czoy5W8KFZIrJ2++KStpjRkjVc2hoU5HpoKRJl/lF+68Uy4NVqggY3QvvCAvnMePOx2Z8nUpKbKkZbt20tY0MlLm7/bs6XRkKphp8lV+46qrpCimQQPYtAkGDoQff3Q6KuXr/v1Xpq6tWQOXXAK//AJeamOuVI40+Sq/UrWqjPneeiskJcEDD0g1tAeWIVYB4JZbZCrRoUMybW3VKrjiCqejUkqTb57deeedlCpViocfftjpUIJWZCR89x0884wUzXz1lbQF1HnAKou1MG6cLF954oRUyy9eLA1blPIFmnzzqFevXkyaNCnPP7dr1y6io6OpW7cuDRo04KuvvvJAdMGjUCF4/30YNQpCQqSIpk4deaFVwS09XebrPvGE9At/8UWYOlUK9pTyFZp88yg6OprIyMg8/1yhQoUYMWIEGzZsYMGCBfTu3ZsTmikK7Omn4fvvoWhRGQe++Wa5xKiC0/HjcNddsj50oUJSHT94sLxBU8qX6J+kl1SsWJFGjRoBUKFCBcqUKcPRo0cdjiow3HqrFNFUrSofa9SAv/92OirlbXv2SK/mWbOgVCkpxuva1emolDo/Tb4OWLNmDenp6VStWtXpUAJGgwZSTHPVVXLpuVkzmD/f6aiUt6xYIc/9vn1Qs6ZMKdKFOJQv0+TrZUePHqVz586MHz/e6VACTsWKsGQJ3H23XH685Rb46COno1Ke9v33kmj375eVsFauhKgop6NS6sI0+brR0KFDMcacc3vllVcASElJoWPHjvTv35/rrrvO4WgDU0QETJ8Ozz8vFa/du8OTT0rhjQos1kqVe4cOcPIkPPSQXGouU8bpyJTKnSbfPGrdujWdOnViwYIFVKlShRUrVpz6Xs+ePdm3b9+p27PPPkuFChXo3Lkz1lq6du1Kq1atdJqSh4WEwJAh0gs6JATGjpWzYa1vCxxpaVJs9/LL8vV//ysrFRUu7GxcSrmqkNMB+JuFCxcCEB8ff07Vc2Rk5Kn7hgwZwpQpU4iJieGyyy5j2bJlTJs2jQYNGjBz5kwAPv/8c+rXr+/dAwgijz0mxVd33QUzZ8qY4KJFTkelCurEiVCio2H5ckm2n30G99/vdFRK5Y0mXw946623GD16NIsXLyYqc/Dp+uuvJ0OvfXpdq1YyBtiqlayI1LQpvPpqUS3G8VM7dsDTTzdm+3YoXlyWndQRHOWPfOay82uvyQ2kWGLTJunF2qSJ3PfsszB8uHxeqRLs3QsxMacrGrt3l+XnQDogxcdLIcbtt8t9DzwAX34pnxuTvxizj+MWL178nLFdgEGDBjF69GhiYmJOJV7lrDp1ZAH15s1lKcInn7ySOXOcjkrl1erV0KgRbN9ejMsvhz/+0MSr/Ji11iO3qKgom5MNGzbk+D1ftnPnTtuyZUt7+eWX23r16tnp06ef8f3XX3/dVq1a1W7evNmhCN0jLi7uvPf76/OWJSnJ2sdu3mmrsNMaY+277zodkfu1bNnSNmzY0Okw3G7CBGsLF7a2CjvtTVF/2mPHnI7IcxYvXux0CB4TyMdmrbXAb9bFHOkzZ77+IHuXqpkzZ57RpWrQoEGMHDmSqVOnUrRoUfbv38/+/ftJTk52OGqVpUgR+GheVVp3TcNa6NtXinbS0pyOTOUkLU2ep65dZWnAW7tXpf/ow5Qs6XRkShWMJt88yN6lqnz58qe6VFlrGTZsGEeOHKF58+ZUrFjx1G358uUOR62yM9On0a/qZ0yaBGFhMHq0rHqzb5/TkamzHT4M118P770nrSJHj4ZxraZR8WetmlP+Twuu8umPP/44o0vVcV3V3T+MHUvl2FjqrX2DmjVljuiaNdC4MUybBi1bOh2gAvj9d5kellVYNWeOJGKi5fnjjTecDlGpAtEz33w4evQoPXr00C5Vfu666+Cvv+DGG+HAAfk4dKguTegka2W1qquvlsR79dXyHF1/vdORKeVemnzzKKtLVZ8+fbRLVQAoX17WfO3TR174X3hB2lIePOh0ZMHnyBFo3Rp695aOZI89Bj//DFWqOB2ZUu6nyTcPbLYuVffrrP6AUagQvPsufPedXOKcPx+uuAKdjuRFS5fKNKKffpLlIb/5RjqUFSnidGRKeYYm3zxYvnw506ZNY+bMmTRv3pxGjRqxfv16p8NSbtKhA6xfL835Dx2C9u3l7Csx0enIAldSkszhv+EGmYPdrBn8+ad0JVMqkGnBVR5k71J1vvaSyg98/TV/LV9O8xy+Xa2aNG8ZNgxefBE+/VQufX72mY47utsvv8CDD8rYrjGShN98U6rQc5TL86eUv9AzXxVcypQhtUSJC24SEiJjv7//DvXqwebNcjb8xBMQG+ulOANYYiL06ycdx7Zvh9q1ZS3mYcNySbzg0vOnlD/Q5KuCy4QJVJg3z6VNGzaE336DAQNkXHjcOLj0UhmP1IrovLNWxtVr1ZJWscbIm5y1a6Wq2SV5eP6U8mWafFVwyeOLd5Eicil07Vpo0ACOHYP//Ed6iv/9t+fCDDSbN8sc6o4dpS/7ZZfJghdvv53HoipNvipAaPJVygX16kkj/7FjpRr355+hfn1pT3n4sNPR+a7jx+H556FuXaloDg+HkSPljcs11zgdnVLO0eSrlItCQmTcd+tW6NFD5qKOHg01a8pl1KQkpyP0HUlJ8ju59FIZy01NhS5dYNs2+L//k8v4SgUzx5Kv1UEzv6LP12nlysGHH8K6dVKIFRcnBURVqsCIEcGdhFNT4ZNP5A1Jv35w9KgUVv3yC0yYIE1NlFIOJd+wsDCSgvkVyg8lJSURlmspanCpXx+WLJFmHLVqSaLp0wcqVJCmHZkLXgWFhAR541G5MnTrJgtVREXJYvdLl8r8XaXUaY4k33LlyrFnzx4SExP1jMrHWWtJTExkz549lCtXzulwCu6HH/jf22+77eGMgVtvhY0bYdYsGduMi5M5qxUqQK9eMp0mUB04AAMHyhltnz7SnKRqVZgyRcZ127WT35HbuPn5U8opjoy8FC9eHIC9e/eSmprqRAgFlpycTJEA7X139rGFhYVRvnz5U8+bX4uIIMMDz5sxcPvt0hVrzhwYPFiqeUeOhFGjoFUr6N9fPob4eaVFRgYsXCjHNm8epKfL/VddBS+/LL8Djx2jh54/pbzNsbKH4sWL+/WLeUxMDI0bN3Y6DI8I5GNjzBgqbdokc4U8wBhJPu3bw+rVkninTIFFi+RWtix07iyLw19xhUdC8JjNm2HqVEm6hw7JfcbAHXfAc8/J2K7Hefj5U8pbtOZQBZfp0ynnpTZV11wDn38u1b7jx0sh0s6dUgU8fLjMde3SRS5bN27s5suzbrJhA3z9tbyB+Oef0/dXqgQ9e8Ijj8g4r9d48flTypNcujhkjCltjPnWGHPCGLPDGPOApwNTKlBUqACvvCLTbJYulWlKxYrJmeTLL0OTJlCxoizi8NVX0oTCKfv3S6J97DF5c1CvHrz6qiTeIkWkF/P8+bBrl4z1ejXxKhVAXD3zHQ2cBMoDjYA5xph11tq/PBaZUgEmJEQWZ7j+elkwfv58mD1bziwPHJBFHD79VLYtV04u47ZsKZ216tSRJO7Os+MjR2S61B9/SAevxYthz54zt4mMlLHsBx+Em26CwoXdt3+lglmuydcYUxS4G7jCWpsALDPGzAIeBvp7OD6lAlLhwrKEYYcO0jP6f/+TaumFCyURHjwI334rtyxFi0KNGjJWXLKkzCsuX17GkcPDpXHF8eNw4kQhli2D+PjTt+PHpTnI7t2wYwds2SL3ny08XN4cREdDmzZyOVwbYijlfia3qT7GmMbAcmttRLb7+gEtrbW35/RzERER9poA7h8XGxtLyZIlnQ7DIwL52Fi7lrS0NApddZXTkeTIWln5Jy5OkmZCAqSkQFqaKz+9NvNjo1y3DAmBiAhJ6sWLy6XwyEjfHHs+xQ+ev4IK5P+/QD42gCVLlqyx1rr0x+lK8m0BfGWtrZDtvseBB6210Wdt2x3onvnlFcCfeYjb35QBArWrbyAfG+jx+Ts9Pv8VyMcGUNta69JC765cUEoAzp4TVBw456KVtXY8MB7AGPObq+8A/FEgH18gHxvo8fk7PT7/FcjHBnJ8rm7rSrXzJqCQMaZWtvsaAlpspZRSSuVDrsnXWnsCmAG8YYwpaoxpDtwBfO7p4JRSSqlA5GoTuCeBcOAgMAXo6cI0o/EFCcwPBPLxBfKxgR6fv9Pj81+BfGyQh+PLteBKKaWUUu7l5y3elVJKKf+jyVcppZTyMq8kX2NMLWNMsjFmsjf25y3GmMnGmH3GmDhjzCZjTDenY3IXY0xhY8wnmb28440xa40xtzgdlzsZY542xvxmjEkxxkxwOp6CCuQe7IH2XJ0t0P/fAvm1Mru85DpvnfmOBn710r686S2gurW2ONABGGSMaeJwTO5SCNgFtARKAAOB6caY6g7G5G57gUHAp04H4ibZe7A/CIw1xtRzNiS3CbTn6myB/v8WyK+V2bmc6zyefI0x9wGxwCJP78vbrLV/WWtTsr7MvNV0MCS3sdaesNa+Zq3dbq3NsNbOBrYBAfMPY62dYa2dCRxxOpaCytaD/WVrbYK1dhmQ1YPd7wXSc3U+gf7/FsivlVnymus8mnyNMcWBN4C+ntyPk4wxY4wxicA/wD7gB4dD8ghjTHkgCm2u4quigDRr7aZs960DAuXMN6gE4v9bIL9W5ifXefrM97/AJ9ba3R7ej2OstU8CkUALpBlJyoV/wv8YY8KAL4CJ1tp/ctteOaIYEHfWfceRv03lRwL1/y3AXyvznOvynXyNMTHGGJvDbZkxphHQGngvv/twUm7Hl31ba2165mW+KkBPZyLOG1ePzxgTgnQzOwk87VjAeZSX5y9AuNyDXfkuf/1/c5U/vlbmJr+5Lt8rdZ69otF5AuoNVAd2GlmjrBgQaoypa629Mr/79Zbcji8HhfCTcQxXjs/IE/cJUsBzq7U21dNxuUs+nz9/dqoHu7X238z7tAe7H/Hn/7d88JvXShdEk49c58nLzuORX26jzNuHwBygrQf36TXGmHLGmPuMMcWMMaHGmLbA/QRWYdlY4HLgdmttktPBuJsxppAxpggQivyzFDHG+OXS8YHegz2QnqsLCMj/tyB4rcxXrvNY8rXWJlpr92fdkMtiydbaQ57ap5dZ5LLJbuAY8A7Q21o7y9Go3MQYcwnQA/lj2m+MSci8PehwaO40EEgC+gMPZX4+0NGICiY/Pdj9RaA9V2cI8P+3gH6tzG+u097OSimllJdpe0mllFLKyzT5KqWUUl6myVcppZTyMk2+SimllJdp8lVKKaW8TJOvUkop5WWafJVSSikv0+SrlFJKeZkmX6WUUsrLNPkqFQCMMc/nsILTG07HppQ6l7aXVCoAGGMigaLZ7uoHPAi0sNZudiYqpVRONPkqFWCMMS8AzwCtrLUbnY5HKXWuQFuSS6mgZowZADwF3Git3eR0PEqp89Pkq1SAMMYMBJ4AovVSs1K+TZOvUgHAGPMK0A1oaa3d4nQ8SqkL0+SrlJ/LPON9BugAnDDGVMj8Vqy1Ntm5yJRSOdGCK6X8mDHGALFA8fN8u7W1dpGXQ1JKuUCTr1JKKeVl2mRDKaWU8jJNvkoppZSXafJVSimlvEyTr1JKKeVlmnyVUkopL9Pkq5RSSnmZJl+llFLKyzT5KqWUUl6myVcppZTysv8HMk22K5kLKGQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 576x252 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(8, 3.5))\n",
    "z = np.linspace(-4, 4, 200)\n",
    "plt.plot(z, huber_fn(0, z), \"b-\", linewidth=2, label=\"huber($z$)\")\n",
    "plt.plot(z, z**2 / 2, \"b:\", linewidth=1, label=r\"$\\frac{1}{2}z^2$\")\n",
    "plt.plot([-1, -1], [0, huber_fn(0., -1.)], \"r--\")\n",
    "plt.plot([1, 1], [0, huber_fn(0., 1.)], \"r--\")\n",
    "plt.gca().axhline(y=0, color='k')\n",
    "plt.gca().axvline(x=0, color='k')\n",
    "plt.axis([-4, 4, 0, 4])\n",
    "plt.grid(True)\n",
    "plt.xlabel(\"$z$\")\n",
    "plt.legend(fontsize=14)\n",
    "plt.title(\"Huber loss\", fontsize=14)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_shape = X_train.shape[1:]\n",
    "\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n",
    "                       input_shape=input_shape),\n",
    "    keras.layers.Dense(1),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=huber_fn, optimizer=\"nadam\", metrics=[\"mae\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples, validate on 3870 samples\n",
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 1s 91us/sample - loss: 0.6247 - mae: 0.9969 - val_loss: 0.2884 - val_mae: 0.5913\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 56us/sample - loss: 0.2193 - mae: 0.5176 - val_loss: 0.2361 - val_mae: 0.5232\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7f9f8834ec90>"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Saving/Loading Models with Custom Objects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(\"my_model_with_a_custom_loss.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.load_model(\"my_model_with_a_custom_loss.h5\",\n",
    "                                custom_objects={\"huber_fn\": huber_fn})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples, validate on 3870 samples\n",
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 1s 76us/sample - loss: 0.2056 - mae: 0.4982 - val_loss: 0.2170 - val_mae: 0.5037\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 56us/sample - loss: 0.2006 - mae: 0.4911 - val_loss: 0.2097 - val_mae: 0.4908\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7f9f88115a50>"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_huber(threshold=1.0):\n",
    "    def huber_fn(y_true, y_pred):\n",
    "        error = y_true - y_pred\n",
    "        is_small_error = tf.abs(error) < threshold\n",
    "        squared_loss = tf.square(error) / 2\n",
    "        linear_loss  = threshold * tf.abs(error) - threshold**2 / 2\n",
    "        return tf.where(is_small_error, squared_loss, linear_loss)\n",
    "    return huber_fn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=create_huber(2.0), optimizer=\"nadam\", metrics=[\"mae\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples, validate on 3870 samples\n",
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 1s 88us/sample - loss: 0.2229 - mae: 0.4893 - val_loss: 0.2525 - val_mae: 0.4973\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 58us/sample - loss: 0.2189 - mae: 0.4856 - val_loss: 0.2338 - val_mae: 0.4765\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7f9e1153d410>"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(\"my_model_with_a_custom_loss_threshold_2.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.load_model(\"my_model_with_a_custom_loss_threshold_2.h5\",\n",
    "                                custom_objects={\"huber_fn\": create_huber(2.0)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples, validate on 3870 samples\n",
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 1s 78us/sample - loss: 0.2148 - mae: 0.4796 - val_loss: 0.2111 - val_mae: 0.4713\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 57us/sample - loss: 0.2123 - mae: 0.4775 - val_loss: 0.1970 - val_mae: 0.4534\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7f9e10ad5c50>"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "class HuberLoss(keras.losses.Loss):\n",
    "    def __init__(self, threshold=1.0, **kwargs):\n",
    "        self.threshold = threshold\n",
    "        super().__init__(**kwargs)\n",
    "    def call(self, y_true, y_pred):\n",
    "        error = y_true - y_pred\n",
    "        is_small_error = tf.abs(error) < self.threshold\n",
    "        squared_loss = tf.square(error) / 2\n",
    "        linear_loss  = self.threshold * tf.abs(error) - self.threshold**2 / 2\n",
    "        return tf.where(is_small_error, squared_loss, linear_loss)\n",
    "    def get_config(self):\n",
    "        base_config = super().get_config()\n",
    "        return {**base_config, \"threshold\": self.threshold}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n",
    "                       input_shape=input_shape),\n",
    "    keras.layers.Dense(1),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=HuberLoss(2.), optimizer=\"nadam\", metrics=[\"mae\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples, validate on 3870 samples\n",
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 1s 87us/sample - loss: 0.6934 - mae: 0.8818 - val_loss: 0.3288 - val_mae: 0.5451\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 58us/sample - loss: 0.2402 - mae: 0.5077 - val_loss: 0.2315 - val_mae: 0.4861\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7f9e109bc890>"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(\"my_model_with_a_custom_loss_class.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "#model = keras.models.load_model(\"my_model_with_a_custom_loss_class.h5\", # TODO: check PR #25956\n",
    "#                                custom_objects={\"HuberLoss\": HuberLoss})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples, validate on 3870 samples\n",
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 1s 57us/sample - loss: 0.2263 - mae: 0.4946 - val_loss: 0.2285 - val_mae: 0.4851\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 57us/sample - loss: 0.2217 - mae: 0.4913 - val_loss: 0.1994 - val_mae: 0.4635\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7f9e107ac650>"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "#model = keras.models.load_model(\"my_model_with_a_custom_loss_class.h5\",  # TODO: check PR #25956\n",
    "#                                custom_objects={\"HuberLoss\": HuberLoss})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.0"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.loss.threshold"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Other Custom Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "def my_softplus(z): # return value is just tf.nn.softplus(z)\n",
    "    return tf.math.log(tf.exp(z) + 1.0)\n",
    "\n",
    "def my_glorot_initializer(shape, dtype=tf.float32):\n",
    "    stddev = tf.sqrt(2. / (shape[0] + shape[1]))\n",
    "    return tf.random.normal(shape, stddev=stddev, dtype=dtype)\n",
    "\n",
    "def my_l1_regularizer(weights):\n",
    "    return tf.reduce_sum(tf.abs(0.01 * weights))\n",
    "\n",
    "def my_positive_weights(weights): # return value is just tf.nn.relu(weights)\n",
    "    return tf.where(weights < 0., tf.zeros_like(weights), weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "layer = keras.layers.Dense(1, activation=my_softplus,\n",
    "                           kernel_initializer=my_glorot_initializer,\n",
    "                           kernel_regularizer=my_l1_regularizer,\n",
    "                           kernel_constraint=my_positive_weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n",
    "                       input_shape=input_shape),\n",
    "    keras.layers.Dense(1, activation=my_softplus,\n",
    "                       kernel_regularizer=my_l1_regularizer,\n",
    "                       kernel_constraint=my_positive_weights,\n",
    "                       kernel_initializer=my_glorot_initializer),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=\"mse\", optimizer=\"nadam\", metrics=[\"mae\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples, validate on 3870 samples\n",
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 1s 96us/sample - loss: 1.4486 - mae: 0.8727 - val_loss: 2.4208 - val_mae: 0.5681\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 58us/sample - loss: 0.5848 - mae: 0.5260 - val_loss: 1.6040 - val_mae: 0.5122\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7f9e11d466d0>"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(\"my_model_with_many_custom_parts.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.load_model(\n",
    "    \"my_model_with_many_custom_parts.h5\",\n",
    "    custom_objects={\n",
    "       \"my_l1_regularizer\": my_l1_regularizer,\n",
    "       \"my_positive_weights\": my_positive_weights,\n",
    "       \"my_glorot_initializer\": my_glorot_initializer,\n",
    "       \"my_softplus\": my_softplus,\n",
    "    })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyL1Regularizer(keras.regularizers.Regularizer):\n",
    "    def __init__(self, factor):\n",
    "        self.factor = factor\n",
    "    def __call__(self, weights):\n",
    "        return tf.reduce_sum(tf.abs(self.factor * weights))\n",
    "    def get_config(self):\n",
    "        return {\"factor\": self.factor}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n",
    "                       input_shape=input_shape),\n",
    "    keras.layers.Dense(1, activation=my_softplus,\n",
    "                       kernel_regularizer=MyL1Regularizer(0.01),\n",
    "                       kernel_constraint=my_positive_weights,\n",
    "                       kernel_initializer=my_glorot_initializer),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=\"mse\", optimizer=\"nadam\", metrics=[\"mae\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples, validate on 3870 samples\n",
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 1s 90us/sample - loss: 1.4486 - mae: 0.8727 - val_loss: 2.4208 - val_mae: 0.5681\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 58us/sample - loss: 0.5848 - mae: 0.5260 - val_loss: 1.6040 - val_mae: 0.5122\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7f9e12d2d8d0>"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(\"my_model_with_many_custom_parts.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.load_model(\n",
    "    \"my_model_with_many_custom_parts.h5\",\n",
    "    custom_objects={\n",
    "       \"MyL1Regularizer\": MyL1Regularizer,\n",
    "       \"my_positive_weights\": my_positive_weights,\n",
    "       \"my_glorot_initializer\": my_glorot_initializer,\n",
    "       \"my_softplus\": my_softplus,\n",
    "    })"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Custom Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n",
    "                       input_shape=input_shape),\n",
    "    keras.layers.Dense(1),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=\"mse\", optimizer=\"nadam\", metrics=[create_huber(2.0)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples\n",
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 1s 73us/sample - loss: 2.0326 - huber_fn: 0.9038\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 47us/sample - loss: 0.5862 - huber_fn: 0.2682\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7f9e1296d390>"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Warning**: if you use the same function as the loss and a metric, you may be surprised to see different results. This is generally just due to floating point precision errors: even though the mathematical equations are equivalent, the operations are not run in the same order, which can lead to small differences. Moreover, when using sample weights, there's more than just precision errors:\n",
    "* the loss since the start of the epoch is the mean of all batch losses seen so far. Each batch loss is the sum of the weighted instance losses divided by the _batch size_ (not the sum of weights, so the batch loss is _not_ the weighted mean of the losses).\n",
    "* the metric since the start of the epoch is equal to the sum of weighted instance losses divided by sum of all weights seen so far. In other words, it is the weighted mean of all the instance losses. Not the same thing.\n",
    "\n",
    "If you do the math, you will find that loss = metric * mean of sample weights (plus some floating point precision error)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=create_huber(2.0), optimizer=\"nadam\", metrics=[create_huber(2.0)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Train on 11610 samples\n",
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 1s 84us/sample - loss: 0.1174 - huber_fn: 0.2375\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 52us/sample - loss: 0.1137 - huber_fn: 0.2306\n"
     ]
    }
   ],
   "source": [
    "sample_weight = np.random.rand(len(y_train))\n",
    "history = model.fit(X_train_scaled, y_train, epochs=2, sample_weight=sample_weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.11744692476015477, 0.11787289870922694)"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "history.history[\"loss\"][0], history.history[\"huber_fn\"][0] * sample_weight.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Streaming metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=0.8>"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision = keras.metrics.Precision()\n",
    "precision([0, 1, 1, 1, 0, 1, 0, 1], [1, 1, 0, 1, 0, 1, 0, 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=0.5>"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision([0, 1, 0, 0, 1, 0, 1, 1], [1, 0, 1, 1, 0, 0, 0, 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=0.5>"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision.result()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Variable 'true_positives:0' shape=(1,) dtype=float32, numpy=array([4.], dtype=float32)>,\n",
       " <tf.Variable 'false_positives:0' shape=(1,) dtype=float32, numpy=array([4.], dtype=float32)>]"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision.variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "precision.reset_states()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Creating a streaming metric:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "class HuberMetric(keras.metrics.Metric):\n",
    "    def __init__(self, threshold=1.0, **kwargs):\n",
    "        super().__init__(**kwargs) # handles base args (e.g., dtype)\n",
    "        self.threshold = threshold\n",
    "        #self.huber_fn = create_huber(threshold) # TODO: investigate why this fails\n",
    "        self.total = self.add_weight(\"total\", initializer=\"zeros\")\n",
    "        self.count = self.add_weight(\"count\", initializer=\"zeros\")\n",
    "    def huber_fn(self, y_true, y_pred): # workaround\n",
    "        error = y_true - y_pred\n",
    "        is_small_error = tf.abs(error) < self.threshold\n",
    "        squared_loss = tf.square(error) / 2\n",
    "        linear_loss  = self.threshold * tf.abs(error) - self.threshold**2 / 2\n",
    "        return tf.where(is_small_error, squared_loss, linear_loss)\n",
    "    def update_state(self, y_true, y_pred, sample_weight=None):\n",
    "        metric = self.huber_fn(y_true, y_pred)\n",
    "        self.total.assign_add(tf.reduce_sum(metric))\n",
    "        self.count.assign_add(tf.cast(tf.size(y_true), tf.float32))\n",
    "    def result(self):\n",
    "        return self.total / self.count\n",
    "    def get_config(self):\n",
    "        base_config = super().get_config()\n",
    "        return {**base_config, \"threshold\": self.threshold}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Warning**: when running the following cell, if you get autograph warnings such as `WARNING:tensorflow:AutoGraph could not transform [...] and will run it as-is`, then please install version 0.2.2 of the gast library (e.g., by running `!pip install gast==0.2.2`), then restart the kernel and run this notebook again from the beginning (see [autograph issue #1](https://github.com/tensorflow/autograph/issues/1) for more details):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=14.0>"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m = HuberMetric(2.)\n",
    "\n",
    "# total = 2 * |10 - 2| - 2²/2 = 14\n",
    "# count = 1\n",
    "# result = 14 / 1 = 14\n",
    "m(tf.constant([[2.]]), tf.constant([[10.]])) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=7.0>"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# total = total + (|1 - 0|² / 2) + (2 * |9.25 - 5| - 2² / 2) = 14 + 7 = 21\n",
    "# count = count + 2 = 3\n",
    "# result = total / count = 21 / 3 = 7\n",
    "m(tf.constant([[0.], [5.]]), tf.constant([[1.], [9.25]]))\n",
    "\n",
    "m.result()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Variable 'total:0' shape=() dtype=float32, numpy=21.0>,\n",
       " <tf.Variable 'count:0' shape=() dtype=float32, numpy=3.0>]"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Variable 'total:0' shape=() dtype=float32, numpy=0.0>,\n",
       " <tf.Variable 'count:0' shape=() dtype=float32, numpy=0.0>]"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.reset_states()\n",
    "m.variables"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's check that the `HuberMetric` class works well:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n",
    "                       input_shape=input_shape),\n",
    "    keras.layers.Dense(1),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=create_huber(2.0), optimizer=\"nadam\", metrics=[HuberMetric(2.0)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples\n",
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 1s 74us/sample - loss: 0.8646 - huber_metric: 0.8646\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 48us/sample - loss: 0.2564 - huber_metric: 0.2564\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7f9e1261d550>"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled.astype(np.float32), y_train.astype(np.float32), epochs=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(\"my_model_with_a_custom_metric.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "#model = keras.models.load_model(\"my_model_with_a_custom_metric.h5\",           # TODO: check PR #25956\n",
    "#                                custom_objects={\"huber_fn\": create_huber(2.0),\n",
    "#                                                \"HuberMetric\": HuberMetric})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples\n",
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 1s 48us/sample - loss: 0.2349 - huber_metric: 0.2349\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 48us/sample - loss: 0.2286 - huber_metric: 0.2286\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7f9e11b18c90>"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled.astype(np.float32), y_train.astype(np.float32), epochs=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Warning**: In TF 2.2, tf.keras adds an extra first metric in `model.metrics` at position 0 (see [TF issue #38150](https://github.com/tensorflow/tensorflow/issues/38150)). This forces us to use `model.metrics[-1]` rather than `model.metrics[0]` to access the `HuberMetric`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.0"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.metrics[-1].threshold"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Looks like it works fine! More simply, we could have created the class like this:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "class HuberMetric(keras.metrics.Mean):\n",
    "    def __init__(self, threshold=1.0, name='HuberMetric', dtype=None):\n",
    "        self.threshold = threshold\n",
    "        self.huber_fn = create_huber(threshold)\n",
    "        super().__init__(name=name, dtype=dtype)\n",
    "    def update_state(self, y_true, y_pred, sample_weight=None):\n",
    "        metric = self.huber_fn(y_true, y_pred)\n",
    "        super(HuberMetric, self).update_state(metric, sample_weight)\n",
    "    def get_config(self):\n",
    "        base_config = super().get_config()\n",
    "        return {**base_config, \"threshold\": self.threshold}        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This class handles shapes better, and it also supports sample weights."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n",
    "                       input_shape=input_shape),\n",
    "    keras.layers.Dense(1),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=keras.losses.Huber(2.0), optimizer=\"nadam\", weighted_metrics=[HuberMetric(2.0)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Train on 11610 samples\n",
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 1s 94us/sample - loss: 0.4330 - HuberMetric: 0.8725\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 55us/sample - loss: 0.1286 - HuberMetric: 0.2592\n"
     ]
    }
   ],
   "source": [
    "sample_weight = np.random.rand(len(y_train))\n",
    "history = model.fit(X_train_scaled.astype(np.float32), y_train.astype(np.float32),\n",
    "                    epochs=2, sample_weight=sample_weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.43299586604013207, 0.43299578338297107)"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "history.history[\"loss\"][0], history.history[\"HuberMetric\"][0] * sample_weight.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(\"my_model_with_a_custom_metric_v2.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "#model = keras.models.load_model(\"my_model_with_a_custom_metric_v2.h5\",        # TODO: check PR #25956\n",
    "#                                custom_objects={\"HuberMetric\": HuberMetric})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples\n",
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 1s 75us/sample - loss: 0.2363 - HuberMetric: 0.2363\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 49us/sample - loss: 0.2274 - HuberMetric: 0.2274\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7f9e11b1ea50>"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled.astype(np.float32), y_train.astype(np.float32), epochs=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Warning**: In TF 2.2, tf.keras adds an extra first metric in `model.metrics` at position 0 (see [TF issue #38150](https://github.com/tensorflow/tensorflow/issues/38150)). This forces us to use `model.metrics[-1]` rather than `model.metrics[0]` to access the `HuberMetric`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.0"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.metrics[-1].threshold"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Custom Layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "exponential_layer = keras.layers.Lambda(lambda x: tf.exp(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3,), dtype=float32, numpy=array([0.36787945, 1.        , 2.7182817 ], dtype=float32)>"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "exponential_layer([-1., 0., 1.])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Adding an exponential layer at the output of a regression model can be useful if the values to predict are positive and with very different scales (e.g., 0.001, 10., 10000):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples, validate on 3870 samples\n",
      "Epoch 1/5\n",
      "11610/11610 [==============================] - 1s 79us/sample - loss: 1652499274263224909824.0000 - val_loss: inf\n",
      "Epoch 2/5\n",
      "11610/11610 [==============================] - 1s 52us/sample - loss: 310651844421202280448.0000 - val_loss: inf\n",
      "Epoch 3/5\n",
      "11610/11610 [==============================] - 1s 52us/sample - loss: 310651844421202280448.0000 - val_loss: inf\n",
      "Epoch 4/5\n",
      "11610/11610 [==============================] - 1s 51us/sample - loss: 310651844421202280448.0000 - val_loss: inf\n",
      "Epoch 5/5\n",
      "11610/11610 [==============================] - 1s 52us/sample - loss: 310651844421202280448.0000 - val_loss: inf\n",
      "5160/5160 [==============================] - 0s 20us/sample - loss: 561417.6699\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "561417.6698863751"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"relu\", input_shape=input_shape),\n",
    "    keras.layers.Dense(1),\n",
    "    exponential_layer\n",
    "])\n",
    "model.compile(loss=\"mse\", optimizer=\"nadam\")\n",
    "model.fit(X_train_scaled, y_train, epochs=5,\n",
    "          validation_data=(X_valid_scaled, y_valid))\n",
    "model.evaluate(X_test_scaled, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyDense(keras.layers.Layer):\n",
    "    def __init__(self, units, activation=None, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.units = units\n",
    "        self.activation = keras.activations.get(activation)\n",
    "\n",
    "    def build(self, batch_input_shape):\n",
    "        self.kernel = self.add_weight(\n",
    "            name=\"kernel\", shape=[batch_input_shape[-1], self.units],\n",
    "            initializer=\"glorot_normal\")\n",
    "        self.bias = self.add_weight(\n",
    "            name=\"bias\", shape=[self.units], initializer=\"zeros\")\n",
    "        super().build(batch_input_shape) # must be at the end\n",
    "\n",
    "    def call(self, X):\n",
    "        return self.activation(X @ self.kernel + self.bias)\n",
    "\n",
    "    def compute_output_shape(self, batch_input_shape):\n",
    "        return tf.TensorShape(batch_input_shape.as_list()[:-1] + [self.units])\n",
    "\n",
    "    def get_config(self):\n",
    "        base_config = super().get_config()\n",
    "        return {**base_config, \"units\": self.units,\n",
    "                \"activation\": keras.activations.serialize(self.activation)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.Sequential([\n",
    "    MyDense(30, activation=\"relu\", input_shape=input_shape),\n",
    "    MyDense(1)\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples, validate on 3870 samples\n",
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 1s 80us/sample - loss: 2.1425 - val_loss: 1.2779\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 52us/sample - loss: 0.6367 - val_loss: 0.5267\n",
      "5160/5160 [==============================] - 0s 19us/sample - loss: 0.5359\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.5359284200409586"
      ]
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.compile(loss=\"mse\", optimizer=\"nadam\")\n",
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))\n",
    "model.evaluate(X_test_scaled, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(\"my_model_with_a_custom_layer.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.load_model(\"my_model_with_a_custom_layer.h5\",\n",
    "                                custom_objects={\"MyDense\": MyDense})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyMultiLayer(keras.layers.Layer):\n",
    "    def call(self, X):\n",
    "        X1, X2 = X\n",
    "        return X1 + X2, X1 * X2\n",
    "\n",
    "    def compute_output_shape(self, batch_input_shape):\n",
    "        batch_input_shape1, batch_input_shape2 = batch_input_shape\n",
    "        return [batch_input_shape1, batch_input_shape2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs1 = keras.layers.Input(shape=[2])\n",
    "inputs2 = keras.layers.Input(shape=[2])\n",
    "outputs1, outputs2 = MyMultiLayer()((inputs1, inputs2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's create a layer with a different behavior during training and testing:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [],
   "source": [
    "class AddGaussianNoise(keras.layers.Layer):\n",
    "    def __init__(self, stddev, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.stddev = stddev\n",
    "\n",
    "    def call(self, X, training=None):\n",
    "        if training:\n",
    "            noise = tf.random.normal(tf.shape(X), stddev=self.stddev)\n",
    "            return X + noise\n",
    "        else:\n",
    "            return X\n",
    "\n",
    "    def compute_output_shape(self, batch_input_shape):\n",
    "        return batch_input_shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples, validate on 3870 samples\n",
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 1s 79us/sample - loss: 0.4776 - val_loss: 0.5223\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 51us/sample - loss: 0.4213 - val_loss: 0.3807\n",
      "5160/5160 [==============================] - 0s 20us/sample - loss: 0.3977\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.39774651444235515"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.compile(loss=\"mse\", optimizer=\"nadam\")\n",
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))\n",
    "model.evaluate(X_test_scaled, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Custom Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_new_scaled = X_test_scaled"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ResidualBlock(keras.layers.Layer):\n",
    "    def __init__(self, n_layers, n_neurons, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.hidden = [keras.layers.Dense(n_neurons, activation=\"elu\",\n",
    "                                          kernel_initializer=\"he_normal\")\n",
    "                       for _ in range(n_layers)]\n",
    "\n",
    "    def call(self, inputs):\n",
    "        Z = inputs\n",
    "        for layer in self.hidden:\n",
    "            Z = layer(Z)\n",
    "        return inputs + Z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ResidualRegressor(keras.models.Model):\n",
    "    def __init__(self, output_dim, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.hidden1 = keras.layers.Dense(30, activation=\"elu\",\n",
    "                                          kernel_initializer=\"he_normal\")\n",
    "        self.block1 = ResidualBlock(2, 30)\n",
    "        self.block2 = ResidualBlock(2, 30)\n",
    "        self.out = keras.layers.Dense(output_dim)\n",
    "\n",
    "    def call(self, inputs):\n",
    "        Z = self.hidden1(inputs)\n",
    "        for _ in range(1 + 3):\n",
    "            Z = self.block1(Z)\n",
    "        Z = self.block2(Z)\n",
    "        return self.out(Z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples\n",
      "Epoch 1/5\n",
      "11610/11610 [==============================] - 2s 139us/sample - loss: 11.4379\n",
      "Epoch 2/5\n",
      "11610/11610 [==============================] - 1s 82us/sample - loss: 1.7623\n",
      "Epoch 3/5\n",
      "11610/11610 [==============================] - 1s 82us/sample - loss: 1.4746\n",
      "Epoch 4/5\n",
      "11610/11610 [==============================] - 1s 82us/sample - loss: 0.6453\n",
      "Epoch 5/5\n",
      "11610/11610 [==============================] - 1s 82us/sample - loss: 0.5190\n",
      "5160/5160 [==============================] - 0s 42us/sample - loss: 0.6061\n"
     ]
    }
   ],
   "source": [
    "model = ResidualRegressor(1)\n",
    "model.compile(loss=\"mse\", optimizer=\"nadam\")\n",
    "history = model.fit(X_train_scaled, y_train, epochs=5)\n",
    "score = model.evaluate(X_test_scaled, y_test)\n",
    "y_pred = model.predict(X_new_scaled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/haesun/anaconda3/envs/homl2/lib/python3.7/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1786: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n",
      "INFO:tensorflow:Assets written to: my_custom_model.ckpt/assets\n"
     ]
    }
   ],
   "source": [
    "model.save(\"my_custom_model.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.load_model(\"my_custom_model.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples\n",
      "Epoch 1/5\n",
      "11610/11610 [==============================] - 2s 149us/sample - loss: 0.6721\n",
      "Epoch 2/5\n",
      "11610/11610 [==============================] - 1s 82us/sample - loss: 0.4372\n",
      "Epoch 3/5\n",
      "11610/11610 [==============================] - 1s 82us/sample - loss: 0.5437\n",
      "Epoch 4/5\n",
      "11610/11610 [==============================] - 1s 82us/sample - loss: 0.4754\n",
      "Epoch 5/5\n",
      "11610/11610 [==============================] - 1s 82us/sample - loss: 0.4668\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(X_train_scaled, y_train, epochs=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We could have defined the model using the sequential API instead:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [],
   "source": [
    "block1 = ResidualBlock(2, 30)\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"elu\", kernel_initializer=\"he_normal\"),\n",
    "    block1, block1, block1, block1,\n",
    "    ResidualBlock(2, 30),\n",
    "    keras.layers.Dense(1)\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples\n",
      "Epoch 1/5\n",
      "11610/11610 [==============================] - 1s 121us/sample - loss: 0.9281\n",
      "Epoch 2/5\n",
      "11610/11610 [==============================] - 1s 72us/sample - loss: 0.4480\n",
      "Epoch 3/5\n",
      "11610/11610 [==============================] - 1s 72us/sample - loss: 0.7364\n",
      "Epoch 4/5\n",
      "11610/11610 [==============================] - 1s 72us/sample - loss: 0.4003\n",
      "Epoch 5/5\n",
      "11610/11610 [==============================] - 1s 72us/sample - loss: 0.5058\n",
      "5160/5160 [==============================] - 0s 35us/sample - loss: 0.4289\n"
     ]
    }
   ],
   "source": [
    "model.compile(loss=\"mse\", optimizer=\"nadam\")\n",
    "history = model.fit(X_train_scaled, y_train, epochs=5)\n",
    "score = model.evaluate(X_test_scaled, y_test)\n",
    "y_pred = model.predict(X_new_scaled)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Losses and Metrics Based on Model Internals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ReconstructingRegressor(keras.models.Model):\n",
    "    def __init__(self, output_dim, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.hidden = [keras.layers.Dense(30, activation=\"selu\",\n",
    "                                          kernel_initializer=\"lecun_normal\")\n",
    "                       for _ in range(5)]\n",
    "        self.out = keras.layers.Dense(output_dim)\n",
    "        # TODO: check https://github.com/tensorflow/tensorflow/issues/26260\n",
    "        #self.reconstruction_mean = keras.metrics.Mean(name=\"reconstruction_error\")\n",
    "\n",
    "    def build(self, batch_input_shape):\n",
    "        n_inputs = batch_input_shape[-1]\n",
    "        self.reconstruct = keras.layers.Dense(n_inputs)\n",
    "        super().build(batch_input_shape)\n",
    "\n",
    "    def call(self, inputs, training=None):\n",
    "        Z = inputs\n",
    "        for layer in self.hidden:\n",
    "            Z = layer(Z)\n",
    "        reconstruction = self.reconstruct(Z)\n",
    "        recon_loss = tf.reduce_mean(tf.square(reconstruction - inputs))\n",
    "        self.add_loss(0.05 * recon_loss)\n",
    "        #if training:\n",
    "        #    result = self.reconstruction_mean(recon_loss)\n",
    "        #    self.add_metric(result)\n",
    "        return self.out(Z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples\n",
      "Epoch 1/2\n",
      "11610/11610 [==============================] - 2s 134us/sample - loss: 0.7647\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 78us/sample - loss: 0.4113\n"
     ]
    }
   ],
   "source": [
    "model = ReconstructingRegressor(1)\n",
    "model.compile(loss=\"mse\", optimizer=\"nadam\")\n",
    "history = model.fit(X_train_scaled, y_train, epochs=2)\n",
    "y_pred = model.predict(X_test_scaled)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Computing Gradients with Autodiff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "def f(w1, w2):\n",
    "    return 3 * w1 ** 2 + 2 * w1 * w2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "36.000003007075065"
      ]
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w1, w2 = 5, 3\n",
    "eps = 1e-6\n",
    "(f(w1 + eps, w2) - f(w1, w2)) / eps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10.000000003174137"
      ]
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(f(w1, w2 + eps) - f(w1, w2)) / eps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "w1, w2 = tf.Variable(5.), tf.Variable(3.)\n",
    "with tf.GradientTape() as tape:\n",
    "    z = f(w1, w2)\n",
    "\n",
    "gradients = tape.gradient(z, [w1, w2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(), dtype=float32, numpy=36.0>,\n",
       " <tf.Tensor: shape=(), dtype=float32, numpy=10.0>]"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GradientTape.gradient can only be called once on non-persistent tapes.\n"
     ]
    }
   ],
   "source": [
    "with tf.GradientTape() as tape:\n",
    "    z = f(w1, w2)\n",
    "\n",
    "dz_dw1 = tape.gradient(z, w1)\n",
    "try:\n",
    "    dz_dw2 = tape.gradient(z, w2)\n",
    "except RuntimeError as ex:\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.GradientTape(persistent=True) as tape:\n",
    "    z = f(w1, w2)\n",
    "\n",
    "dz_dw1 = tape.gradient(z, w1)\n",
    "dz_dw2 = tape.gradient(z, w2) # works now!\n",
    "del tape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(<tf.Tensor: shape=(), dtype=float32, numpy=36.0>,\n",
       " <tf.Tensor: shape=(), dtype=float32, numpy=10.0>)"
      ]
     },
     "execution_count": 171,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dz_dw1, dz_dw2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [],
   "source": [
    "c1, c2 = tf.constant(5.), tf.constant(3.)\n",
    "with tf.GradientTape() as tape:\n",
    "    z = f(c1, c2)\n",
    "\n",
    "gradients = tape.gradient(z, [c1, c2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[None, None]"
      ]
     },
     "execution_count": 173,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.GradientTape() as tape:\n",
    "    tape.watch(c1)\n",
    "    tape.watch(c2)\n",
    "    z = f(c1, c2)\n",
    "\n",
    "gradients = tape.gradient(z, [c1, c2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(), dtype=float32, numpy=36.0>,\n",
       " <tf.Tensor: shape=(), dtype=float32, numpy=10.0>]"
      ]
     },
     "execution_count": 175,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(), dtype=float32, numpy=136.0>,\n",
       " <tf.Tensor: shape=(), dtype=float32, numpy=30.0>]"
      ]
     },
     "execution_count": 176,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with tf.GradientTape() as tape:\n",
    "    z1 = f(w1, w2 + 2.)\n",
    "    z2 = f(w1, w2 + 5.)\n",
    "    z3 = f(w1, w2 + 7.)\n",
    "\n",
    "tape.gradient([z1, z2, z3], [w1, w2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.GradientTape(persistent=True) as tape:\n",
    "    z1 = f(w1, w2 + 2.)\n",
    "    z2 = f(w1, w2 + 5.)\n",
    "    z3 = f(w1, w2 + 7.)\n",
    "\n",
    "tf.reduce_sum(tf.stack([tape.gradient(z, [w1, w2]) for z in (z1, z2, z3)]), axis=0)\n",
    "del tape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.GradientTape(persistent=True) as hessian_tape:\n",
    "    with tf.GradientTape() as jacobian_tape:\n",
    "        z = f(w1, w2)\n",
    "    jacobians = jacobian_tape.gradient(z, [w1, w2])\n",
    "hessians = [hessian_tape.gradient(jacobian, [w1, w2])\n",
    "            for jacobian in jacobians]\n",
    "del hessian_tape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(), dtype=float32, numpy=36.0>,\n",
       " <tf.Tensor: shape=(), dtype=float32, numpy=10.0>]"
      ]
     },
     "execution_count": 179,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jacobians"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[<tf.Tensor: shape=(), dtype=float32, numpy=6.0>,\n",
       "  <tf.Tensor: shape=(), dtype=float32, numpy=2.0>],\n",
       " [<tf.Tensor: shape=(), dtype=float32, numpy=2.0>, None]]"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hessians"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(), dtype=float32, numpy=30.0>, None]"
      ]
     },
     "execution_count": 181,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def f(w1, w2):\n",
    "    return 3 * w1 ** 2 + tf.stop_gradient(2 * w1 * w2)\n",
    "\n",
    "with tf.GradientTape() as tape:\n",
    "    z = f(w1, w2)\n",
    "\n",
    "tape.gradient(z, [w1, w2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(), dtype=float32, numpy=nan>]"
      ]
     },
     "execution_count": 182,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = tf.Variable(100.)\n",
    "with tf.GradientTape() as tape:\n",
    "    z = my_softplus(x)\n",
    "\n",
    "tape.gradient(z, [x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=30.0>"
      ]
     },
     "execution_count": 183,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.math.log(tf.exp(tf.constant(30., dtype=tf.float32)) + 1.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(1,), dtype=float32, numpy=array([nan], dtype=float32)>]"
      ]
     },
     "execution_count": 184,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = tf.Variable([100.])\n",
    "with tf.GradientTape() as tape:\n",
    "    z = my_softplus(x)\n",
    "\n",
    "tape.gradient(z, [x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [],
   "source": [
    "@tf.custom_gradient\n",
    "def my_better_softplus(z):\n",
    "    exp = tf.exp(z)\n",
    "    def my_softplus_gradients(grad):\n",
    "        return grad / (1 + 1 / exp)\n",
    "    return tf.math.log(exp + 1), my_softplus_gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [],
   "source": [
    "def my_better_softplus(z):\n",
    "    return tf.where(z > 30., z, tf.math.log(tf.exp(z) + 1.))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(<tf.Tensor: shape=(1,), dtype=float32, numpy=array([1000.], dtype=float32)>,\n",
       " [<tf.Tensor: shape=(1,), dtype=float32, numpy=array([nan], dtype=float32)>])"
      ]
     },
     "execution_count": 187,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = tf.Variable([1000.])\n",
    "with tf.GradientTape() as tape:\n",
    "    z = my_better_softplus(x)\n",
    "\n",
    "z, tape.gradient(z, [x])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Computing Gradients Using Autodiff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [],
   "source": [
    "l2_reg = keras.regularizers.l2(0.05)\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"elu\", kernel_initializer=\"he_normal\",\n",
    "                       kernel_regularizer=l2_reg),\n",
    "    keras.layers.Dense(1, kernel_regularizer=l2_reg)\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [],
   "source": [
    "def random_batch(X, y, batch_size=32):\n",
    "    idx = np.random.randint(len(X), size=batch_size)\n",
    "    return X[idx], y[idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_status_bar(iteration, total, loss, metrics=None):\n",
    "    metrics = \" - \".join([\"{}: {:.4f}\".format(m.name, m.result())\n",
    "                         for m in [loss] + (metrics or [])])\n",
    "    end = \"\" if iteration < total else \"\\n\"\n",
    "    print(\"\\r{}/{} - \".format(iteration, total) + metrics,\n",
    "          end=end)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50/50 - loss: 0.0900 - mean_square: 858.5000\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "mean_loss = keras.metrics.Mean(name=\"loss\")\n",
    "mean_square = keras.metrics.Mean(name=\"mean_square\")\n",
    "for i in range(1, 50 + 1):\n",
    "    loss = 1 / i\n",
    "    mean_loss(loss)\n",
    "    mean_square(i ** 2)\n",
    "    print_status_bar(i, 50, mean_loss, [mean_square])\n",
    "    time.sleep(0.05)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A fancier version with a progress bar:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [],
   "source": [
    "def progress_bar(iteration, total, size=30):\n",
    "    running = iteration < total\n",
    "    c = \">\" if running else \"=\"\n",
    "    p = (size - 1) * iteration // total\n",
    "    fmt = \"{{:-{}d}}/{{}} [{{}}]\".format(len(str(total)))\n",
    "    params = [iteration, total, \"=\" * p + c + \".\" * (size - p - 1)]\n",
    "    return fmt.format(*params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' 3500/10000 [=>....]'"
      ]
     },
     "execution_count": 194,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "progress_bar(3500, 10000, size=6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_status_bar(iteration, total, loss, metrics=None, size=30):\n",
    "    metrics = \" - \".join([\"{}: {:.4f}\".format(m.name, m.result())\n",
    "                         for m in [loss] + (metrics or [])])\n",
    "    end = \"\" if iteration < total else \"\\n\"\n",
    "    print(\"\\r{} - {}\".format(progress_bar(iteration, total), metrics), end=end)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50/50 [==============================] - loss: 0.0900 - mean_square: 858.5000\n"
     ]
    }
   ],
   "source": [
    "mean_loss = keras.metrics.Mean(name=\"loss\")\n",
    "mean_square = keras.metrics.Mean(name=\"mean_square\")\n",
    "for i in range(1, 50 + 1):\n",
    "    loss = 1 / i\n",
    "    mean_loss(loss)\n",
    "    mean_square(i ** 2)\n",
    "    print_status_bar(i, 50, mean_loss, [mean_square])\n",
    "    time.sleep(0.05)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = 5\n",
    "batch_size = 32\n",
    "n_steps = len(X_train) // batch_size\n",
    "optimizer = keras.optimizers.Nadam(lr=0.01)\n",
    "loss_fn = keras.losses.mean_squared_error\n",
    "mean_loss = keras.metrics.Mean()\n",
    "metrics = [keras.metrics.MeanAbsoluteError()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "WARNING:tensorflow:Layer dense is casting an input tensor from dtype float64 to the layer's dtype of float32, which is new behavior in TensorFlow 2.  The layer has dtype float32 because it's dtype defaults to floatx.\n",
      "\n",
      "If you intended to run this layer in float32, you can safely ignore this warning. If in doubt, this warning is likely only an issue if you are porting a TensorFlow 1.X model to TensorFlow 2.\n",
      "\n",
      "To change all layers to have dtype float64 by default, call `tf.keras.backend.set_floatx('float64')`. To change just this layer, pass dtype='float64' to the layer constructor. If you are the author of this layer, you can disable autocasting by passing autocast=False to the base Layer constructor.\n",
      "\n",
      "11610/11610 [==============================] - mean: 1.3955 - mean_absolute_error: 0.5722\n",
      "Epoch 2/5\n",
      "11610/11610 [==============================] - mean: 0.6774 - mean_absolute_error: 0.5280\n",
      "Epoch 3/5\n",
      "11610/11610 [==============================] - mean: 0.6351 - mean_absolute_error: 0.5177\n",
      "Epoch 4/5\n",
      "11610/11610 [==============================] - mean: 0.6384 - mean_absolute_error: 0.5181\n",
      "Epoch 5/5\n",
      "11610/11610 [==============================] - mean: 0.6440 - mean_absolute_error: 0.5222\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(1, n_epochs + 1):\n",
    "    print(\"Epoch {}/{}\".format(epoch, n_epochs))\n",
    "    for step in range(1, n_steps + 1):\n",
    "        X_batch, y_batch = random_batch(X_train_scaled, y_train)\n",
    "        with tf.GradientTape() as tape:\n",
    "            y_pred = model(X_batch)\n",
    "            main_loss = tf.reduce_mean(loss_fn(y_batch, y_pred))\n",
    "            loss = tf.add_n([main_loss] + model.losses)\n",
    "        gradients = tape.gradient(loss, model.trainable_variables)\n",
    "        optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n",
    "        for variable in model.variables:\n",
    "            if variable.constraint is not None:\n",
    "                variable.assign(variable.constraint(variable))\n",
    "        mean_loss(loss)\n",
    "        for metric in metrics:\n",
    "            metric(y_batch, y_pred)\n",
    "        print_status_bar(step * batch_size, len(y_train), mean_loss, metrics)\n",
    "    print_status_bar(len(y_train), len(y_train), mean_loss, metrics)\n",
    "    for metric in [mean_loss] + metrics:\n",
    "        metric.reset_states()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "14707283021347b4a5942f460201587c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='All epochs', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3cb414b89b8240588717e57a55f9b025",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Epoch 1/5', max=362.0, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b193995f83f543328f10aa146e2990e1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Epoch 2/5', max=362.0, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6375839d7f0c4afe968f37bb167507ff",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Epoch 3/5', max=362.0, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ab550558c8924aa9813619dac4222d78",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Epoch 4/5', max=362.0, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6bc40ecbb1a140ee926f986c1659b805",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Epoch 5/5', max=362.0, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    from tqdm.notebook import trange\n",
    "    from collections import OrderedDict\n",
    "    with trange(1, n_epochs + 1, desc=\"All epochs\") as epochs:\n",
    "        for epoch in epochs:\n",
    "            with trange(1, n_steps + 1, desc=\"Epoch {}/{}\".format(epoch, n_epochs)) as steps:\n",
    "                for step in steps:\n",
    "                    X_batch, y_batch = random_batch(X_train_scaled, y_train)\n",
    "                    with tf.GradientTape() as tape:\n",
    "                        y_pred = model(X_batch)\n",
    "                        main_loss = tf.reduce_mean(loss_fn(y_batch, y_pred))\n",
    "                        loss = tf.add_n([main_loss] + model.losses)\n",
    "                    gradients = tape.gradient(loss, model.trainable_variables)\n",
    "                    optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n",
    "                    for variable in model.variables:\n",
    "                        if variable.constraint is not None:\n",
    "                            variable.assign(variable.constraint(variable))                    \n",
    "                    status = OrderedDict()\n",
    "                    mean_loss(loss)\n",
    "                    status[\"loss\"] = mean_loss.result().numpy()\n",
    "                    for metric in metrics:\n",
    "                        metric(y_batch, y_pred)\n",
    "                        status[metric.name] = metric.result().numpy()\n",
    "                    steps.set_postfix(status)\n",
    "            for metric in [mean_loss] + metrics:\n",
    "                metric.reset_states()\n",
    "except ImportError as ex:\n",
    "    print(\"To run this cell, please install tqdm, ipywidgets and restart Jupyter\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TensorFlow Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cube(x):\n",
    "    return x ** 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8"
      ]
     },
     "execution_count": 202,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cube(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=8.0>"
      ]
     },
     "execution_count": 203,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cube(tf.constant(2.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.eager.def_function.Function at 0x7f9e10b1ac50>"
      ]
     },
     "execution_count": 204,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf_cube = tf.function(cube)\n",
    "tf_cube"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=int32, numpy=8>"
      ]
     },
     "execution_count": 205,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf_cube(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=8.0>"
      ]
     },
     "execution_count": 206,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf_cube(tf.constant(2.0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TF Functions and Concrete Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.framework.func_graph.FuncGraph at 0x7f9e12175290>"
      ]
     },
     "execution_count": 207,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concrete_function = tf_cube.get_concrete_function(tf.constant(2.0))\n",
    "concrete_function.graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=8.0>"
      ]
     },
     "execution_count": 208,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concrete_function(tf.constant(2.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 209,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concrete_function is tf_cube.get_concrete_function(tf.constant(2.0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Exploring Function Definitions and Graphs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.framework.func_graph.FuncGraph at 0x7f9e12175290>"
      ]
     },
     "execution_count": 210,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concrete_function.graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Operation 'x' type=Placeholder>,\n",
       " <tf.Operation 'pow/y' type=Const>,\n",
       " <tf.Operation 'pow' type=Pow>,\n",
       " <tf.Operation 'Identity' type=Identity>]"
      ]
     },
     "execution_count": 211,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ops = concrete_function.graph.get_operations()\n",
    "ops"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor 'x:0' shape=() dtype=float32>,\n",
       " <tf.Tensor 'pow/y:0' shape=() dtype=float32>]"
      ]
     },
     "execution_count": 212,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pow_op = ops[2]\n",
    "list(pow_op.inputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor 'pow:0' shape=() dtype=float32>]"
      ]
     },
     "execution_count": 213,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pow_op.outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Operation 'x' type=Placeholder>"
      ]
     },
     "execution_count": 214,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concrete_function.graph.get_operation_by_name('x')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'Identity:0' shape=() dtype=float32>"
      ]
     },
     "execution_count": 215,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concrete_function.graph.get_tensor_by_name('Identity:0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "name: \"__inference_cube_999576\"\n",
       "input_arg {\n",
       "  name: \"x\"\n",
       "  type: DT_FLOAT\n",
       "}\n",
       "output_arg {\n",
       "  name: \"identity\"\n",
       "  type: DT_FLOAT\n",
       "}"
      ]
     },
     "execution_count": 216,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concrete_function.function_def.signature"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How TF Functions Trace Python Functions to Extract Their Computation Graphs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [],
   "source": [
    "@tf.function\n",
    "def tf_cube(x):\n",
    "    print(\"print:\", x)\n",
    "    return x ** 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "print: Tensor(\"x:0\", shape=(), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "result = tf_cube(tf.constant(2.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=8.0>"
      ]
     },
     "execution_count": 219,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "print: 2\n",
      "print: 3\n",
      "print: Tensor(\"x:0\", shape=(1, 2), dtype=float32)\n",
      "print: Tensor(\"x:0\", shape=(2, 2), dtype=float32)\n",
      "WARNING:tensorflow:5 out of the last 5 calls to <function tf_cube at 0x7f9e112a6290> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
      "print: Tensor(\"x:0\", shape=(3, 2), dtype=float32)\n",
      "WARNING:tensorflow:6 out of the last 6 calls to <function tf_cube at 0x7f9e112a6290> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n"
     ]
    }
   ],
   "source": [
    "result = tf_cube(2)\n",
    "result = tf_cube(3)\n",
    "result = tf_cube(tf.constant([[1., 2.]])) # New shape: trace!\n",
    "result = tf_cube(tf.constant([[3., 4.], [5., 6.]])) # New shape: trace!\n",
    "result = tf_cube(tf.constant([[7., 8.], [9., 10.], [11., 12.]])) # no trace"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It is also possible to specify a particular input signature:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {},
   "outputs": [],
   "source": [
    "@tf.function(input_signature=[tf.TensorSpec([None, 28, 28], tf.float32)])\n",
    "def shrink(images):\n",
    "    print(\"Tracing\", images)\n",
    "    return images[:, ::2, ::2] # drop half the rows and columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tracing Tensor(\"images:0\", shape=(None, 28, 28), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "img_batch_1 = tf.random.uniform(shape=[100, 28, 28])\n",
    "img_batch_2 = tf.random.uniform(shape=[50, 28, 28])\n",
    "preprocessed_images = shrink(img_batch_1) # Traces the function.\n",
    "preprocessed_images = shrink(img_batch_2) # Reuses the same concrete function."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python inputs incompatible with input_signature:\n",
      "  inputs: (\n",
      "    tf.Tensor(\n",
      "[[[0.7413678  0.62854624]\n",
      "  [0.01738465 0.3431449 ]]\n",
      "\n",
      " [[0.51063764 0.3777541 ]\n",
      "  [0.07321596 0.02137029]]], shape=(2, 2, 2), dtype=float32))\n",
      "  input_signature: (\n",
      "    TensorSpec(shape=(None, 28, 28), dtype=tf.float32, name=None))\n"
     ]
    }
   ],
   "source": [
    "img_batch_3 = tf.random.uniform(shape=[2, 2, 2])\n",
    "try:\n",
    "    preprocessed_images = shrink(img_batch_3)  # rejects unexpected types or shapes\n",
    "except ValueError as ex:\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Using Autograph To Capture Control Flow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A \"static\" `for` loop using `range()`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {},
   "outputs": [],
   "source": [
    "@tf.function\n",
    "def add_10(x):\n",
    "    for i in range(10):\n",
    "        x += 1\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=int32, numpy=15>"
      ]
     },
     "execution_count": 226,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "add_10(tf.constant(5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Operation 'x' type=Placeholder>,\n",
       " <tf.Operation 'add/y' type=Const>,\n",
       " <tf.Operation 'add' type=AddV2>,\n",
       " <tf.Operation 'add_1/y' type=Const>,\n",
       " <tf.Operation 'add_1' type=AddV2>,\n",
       " <tf.Operation 'add_2/y' type=Const>,\n",
       " <tf.Operation 'add_2' type=AddV2>,\n",
       " <tf.Operation 'add_3/y' type=Const>,\n",
       " <tf.Operation 'add_3' type=AddV2>,\n",
       " <tf.Operation 'add_4/y' type=Const>,\n",
       " <tf.Operation 'add_4' type=AddV2>,\n",
       " <tf.Operation 'add_5/y' type=Const>,\n",
       " <tf.Operation 'add_5' type=AddV2>,\n",
       " <tf.Operation 'add_6/y' type=Const>,\n",
       " <tf.Operation 'add_6' type=AddV2>,\n",
       " <tf.Operation 'add_7/y' type=Const>,\n",
       " <tf.Operation 'add_7' type=AddV2>,\n",
       " <tf.Operation 'add_8/y' type=Const>,\n",
       " <tf.Operation 'add_8' type=AddV2>,\n",
       " <tf.Operation 'add_9/y' type=Const>,\n",
       " <tf.Operation 'add_9' type=AddV2>,\n",
       " <tf.Operation 'Identity' type=Identity>]"
      ]
     },
     "execution_count": 227,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "add_10.get_concrete_function(tf.constant(5)).graph.get_operations()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A \"dynamic\" loop using `tf.while_loop()`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "metadata": {},
   "outputs": [],
   "source": [
    "@tf.function\n",
    "def add_10(x):\n",
    "    condition = lambda i, x: tf.less(i, 10)\n",
    "    body = lambda i, x: (tf.add(i, 1), tf.add(x, 1))\n",
    "    final_i, final_x = tf.while_loop(condition, body, [tf.constant(0), x])\n",
    "    return final_x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=int32, numpy=15>"
      ]
     },
     "execution_count": 229,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "add_10(tf.constant(5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Operation 'x' type=Placeholder>,\n",
       " <tf.Operation 'Const' type=Const>,\n",
       " <tf.Operation 'while/maximum_iterations' type=Const>,\n",
       " <tf.Operation 'while/loop_counter' type=Const>,\n",
       " <tf.Operation 'while' type=StatelessWhile>,\n",
       " <tf.Operation 'Identity' type=Identity>]"
      ]
     },
     "execution_count": 230,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "add_10.get_concrete_function(tf.constant(5)).graph.get_operations()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A \"dynamic\" `for` loop using `tf.range()` (captured by autograph):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {},
   "outputs": [],
   "source": [
    "@tf.function\n",
    "def add_10(x):\n",
    "    for i in tf.range(10):\n",
    "        x = x + 1\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Operation 'x' type=Placeholder>,\n",
       " <tf.Operation 'range/start' type=Const>,\n",
       " <tf.Operation 'range/limit' type=Const>,\n",
       " <tf.Operation 'range/delta' type=Const>,\n",
       " <tf.Operation 'range' type=Range>,\n",
       " <tf.Operation 'while/maximum_iterations' type=Const>,\n",
       " <tf.Operation 'while/loop_counter' type=Const>,\n",
       " <tf.Operation 'while' type=StatelessWhile>,\n",
       " <tf.Operation 'Identity' type=Identity>]"
      ]
     },
     "execution_count": 232,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "add_10.get_concrete_function(tf.constant(0)).graph.get_operations()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Handling Variables and Other Resources in TF Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "metadata": {},
   "outputs": [],
   "source": [
    "counter = tf.Variable(0)\n",
    "\n",
    "@tf.function\n",
    "def increment(counter, c=1):\n",
    "    return counter.assign_add(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 234,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=int32, numpy=2>"
      ]
     },
     "execution_count": 234,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "increment(counter)\n",
    "increment(counter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 235,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "name: \"counter\"\n",
       "type: DT_RESOURCE"
      ]
     },
     "execution_count": 235,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function_def = increment.get_concrete_function(counter).function_def\n",
    "function_def.signature.input_arg[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 236,
   "metadata": {},
   "outputs": [],
   "source": [
    "counter = tf.Variable(0)\n",
    "\n",
    "@tf.function\n",
    "def increment(c=1):\n",
    "    return counter.assign_add(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=int32, numpy=2>"
      ]
     },
     "execution_count": 237,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "increment()\n",
    "increment()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "name: \"assignaddvariableop_resource\"\n",
       "type: DT_RESOURCE"
      ]
     },
     "execution_count": 238,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function_def = increment.get_concrete_function().function_def\n",
    "function_def.signature.input_arg[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Counter:\n",
    "    def __init__(self):\n",
    "        self.counter = tf.Variable(0)\n",
    "\n",
    "    @tf.function\n",
    "    def increment(self, c=1):\n",
    "        return self.counter.assign_add(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=int32, numpy=2>"
      ]
     },
     "execution_count": 240,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c = Counter()\n",
    "c.increment()\n",
    "c.increment()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"def tf__add_10(x):\\n  do_return = False\\n  retval_ = ag__.UndefinedReturnValue()\\n  with ag__.FunctionScope('add_10', 'fscope', ag__.ConversionOptions(recursive=True, user_requested=True, optional_features=(), internal_convert_user_code=True)) as fscope:\\n\\n    def get_state():\\n      return ()\\n\\n    def set_state(_):\\n      pass\\n\\n    def loop_body(iterates, x):\\n      i = iterates\\n      x += 1\\n      return x,\\n    x, = ag__.for_stmt(ag__.converted_call(tf.range, (10,), None, fscope), None, loop_body, get_state, set_state, (x,), ('x',), ())\\n    do_return = True\\n    retval_ = fscope.mark_return_value(x)\\n  do_return,\\n  return ag__.retval(retval_)\\n\""
      ]
     },
     "execution_count": 241,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "@tf.function\n",
    "def add_10(x):\n",
    "    for i in tf.range(10):\n",
    "        x += 1\n",
    "    return x\n",
    "\n",
    "tf.autograph.to_code(add_10.python_function)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "metadata": {},
   "outputs": [],
   "source": [
    "def display_tf_code(func):\n",
    "    from IPython.display import display, Markdown\n",
    "    if hasattr(func, \"python_function\"):\n",
    "        func = func.python_function\n",
    "    code = tf.autograph.to_code(func)\n",
    "    display(Markdown('```python\\n{}\\n```'.format(code)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "```python\n",
       "def tf__add_10(x):\n",
       "  do_return = False\n",
       "  retval_ = ag__.UndefinedReturnValue()\n",
       "  with ag__.FunctionScope('add_10', 'fscope', ag__.ConversionOptions(recursive=True, user_requested=True, optional_features=(), internal_convert_user_code=True)) as fscope:\n",
       "\n",
       "    def get_state():\n",
       "      return ()\n",
       "\n",
       "    def set_state(_):\n",
       "      pass\n",
       "\n",
       "    def loop_body(iterates, x):\n",
       "      i = iterates\n",
       "      x += 1\n",
       "      return x,\n",
       "    x, = ag__.for_stmt(ag__.converted_call(tf.range, (10,), None, fscope), None, loop_body, get_state, set_state, (x,), ('x',), ())\n",
       "    do_return = True\n",
       "    retval_ = fscope.mark_return_value(x)\n",
       "  do_return,\n",
       "  return ag__.retval(retval_)\n",
       "\n",
       "```"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display_tf_code(add_10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using TF Functions with tf.keras (or Not)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "By default, tf.keras will automatically convert your custom code into TF Functions, no need to use\n",
    "`tf.function()`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Custom loss function\n",
    "def my_mse(y_true, y_pred):\n",
    "    print(\"Tracing loss my_mse()\")\n",
    "    return tf.reduce_mean(tf.square(y_pred - y_true))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Custom metric function\n",
    "def my_mae(y_true, y_pred):\n",
    "    print(\"Tracing metric my_mae()\")\n",
    "    return tf.reduce_mean(tf.abs(y_pred - y_true))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 246,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Custom layer\n",
    "class MyDense(keras.layers.Layer):\n",
    "    def __init__(self, units, activation=None, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.units = units\n",
    "        self.activation = keras.activations.get(activation)\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        self.kernel = self.add_weight(name='kernel', \n",
    "                                      shape=(input_shape[1], self.units),\n",
    "                                      initializer='uniform',\n",
    "                                      trainable=True)\n",
    "        self.biases = self.add_weight(name='bias', \n",
    "                                      shape=(self.units,),\n",
    "                                      initializer='zeros',\n",
    "                                      trainable=True)\n",
    "        super().build(input_shape)\n",
    "\n",
    "    def call(self, X):\n",
    "        print(\"Tracing MyDense.call()\")\n",
    "        return self.activation(X @ self.kernel + self.biases)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Custom model\n",
    "class MyModel(keras.models.Model):\n",
    "    def __init__(self, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.hidden1 = MyDense(30, activation=\"relu\")\n",
    "        self.hidden2 = MyDense(30, activation=\"relu\")\n",
    "        self.output_ = MyDense(1)\n",
    "\n",
    "    def call(self, input):\n",
    "        print(\"Tracing MyModel.call()\")\n",
    "        hidden1 = self.hidden1(input)\n",
    "        hidden2 = self.hidden2(hidden1)\n",
    "        concat = keras.layers.concatenate([input, hidden2])\n",
    "        output = self.output_(concat)\n",
    "        return output\n",
    "\n",
    "model = MyModel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=my_mse, optimizer=\"nadam\", metrics=[my_mae])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing metric my_mae()\n",
      "Tracing loss my_mse()\n",
      "Train on 11610 samples, validate on 3870 samples\n",
      "Epoch 1/2\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "10976/11610 [===========================>..] - ETA: 0s - loss: 1.3269 - my_mae: 0.7980Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "11610/11610 [==============================] - 1s 95us/sample - loss: 1.2838 - my_mae: 0.7826 - val_loss: 0.4503 - val_my_mae: 0.4879\n",
      "Epoch 2/2\n",
      "11610/11610 [==============================] - 1s 60us/sample - loss: 0.4418 - my_mae: 0.4782 - val_loss: 0.7718 - val_my_mae: 0.4583\n",
      "5160/5160 [==============================] - 0s 22us/sample - loss: 0.4174 - my_mae: 0.4584\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.4173873696216317, 0.4584151]"
      ]
     },
     "execution_count": 250,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))\n",
    "model.evaluate(X_test_scaled, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can turn this off by creating the model with `dynamic=True` (or calling `super().__init__(dynamic=True, **kwargs)` in the model's constructor):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = MyModel(dynamic=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=my_mse, optimizer=\"nadam\", metrics=[my_mae])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Not the custom code will be called at each iteration. Let's fit, validate and evaluate with tiny datasets to avoid getting too much output:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[5.504646301269531, 2.056248]"
      ]
     },
     "execution_count": 254,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled[:64], y_train[:64], epochs=1,\n",
    "          validation_data=(X_valid_scaled[:64], y_valid[:64]), verbose=0)\n",
    "model.evaluate(X_test_scaled[:64], y_test[:64], verbose=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Alternatively, you can compile a model with `run_eagerly=True`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 255,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = MyModel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=my_mse, optimizer=\"nadam\", metrics=[my_mae], run_eagerly=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[5.504646301269531, 2.056248]"
      ]
     },
     "execution_count": 258,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled[:64], y_train[:64], epochs=1,\n",
    "          validation_data=(X_valid_scaled[:64], y_valid[:64]), verbose=0)\n",
    "model.evaluate(X_test_scaled[:64], y_test[:64], verbose=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Custom Optimizers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Defining custom optimizers is not very common, but in case you are one of the happy few who gets to write one, here is an example:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyMomentumOptimizer(keras.optimizers.Optimizer):\n",
    "    def __init__(self, learning_rate=0.001, momentum=0.9, name=\"MyMomentumOptimizer\", **kwargs):\n",
    "        \"\"\"Call super().__init__() and use _set_hyper() to store hyperparameters\"\"\"\n",
    "        super().__init__(name, **kwargs)\n",
    "        self._set_hyper(\"learning_rate\", kwargs.get(\"lr\", learning_rate)) # handle lr=learning_rate\n",
    "        self._set_hyper(\"decay\", self._initial_decay) # \n",
    "        self._set_hyper(\"momentum\", momentum)\n",
    "    \n",
    "    def _create_slots(self, var_list):\n",
    "        \"\"\"For each model variable, create the optimizer variable associated with it.\n",
    "        TensorFlow calls these optimizer variables \"slots\".\n",
    "        For momentum optimization, we need one momentum slot per model variable.\n",
    "        \"\"\"\n",
    "        for var in var_list:\n",
    "            self.add_slot(var, \"momentum\")\n",
    "\n",
    "    @tf.function\n",
    "    def _resource_apply_dense(self, grad, var):\n",
    "        \"\"\"Update the slots and perform one optimization step for one model variable\n",
    "        \"\"\"\n",
    "        var_dtype = var.dtype.base_dtype\n",
    "        lr_t = self._decayed_lr(var_dtype) # handle learning rate decay\n",
    "        momentum_var = self.get_slot(var, \"momentum\")\n",
    "        momentum_hyper = self._get_hyper(\"momentum\", var_dtype)\n",
    "        momentum_var.assign(momentum_var * momentum_hyper - (1. - momentum_hyper)* grad)\n",
    "        var.assign_add(momentum_var * lr_t)\n",
    "\n",
    "    def _resource_apply_sparse(self, grad, var):\n",
    "        raise NotImplementedError\n",
    "\n",
    "    def get_config(self):\n",
    "        base_config = super().get_config()\n",
    "        return {\n",
    "            **base_config,\n",
    "            \"learning_rate\": self._serialize_hyperparameter(\"learning_rate\"),\n",
    "            \"decay\": self._serialize_hyperparameter(\"decay\"),\n",
    "            \"momentum\": self._serialize_hyperparameter(\"momentum\"),\n",
    "        }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11610 samples\n",
      "Epoch 1/5\n",
      "11610/11610 [==============================] - 1s 48us/sample - loss: 3.8096\n",
      "Epoch 2/5\n",
      "11610/11610 [==============================] - 0s 32us/sample - loss: 1.4864\n",
      "Epoch 3/5\n",
      "11610/11610 [==============================] - 0s 31us/sample - loss: 0.9173\n",
      "Epoch 4/5\n",
      "11610/11610 [==============================] - 0s 31us/sample - loss: 0.7593\n",
      "Epoch 5/5\n",
      "11610/11610 [==============================] - 0s 30us/sample - loss: 0.7036\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7f9e10ddad10>"
      ]
     },
     "execution_count": 261,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = keras.models.Sequential([keras.layers.Dense(1, input_shape=[8])])\n",
    "model.compile(loss=\"mse\", optimizer=MyMomentumOptimizer())\n",
    "model.fit(X_train_scaled, y_train, epochs=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exercises"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. to 11.\n",
    "See Appendix A."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 12. Implement a custom layer that performs _Layer Normalization_\n",
    "_We will use this type of layer in Chapter 15 when using Recurrent Neural Networks._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### a.\n",
    "_Exercise: The `build()` method should define two trainable weights *α* and *β*, both of shape `input_shape[-1:]` and data type `tf.float32`. *α* should be initialized with 1s, and *β* with 0s._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Solution: see below."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b.\n",
    "_Exercise: The `call()` method should compute the mean_ μ _and standard deviation_ σ _of each instance's features. For this, you can use `tf.nn.moments(inputs, axes=-1, keepdims=True)`, which returns the mean μ and the variance σ<sup>2</sup> of all instances (compute the square root of the variance to get the standard deviation). Then the function should compute and return *α*⊗(*X* - μ)/(σ + ε) + *β*, where ⊗ represents itemwise multiplication (`*`) and ε is a smoothing term (small constant to avoid division by zero, e.g., 0.001)._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 262,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LayerNormalization(keras.layers.Layer):\n",
    "    def __init__(self, eps=0.001, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.eps = eps\n",
    "\n",
    "    def build(self, batch_input_shape):\n",
    "        self.alpha = self.add_weight(\n",
    "            name=\"alpha\", shape=batch_input_shape[-1:],\n",
    "            initializer=\"ones\")\n",
    "        self.beta = self.add_weight(\n",
    "            name=\"beta\", shape=batch_input_shape[-1:],\n",
    "            initializer=\"zeros\")\n",
    "        super().build(batch_input_shape) # must be at the end\n",
    "\n",
    "    def call(self, X):\n",
    "        mean, variance = tf.nn.moments(X, axes=-1, keepdims=True)\n",
    "        return self.alpha * (X - mean) / (tf.sqrt(variance + self.eps)) + self.beta\n",
    "\n",
    "    def compute_output_shape(self, batch_input_shape):\n",
    "        return batch_input_shape\n",
    "\n",
    "    def get_config(self):\n",
    "        base_config = super().get_config()\n",
    "        return {**base_config, \"eps\": self.eps}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note that making _ε_ a hyperparameter (`eps`) was not compulsory. Also note that it's preferable to compute `tf.sqrt(variance + self.eps)` rather than `tf.sqrt(variance) + self.eps`. Indeed, the derivative of sqrt(z) is undefined when z=0, so training will bomb whenever the variance vector has at least one component equal to 0. Adding _ε_ within the square root guarantees that this will never happen."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### c.\n",
    "_Exercise: Ensure that your custom layer produces the same (or very nearly the same) output as the `keras.layers.LayerNormalization` layer._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's create one instance of each class, apply them to some data (e.g., the training set), and ensure that the difference is negligeable."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 263,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=3.80914e-08>"
      ]
     },
     "execution_count": 263,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = X_train.astype(np.float32)\n",
    "\n",
    "custom_layer_norm = LayerNormalization()\n",
    "keras_layer_norm = keras.layers.LayerNormalization()\n",
    "\n",
    "tf.reduce_mean(keras.losses.mean_absolute_error(\n",
    "    keras_layer_norm(X), custom_layer_norm(X)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Yep, that's close enough. To be extra sure, let's make alpha and beta completely random and compare again:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 264,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=1.695759e-08>"
      ]
     },
     "execution_count": 264,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "random_alpha = np.random.rand(X.shape[-1])\n",
    "random_beta = np.random.rand(X.shape[-1])\n",
    "\n",
    "custom_layer_norm.set_weights([random_alpha, random_beta])\n",
    "keras_layer_norm.set_weights([random_alpha, random_beta])\n",
    "\n",
    "tf.reduce_mean(keras.losses.mean_absolute_error(\n",
    "    keras_layer_norm(X), custom_layer_norm(X)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Still a negligeable difference! Our custom layer works fine."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 13. Train a model using a custom training loop to tackle the Fashion MNIST dataset\n",
    "_The Fashion MNIST dataset was introduced in Chapter 10._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### a.\n",
    "_Exercise: Display the epoch, iteration, mean training loss, and mean accuracy over each epoch (updated at each iteration), as well as the validation loss and accuracy at the end of each epoch._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 265,
   "metadata": {},
   "outputs": [],
   "source": [
    "(X_train_full, y_train_full), (X_test, y_test) = keras.datasets.fashion_mnist.load_data()\n",
    "X_train_full = X_train_full.astype(np.float32) / 255.\n",
    "X_valid, X_train = X_train_full[:5000], X_train_full[5000:]\n",
    "y_valid, y_train = y_train_full[:5000], y_train_full[5000:]\n",
    "X_test = X_test.astype(np.float32) / 255."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 266,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 267,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.Sequential([\n",
    "    keras.layers.Flatten(input_shape=[28, 28]),\n",
    "    keras.layers.Dense(100, activation=\"relu\"),\n",
    "    keras.layers.Dense(10, activation=\"softmax\"),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 268,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = 5\n",
    "batch_size = 32\n",
    "n_steps = len(X_train) // batch_size\n",
    "optimizer = keras.optimizers.Nadam(lr=0.01)\n",
    "loss_fn = keras.losses.sparse_categorical_crossentropy\n",
    "mean_loss = keras.metrics.Mean()\n",
    "metrics = [keras.metrics.SparseCategoricalAccuracy()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 269,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c827a90455f148ab9291c6a348e95a43",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='All epochs', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "57f277d1b9174ae4a303a0801ae8de0d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Epoch 1/5', max=1718.0, style=ProgressStyle(description_w…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c5bbaa672c8f45a7a5085137f72f0e85",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Epoch 2/5', max=1718.0, style=ProgressStyle(description_w…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9890f3433e044373a6dfd545beda7840",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Epoch 3/5', max=1718.0, style=ProgressStyle(description_w…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "50c6022ff19a4e0a971ba6051d2ea160",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Epoch 4/5', max=1718.0, style=ProgressStyle(description_w…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d769a81adb6c4a5895eb9cdd4275bb00",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Epoch 5/5', max=1718.0, style=ProgressStyle(description_w…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "with trange(1, n_epochs + 1, desc=\"All epochs\") as epochs:\n",
    "    for epoch in epochs:\n",
    "        with trange(1, n_steps + 1, desc=\"Epoch {}/{}\".format(epoch, n_epochs)) as steps:\n",
    "            for step in steps:\n",
    "                X_batch, y_batch = random_batch(X_train, y_train)\n",
    "                with tf.GradientTape() as tape:\n",
    "                    y_pred = model(X_batch)\n",
    "                    main_loss = tf.reduce_mean(loss_fn(y_batch, y_pred))\n",
    "                    loss = tf.add_n([main_loss] + model.losses)\n",
    "                gradients = tape.gradient(loss, model.trainable_variables)\n",
    "                optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n",
    "                for variable in model.variables:\n",
    "                    if variable.constraint is not None:\n",
    "                        variable.assign(variable.constraint(variable))                    \n",
    "                status = OrderedDict()\n",
    "                mean_loss(loss)\n",
    "                status[\"loss\"] = mean_loss.result().numpy()\n",
    "                for metric in metrics:\n",
    "                    metric(y_batch, y_pred)\n",
    "                    status[metric.name] = metric.result().numpy()\n",
    "                steps.set_postfix(status)\n",
    "            y_pred = model(X_valid)\n",
    "            status[\"val_loss\"] = np.mean(loss_fn(y_valid, y_pred))\n",
    "            status[\"val_accuracy\"] = np.mean(keras.metrics.sparse_categorical_accuracy(\n",
    "                tf.constant(y_valid, dtype=np.float32), y_pred))\n",
    "            steps.set_postfix(status)\n",
    "        for metric in [mean_loss] + metrics:\n",
    "            metric.reset_states()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b.\n",
    "_Exercise: Try using a different optimizer with a different learning rate for the upper layers and the lower layers._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 270,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 271,
   "metadata": {},
   "outputs": [],
   "source": [
    "lower_layers = keras.models.Sequential([\n",
    "    keras.layers.Flatten(input_shape=[28, 28]),\n",
    "    keras.layers.Dense(100, activation=\"relu\"),\n",
    "])\n",
    "upper_layers = keras.models.Sequential([\n",
    "    keras.layers.Dense(10, activation=\"softmax\"),\n",
    "])\n",
    "model = keras.models.Sequential([\n",
    "    lower_layers, upper_layers\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 272,
   "metadata": {},
   "outputs": [],
   "source": [
    "lower_optimizer = keras.optimizers.SGD(lr=1e-4)\n",
    "upper_optimizer = keras.optimizers.Nadam(lr=1e-3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 273,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = 5\n",
    "batch_size = 32\n",
    "n_steps = len(X_train) // batch_size\n",
    "loss_fn = keras.losses.sparse_categorical_crossentropy\n",
    "mean_loss = keras.metrics.Mean()\n",
    "metrics = [keras.metrics.SparseCategoricalAccuracy()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 274,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "78db61f5cd39438ba9c0641aa8ee16b2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='All epochs', max=5.0, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d4729c86914e4917b355f50c7d46d043",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Epoch 1/5', max=1718.0, style=ProgressStyle(description_w…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "17bd45c19073458b90817535f75fc8e8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Epoch 2/5', max=1718.0, style=ProgressStyle(description_w…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "839f25aa89a0430493459f85d48bba4e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Epoch 3/5', max=1718.0, style=ProgressStyle(description_w…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4cca5d87f6ee4f6497aa520b255be44a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Epoch 4/5', max=1718.0, style=ProgressStyle(description_w…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "771fc2da343f4805a7ce672fd514a2cf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Epoch 5/5', max=1718.0, style=ProgressStyle(description_w…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "with trange(1, n_epochs + 1, desc=\"All epochs\") as epochs:\n",
    "    for epoch in epochs:\n",
    "        with trange(1, n_steps + 1, desc=\"Epoch {}/{}\".format(epoch, n_epochs)) as steps:\n",
    "            for step in steps:\n",
    "                X_batch, y_batch = random_batch(X_train, y_train)\n",
    "                with tf.GradientTape(persistent=True) as tape:\n",
    "                    y_pred = model(X_batch)\n",
    "                    main_loss = tf.reduce_mean(loss_fn(y_batch, y_pred))\n",
    "                    loss = tf.add_n([main_loss] + model.losses)\n",
    "                for layers, optimizer in ((lower_layers, lower_optimizer),\n",
    "                                          (upper_layers, upper_optimizer)):\n",
    "                    gradients = tape.gradient(loss, layers.trainable_variables)\n",
    "                    optimizer.apply_gradients(zip(gradients, layers.trainable_variables))\n",
    "                del tape\n",
    "                for variable in model.variables:\n",
    "                    if variable.constraint is not None:\n",
    "                        variable.assign(variable.constraint(variable))                    \n",
    "                status = OrderedDict()\n",
    "                mean_loss(loss)\n",
    "                status[\"loss\"] = mean_loss.result().numpy()\n",
    "                for metric in metrics:\n",
    "                    metric(y_batch, y_pred)\n",
    "                    status[metric.name] = metric.result().numpy()\n",
    "                steps.set_postfix(status)\n",
    "            y_pred = model(X_valid)\n",
    "            status[\"val_loss\"] = np.mean(loss_fn(y_valid, y_pred))\n",
    "            status[\"val_accuracy\"] = np.mean(keras.metrics.sparse_categorical_accuracy(\n",
    "                tf.constant(y_valid, dtype=np.float32), y_pred))\n",
    "            steps.set_postfix(status)\n",
    "        for metric in [mean_loss] + metrics:\n",
    "            metric.reset_states()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}