cifar-10-EfficientNetV2small(3).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Cifar 10 example\n",
    "\n",
    "## Tested on GCP n2 series, provsioned with 2 Vcpu and 7 GB / container "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "tags": [
     "skip"
    ]
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Collecting jax==0.4.1\n",
      "  Downloading jax-0.4.1.tar.gz (1.2 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m50.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hCollecting jaxlib==0.4.1\n",
      "  Downloading jaxlib-0.4.1-cp38-cp38-manylinux2014_x86_64.whl (71.3 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m71.3/71.3 MB\u001b[0m \u001b[31m24.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hCollecting pendulum==2.1.2\n",
      "  Downloading pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl (155 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m155.7/155.7 KB\u001b[0m \u001b[31m29.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting tensorflow==2.11.0\n",
      "  Downloading tensorflow-2.11.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (588.3 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m588.3/588.3 MB\u001b[0m \u001b[31m4.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hCollecting numpy==1.24.0\n",
      "  Downloading numpy-1.24.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (17.3 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m17.3/17.3 MB\u001b[0m \u001b[31m109.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hCollecting pandas==1.5.2\n",
      "  Downloading pandas-1.5.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.2 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.2/12.2 MB\u001b[0m \u001b[31m138.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hCollecting pyvis==0.3.1\n",
      "  Downloading pyvis-0.3.1.tar.gz (748 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m748.9/748.9 KB\u001b[0m \u001b[31m94.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hCollecting plotly==5.11.0\n",
      "  Downloading plotly-5.11.0-py2.py3-none-any.whl (15.3 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m15.3/15.3 MB\u001b[0m \u001b[31m84.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hCollecting matplotlib==3.6.2\n",
      "  Downloading matplotlib-3.6.2-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (9.4 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m9.4/9.4 MB\u001b[0m \u001b[31m178.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hCollecting tensorflow-text==2.11.0\n",
      "  Downloading tensorflow_text-2.11.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (5.8 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.8/5.8 MB\u001b[0m \u001b[31m127.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hCollecting imageio==2.25.0\n",
      "  Downloading imageio-2.25.0-py3-none-any.whl (3.4 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m149.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting opt_einsum\n",
      "  Downloading opt_einsum-3.3.0-py3-none-any.whl (65 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m65.5/65.5 KB\u001b[0m \u001b[31m19.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: scipy>=1.5 in /usr/local/lib/python3.8/dist-packages (from jax==0.4.1->-r requirements.txt (line 1)) (1.6.3)\n",
      "Requirement already satisfied: python-dateutil<3.0,>=2.6 in /usr/local/lib/python3.8/dist-packages (from pendulum==2.1.2->-r requirements.txt (line 3)) (2.8.2)\n",
      "Collecting pytzdata>=2020.1\n",
      "  Downloading pytzdata-2020.1-py2.py3-none-any.whl (489 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m490.0/490.0 KB\u001b[0m \u001b[31m68.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting tensorflow-estimator<2.12,>=2.11.0\n",
      "  Downloading tensorflow_estimator-2.11.0-py2.py3-none-any.whl (439 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m439.2/439.2 KB\u001b[0m \u001b[31m81.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting protobuf<3.20,>=3.9.2\n",
      "  Downloading protobuf-3.19.6-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.1 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m72.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting google-pasta>=0.1.1\n",
      "  Downloading google_pasta-0.2.0-py3-none-any.whl (57 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m57.5/57.5 KB\u001b[0m \u001b[31m16.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: setuptools in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.11.0->-r requirements.txt (line 4)) (62.0.0)\n",
      "Collecting astunparse>=1.6.0\n",
      "  Downloading astunparse-1.6.3-py2.py3-none-any.whl (12 kB)\n",
      "Collecting gast<=0.4.0,>=0.2.1\n",
      "  Downloading gast-0.4.0-py3-none-any.whl (9.8 kB)\n",
      "Collecting libclang>=13.0.0\n",
      "  Downloading libclang-15.0.6.1-py2.py3-none-manylinux2010_x86_64.whl (21.5 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m21.5/21.5 MB\u001b[0m \u001b[31m111.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hCollecting absl-py>=1.0.0\n",
      "  Downloading absl_py-1.4.0-py3-none-any.whl (126 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m126.5/126.5 KB\u001b[0m \u001b[31m32.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.11.0->-r requirements.txt (line 4)) (21.3)\n",
      "Requirement already satisfied: grpcio<2.0,>=1.24.3 in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.11.0->-r requirements.txt (line 4)) (1.50.0)\n",
      "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.11.0->-r requirements.txt (line 4)) (2.0.1)\n",
      "Requirement already satisfied: typing-extensions>=3.6.6 in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.11.0->-r requirements.txt (line 4)) (3.10.0.2)\n",
      "Collecting flatbuffers>=2.0\n",
      "  Downloading flatbuffers-23.3.3-py2.py3-none-any.whl (26 kB)\n",
      "Collecting tensorflow-io-gcs-filesystem>=0.23.1\n",
      "  Downloading tensorflow_io_gcs_filesystem-0.31.0-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (2.4 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.4/2.4 MB\u001b[0m \u001b[31m163.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting tensorboard<2.12,>=2.11\n",
      "  Downloading tensorboard-2.11.2-py3-none-any.whl (6.0 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.0/6.0 MB\u001b[0m \u001b[31m181.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.11.0->-r requirements.txt (line 4)) (1.16.0)\n",
      "Collecting keras<2.12,>=2.11.0\n",
      "  Downloading keras-2.11.0-py2.py3-none-any.whl (1.7 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.7/1.7 MB\u001b[0m \u001b[31m151.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: wrapt>=1.11.0 in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.11.0->-r requirements.txt (line 4)) (1.14.1)\n",
      "Collecting h5py>=2.9.0\n",
      "  Downloading h5py-3.8.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (4.7 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.7/4.7 MB\u001b[0m \u001b[31m156.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.8/dist-packages (from pandas==1.5.2->-r requirements.txt (line 6)) (2022.5)\n",
      "Requirement already satisfied: jinja2>=2.9.6 in /usr/local/lib/python3.8/dist-packages (from pyvis==0.3.1->-r requirements.txt (line 7)) (3.0.3)\n",
      "Requirement already satisfied: networkx>=1.11 in /usr/local/lib/python3.8/dist-packages (from pyvis==0.3.1->-r requirements.txt (line 7)) (2.6.3)\n",
      "Requirement already satisfied: ipython>=5.3.0 in /usr/local/lib/python3.8/dist-packages (from pyvis==0.3.1->-r requirements.txt (line 7)) (8.5.0)\n",
      "Requirement already satisfied: jsonpickle>=1.4.1 in /usr/local/lib/python3.8/dist-packages (from pyvis==0.3.1->-r requirements.txt (line 7)) (2.2.0)\n",
      "Collecting tenacity>=6.2.0\n",
      "  Downloading tenacity-8.2.2-py3-none-any.whl (24 kB)\n",
      "Collecting contourpy>=1.0.1\n",
      "  Downloading contourpy-1.0.7-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (300 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m300.0/300.0 KB\u001b[0m \u001b[31m65.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.8/dist-packages (from matplotlib==3.6.2->-r requirements.txt (line 9)) (1.4.4)\n",
      "Collecting fonttools>=4.22.0\n",
      "  Downloading fonttools-4.39.0-py3-none-any.whl (1.0 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/1.0 MB\u001b[0m \u001b[31m125.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: pyparsing>=2.2.1 in /usr/local/lib/python3.8/dist-packages (from matplotlib==3.6.2->-r requirements.txt (line 9)) (3.0.9)\n",
      "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.8/dist-packages (from matplotlib==3.6.2->-r requirements.txt (line 9)) (0.11.0)\n",
      "Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.8/dist-packages (from matplotlib==3.6.2->-r requirements.txt (line 9)) (9.2.0)\n",
      "Collecting tensorflow-hub>=0.8.0\n",
      "  Downloading tensorflow_hub-0.12.0-py2.py3-none-any.whl (108 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m108.8/108.8 KB\u001b[0m \u001b[31m30.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: wheel<1.0,>=0.23.0 in /usr/lib/python3/dist-packages (from astunparse>=1.6.0->tensorflow==2.11.0->-r requirements.txt (line 4)) (0.30.0)\n",
      "Requirement already satisfied: stack-data in /usr/local/lib/python3.8/dist-packages (from ipython>=5.3.0->pyvis==0.3.1->-r requirements.txt (line 7)) (0.5.1)\n",
      "Requirement already satisfied: jedi>=0.16 in /usr/local/lib/python3.8/dist-packages (from ipython>=5.3.0->pyvis==0.3.1->-r requirements.txt (line 7)) (0.18.1)\n",
      "Requirement already satisfied: pexpect>4.3 in /usr/local/lib/python3.8/dist-packages (from ipython>=5.3.0->pyvis==0.3.1->-r requirements.txt (line 7)) (4.8.0)\n",
      "Requirement already satisfied: decorator in /usr/local/lib/python3.8/dist-packages (from ipython>=5.3.0->pyvis==0.3.1->-r requirements.txt (line 7)) (5.1.1)\n",
      "Requirement already satisfied: pickleshare in /usr/local/lib/python3.8/dist-packages (from ipython>=5.3.0->pyvis==0.3.1->-r requirements.txt (line 7)) (0.7.5)\n",
      "Requirement already satisfied: matplotlib-inline in /usr/local/lib/python3.8/dist-packages (from ipython>=5.3.0->pyvis==0.3.1->-r requirements.txt (line 7)) (0.1.6)\n",
      "Requirement already satisfied: traitlets>=5 in /usr/local/lib/python3.8/dist-packages (from ipython>=5.3.0->pyvis==0.3.1->-r requirements.txt (line 7)) (5.5.0)\n",
      "Requirement already satisfied: backcall in /usr/local/lib/python3.8/dist-packages (from ipython>=5.3.0->pyvis==0.3.1->-r requirements.txt (line 7)) (0.2.0)\n",
      "Requirement already satisfied: pygments>=2.4.0 in /usr/local/lib/python3.8/dist-packages (from ipython>=5.3.0->pyvis==0.3.1->-r requirements.txt (line 7)) (2.13.0)\n",
      "Requirement already satisfied: prompt-toolkit<3.1.0,>3.0.1 in /usr/local/lib/python3.8/dist-packages (from ipython>=5.3.0->pyvis==0.3.1->-r requirements.txt (line 7)) (3.0.31)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.8/dist-packages (from jinja2>=2.9.6->pyvis==0.3.1->-r requirements.txt (line 7)) (2.1.1)\n",
      "Collecting scipy>=1.5\n",
      "  Downloading scipy-1.10.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (34.5 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m34.5/34.5 MB\u001b[0m \u001b[31m83.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hCollecting tensorboard-plugin-wit>=1.6.0\n",
      "  Downloading tensorboard_plugin_wit-1.8.1-py3-none-any.whl (781 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m781.3/781.3 KB\u001b[0m \u001b[31m100.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.8/dist-packages (from tensorboard<2.12,>=2.11->tensorflow==2.11.0->-r requirements.txt (line 4)) (2.28.1)\n",
      "Collecting markdown>=2.6.8\n",
      "  Downloading Markdown-3.4.1-py3-none-any.whl (93 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m93.3/93.3 KB\u001b[0m \u001b[31m29.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting tensorboard-data-server<0.7.0,>=0.6.0\n",
      "  Downloading tensorboard_data_server-0.6.1-py3-none-manylinux2010_x86_64.whl (4.9 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.9/4.9 MB\u001b[0m \u001b[31m158.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.8/dist-packages (from tensorboard<2.12,>=2.11->tensorflow==2.11.0->-r requirements.txt (line 4)) (1.34.0)\n",
      "Collecting werkzeug>=1.0.1\n",
      "  Downloading Werkzeug-2.2.3-py3-none-any.whl (233 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m233.6/233.6 KB\u001b[0m \u001b[31m50.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting google-auth-oauthlib<0.5,>=0.4.1\n",
      "  Downloading google_auth_oauthlib-0.4.6-py2.py3-none-any.whl (18 kB)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.8/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.12,>=2.11->tensorflow==2.11.0->-r requirements.txt (line 4)) (0.2.8)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.8/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.12,>=2.11->tensorflow==2.11.0->-r requirements.txt (line 4)) (4.9)\n",
      "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.8/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.12,>=2.11->tensorflow==2.11.0->-r requirements.txt (line 4)) (4.2.2)\n",
      "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.8/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.12,>=2.11->tensorflow==2.11.0->-r requirements.txt (line 4)) (1.3.1)\n",
      "Requirement already satisfied: parso<0.9.0,>=0.8.0 in /usr/local/lib/python3.8/dist-packages (from jedi>=0.16->ipython>=5.3.0->pyvis==0.3.1->-r requirements.txt (line 7)) (0.8.3)\n",
      "Requirement already satisfied: importlib-metadata>=4.4 in /usr/local/lib/python3.8/dist-packages (from markdown>=2.6.8->tensorboard<2.12,>=2.11->tensorflow==2.11.0->-r requirements.txt (line 4)) (5.0.0)\n",
      "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.8/dist-packages (from pexpect>4.3->ipython>=5.3.0->pyvis==0.3.1->-r requirements.txt (line 7)) (0.7.0)\n",
      "Requirement already satisfied: wcwidth in /usr/local/lib/python3.8/dist-packages (from prompt-toolkit<3.1.0,>3.0.1->ipython>=5.3.0->pyvis==0.3.1->-r requirements.txt (line 7)) (0.2.5)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests<3,>=2.21.0->tensorboard<2.12,>=2.11->tensorflow==2.11.0->-r requirements.txt (line 4)) (2021.5.30)\n",
      "Requirement already satisfied: charset-normalizer<3,>=2 in /usr/local/lib/python3.8/dist-packages (from requests<3,>=2.21.0->tensorboard<2.12,>=2.11->tensorflow==2.11.0->-r requirements.txt (line 4)) (2.1.1)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests<3,>=2.21.0->tensorboard<2.12,>=2.11->tensorflow==2.11.0->-r requirements.txt (line 4)) (3.2)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.8/dist-packages (from requests<3,>=2.21.0->tensorboard<2.12,>=2.11->tensorflow==2.11.0->-r requirements.txt (line 4)) (1.26.12)\n",
      "Requirement already satisfied: pure-eval in /usr/local/lib/python3.8/dist-packages (from stack-data->ipython>=5.3.0->pyvis==0.3.1->-r requirements.txt (line 7)) (0.2.2)\n",
      "Requirement already satisfied: asttokens in /usr/local/lib/python3.8/dist-packages (from stack-data->ipython>=5.3.0->pyvis==0.3.1->-r requirements.txt (line 7)) (2.0.8)\n",
      "Requirement already satisfied: executing in /usr/local/lib/python3.8/dist-packages (from stack-data->ipython>=5.3.0->pyvis==0.3.1->-r requirements.txt (line 7)) (1.1.1)\n",
      "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.8/dist-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard<2.12,>=2.11->tensorflow==2.11.0->-r requirements.txt (line 4)) (3.10.0)\n",
      "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.8/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard<2.12,>=2.11->tensorflow==2.11.0->-r requirements.txt (line 4)) (0.4.8)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.8/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.12,>=2.11->tensorflow==2.11.0->-r requirements.txt (line 4)) (3.2.2)\n",
      "Building wheels for collected packages: jax, pyvis\n",
      "  Building wheel for jax (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for jax: filename=jax-0.4.1-py3-none-any.whl size=1338540 sha256=13a89ea073e7f82857662c399880178eefbe99750d162063b39fb9bd934b93a7\n",
      "  Stored in directory: /home/jovyan/.cache/pip/wheels/e2/c2/cc/ac766a5c8ed28aec746fc292690d3e2e3790b554d2a6abacb7\n",
      "  Building wheel for pyvis (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for pyvis: filename=pyvis-0.3.1-py3-none-any.whl size=756330 sha256=77d0719a41ff8cf2821e70e4c3027177f33f78c5be5a7338ba3d3ac55f0d5710\n",
      "  Stored in directory: /home/jovyan/.cache/pip/wheels/a4/0c/61/8469ca276f96ab772c3acc7f47d71e9737cbdf6f446f017f48\n",
      "Successfully built jax pyvis\n",
      "Installing collected packages: tensorboard-plugin-wit, libclang, flatbuffers, werkzeug, tensorflow-io-gcs-filesystem, tensorflow-estimator, tensorboard-data-server, tenacity, pytzdata, protobuf, numpy, keras, google-pasta, gast, fonttools, astunparse, absl-py, tensorflow-hub, scipy, plotly, pendulum, pandas, opt_einsum, markdown, imageio, h5py, contourpy, matplotlib, jaxlib, jax, google-auth-oauthlib, tensorboard, pyvis, tensorflow, tensorflow-text\n",
      "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "yellowbrick 1.3 requires numpy<1.20,>=1.16.0, but you have numpy 1.24.0 which is incompatible.\n",
      "ml-metadata 0.30.0 requires absl-py<0.13,>=0.9, but you have absl-py 1.4.0 which is incompatible.\n",
      "kubeflow-kale 2.0rc3.post0+6.g3f6dfd87b requires matplotlib<3.4.0, but you have matplotlib 3.6.2 which is incompatible.\n",
      "kubeflow-kale 2.0rc3.post0+6.g3f6dfd87b requires scipy<1.7, but you have scipy 1.10.1 which is incompatible.\n",
      "kserve 0.8.0 requires numpy~=1.19.2, but you have numpy 1.24.0 which is incompatible.\n",
      "kfp 1.7.post0+41.gb3589b6c6 requires absl-py<=0.11,>=0.9, but you have absl-py 1.4.0 which is incompatible.\u001b[0m\u001b[31m\n",
      "\u001b[0mSuccessfully installed absl-py-1.4.0 astunparse-1.6.3 contourpy-1.0.7 flatbuffers-23.3.3 fonttools-4.39.0 gast-0.4.0 google-auth-oauthlib-0.4.6 google-pasta-0.2.0 h5py-3.8.0 imageio-2.25.0 jax-0.4.1 jaxlib-0.4.1 keras-2.11.0 libclang-15.0.6.1 markdown-3.4.1 matplotlib-3.6.2 numpy-1.24.0 opt_einsum-3.3.0 pandas-1.5.2 pendulum-2.1.2 plotly-5.11.0 protobuf-3.19.6 pytzdata-2020.1 pyvis-0.3.1 scipy-1.10.1 tenacity-8.2.2 tensorboard-2.11.2 tensorboard-data-server-0.6.1 tensorboard-plugin-wit-1.8.1 tensorflow-2.11.0 tensorflow-estimator-2.11.0 tensorflow-hub-0.12.0 tensorflow-io-gcs-filesystem-0.31.0 tensorflow-text-2.11.0 werkzeug-2.2.3\n",
      "\u001b[33mWARNING: You are using pip version 22.0.4; however, version 23.0.1 is available.\n",
      "You should consider upgrading via the '/usr/bin/python3 -m pip install --upgrade pip' command.\u001b[0m\u001b[33m\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "# Mark in Kale as \"skip cell\", but run this and the next cell first\n",
    "# Install required packages\n",
    "! pip3 install -r requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "tags": [
     "skip"
    ]
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'status': 'ok', 'restart': True}"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Mark in Kale as \"skip cell\", but run this after the last cell first\n",
    "# Restart the kernel to apply the updates\n",
    "import sys\n",
    "import IPython\n",
    "app = IPython.Application.instance()\n",
    "app.kernel.do_shutdown(True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "tags": [
     "imports"
    ]
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-12 18:06:14.299663: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 AVX512F AVX512_VNNI FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2023-03-12 18:06:14.406793: I tensorflow/core/util/port.cc:104] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",
      "2023-03-12 18:06:14.410178: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory\n",
      "2023-03-12 18:06:14.410191: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n",
      "2023-03-12 18:06:15.050834: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory\n",
      "2023-03-12 18:06:15.050916: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory\n",
      "2023-03-12 18:06:15.050922: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.\n",
      "2023-03-12 18:06:16.332719: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory\n",
      "2023-03-12 18:06:16.332743: W tensorflow/compiler/xla/stream_executor/cuda/cuda_driver.cc:265] failed call to cuInit: UNKNOWN ERROR (303)\n",
      "2023-03-12 18:06:16.332758: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (efficientnet2sm-0): /proc/driver/nvidia/version does not exist\n",
      "2023-03-12 18:06:16.332978: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 AVX512F AVX512_VNNI FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
     ]
    }
   ],
   "source": [
    "# Mark in Kale as Imports\n",
    "\n",
    "import tensorflow as tf\n",
    "import tensorflow_hub as hub\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import imageio.v3 as iio\n",
    "# from multiprocessing import Pool  # , Process\n",
    "from cerebros.simplecerebrosrandomsearch.simple_cerebros_random_search\\\n",
    "    import SimpleCerebrosRandomSearch\n",
    "import pendulum\n",
    "from cerebros.units.units import DenseUnit\n",
    "from cerebros.denseautomlstructuralcomponent.dense_automl_structural_component\\\n",
    "    import zero_7_exp_decay, zero_95_exp_decay, simple_sigmoid\n",
    "from ast import literal_eval\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "tags": [
     "pipeline-parameters"
    ]
   },
   "outputs": [],
   "source": [
    "# Mark in Kale as Pipeline params:\n",
    "\n",
    "activation = 'relu'\n",
    "predecessor_level_connection_affinity_factor_first = 2.0\n",
    "predecessor_level_connection_affinity_factor_main = 0.97\n",
    "max_consecutive_lateral_connections = 5\n",
    "p_lateral_connection = 0.97\n",
    "num_lateral_connection_tries_per_unit = 2\n",
    "learning_rate = 0.001\n",
    "epochs = 10  # [1, 100]\n",
    "batch_size = 20\n",
    "maximum_levels = 4  # [3,7]\n",
    "maximum_units_per_level = 7  # [2,10]\n",
    "maximum_neurons_per_unit = 4  # [2,20]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "tags": [
     "block:data_ingest"
    ]
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "labels_tensor_ohe shape: (2500, 10)\n",
      "data_tensor shape: (2500, 32, 32, 3)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# Mark in Kale as Pipeline step data_ingest\n",
    "\n",
    "\n",
    "### Global configurables:\n",
    "\n",
    "INPUT_SHAPES = [(32, 32, 3)]  # resize from ]\n",
    "RESIZE_TO = (384, 384, 3)\n",
    "OUTPUT_SHAPES = [10]\n",
    "\n",
    "# Read in the data set and make it useable\n",
    "\n",
    "ciphar10_metadata = pd.read_csv(\"cifar10-mini/file_metadata.csv\")\n",
    "\n",
    "ciphar10_train = ciphar10_metadata.query(\"data_set == 'train'\")\n",
    "ciphar10_test = ciphar10_metadata.query(\"data_set == 'test'\")\n",
    "\n",
    "def make_dataset(dataset):\n",
    "    images = []\n",
    "    labels = []\n",
    "    for i in np.arange(ciphar10_metadata.shape[0]):\n",
    "        imfile = ciphar10_metadata.loc[i]['file_name']\n",
    "\n",
    "        # Debug delete\n",
    "        # print(f\"$$$$: attempting file: {imfile}\")\n",
    "\n",
    "        img = iio.imread(imfile)\n",
    "\n",
    "        images.append(np.array(img))\n",
    "        labels.append(int(ciphar10_metadata.loc[i]['label']))\n",
    "    data_tensor = tf.constant(images)\n",
    "    labels_tensor = tf.constant(labels)\n",
    "    labels_tensor_ohe = tf.one_hot(indices=labels_tensor,\n",
    "                                   depth=10)\n",
    "    print(f\"labels_tensor_ohe shape: {labels_tensor_ohe.shape}\")\n",
    "    print(f\"data_tensor shape: {data_tensor.shape}\")\n",
    "    return data_tensor, labels_tensor_ohe\n",
    "\n",
    "\n",
    "selected_x_train, selected_y_train_ohe =\\\n",
    "    make_dataset(ciphar10_train)\n",
    "\n",
    "training_x = [selected_x_train]\n",
    "train_labels = [selected_y_train_ohe]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "block:train",
     "prev:data_ingest"
    ]
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Compiled the loaded model, but the compiled metrics have yet to be built. `model.compile_metrics` will be empty until you train or evaluate the model.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:Found untraced functions such as _jit_compiled_convolution_op, _jit_compiled_convolution_op, _jit_compiled_convolution_op, _jit_compiled_convolution_op, _jit_compiled_convolution_op while saving (showing 5 of 339). These functions will not be directly callable after loading.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: test/assets\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: test/assets\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SimpleCerebrosRandomSearch.input_shapes: [(32, 32, 3)]\n",
      "nan\n",
      ">nnf>ceil\n",
      "k is: 0 value is: [{'1': <class 'cerebros.units.units.InputUnit'>}]\n",
      "0\n",
      "k is: 1 value is: [{'2': <class 'cerebros.units.units.DenseUnit'>}, {'2': <class 'cerebros.units.units.DenseUnit'>}]\n",
      "1\n",
      "Trying to create level 1\n",
      "We think level 1's predecessors are: [0]\n",
      "k is: 2 value is: [{'3': <class 'cerebros.units.units.DenseUnit'>}, {'2': <class 'cerebros.units.units.DenseUnit'>}]\n",
      "2\n",
      "Trying to create level 2\n",
      "We think level 2's predecessors are: [0, 1]\n",
      "k is: 3 value is: [{'10': <class 'cerebros.units.units.FinalDenseUnit'>}]\n",
      "3\n",
      "Trying to create Final level 3\n",
      "Trying to create level 3\n",
      "We think level final level 3's predecessors are: [0, 1, 2]\n",
      "levels:\n",
      "[0, 1, 2, 3]\n",
      "{'0': 'InputUnitModule'}\n",
      "InputLevel.input_shapes [(32, 32, 3)]\n",
      "{'2': <class 'cerebros.units.units.DenseUnit'>}\n",
      "{'2': <class 'cerebros.units.units.DenseUnit'>}\n",
      "{'3': <class 'cerebros.units.units.DenseUnit'>}\n",
      "{'2': <class 'cerebros.units.units.DenseUnit'>}\n",
      "Debug: I am 3 selecting 2\n",
      "debug: meta_level_number\n",
      "debug: meta_level_number\n",
      "debug: meta_level_number\n",
      "debug: meta_level_number\n",
      "debug: meta_level_number\n",
      "Setting levels_unmaterialized[0] level_number 0 to have first successor: levels_unmaterialized[:1], having level_numbers of [1, 2, 3]\n",
      "Setting levels_unmaterialized[1] level_number 1 to have first successor: levels_unmaterialized[:2], having level_numbers of [2, 3]\n",
      "Setting levels_unmaterialized[2] level_number 2 to have first successor: levels_unmaterialized[:3], having level_numbers of [3]\n",
      "Debug: successor_connectivity_errors_2d []\n",
      "$$$$$$>>>>> Base model: <keras.engine.functional.Functional object at 0x7f9e1e255310>\n",
      "InputUnit.input_shape: (32, 32, 3)\n",
      "{'2': <class 'cerebros.units.units.DenseUnit'>}\n",
      "{'2': <class 'cerebros.units.units.DenseUnit'>}\n",
      "debug: meta_level_number\n",
      "debug: meta_level_number\n",
      "Debug: successor_connectivity_errors_2d []\n",
      "Debug: successor_connectivity_errors_2d []\n",
      "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_0 called\n",
      "materialized network layers\n",
      "[<KerasTensor: shape=(None, 1280) dtype=float32 (created by layer 'model_7')>, <KerasTensor: shape=(None, 1280) dtype=float32 (created by layer 'model_7')>, <KerasTensor: shape=(None, 1280) dtype=float32 (created by layer 'model_7')>, <KerasTensor: shape=(None, 1280) dtype=float32 (created by layer 'model_7')>]\n",
      "materialized_predecessor_units [<KerasTensor: shape=(None, 1280) dtype=float32 (created by layer 'model_7')>, <KerasTensor: shape=(None, 1280) dtype=float32 (created by layer 'model_7')>, <KerasTensor: shape=(None, 1280) dtype=float32 (created by layer 'model_7')>, <KerasTensor: shape=(None, 1280) dtype=float32 (created by layer 'model_7')>]\n",
      "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_1 called\n",
      "materialized network layers\n",
      "[<KerasTensor: shape=(None, 1280) dtype=float32 (created by layer 'model_7')>, <KerasTensor: shape=(None, 1280) dtype=float32 (created by layer 'model_7')>, <KerasTensor: shape=(None, 1280) dtype=float32 (created by layer 'model_7')>, <KerasTensor: shape=(None, 1280) dtype=float32 (created by layer 'model_7')>]\n",
      "materialized_predecessor_units [<KerasTensor: shape=(None, 1280) dtype=float32 (created by layer 'model_7')>, <KerasTensor: shape=(None, 1280) dtype=float32 (created by layer 'model_7')>, <KerasTensor: shape=(None, 1280) dtype=float32 (created by layer 'model_7')>, <KerasTensor: shape=(None, 1280) dtype=float32 (created by layer 'model_7')>]\n",
      "{'3': <class 'cerebros.units.units.DenseUnit'>}\n",
      "{'2': <class 'cerebros.units.units.DenseUnit'>}\n",
      "debug: meta_level_number\n",
      "debug: meta_level_number\n",
      "Debug: successor_connectivity_errors_2d []\n",
      "Debug: successor_connectivity_errors_2d []\n",
      "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000002_tr_0_DenseUnit_0000000000000002_tr_0_0 called\n",
      "materialized network layers\n",
      "[<KerasTensor: shape=(None, 1280) dtype=float32 (created by layer 'model_7')>, <KerasTensor: shape=(None, 2) dtype=float32 (created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_1_dns_')>, <KerasTensor: shape=(None, 2) dtype=float32 (created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_0_dns_')>, <KerasTensor: shape=(None, 1280) dtype=float32 (created by layer 'model_7')>, <KerasTensor: shape=(None, 2) dtype=float32 (created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_1_dns_')>, <KerasTensor: shape=(None, 2) dtype=float32 (created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_1_dns_')>]\n",
      "materialized_predecessor_units [<KerasTensor: shape=(None, 1280) dtype=float32 (created by layer 'model_7')>, <KerasTensor: shape=(None, 2) dtype=float32 (created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_1_dns_')>, <KerasTensor: shape=(None, 2) dtype=float32 (created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_0_dns_')>, <KerasTensor: shape=(None, 1280) dtype=float32 (created by layer 'model_7')>, <KerasTensor: shape=(None, 2) dtype=float32 (created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_1_dns_')>, <KerasTensor: shape=(None, 2) dtype=float32 (created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_1_dns_')>]\n",
      "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000002_tr_0_DenseUnit_0000000000000002_tr_0_1 called\n",
      "materialized network layers\n",
      "[<KerasTensor: shape=(None, 1280) dtype=float32 (created by layer 'model_7')>, <KerasTensor: shape=(None, 2) dtype=float32 (created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_1_dns_')>, <KerasTensor: shape=(None, 2) dtype=float32 (created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_0_dns_')>, <KerasTensor: shape=(None, 1280) dtype=float32 (created by layer 'model_7')>, <KerasTensor: shape=(None, 2) dtype=float32 (created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_0_dns_')>, <KerasTensor: shape=(None, 2) dtype=float32 (created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_1_dns_')>]\n",
      "materialized_predecessor_units [<KerasTensor: shape=(None, 1280) dtype=float32 (created by layer 'model_7')>, <KerasTensor: shape=(None, 2) dtype=float32 (created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_1_dns_')>, <KerasTensor: shape=(None, 2) dtype=float32 (created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_0_dns_')>, <KerasTensor: shape=(None, 1280) dtype=float32 (created by layer 'model_7')>, <KerasTensor: shape=(None, 2) dtype=float32 (created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_0_dns_')>, <KerasTensor: shape=(None, 2) dtype=float32 (created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_1_dns_')>]\n",
      "{'10': <class 'cerebros.units.units.FinalDenseUnit'>}\n",
      "debug: meta_level_number\n",
      "Debug: successor_connectivity_errors_2d []\n",
      "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000003_tr_0_FinalDenseUnit_0000000000000003_tr_0_0 called\n",
      "materialized network layers\n",
      "[<KerasTensor: shape=(None, 3) dtype=float32 (created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000002_tr_0_DenseUnit_0000000000000002_tr_0_0_dns_')>, <KerasTensor: shape=(None, 2) dtype=float32 (created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000002_tr_0_DenseUnit_0000000000000002_tr_0_1_dns_')>, <KerasTensor: shape=(None, 2) dtype=float32 (created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000002_tr_0_DenseUnit_0000000000000002_tr_0_1_dns_')>, <KerasTensor: shape=(None, 3) dtype=float32 (created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000002_tr_0_DenseUnit_0000000000000002_tr_0_0_dns_')>, <KerasTensor: shape=(None, 1280) dtype=float32 (created by layer 'model_7')>, <KerasTensor: shape=(None, 2) dtype=float32 (created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_1_dns_')>, <KerasTensor: shape=(None, 3) dtype=float32 (created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000002_tr_0_DenseUnit_0000000000000002_tr_0_0_dns_')>, <KerasTensor: shape=(None, 3) dtype=float32 (created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000002_tr_0_DenseUnit_0000000000000002_tr_0_0_dns_')>, <KerasTensor: shape=(None, 1280) dtype=float32 (created by layer 'model_7')>, <KerasTensor: shape=(None, 2) dtype=float32 (created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_1_dns_')>, <KerasTensor: shape=(None, 2) dtype=float32 (created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000002_tr_0_DenseUnit_0000000000000002_tr_0_1_dns_')>, <KerasTensor: shape=(None, 3) dtype=float32 (created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000002_tr_0_DenseUnit_0000000000000002_tr_0_0_dns_')>]\n",
      "materialized_predecessor_units [<KerasTensor: shape=(None, 3) dtype=float32 (created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000002_tr_0_DenseUnit_0000000000000002_tr_0_0_dns_')>, <KerasTensor: shape=(None, 2) dtype=float32 (created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000002_tr_0_DenseUnit_0000000000000002_tr_0_1_dns_')>, <KerasTensor: shape=(None, 2) dtype=float32 (created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000002_tr_0_DenseUnit_0000000000000002_tr_0_1_dns_')>, <KerasTensor: shape=(None, 3) dtype=float32 (created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000002_tr_0_DenseUnit_0000000000000002_tr_0_0_dns_')>, <KerasTensor: shape=(None, 1280) dtype=float32 (created by layer 'model_7')>, <KerasTensor: shape=(None, 2) dtype=float32 (created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_1_dns_')>, <KerasTensor: shape=(None, 3) dtype=float32 (created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000002_tr_0_DenseUnit_0000000000000002_tr_0_0_dns_')>, <KerasTensor: shape=(None, 3) dtype=float32 (created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000002_tr_0_DenseUnit_0000000000000002_tr_0_0_dns_')>, <KerasTensor: shape=(None, 1280) dtype=float32 (created by layer 'model_7')>, <KerasTensor: shape=(None, 2) dtype=float32 (created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_1_dns_')>, <KerasTensor: shape=(None, 2) dtype=float32 (created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000002_tr_0_DenseUnit_0000000000000002_tr_0_1_dns_')>, <KerasTensor: shape=(None, 3) dtype=float32 (created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000002_tr_0_DenseUnit_0000000000000002_tr_0_0_dns_')>]\n",
      "inputs\n",
      "KerasTensor(type_spec=TensorSpec(shape=(None, 32, 32, 3), dtype=tf.float32, name='NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp'), name='NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp', description=\"created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp'\")\n",
      "\n",
      "outputs\n",
      "KerasTensor(type_spec=TensorSpec(shape=(None, 10), dtype=tf.float32, name=None), name='NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000003_tr_0_FinalDenseUnit_0000000000000003_tr_0_0_dns_/Softmax:0', description=\"created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000003_tr_0_FinalDenseUnit_0000000000000003_tr_0_0_dns_'\")\n",
      "Model: \"NeuralNetworkFuture_0000000000000nan_tr_0_nn_materialized\"\n",
      "__________________________________________________________________________________________________\n",
      " Layer (type)                   Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      " NeuralNetworkFuture_0000000000  [(None, 32, 32, 3)]  0          []                               \n",
      " 000nan_tr_0_InputLevel_0000000                                                                   \n",
      " 000000000_tr_0_InputUnit_00000                                                                   \n",
      " 00000000000_tr_0_0_inp (InputL                                                                   \n",
      " ayer)                                                                                            \n",
      "                                                                                                  \n",
      " model_7 (Functional)           (None, 1280)         117746848   ['NeuralNetworkFuture_00000000000\n",
      "                                                                 00nan_tr_0_InputLevel_00000000000\n",
      "                                                                 00000_tr_0_InputUnit_000000000000\n",
      "                                                                 0000_tr_0_0_inp[0][0]']          \n",
      "                                                                                                  \n",
      " NeuralNetworkFuture_0000000000  (None, 5120)        0           ['model_7[0][0]',                \n",
      " 000nan_tr_0_DenseLevel_0000000                                   'model_7[0][0]',                \n",
      " 000000001_tr_0_DenseUnit_00000                                   'model_7[0][0]',                \n",
      " 00000000001_tr_0_1_cat_ (Conca                                   'model_7[0][0]']                \n",
      " tenate)                                                                                          \n",
      "                                                                                                  \n",
      " NeuralNetworkFuture_0000000000  (None, 5120)        0           ['model_7[0][0]',                \n",
      " 000nan_tr_0_DenseLevel_0000000                                   'model_7[0][0]',                \n",
      " 000000001_tr_0_DenseUnit_00000                                   'model_7[0][0]',                \n",
      " 00000000001_tr_0_0_cat_ (Conca                                   'model_7[0][0]']                \n",
      " tenate)                                                                                          \n",
      "                                                                                                  \n",
      " NeuralNetworkFuture_0000000000  (None, 5120)        20480       ['NeuralNetworkFuture_00000000000\n",
      " 000nan_tr_0_DenseLevel_0000000                                  00nan_tr_0_DenseLevel_00000000000\n",
      " 000000001_tr_0_DenseUnit_00000                                  00001_tr_0_DenseUnit_000000000000\n",
      " 00000000001_tr_0_1_btn_ (Batch                                  0001_tr_0_1_cat_[0][0]']         \n",
      " Normalization)                                                                                   \n",
      "                                                                                                  \n",
      " NeuralNetworkFuture_0000000000  (None, 5120)        20480       ['NeuralNetworkFuture_00000000000\n",
      " 000nan_tr_0_DenseLevel_0000000                                  00nan_tr_0_DenseLevel_00000000000\n",
      " 000000001_tr_0_DenseUnit_00000                                  00001_tr_0_DenseUnit_000000000000\n",
      " 00000000001_tr_0_0_btn_ (Batch                                  0001_tr_0_0_cat_[0][0]']         \n",
      " Normalization)                                                                                   \n",
      "                                                                                                  \n",
      " NeuralNetworkFuture_0000000000  (None, 2)           10242       ['NeuralNetworkFuture_00000000000\n",
      " 000nan_tr_0_DenseLevel_0000000                                  00nan_tr_0_DenseLevel_00000000000\n",
      " 000000001_tr_0_DenseUnit_00000                                  00001_tr_0_DenseUnit_000000000000\n",
      " 00000000001_tr_0_1_dns_ (Dense                                  0001_tr_0_1_btn_[0][0]']         \n",
      " )                                                                                                \n",
      "                                                                                                  \n",
      " NeuralNetworkFuture_0000000000  (None, 2)           10242       ['NeuralNetworkFuture_00000000000\n",
      " 000nan_tr_0_DenseLevel_0000000                                  00nan_tr_0_DenseLevel_00000000000\n",
      " 000000001_tr_0_DenseUnit_00000                                  00001_tr_0_DenseUnit_000000000000\n",
      " 00000000001_tr_0_0_dns_ (Dense                                  0001_tr_0_0_btn_[0][0]']         \n",
      " )                                                                                                \n",
      "                                                                                                  \n",
      " NeuralNetworkFuture_0000000000  (None, 2568)        0           ['model_7[0][0]',                \n",
      " 000nan_tr_0_DenseLevel_0000000                                   'NeuralNetworkFuture_00000000000\n",
      " 000000002_tr_0_DenseUnit_00000                                  00nan_tr_0_DenseLevel_00000000000\n",
      " 00000000002_tr_0_0_cat_ (Conca                                  00001_tr_0_DenseUnit_000000000000\n",
      " tenate)                                                         0001_tr_0_1_dns_[0][0]',         \n",
      "                                                                  'NeuralNetworkFuture_00000000000\n",
      "                                                                 00nan_tr_0_DenseLevel_00000000000\n",
      "                                                                 00001_tr_0_DenseUnit_000000000000\n",
      "                                                                 0001_tr_0_0_dns_[0][0]',         \n",
      "                                                                  'model_7[0][0]',                \n",
      "                                                                  'NeuralNetworkFuture_00000000000\n",
      "                                                                 00nan_tr_0_DenseLevel_00000000000\n",
      "                                                                 00001_tr_0_DenseUnit_000000000000\n",
      "                                                                 0001_tr_0_1_dns_[0][0]',         \n",
      "                                                                  'NeuralNetworkFuture_00000000000\n",
      "                                                                 00nan_tr_0_DenseLevel_00000000000\n",
      "                                                                 00001_tr_0_DenseUnit_000000000000\n",
      "                                                                 0001_tr_0_1_dns_[0][0]']         \n",
      "                                                                                                  \n",
      " NeuralNetworkFuture_0000000000  (None, 2568)        0           ['model_7[0][0]',                \n",
      " 000nan_tr_0_DenseLevel_0000000                                   'NeuralNetworkFuture_00000000000\n",
      " 000000002_tr_0_DenseUnit_00000                                  00nan_tr_0_DenseLevel_00000000000\n",
      " 00000000002_tr_0_1_cat_ (Conca                                  00001_tr_0_DenseUnit_000000000000\n",
      " tenate)                                                         0001_tr_0_1_dns_[0][0]',         \n",
      "                                                                  'NeuralNetworkFuture_00000000000\n",
      "                                                                 00nan_tr_0_DenseLevel_00000000000\n",
      "                                                                 00001_tr_0_DenseUnit_000000000000\n",
      "                                                                 0001_tr_0_0_dns_[0][0]',         \n",
      "                                                                  'model_7[0][0]',                \n",
      "                                                                  'NeuralNetworkFuture_00000000000\n",
      "                                                                 00nan_tr_0_DenseLevel_00000000000\n",
      "                                                                 00001_tr_0_DenseUnit_000000000000\n",
      "                                                                 0001_tr_0_0_dns_[0][0]',         \n",
      "                                                                  'NeuralNetworkFuture_00000000000\n",
      "                                                                 00nan_tr_0_DenseLevel_00000000000\n",
      "                                                                 00001_tr_0_DenseUnit_000000000000\n",
      "                                                                 0001_tr_0_1_dns_[0][0]']         \n",
      "                                                                                                  \n",
      " NeuralNetworkFuture_0000000000  (None, 2568)        10272       ['NeuralNetworkFuture_00000000000\n",
      " 000nan_tr_0_DenseLevel_0000000                                  00nan_tr_0_DenseLevel_00000000000\n",
      " 000000002_tr_0_DenseUnit_00000                                  00002_tr_0_DenseUnit_000000000000\n",
      " 00000000002_tr_0_0_btn_ (Batch                                  0002_tr_0_0_cat_[0][0]']         \n",
      " Normalization)                                                                                   \n",
      "                                                                                                  \n",
      " NeuralNetworkFuture_0000000000  (None, 2568)        10272       ['NeuralNetworkFuture_00000000000\n",
      " 000nan_tr_0_DenseLevel_0000000                                  00nan_tr_0_DenseLevel_00000000000\n",
      " 000000002_tr_0_DenseUnit_00000                                  00002_tr_0_DenseUnit_000000000000\n",
      " 00000000002_tr_0_1_btn_ (Batch                                  0002_tr_0_1_cat_[0][0]']         \n",
      " Normalization)                                                                                   \n",
      "                                                                                                  \n",
      " NeuralNetworkFuture_0000000000  (None, 3)           7707        ['NeuralNetworkFuture_00000000000\n",
      " 000nan_tr_0_DenseLevel_0000000                                  00nan_tr_0_DenseLevel_00000000000\n",
      " 000000002_tr_0_DenseUnit_00000                                  00002_tr_0_DenseUnit_000000000000\n",
      " 00000000002_tr_0_0_dns_ (Dense                                  0002_tr_0_0_btn_[0][0]']         \n",
      " )                                                                                                \n",
      "                                                                                                  \n",
      " NeuralNetworkFuture_0000000000  (None, 2)           5138        ['NeuralNetworkFuture_00000000000\n",
      " 000nan_tr_0_DenseLevel_0000000                                  00nan_tr_0_DenseLevel_00000000000\n",
      " 000000002_tr_0_DenseUnit_00000                                  00002_tr_0_DenseUnit_000000000000\n",
      " 00000000002_tr_0_1_dns_ (Dense                                  0002_tr_0_1_btn_[0][0]']         \n",
      " )                                                                                                \n",
      "                                                                                                  \n",
      " NeuralNetworkFuture_0000000000  (None, 2585)        0           ['NeuralNetworkFuture_00000000000\n",
      " 000nan_tr_0_FinalDenseLevel_00                                  00nan_tr_0_DenseLevel_00000000000\n",
      " 00000000000003_tr_0_FinalDense                                  00002_tr_0_DenseUnit_000000000000\n",
      " Unit_0000000000000003_tr_0_0_c                                  0002_tr_0_0_dns_[0][0]',         \n",
      " at_ (Concatenate)                                                'NeuralNetworkFuture_00000000000\n",
      "                                                                 00nan_tr_0_DenseLevel_00000000000\n",
      "                                                                 00002_tr_0_DenseUnit_000000000000\n",
      "                                                                 0002_tr_0_1_dns_[0][0]',         \n",
      "                                                                  'NeuralNetworkFuture_00000000000\n",
      "                                                                 00nan_tr_0_DenseLevel_00000000000\n",
      "                                                                 00002_tr_0_DenseUnit_000000000000\n",
      "                                                                 0002_tr_0_1_dns_[0][0]',         \n",
      "                                                                  'NeuralNetworkFuture_00000000000\n",
      "                                                                 00nan_tr_0_DenseLevel_00000000000\n",
      "                                                                 00002_tr_0_DenseUnit_000000000000\n",
      "                                                                 0002_tr_0_0_dns_[0][0]',         \n",
      "                                                                  'model_7[0][0]',                \n",
      "                                                                  'NeuralNetworkFuture_00000000000\n",
      "                                                                 00nan_tr_0_DenseLevel_00000000000\n",
      "                                                                 00001_tr_0_DenseUnit_000000000000\n",
      "                                                                 0001_tr_0_1_dns_[0][0]',         \n",
      "                                                                  'NeuralNetworkFuture_00000000000\n",
      "                                                                 00nan_tr_0_DenseLevel_00000000000\n",
      "                                                                 00002_tr_0_DenseUnit_000000000000\n",
      "                                                                 0002_tr_0_0_dns_[0][0]',         \n",
      "                                                                  'NeuralNetworkFuture_00000000000\n",
      "                                                                 00nan_tr_0_DenseLevel_00000000000\n",
      "                                                                 00002_tr_0_DenseUnit_000000000000\n",
      "                                                                 0002_tr_0_0_dns_[0][0]',         \n",
      "                                                                  'model_7[0][0]',                \n",
      "                                                                  'NeuralNetworkFuture_00000000000\n",
      "                                                                 00nan_tr_0_DenseLevel_00000000000\n",
      "                                                                 00001_tr_0_DenseUnit_000000000000\n",
      "                                                                 0001_tr_0_1_dns_[0][0]',         \n",
      "                                                                  'NeuralNetworkFuture_00000000000\n",
      "                                                                 00nan_tr_0_DenseLevel_00000000000\n",
      "                                                                 00002_tr_0_DenseUnit_000000000000\n",
      "                                                                 0002_tr_0_1_dns_[0][0]',         \n",
      "                                                                  'NeuralNetworkFuture_00000000000\n",
      "                                                                 00nan_tr_0_DenseLevel_00000000000\n",
      "                                                                 00002_tr_0_DenseUnit_000000000000\n",
      "                                                                 0002_tr_0_0_dns_[0][0]']         \n",
      "                                                                                                  \n",
      " NeuralNetworkFuture_0000000000  (None, 2585)        10340       ['NeuralNetworkFuture_00000000000\n",
      " 000nan_tr_0_FinalDenseLevel_00                                  00nan_tr_0_FinalDenseLevel_000000\n",
      " 00000000000003_tr_0_FinalDense                                  0000000003_tr_0_FinalDenseUnit_00\n",
      " Unit_0000000000000003_tr_0_0_b                                  00000000000003_tr_0_0_cat_[0][0]'\n",
      " tn_ (BatchNormalization)                                        ]                                \n",
      "                                                                                                  \n",
      " NeuralNetworkFuture_0000000000  (None, 10)          25860       ['NeuralNetworkFuture_00000000000\n",
      " 000nan_tr_0_FinalDenseLevel_00                                  00nan_tr_0_FinalDenseLevel_000000\n",
      " 00000000000003_tr_0_FinalDense                                  0000000003_tr_0_FinalDenseUnit_00\n",
      " Unit_0000000000000003_tr_0_0_d                                  00000000000003_tr_0_0_btn_[0][0]'\n",
      " ns_ (Dense)                                                     ]                                \n",
      "                                                                                                  \n",
      "==================================================================================================\n",
      "Total params: 117,877,881\n",
      "Trainable params: 914,311\n",
      "Non-trainable params: 116,963,570\n",
      "__________________________________________________________________________________________________\n",
      "None\n",
      "Epoch 1/10\n",
      "82/82 [==============================] - 605s 7s/step - loss: 0.6668 - top_1_categorical_accuracy: 0.8437 - val_loss: 0.4206 - val_top_1_categorical_accuracy: 0.8789\n",
      "Epoch 2/10\n",
      "82/82 [==============================] - ETA: 0s - loss: 0.2678 - top_1_categorical_accuracy: 0.9292"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn [13], line 137\u001b[0m\n\u001b[1;32m     89\u001b[0m meta_trial_number \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mstr\u001b[39m(\u001b[38;5;28mint\u001b[39m(np\u001b[38;5;241m.\u001b[39mrandom\u001b[38;5;241m.\u001b[39mrandom() \u001b[38;5;241m*\u001b[39m \u001b[38;5;241m10\u001b[39m \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39m \u001b[38;5;241m12\u001b[39m))\n\u001b[1;32m     91\u001b[0m cerebros_automl \u001b[38;5;241m=\u001b[39m SimpleCerebrosRandomSearch(\n\u001b[1;32m     92\u001b[0m     unit_type\u001b[38;5;241m=\u001b[39mDenseUnit,\n\u001b[1;32m     93\u001b[0m     input_shapes\u001b[38;5;241m=\u001b[39mINPUT_SHAPES,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    134\u001b[0m     meta_trial_number\u001b[38;5;241m=\u001b[39mmeta_trial_number,\n\u001b[1;32m    135\u001b[0m     base_models\u001b[38;5;241m=\u001b[39m[embedding_model])\n\u001b[1;32m    136\u001b[0m val_top_1_categorical_accuracy \u001b[38;5;241m=\u001b[39m\\\n\u001b[0;32m--> 137\u001b[0m     \u001b[43mcerebros_automl\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun_random_search\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/cerebros-core-algorithm-alpha/cerebros/simplecerebrosrandomsearch/simple_cerebros_random_search.py:521\u001b[0m, in \u001b[0;36mSimpleCerebrosRandomSearch.run_random_search\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    517\u001b[0m lock \u001b[38;5;241m=\u001b[39m Lock()\n\u001b[1;32m    518\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m np\u001b[38;5;241m.\u001b[39marange(\n\u001b[1;32m    519\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnumber_of_tries_per_architecture_moity):\n\u001b[1;32m    520\u001b[0m     processes\u001b[38;5;241m.\u001b[39mappend(\n\u001b[0;32m--> 521\u001b[0m         Process(target\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun_moity_permutations\u001b[49m\u001b[43m(\u001b[49m\u001b[43mspec\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mspec\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msubtrial_number\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msubtrial_number\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlock\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlock\u001b[49m\u001b[43m)\u001b[49m))\n\u001b[1;32m    522\u001b[0m     subtrial_number \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m\n\u001b[1;32m    523\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mseed \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m\n",
      "File \u001b[0;32m~/cerebros-core-algorithm-alpha/cerebros/simplecerebrosrandomsearch/simple_cerebros_random_search.py:478\u001b[0m, in \u001b[0;36mSimpleCerebrosRandomSearch.run_moity_permutations\u001b[0;34m(self, spec, subtrial_number, lock)\u001b[0m\n\u001b[1;32m    475\u001b[0m \u001b[38;5;28mprint\u001b[39m(nnf\u001b[38;5;241m.\u001b[39mmaterialized_neural_network\u001b[38;5;241m.\u001b[39msummary())\n\u001b[1;32m    476\u001b[0m nnf\u001b[38;5;241m.\u001b[39mget_graph()\n\u001b[0;32m--> 478\u001b[0m history \u001b[38;5;241m=\u001b[39m \u001b[43mneural_network\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtraining_data\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    479\u001b[0m \u001b[43m                             \u001b[49m\u001b[43my\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlabels\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    480\u001b[0m \u001b[43m                             \u001b[49m\u001b[43mepochs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mepochs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    481\u001b[0m \u001b[43m                             \u001b[49m\u001b[43mbatch_size\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbatch_size\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    482\u001b[0m \u001b[43m                             \u001b[49m\u001b[38;5;66;43;03m# callbacks=[early_stopping,\u001b[39;49;00m\n\u001b[1;32m    483\u001b[0m \u001b[43m                             \u001b[49m\u001b[38;5;66;43;03m#            tensor_board],\u001b[39;49;00m\n\u001b[1;32m    484\u001b[0m \u001b[43m                             \u001b[49m\u001b[43mvalidation_split\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvalidation_split\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    485\u001b[0m oracle_0 \u001b[38;5;241m=\u001b[39m pd\u001b[38;5;241m.\u001b[39mDataFrame(history\u001b[38;5;241m.\u001b[39mhistory)\n\u001b[1;32m    487\u001b[0m model_architectures_folder \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mproject_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m/model_architectures\u001b[39m\u001b[38;5;124m\"\u001b[39m\n",
      "File \u001b[0;32m~/.local/lib/python3.8/site-packages/keras/utils/traceback_utils.py:65\u001b[0m, in \u001b[0;36mfilter_traceback.<locals>.error_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     63\u001b[0m filtered_tb \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m     64\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 65\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     66\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m     67\u001b[0m     filtered_tb \u001b[38;5;241m=\u001b[39m _process_traceback_frames(e\u001b[38;5;241m.\u001b[39m__traceback__)\n",
      "File \u001b[0;32m~/.local/lib/python3.8/site-packages/keras/engine/training.py:1694\u001b[0m, in \u001b[0;36mModel.fit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m   1679\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mgetattr\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m_eval_data_handler\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m) \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m   1680\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_eval_data_handler \u001b[38;5;241m=\u001b[39m data_adapter\u001b[38;5;241m.\u001b[39mget_data_handler(\n\u001b[1;32m   1681\u001b[0m         x\u001b[38;5;241m=\u001b[39mval_x,\n\u001b[1;32m   1682\u001b[0m         y\u001b[38;5;241m=\u001b[39mval_y,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1692\u001b[0m         steps_per_execution\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_steps_per_execution,\n\u001b[1;32m   1693\u001b[0m     )\n\u001b[0;32m-> 1694\u001b[0m val_logs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mevaluate\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1695\u001b[0m \u001b[43m    \u001b[49m\u001b[43mx\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mval_x\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1696\u001b[0m \u001b[43m    \u001b[49m\u001b[43my\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mval_y\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1697\u001b[0m \u001b[43m    \u001b[49m\u001b[43msample_weight\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mval_sample_weight\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1698\u001b[0m \u001b[43m    \u001b[49m\u001b[43mbatch_size\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mvalidation_batch_size\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mbatch_size\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1699\u001b[0m \u001b[43m    \u001b[49m\u001b[43msteps\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mvalidation_steps\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1700\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcallbacks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcallbacks\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1701\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmax_queue_size\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmax_queue_size\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1702\u001b[0m \u001b[43m    \u001b[49m\u001b[43mworkers\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mworkers\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1703\u001b[0m \u001b[43m    \u001b[49m\u001b[43muse_multiprocessing\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_multiprocessing\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1704\u001b[0m \u001b[43m    \u001b[49m\u001b[43mreturn_dict\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m   1705\u001b[0m \u001b[43m    \u001b[49m\u001b[43m_use_cached_eval_dataset\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m   1706\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1707\u001b[0m val_logs \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m   1708\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mval_\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m+\u001b[39m name: val \u001b[38;5;28;01mfor\u001b[39;00m name, val \u001b[38;5;129;01min\u001b[39;00m val_logs\u001b[38;5;241m.\u001b[39mitems()\n\u001b[1;32m   1709\u001b[0m }\n\u001b[1;32m   1710\u001b[0m epoch_logs\u001b[38;5;241m.\u001b[39mupdate(val_logs)\n",
      "File \u001b[0;32m~/.local/lib/python3.8/site-packages/keras/utils/traceback_utils.py:65\u001b[0m, in \u001b[0;36mfilter_traceback.<locals>.error_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     63\u001b[0m filtered_tb \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m     64\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 65\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     66\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m     67\u001b[0m     filtered_tb \u001b[38;5;241m=\u001b[39m _process_traceback_frames(e\u001b[38;5;241m.\u001b[39m__traceback__)\n",
      "File \u001b[0;32m~/.local/lib/python3.8/site-packages/keras/engine/training.py:2040\u001b[0m, in \u001b[0;36mModel.evaluate\u001b[0;34m(self, x, y, batch_size, verbose, sample_weight, steps, callbacks, max_queue_size, workers, use_multiprocessing, return_dict, **kwargs)\u001b[0m\n\u001b[1;32m   2036\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m tf\u001b[38;5;241m.\u001b[39mprofiler\u001b[38;5;241m.\u001b[39mexperimental\u001b[38;5;241m.\u001b[39mTrace(\n\u001b[1;32m   2037\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtest\u001b[39m\u001b[38;5;124m\"\u001b[39m, step_num\u001b[38;5;241m=\u001b[39mstep, _r\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m\n\u001b[1;32m   2038\u001b[0m ):\n\u001b[1;32m   2039\u001b[0m     callbacks\u001b[38;5;241m.\u001b[39mon_test_batch_begin(step)\n\u001b[0;32m-> 2040\u001b[0m     tmp_logs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtest_function\u001b[49m\u001b[43m(\u001b[49m\u001b[43miterator\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   2041\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m data_handler\u001b[38;5;241m.\u001b[39mshould_sync:\n\u001b[1;32m   2042\u001b[0m         context\u001b[38;5;241m.\u001b[39masync_wait()\n",
      "File \u001b[0;32m~/.local/lib/python3.8/site-packages/tensorflow/python/util/traceback_utils.py:150\u001b[0m, in \u001b[0;36mfilter_traceback.<locals>.error_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    148\u001b[0m filtered_tb \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m    149\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 150\u001b[0m   \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    151\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m    152\u001b[0m   filtered_tb \u001b[38;5;241m=\u001b[39m _process_traceback_frames(e\u001b[38;5;241m.\u001b[39m__traceback__)\n",
      "File \u001b[0;32m~/.local/lib/python3.8/site-packages/tensorflow/python/eager/polymorphic_function/polymorphic_function.py:880\u001b[0m, in \u001b[0;36mFunction.__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    877\u001b[0m compiler \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mxla\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_jit_compile \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnonXla\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    879\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m OptionalXlaContext(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_jit_compile):\n\u001b[0;32m--> 880\u001b[0m   result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwds\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    882\u001b[0m new_tracing_count \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mexperimental_get_tracing_count()\n\u001b[1;32m    883\u001b[0m without_tracing \u001b[38;5;241m=\u001b[39m (tracing_count \u001b[38;5;241m==\u001b[39m new_tracing_count)\n",
      "File \u001b[0;32m~/.local/lib/python3.8/site-packages/tensorflow/python/eager/polymorphic_function/polymorphic_function.py:919\u001b[0m, in \u001b[0;36mFunction._call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    916\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_lock\u001b[38;5;241m.\u001b[39mrelease()\n\u001b[1;32m    917\u001b[0m \u001b[38;5;66;03m# In this case we have not created variables on the first call. So we can\u001b[39;00m\n\u001b[1;32m    918\u001b[0m \u001b[38;5;66;03m# run the first trace but we should fail if variables are created.\u001b[39;00m\n\u001b[0;32m--> 919\u001b[0m results \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_variable_creation_fn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwds\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    920\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_created_variables \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m ALLOW_DYNAMIC_VARIABLE_CREATION:\n\u001b[1;32m    921\u001b[0m   \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCreating variables on a non-first call to a function\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    922\u001b[0m                    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m decorated with tf.function.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
      "File \u001b[0;32m~/.local/lib/python3.8/site-packages/tensorflow/python/eager/polymorphic_function/tracing_compiler.py:134\u001b[0m, in \u001b[0;36mTracingCompiler.__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    131\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_lock:\n\u001b[1;32m    132\u001b[0m   (concrete_function,\n\u001b[1;32m    133\u001b[0m    filtered_flat_args) \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_maybe_define_function(args, kwargs)\n\u001b[0;32m--> 134\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mconcrete_function\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_flat\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    135\u001b[0m \u001b[43m    \u001b[49m\u001b[43mfiltered_flat_args\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcaptured_inputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconcrete_function\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcaptured_inputs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/.local/lib/python3.8/site-packages/tensorflow/python/eager/polymorphic_function/monomorphic_function.py:1745\u001b[0m, in \u001b[0;36mConcreteFunction._call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1741\u001b[0m possible_gradient_type \u001b[38;5;241m=\u001b[39m gradients_util\u001b[38;5;241m.\u001b[39mPossibleTapeGradientTypes(args)\n\u001b[1;32m   1742\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (possible_gradient_type \u001b[38;5;241m==\u001b[39m gradients_util\u001b[38;5;241m.\u001b[39mPOSSIBLE_GRADIENT_TYPES_NONE\n\u001b[1;32m   1743\u001b[0m     \u001b[38;5;129;01mand\u001b[39;00m executing_eagerly):\n\u001b[1;32m   1744\u001b[0m   \u001b[38;5;66;03m# No tape is watching; skip to running the function.\u001b[39;00m\n\u001b[0;32m-> 1745\u001b[0m   \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_build_call_outputs(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_inference_function\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcall\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1746\u001b[0m \u001b[43m      \u001b[49m\u001b[43mctx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcancellation_manager\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcancellation_manager\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[1;32m   1747\u001b[0m forward_backward \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_select_forward_and_backward_functions(\n\u001b[1;32m   1748\u001b[0m     args,\n\u001b[1;32m   1749\u001b[0m     possible_gradient_type,\n\u001b[1;32m   1750\u001b[0m     executing_eagerly)\n\u001b[1;32m   1751\u001b[0m forward_function, args_with_tangents \u001b[38;5;241m=\u001b[39m forward_backward\u001b[38;5;241m.\u001b[39mforward()\n",
      "File \u001b[0;32m~/.local/lib/python3.8/site-packages/tensorflow/python/eager/polymorphic_function/monomorphic_function.py:378\u001b[0m, in \u001b[0;36m_EagerDefinedFunction.call\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m    376\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m _InterpolateFunctionError(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m    377\u001b[0m   \u001b[38;5;28;01mif\u001b[39;00m cancellation_manager \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 378\u001b[0m     outputs \u001b[38;5;241m=\u001b[39m \u001b[43mexecute\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    379\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43mstr\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msignature\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mname\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    380\u001b[0m \u001b[43m        \u001b[49m\u001b[43mnum_outputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_num_outputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    381\u001b[0m \u001b[43m        \u001b[49m\u001b[43minputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    382\u001b[0m \u001b[43m        \u001b[49m\u001b[43mattrs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mattrs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    383\u001b[0m \u001b[43m        \u001b[49m\u001b[43mctx\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mctx\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    384\u001b[0m   \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    385\u001b[0m     outputs \u001b[38;5;241m=\u001b[39m execute\u001b[38;5;241m.\u001b[39mexecute_with_cancellation(\n\u001b[1;32m    386\u001b[0m         \u001b[38;5;28mstr\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msignature\u001b[38;5;241m.\u001b[39mname),\n\u001b[1;32m    387\u001b[0m         num_outputs\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_num_outputs,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    390\u001b[0m         ctx\u001b[38;5;241m=\u001b[39mctx,\n\u001b[1;32m    391\u001b[0m         cancellation_manager\u001b[38;5;241m=\u001b[39mcancellation_manager)\n",
      "File \u001b[0;32m~/.local/lib/python3.8/site-packages/tensorflow/python/eager/execute.py:52\u001b[0m, in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     50\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m     51\u001b[0m   ctx\u001b[38;5;241m.\u001b[39mensure_initialized()\n\u001b[0;32m---> 52\u001b[0m   tensors \u001b[38;5;241m=\u001b[39m \u001b[43mpywrap_tfe\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mTFE_Py_Execute\u001b[49m\u001b[43m(\u001b[49m\u001b[43mctx\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_handle\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdevice_name\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mop_name\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     53\u001b[0m \u001b[43m                                      \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mattrs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnum_outputs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     54\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m core\u001b[38;5;241m.\u001b[39m_NotOkStatusException \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m     55\u001b[0m   \u001b[38;5;28;01mif\u001b[39;00m name \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# Mark in Kale as Pipeline step train; Requires data_ingest\n",
    "\n",
    "# base_new = tf.keras.applications.MobileNetV3Large(\n",
    "#     input_shape=None,\n",
    "#     alpha=1.0,\n",
    "#     minimalistic=False,\n",
    "#     include_top=True,\n",
    "#     weights=\"imagenet\",\n",
    "#     input_tensor=None,\n",
    "#     classes=1000,\n",
    "#     pooling=None,\n",
    "#     dropout_rate=0.2,\n",
    "#     classifier_activation=\"softmax\",\n",
    "#     include_preprocessing=True,\n",
    "# )\n",
    "\n",
    "# for layer in base_new.layers:\n",
    "#     layer.trainable = True\n",
    "\n",
    "# last_relevant_layer = base_new.layers[-2]\n",
    "# # last_relevant_layer_extracted = last_relevant_layer #.output[0][0][0]\n",
    "# base_embedding = tf.keras.Model(inputs=base_new.layers[0].input,\n",
    "#                                 outputs=last_relevant_layer.output)\n",
    "\n",
    "\n",
    "# image_input_0 = tf.keras.layers.Input(shape=INPUT_SHAPES[0])\n",
    "# resizing = tf.keras.layers.Resizing(\n",
    "#     height=RESIZE_TO[0],\n",
    "#     width=RESIZE_TO[1],\n",
    "#     interpolation='bilinear',\n",
    "#     crop_to_aspect_ratio=False)\n",
    "# resized = resizing(image_input_0)\n",
    "# embedded = base_embedding(resized)\n",
    "\n",
    "# embedding_model = tf.keras.Model(image_input_0,\n",
    "#                                  embedded)\n",
    "\n",
    "\n",
    "# mod_with_fc_raw = tf.keras.applications.EfficientNetV2L(\n",
    "#     include_top=True,\n",
    "#     weights=\"imagenet\",\n",
    "#     input_tensor=None,\n",
    "#     input_shape=None,\n",
    "#     pooling=None,\n",
    "#     classes=1000,\n",
    "#     classifier_activation=\"softmax\",\n",
    "#     include_preprocessing=True,\n",
    "# )\n",
    "\n",
    "mod_with_fc_raw = tf.keras.applications.EfficientNetV2S(\n",
    "    include_top=True,\n",
    "    weights=\"imagenet\",\n",
    "    input_tensor=None,\n",
    "    input_shape=None,\n",
    "    pooling=None,\n",
    "    classes=1000,\n",
    "    classifier_activation=\"softmax\",\n",
    "    include_preprocessing=True,\n",
    ")\n",
    "\n",
    "\n",
    "# Make the deepest conv2d layer trainable, leave everything else\n",
    "# as not trainable\n",
    "for layer in mod_with_fc_raw.layers:\n",
    "    layer.trainable = False\n",
    "# Last conv2d layer. This we want to train .\n",
    "mod_with_fc_raw.layers[-6].trainable = True\n",
    "\n",
    "# Create the final base model\n",
    "# (remove the final Dense and BatchNormalization layers ...) \n",
    "efficient_net_b_7_transferable_base_model =\\\n",
    "    tf.keras.Model(inputs=mod_with_fc_raw.layers[0].input, \n",
    "                    outputs=mod_with_fc_raw.layers[-3].output)\n",
    "\n",
    "image_input_0 = tf.keras.layers.Input(shape=INPUT_SHAPES[0])\n",
    "resizing = tf.keras.layers.Resizing(\n",
    "    height=RESIZE_TO[0],\n",
    "    width=RESIZE_TO[1],\n",
    "    interpolation='bilinear',\n",
    "    crop_to_aspect_ratio=False)\n",
    "resized = resizing(image_input_0)\n",
    "embedded = efficient_net_b_7_transferable_base_model(resized)\n",
    "\n",
    "embedding_model = tf.keras.Model(image_input_0,\n",
    "                                 embedded)\n",
    "\n",
    "# embedding_model.save('test')\n",
    "\n",
    "# Final training task\n",
    "\n",
    "TIME = pendulum.now(tz='America/New_York').__str__()[:16]\\\n",
    "    .replace('T', '_')\\\n",
    "    .replace(':', '_')\\\n",
    "    .replace('-', '_')\n",
    "PROJECT_NAME = f'{TIME}_cerebros_auto_ml_test'\n",
    "\n",
    "meta_trial_number = str(int(np.random.random() * 10 ** 12))\n",
    "\n",
    "cerebros_automl = SimpleCerebrosRandomSearch(\n",
    "    unit_type=DenseUnit,\n",
    "    input_shapes=INPUT_SHAPES,\n",
    "    output_shapes=OUTPUT_SHAPES,\n",
    "    training_data=training_x,\n",
    "    labels=train_labels,\n",
    "    validation_split=0.35,\n",
    "    direction='maximize',\n",
    "    metric_to_rank_by=\"val_top_1_categorical_accuracy\",\n",
    "    minimum_levels=2,\n",
    "    maximum_levels=maximum_levels,\n",
    "    minimum_units_per_level=1,\n",
    "    maximum_units_per_level=maximum_units_per_level,\n",
    "    minimum_neurons_per_unit=1,\n",
    "    maximum_neurons_per_unit=maximum_neurons_per_unit,\n",
    "    activation=activation,\n",
    "    final_activation='softmax',\n",
    "    number_of_architecture_moities_to_try=2,\n",
    "    number_of_tries_per_architecture_moity=1,\n",
    "    minimum_skip_connection_depth=1,\n",
    "    maximum_skip_connection_depth=7,\n",
    "    predecessor_level_connection_affinity_factor_first=predecessor_level_connection_affinity_factor_first,\n",
    "    predecessor_level_connection_affinity_factor_first_rounding_rule='ceil',\n",
    "    predecessor_level_connection_affinity_factor_main=predecessor_level_connection_affinity_factor_main,\n",
    "    predecessor_level_connection_affinity_factor_main_rounding_rule='ceil',\n",
    "    predecessor_level_connection_affinity_factor_decay_main=zero_7_exp_decay,\n",
    "    seed=8675309,\n",
    "    max_consecutive_lateral_connections=max_consecutive_lateral_connections,\n",
    "    gate_after_n_lateral_connections=3,\n",
    "    gate_activation_function=simple_sigmoid,\n",
    "    p_lateral_connection=p_lateral_connection,\n",
    "    p_lateral_connection_decay=zero_95_exp_decay,\n",
    "    num_lateral_connection_tries_per_unit=num_lateral_connection_tries_per_unit,\n",
    "    learning_rate=learning_rate,\n",
    "    loss=tf.keras.losses.CategoricalCrossentropy(),\n",
    "    metrics=[tf.keras.metrics.TopKCategoricalAccuracy(\n",
    "                k=1, name='top_1_categorical_accuracy')\n",
    "             ],\n",
    "    epochs=epochs,\n",
    "    project_name=f\"{PROJECT_NAME}_meta_{meta_trial_number}\",\n",
    "    # use_multiprocessing_for_multiple_neural_networks=False,  # pull this param\n",
    "    model_graphs='model_graphs',\n",
    "    batch_size=batch_size,\n",
    "    meta_trial_number=meta_trial_number,\n",
    "    base_models=[embedding_model])\n",
    "val_top_1_categorical_accuracy =\\\n",
    "    cerebros_automl.run_random_search()\n",
    "# Mark in Kale as Pipeline metrics Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "pipeline-metrics"
    ]
   },
   "outputs": [],
   "source": [
    "print(val_top_1_categorical_accuracy)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "kubeflow_notebook": {
   "autosnapshot": true,
   "deploy_config": {},
   "docker_image": "gcr.io/arrikto/jupyter-kale-py38@sha256:2e1ce3427b780c0c78e7cfec527ee10c391092fdc4a8344cd76f8b83c61c5234",
   "experiment_name": "image-classification-sm",
   "katib_metadata": {
    "algorithm": {
     "algorithmName": "bayesianoptimization",
     "algorithmSettings": [
      {
       "name": "random_state",
       "value": "10"
      },
      {
       "name": "acq_optimizer",
       "value": "auto"
      },
      {
       "name": "acq_func",
       "value": "gp_hedge"
      },
      {
       "name": "base_estimator",
       "value": "GP"
      }
     ]
    },
    "maxFailedTrialCount": 10,
    "maxTrialCount": 50,
    "objective": {
     "additionalMetricNames": [],
     "goal": 1,
     "objectiveMetricName": "val-top-1-categorical-accuracy",
     "type": "maximize"
    },
    "parallelTrialCount": 6,
    "parameters": [
     {
      "feasibleSpace": {
       "list": [
        "relu",
        "elu",
        "gelu"
       ]
      },
      "name": "activation",
      "parameterType": "categorical"
     },
     {
      "feasibleSpace": {
       "max": "40",
       "min": "1",
       "step": "0.1"
      },
      "name": "predecessor_level_connection_affinity_factor_first",
      "parameterType": "double"
     },
     {
      "feasibleSpace": {
       "max": "65",
       "min": "0.1",
       "step": "0.1"
      },
      "name": "predecessor_level_connection_affinity_factor_main",
      "parameterType": "double"
     },
     {
      "feasibleSpace": {
       "max": "40",
       "min": "2",
       "step": "1"
      },
      "name": "max_consecutive_lateral_connections",
      "parameterType": "int"
     },
     {
      "feasibleSpace": {
       "max": "50",
       "min": "0.1",
       "step": "0.1"
      },
      "name": "p_lateral_connection",
      "parameterType": "double"
     },
     {
      "feasibleSpace": {
       "max": "40",
       "min": "1",
       "step": "1"
      },
      "name": "num_lateral_connection_tries_per_unit",
      "parameterType": "int"
     },
     {
      "feasibleSpace": {
       "max": "0.3",
       "min": "0.00001",
       "step": "0.00001"
      },
      "name": "learning_rate",
      "parameterType": "double"
     },
     {
      "feasibleSpace": {
       "max": "7",
       "min": "1",
       "step": "1"
      },
      "name": "epochs",
      "parameterType": "int"
     },
     {
      "feasibleSpace": {
       "max": "40",
       "min": "5",
       "step": "1"
      },
      "name": "batch_size",
      "parameterType": "int"
     },
     {
      "feasibleSpace": {
       "max": "8",
       "min": "2"
      },
      "name": "maximum_levels",
      "parameterType": "int"
     },
     {
      "feasibleSpace": {
       "max": "15",
       "min": "2",
       "step": "1"
      },
      "name": "maximum_units_per_level",
      "parameterType": "int"
     },
     {
      "feasibleSpace": {
       "max": "10",
       "min": "2",
       "step": "1"
      },
      "name": "maximum_neurons_per_unit",
      "parameterType": "int"
     }
    ]
   },
   "katib_run": true,
   "pipeline_description": "Cifar 10 example with EfficientnnetV2L",
   "pipeline_name": "image-classification-sm",
   "snapshot_volumes": true,
   "volume_access_mode": "rwm",
   "volumes": [
    {
     "annotations": [],
     "mount_point": "/home/jovyan",
     "name": "efficientnet2sm-workspace-fpg9l",
     "size": 5,
     "size_type": "Gi",
     "snapshot": false,
     "type": "clone"
    }
   ]
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}