From 43b6459f28cfad3c4075bad53e242a516c7b6eb1 Mon Sep 17 00:00:00 2001 From: Aba Date: Tue, 1 Aug 2023 13:32:11 -0700 Subject: [PATCH] Refactor --- test/py/resnet18_bundle_api.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/py/resnet18_bundle_api.ipynb b/test/py/resnet18_bundle_api.ipynb index 10beb92..af41c3d 100644 --- a/test/py/resnet18_bundle_api.ipynb +++ b/test/py/resnet18_bundle_api.ipynb @@ -1 +1 @@ -{"cells":[{"cell_type":"code","execution_count":1,"metadata":{"id":"ABy3xAE8uW__"},"outputs":[{"name":"stderr","output_type":"stream","text":["c:\\ProgramData\\Miniconda3\\envs\\qkeras\\lib\\site-packages\\numpy\\_distributor_init.py:30: UserWarning: loaded more than 1 DLL from .libs:\n","c:\\ProgramData\\Miniconda3\\envs\\qkeras\\lib\\site-packages\\numpy\\.libs\\libopenblas.FB5AE2TYXYH2IJRDKGDGQ3XBKLKTF43H.gfortran-win_amd64.dll\n","c:\\ProgramData\\Miniconda3\\envs\\qkeras\\lib\\site-packages\\numpy\\.libs\\libopenblas64__v0.3.21-gcc_10_3_0.dll\n"," warnings.warn(\"loaded more than 1 DLL from .libs:\"\n"]}],"source":["from qkeras import *\n","from tensorflow.keras.layers import Input, AveragePooling2D, Flatten, Softmax, Add, ZeroPadding2D, MaxPooling2D\n","import numpy as np\n","from collections import namedtuple\n","import pickle\n","import math\n","import tensorflow as tf\n","from tensorflow.keras.optimizers import Adam\n","from tensorflow.keras.callbacks import ModelCheckpoint, LearningRateScheduler\n","from tensorflow.keras.callbacks import ReduceLROnPlateau\n","from tensorflow.keras.preprocessing.image import ImageDataGenerator\n","from tensorflow.keras.datasets import cifar10\n","from tensorflow.keras.utils import plot_model\n","from tensorflow.keras.utils import to_categorical\n","from qkeras.utils import model_save_quantized_weights"]},{"cell_type":"code","execution_count":2,"metadata":{"id":"vG3iXUBnuXAB"},"outputs":[],"source":["def load_data(num_classes=10, subtract_pixel_mean=True):\n"," \"\"\"\n"," Load CIFAR10 data and normalize\n"," \"\"\"\n"," (x_train, y_train), (x_test, y_test) = cifar10.load_data()\n","\n"," # input image dimensions.\n"," input_shape = x_train.shape[1:]\n","\n"," # normalize data.\n"," x_train = x_train.astype('float32') / 128.0 - 1.0\n"," x_test = x_test.astype('float32') / 128.0 - 1.0\n","\n"," # if subtract pixel mean is enabled\n"," if subtract_pixel_mean:\n"," x_train_mean = np.mean(x_train, axis=0)\n"," x_train -= x_train_mean\n"," x_test -= x_train_mean\n","\n"," print('x_train shape:', x_train.shape)\n"," print(x_train.shape[0], 'train samples')\n"," print(x_test.shape[0], 'test samples')\n"," print('y_train shape:', y_train.shape)\n","\n"," # convert class vectors to binary class matrices,\n"," # i.e., one hot encodings\n"," y_train = to_categorical(y_train, num_classes)\n"," y_test = to_categorical(y_test, num_classes)\n","\n"," return x_train, y_train, x_test, y_test\n"]},{"cell_type":"code","execution_count":3,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"ZtMyMdKTuXAC","outputId":"24e6c7d8-b3bc-4b07-a12f-3d3955194087"},"outputs":[{"name":"stdout","output_type":"stream","text":["x_train shape: (50000, 32, 32, 3)\n","50000 train samples\n","10000 test samples\n","y_train shape: (50000, 1)\n"]}],"source":["x_train, y_train, x_test, y_test = load_data(10, False)"]},{"cell_type":"code","execution_count":4,"metadata":{"id":"FtdLlJbzuXAC"},"outputs":[],"source":["input_shape = x_train.shape[1:-1] + (3,)\n","np.random.seed(1)\n","\n","a_0 = 'quantized_relu(8,0,negative_slope=0.125)'\n","a_1 = 'quantized_relu(8,1,negative_slope=0.125)'\n","a_2 = 'quantized_relu(8,2,negative_slope=0.125)'\n","a_3 = 'quantized_relu(8,3,negative_slope=0.125)'\n","\n","q_0 = 'quantized_bits(8,0,False,True,1)'\n","q_1 = 'quantized_bits(8,1,False,True,1)'\n","q_2 = 'quantized_bits(8,2,False,True,1)'\n","q_3 = 'quantized_bits(8,3,False,True,1)'\n","\n","q_t = 'quantized_bits(8,0,False,True,1)'\n","\n","np.random.seed(42)\n","#preamble = './drive/MyDrive/resnet/'\n","preamble = ''\n","USE_BIAS = True"]},{"cell_type":"code","execution_count":5,"metadata":{"id":"qgoMo_ima_OB"},"outputs":[{"name":"stdout","output_type":"stream","text":["Model: \"model\"\n","__________________________________________________________________________________________________\n"," Layer (type) Output Shape Param # Connected to \n","==================================================================================================\n"," input (InputLayer) [(None, 32, 32, 3)] 0 [] \n"," \n"," q_activation (QActivation) (None, 32, 32, 3) 0 ['input[0][0]'] \n"," \n"," bundle (Bundle) (None, 16, 16, 64) 9729 ['q_activation[0][0]'] \n","|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n","| q_activation_1 (QActivation) multiple 0 [] |\n","| |\n","| q_conv2d_batchnorm (QConv2DBat multiple 9729 [] |\n","| chnorm) |\n","| |\n","| q_activation_2 (QActivation) multiple 0 [] |\n","| |\n","| max_pooling2d (MaxPooling2D) multiple 0 [] |\n","¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯\n"," bundle_1 (Bundle) (None, 16, 16, 64) 37185 ['bundle[0][0]'] \n","|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n","| q_activation_3 (QActivation) multiple 0 [] |\n","| |\n","| q_conv2d_batchnorm_1 (QConv2DB multiple 37185 [] |\n","| atchnorm) |\n","¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯\n"," bundle_2 (Bundle) (None, 16, 16, 64) 37185 ['bundle_1[0][0]', \n"," 'bundle[0][0]'] \n","|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n","| q_activation_4 (QActivation) multiple 0 [] |\n","| |\n","| q_conv2d_batchnorm_2 (QConv2DB multiple 37185 [] |\n","| atchnorm) |\n","| |\n","| q_activation_5 (QActivation) multiple 0 [] |\n","¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯\n"," bundle_3 (Bundle) (None, 16, 16, 64) 37185 ['bundle_2[0][0]'] \n","|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n","| q_activation_6 (QActivation) multiple 0 [] |\n","| |\n","| q_conv2d_batchnorm_3 (QConv2DB multiple 37185 [] |\n","| atchnorm) |\n","¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯\n"," bundle_4 (Bundle) (None, 16, 16, 64) 37185 ['bundle_3[0][0]', \n"," 'bundle_2[0][0]'] \n","|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n","| q_activation_7 (QActivation) multiple 0 [] |\n","| |\n","| q_conv2d_batchnorm_4 (QConv2DB multiple 37185 [] |\n","| atchnorm) |\n","| |\n","| q_activation_8 (QActivation) multiple 0 [] |\n","¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯\n"," bundle_5 (Bundle) (None, 8, 8, 128) 74369 ['bundle_4[0][0]'] \n","|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n","| q_activation_9 (QActivation) multiple 0 [] |\n","| |\n","| q_conv2d_batchnorm_5 (QConv2DB multiple 74369 [] |\n","| atchnorm) |\n","¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯\n"," bundle_6 (Bundle) (None, 8, 8, 128) 148097 ['bundle_5[0][0]'] \n","|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n","| q_activation_10 (QActivation) multiple 0 [] |\n","| |\n","| q_conv2d_batchnorm_6 (QConv2DB multiple 148097 [] |\n","| atchnorm) |\n","¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯\n"," bundle_7 (Bundle) (None, 8, 8, 128) 8833 ['bundle_4[0][0]', \n"," 'bundle_6[0][0]'] \n","|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n","| q_activation_11 (QActivation) multiple 0 [] |\n","| |\n","| q_conv2d_batchnorm_7 (QConv2DB multiple 8833 [] |\n","| atchnorm) |\n","| |\n","| q_activation_12 (QActivation) multiple 0 [] |\n","¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯\n"," bundle_8 (Bundle) (None, 8, 8, 128) 148097 ['bundle_7[0][0]'] \n","|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n","| q_activation_13 (QActivation) multiple 0 [] |\n","| |\n","| q_conv2d_batchnorm_8 (QConv2DB multiple 148097 [] |\n","| atchnorm) |\n","¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯\n"," bundle_9 (Bundle) (None, 8, 8, 128) 17025 ['bundle_8[0][0]', \n"," 'bundle_7[0][0]'] \n","|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n","| q_activation_14 (QActivation) multiple 0 [] |\n","| |\n","| q_conv2d_batchnorm_9 (QConv2DB multiple 17025 [] |\n","| atchnorm) |\n","| |\n","| q_activation_15 (QActivation) multiple 0 [] |\n","¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯\n"," bundle_10 (Bundle) (None, 4, 4, 256) 296193 ['bundle_9[0][0]'] \n","|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n","| q_activation_16 (QActivation) multiple 0 [] |\n","| |\n","| q_conv2d_batchnorm_10 (QConv2D multiple 296193 [] |\n","| Batchnorm) |\n","¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯\n"," bundle_11 (Bundle) (None, 4, 4, 256) 591105 ['bundle_10[0][0]'] \n","|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n","| q_activation_17 (QActivation) multiple 0 [] |\n","| |\n","| q_conv2d_batchnorm_11 (QConv2D multiple 591105 [] |\n","| Batchnorm) |\n","¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯\n"," bundle_12 (Bundle) (None, 4, 4, 256) 34049 ['bundle_9[0][0]', \n"," 'bundle_11[0][0]'] \n","|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n","| q_activation_18 (QActivation) multiple 0 [] |\n","| |\n","| q_conv2d_batchnorm_12 (QConv2D multiple 34049 [] |\n","| Batchnorm) |\n","| |\n","| q_activation_19 (QActivation) multiple 0 [] |\n","¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯\n"," bundle_13 (Bundle) (None, 4, 4, 256) 591105 ['bundle_12[0][0]'] \n","|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n","| q_activation_20 (QActivation) multiple 0 [] |\n","| |\n","| q_conv2d_batchnorm_13 (QConv2D multiple 591105 [] |\n","| Batchnorm) |\n","¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯\n"," bundle_14 (Bundle) (None, 4, 4, 256) 66817 ['bundle_13[0][0]', \n"," 'bundle_12[0][0]'] \n","|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n","| q_activation_21 (QActivation) multiple 0 [] |\n","| |\n","| q_conv2d_batchnorm_14 (QConv2D multiple 66817 [] |\n","| Batchnorm) |\n","| |\n","| q_activation_22 (QActivation) multiple 0 [] |\n","¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯\n"," bundle_15 (Bundle) (None, 2, 2, 512) 1182209 ['bundle_14[0][0]'] \n","|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n","| q_activation_23 (QActivation) multiple 0 [] |\n","| |\n","| q_conv2d_batchnorm_15 (QConv2D multiple 1182209 [] |\n","| Batchnorm) |\n","¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯\n"," bundle_16 (Bundle) (None, 2, 2, 512) 2361857 ['bundle_15[0][0]'] \n","|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n","| q_activation_24 (QActivation) multiple 0 [] |\n","| |\n","| q_conv2d_batchnorm_16 (QConv2D multiple 2361857 [] |\n","| Batchnorm) |\n","¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯\n"," bundle_17 (Bundle) (None, 2, 2, 512) 133633 ['bundle_14[0][0]', \n"," 'bundle_16[0][0]'] \n","|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n","| q_activation_25 (QActivation) multiple 0 [] |\n","| |\n","| q_conv2d_batchnorm_17 (QConv2D multiple 133633 [] |\n","| Batchnorm) |\n","| |\n","| q_activation_26 (QActivation) multiple 0 [] |\n","¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯\n"," bundle_18 (Bundle) (None, 2, 2, 512) 2361857 ['bundle_17[0][0]'] \n","|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n","| q_activation_27 (QActivation) multiple 0 [] |\n","| |\n","| q_conv2d_batchnorm_18 (QConv2D multiple 2361857 [] |\n","| Batchnorm) |\n","¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯\n"," bundle_19 (Bundle) (None, 512) 264705 ['bundle_18[0][0]', \n"," 'bundle_17[0][0]'] \n","|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n","| q_activation_28 (QActivation) multiple 0 [] |\n","| |\n","| q_conv2d_batchnorm_19 (QConv2D multiple 264705 [] |\n","| Batchnorm) |\n","| |\n","| q_activation_29 (QActivation) multiple 0 [] |\n","| |\n","| q_activation_30 (QActivation) multiple 0 [] |\n","| |\n","| q_average_pooling2d (QAverageP multiple 0 [] |\n","| ooling2D) |\n","| |\n","| flatten (Flatten) multiple 0 [] |\n","¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯\n"," bundle_20 (Bundle) (None, 10) 5130 ['bundle_19[0][0]'] \n","|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n","| q_activation_31 (QActivation) multiple 0 [] |\n","| |\n","| q_dense (QDense) multiple 5130 [] |\n","| |\n","| activation (Activation) multiple 0 [] |\n","¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯\n","==================================================================================================\n","Total params: 8,443,550\n","Trainable params: 8,433,930\n","Non-trainable params: 9,620\n","__________________________________________________________________________________________________\n","None\n"]}],"source":["class Bundle(tf.keras.Model):\n"," def __init__(self, \n"," core, # dict, Mandaroty: parameters for conv/dense layer, act can be quantization or relu\n"," add=None, # dict, Mandatory if x1 is not None in call(), else ignored\n"," pool=None, # dict, Optional: can only be max or avg\n"," flatten=False, # Optional: set to True to flatten the outputs\n"," softmax=False, # Optional: set to Ture to include floating point softmax layer\n"," **kwargs):\n","\n"," super(Bundle, self).__init__()\n"," \n"," self.core = core\n"," self.add = add\n"," self.pool = pool\n"," self.flatten = flatten\n"," self.softmax = softmax\n"," self.inp = {'tensor':None, 'int': None, 'bits':None, 'frac': None}\n"," self.out = {'tensor':None, 'int': None, 'bits':None, 'frac': None}\n"," self.proc = {'tensor':None, 'int': None, 'bits':None, 'frac': None}\n"," self.w = {'tensor':None, 'int': None, 'bits':None, 'frac': None}\n"," self.b = {'tensor':None, 'int': None, 'bits':None, 'frac': None}\n","\n"," # Store reference to bundle object here, not just a idx number\n"," self.prev_bundle = None\n"," self.add_bundle = None\n","\n"," def extract_act(signature):\n"," ilayer = QActivation(signature)\n"," d = ilayer.quantizer.get_config()\n"," sign_bit = d['keep_negative'] if 'keep_negative' in d else (d['negative_slope'] !=0 if 'negative_slope' in d else (0))\n"," int_bit = d['integer'] if 'integer' in d else 0\n"," frac = d['bits']-int_bit-sign_bit\n","\n"," if isinstance(ilayer.quantizer, quantized_bits):\n"," return { 'layer':ilayer, 'type':'quant', 'bits':d['bits'], 'frac':frac}\n"," elif 'relu' in str(ilayer.quantizer.__class__) and ilayer.quantizer.negative_slope != 0:\n"," return { 'layer':ilayer, 'type':'relu', 'slope':ilayer.quantizer.negative_slope, 'bits':d['bits'], 'frac':frac}\n"," else:\n"," raise Exception(\"Only leaky_relu (relu with negative_slope > 0) is suppported!\")\n","\n"," '''\n"," CORE LAYER\n"," '''\n"," if core['type'] == 'conv':\n"," for i in ['filters', 'kernel_size', 'strides', 'padding', 'kernel_quantizer', 'bias_quantizer', 'use_bias', 'act_str']:\n"," assert i in core, f\"'{i}' must be provided for conv\"\n","\n"," self.core['layer'] = QConv2DBatchnorm(\n"," filters=self.core['filters'], kernel_size=self.core['kernel_size'], strides=self.core['strides'],\n"," padding=self.core['padding'], kernel_quantizer=self.core['kernel_quantizer'], \n"," bias_quantizer=self.core['bias_quantizer'], use_bias=self.core['use_bias'])\n"," \n"," else:\n"," for i in ['units', 'kernel_quantizer', 'bias_quantizer', 'use_bias', 'act_str']:\n"," assert i in self.core, f\"'{i}' must be provided for dense\"\n"," \n"," self.core['layer'] = QDense(\n"," units=self.core['units'], kernel_quantizer=self.core['kernel_quantizer'],\n"," bias_quantizer=self.core['bias_quantizer'], use_bias=self.core['use_bias'])\n","\n"," '''\n"," CORE ACT LAYER\n"," '''\n"," self.core['act'] = extract_act(core['act_str'])\n"," self.out['frac'], self.out['bits'] = self.core['act']['frac'], self.core['act']['bits']\n","\n"," '''\n"," ACT ADD LAYER\n"," '''\n"," if self.add is not None:\n"," self.add['act'] = extract_act(add['act_str'])\n"," self.out['frac'], self.out['bits'] = self.add['act']['frac'], self.add['act']['bits']\n","\n"," '''\n"," POOL LAYER\n"," '''\n"," if pool:\n"," for i in ['type', 'size', 'strides', 'padding']:\n"," assert i in pool, f\"'{i}' must be provided for pool\"\n","\n"," if pool['type'] == 'max':\n"," self.pool_layer = MaxPooling2D(self.pool['size'], strides=self.pool['strides'], padding=self.pool['padding'])\n"," elif pool['type'] == 'avg':\n"," self.pool_layer = QAveragePooling2D(self.pool['size'], strides=self.pool['strides'], padding=self.pool['padding'])\n"," else:\n"," raise Exception(self.pool['type'], \"only avg or max pool is supported for now\")\n"," \n"," self.pool['act'] = extract_act(self.pool['act_str'])\n"," self.out['frac'], self.out['bits'] = self.pool['act']['frac'], self.pool['act']['bits']\n"," else:\n"," self.pool_layer = None\n","\n"," '''\n"," FLATTEN & SOFTMAX LAYERS\n"," '''\n"," self.flatten_layer = Flatten() if self.flatten else None\n","\n"," self.softmax = softmax\n"," self.softmax_layer = Activation(\"softmax\") if self.softmax else None\n"," if softmax:\n"," self.out['frac'], self.out['bits'] = 0, 1\n","\n","\n"," # functions for training\n"," def call(self, x, x_1=None):\n"," if hasattr(x, \"bundle\"):\n"," self.prev_bundle = x.bundle\n"," self.idx = self.prev_bundle.idx + 1\n"," else:\n"," self.prev_bundle = None\n"," self.idx = 0\n","\n"," self.inp['tensor'] = x\n","\n"," x = self.core['layer'](x)\n"," x = self.core['act']['layer'](x)\n"," self.core['tensor'] = x\n","\n"," if x_1 is not None:\n"," if hasattr(x_1, \"bundle\"):\n"," self.add['bundle'] = x_1.bundle\n"," else:\n"," self.add['bundle'] = None\n"," x = Add()([x, x_1])\n"," x = self.add['act']['layer'](x)\n"," self.add['tensor'] = x\n"," if self.pool_layer:\n"," x = self.pool_layer(x)\n"," x = self.pool['act']['layer'](x)\n"," self.pool['tensor'] = x\n"," if self.flatten_layer:\n"," x = self.flatten_layer(x)\n"," if self.softmax_layer:\n"," x = self.softmax_layer(x)\n","\n"," self.out['tensor'] = x\n"," x.bundle = self\n"," return x\n","\n"," # functions to be prepared for exportation\n"," def load_weight_bias(self):\n"," k_tensor = self.core['layer'].get_folded_weights()[0] if isinstance(self.core['layer'], QConv2DBatchnorm) else self.core['layer'].kernel\n"," k = self.core['layer'].kernel_quantizer_internal(k_tensor).numpy()\n"," k_config = self.core['layer'].kernel_quantizer_internal.get_config()\n","\n"," k_frac = k_config['bits']-k_config['integer']-k_config['keep_negative']\n"," k_int = k * 2**k_frac\n"," assert (k_int == k_int.astype(int)).all(), f\"Weights failed integer test for bundle {self.idx}\"\n"," k_int = k_int.astype(int)\n"," self.w = {'tensor':k_tensor, 'int': k_int, 'bits':k_config['bits'], 'frac':k_frac}\n","\n"," if (self.core['type'] == 'conv' and self.core['use_bias']) or (self.core['type'] == 'dense' and self.core['use_bias']):\n"," b_tensor = self.core['layer'].get_folded_weights()[1] if isinstance(self.core['layer'], QConv2DBatchnorm) else self.core['layer'].bias\n"," b = self.core['layer'].bias_quantizer_internal(b_tensor).numpy()\n"," b_config = self.core['layer'].bias_quantizer_internal.get_config()\n"," b_frac = b_config['bits']-b_config['integer']-b_config['keep_negative']\n"," b_int = b * 2**b_frac\n"," assert (b_int == b_int.astype(int)).all(), f\"Bias failed integer test for bundle {self.idx}\"\n"," b_int = b_int.astype(int)\n"," self.b = {'tensor':b_tensor, 'int':b_int, 'bits':b_config['bits'], 'frac':b_frac}\n","\n","\n"," def process_val(self, function, inp = None):\n"," if inp is not None: # independant mode\n"," self.inp = inp\n"," else: # chained mode\n"," # ToDo: do not rely on external(global) variables!\n"," self.inp = self.prev_bundle.out\n"," assert self.idx > 0, \"input must be provided manually for the first bundle\"\n"," \n"," '''\n"," Integer test for output\n"," '''\n"," self.out['int'] = self.out['tensor'].numpy() * 2**self.out['frac']\n"," if self.softmax is None:\n"," assert (self.out['int'] == self.out['int'].astype(int)).all(), f\"Output tensor of bundle {self.idx} is not a fixed point\"\n"," self.out['int'] = self.out['int'].astype(int)\n","\n"," self.load_weight_bias()\n","\n"," clog2_add = int(np.ceil(np.log2(np.prod(self.w['int'].shape[:-1]))))\n"," self.proc['bits'] = self.inp['bits'] + self.w['bits'] + clog2_add\n"," self.proc['frac'] = self.inp['frac'] + self.w['frac']\n"," self.proc['int'] = function(self.inp['int'], self.w['int'])\n","\n","\n"," if self.b is not None:\n"," self.proc['int'] += self.b['int'] * 2** (self.proc['frac'] - self.b['frac'])\n","\n","\n"," if 'strides' in self.core and self.core['strides'] != (1,1):\n"," SH, SW = self.core['strides']\n"," N, XH, XW, C = self.proc['int'].shape\n"," YH, YW = XH//SH, XW//SW\n"," self.proc['int'] = self.proc['int'].reshape(N, YH, SH, YW, SW, C)\n"," ind = -1 if self.w['int'].shape[0] > 1 else 0\n"," self.proc['int'] = self.proc['int'][:,:,ind,:,ind,:]\n","\n"," def apply_act(act_dict):\n"," x = self.proc['int'].astype(np.float32)\n"," frac, bits = act_dict['frac'], act_dict['bits']\n","\n"," if act_dict['type'] == 'quant':\n"," x *= 2**(frac-self.proc['frac'])\n"," x = np.around(x)\n"," x = np.clip(x, -2**(bits-1), 2**(bits-1)-1).astype(int)\n","\n"," elif act_dict['type'] == 'relu':\n"," x *= 2**(frac-self.proc['frac'])\n"," x = np.clip(x, -2**(bits-1), 2**(bits-1)-1)\n"," x = np.maximum(x * act_dict['slope'], x)\n"," x = np.around(x)\n"," x = np.clip(x,-2**(bits-1), 2**(bits-1)-1).astype(int)\n"," else:\n"," raise Exception('Only relu is supported yet')\n","\n"," self.proc['int'], self.proc['bits'], self.proc['frac'] = x, bits, frac\n","\n"," apply_act(self.core['act'])\n"," assert np.all(self.proc['int'] == self.core['tensor'].numpy() * 2**self.proc['frac']), f\"Core + act output of bundle {self.idx} is not fixed point\"\n","\n","\n"," if self.add is not None:\n"," a = self.add['bundle']\n"," out_frac_add, out_bits_add = max(self.proc['frac'], a.out['frac']), max(self.proc['bits'], a.out['bits'])\n","\n"," a_arr_cast = a.out['int'] * 2** (out_frac_add - a.out['frac'])\n"," out_arr_cast = self.proc['int'] * 2 **(out_frac_add - self.proc['frac'])\n","\n"," self.proc['int'] = out_arr_cast.astype(np.int64) + a_arr_cast.astype(np.int64)\n"," self.proc['bits'], self.proc['frac'] = out_bits_add, out_frac_add\n"," apply_act(self.add['act'])\n","\n"," assert np.all(self.proc['int'] == self.add['tensor'].numpy() * 2**self.proc['frac']), f\"Add + act output of bundle {self.idx} is not a fixed point\"\n","\n"," if self.pool_layer:\n"," if self.pool['type'] == 'max':\n"," pStride = self.pool['strides']\n"," pSize = self.pool['size']\n","\n"," def findMax(InArray, p, q):\n"," results = np.zeros((InArray.shape[0], InArray.shape[3]))\n"," results -= math.inf\n"," for i in range(p, p+pSize[0]):\n"," for j in range(q, q+pSize[1]):\n"," if i >=0 and j>=0 and i < InArray.shape[1] and j < InArray.shape[2]:\n"," cand = InArray[:,i,j,:]\n"," results = np.maximum(results, cand)\n"," return results\n","\n"," def HotFixMaxPool2D(InArray):\n"," if pStride[0]!=pStride[1] or pSize[0]!=pSize[1]:\n"," raise Exception('Only square stride and size is supported')\n"," if pSize[0]/2 == 0:\n"," raise Exception('Maxpool size should be odd')\n","\n"," pad = (pSize[0]-1)//2\n","\n"," inShape = InArray.shape\n"," assert len(inShape) == 4\n"," OutArray = np.zeros((inShape[0], inShape[1]//pStride[0], inShape[2]//pStride[1], inShape[3]))\n"," # Start point, should include pad\n"," st_p, st_q = -pad, -pad\n","\n"," for i in range(OutArray.shape[1]):\n"," for j in range(OutArray.shape[2]):\n"," p, q = st_p + i*pStride[0] + pStride[0]-1, st_q + j*pStride[1] + pStride[1]-1\n"," OutArray[:,i,j,:] = findMax(InArray, p, q)\n","\n"," return OutArray\n","\n"," self.proc['int'] = HotFixMaxPool2D(self.proc['int']).astype(int)\n","\n"," elif self.pool['type'] == 'avg':\n"," assert self.pool['size'] == self.pool['strides']\n"," KH, KW = self.pool['size']\n"," N, H, W, C = self.proc['int'].shape\n"," self.proc['int'] = self.proc['int'].reshape(N, H//KH, KH, W//KW, KW, C).mean(axis=(2,4))\n"," # NO need for clipping, as act_pool in place!\n"," apply_act(self.pool['act'])\n"," assert np.all(self.proc['int'] == self.pool['tensor'].numpy() * 2**self.proc['frac']), f\"Pool + act output of bundle {self.idx} is not a fixed point\"\n","\n"," if self.flatten:\n"," self.proc['int'] = self.proc['int'].reshape(self.proc['int'].shape[0],-1)\n","\n","\n"," if self.softmax:\n"," self.proc['int'] = self.proc['int'] / 2**self.proc['frac']\n"," exp = np.exp(self.proc['int'] - self.proc['int'].max())\n"," self.proc['int'] = exp/np.sum(exp, axis=1)[0]\n"," assert np.all(np.argmax(self.out['int'], axis=-1) == np.argmax(self.proc['int'], axis=-1))\n"," else:\n"," assert np.all(self.proc['int'] == self.out['int']), f\"Overall output of bundle {self.idx} is not a fixed point\"\n","\n"," return\n","\n","\n","'''\n","Build Model\n","'''\n","\n","x = x_in = Input(input_shape, name='input')\n","x = QActivation(q_0)(x)\n","\n","x = x1 = Bundle(\n"," core= {'type':'conv', 'filters':64, 'kernel_size':(7,7), 'strides':(2,2), 'padding':'same', 'kernel_quantizer':q_0, 'bias_quantizer':q_0, 'use_bias':USE_BIAS, 'act_str':a_0},\n"," pool= {'type':'max', 'size':(3,3), 'strides':(1,1), 'padding':'same', 'act_str': q_0}\n"," )(x)\n","\n","# block 0\n","x = Bundle(\n"," core= {'type':'conv', 'filters':64, 'kernel_size':(3,3), 'strides':(1,1), 'padding':'same', 'kernel_quantizer':q_0, 'bias_quantizer':q_0, 'use_bias':USE_BIAS, 'act_str':a_0}\n"," )(x)\n","\n","\n","x = x1 = Bundle(\n"," core= {'type':'conv', 'filters':64, 'kernel_size':(3,3), 'strides':(1,1), 'padding':'same', 'kernel_quantizer':q_0, 'bias_quantizer':q_0, 'use_bias':USE_BIAS, 'act_str':q_1},\n"," add= {'act_str': a_0}\n"," )(x, x1)\n","\n","x = Bundle(\n"," core= {'type':'conv', 'filters':64, 'kernel_size':(3,3), 'strides':(1,1), 'padding':'same', 'kernel_quantizer':q_0, 'bias_quantizer':q_0, 'use_bias':USE_BIAS, 'act_str':a_0}\n"," )(x)\n","x = x1 = Bundle(\n"," core= {'type':'conv', 'filters':64, 'kernel_size':(3,3), 'strides':(1,1), 'padding':'same', 'kernel_quantizer':q_0, 'bias_quantizer':q_0, 'use_bias':USE_BIAS, 'act_str':q_1}, \n"," add= {'act_str': a_1}\n"," )(x, x1)\n","\n","# block 1\n","x1 = Bundle(\n"," core= {'type':'conv', 'filters':128, 'kernel_size':(3,3), 'strides':(2,2), 'padding':'same', 'kernel_quantizer':q_0, 'bias_quantizer':q_0, 'use_bias':USE_BIAS, 'act_str':a_1}\n"," )(x1)\n","x1 = Bundle(\n"," core= {'type':'conv', 'filters':128, 'kernel_size':(3,3), 'strides':(1,1), 'padding':'same', 'kernel_quantizer':q_0, 'bias_quantizer':q_0, 'use_bias':USE_BIAS, 'act_str':q_2}\n"," )(x1)\n","x = x1 = Bundle(\n"," core= {'type':'conv', 'filters':128, 'kernel_size':(1,1), 'strides':(2,2), 'padding':'same', 'kernel_quantizer':q_0, 'bias_quantizer':q_0, 'use_bias':USE_BIAS, 'act_str':q_2},\n"," add={'act_str':a_2}\n"," )(x, x1)\n","\n","x = Bundle(\n"," core= {'type':'conv', 'filters':128, 'kernel_size':(3,3), 'strides':(1,1), 'padding':'same', 'kernel_quantizer':q_0, 'bias_quantizer':q_0, 'use_bias':USE_BIAS, 'act_str':a_1}\n"," )(x)\n","x = x1 = Bundle(\n"," core= {'type':'conv', 'filters':128, 'kernel_size':(1,1), 'strides':(1,1), 'padding':'same', 'kernel_quantizer':q_0, 'bias_quantizer':q_0, 'use_bias':USE_BIAS, 'act_str':q_2},\n"," add={'act_str':a_2}\n"," )(x, x1)\n","\n","#block 2\n","x1 = Bundle(\n"," core= {'type':'conv', 'filters':256, 'kernel_size':(3,3), 'strides':(2,2), 'padding':'same', 'kernel_quantizer':q_0, 'bias_quantizer':q_0, 'use_bias':USE_BIAS, 'act_str':a_1}\n"," )(x1)\n","x1 = Bundle(\n"," core= {'type':'conv', 'filters':256, 'kernel_size':(3,3), 'strides':(1,1), 'padding':'same', 'kernel_quantizer':q_0, 'bias_quantizer':q_0, 'use_bias':USE_BIAS, 'act_str':q_2}\n"," )(x1)\n","x = x1 = Bundle(\n"," core = {'type':'conv', 'filters':256, 'kernel_size':(1,1), 'strides':(2,2), 'padding':'same', 'kernel_quantizer':q_0, 'bias_quantizer':q_0, 'use_bias':USE_BIAS, 'act_str':q_3},\n"," add= {'act_str':a_3}\n"," )(x, x1)\n","\n","x = Bundle(\n"," core = {'type':'conv', 'filters':256, 'kernel_size':(3,3), 'strides':(1,1), 'padding':'same', 'kernel_quantizer':q_0, 'bias_quantizer':q_0, 'use_bias':USE_BIAS, 'act_str':a_2}\n"," )(x)\n","x = x1 = Bundle(\n"," core = {'type':'conv', 'filters':256, 'kernel_size':(1,1), 'strides':(1,1), 'padding':'same', 'kernel_quantizer':q_0, 'bias_quantizer':q_0, 'use_bias':USE_BIAS, 'act_str':q_3},\n"," add= {'act_str':a_3}\n"," )(x, x1)\n","\n","#block 3\n","x1 = Bundle(\n"," core={'type':'conv', 'filters':512, 'kernel_size':(3,3), 'strides':(2,2), 'padding':'same', 'kernel_quantizer':q_0, 'bias_quantizer':q_0, 'use_bias':USE_BIAS, 'act_str':a_2}\n"," )(x1)\n","x1 = Bundle(\n"," core= {'type':'conv', 'filters':512, 'kernel_size':(3,3), 'strides':(1,1), 'padding':'same', 'kernel_quantizer':q_0, 'bias_quantizer':q_0, 'use_bias':USE_BIAS, 'act_str':q_3}\n"," )(x1)\n","x = x1 = Bundle(\n"," core= {'type':'conv', 'filters':512, 'kernel_size':(1,1), 'strides':(2,2), 'padding':'same', 'kernel_quantizer':q_0, 'bias_quantizer':q_0, 'use_bias':USE_BIAS, 'act_str':q_3}, \n"," add= {'act_str':a_3}\n"," )(x, x1)\n","\n","x = Bundle(\n"," core= {'type':'conv', 'filters':512, 'kernel_size':(3,3), 'strides':(1,1), 'padding':'same', 'kernel_quantizer':q_0, 'bias_quantizer':q_0, 'use_bias':USE_BIAS, 'act_str':a_2}\n"," )(x)\n","x = Bundle(\n"," core= {'type':'conv', 'filters':512, 'kernel_size':(1,1), 'strides':(1,1), 'padding':'same', 'kernel_quantizer':q_0, 'bias_quantizer':q_0, 'use_bias':USE_BIAS, 'act_str':q_3}, \n"," add= {'act_str':a_3},\n"," pool= {'type':'avg', 'size':(2,2), 'strides':(2,2), 'padding':'valid', 'act_str': q_3},\n"," flatten=True\n"," )(x, x1)\n","\n","x = Bundle(\n"," core= {'type':'dense', 'units':10, 'kernel_quantizer':q_2, 'bias_quantizer':q_2, 'use_bias':USE_BIAS, 'act_str': q_3}, \n"," softmax=True)(x)\n","\n","model = Model(inputs=x_in, outputs=x)\n","print(model.summary(expand_nested=True))\n"]},{"cell_type":"code","execution_count":6,"metadata":{"id":"qSxuQVKda_OC","tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["Learning rate: 0.001\n"]}],"source":["def lr_schedule(epoch):\n"," \"\"\"\n"," Bundles_pre_trainearning Rate Schedule\n"," Learning rate is scheduled to be reduced after 80, 120, 160, 180 epochs.\n"," Called automatically every epoch as part of callbacks during training.\n"," # Arguments\n"," epoch (int): The number of epochs\n"," # Returns\n"," lr (float32): learning rate\n"," \"\"\"\n"," # initial_lr = 1e-4\n"," # lr_decay = 0.99\n"," # lr = initial_lr * (lr_decay ** epoch)\n"," lr = 1e-3 # default 1e-3\n"," if epoch > 180:\n"," lr *= 0.5e-3\n"," elif epoch > 150:\n"," lr *= 1e-2\n"," elif epoch > 100:\n"," lr *= 1e-1\n"," elif epoch > 50:\n"," lr *= 1e-1\n"," print('Learning rate: ', lr)\n"," return lr\n","\n","preamble = ''\n","model_file_path = preamble+'resnet18.h5'\n","checkpoint = ModelCheckpoint(filepath=model_file_path,\n"," monitor='val_acc',\n"," verbose=1,\n"," save_best_only=True)\n","lr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.1),\n"," cooldown=0,\n"," patience=5,\n"," min_lr=0.5e-6)\n","lr_scheduler = LearningRateScheduler(lr_schedule)\n","\n","callbacks = [checkpoint, lr_reducer, lr_scheduler]\n","\n","NB_EPOCH = 200\n","BATCH_SIZE = 256\n","VERBOSE = 1\n","VALIDATION_SPLIT = 0.1\n","RELU_NEG_SLOPE = 0.125\n","\n","model.compile(loss='categorical_crossentropy',\n"," optimizer=Adam(learning_rate=lr_schedule(0)), metrics=['acc'])\n","\n","# model.fit(x_train, y_train,\n","# batch_size=BATCH_SIZE,\n","# epochs=NB_EPOCH,\n","# validation_data=(x_test, y_test),\n","# shuffle=True,\n","# callbacks=callbacks)"]},{"cell_type":"code","execution_count":7,"metadata":{"id":"xwApBeKxa_OD","tags":[]},"outputs":[],"source":["#arch = model.to_json()\n","#with open(\"bundle_resnet_18.json\", 'w') as arch_file:\n","# arch_file.write(arch)"]},{"cell_type":"code","execution_count":8,"metadata":{"id":"I6_vo0gYa_OD"},"outputs":[],"source":["def conv(x,w):\n"," x = x.astype(np.int32)\n"," w = w.astype(np.int32)\n"," return tf.keras.backend.conv2d(x, w, padding='same').numpy()"]},{"cell_type":"code","execution_count":9,"metadata":{"id":"VPp5nSPYa_OD","outputId":"968bf09a-7290-4544-fcda-e458b29b7777"},"outputs":[{"name":"stdout","output_type":"stream","text":["(4, 32, 32, 3)\n"]}],"source":["XN = 4\n","x = np.random.randn(XN, *model.input.shape[1:])\n","x = np.clip(x, -1.0, 1.0)\n","print(x.shape)\n","\n","pre_layer = model.layers[1]\n","temp_model = Model(inputs=model.input, outputs=pre_layer.output)\n","x_init = temp_model(x, training=False)\n","\n","# for i, layer in enumerate(model.layers[2:]): # disregard input and initial quant layers\n","# print(layer.name)\n","# temp_model = Model(inputs=model.inp, outputs=layer.out)\n","# y = temp_model(x, training=False).numpy()\n","# x_init = x_init if i == 0 else None\n","# layer.prepare_val(y, x_init)"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":[]},{"cell_type":"code","execution_count":10,"metadata":{},"outputs":[{"data":{"text/plain":["7"]},"execution_count":10,"metadata":{},"output_type":"execute_result"}],"source":["model.layers[2].out['frac']"]},{"cell_type":"code","execution_count":11,"metadata":{"id":"LGQRt23Na_OD"},"outputs":[],"source":["init_layer = model.layers[1]\n","temp_model = Model(inputs=model.input, outputs=init_layer.output)\n","inp ={ 'tensor': temp_model(x, training=False), 'bits':8, 'frac':7}\n","inp['int'] = inp['tensor'].numpy() * 2**inp['frac']\n","\n","\n","y = model(x)\n","\n","model.layers[2].process_val(conv, inp)\n","for j in range(3, len(model.layers)-1):\n"," model.layers[j].process_val(conv)\n","\n","model.layers[-1].process_val((lambda x, w : x @ w))"]},{"cell_type":"code","execution_count":12,"metadata":{},"outputs":[{"data":{"text/plain":["True"]},"execution_count":12,"metadata":{},"output_type":"execute_result"}],"source":["tf.executing_eagerly()"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":[]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":[]}],"metadata":{"accelerator":"GPU","colab":{"gpuType":"V100","machine_shape":"hm","provenance":[]},"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.9"}},"nbformat":4,"nbformat_minor":0} +{"cells":[{"cell_type":"code","execution_count":1,"metadata":{"id":"ABy3xAE8uW__"},"outputs":[{"name":"stderr","output_type":"stream","text":["c:\\ProgramData\\Miniconda3\\envs\\qkeras\\lib\\site-packages\\numpy\\_distributor_init.py:30: UserWarning: loaded more than 1 DLL from .libs:\n","c:\\ProgramData\\Miniconda3\\envs\\qkeras\\lib\\site-packages\\numpy\\.libs\\libopenblas.FB5AE2TYXYH2IJRDKGDGQ3XBKLKTF43H.gfortran-win_amd64.dll\n","c:\\ProgramData\\Miniconda3\\envs\\qkeras\\lib\\site-packages\\numpy\\.libs\\libopenblas64__v0.3.21-gcc_10_3_0.dll\n"," warnings.warn(\"loaded more than 1 DLL from .libs:\"\n"]}],"source":["from qkeras import *\n","from tensorflow.keras.layers import Input, AveragePooling2D, Flatten, Softmax, Add, ZeroPadding2D, MaxPooling2D\n","import numpy as np\n","from collections import namedtuple\n","import pickle\n","import math\n","import tensorflow as tf\n","from tensorflow.keras.optimizers import Adam\n","from tensorflow.keras.callbacks import ModelCheckpoint, LearningRateScheduler\n","from tensorflow.keras.callbacks import ReduceLROnPlateau\n","from tensorflow.keras.preprocessing.image import ImageDataGenerator\n","from tensorflow.keras.datasets import cifar10\n","from tensorflow.keras.utils import plot_model\n","from tensorflow.keras.utils import to_categorical\n","from qkeras.utils import model_save_quantized_weights"]},{"cell_type":"code","execution_count":2,"metadata":{"id":"vG3iXUBnuXAB"},"outputs":[],"source":["def load_data(num_classes=10, subtract_pixel_mean=True):\n"," \"\"\"\n"," Load CIFAR10 data and normalize\n"," \"\"\"\n"," (x_train, y_train), (x_test, y_test) = cifar10.load_data()\n","\n"," # input image dimensions.\n"," input_shape = x_train.shape[1:]\n","\n"," # normalize data.\n"," x_train = x_train.astype('float32') / 128.0 - 1.0\n"," x_test = x_test.astype('float32') / 128.0 - 1.0\n","\n"," # if subtract pixel mean is enabled\n"," if subtract_pixel_mean:\n"," x_train_mean = np.mean(x_train, axis=0)\n"," x_train -= x_train_mean\n"," x_test -= x_train_mean\n","\n"," print('x_train shape:', x_train.shape)\n"," print(x_train.shape[0], 'train samples')\n"," print(x_test.shape[0], 'test samples')\n"," print('y_train shape:', y_train.shape)\n","\n"," # convert class vectors to binary class matrices,\n"," # i.e., one hot encodings\n"," y_train = to_categorical(y_train, num_classes)\n"," y_test = to_categorical(y_test, num_classes)\n","\n"," return x_train, y_train, x_test, y_test\n"]},{"cell_type":"code","execution_count":3,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"ZtMyMdKTuXAC","outputId":"24e6c7d8-b3bc-4b07-a12f-3d3955194087"},"outputs":[{"name":"stdout","output_type":"stream","text":["x_train shape: (50000, 32, 32, 3)\n","50000 train samples\n","10000 test samples\n","y_train shape: (50000, 1)\n"]}],"source":["x_train, y_train, x_test, y_test = load_data(10, False)"]},{"cell_type":"code","execution_count":4,"metadata":{"id":"FtdLlJbzuXAC"},"outputs":[],"source":["input_shape = x_train.shape[1:-1] + (3,)\n","np.random.seed(1)\n","\n","a_0 = 'quantized_relu(8,0,negative_slope=0.125)'\n","a_1 = 'quantized_relu(8,1,negative_slope=0.125)'\n","a_2 = 'quantized_relu(8,2,negative_slope=0.125)'\n","a_3 = 'quantized_relu(8,3,negative_slope=0.125)'\n","\n","q_0 = 'quantized_bits(8,0,False,True,1)'\n","q_1 = 'quantized_bits(8,1,False,True,1)'\n","q_2 = 'quantized_bits(8,2,False,True,1)'\n","q_3 = 'quantized_bits(8,3,False,True,1)'\n","\n","q_t = 'quantized_bits(8,0,False,True,1)'\n","\n","np.random.seed(42)\n","#preamble = './drive/MyDrive/resnet/'\n","preamble = ''\n","USE_BIAS = True"]},{"cell_type":"code","execution_count":5,"metadata":{"id":"qgoMo_ima_OB"},"outputs":[{"name":"stdout","output_type":"stream","text":["Model: \"model\"\n","__________________________________________________________________________________________________\n"," Layer (type) Output Shape Param # Connected to \n","==================================================================================================\n"," input (InputLayer) [(None, 32, 32, 3)] 0 [] \n"," \n"," q_activation (QActivation) (None, 32, 32, 3) 0 ['input[0][0]'] \n"," \n"," bundle (Bundle) (None, 16, 16, 64) 9729 ['q_activation[0][0]'] \n","|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n","| q_activation_1 (QActivation) multiple 0 [] |\n","| |\n","| q_conv2d_batchnorm (QConv2DBat multiple 9729 [] |\n","| chnorm) |\n","| |\n","| q_activation_2 (QActivation) multiple 0 [] |\n","| |\n","| max_pooling2d (MaxPooling2D) multiple 0 [] |\n","¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯\n"," bundle_1 (Bundle) (None, 16, 16, 64) 37185 ['bundle[0][0]'] \n","|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n","| q_activation_3 (QActivation) multiple 0 [] |\n","| |\n","| q_conv2d_batchnorm_1 (QConv2DB multiple 37185 [] |\n","| atchnorm) |\n","¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯\n"," bundle_2 (Bundle) (None, 16, 16, 64) 37185 ['bundle_1[0][0]', \n"," 'bundle[0][0]'] \n","|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n","| q_activation_4 (QActivation) multiple 0 [] |\n","| |\n","| q_conv2d_batchnorm_2 (QConv2DB multiple 37185 [] |\n","| atchnorm) |\n","| |\n","| q_activation_5 (QActivation) multiple 0 [] |\n","¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯\n"," bundle_3 (Bundle) (None, 16, 16, 64) 37185 ['bundle_2[0][0]'] \n","|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n","| q_activation_6 (QActivation) multiple 0 [] |\n","| |\n","| q_conv2d_batchnorm_3 (QConv2DB multiple 37185 [] |\n","| atchnorm) |\n","¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯\n"," bundle_4 (Bundle) (None, 16, 16, 64) 37185 ['bundle_3[0][0]', \n"," 'bundle_2[0][0]'] \n","|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n","| q_activation_7 (QActivation) multiple 0 [] |\n","| |\n","| q_conv2d_batchnorm_4 (QConv2DB multiple 37185 [] |\n","| atchnorm) |\n","| |\n","| q_activation_8 (QActivation) multiple 0 [] |\n","¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯\n"," bundle_5 (Bundle) (None, 8, 8, 128) 74369 ['bundle_4[0][0]'] \n","|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n","| q_activation_9 (QActivation) multiple 0 [] |\n","| |\n","| q_conv2d_batchnorm_5 (QConv2DB multiple 74369 [] |\n","| atchnorm) |\n","¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯\n"," bundle_6 (Bundle) (None, 8, 8, 128) 148097 ['bundle_5[0][0]'] \n","|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n","| q_activation_10 (QActivation) multiple 0 [] |\n","| |\n","| q_conv2d_batchnorm_6 (QConv2DB multiple 148097 [] |\n","| atchnorm) |\n","¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯\n"," bundle_7 (Bundle) (None, 8, 8, 128) 8833 ['bundle_4[0][0]', \n"," 'bundle_6[0][0]'] \n","|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n","| q_activation_11 (QActivation) multiple 0 [] |\n","| |\n","| q_conv2d_batchnorm_7 (QConv2DB multiple 8833 [] |\n","| atchnorm) |\n","| |\n","| q_activation_12 (QActivation) multiple 0 [] |\n","¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯\n"," bundle_8 (Bundle) (None, 8, 8, 128) 148097 ['bundle_7[0][0]'] \n","|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n","| q_activation_13 (QActivation) multiple 0 [] |\n","| |\n","| q_conv2d_batchnorm_8 (QConv2DB multiple 148097 [] |\n","| atchnorm) |\n","¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯\n"," bundle_9 (Bundle) (None, 8, 8, 128) 17025 ['bundle_8[0][0]', \n"," 'bundle_7[0][0]'] \n","|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n","| q_activation_14 (QActivation) multiple 0 [] |\n","| |\n","| q_conv2d_batchnorm_9 (QConv2DB multiple 17025 [] |\n","| atchnorm) |\n","| |\n","| q_activation_15 (QActivation) multiple 0 [] |\n","¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯\n"," bundle_10 (Bundle) (None, 4, 4, 256) 296193 ['bundle_9[0][0]'] \n","|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n","| q_activation_16 (QActivation) multiple 0 [] |\n","| |\n","| q_conv2d_batchnorm_10 (QConv2D multiple 296193 [] |\n","| Batchnorm) |\n","¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯\n"," bundle_11 (Bundle) (None, 4, 4, 256) 591105 ['bundle_10[0][0]'] \n","|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n","| q_activation_17 (QActivation) multiple 0 [] |\n","| |\n","| q_conv2d_batchnorm_11 (QConv2D multiple 591105 [] |\n","| Batchnorm) |\n","¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯\n"," bundle_12 (Bundle) (None, 4, 4, 256) 34049 ['bundle_9[0][0]', \n"," 'bundle_11[0][0]'] \n","|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n","| q_activation_18 (QActivation) multiple 0 [] |\n","| |\n","| q_conv2d_batchnorm_12 (QConv2D multiple 34049 [] |\n","| Batchnorm) |\n","| |\n","| q_activation_19 (QActivation) multiple 0 [] |\n","¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯\n"," bundle_13 (Bundle) (None, 4, 4, 256) 591105 ['bundle_12[0][0]'] \n","|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n","| q_activation_20 (QActivation) multiple 0 [] |\n","| |\n","| q_conv2d_batchnorm_13 (QConv2D multiple 591105 [] |\n","| Batchnorm) |\n","¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯\n"," bundle_14 (Bundle) (None, 4, 4, 256) 66817 ['bundle_13[0][0]', \n"," 'bundle_12[0][0]'] \n","|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n","| q_activation_21 (QActivation) multiple 0 [] |\n","| |\n","| q_conv2d_batchnorm_14 (QConv2D multiple 66817 [] |\n","| Batchnorm) |\n","| |\n","| q_activation_22 (QActivation) multiple 0 [] |\n","¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯\n"," bundle_15 (Bundle) (None, 2, 2, 512) 1182209 ['bundle_14[0][0]'] \n","|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n","| q_activation_23 (QActivation) multiple 0 [] |\n","| |\n","| q_conv2d_batchnorm_15 (QConv2D multiple 1182209 [] |\n","| Batchnorm) |\n","¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯\n"," bundle_16 (Bundle) (None, 2, 2, 512) 2361857 ['bundle_15[0][0]'] \n","|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n","| q_activation_24 (QActivation) multiple 0 [] |\n","| |\n","| q_conv2d_batchnorm_16 (QConv2D multiple 2361857 [] |\n","| Batchnorm) |\n","¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯\n"," bundle_17 (Bundle) (None, 2, 2, 512) 133633 ['bundle_14[0][0]', \n"," 'bundle_16[0][0]'] \n","|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n","| q_activation_25 (QActivation) multiple 0 [] |\n","| |\n","| q_conv2d_batchnorm_17 (QConv2D multiple 133633 [] |\n","| Batchnorm) |\n","| |\n","| q_activation_26 (QActivation) multiple 0 [] |\n","¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯\n"," bundle_18 (Bundle) (None, 2, 2, 512) 2361857 ['bundle_17[0][0]'] \n","|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n","| q_activation_27 (QActivation) multiple 0 [] |\n","| |\n","| q_conv2d_batchnorm_18 (QConv2D multiple 2361857 [] |\n","| Batchnorm) |\n","¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯\n"," bundle_19 (Bundle) (None, 512) 264705 ['bundle_18[0][0]', \n"," 'bundle_17[0][0]'] \n","|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n","| q_activation_28 (QActivation) multiple 0 [] |\n","| |\n","| q_conv2d_batchnorm_19 (QConv2D multiple 264705 [] |\n","| Batchnorm) |\n","| |\n","| q_activation_29 (QActivation) multiple 0 [] |\n","| |\n","| q_activation_30 (QActivation) multiple 0 [] |\n","| |\n","| q_average_pooling2d (QAverageP multiple 0 [] |\n","| ooling2D) |\n","| |\n","| flatten (Flatten) multiple 0 [] |\n","¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯\n"," bundle_20 (Bundle) (None, 10) 5130 ['bundle_19[0][0]'] \n","|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n","| q_activation_31 (QActivation) multiple 0 [] |\n","| |\n","| q_dense (QDense) multiple 5130 [] |\n","| |\n","| activation (Activation) multiple 0 [] |\n","¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯\n","==================================================================================================\n","Total params: 8,443,550\n","Trainable params: 8,433,930\n","Non-trainable params: 9,620\n","__________________________________________________________________________________________________\n","None\n"]}],"source":["class Bundle(tf.keras.Model):\n"," def __init__(self, \n"," core, # dict, Mandaroty: parameters for conv/dense layer, act can be quantization or relu\n"," add=None, # dict, Mandatory if x1 is not None in call(), else ignored\n"," pool=None, # dict, Optional: can only be max or avg\n"," flatten=False, # Optional: set to True to flatten the outputs\n"," softmax=False, # Optional: set to Ture to include floating point softmax layer\n"," **kwargs):\n","\n"," super(Bundle, self).__init__()\n"," \n"," self.core = core\n"," self.add = add\n"," self.pool = pool\n"," self.flatten = flatten\n"," self.softmax = softmax\n"," self.inp = {'tensor':None, 'int': None, 'bits':None, 'frac': None}\n"," self.out = {'tensor':None, 'int': None, 'bits':None, 'frac': None}\n"," self.proc = {'tensor':None, 'int': None, 'bits':None, 'frac': None}\n"," self.w = {'tensor':None, 'int': None, 'bits':None, 'frac': None}\n"," self.b = {'tensor':None, 'int': None, 'bits':None, 'frac': None}\n","\n"," # Store reference to bundle object here, not just a idx number\n"," self.prev_bundle = None\n"," self.add_bundle = None\n","\n"," def extract_act(signature):\n"," ilayer = QActivation(signature)\n"," d = ilayer.quantizer.get_config()\n"," sign_bit = d['keep_negative'] if 'keep_negative' in d else (d['negative_slope'] !=0 if 'negative_slope' in d else (0))\n"," int_bit = d['integer'] if 'integer' in d else 0\n"," frac = d['bits']-int_bit-sign_bit\n","\n"," if isinstance(ilayer.quantizer, quantized_bits):\n"," return { 'layer':ilayer, 'type':'quant', 'bits':d['bits'], 'frac':frac}\n"," elif 'relu' in str(ilayer.quantizer.__class__) and ilayer.quantizer.negative_slope != 0:\n"," return { 'layer':ilayer, 'type':'relu', 'slope':ilayer.quantizer.negative_slope, 'bits':d['bits'], 'frac':frac}\n"," else:\n"," raise Exception(\"Only leaky_relu (relu with negative_slope > 0) is suppported!\")\n","\n"," '''\n"," CORE LAYER\n"," '''\n"," if core['type'] == 'conv':\n"," for i in ['filters', 'kernel_size', 'strides', 'padding', 'kernel_quantizer', 'bias_quantizer', 'use_bias', 'act_str']:\n"," assert i in core, f\"'{i}' must be provided for conv\"\n","\n"," self.core['layer'] = QConv2DBatchnorm(\n"," filters=self.core['filters'], kernel_size=self.core['kernel_size'], strides=self.core['strides'],\n"," padding=self.core['padding'], kernel_quantizer=self.core['kernel_quantizer'], \n"," bias_quantizer=self.core['bias_quantizer'], use_bias=self.core['use_bias'])\n"," \n"," else:\n"," for i in ['units', 'kernel_quantizer', 'bias_quantizer', 'use_bias', 'act_str']:\n"," assert i in self.core, f\"'{i}' must be provided for dense\"\n"," \n"," self.core['layer'] = QDense(\n"," units=self.core['units'], kernel_quantizer=self.core['kernel_quantizer'],\n"," bias_quantizer=self.core['bias_quantizer'], use_bias=self.core['use_bias'])\n","\n"," '''\n"," CORE ACT LAYER\n"," '''\n"," self.core['act'] = extract_act(core['act_str'])\n"," self.out['frac'], self.out['bits'] = self.core['act']['frac'], self.core['act']['bits']\n","\n"," '''\n"," ACT ADD LAYER\n"," '''\n"," if self.add is not None:\n"," self.add['act'] = extract_act(add['act_str'])\n"," self.out['frac'], self.out['bits'] = self.add['act']['frac'], self.add['act']['bits']\n","\n"," '''\n"," POOL LAYER\n"," '''\n"," if pool:\n"," for i in ['type', 'size', 'strides', 'padding']:\n"," assert i in pool, f\"'{i}' must be provided for pool\"\n","\n"," if pool['type'] == 'max':\n"," self.pool_layer = MaxPooling2D(self.pool['size'], strides=self.pool['strides'], padding=self.pool['padding'])\n"," elif pool['type'] == 'avg':\n"," self.pool_layer = QAveragePooling2D(self.pool['size'], strides=self.pool['strides'], padding=self.pool['padding'])\n"," else:\n"," raise Exception(self.pool['type'], \"only avg or max pool is supported for now\")\n"," \n"," self.pool['act'] = extract_act(self.pool['act_str'])\n"," self.out['frac'], self.out['bits'] = self.pool['act']['frac'], self.pool['act']['bits']\n"," else:\n"," self.pool_layer = None\n","\n"," '''\n"," FLATTEN & SOFTMAX LAYERS\n"," '''\n"," self.flatten_layer = Flatten() if self.flatten else None\n","\n"," self.softmax = softmax\n"," self.softmax_layer = Activation(\"softmax\") if self.softmax else None\n"," if softmax:\n"," self.out['frac'], self.out['bits'] = 0, 1\n","\n","\n"," # functions for training\n"," def call(self, x, x_1=None):\n"," if hasattr(x, \"bundle\"):\n"," self.prev_bundle = x.bundle\n"," self.idx = self.prev_bundle.idx + 1\n"," else:\n"," self.prev_bundle = None\n"," self.idx = 0\n","\n"," self.inp['tensor'] = x\n","\n"," x = self.core['layer'](x)\n"," x = self.core['act']['layer'](x)\n"," self.core['tensor'] = x\n","\n"," if x_1 is not None:\n"," if hasattr(x_1, \"bundle\"):\n"," self.add['bundle'] = x_1.bundle\n"," else:\n"," self.add['bundle'] = None\n"," x = Add()([x, x_1])\n"," x = self.add['act']['layer'](x)\n"," self.add['tensor'] = x\n"," if self.pool_layer:\n"," x = self.pool_layer(x)\n"," x = self.pool['act']['layer'](x)\n"," self.pool['tensor'] = x\n"," if self.flatten_layer:\n"," x = self.flatten_layer(x)\n"," if self.softmax_layer:\n"," x = self.softmax_layer(x)\n","\n"," self.out['tensor'] = x\n"," x.bundle = self\n"," return x\n","\n"," # functions to be prepared for exportation\n"," def load_weight_bias(self):\n"," k_tensor = self.core['layer'].get_folded_weights()[0] if isinstance(self.core['layer'], QConv2DBatchnorm) else self.core['layer'].kernel\n"," k = self.core['layer'].kernel_quantizer_internal(k_tensor).numpy()\n"," k_config = self.core['layer'].kernel_quantizer_internal.get_config()\n","\n"," k_frac = k_config['bits']-k_config['integer']-k_config['keep_negative']\n"," k_int = k * 2**k_frac\n"," assert (k_int == k_int.astype(int)).all(), f\"Weights failed integer test for bundle {self.idx}\"\n"," k_int = k_int.astype(int)\n"," self.w = {'tensor':k_tensor, 'int': k_int, 'bits':k_config['bits'], 'frac':k_frac}\n","\n"," if (self.core['type'] == 'conv' and self.core['use_bias']) or (self.core['type'] == 'dense' and self.core['use_bias']):\n"," b_tensor = self.core['layer'].get_folded_weights()[1] if isinstance(self.core['layer'], QConv2DBatchnorm) else self.core['layer'].bias\n"," b = self.core['layer'].bias_quantizer_internal(b_tensor).numpy()\n"," b_config = self.core['layer'].bias_quantizer_internal.get_config()\n"," b_frac = b_config['bits']-b_config['integer']-b_config['keep_negative']\n"," b_int = b * 2**b_frac\n"," assert (b_int == b_int.astype(int)).all(), f\"Bias failed integer test for bundle {self.idx}\"\n"," b_int = b_int.astype(int)\n"," self.b = {'tensor':b_tensor, 'int':b_int, 'bits':b_config['bits'], 'frac':b_frac}\n","\n","\n"," def process(self, inp = None):\n"," \n"," ''' Integer test for output '''\n"," self.out['int'] = self.out['tensor'].numpy() * 2**self.out['frac']\n"," if self.softmax is None:\n"," assert (self.out['int'] == self.out['int'].astype(int)).all(), f\"Output tensor of bundle {self.idx} is not a fixed point\"\n"," self.out['int'] = self.out['int'].astype(int)\n","\n"," if inp is not None: # independant mode\n"," self.inp = inp\n"," else: # chained mode\n"," # ToDo: do not rely on external(global) variables!\n"," self.inp = self.prev_bundle.out\n"," assert self.idx > 0, \"input must be provided manually for the first bundle\"\n","\n"," self.load_weight_bias()\n"," x = self.inp['int'].astype(np.int32)\n"," w = self.w['int'].astype(np.int32)\n","\n"," if self.core['type'] == 'conv':\n"," self.proc['int'] = tf.keras.backend.conv2d(x, w, padding='same').numpy()\n"," else:\n"," self.proc['int'] = x @ w\n","\n"," self.post_process()\n","\n","\n"," def post_process(self):\n"," \n"," clog2_add = int(np.ceil(np.log2(np.prod(self.w['int'].shape[:-1]))))\n"," self.proc['bits'] = self.inp['bits'] + self.w['bits'] + clog2_add\n"," self.proc['frac'] = self.inp['frac'] + self.w['frac']\n","\n"," if self.b is not None:\n"," self.proc['int'] += self.b['int'] * 2** (self.proc['frac'] - self.b['frac'])\n","\n","\n"," if 'strides' in self.core and self.core['strides'] != (1,1):\n"," SH, SW = self.core['strides']\n"," N, XH, XW, C = self.proc['int'].shape\n"," YH, YW = XH//SH, XW//SW\n"," self.proc['int'] = self.proc['int'].reshape(N, YH, SH, YW, SW, C)\n"," ind = -1 if self.w['int'].shape[0] > 1 else 0\n"," self.proc['int'] = self.proc['int'][:,:,ind,:,ind,:]\n","\n"," def apply_act(act_dict):\n"," x = self.proc['int'].astype(np.float32)\n"," frac, bits = act_dict['frac'], act_dict['bits']\n","\n"," if act_dict['type'] == 'quant':\n"," x *= 2**(frac-self.proc['frac'])\n"," x = np.around(x)\n"," x = np.clip(x, -2**(bits-1), 2**(bits-1)-1).astype(int)\n","\n"," elif act_dict['type'] == 'relu':\n"," x *= 2**(frac-self.proc['frac'])\n"," x = np.clip(x, -2**(bits-1), 2**(bits-1)-1)\n"," x = np.maximum(x * act_dict['slope'], x)\n"," x = np.around(x)\n"," x = np.clip(x,-2**(bits-1), 2**(bits-1)-1).astype(int)\n"," else:\n"," raise Exception('Only relu is supported yet')\n","\n"," self.proc['int'], self.proc['bits'], self.proc['frac'] = x, bits, frac\n","\n"," apply_act(self.core['act'])\n"," assert np.all(self.proc['int'] == self.core['tensor'].numpy() * 2**self.proc['frac']), f\"Core + act output of bundle {self.idx} is not fixed point\"\n","\n","\n"," if self.add is not None:\n"," a = self.add['bundle']\n"," out_frac_add, out_bits_add = max(self.proc['frac'], a.out['frac']), max(self.proc['bits'], a.out['bits'])\n","\n"," a_arr_cast = a.out['int'] * 2** (out_frac_add - a.out['frac'])\n"," out_arr_cast = self.proc['int'] * 2 **(out_frac_add - self.proc['frac'])\n","\n"," self.proc['int'] = out_arr_cast.astype(np.int64) + a_arr_cast.astype(np.int64)\n"," self.proc['bits'], self.proc['frac'] = out_bits_add, out_frac_add\n"," apply_act(self.add['act'])\n","\n"," assert np.all(self.proc['int'] == self.add['tensor'].numpy() * 2**self.proc['frac']), f\"Add + act output of bundle {self.idx} is not a fixed point\"\n","\n"," if self.pool_layer:\n"," if self.pool['type'] == 'max':\n"," pStride = self.pool['strides']\n"," pSize = self.pool['size']\n","\n"," def findMax(InArray, p, q):\n"," results = np.zeros((InArray.shape[0], InArray.shape[3]))\n"," results -= math.inf\n"," for i in range(p, p+pSize[0]):\n"," for j in range(q, q+pSize[1]):\n"," if i >=0 and j>=0 and i < InArray.shape[1] and j < InArray.shape[2]:\n"," cand = InArray[:,i,j,:]\n"," results = np.maximum(results, cand)\n"," return results\n","\n"," def HotFixMaxPool2D(InArray):\n"," if pStride[0]!=pStride[1] or pSize[0]!=pSize[1]:\n"," raise Exception('Only square stride and size is supported')\n"," if pSize[0]/2 == 0:\n"," raise Exception('Maxpool size should be odd')\n","\n"," pad = (pSize[0]-1)//2\n","\n"," inShape = InArray.shape\n"," assert len(inShape) == 4\n"," OutArray = np.zeros((inShape[0], inShape[1]//pStride[0], inShape[2]//pStride[1], inShape[3]))\n"," # Start point, should include pad\n"," st_p, st_q = -pad, -pad\n","\n"," for i in range(OutArray.shape[1]):\n"," for j in range(OutArray.shape[2]):\n"," p, q = st_p + i*pStride[0] + pStride[0]-1, st_q + j*pStride[1] + pStride[1]-1\n"," OutArray[:,i,j,:] = findMax(InArray, p, q)\n","\n"," return OutArray\n","\n"," self.proc['int'] = HotFixMaxPool2D(self.proc['int']).astype(int)\n","\n"," elif self.pool['type'] == 'avg':\n"," assert self.pool['size'] == self.pool['strides']\n"," KH, KW = self.pool['size']\n"," N, H, W, C = self.proc['int'].shape\n"," self.proc['int'] = self.proc['int'].reshape(N, H//KH, KH, W//KW, KW, C).mean(axis=(2,4))\n"," # NO need for clipping, as act_pool in place!\n"," apply_act(self.pool['act'])\n"," assert np.all(self.proc['int'] == self.pool['tensor'].numpy() * 2**self.proc['frac']), f\"Pool + act output of bundle {self.idx} is not a fixed point\"\n","\n"," if self.flatten:\n"," self.proc['int'] = self.proc['int'].reshape(self.proc['int'].shape[0],-1)\n","\n","\n"," if self.softmax:\n"," self.proc['int'] = self.proc['int'] / 2**self.proc['frac']\n"," exp = np.exp(self.proc['int'] - self.proc['int'].max())\n"," self.proc['int'] = exp/np.sum(exp, axis=1)[0]\n"," assert np.all(np.argmax(self.out['int'], axis=-1) == np.argmax(self.proc['int'], axis=-1))\n"," else:\n"," assert np.all(self.proc['int'] == self.out['int']), f\"Overall output of bundle {self.idx} is not a fixed point\"\n","\n"," return\n","\n","class BModel(tf.keras.Model):\n"," def __init__(self,\n"," **kwargs):\n"," super(BModel, self).__init__()\n","\n","\n","'''\n","Build Model\n","'''\n","\n","x = x_in = Input(input_shape, name='input')\n","x = QActivation(q_0)(x)\n","\n","x = x1 = Bundle(\n"," core= {'type':'conv', 'filters':64, 'kernel_size':(7,7), 'strides':(2,2), 'padding':'same', 'kernel_quantizer':q_0, 'bias_quantizer':q_0, 'use_bias':USE_BIAS, 'act_str':a_0},\n"," pool= {'type':'max', 'size':(3,3), 'strides':(1,1), 'padding':'same', 'act_str': q_0}\n"," )(x)\n","\n","# block 0\n","x = Bundle(\n"," core= {'type':'conv', 'filters':64, 'kernel_size':(3,3), 'strides':(1,1), 'padding':'same', 'kernel_quantizer':q_0, 'bias_quantizer':q_0, 'use_bias':USE_BIAS, 'act_str':a_0}\n"," )(x)\n","\n","\n","x = x1 = Bundle(\n"," core= {'type':'conv', 'filters':64, 'kernel_size':(3,3), 'strides':(1,1), 'padding':'same', 'kernel_quantizer':q_0, 'bias_quantizer':q_0, 'use_bias':USE_BIAS, 'act_str':q_1},\n"," add= {'act_str': a_0}\n"," )(x, x1)\n","\n","x = Bundle(\n"," core= {'type':'conv', 'filters':64, 'kernel_size':(3,3), 'strides':(1,1), 'padding':'same', 'kernel_quantizer':q_0, 'bias_quantizer':q_0, 'use_bias':USE_BIAS, 'act_str':a_0}\n"," )(x)\n","x = x1 = Bundle(\n"," core= {'type':'conv', 'filters':64, 'kernel_size':(3,3), 'strides':(1,1), 'padding':'same', 'kernel_quantizer':q_0, 'bias_quantizer':q_0, 'use_bias':USE_BIAS, 'act_str':q_1}, \n"," add= {'act_str': a_1}\n"," )(x, x1)\n","\n","# block 1\n","x1 = Bundle(\n"," core= {'type':'conv', 'filters':128, 'kernel_size':(3,3), 'strides':(2,2), 'padding':'same', 'kernel_quantizer':q_0, 'bias_quantizer':q_0, 'use_bias':USE_BIAS, 'act_str':a_1}\n"," )(x1)\n","x1 = Bundle(\n"," core= {'type':'conv', 'filters':128, 'kernel_size':(3,3), 'strides':(1,1), 'padding':'same', 'kernel_quantizer':q_0, 'bias_quantizer':q_0, 'use_bias':USE_BIAS, 'act_str':q_2}\n"," )(x1)\n","x = x1 = Bundle(\n"," core= {'type':'conv', 'filters':128, 'kernel_size':(1,1), 'strides':(2,2), 'padding':'same', 'kernel_quantizer':q_0, 'bias_quantizer':q_0, 'use_bias':USE_BIAS, 'act_str':q_2},\n"," add={'act_str':a_2}\n"," )(x, x1)\n","\n","x = Bundle(\n"," core= {'type':'conv', 'filters':128, 'kernel_size':(3,3), 'strides':(1,1), 'padding':'same', 'kernel_quantizer':q_0, 'bias_quantizer':q_0, 'use_bias':USE_BIAS, 'act_str':a_1}\n"," )(x)\n","x = x1 = Bundle(\n"," core= {'type':'conv', 'filters':128, 'kernel_size':(1,1), 'strides':(1,1), 'padding':'same', 'kernel_quantizer':q_0, 'bias_quantizer':q_0, 'use_bias':USE_BIAS, 'act_str':q_2},\n"," add={'act_str':a_2}\n"," )(x, x1)\n","\n","#block 2\n","x1 = Bundle(\n"," core= {'type':'conv', 'filters':256, 'kernel_size':(3,3), 'strides':(2,2), 'padding':'same', 'kernel_quantizer':q_0, 'bias_quantizer':q_0, 'use_bias':USE_BIAS, 'act_str':a_1}\n"," )(x1)\n","x1 = Bundle(\n"," core= {'type':'conv', 'filters':256, 'kernel_size':(3,3), 'strides':(1,1), 'padding':'same', 'kernel_quantizer':q_0, 'bias_quantizer':q_0, 'use_bias':USE_BIAS, 'act_str':q_2}\n"," )(x1)\n","x = x1 = Bundle(\n"," core = {'type':'conv', 'filters':256, 'kernel_size':(1,1), 'strides':(2,2), 'padding':'same', 'kernel_quantizer':q_0, 'bias_quantizer':q_0, 'use_bias':USE_BIAS, 'act_str':q_3},\n"," add= {'act_str':a_3}\n"," )(x, x1)\n","\n","x = Bundle(\n"," core = {'type':'conv', 'filters':256, 'kernel_size':(3,3), 'strides':(1,1), 'padding':'same', 'kernel_quantizer':q_0, 'bias_quantizer':q_0, 'use_bias':USE_BIAS, 'act_str':a_2}\n"," )(x)\n","x = x1 = Bundle(\n"," core = {'type':'conv', 'filters':256, 'kernel_size':(1,1), 'strides':(1,1), 'padding':'same', 'kernel_quantizer':q_0, 'bias_quantizer':q_0, 'use_bias':USE_BIAS, 'act_str':q_3},\n"," add= {'act_str':a_3}\n"," )(x, x1)\n","\n","#block 3\n","x1 = Bundle(\n"," core={'type':'conv', 'filters':512, 'kernel_size':(3,3), 'strides':(2,2), 'padding':'same', 'kernel_quantizer':q_0, 'bias_quantizer':q_0, 'use_bias':USE_BIAS, 'act_str':a_2}\n"," )(x1)\n","x1 = Bundle(\n"," core= {'type':'conv', 'filters':512, 'kernel_size':(3,3), 'strides':(1,1), 'padding':'same', 'kernel_quantizer':q_0, 'bias_quantizer':q_0, 'use_bias':USE_BIAS, 'act_str':q_3}\n"," )(x1)\n","x = x1 = Bundle(\n"," core= {'type':'conv', 'filters':512, 'kernel_size':(1,1), 'strides':(2,2), 'padding':'same', 'kernel_quantizer':q_0, 'bias_quantizer':q_0, 'use_bias':USE_BIAS, 'act_str':q_3}, \n"," add= {'act_str':a_3}\n"," )(x, x1)\n","\n","x = Bundle(\n"," core= {'type':'conv', 'filters':512, 'kernel_size':(3,3), 'strides':(1,1), 'padding':'same', 'kernel_quantizer':q_0, 'bias_quantizer':q_0, 'use_bias':USE_BIAS, 'act_str':a_2}\n"," )(x)\n","x = Bundle(\n"," core= {'type':'conv', 'filters':512, 'kernel_size':(1,1), 'strides':(1,1), 'padding':'same', 'kernel_quantizer':q_0, 'bias_quantizer':q_0, 'use_bias':USE_BIAS, 'act_str':q_3}, \n"," add= {'act_str':a_3},\n"," pool= {'type':'avg', 'size':(2,2), 'strides':(2,2), 'padding':'valid', 'act_str': q_3},\n"," flatten=True\n"," )(x, x1)\n","\n","x = Bundle(\n"," core= {'type':'dense', 'units':10, 'kernel_quantizer':q_2, 'bias_quantizer':q_2, 'use_bias':USE_BIAS, 'act_str': q_3}, \n"," softmax=True)(x)\n","\n","model = Model(inputs=x_in, outputs=x)\n","print(model.summary(expand_nested=True))\n"]},{"cell_type":"code","execution_count":6,"metadata":{"id":"qSxuQVKda_OC","tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["Learning rate: 0.001\n"]}],"source":["def lr_schedule(epoch):\n"," \"\"\"\n"," Bundles_pre_trainearning Rate Schedule\n"," Learning rate is scheduled to be reduced after 80, 120, 160, 180 epochs.\n"," Called automatically every epoch as part of callbacks during training.\n"," # Arguments\n"," epoch (int): The number of epochs\n"," # Returns\n"," lr (float32): learning rate\n"," \"\"\"\n"," # initial_lr = 1e-4\n"," # lr_decay = 0.99\n"," # lr = initial_lr * (lr_decay ** epoch)\n"," lr = 1e-3 # default 1e-3\n"," if epoch > 180:\n"," lr *= 0.5e-3\n"," elif epoch > 150:\n"," lr *= 1e-2\n"," elif epoch > 100:\n"," lr *= 1e-1\n"," elif epoch > 50:\n"," lr *= 1e-1\n"," print('Learning rate: ', lr)\n"," return lr\n","\n","preamble = ''\n","model_file_path = preamble+'resnet18.h5'\n","checkpoint = ModelCheckpoint(filepath=model_file_path,\n"," monitor='val_acc',\n"," verbose=1,\n"," save_best_only=True)\n","lr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.1),\n"," cooldown=0,\n"," patience=5,\n"," min_lr=0.5e-6)\n","lr_scheduler = LearningRateScheduler(lr_schedule)\n","\n","callbacks = [checkpoint, lr_reducer, lr_scheduler]\n","\n","NB_EPOCH = 200\n","BATCH_SIZE = 256\n","VERBOSE = 1\n","VALIDATION_SPLIT = 0.1\n","RELU_NEG_SLOPE = 0.125\n","\n","model.compile(loss='categorical_crossentropy',\n"," optimizer=Adam(learning_rate=lr_schedule(0)), metrics=['acc'])\n","\n","# model.fit(x_train, y_train,\n","# batch_size=BATCH_SIZE,\n","# epochs=NB_EPOCH,\n","# validation_data=(x_test, y_test),\n","# shuffle=True,\n","# callbacks=callbacks)"]},{"cell_type":"code","execution_count":7,"metadata":{"id":"LGQRt23Na_OD"},"outputs":[],"source":["XN = 4\n","x = np.random.randn(XN, *model.input.shape[1:])\n","x = np.clip(x, -1.0, 1.0)\n","\n","inp_act_model = Model(inputs=model.input, outputs=model.layers[1].output)\n","inp ={ 'tensor': inp_act_model(x, training=False), 'bits':8, 'frac':7}\n","inp['int'] = inp['tensor'].numpy() * 2**inp['frac']\n","\n","y = model(x)\n","\n","model.layers[2].process(inp)\n","for layer in model.layers[3:]:\n"," layer.process()"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":[]}],"metadata":{"accelerator":"GPU","colab":{"gpuType":"V100","machine_shape":"hm","provenance":[]},"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.9"}},"nbformat":4,"nbformat_minor":0}