VGGish_implementation.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "VGGish_implementation.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2WRXN7tjOTPs",
        "colab_type": "text"
      },
      "source": [
        "## **Installing Dependencies**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nWvIJE3mOVBy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install tensorflow==1.15\n",
        "!pip install scipy\n",
        "!pip install resampy tensorflow six\n",
        "!pip install tf_slim six soundfile"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VrKABv2hOf02",
        "colab_type": "text"
      },
      "source": [
        "Clonning tensorflow repo"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DUuIB68jOVNX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!git clone https://github.com/tensorflow/models.git"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iasG2Qe7OVO3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Check to see where are in the kernel's file system.\n",
        "!pwd"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ybeIJhIuOVRv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Grab the VGGish model required weights and paramters\n",
        "!curl -O https://storage.googleapis.com/audioset/vggish_model.ckpt\n",
        "!curl -O https://storage.googleapis.com/audioset/vggish_pca_params.npz"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "umSnYzczOVXS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Verify the location of the AudioSet source files\n",
        "!ls models/research/audioset/vggish/"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AVMwxbNlO0Do",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Copy the source files to the current directory.\n",
        "!cp models/research/audioset/vggish/* ."
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U5tkqJYWO0FZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Run the test, which also loads all the necessary functions.\n",
        "from vggish_smoke_test import *"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y7VL-osvO0J1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Run the test, which also loads all the necessary functions.\n",
        "\n",
        "# from vggish_smoke_test import *\n",
        "# either run above line or below line, both will give same results. \n",
        "!python vggish_smoke_test.py"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KpmeveKsO0OK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# this is just for getting embeddings from each wav file\n",
        "# ran it for test purpose.\n",
        "\n",
        "!python vggish_inference_demo.py --wav_file \"/content/drive/My Drive/test_dataset/not_sick/audioset_-21_SXelVNo_30_35.wav\"\\\n",
        "                                    --tfrecord_file \"/content/TfrecordFile\" \\\n",
        "                                    --checkpoint \"/content/vggish_model.ckpt\" \\\n",
        "                                    --pca_params \"/content/vggish_pca_params.npz\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yLUh0dQNPFIz",
        "colab_type": "text"
      },
      "source": [
        "**USING VGGish for extracting embeddings**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vPwtmRM6O0P9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import vggish_slim\n",
        "import vggish_params\n",
        "import vggish_input\n",
        "import tensorflow as tf\n",
        "import numpy as np\n",
        "\n",
        "def CreateVGGishNetwork(hop_size=0.96):   # Hop size is in seconds.\n",
        "  \"\"\"Define VGGish model, load the checkpoint, and return a dictionary that points\n",
        "  to the different tensors defined by the model.\n",
        "  \"\"\"\n",
        "  vggish_slim.define_vggish_slim()\n",
        "  checkpoint_path = 'vggish_model.ckpt'\n",
        "  vggish_params.EXAMPLE_HOP_SECONDS = hop_size\n",
        "  \n",
        "  vggish_slim.load_vggish_slim_checkpoint(sess, checkpoint_path)\n",
        "\n",
        "  features_tensor = sess.graph.get_tensor_by_name(\n",
        "      vggish_params.INPUT_TENSOR_NAME)\n",
        "  embedding_tensor = sess.graph.get_tensor_by_name(\n",
        "      vggish_params.OUTPUT_TENSOR_NAME)\n",
        "\n",
        "  layers = {'conv1': 'vggish/conv1/Relu',\n",
        "            'pool1': 'vggish/pool1/MaxPool',\n",
        "            'conv2': 'vggish/conv2/Relu',\n",
        "            'pool2': 'vggish/pool2/MaxPool',\n",
        "            'conv3': 'vggish/conv3/conv3_2/Relu',\n",
        "            'pool3': 'vggish/pool3/MaxPool',\n",
        "            'conv4': 'vggish/conv4/conv4_2/Relu',\n",
        "            'pool4': 'vggish/pool4/MaxPool',\n",
        "            'fc1': 'vggish/fc1/fc1_2/Relu',\n",
        "            'fc2': 'vggish/fc2/Relu',\n",
        "            'embedding': 'vggish/embedding',\n",
        "            'features': 'vggish/input_features',\n",
        "         }\n",
        "  g = tf.get_default_graph()\n",
        "  for k in layers:\n",
        "    layers[k] = g.get_tensor_by_name( layers[k] + ':0')\n",
        "    \n",
        "  return {'features': features_tensor,\n",
        "          'embedding': embedding_tensor,\n",
        "          'layers': layers,\n",
        "         }"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "euQcMmRCO0UU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def ProcessWithVGGish(vgg, x, sr):\n",
        "  '''Run the VGGish model, starting with a sound (x) at sample rate\n",
        "  (sr). Return a whitened version of the embeddings. Sound must be scaled to be\n",
        "  floats between -1 and +1.'''\n",
        "\n",
        "  # Produce a batch of log mel spectrogram examples.\n",
        "  input_batch = vggish_input.waveform_to_examples(x, sr)\n",
        "  # print('Log Mel Spectrogram example: ', input_batch[0])\n",
        "\n",
        "  [embedding_batch] = sess.run([vgg['embedding']],\n",
        "                               feed_dict={vgg['features']: input_batch})\n",
        "\n",
        "  # Postprocess the results to produce whitened quantized embeddings.\n",
        "  pca_params_path = 'vggish_pca_params.npz'\n",
        "\n",
        "  pproc = vggish_postprocess.Postprocessor(pca_params_path)\n",
        "  postprocessed_batch = pproc.postprocess(embedding_batch)\n",
        "  # print('Postprocessed VGGish embedding: ', postprocessed_batch[0])\n",
        "  return postprocessed_batch[0]\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Nv-F0U9nO0Yw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Test these new functions with the original test.\n",
        "\n",
        "tf.reset_default_graph()\n",
        "sess = tf.Session()\n",
        "\n",
        "vgg = CreateVGGishNetwork(0.01)\n",
        "\n",
        "# Generate a 1 kHz sine wave at 44.1 kHz (we use a high sampling rate\n",
        "# to test resampling to 16 kHz during feature extraction).\n",
        "num_secs = 3\n",
        "freq = 1000\n",
        "sr = 44100\n",
        "t = np.linspace(0, num_secs, int(num_secs * sr))\n",
        "x = np.sin(2 * np.pi * freq * t)  # Unit amplitude input signal\n",
        "\n",
        "postprocessed_batch = ProcessWithVGGish(vgg, x, sr)\n",
        "\n",
        "# print('Postprocessed VGGish embedding: ', postprocessed_batch[0])\n",
        "expected_postprocessed_mean = 123.0\n",
        "expected_postprocessed_std = 75.0\n",
        "np.testing.assert_allclose(\n",
        "    [np.mean(postprocessed_batch), np.std(postprocessed_batch)],\n",
        "    [expected_postprocessed_mean, expected_postprocessed_std],\n",
        "    rtol=rel_error)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Dh5r7Z7-O0dQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def EmbeddingsFromVGGish(vgg, x, sr):\n",
        "  '''Run the VGGish model, starting with a sound (x) at sample rate\n",
        "  (sr). Return a dictionary of embeddings from the different layers\n",
        "  of the model.'''\n",
        "  # Produce a batch of log mel spectrogram examples.\n",
        "  input_batch = vggish_input.waveform_to_examples(x, sr)\n",
        "  # print('Log Mel Spectrogram example: ', input_batch[0])\n",
        "\n",
        "  layer_names = vgg['layers'].keys()\n",
        "  tensors = [vgg['layers'][k] for k in layer_names]\n",
        "  \n",
        "  results = sess.run(tensors,\n",
        "                     feed_dict={vgg['features']: input_batch})\n",
        "\n",
        "  resdict = {}\n",
        "  for i, k in enumerate(layer_names):\n",
        "    resdict[k] = results[i]\n",
        "    \n",
        "  return resdict"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YTnw4WD_O0gw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "resdict = EmbeddingsFromVGGish(vgg, x, sr)\n",
        "\n",
        "for k in resdict:\n",
        "  print( k, resdict[k].shape)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c1moRMN6O0bs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# My small test\n",
        "\n",
        "import librosa, librosa.display\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "\n",
        "\n",
        "file = \"/content/drive/My Drive/test_dataset/not_sick/audioset_-21_SXelVNo_30_35.wav\"\n",
        "sr = 44100         # by default sample rate for analyzing audio data.\n",
        "duration = 4\n",
        "samples_per_track = sr * duration\n",
        "\n",
        "signal, sr = librosa.load(file, sr)\n",
        "print(signal.shape)\n",
        "print(signal.shape)\n",
        "\n",
        "resdict = EmbeddingsFromVGGish(vgg, signal, sr)\n",
        "\n",
        "for k in resdict:\n",
        "  print( k, resdict[k].shape)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yc8mBHfaO0XZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print(\"This is features shape: \",resdict[\"features\"].shape)\n",
        "print(\"This is embedding shape: \", resdict[\"embedding\"].shape)\n",
        "\n",
        "# If you want to make these embeddings more quantized so that it can run on edge then one can use vgg_postprocess.py file in order to make\n",
        "# these embeddings more quantized and it converts them into 8 bit integer.\n",
        "# or you can say it returns An nparray of the same shape as the input but of type uint8,\n",
        "# containing the PCA-transformed and quantized version of the input.\n",
        "\n",
        "# this postprocess file applies:\n",
        "    # Apply PCA.\n",
        "    # - Embeddings come in as [batch_size, embedding_size].\n",
        "    # - Transpose to [embedding_size, batch_size].\n",
        "    # - Subtract pca_means column vector from each column.\n",
        "    # - Premultiply by PCA matrix of shape [output_dims, input_dims]\n",
        "    #   where both are are equal to embedding_size in our case.\n",
        "    # - Transpose result back to [batch_size, embedding_size]."
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0Oca9FjsO0Sx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# For plotting embeddings\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "plt.imshow(resdict['embedding'],\n",
        "           aspect='auto', cmap='binary')\n",
        "plt.xlabel('Embedding Dimension')\n",
        "plt.ylabel('Time (frame number)')\n",
        "plt.title('Embedded Representation from a wav file')\n",
        "plt.grid(False);"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WTQPS21oQb53",
        "colab_type": "text"
      },
      "source": [
        "### **For extracting features from Audio dataset**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zCoXBr_MPvqx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import os\n",
        "import librosa, librosa.display\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import json\n",
        "\n",
        "dataset_path = \"/content/drive/My Drive/test_dataset\"\n",
        "json_path = \"embeddings2.json\"\n",
        "sr = 44100\n",
        "\n",
        "duration = 4\n",
        "samples_per_track = sr * duration\n",
        "\n",
        "\n",
        "data_store = {\n",
        "        \"embeddings\": [],\n",
        "        \"labels\": []\n",
        "    }\n",
        "\n",
        "for i, (dirpath, dirname, filenames) in enumerate(os.walk(dataset_path)):\n",
        "  if dirpath is not dataset_path:\n",
        "    for f in filenames:\n",
        "      file_path = os.path.join(dirpath, f)\n",
        "      data, sr = librosa.load(file_path, sr)\n",
        "      \n",
        "      resdict = EmbeddingsFromVGGish(vgg, data[:samples_per_track], sr)\n",
        "      print(resdict[\"embedding\"].shape)\n",
        "      data = resdict['embedding']\n",
        "      data_store[\"embeddings\"].append(data.tolist())\n",
        "      data_store[\"labels\"].append(i-1)\n",
        "  \n",
        "# final step: saving everything as a json file            \n",
        "with open(json_path, \"w\") as fp:\n",
        "    json.dump(data_store, fp, indent = 4)  # indent mean spaces while writing. fp mean file_path  "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h_dDsapfQSJU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Not necessary lines\n",
        "\n",
        "t = np.array(data_store[\"embeddings\"])\n",
        "print(t.shape)\n",
        "y = np.array(data_store[\"labels\"])\n",
        "j = y[..., np.newaxis]\n",
        "print(j[1000])\n",
        "print(j.shape)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SXXDeBSDUJ5L",
        "colab_type": "text"
      },
      "source": [
        "#### **Building a binary class model**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hCUNxI51QSMJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import json\n",
        "import numpy as np\n",
        "from sklearn.model_selection import train_test_split\n",
        "import tensorflow.keras as keras\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "DATASET_PATH = \"embeddings.json\"\n",
        "\n",
        "def load_data(dataset_path):\n",
        "    with open(dataset_path, \"r\") as fp:\n",
        "        data = json.load(fp)\n",
        "        \n",
        "        \n",
        "    # converting lists into numpy arrays\n",
        "    X = np.array(data[\"embeddings\"])\n",
        "    y = np.array(data[\"labels\"])\n",
        "    \n",
        "    return X, y\n",
        "\n",
        "def plot_history(history):\n",
        "    \n",
        "    fig, axis = plt.subplots(2)\n",
        "    \n",
        "    # create accuracy subplot\n",
        "    axis[0].plot(history.history[\"acc\"], label = \"train accuracy\" )\n",
        "    axis[0].plot(history.history[\"val_acc\"], label = \"test_accuracy\" )\n",
        "    axis[0].set_ylabel(\"Accuracy\")\n",
        "    axis[0].set_xlabel(\"Epochs\")\n",
        "    axis[0].legend(loc = \"lower right\")\n",
        "    axis[0].set_title(\"Acccuracy eval\")\n",
        "    \n",
        "    \n",
        "    # create error subplot\n",
        "    axis[1].plot(history.history[\"loss\"], label = \"train error\" )\n",
        "    axis[1].plot(history.history[\"val_loss\"], label = \"test error\" )\n",
        "    axis[1].set_ylabel(\"Error\")\n",
        "    axis[1].set_xlabel(\"Epochs\")\n",
        "    axis[1].legend(loc = \"upper right\")\n",
        "    axis[1].set_title(\"Error eval\")\n",
        "    \n",
        "    plt.show()\n",
        "\n",
        "def prepare_datasets(test_size, validation_size):\n",
        "    \n",
        "    # load dataset\n",
        "    X, y = load_data(DATASET_PATH)\n",
        "    \n",
        "    # create train/test split\n",
        "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = test_size, random_state=42)\n",
        "    \n",
        "    # create train/validataion split\n",
        "    X_train, X_validation, y_train, y_validation = train_test_split(X_train, y_train, test_size= validation_size, random_state=32)\n",
        "    \n",
        "    print(X_train.shape)\n",
        "    \n",
        "    # so in X_train we have 3d array --> (sample_size, mfcc_vector, mfcc_coefficient)\n",
        "    # so we have to introduce another dimension in it. and make it 4d array. \n",
        "    X_train = X_train[... , np.newaxis]\n",
        "    # ... 3dots mean keep the rest of the shape same.\n",
        "    X_validation = X_validation[..., np.newaxis]\n",
        "    X_test = X_test[..., np.newaxis]\n",
        "    \n",
        "    print(X_train.shape)\n",
        "    print(X_validation.shape)\n",
        "    print(X_test.shape)\n",
        "    \n",
        "    \n",
        "    return X_train, X_validation, X_test, y_train, y_validation, y_test\n",
        "    \n",
        "\n",
        "def build_model(input_shape):\n",
        "    \n",
        "    # create model\n",
        "    model = keras.Sequential()\n",
        "    \n",
        "    # 1st conv layer\n",
        "    model.add(keras.layers.Conv2D(32, (3, 3), padding='same', activation = \"relu\", input_shape = input_shape))\n",
        "    model.add(keras.layers.MaxPooling2D((3, 3), strides=(2, 2), padding='same'))\n",
        "#     model.add(keras.layers.MaxPooling2D(pool_size=(3, 3)))\n",
        "    model.add(keras.layers.BatchNormalization())\n",
        "    \n",
        "    # 2nd conv layer\n",
        "    model.add(keras.layers.Conv2D(32, (3, 3), padding='same', activation = \"relu\"))\n",
        "    model.add(keras.layers.MaxPooling2D((3, 3), strides=(2, 2), padding='same'))\n",
        "#     model.add(keras.layers.MaxPooling2D(pool_size=(3, 3)))\n",
        "    model.add(keras.layers.BatchNormalization())\n",
        "\n",
        "     # 3rd conv layer\n",
        "    model.add(keras.layers.Conv2D(32, (2, 2), padding='same', activation = \"relu\"))\n",
        "    model.add(keras.layers.MaxPooling2D((2, 2), strides=(2, 2), padding='same'))\n",
        "    model.add(keras.layers.BatchNormalization())\n",
        "\n",
        "    # Flatten the output and feed it into dense layer\n",
        "    model.add(keras.layers.Flatten())\n",
        "    \n",
        "    # 1st dense layer\n",
        "    model.add(keras.layers.Dense(128, activation = \"relu\", kernel_regularizer=keras.regularizers.l2(0.001)))\n",
        "    model.add(keras.layers.Dropout(0.8))\n",
        "\n",
        "    # 2nd dense layer\n",
        "    model.add(keras.layers.Dense(64, activation = \"relu\", kernel_regularizer=keras.regularizers.l2(0.001)))\n",
        "    model.add(keras.layers.Dropout(0.6))\n",
        "    \n",
        "    # output layer\n",
        "    model.add(keras.layers.Dense(1, activation = \"sigmoid\"))\n",
        "    \n",
        "    return model\n",
        "\n",
        "\n",
        "def predict(model, X, y):\n",
        "    \n",
        "    # add a dimension to input data for sample - model.predict() expects a 4d array in this case\n",
        "    X = X[np.newaxis, ...] # array shape (1, 130, 13, 1)\n",
        "    print(X.shape)\n",
        "\n",
        "    # perform prediction\n",
        "    prediction = model.predict(X)\n",
        "\n",
        "    # get index with max value\n",
        "    predicted_index = np.argmax(prediction, axis=1)\n",
        "\n",
        "    print(\"Target: {}, Predicted label: {}\".format(y, predicted_index))\n",
        "\n",
        "\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "\n",
        "    # Create train, validation and test set\n",
        "    X_train, X_validation, X_test, y_train, y_validation, y_test = prepare_datasets(0.25, 0.20)\n",
        "    \n",
        "    input_shape = (X_train.shape[1], X_train.shape[2], 1)\n",
        "    \n",
        "    # build the CNN network architecture\n",
        "    model = build_model(input_shape)\n",
        "    \n",
        "    # compile the network\n",
        "    optimizer = keras.optimizers.SGD(learning_rate = 0.01, momentum=0.9, decay=0.01, nesterov=False,)\n",
        "    model.compile(optimizer = optimizer,\n",
        "                  loss = keras.losses.BinaryCrossentropy(),\n",
        "                  metrics = [\"accuracy\"]\n",
        "                 )\n",
        "    \n",
        "    model.summary()\n",
        "    \n",
        "    # train the CNN\n",
        "    history = model.fit(X_train,\n",
        "              y_train,\n",
        "              validation_data = (X_validation, y_validation),\n",
        "              batch_size = 32,\n",
        "              epochs = 70\n",
        "             )\n",
        "    \n",
        "    # plot accuracy/error for training and validation\n",
        "    plot_history(history)\n",
        "    \n",
        "    # evaluate the CNN on the test set\n",
        "    test_error, test_accuracy = model.evaluate(X_test, y_test, verbose = 1)\n",
        "    print(\"Accuracy on test set is {}\".format(test_accuracy))\n",
        "    \n",
        "    # pick a sample to predict from the test set\n",
        "    X_to_predict = X_test[50]\n",
        "    y_to_predict = y_test[50]\n",
        "\n",
        "    # predict sample\n",
        "    predict(model, X_to_predict, y_to_predict)\n",
        "    "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ld6GcC7MQSQE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.metrics import confusion_matrix\n",
        "\n",
        "yhat_probs = model.predict(X_test, verbose=0)\n",
        "tt = np.argmax(yhat_probs, axis = 1)\n",
        "\n",
        "cm=confusion_matrix(y_test, tt)\n",
        "print(cm)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p2WRQDrWUcK3",
        "colab_type": "text"
      },
      "source": [
        "#### **Computing different Features**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3ac2R8-SPvs0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# For log mel spectogram\n",
        "\n",
        "from mel_features import log_mel_spectrogram\n",
        "\n",
        "path = \"/content/drive/My Drive/small_dataset/not_sick/Copy of audioset_EQzYcBJ1Dec_100_105.wav\"\n",
        "\n",
        "sr = 44100\n",
        "duration = 4\n",
        "samples_per_track = sr * duration\n",
        "\n",
        "signal, sr = librosa.load(path, sr)\n",
        "hh = log_mel_spectrogram(signal,\n",
        "                        audio_sample_rate=sr,\n",
        "                        log_offset=0.0,\n",
        "                        window_length_secs=0.025,\n",
        "                        hop_length_secs=0.010,)\n",
        "\n",
        "print(hh.shape)\n",
        "\n",
        "\n",
        "import librosa, librosa.display\n",
        "  \n",
        "librosa.display.specshow(hh, sr = sr, hop_length = 0.010)\n",
        "plt.xlabel(\"Time\")\n",
        "plt.ylabel(\"MFCC Coeffiecient\")\n",
        "plt.colorbar()\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FZ-DkT1LPvw9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# yeh tarka hai direct wav file vala\n",
        "# model input\n",
        "from vggish_input import wavfile_to_examples\n",
        "\n",
        "path = \"/content/drive/My Drive/small_dataset/not_sick/Copy of audioset_EQzYcBJ1Dec_100_105.wav\"\n",
        "\n",
        "jj = wavfile_to_examples(path)\n",
        "print(jj.shape)\n",
        "\n",
        "\n",
        "import librosa, librosa.display\n",
        "\n",
        "plt.imshow(jj[300])\n",
        "plt.xlabel(\"no of sample points\")\n",
        "plt.ylabel(\"Log mel spectogram\")\n",
        "plt.colorbar()\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YFFDR6CfPv1H",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# trying waveform to example method\n",
        "# yeh tarika hai librosa k through load krny ka\n",
        "# model input\n",
        "\n",
        "from vggish_input import waveform_to_examples\n",
        "\n",
        "path = \"/content/drive/My Drive/small_dataset/not_sick/Copy of audioset_EQzYcBJ1Dec_100_105.wav\"\n",
        "\n",
        "signal, sr = librosa.load(path, sr)\n",
        "\n",
        "kk = waveform_to_examples(signal, sr)\n",
        "print(kk.shape)\n",
        "print(signal.shape)\n",
        "\n",
        "import librosa, librosa.display\n",
        "\n",
        "plt.imshow(kk[300])\n",
        "plt.xlabel(\"no of sample points\")\n",
        "plt.ylabel(\"Log mel spectogram\")\n",
        "plt.colorbar()\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3cUrDEpiVPZV",
        "colab_type": "text"
      },
      "source": [
        "# **Now Working With keras Implementation of VGGish**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s6b2__gKPv4i",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!git clone https://github.com/furqan4545/VGGish-Keras.git\n",
        "!pip install sound"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iBnWRFH1Vbup",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Copy the source files to the current directory.\n",
        "!cp /content/VGGish-Keras/* ."
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VewAo2UzVkj6",
        "colab_type": "text"
      },
      "source": [
        "#### **Model 1 with MFCC coefficient**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ia_Za9TpVbwo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from preprocess_sound import preprocess_sound\n",
        "import numpy as np\n",
        "from scipy.io import wavfile\n",
        "\n",
        "from keras import backend as K\n",
        "from keras import optimizers\n",
        "from keras.layers import Flatten, Input, Dense, GlobalMaxPooling2D\n",
        "from keras.models import Model\n",
        "from keras.layers import GlobalAveragePooling2D\n",
        "from vggish import VGGish\n",
        "from preprocess_sound import preprocess_sound\n",
        "\n",
        "import json\n",
        "from sklearn.model_selection import train_test_split\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "DATASET_PATH = \"/content/drive/My Drive/sick_sound.json\"\n",
        "\n",
        "def load_data(dataset_path):\n",
        "    with open(dataset_path, \"r\") as fp:\n",
        "        data = json.load(fp)\n",
        "        \n",
        "        \n",
        "    # converting lists into numpy arrays\n",
        "    X = np.array(data[\"mfcc\"])\n",
        "    y = np.array(data[\"labels\"])\n",
        "    \n",
        "    return X, y\n",
        "\n",
        "def prepare_datasets(test_size, validation_size):\n",
        "    \n",
        "    # load dataset\n",
        "    X, y = load_data(DATASET_PATH)\n",
        "    \n",
        "    # create train/test split\n",
        "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = test_size, random_state=42)\n",
        "    \n",
        "    # create train/validataion split\n",
        "    X_train, X_validation, y_train, y_validation = train_test_split(X_train, y_train, test_size= validation_size, random_state=32)\n",
        "    \n",
        "    print(X_train.shape)\n",
        "    \n",
        "    # so in X_train we have 3d array --> (sample_size, mfcc_vector, mfcc_coefficient)\n",
        "    # so we have to introduce another dimension in it. and make it 4d array. \n",
        "    X_train = X_train[... , np.newaxis]\n",
        "    # ... 3dots mean keep the rest of the shape same.\n",
        "    X_validation = X_validation[..., np.newaxis]\n",
        "    X_test = X_test[..., np.newaxis]\n",
        "    \n",
        "    print(X_train.shape)\n",
        "    print(X_validation.shape)\n",
        "    print(X_test.shape)\n",
        "    \n",
        "    return X_train, X_validation, X_test, y_train, y_validation, y_test\n",
        "\n",
        "def plot_history(history):\n",
        "    \n",
        "    fig, axis = plt.subplots(2)\n",
        "    \n",
        "    # create accuracy subplot\n",
        "    axis[0].plot(history.history[\"accuracy\"], label = \"train accuracy\" )\n",
        "    axis[0].plot(history.history[\"val_accuracy\"], label = \"test_accuracy\" )\n",
        "    axis[0].set_ylabel(\"Accuracy\")\n",
        "    axis[0].set_xlabel(\"Epochs\")\n",
        "    axis[0].legend(loc = \"lower right\")\n",
        "    axis[0].set_title(\"Acccuracy eval\")\n",
        "    \n",
        "    \n",
        "    # create error subplot\n",
        "    axis[1].plot(history.history[\"loss\"], label = \"train error\" )\n",
        "    axis[1].plot(history.history[\"val_loss\"], label = \"test error\" )\n",
        "    axis[1].set_ylabel(\"Error\")\n",
        "    axis[1].set_xlabel(\"Epochs\")\n",
        "    axis[1].legend(loc = \"upper right\")\n",
        "    axis[1].set_title(\"Error eval\")\n",
        "    \n",
        "    plt.show()\n",
        "\n",
        "\n",
        "\n",
        "def predict(model, X, y):\n",
        "    \n",
        "    # add a dimension to input data for sample - model.predict() expects a 4d array in this case\n",
        "    X = X[np.newaxis, ...] # array shape (1, 130, 13, 1)\n",
        "    print(X.shape)\n",
        "\n",
        "    # perform prediction\n",
        "    prediction = model.predict(X)\n",
        "\n",
        "    # get index with max value\n",
        "    predicted_index = np.argmax(prediction, axis=1)\n",
        "\n",
        "    print(\"Target: {}, Predicted label: {}\".format(y, predicted_index))\n",
        "\n",
        "\n",
        "\n",
        "# Create train, validation and test set\n",
        "X_train, X_validation, X_test, y_train, y_validation, y_test = prepare_datasets(0.25, 0.20)\n",
        "\n",
        "input_shape = (X_train.shape[1], X_train.shape[2], 1)\n",
        "\n",
        "new_input = Input(shape=(108, 13, 1))\n",
        "sound_model = VGGish(include_top=False, load_weights=True, input_tensor = new_input)\n",
        "x = sound_model.get_layer(name=\"conv4/conv4_2\").output\n",
        "# output_layer = GlobalAveragePooling2D()(x)\n",
        "flat1 = GlobalAveragePooling2D()(x)\n",
        "# flat1 = Flatten()(x)\n",
        "class1 = Dense(1024, activation = 'relu')(flat1)\n",
        "outputss = Dense(2, activation = \"softmax\")(class1)\n",
        "\n",
        "# define new model\n",
        "model = Model(input = sound_model.input, output = outputss)\n",
        "model.summary()\n",
        "\n",
        "for layer in sound_model.layers:\n",
        "  layer.trainable = False\n",
        "\n",
        "# compile the network\n",
        "optimizer = keras.optimizers.Adam(learning_rate = 0.01, momentum=0.9, decay=0.01, nesterov=False,)\n",
        "model.compile(optimizer = optimizer,\n",
        "              loss = \"sparse_categorical_crossentropy\",\n",
        "              metrics = [\"accuracy\"]\n",
        "              )\n",
        "\n",
        "# train the CNN\n",
        "history = model.fit(X_train,\n",
        "          y_train,\n",
        "          validation_data = (X_validation, y_validation),\n",
        "          batch_size = 64,\n",
        "          epochs = 50\n",
        "          )\n",
        "\n",
        "# plot accuracy/error for training and validation\n",
        "plot_history(history)\n",
        "\n",
        "# evaluate the CNN on the test set\n",
        "test_error, test_accuracy = model.evaluate(X_test, y_test, verbose = 1)\n",
        "print(\"Accuracy on test set is {}\".format(test_accuracy))\n",
        "\n",
        "# pick a sample to predict from the test set\n",
        "X_to_predict = X_test[50]\n",
        "y_to_predict = y_test[50]\n",
        "\n",
        "# predict sample\n",
        "predict(model, X_to_predict, y_to_predict)\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CUfNTX5fWI_t",
        "colab_type": "text"
      },
      "source": [
        "#### **Model 2 with Embeddings**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1LuU0v-BVb2v",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from preprocess_sound import preprocess_sound\n",
        "import numpy as np\n",
        "\n",
        "from keras import backend as K\n",
        "from keras import optimizers\n",
        "from keras.layers import Flatten, Input, Dense, GlobalMaxPooling2D\n",
        "from keras.models import Model\n",
        "from keras.layers import GlobalAveragePooling2D\n",
        "from vggish import VGGish\n",
        "from preprocess_sound import preprocess_sound\n",
        "\n",
        "\n",
        "import json\n",
        "from sklearn.model_selection import train_test_split\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "DATASET_PATH = \"/content/embeddings2.json\"\n",
        "\n",
        "def load_data(dataset_path):\n",
        "    with open(dataset_path, \"r\") as fp:\n",
        "        data = json.load(fp)\n",
        "        \n",
        "        \n",
        "    # converting lists into numpy arrays\n",
        "    X = np.array(data[\"embeddings\"])\n",
        "    y = np.array(data[\"labels\"])\n",
        "    j = y[..., np.newaxis]\n",
        "\n",
        "    \n",
        "    return X, j\n",
        "\n",
        "def prepare_datasets(test_size, validation_size):\n",
        "    \n",
        "    # load dataset\n",
        "    X, y = load_data(DATASET_PATH)\n",
        "    \n",
        "    # create train/test split\n",
        "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = test_size, random_state=42)\n",
        "    \n",
        "    # create train/validataion split\n",
        "    X_train, X_validation, y_train, y_validation = train_test_split(X_train, y_train, test_size= validation_size, random_state=32)\n",
        "    \n",
        "    print(X_train.shape)\n",
        "    \n",
        "    # so in X_train we have 3d array --> (sample_size, mfcc_vector, mfcc_coefficient)\n",
        "    # so we have to introduce another dimension in it. and make it 4d array. \n",
        "    X_train = X_train[... , np.newaxis]\n",
        "    # ... 3dots mean keep the rest of the shape same.\n",
        "    X_validation = X_validation[..., np.newaxis]\n",
        "    X_test = X_test[..., np.newaxis]\n",
        "    \n",
        "    print(X_train.shape)\n",
        "    print(X_validation.shape)\n",
        "    print(X_test.shape)\n",
        "    \n",
        "    return X_train, X_validation, X_test, y_train, y_validation, y_test\n",
        "\n",
        "def plot_history(history):\n",
        "    \n",
        "    fig, axis = plt.subplots(2)\n",
        "    \n",
        "    # create accuracy subplot\n",
        "    axis[0].plot(history.history[\"accuracy\"], label = \"train accuracy\" )\n",
        "    axis[0].plot(history.history[\"val_accuracy\"], label = \"test_accuracy\" )\n",
        "    axis[0].set_ylabel(\"Accuracy\")\n",
        "    axis[0].set_xlabel(\"Epochs\")\n",
        "    axis[0].legend(loc = \"lower right\")\n",
        "    axis[0].set_title(\"Acccuracy eval\")\n",
        "    \n",
        "    \n",
        "    # create error subplot\n",
        "    axis[1].plot(history.history[\"loss\"], label = \"train error\" )\n",
        "    axis[1].plot(history.history[\"val_loss\"], label = \"test error\" )\n",
        "    axis[1].set_ylabel(\"Error\")\n",
        "    axis[1].set_xlabel(\"Epochs\")\n",
        "    axis[1].legend(loc = \"upper right\")\n",
        "    axis[1].set_title(\"Error eval\")\n",
        "    \n",
        "    plt.show()\n",
        "\n",
        "\n",
        "\n",
        "def predict(model, X, y):\n",
        "    \n",
        "    # add a dimension to input data for sample - model.predict() expects a 4d array in this case\n",
        "    X = X[np.newaxis, ...] # array shape (1, 130, 13, 1)\n",
        "    print(X.shape)\n",
        "\n",
        "    # perform prediction\n",
        "    prediction = model.predict(X)\n",
        "\n",
        "    # get index with max value\n",
        "    predicted_index = np.argmax(prediction, axis=1)\n",
        "\n",
        "    print(\"Target: {}, Predicted label: {}\".format(y, predicted_index))\n",
        "\n",
        "\n",
        "\n",
        "# Create train, validation and test set\n",
        "X_train, X_validation, X_test, y_train, y_validation, y_test = prepare_datasets(0.25, 0.20)\n",
        "\n",
        "input_shape = (X_train.shape[1], X_train.shape[2], 1)\n",
        "\n",
        "new_input = Input(shape=(303, 128, 1))\n",
        "sound_model = VGGish(include_top=False, load_weights=True, input_tensor = new_input)\n",
        "x = sound_model.get_layer(name=\"conv4/conv4_2\").output\n",
        "flat1 = GlobalMaxPooling2D()(x)\n",
        "# flat1 = GlobalAveragePooling2D()(x)\n",
        "# flat1 = Flatten()(x)\n",
        "class1 = Dense(1024, activation = 'relu')(flat1)\n",
        "class2 = Dense(512, activation = 'relu')(class1)\n",
        "class3 = Dense(256, activation = 'relu')(class2)\n",
        "outputss = Dense(1, activation = \"sigmoid\")(class3)\n",
        "\n",
        "# define new model\n",
        "model = Model(input = sound_model.input, output = outputss)\n",
        "model.summary()\n",
        "\n",
        "for layer in sound_model.layers:\n",
        "  layer.trainable = False\n",
        "\n",
        "# compile the network\n",
        "optimizer = optimizers.Adam(learning_rate=0.005)\n",
        "model.compile(optimizer = optimizer,\n",
        "              loss = keras.losses.BinaryCrossentropy(),\n",
        "              metrics = [\"accuracy\"]\n",
        "              )\n",
        "\n",
        "# train the CNN\n",
        "history = model.fit(X_train,\n",
        "          y_train,\n",
        "          validation_data = (X_validation, y_validation),\n",
        "          batch_size = 32,\n",
        "          epochs = 30\n",
        "          )\n",
        "\n",
        "# plot accuracy/error for training and validation\n",
        "plot_history(history)\n",
        "\n",
        "# evaluate the CNN on the test set\n",
        "test_error, test_accuracy = model.evaluate(X_test, y_test, verbose = 1)\n",
        "print(\"Accuracy on test set is {}\".format(test_accuracy))\n",
        "\n",
        "# pick a sample to predict from the test set\n",
        "X_to_predict = X_test[50]\n",
        "y_to_predict = y_test[50]\n",
        "\n",
        "# predict sample\n",
        "predict(model, X_to_predict, y_to_predict)\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ko11t52JWaLe",
        "colab_type": "text"
      },
      "source": [
        "#### **Model 3 using Softmax and callbacks**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HZoZuKKpVb5O",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from preprocess_sound import preprocess_sound\n",
        "import numpy as np\n",
        "from keras.callbacks import ReduceLROnPlateau, EarlyStopping\n",
        "\n",
        "from keras import backend as K\n",
        "from keras import optimizers\n",
        "from keras.regularizers import l2\n",
        "from keras.layers import Flatten, Input, Dense, GlobalMaxPooling2D\n",
        "from keras.models import Model\n",
        "from keras.layers import GlobalAveragePooling2D, BatchNormalization, Dropout\n",
        "from vggish import VGGish\n",
        "from preprocess_sound import preprocess_sound\n",
        "\n",
        "\n",
        "import json\n",
        "from sklearn.model_selection import train_test_split\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "\n",
        "DATASET_PATH = \"/content/embeddings2.json\"\n",
        "\n",
        "def load_data(dataset_path):\n",
        "    with open(dataset_path, \"r\") as fp:\n",
        "        data = json.load(fp)\n",
        "        \n",
        "        \n",
        "    # converting lists into numpy arrays\n",
        "    X = np.array(data[\"embeddings\"])\n",
        "    y = np.array(data[\"labels\"])\n",
        "    j = y[..., np.newaxis]\n",
        "    \n",
        "    return X, j\n",
        "\n",
        "def prepare_datasets(test_size, validation_size):\n",
        "    \n",
        "    # load dataset\n",
        "    X, y = load_data(DATASET_PATH)\n",
        "    \n",
        "    # create train/test split\n",
        "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = test_size, random_state=42)\n",
        "    \n",
        "    # create train/validataion split\n",
        "    X_train, X_validation, y_train, y_validation = train_test_split(X_train, y_train, test_size= validation_size, random_state=32)\n",
        "    \n",
        "    print(X_train.shape)\n",
        "    \n",
        "    # so in X_train we have 3d array --> (sample_size, mfcc_vector, mfcc_coefficient)\n",
        "    # so we have to introduce another dimension in it. and make it 4d array. \n",
        "    X_train = X_train[... , np.newaxis]\n",
        "    # ... 3dots mean keep the rest of the shape same.\n",
        "    X_validation = X_validation[..., np.newaxis]\n",
        "    X_test = X_test[..., np.newaxis]\n",
        "    \n",
        "    print(X_train.shape)\n",
        "    print(X_validation.shape)\n",
        "    print(X_test.shape)\n",
        "    \n",
        "    return X_train, X_validation, X_test, y_train, y_validation, y_test\n",
        "\n",
        "def plot_history(history):\n",
        "    \n",
        "    fig, axis = plt.subplots(2)\n",
        "    \n",
        "    # create accuracy subplot\n",
        "    axis[0].plot(history.history[\"accuracy\"], label = \"train accuracy\" )\n",
        "    axis[0].plot(history.history[\"val_accuracy\"], label = \"test_accuracy\" )\n",
        "    axis[0].set_ylabel(\"Accuracy\")\n",
        "    axis[0].set_xlabel(\"Epochs\")\n",
        "    axis[0].legend(loc = \"lower right\")\n",
        "    axis[0].set_title(\"Acccuracy eval\")\n",
        "    \n",
        "    \n",
        "    # create error subplot\n",
        "    axis[1].plot(history.history[\"loss\"], label = \"train error\" )\n",
        "    axis[1].plot(history.history[\"val_loss\"], label = \"test error\" )\n",
        "    axis[1].set_ylabel(\"Error\")\n",
        "    axis[1].set_xlabel(\"Epochs\")\n",
        "    axis[1].legend(loc = \"upper right\")\n",
        "    axis[1].set_title(\"Error eval\")\n",
        "    \n",
        "    plt.show()\n",
        "\n",
        "\n",
        "\n",
        "def predict(model, X, y):\n",
        "    \n",
        "    # add a dimension to input data for sample - model.predict() expects a 4d array in this case\n",
        "    X = X[np.newaxis, ...] # array shape (1, 130, 13, 1)\n",
        "    print(X.shape)\n",
        "\n",
        "    # perform prediction\n",
        "    prediction = model.predict(X)\n",
        "\n",
        "    # get index with max value\n",
        "    predicted_index = np.argmax(prediction, axis=1)\n",
        "\n",
        "    print(\"Target: {}, Predicted label: {}\".format(y, predicted_index))\n",
        "\n",
        "\n",
        "\n",
        "# Create train, validation and test set\n",
        "X_train, X_validation, X_test, y_train, y_validation, y_test = prepare_datasets(0.25, 0.20)\n",
        "\n",
        "input_shape = (X_train.shape[1], X_train.shape[2], 1)\n",
        "\n",
        "new_input = Input(shape=(303, 128, 1))\n",
        "sound_model = VGGish(include_top=False, load_weights=True, input_tensor = new_input)\n",
        "\n",
        "for layer in sound_model.layers:\n",
        "  layer.trainable = False\n",
        "\n",
        "x = sound_model.get_layer(name=\"conv4/conv4_2\").output\n",
        "flat1 = GlobalAveragePooling2D()(x)\n",
        "# flat1 = GlobalAveragePooling2D()(x)\n",
        "# flat1 = Flatten()(x)\n",
        "rt =  BatchNormalization()(flat1)\n",
        "class1 = Dense(1024, activation = 'relu', kernel_regularizer=l2(0.01))(rt)\n",
        "class1 = Dropout(0.5)(class1)\n",
        "class2 = Dense(512, activation = 'relu', kernel_regularizer=l2(0.01))(class1)\n",
        "class2 = Dropout(0.5)(class2)\n",
        "class2 = Dense(512, activation = 'relu', kernel_regularizer=l2(0.01))(class2)\n",
        "class2 = Dropout(0.5)(class2)\n",
        "class3 = Dense(256, activation = 'relu', kernel_regularizer=l2(0.01))(class2)\n",
        "outputss = Dense(2, activation = \"softmax\")(class3)\n",
        "\n",
        "# define new model\n",
        "model = Model(input = sound_model.input, output = outputss)\n",
        "model.summary()\n",
        "\n",
        "\n",
        "\n",
        "reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, mode = \"auto\",\n",
        "                              patience=5, min_delta=0.0001, cooldown=0, min_lr=0.001)\n",
        "\n",
        "early_stoping = EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=6, verbose=1,\n",
        "              mode='auto', baseline=None, restore_best_weights=True)\n",
        "\n",
        "# compile the network\n",
        "optimizer = optimizers.Adam(learning_rate=0.0001)\n",
        "model.compile(optimizer = optimizer,\n",
        "              loss = \"sparse_categorical_crossentropy\",\n",
        "              metrics = [\"accuracy\"]\n",
        "              )\n",
        "\n",
        "# train the CNN\n",
        "history = model.fit(X_train,\n",
        "          y_train,\n",
        "          validation_data = (X_validation, y_validation),\n",
        "          batch_size = 32,\n",
        "          epochs = 350,\n",
        "          callbacks = [reduce_lr, early_stoping]\n",
        "          )\n",
        "\n",
        "# plot accuracy/error for training and validation\n",
        "plot_history(history)\n",
        "\n",
        "# evaluate the CNN on the test set\n",
        "test_error, test_accuracy = model.evaluate(X_test, y_test, verbose = 1)\n",
        "print(\"Accuracy on test set is {}\".format(test_accuracy))\n",
        "\n",
        "# pick a sample to predict from the test set\n",
        "X_to_predict = X_test[50]\n",
        "y_to_predict = y_test[50]\n",
        "\n",
        "# predict sample\n",
        "predict(model, X_to_predict, y_to_predict)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YvXkkYUYVb02",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N4mSDe7-Pvzq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vxBdhbOyPvve",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xPu95bb0O0Io",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}