AudioTagger.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "AudioTagger",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/QColeman97/AudioTagger/blob/master/AudioTagger.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1r-IRKqSNCF8",
        "colab_type": "code",
        "outputId": "6f183737-5730-4bc0-99f3-4dd0734b4dec",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7D7P93WP3rs-",
        "colab_type": "code",
        "outputId": "425eb3e3-e708-450e-8dcf-845d7455fc85",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "!pip install sox"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: sox in /usr/local/lib/python3.6/dist-packages (1.3.7)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BntxWSSKakG2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np \n",
        "import pandas as pd \n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns # for data visualization \n",
        "\n",
        "import IPython\n",
        "import IPython.display as ipd #To play sound in notebook\n",
        "import scipy as sci\n",
        "import wave \n",
        "from pathlib import Path\n",
        "\n",
        "from scipy.fftpack import fft #Fast Fourier Transformation \n",
        "from scipy.io import wavfile \n",
        "\n",
        "import librosa "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WUp-xuxBUHie",
        "colab_type": "code",
        "outputId": "ae5ba669-765a-48e4-ec73-2b9923b74bdd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "import os \n",
        "print(os.listdir(\"drive/My Drive/CSC490Final-AudioTagger\"))"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['FSDKaggle2018.audio_test', 'FSDKaggle2018.audio_train', 'test_post_competition_scoring_clips.csv', 'train_post_competition.csv']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K7_TeQythJZs",
        "colab_type": "code",
        "outputId": "63141a9d-c540-499a-da8a-cb35baccbb0b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "INPUT_PATH =\"drive/My Drive/CSC490Final-AudioTagger/\"\n",
        "audio_train_file = (INPUT_PATH + \"FSDKaggle2018.audio_train\")\n",
        "audio_test_file = (INPUT_PATH + \"FSDKaggle2018.audio_test\")\n",
        "train= pd.read_csv(INPUT_PATH + \"train_post_competition.csv\")\n",
        "\n",
        "\n",
        "#scipy.wavfile.read returns rate of wave, and # of data read\n",
        "filename = '/001ca53d.wav'\n",
        "sample_rate, samples = wavfile.read(str(audio_train_file) + filename)\n",
        "print(samples)\n",
        "print(train.shape)"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[-33 -32 -34 ...  -1  -1  -1]\n",
            "(9473, 5)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BzbjcBaw7G5v",
        "colab_type": "code",
        "outputId": "682461d7-e6e0-4373-e58e-25894af0a407",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 153
        }
      },
      "source": [
        "print(train.head())"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "          fname         label  ...  freesound_id             license\n",
            "0  00044347.wav        Hi-hat  ...         28739         Attribution\n",
            "1  001ca53d.wav     Saxophone  ...        358827         Attribution\n",
            "2  002d256b.wav       Trumpet  ...         10897  Creative Commons 0\n",
            "3  0033e230.wav  Glockenspiel  ...        325017         Attribution\n",
            "4  00353774.wav         Cello  ...        195688         Attribution\n",
            "\n",
            "[5 rows x 5 columns]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8ZLKMxHB3Qqc",
        "colab_type": "text"
      },
      "source": [
        "# DATA PREPROCESSING \n",
        "Cut out silent parts \n",
        "\n",
        "Normalize wave form"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F5RnmkK0I8iy",
        "colab_type": "code",
        "outputId": "c2c15372-25de-420b-d892-46b4cff1264f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "df_train = pd.read_csv(INPUT_PATH +'/train_post_competition.csv')\n",
        "df_test = pd.read_csv(INPUT_PATH + '/test_post_competition_scoring_clips.csv')\n",
        "labels = df_train.label.unique()\n",
        "label2int = {label:index for index, label in enumerate(labels)}\n",
        "num_class = len(labels)\n",
        "#Indices of manually verified training data\n",
        "verifed_train = np.array(df_train[df_train.manually_verified == 1].index)\n",
        "#array of labels in number form (0 = hi-hat, 1 = saxophone, etc)\n",
        "plain_y_train = np.array([label2int[label] for label in df_train.label])\n",
        "\n",
        "#np.set_printoptions(threshold=np.inf)\n",
        "plain_y_train"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([ 0,  1,  2, ..., 12, 20, 17])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fv0TgNsTJBMp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "'''Two approaches:\n",
        "  1) LH uses highest feature with only beginning sound. Useful info\n",
        "     are usually in begging part of sample and support. \n",
        "     \n",
        "  2) Splits sample and uses one that are long but with coarser feature. \n",
        "     Good for samples with contents in the middle or later part. Perfect \n",
        "     for ones that use entire wav file  \n",
        "'''\n",
        "confLH, confX = {}, {}\n",
        "confs = [confLH, confX]\n",
        "confLH['folder'] = Path('LH')\n",
        "confX['folder'] = Path('X')\n",
        "\n",
        "#configs for confLH: highest resolutions\n",
        "\n",
        "confLH['sampling_rate'] = 44100\n",
        "confLH['duration'] = 4\n",
        "confLH['hop_length'] = 882 # 20ms\n",
        "confLH['fmin'] = 20\n",
        "confLH['fmax'] = confLH['sampling_rate'] // 2\n",
        "confLH['n_mels'] = 128\n",
        "confLH['n_fft'] = confLH['n_mels'] * 20\n",
        "confLH['audio_split'] = 'head'\n",
        "confLH['samples'] = confLH['sampling_rate'] * confLH['duration']\n",
        "confLH['dims'] = (confLH['n_mels'], 1 + \n",
        "                  int(np.floor(confLH['samples']/confLH['hop_length'])), 1)\n",
        "\n",
        "\n",
        "# Approach X uses longer sound, then it uses suppressed \n",
        "confX['sampling_rate'] = 26000\n",
        "confX['duration'] = 6\n",
        "confX['hop_length'] = 520 # 20ms\n",
        "confX['fmin'] = 20\n",
        "confX['fmax'] = confX['sampling_rate'] // 2\n",
        "confX['n_mels'] = 48\n",
        "confX['n_fft'] = confX['n_mels'] * 20\n",
        "confX['audio_split'] = 'dont_crop'\n",
        "confX['samples'] = confX['sampling_rate'] * confX['duration']\n",
        "confX['dims'] = (confX['n_mels'], 1 + \n",
        "                  int(np.floor(confX['samples']/confX['hop_length'])), 1)\n",
        "\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "txYQUju9Pk2O",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import librosa\n",
        "import librosa.display\n",
        "\n",
        "def read_audio(conf, pathname):\n",
        "    #return audio time series and sampling rate \n",
        "    y, sr = librosa.load(pathname, sr=conf['sampling_rate'])\n",
        "    # trim silence\n",
        "    if 0 < len(y):\n",
        "        y, _ = librosa.effects.trim(y) # trim, top_db=default(60)\n",
        "    # make it unified length to conf.samples\n",
        "    if len(y) > conf['samples']: # long enough\n",
        "        if conf['audio_split'] == 'head':\n",
        "            y = y[0:0+conf['samples']]\n",
        "    else: # pad blank\n",
        "        padding = conf['samples'] - len(y)    # add padding at both ends\n",
        "        offset = padding // 2\n",
        "        y = np.pad(y, (offset, conf['samples'] - len(y) - offset), 'constant')\n",
        "    return y\n",
        "\n",
        "def audio_to_melspectrogram(conf, audio):\n",
        "    spectrogram = librosa.feature.melspectrogram(audio, \n",
        "                                                 sr=conf['sampling_rate'],\n",
        "                                                 n_mels=conf['n_mels'],\n",
        "                                                 hop_length=conf['hop_length'],\n",
        "                                                 n_fft=conf['n_fft'],\n",
        "                                                 fmin=conf['fmin'],\n",
        "                                                 fmax=conf['fmax'])\n",
        "    #convert spectrogram to decibel\n",
        "    spectrogram = librosa.power_to_db(spectrogram)\n",
        "    spectrogram = spectrogram.astype(np.float32)\n",
        "    return spectrogram\n",
        "\n",
        "def show_melspectrogram(mels, conf):\n",
        "    librosa.display.specshow(mels, x_axis='time', y_axis='mel', \n",
        "                             sr=conf['sampling_rate'], hop_length=conf['hop_length'],\n",
        "                            fmin=conf['fmin'], fmax=conf['fmax'])\n",
        "    plt.colorbar(format='%+2.0f dB')\n",
        "    plt.title('Log-frequency power spectrogram')\n",
        "    plt.show()\n",
        "\n",
        "def read_as_melspectrogram(conf, pathname, debug_display=False):\n",
        "    x = read_audio(conf, pathname)\n",
        "    mels = audio_to_melspectrogram(conf, x)\n",
        "    if debug_display:\n",
        "        IPython.display.display(IPython.display.Audio(x, rate=conf['sampling_rate']))\n",
        "        show_melspectrogram(mels, conf)\n",
        "    return mels"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HRrxlCENVBRO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#spectograms are ndarray \n",
        "\n",
        "#LH method\n",
        "mels1 = read_as_melspectrogram(confLH, audio_train_file + '/' +\n",
        "                       df_train.fname[0], debug_display=False)\n",
        "mels_LH2 = read_as_melspectrogram(confLH, audio_train_file + '/' +\n",
        "                                  df_train.fname[100], debug_display=False)\n",
        "\n",
        "# X method \n",
        "mels2 = read_as_melspectrogram(confX, audio_train_file + '/' + \n",
        "                       df_train.fname[0], debug_display=False)\n",
        "mels_X2 = read_as_melspectrogram(confX, audio_train_file + '/' + \n",
        "                       df_train.fname[1], debug_display=False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GOZIHtjckMis",
        "colab_type": "code",
        "outputId": "263c7e21-ddfb-4823-964c-a86540c3add0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "print(mels1.size)\n",
        "print(mels_LH2.size)\n",
        "print(mels2.size)\n",
        "print(mels_X2.size)"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "25728\n",
            "25728\n",
            "27744\n",
            "24816\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5cPhP8mb3ZMr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def split_long_data(conf, X):\n",
        "    # Splits long mel-spectrogram data with small overlap\n",
        "    L = X.shape[1]\n",
        "    one_length = conf['dims'][1]\n",
        "    loop_length = int(one_length * 0.9)\n",
        "    min_length = int(one_length * 0.2)\n",
        "    print(' sample length', L, 'to cut every', one_length)\n",
        "    for idx in range(L // loop_length):\n",
        "        cur = loop_length * idx\n",
        "        rest = L - cur\n",
        "        if one_length <= rest:\n",
        "            yield X[:, cur:cur+one_length]\n",
        "        elif min_length <= rest:\n",
        "            cur = L - one_length\n",
        "            yield X[:, cur:cur+one_length]\n",
        "\n",
        "def convert_X(df, conf, datapath):\n",
        "    # Convert all files listed on df.fname\n",
        "    # Then generates X (contains mel-spectrograms)\n",
        "    # and index mapping to original sample order\n",
        "    X = []\n",
        "    index_map = []\n",
        "    for i, fname in enumerate(df.fname):\n",
        "        print('processing', fname)\n",
        "        data = read_as_melspectrogram(conf, datapath + '/' + fname)\n",
        "        for chunk in split_long_data(conf, data):\n",
        "            X.append(np.expand_dims(chunk, axis=-1))\n",
        "            index_map.append(i)\n",
        "    return np.array(X), np.array(index_map)\n",
        "\n",
        "def convert_y_train(idx_train, plain_y_train):\n",
        "    return np.array( [plain_y_train[idx] for idx in idx_train])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eWKSBRHYvqZX",
        "colab_type": "code",
        "outputId": "90dd2a66-0cd2-46c0-ce70-73204a080649",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 578
        }
      },
      "source": [
        "print('All samples will be cut per split length (=duration)')\n",
        "#X_train, idx_train = convert_X(df_train[:10], confX, audio_train_file)\n",
        "X_train, idx_train = convert_X(df_train, confX, audio_train_file)\n",
        "y_train = convert_y_train(idx_train, plain_y_train)\n",
        "print('Now original 10 samples were cut into ', len(idx_train), 'samples.')\n",
        "print()\n",
        "print('idx_train holds original sample index, y_train is also converted to have the same length with X_train/idx_train.')\n",
        "print('idx_train', idx_train)\n",
        "print('y_train', y_train)"
      ],
      "execution_count": 51,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "All samples will be cut per split length (=duration)\n",
            "processing 00044347.wav\n",
            " sample length 578 to cut every 301\n",
            "processing 001ca53d.wav\n",
            " sample length 517 to cut every 301\n",
            "processing 002d256b.wav\n",
            " sample length 301 to cut every 301\n",
            "processing 0033e230.wav\n",
            " sample length 301 to cut every 301\n",
            "processing 00353774.wav\n",
            " sample length 301 to cut every 301\n",
            "processing 003b91e8.wav\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-51-a50e8cb8f3ce>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'All samples will be cut per split length (=duration)'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;31m#X_train, idx_train = convert_X(df_train[:10], confX, audio_train_file)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0midx_train\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconvert_X\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconfX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maudio_train_file\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0my_train\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconvert_y_train\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0midx_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mplain_y_train\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Now original 10 samples were cut into '\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0midx_train\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'samples.'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-34-92505721b765>\u001b[0m in \u001b[0;36mconvert_X\u001b[0;34m(df, conf, datapath)\u001b[0m\n\u001b[1;32m     23\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfname\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'processing'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 25\u001b[0;31m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mread_as_melspectrogram\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdatapath\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m'/'\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mfname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     26\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mchunk\u001b[0m \u001b[0;32min\u001b[0m \u001b[0msplit_long_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     27\u001b[0m             \u001b[0mX\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexpand_dims\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-31-4da33ff0df89>\u001b[0m in \u001b[0;36mread_as_melspectrogram\u001b[0;34m(conf, pathname, debug_display)\u001b[0m\n\u001b[1;32m     40\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     41\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mread_as_melspectrogram\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpathname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdebug_display\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 42\u001b[0;31m     \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mread_audio\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpathname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     43\u001b[0m     \u001b[0mmels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0maudio_to_melspectrogram\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     44\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mdebug_display\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-31-4da33ff0df89>\u001b[0m in \u001b[0;36mread_audio\u001b[0;34m(conf, pathname)\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mread_audio\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpathname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0;31m#return audio time series and sampling rate\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m     \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlibrosa\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpathname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msr\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'sampling_rate'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m     \u001b[0;31m# trim silence\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;36m0\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/librosa/core/audio.py\u001b[0m in \u001b[0;36mload\u001b[0;34m(path, sr, mono, offset, duration, dtype, res_type)\u001b[0m\n\u001b[1;32m    165\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    166\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0msr\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 167\u001b[0;31m             \u001b[0my\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresample\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msr_native\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mres_type\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mres_type\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    168\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    169\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/librosa/core/audio.py\u001b[0m in \u001b[0;36mresample\u001b[0;34m(y, orig_sr, target_sr, res_type, fix, scale, **kwargs)\u001b[0m\n\u001b[1;32m    291\u001b[0m         \u001b[0my_hat\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mscipy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msignal\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresample\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_samples\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    292\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 293\u001b[0;31m         \u001b[0my_hat\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresampy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresample\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0morig_sr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_sr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfilter\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mres_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    294\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    295\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfix\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/resampy/core.py\u001b[0m in \u001b[0;36mresample\u001b[0;34m(x, sr_orig, sr_new, axis, filter, **kwargs)\u001b[0m\n\u001b[1;32m    111\u001b[0m     \u001b[0mx_2d\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mswapaxes\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    112\u001b[0m     \u001b[0my_2d\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mswapaxes\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 113\u001b[0;31m     \u001b[0mresample_f\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_2d\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_2d\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_ratio\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minterp_win\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minterp_delta\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprecision\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    114\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    115\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6zOO1tc23ikz",
        "colab_type": "text"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ixEy2dGsyERm",
        "colab_type": "text"
      },
      "source": [
        "## **Dataset for training**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FI84hzs_yDx_",
        "colab_type": "code",
        "outputId": "bc25ed7d-165e-44a2-cbd3-1cdbf6a1f01b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 231
        }
      },
      "source": [
        "def datapath(conf, filename):\n",
        "    return conf['folder'] / filename\n",
        "\n",
        "def loaddata(conf, filename):\n",
        "    return np.load(conf['folder'] / filename)\n",
        "\n",
        "#### This is Toy example by default ####\n",
        "TRYING_AS_TOY = True # False if you like creating full set\n",
        "\n",
        "for conf in confs:\n",
        "    conf['folder'].mkdir(parents=True, exist_ok=True)\n",
        "    if TRYING_AS_TOY:\n",
        "        for file in ['X_train', 'y_train', 'idx_train', 'X_test', 'idx_test']:\n",
        "            shutil.copy(EXTRA/datapath(conf, file+'.npy'), datapath(conf, file+'.npy'))\n",
        "        plain_y_train = np.load(EXTRA/'toy_plain_y_train.npy')\n",
        "        train_verified_idx = np.load(EXTRA/'toy_train_verified_idx.npy')\n",
        "        train_blacklist_index = np.load(EXTRA / 'toy_train_blacklist.npy')\n",
        "    else:\n",
        "        if not os.path.exists(datapath(conf, 'X_train.npy')):\n",
        "            X_train, idx_train = convert_X(df_train, conf, audio_train_file)\n",
        "            y_train = convert_y_train(idx_train, plain_y_train)\n",
        "            np.save(datapath(conf, 'X_train.npy'), X_train)\n",
        "            np.save(datapath(conf, 'y_train.npy'), y_train)\n",
        "            np.save(datapath(conf, 'idx_train.npy'), idx_train)\n",
        "\n",
        "            X_test, idx_test = convert_X(df_test, conf, audio_test_file)\n",
        "            np.save(datapath(conf, 'X_test.npy'), X_test)\n",
        "            np.save(datapath(conf, 'idx_test.npy'), idx_test)"
      ],
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-44-0522978656dd>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     12\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mTRYING_AS_TOY\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mfile\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m'X_train'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'y_train'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'idx_train'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'X_test'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'idx_test'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 14\u001b[0;31m             \u001b[0mshutil\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mEXTRA\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0mdatapath\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfile\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m'.npy'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdatapath\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfile\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m'.npy'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     15\u001b[0m         \u001b[0mplain_y_train\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mEXTRA\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0;34m'toy_plain_y_train.npy'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m         \u001b[0mtrain_verified_idx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mEXTRA\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0;34m'toy_train_verified_idx.npy'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'shutil' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Dxb43lu8yYx_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Mx-7pQbGXgD_",
        "colab_type": "text"
      },
      "source": [
        "# Models "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "z38JCiAvjwmI",
        "colab_type": "text"
      },
      "source": [
        "Generic 2D CNN model from lab3 made by Quinn"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3MKtggMZXigW",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "6d5aff47-8027-4f4b-a08c-2412403ae7ec"
      },
      "source": [
        "import keras\n",
        "from keras.layers import Dense, Conv2D, AveragePooling2D\n",
        "from keras.layers import SeparableConv1D, BatchNormalization, Flatten, Dropout, GlobalAveragePooling1D,MaxPooling1D\n",
        "from keras.models import Model, Sequential \n",
        "  \n",
        "from keras.callbacks import EarlyStopping, LearningRateScheduler, ModelCheckpoint, TensorBoard, ReduceLROnPlateau\n",
        "  \n",
        "def make_model(conf):\n",
        "    input_shape = conf['dims']\n",
        "    \n",
        "    nn = Sequential()\n",
        "    nn.add(layers.SeparableConv2D(64, (3, 3), padding = 'same',\n",
        "                                  activation = 'relu',\n",
        "                                  input_shape = input_shape))\n",
        "    # Shape: (126, 498, 64)\n",
        "    nn.add(layers.BatchNormalization())\n",
        "    nn.add(layers.SeparableConv2D(64, (3, 3), padding = 'same',\n",
        "                                  activation = 'relu'))\n",
        "    # Shape: (124, 496, 64)\n",
        "    nn.add(layers.BatchNormalization())\n",
        "    nn.add(layers.MaxPooling2D((2, 2)))\n",
        "    # Shape: (62, 248, 64)\n",
        "    nn.add(layers.Dropout(0.3))\n",
        "\n",
        "    nn.add(layers.SeparableConv2D(128, (3, 3), padding = 'same',\n",
        "                                  activation = 'relu'))\n",
        "    # Shape: (60, 246, 128)\n",
        "    nn.add(layers.BatchNormalization())\n",
        "    nn.add(layers.SeparableConv2D(128, (3, 3), padding = 'same',\n",
        "                                  activation = 'relu'))\n",
        "    # Shape: (58, 244, 128)\n",
        "    nn.add(layers.BatchNormalization())\n",
        "    nn.add(layers.MaxPooling2D((2, 2)))\n",
        "    # Shape: (29, 122, 128)\n",
        "    nn.add(layers.Dropout(0.3))\n",
        "\n",
        "    # Possibly make this block more like the others\n",
        "    nn.add(layers.SeparableConv2D(256, (3, 3), padding = 'same',\n",
        "                                  activation = 'relu'))\n",
        "    # Shape: (27, 120, 256)\n",
        "    nn.add(layers.BatchNormalization())\n",
        "    nn.add(layers.Dropout(0.3))\n",
        "    nn.add(layers.SeparableConv2D(256, (3, 3), padding = 'same',\n",
        "                                  activation = 'relu'))\n",
        "    # Shape: (25, 118, 256)\n",
        "    nn.add(layers.BatchNormalization())\n",
        "    nn.add(layers.Dropout(0.3))\n",
        "    nn.add(layers.SeparableConv2D(256, (3, 3), padding = 'same',\n",
        "                                  activation = 'relu'))\n",
        "    # Shape: (23, 116, 256)\n",
        "    nn.add(layers.BatchNormalization())\n",
        "    nn.add(layers.Dropout(0.3))\n",
        "    nn.add(layers.SeparableConv2D(256, (3, 3), padding = 'same',\n",
        "                                  activation = 'relu'))\n",
        "    # Shape: (21, 114, 256)\n",
        "    nn.add(layers.BatchNormalization())\n",
        "    nn.add(layers.MaxPooling2D((2, 2)))\n",
        "    # Shape: (10, 57, 256)\n",
        "    nn.add(layers.Dropout(0.3))\n",
        "\n",
        "    nn.add(layers.SeparableConv2D(512, (3, 3), padding = 'same',\n",
        "                                  activation = 'relu'))\n",
        "    # Shape: (8, 55, 512)\n",
        "    nn.add(layers.BatchNormalization())\n",
        "    nn.add(layers.SeparableConv2D(512, (3, 3), padding = 'same',\n",
        "                                  activation = 'relu'))\n",
        "    # Shape: (6, 53, 512)\n",
        "    nn.add(layers.BatchNormalization())\n",
        "    nn.add(layers.MaxPooling2D((2, 2)))\n",
        "    # Shape: (3, 26, 512)\n",
        "    nn.add(layers.Dropout(0.3))\n",
        "\n",
        "    nn.add(layers.SeparableConv2D(512, (3, 3), padding = 'same',\n",
        "                                  activation = 'relu'))\n",
        "    # Shape: (1, 24, 512)\n",
        "    nn.add(layers.BatchNormalization())\n",
        "    nn.add(layers.SeparableConv2D(512, (3, 3), padding = 'same',\n",
        "                                  activation = 'relu'))\n",
        "    nn.add(layers.BatchNormalization())\n",
        "    nn.add(layers.GlobalAveragePooling2D())\n",
        "\n",
        "    nn.add(layers.Dense(41, activation = 'softmax'))\n",
        "    \n",
        "    model.compile(optimizer= 'rmsprop', loss = 'categorical_crossentropy',\n",
        "                  metrics = ['accuracy'])\n",
        "    \n",
        "    model.summary()\n",
        "    \n",
        "    return nn\n",
        "  \n",
        "def train_model(conf, fold, model, train_steps_per_epoch, valid_steps_per_epoch,\n",
        "                init_best_weights= False, this_epochs = None):\n",
        "\n",
        "    callbacks = [ModelCheckpoint(str(datapath(conf, 'best_%d.h5' % fold)),\n",
        "                        monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=True),\n",
        "        TensorBoard(log_dir=str(datapath(conf, 'logs%s/fold_%d' % (conf['folder'], fold))), write_graph=True)\n",
        "    ]\n",
        "    \n",
        "    if model is None:\n",
        "        model = make_model(conf)\n",
        "        weight_filename = str(init_best_weights)\n",
        "        if weight_filename == 'True' : \n",
        "            weight_filename = str(datapath(conf, 'best_%d.h5' % fold))\n",
        "        if weight_filename is not 'False':\n",
        "            print('Initializing model with last best weights:', weight_filename)\n",
        "            model.load_weights(weight_filename)\n",
        "    #Training model\n",
        "    partial_x_train = X_train[:7105]\n",
        "    x_val = X_train[7105:]\n",
        "    \n",
        "    partial_y_train = y_train[:7105]\n",
        "    y_val = y_train[:7105]\n",
        "    \n",
        "    history = model.fit(partial_x_train, partial_y_train, callbacks = callbacks, \n",
        "                       steps_per_epoch = train_steps_per_epoch,\n",
        "                       epochs = 10, \n",
        "                       batch_size = 32,\n",
        "                       validation_data = (x_val, y_val),\n",
        "                       verbose = 1 )\n",
        "    \n",
        "    \n",
        "    return model, history \n",
        "\n",
        "\n",
        "def run_k_fold(train_data, train_labels,model):\n",
        "    k = 4\n",
        "    num_val = len(train_data) // k\n",
        "    num_train = len(train_labels) - num_val\n",
        "    all_val_acc_histories, all_val_loss_histories = [], []\n",
        "    for x in range(k):\n",
        "        print(\"Fold %d \" % k )\n",
        "        val_data = train_data[x * num_val: (x + 1) * num_val]\n",
        "        val_labels = train_labels[x * num_val: (x + 1) * num_val]\n",
        "\n",
        "        partial_train_data = np.concatenate(\n",
        "            [train_data[: x * num_val], train_data[(x + 1) * num_val:]],\n",
        "            axis = 0)\n",
        "        partial_train_labels = np.concatenate(\n",
        "            [train_labels[: x * num_val],\n",
        "             train_labels[(x + 1) * num_val:]],\n",
        "            axis = 0)\n",
        "\n",
        "    hst = model.fit(partial_train_data, partial_train_labels, batch_size = 2,\n",
        "                    epochs = 10, validation_data = (val_data, val_labels),)\n",
        "\n",
        "    hst = hst.history\n",
        "    all_val_loss_histories.append(hst['val_loss'])\n",
        "    all_val_acc_histories.append(hst['val_acc'])\n",
        "    \n",
        "           \n",
        "    return np.mean(all_val_loss_histories, axis=0), np.mean(all_val_acc_histories, axis=0), hst, model\n",
        "  \n",
        "    \n",
        "def samplewise_mean_X(X):\n",
        "    for i in range(len(X)):\n",
        "        X[i] -= np.mean(X[i], keepdims=True)\n",
        "        X[i] /= (np.std(X[i], keepdims=True) + 1.0)\n",
        "\n",
        "  \n",
        "''' \n",
        "def run_fold(conf, fold, dataset, model=None, init_best_weights=False, eval_only=False):\n",
        "    X_train, y_train, idx_train, all_X_train, all_y_train, all_idx_train, X_test, idx_test = dataset\n",
        "    print('----- Fold#%d ----' % fold)\n",
        "    # c. Cross validation split & balance # of samples\n",
        "    _Xtrain, _ytrain, _Xvalid, _yvalid = \\\n",
        "        get_cross_valid_fold_balanced(conf, fold, X_train, y_train, idx_train)\n",
        "\n",
        "    # d. Train model\n",
        "    train_generator, valid_generator, plain_datagen = \\\n",
        "        create_generators(conf, _Xtrain, _ytrain, _Xvalid, _yvalid)\n",
        "    train_steps_per_epoch, valid_steps_per_epoch = \\\n",
        "        get_steps_per_epoch(conf, _Xtrain, _Xvalid)\n",
        "    \n",
        "    \n",
        "    model, history = train_model(conf, fold, model,\n",
        "                                train_steps_per_epoch, valid_steps_per_epoch,\n",
        "                                 init_best_weights=init_best_weights,\n",
        "                                this_epochs=0 if eval_only else None)\n",
        "\n",
        "    # e. Evaluate with all train sample\n",
        "    model.load_weights(datapath(conf, 'best_%d.h5' % fold))\n",
        "    acc, acc_v = evaluate_fold(conf, fold, 'train_predictions_%d.npy', model, plain_datagen,\n",
        "                               all_X_train, all_idx_train, all_y_train, train_verified_idx)\n",
        "    evaluate_fold(conf, fold, 'test_predictions_%d.npy', model, plain_datagen, X_test, idx_test)\n",
        "\n",
        "    print('Trainset accuracy =', acc, '(tested all over the original training set)')\n",
        "    print('Verified samples accuracy =', acc_v, '(tested over manually verified samples only)')\n",
        "    return acc, acc_v, history, model, plain_datagen\n",
        "'''  \n",
        "   "
      ],
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\" \\ndef run_fold(conf, fold, dataset, model=None, init_best_weights=False, eval_only=False):\\n    X_train, y_train, idx_train, all_X_train, all_y_train, all_idx_train, X_test, idx_test = dataset\\n    print('----- Fold#%d ----' % fold)\\n    # c. Cross validation split & balance # of samples\\n    _Xtrain, _ytrain, _Xvalid, _yvalid =         get_cross_valid_fold_balanced(conf, fold, X_train, y_train, idx_train)\\n\\n    # d. Train model\\n    train_generator, valid_generator, plain_datagen =         create_generators(conf, _Xtrain, _ytrain, _Xvalid, _yvalid)\\n    train_steps_per_epoch, valid_steps_per_epoch =         get_steps_per_epoch(conf, _Xtrain, _Xvalid)\\n    \\n    \\n    model, history = train_model(conf, fold, model,\\n                                train_steps_per_epoch, valid_steps_per_epoch,\\n                                 init_best_weights=init_best_weights,\\n                                this_epochs=0 if eval_only else None)\\n\\n    # e. Evaluate with all train sample\\n    model.load_weights(datapath(conf, 'best_%d.h5' % fold))\\n    acc, acc_v = evaluate_fold(conf, fold, 'train_predictions_%d.npy', model, plain_datagen,\\n                               all_X_train, all_idx_train, all_y_train, train_verified_idx)\\n    evaluate_fold(conf, fold, 'test_predictions_%d.npy', model, plain_datagen, X_test, idx_test)\\n\\n    print('Trainset accuracy =', acc, '(tested all over the original training set)')\\n    print('Verified samples accuracy =', acc_v, '(tested over manually verified samples only)')\\n    return acc, acc_v, history, model, plain_datagen\\n\""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 45
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vLivCph1_G5N",
        "colab_type": "text"
      },
      "source": [
        "# Running Model "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g1RPrbQRkhAA",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        },
        "outputId": "d72c088c-7021-4338-d2d7-9601021a5705"
      },
      "source": [
        "for conf in [confX]: # Running confX only, change this to confs if you need running both confX and confLH\n",
        "    print('== Attempt [%s] ==' % conf['folder'])\n",
        "\n",
        "    # a. Load all dataset -> all_(X|y|idx)_train, (X|idx)_test\n",
        "    all_X_train, all_y_train, all_idx_train = \\\n",
        "        loaddata(conf, 'X_train.npy'), \\\n",
        "        keras.utils.to_categorical(loaddata(conf, 'y_train.npy')), \\\n",
        "        loaddata(conf, 'idx_train.npy')\n",
        "    X_test, idx_test = loaddata(conf, 'X_test.npy'), loaddata(conf, 'idx_test.npy')\n",
        "    print('Loaded trainset:%d, testset:%d samples.' % (len(all_X_train), len(X_test)))\n",
        "\n",
        "    # a'. Normalize samplewise if requested\n",
        "    \n",
        "    samplewise_mean_X(all_X_train)\n",
        "    samplewise_mean_X(X_test)\n",
        "\n",
        "    # b. Removing samples on the blacklist -> X|y|idx\n",
        "    '''\n",
        "    whitelist = [idx for idx in range(len(all_idx_train)) if all_idx_train[idx] not in train_blacklist_index]\n",
        "    X_train, y_train, idx_train = \\\n",
        "        all_X_train[whitelist], all_y_train[whitelist], all_idx_train[whitelist]\n",
        "    print('Filtered samples on blacklist, now trainset has %d samples' % len(idx_train))\n",
        "    '''\n",
        "    # Train folds\n",
        "    \n",
        "    work = {'train_acc': [],\n",
        "            'train_acc_verified': [],\n",
        "            'history': []}\n",
        "    for fold in range(5):\n",
        "        acc, acc_verified, history, model, _ = run_fold(conf, fold,\n",
        "                [X_train, y_train, idx_train, all_X_train, all_y_train, all_idx_train, X_test, idx_test],\n",
        "                model=None,\n",
        "                init_best_weights=EXTRA / 'X48_AlexNet_00696.h5',\n",
        "                eval_only=False)\n",
        "        work['history'].append(history)\n",
        "        work['train_acc'].append(acc)\n",
        "        work['train_acc_verified'].append(acc_verified)\n",
        "\n",
        "    print('___ training finished ___')\n"
      ],
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "== Attempt [X] ==\n",
            "Loaded trainset:13250, testset:337 samples.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-49-da40cde2dee3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     25\u001b[0m             'history': []}\n\u001b[1;32m     26\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mfold\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 27\u001b[0;31m         acc, acc_verified, history, model, _ = run_fold(conf, fold,\n\u001b[0m\u001b[1;32m     28\u001b[0m                 \u001b[0;34m[\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0midx_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mall_X_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mall_y_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mall_idx_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0midx_test\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     29\u001b[0m                 \u001b[0mmodel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'run_fold' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KxZM7oWj_ftc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}