KerasSpeakerIdentification.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import librosa\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Activation, Dropout, Reshape, Flatten\n",
    "import keras\n",
    "import seaborn as sns\n",
    "import pydub\n",
    "import pickle\n",
    "import librosa.display\n",
    "import IPython.display\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.style as ms\n",
    "sys.path.append('data_prep/')\n",
    "import speech_data\n",
    "import segment_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# imports\n",
    "training = '/home/cc/threshold/'\n",
    "validation = '/home/cc/Data/speaker10Val/'\n",
    "num_speakers = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of files: 123\n",
      "(2, ' speakers: ', ['arun', 'andrew'])\n"
     ]
    }
   ],
   "source": [
    "# calculate the mfcc, delta, delta-delta matrices for training\n",
    "X = []\n",
    "Y = []\n",
    "speakers = speech_data.get_speakers(training)\n",
    "hop_length = 128 # default = 512\n",
    "for f in os.listdir(training):\n",
    "    Y.append(speech_data.one_hot_from_item(speech_data.speaker(f), speakers))\n",
    "    y, sr = librosa.load(training + f)\n",
    "    mfcc = librosa.feature.mfcc(y=y, sr=sr, n_mfcc=20, hop_length=hop_length)\n",
    "    delta = librosa.feature.delta(mfcc)\n",
    "    delta2 = librosa.feature.delta(mfcc, order=2)\n",
    "    X.append(np.c_[mfcc,delta,delta2])\n",
    "#pickle.dump(X, open('/home/cc/pickle_files/speaker251_train_2secX.p', 'wb'))\n",
    "#pickle.dump(Y, open('/home/cc/pickle_files/speaker251_train_2secY.p', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of files: 86\n",
      "(10, ' speakers: ', ['Speaker1', 'Speaker0', 'Speaker3', 'Speaker2', 'Speaker5', 'Speaker4', 'Speaker7', 'Speaker6', 'Speaker9', 'Speaker8'])\n"
     ]
    }
   ],
   "source": [
    "# calculate the mfcc, delta, delta-delta matrices for validation\n",
    "Xval = []\n",
    "Yval = []\n",
    "speakers = speech_data.get_speakers(validation)\n",
    "hop_length = 128 # default = 512\n",
    "for f in os.listdir(validation):\n",
    "    Yval.append(speech_data.one_hot_from_item(speech_data.speaker(f), speakers))\n",
    "    y, sr = librosa.load(validation + f)\n",
    "    mfcc = librosa.feature.mfcc(y=y, sr=sr, n_mfcc=20, hop_length=hop_length)\n",
    "    delta = librosa.feature.delta(mfcc)\n",
    "    delta2 = librosa.feature.delta(mfcc, order=2)\n",
    "    Xval.append(np.c_[mfcc,delta,delta2])\n",
    "#pickle.dump(Xval, open('/home/cc/pickle_files/speaker251_val_2secX.p', 'wb'))\n",
    "#pickle.dump(Yval, open('/home/cc/pickle_files/speaker251_val_2secY.p', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of files: 123\n",
      "(2, ' speakers: ', ['arun', 'andrew'])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(123, 20, 519)"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "speakers = speech_data.get_speakers(training)\n",
    "#X = pickle.load(open('/home/cc/pickle_files/speaker40_train_2secX.p', 'rb'))\n",
    "#Y = pickle.load(open('/home/cc/pickle_files/speaker40_train_2secY.p', 'rb'))\n",
    "#Xval = pickle.load(open('/home/cc/pickle_files/speaker40_val_2secX.p', 'rb'))\n",
    "#Yval = pickle.load(open('/home/cc/pickle_files/speaker40_val_2secY.p', 'rb'))\n",
    "x_train = np.array(X)\n",
    "y_train = np.array(Y)\n",
    "x_val = np.array(Xval)\n",
    "y_val = np.array(Yval)\n",
    "x_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# prepare the keras model - using Sequential for a linear stack of layers\n",
    "model = Sequential()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# import / add layers to the model\n",
    "# start with a dense layer - fully connected\n",
    "model.add(Dense(units=128, activation='tanh', input_shape=(20, 519,)))\n",
    "model.add(Dense(units=128, activation='tanh'))\n",
    "model.add(Dense(units=128, activation='tanh'))\n",
    "model.add(Dropout(rate=0.5))\n",
    "model.add(Flatten())\n",
    "model.add(Dense(units=2, activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# configure model's learning process\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "# create the callback functions\n",
    "early_stopping = keras.callbacks.EarlyStopping(monitor='val_loss', patience=0, verbose=1, mode='max')\n",
    "csv_logger = keras.callbacks.CSVLogger('training.log')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 110 samples, validate on 13 samples\n",
      "Epoch 1/10\n",
      "110/110 [==============================] - 0s - loss: 0.7680 - acc: 0.5545 - val_loss: 0.4232 - val_acc: 0.8462\n",
      "Epoch 2/10\n",
      "110/110 [==============================] - 0s - loss: 0.1975 - acc: 0.9636 - val_loss: 0.3738 - val_acc: 0.8462\n",
      "Epoch 3/10\n",
      "110/110 [==============================] - 0s - loss: 0.0893 - acc: 0.9818 - val_loss: 0.2898 - val_acc: 0.8462\n",
      "Epoch 4/10\n",
      "110/110 [==============================] - 0s - loss: 0.0438 - acc: 1.0000 - val_loss: 0.2220 - val_acc: 0.7692\n",
      "Epoch 5/10\n",
      "110/110 [==============================] - 0s - loss: 0.0226 - acc: 1.0000 - val_loss: 0.1764 - val_acc: 0.9231\n",
      "Epoch 6/10\n",
      "110/110 [==============================] - 0s - loss: 0.0102 - acc: 1.0000 - val_loss: 0.1589 - val_acc: 0.9231\n",
      "Epoch 7/10\n",
      "110/110 [==============================] - 0s - loss: 0.0089 - acc: 1.0000 - val_loss: 0.1482 - val_acc: 1.0000\n",
      "Epoch 8/10\n",
      "110/110 [==============================] - 0s - loss: 0.0086 - acc: 1.0000 - val_loss: 0.1405 - val_acc: 1.0000\n",
      "Epoch 9/10\n",
      "110/110 [==============================] - 0s - loss: 0.0097 - acc: 1.0000 - val_loss: 0.1312 - val_acc: 1.0000\n",
      "Epoch 10/10\n",
      "110/110 [==============================] - 0s - loss: 0.0050 - acc: 1.0000 - val_loss: 0.1205 - val_acc: 1.0000\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7fb78b8bec50>"
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# now fit to model\n",
    "#model.fit(x_train, y_train, epochs=10, validation_data=(x_val, y_val), callbacks=[csv_logger])\n",
    "model.fit(x_train, y_train, epochs=10, validation_split=0.1, callbacks=[csv_logger])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_163 (Dense)            (None, 20, 128)           66560     \n",
      "_________________________________________________________________\n",
      "dense_164 (Dense)            (None, 20, 128)           16512     \n",
      "_________________________________________________________________\n",
      "dense_165 (Dense)            (None, 20, 128)           16512     \n",
      "_________________________________________________________________\n",
      "dropout_34 (Dropout)         (None, 20, 128)           0         \n",
      "_________________________________________________________________\n",
      "flatten_33 (Flatten)         (None, 2560)              0         \n",
      "_________________________________________________________________\n",
      "dense_166 (Dense)            (None, 2)                 5122      \n",
      "=================================================================\n",
      "Total params: 104,706\n",
      "Trainable params: 104,706\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Inputs: (None, 20, 519)\n",
      "Outputs: (None, 2)\n",
      "Actual input: (123, 20, 519)\n",
      "Actual output: (123, 2)\n"
     ]
    }
   ],
   "source": [
    "model.summary()\n",
    "print \"Inputs: {}\".format(model.input_shape)\n",
    "print \"Outputs: {}\".format(model.output_shape)\n",
    "print \"Actual input: {}\".format(x_train.shape)\n",
    "print \"Actual output: {}\".format(y_train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of files: 15\n",
      "(2, ' speakers: ', ['arun', 'andrew'])\n"
     ]
    }
   ],
   "source": [
    "# now to evaluate the model\n",
    "testing = '/home/cc/Data/ociTest//'\n",
    "# calculate the mfcc matrices for testing\n",
    "Xtest = []\n",
    "Ytest = []\n",
    "speakers = speech_data.get_speakers(testing)\n",
    "hop_length = 128\n",
    "for f in os.listdir(testing):\n",
    "    Ytest.append(speech_data.one_hot_from_item(speech_data.speaker(f), speakers))\n",
    "    y, sr = librosa.load(testing + f)\n",
    "    mfcc = librosa.feature.mfcc(y=y, sr=sr, n_mfcc=20, hop_length=hop_length)\n",
    "    delta = librosa.feature.delta(mfcc)\n",
    "    delta2 = librosa.feature.delta(mfcc, order=2)\n",
    "    Xtest.append(np.c_[mfcc,delta,delta2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15/15 [==============================] - 0s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.17391367256641388, 0.86666667461395264]"
      ]
     },
     "execution_count": 158,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_test = np.array(Xtest)\n",
    "y_test = np.array(Ytest)\n",
    "model.evaluate(x_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['loss', 'acc']"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.metrics_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}