SSInLOOP.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from mat4py import loadmat\n",
    "import scipy.io\n",
    "from copy import deepcopy\n",
    "import pandas as pd\n",
    "\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn import svm\n",
    "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n",
    "\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "import tensorflow \n",
    "from tensorflow import keras\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.utils import to_categorical\n",
    "\n",
    "import matplotlib.pyplot as plot\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import csv\n",
    "from tabnanny import verbose"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def My_Model(X,y,type_model,f=100,s=70,t=50,l=6):\n",
    "    # This function creates the classifier with different parameters\n",
    "    # INPUTS\n",
    "    # X -> Features matrix\n",
    "    # y -> label vector\n",
    "    # f -> number of neurons of first layer \n",
    "    # s -> number of neurons of second layer\n",
    "    # t -> number of neurons of third layer\n",
    "    # l -> number of neurons of fourth layer\n",
    "    # OUTPUTS\n",
    "    # trained model\n",
    "    # mean\n",
    "    # standard deviation\n",
    "    # minimum accuracy of K-Folds\n",
    "    # maximum accuracy of K-Folds \n",
    "    # accurcy per movement\n",
    "    # predictions vector\n",
    "    # true labels vector\n",
    "    # movements accuracies per fold\n",
    "    try:\n",
    "        # Selection of the model\n",
    "        if(type_model == 'MLP'):\n",
    "            ''' Multi Layer Perceptron '''\n",
    "            model = MLPClassifier(hidden_layer_sizes=(f,s,t,l),\n",
    "                            max_iter=1000,\n",
    "                            activation = 'tanh',\n",
    "                            solver='adam', \n",
    "                            #warm_start=True,\n",
    "                            shuffle=True,\n",
    "                            early_stopping=False)\n",
    "            \n",
    "        elif(type_model == 'SVML'):\n",
    "            ''' Support Vector Machine '''\n",
    "            model = svm.SVC(kernel='linear', C=1, decision_function_shape='ovo')\n",
    "        elif(type_model == 'SVMR'):\n",
    "            model = svm.SVC(kernel='rbf', C=1, decision_function_shape='ovo')\n",
    "            \n",
    "        elif(type_model == 'LDA'):\n",
    "            ''' Linear Discriminant Analysis '''\n",
    "            model = LinearDiscriminantAnalysis()\n",
    "            \n",
    "        elif(type_model == 'NN'):\n",
    "            ''' Neural Network '''\n",
    "            input_shape = (X.shape[1],)\n",
    "            kfold = StratifiedKFold(n_splits=5, shuffle=True) # 5-fold\n",
    "            loss_per_fold = []\n",
    "            accuracy_per_fold = []\n",
    "            n_classes = len(np.unique(y))\n",
    "            trials = len(y)/6\n",
    "            m1 = 0\n",
    "            m2 = 0\n",
    "            m3 = 0\n",
    "            m4 = 0\n",
    "            m5 = 0\n",
    "            m6 = 0\n",
    "            mov_acc = []\n",
    "            mov_acc_per_fold = []\n",
    "            y_pred_arr = []\n",
    "            y_true = []\n",
    "            \n",
    "            y_categorical = to_categorical(y-1, n_classes)\n",
    "            \n",
    "            for train_index, test_index in kfold.split(X, y):\n",
    "                X_train, X_test = X[train_index], X[test_index]\n",
    "                y_train, y_test = y_categorical[train_index], y_categorical[test_index] \n",
    "                len_per_fold = len(y_test)\n",
    "                \n",
    "                # NN Arquitecture\n",
    "                model = Sequential()\n",
    "                model.add(keras.layers.InputLayer(input_shape=input_shape))\n",
    "                model.add(Dense(f, activation='relu'))\n",
    "                model.add(Dense(s, activation='relu'))\n",
    "                model.add(Dense(t, activation='relu'))\n",
    "                model.add(Dense(l, activation='softmax'))\n",
    "                \n",
    "                # Compile model and train\n",
    "                model.compile(loss='mean_squared_error', optimizer='adam', metrics=['accuracy'])\n",
    "                history = model.fit(X_train, y_train, epochs=120, batch_size=30, verbose=0)\n",
    "                \n",
    "                ##########################################################\n",
    "                # Get classes predictions\n",
    "                for i in range(0,len(y_test)):\n",
    "                    y_hat = model.predict(X_test[i,:].reshape(1,-1),verbose=0)\n",
    "                    y_hat = np.argmax(y_hat, axis=1) + 1\n",
    "                    y_target = np.argmax(y_test[i], axis=0) + 1\n",
    "                    \n",
    "                    y_pred_arr.append(y_hat)\n",
    "                    y_true.append(y_target)\n",
    "                    \n",
    "                    if(y_hat == 1 and y_target == 1):\n",
    "                        m1 += 1\n",
    "                    elif(y_hat == 2 and y_target == 2):\n",
    "                        m2 += 1\n",
    "                    elif(y_hat == 3 and y_target == 3):\n",
    "                        m3 += 1\n",
    "                    elif(y_hat == 4 and y_target == 4):\n",
    "                        m4 += 1\n",
    "                    elif(y_hat == 5 and y_target == 5):\n",
    "                        m5 += 1\n",
    "                    elif(y_hat == 6 and y_target == 6):\n",
    "                        m6 += 1   \n",
    "                ##########################################################\n",
    "                \n",
    "                # Metrics\n",
    "                scores = model.evaluate(X_test, y_test,verbose=0)\n",
    "                loss_per_fold.append(scores[0])\n",
    "                accuracy_per_fold.append(scores[1])\n",
    "                # Obtain accuracy per movement per fold\n",
    "                mov_acc_per_fold.append(m1/len_per_fold*100),mov_acc_per_fold.append(m2/len_per_fold*100),\n",
    "                mov_acc_per_fold.append(m3/len_per_fold*100),mov_acc_per_fold.append(m4/len_per_fold*100),\n",
    "                mov_acc_per_fold.append(m5/len_per_fold*100),mov_acc_per_fold.append(m6/len_per_fold*100)\n",
    "                \n",
    "            # Obtain accuracy per movement\n",
    "            mov_acc.append(np.round(m1/trials*100,3)), mov_acc.append(np.round(m2/trials*100,3)), \n",
    "            mov_acc.append(np.round(m3/trials*100,3)), mov_acc.append(np.round(m4/trials*100,3)), \n",
    "            mov_acc.append(np.round(m5/trials*100,3)), mov_acc.append(np.round(m6/trials*100,3))\n",
    "            print(\"Classes Accuracy:\", mov_acc)\n",
    "            print('NN Average Accuracy: %.3f (%.3f)' % (np.mean(accuracy_per_fold), np.std(accuracy_per_fold)))\n",
    "            return model, np.mean(accuracy_per_fold), np.std(accuracy_per_fold), np.min(accuracy_per_fold), np.max(accuracy_per_fold), mov_acc, y_pred_arr, y_true, mov_acc_per_fold\n",
    "        \n",
    "        ######################################################################\n",
    "        if(type_model != 'NN'):\n",
    "            print('Starting K-Fold Cross Validation...')\n",
    "            # Create StratifiedKFold object.\n",
    "            k = StratifiedKFold(n_splits=5, shuffle=True, random_state=1)\n",
    "            lst_accu_stratified = []\n",
    "            model_kfold_cv = model\n",
    "            trials = len(y)/6\n",
    "            m1 = 0\n",
    "            m2 = 0\n",
    "            m3 = 0\n",
    "            m4 = 0\n",
    "            m5 = 0\n",
    "            m6 = 0\n",
    "            mov_acc = []\n",
    "            mov_acc_per_fold = []\n",
    "            y_pred_arr = []\n",
    "            y_true = []\n",
    "                \n",
    "            for train_index, test_index in k.split(X, y):\n",
    "                x_train_fold, x_test_fold = X[train_index], X[test_index]\n",
    "                y_train_fold, y_test_fold = y[train_index], y[test_index]\n",
    "                model_kfold_cv.fit(x_train_fold, y_train_fold)\n",
    "                len_per_fold = len(y_test_fold)\n",
    "                ##########################################################\n",
    "                # Get classes predictions\n",
    "                for i in range(0,len(y_test_fold)):\n",
    "                    y_hat = model_kfold_cv.predict(x_test_fold[i].reshape(1,-1))\n",
    "                    y_pred_arr.append(y_hat)\n",
    "                    y_true.append(y_test_fold[i])\n",
    "                    if(y_hat == y_test_fold[i] and y_hat == 1):\n",
    "                        m1 += 1\n",
    "                    elif(y_hat == y_test_fold[i] and y_hat == 2):\n",
    "                        m2 += 1\n",
    "                    elif(y_hat == y_test_fold[i] and y_hat == 3):\n",
    "                        m3 += 1\n",
    "                    elif(y_hat == y_test_fold[i] and y_hat == 4):\n",
    "                        m4 += 1\n",
    "                    elif(y_hat == y_test_fold[i] and y_hat == 5):\n",
    "                        m5 += 1\n",
    "                    elif(y_hat == y_test_fold[i] and y_hat == 6):\n",
    "                        m6 += 1       \n",
    "                ##########################################################\n",
    "                lst_accu_stratified.append(model_kfold_cv.score(x_test_fold, y_test_fold))\n",
    "                #######################################################################\n",
    "                # Obtain accuracy per movement per fold\n",
    "                mov_acc_per_fold.append(m1/len_per_fold*100),mov_acc_per_fold.append(m2/len_per_fold*100),\n",
    "                mov_acc_per_fold.append(m3/len_per_fold*100),mov_acc_per_fold.append(m4/len_per_fold*100),\n",
    "                mov_acc_per_fold.append(m5/len_per_fold*100),mov_acc_per_fold.append(m6/len_per_fold*100)\n",
    "                \n",
    "            # Obtain accuracy per movement\n",
    "            mov_acc.append(np.round(m1/trials*100,3)), mov_acc.append(np.round(m2/trials*100,3)), \n",
    "            mov_acc.append(np.round(m3/trials*100,3)), mov_acc.append(np.round(m4/trials*100,3)), \n",
    "            mov_acc.append(np.round(m5/trials*100,3)), mov_acc.append(np.round(m6/trials*100,3))\n",
    "            print(\"Classes Accuracy:\", mov_acc)\n",
    "            print(type_model, 'Mean Accuracy: %.3f (%.3f)' % (np.mean(lst_accu_stratified), np.std(lst_accu_stratified)))\n",
    "        \n",
    "        # Train Model\n",
    "        model.fit(X,y)\n",
    "        return model, np.mean(lst_accu_stratified), np.std(lst_accu_stratified), np.min(lst_accu_stratified), np.max(lst_accu_stratified), mov_acc, y_pred_arr, y_true, mov_acc_per_fold\n",
    "    except:\n",
    "        print(type_model, 'Not Found')\n",
    "        \n",
    "def My_model_predict(model, X_test,y_test,type_model,pca,scaling,prt=1):\n",
    "    # This function predicts by a given model and compares it against true label\n",
    "    # INPUT\n",
    "    # model -> trained model\n",
    "    # X_test -> testing data\n",
    "    # y_test -> testin data label\n",
    "    # type_model -> which model was trained?\n",
    "    # pca -> pca transformation object\n",
    "    # scaling -> scaling transformation object\n",
    "    # prt -> weather to print or not \n",
    "    # OUTPUTS\n",
    "    # accuracy\n",
    "    # predictions vector\n",
    "    predictions = []\n",
    "    # Make Predictions\n",
    "    if(prt ==1):\n",
    "        print(\"Starting Predictions...\")\n",
    "    test_acc = 0\n",
    "    for i in range(0,len(y_test)):\n",
    "        X = X_test[i,:]\n",
    "        #print(X.shape)\n",
    "        X= np.reshape(X, (1, -1))\n",
    "        #print(X.shape)\n",
    "        if X.shape[1] == 40:\n",
    "            X = pca.transform(X)\n",
    "            X = scaling.transform(X)\n",
    "        #print(X.shape)\n",
    "        \n",
    "        if type_model != 'NN':\n",
    "            y_hat = model.predict(X)\n",
    "            predictions.append(y_hat)\n",
    "            if y_hat == y_test[i]:\n",
    "                test_acc += 1\n",
    "        else:\n",
    "            y_hat = model.predict(X,verbose=0)\n",
    "            y_hat = np.argmax(y_hat, axis=1) + 1\n",
    "            predictions.append(y_hat[0])\n",
    "            if y_hat == y_test[i]:\n",
    "                 test_acc += 1\n",
    "                \n",
    "        \n",
    "    score = test_acc/len(y_test)*100\n",
    "    if(prt == 1):\n",
    "        print('Test Accuracy: %.3f' % score)\n",
    "    return score, predictions\n",
    "\n",
    "def plot_confusion_matrix(y_true, y_pred, classes,\n",
    "                          normalize=False,\n",
    "                          title=None,\n",
    "                          cmap=plot.cm.Blues):\n",
    "    # This function plots a confusion matrix of the classes by a given model\n",
    "    # INPUTS\n",
    "    # y_true -> correct labels vector\n",
    "    # y_pred -> predicted labels vector\n",
    "    # classes -> unique classes array\n",
    "    # normalize -> print in percentages or not\n",
    "    # title -> title of the plot\n",
    "    # cmap -> cmap of the plot\n",
    "    # OUTPUTS\n",
    "    # axis\n",
    "   \n",
    "    if not title:\n",
    "        if normalize:\n",
    "            title = 'Normalized confusion matrix'\n",
    "        \n",
    "    cm = confusion_matrix(y_true, y_pred)\n",
    "    \n",
    "    if normalize:\n",
    "        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
    "        #print(\"Normalized confusion matrix\")\n",
    "   \n",
    "    #print(cm)\n",
    "\n",
    "    figure, axis = plot.subplots()\n",
    "    im = axis.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "    axis.figure.colorbar(im, ax=axis)\n",
    "\n",
    "    axis.set(xticks=np.arange(cm.shape[1]),\n",
    "           yticks=np.arange(cm.shape[0]),\n",
    "\n",
    "           xticklabels=classes, yticklabels=classes,\n",
    "           title=title,\n",
    "           ylabel='True label',\n",
    "           xlabel='Predicted label')\n",
    "\n",
    "\n",
    "    plot.setp(axis.get_xticklabels(), rotation=45, ha=\"right\",\n",
    "             rotation_mode=\"anchor\")\n",
    "\n",
    "    fmt = '.2f' if normalize else 'd'\n",
    "    thresh = cm.max() / 2.\n",
    "    for i in range(cm.shape[0]):\n",
    "        for j in range(cm.shape[1]):\n",
    "            axis.text(j, i, format(cm[i, j], fmt),\n",
    "                    ha=\"center\", va=\"center\",\n",
    "                    color=\"cyan\" if cm[i, j] > thresh else \"red\")\n",
    "    figure.tight_layout()\n",
    "    return axis\n",
    "\n",
    "\n",
    "def PFvsWS(IL,model, X_test,y_test,type_model,pca,scaling,cm_flag=0):\n",
    "    # This function computes Partial Fitm, Warm Start and makes new predictions with the model\n",
    "    # NOTE: Not every model has partial fit or warm start computation\n",
    "    # INPUTS \n",
    "    # IL -> Partial Fitm or Warm Start\n",
    "    # model -> traine dmodel\n",
    "    # X_test -> feature testing data\n",
    "    # Y_test -> labels testing data\n",
    "    # type_model -> which model was trained\n",
    "    # pca -> pca transformation object\n",
    "    # model -> scaling transformation object\n",
    "    # cm -> print confusion matrix or not, if not, 5 runs of prediction will be made\n",
    "    # OUTPUTS\n",
    "    # percentage of the data used to fit the model\n",
    "    # accuracies\n",
    "    if(type_model != 'MLP'):\n",
    "        print(\"Cannot PF or WS Fit Model\")\n",
    "        return\n",
    "    \n",
    "    if(IL == 'PF'):\n",
    "        print('------- PARTIAL FIT MODEL -------')\n",
    "    else:\n",
    "        print('------- WARM START MODEL -------')\n",
    "        model.warm_start = True\n",
    "        first_layer = model.hidden_layer_sizes[0]\n",
    "        second_layer = model.hidden_layer_sizes[1]\n",
    "        third_layer = model.hidden_layer_sizes[2]\n",
    "        fourth_layer = model.hidden_layer_sizes[3]\n",
    "        model.hidden_layer_sizes = (first_layer,second_layer+30,third_layer+50,fourth_layer)\n",
    "        \n",
    "    X_new = []\n",
    "    for i in range(0,len(y_test)):\n",
    "        X = X_test[i,:]\n",
    "        X = np.reshape(X, (1, -1))\n",
    "        X = pca.transform(X)\n",
    "        X = scaling.transform(X)\n",
    "        X_new.append(X)\n",
    "        \n",
    "    X_new = np.array(X_new)  \n",
    "    X_new = np.reshape(X_new, (240,36))\n",
    "    y_new = y_test\n",
    "    \n",
    "    percent = [0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9]\n",
    "    test_accuracies = []\n",
    "\n",
    "    for i in range(len(percent)):\n",
    "        # Copy model to test\n",
    "        copy_model = deepcopy(model)\n",
    "        \n",
    "        x_train = []\n",
    "        x_test = []\n",
    "        y_train = []\n",
    "        y_test = []\n",
    "        samples = len(X_new)*percent[i]\n",
    "        idx = int(samples/6)\n",
    "        \n",
    "        #print(idx)\n",
    "        for j in range(len(np.unique(y_new))):\n",
    "            x_train.append(X_new[j*40:j*40+idx])\n",
    "            x_test.append(X_new[j*40+idx:j*40+40])\n",
    "            y_train.append(y_new[j*40:j*40+idx])  \n",
    "            y_test.append(y_new[j*40+idx:j*40+40])\n",
    "              \n",
    "        x_train = np.array(x_train)\n",
    "        x_test = np.array(x_test)\n",
    "        y_train = np.array(y_train)\n",
    "        y_test = np.array(y_test)\n",
    "        # reshape y to nx1\n",
    "        y_train = np.reshape(y_train, (len(y_train)*len(y_train[0]),1))\n",
    "        y_test = np.reshape(y_test, (len(y_test)*len(y_test[0]),1))\n",
    "        # reshape x to nx36\n",
    "        x_train = np.reshape(x_train, (len(x_train)*len(x_train[0]),36))\n",
    "        x_test = np.reshape(x_test, (len(x_test)*len(x_test[0]),36))\n",
    "        \n",
    "        y_train = y_train.ravel()\n",
    "        \n",
    "        if(IL == 'PF'):\n",
    "            copy_model.partial_fit(x_train, y_train)\n",
    "        elif(IL == 'WS'):\n",
    "            copy_model.fit(x_train, y_train)\n",
    "        else:\n",
    "            print('IL not found')\n",
    "            exit()\n",
    "        \n",
    "        mean_acc = []\n",
    "        \n",
    "        if(cm_flag == 0):\n",
    "            for m in range(0,5):\n",
    "                score, y_pred = My_model_predict(copy_model,x_test,y_test,type_model,pca,scaling,0)\n",
    "                mean_acc.append(score)\n",
    "            test_accuracies.append(np.mean(mean_acc))\n",
    "        else:\n",
    "            score, y_pred = My_model_predict(copy_model,x_test,y_test,type_model,pca,scaling,1)\n",
    "            test_accuracies.append(score)\n",
    "        \n",
    "            y_pred = np.array(y_pred)\n",
    "            y_pred = np.reshape(y_pred, (y_pred.shape[0],))\n",
    "            plot_confusion_matrix(y_test, \n",
    "                                y_pred, \n",
    "                                classes=np.array(['1','2','3','4','5','6']), \n",
    "                                normalize=True, \n",
    "                                title=str(percent[i]*100)+'% of Data')\n",
    "            plot.show()\n",
    "        \n",
    "    return np.array(percent), np.array(test_accuracies)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---------------------------------------------LOOP ONE------------------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "################### SSI ############################\n",
    "ssi_flag = 1\n",
    "write_csv = 1\n",
    "if (ssi_flag == 1):\n",
    "    subject = 16\n",
    "    if subject < 10:\n",
    "        s = 'S0' + str(subject)\n",
    "    else:   \n",
    "        s = 'S'+str(subject)\n",
    "    #classifiers = ['LDA','SVML','SVMR']\n",
    "    #classifiers = ['MLP1','MLP2','MLP3']\n",
    "    clf = 'NN3'\n",
    "    name_clf = 'NN3'\n",
    "\n",
    "    movs_per = []\n",
    "    train_path = r'SxxAll'\n",
    "    test_path = r'Sxx'\n",
    "    X_train_file = train_path + '\\\\' + s + 'x_train.mat'\n",
    "    y_train_file = train_path + '\\\\' + s + 'y_train.mat'\n",
    "    X_test_file = test_path + '\\\\' + s + 'X_1.mat'\n",
    "    y_test_file = test_path + '\\\\' + s + 'y.mat'\n",
    "    ''' Select a Model '''\n",
    "    name_clf = clf\n",
    "    # For MLP or NN\n",
    "    f = 1\n",
    "    s = 1\n",
    "    t = 1\n",
    "    l = 1\n",
    "    if(name_clf == 'MLP1' or name_clf == 'NN1'):\n",
    "        f=100\n",
    "        s=70\n",
    "        t=30\n",
    "        l=6\n",
    "        if(name_clf == 'MLP1'):\n",
    "            type_model = 'MLP'\n",
    "        else:\n",
    "            type_model = 'NN'\n",
    "    elif(name_clf == 'MLP2' or name_clf == 'NN2'):\n",
    "        f=100\n",
    "        s=100\n",
    "        t=100\n",
    "        l=6\n",
    "        if(name_clf == 'MLP2'):\n",
    "            type_model = 'MLP'\n",
    "        else:\n",
    "            type_model = 'NN'\n",
    "    elif(name_clf == 'MLP3' or name_clf == 'NN3'):\n",
    "        f=150\n",
    "        s=130\n",
    "        t=130\n",
    "        l=6\n",
    "        if(name_clf == 'MLP3'):\n",
    "            type_model = 'MLP'\n",
    "        else:\n",
    "            type_model = 'NN'\n",
    "    else:\n",
    "        type_model = name_clf\n",
    "        \n",
    "    X = X_test_file\n",
    "    y = y_test_file\n",
    "    ''' Read Training Subjects Data and Labels'''\n",
    "    X_subject = scipy.io.loadmat(X)\n",
    "    y_subject = scipy.io.loadmat(y)\n",
    "    X_subject = np.array(X_subject['X_1'])\n",
    "    y_subject = np.array(y_subject['y'])\n",
    "    X = X_subject\n",
    "    y = np.reshape(y_subject,(y_subject.shape[0],))\n",
    "    ''' Feature Selection with PCA '''\n",
    "    pca = PCA(n_components=36)\n",
    "    pca.fit(X)\n",
    "    X = pca.transform(X)\n",
    "    ''' Scaling '''\n",
    "    scaling = StandardScaler(with_mean=True, with_std=True)\n",
    "    scaling.fit(X)\n",
    "    X = scaling.transform(X)\n",
    "    ''' Train Model '''\n",
    "    print(\"------- SINGLE SUBJECT INDEPENDENT -------\")\n",
    "    print('Subject: ',subject)\n",
    "    print('Model:',type_model)\n",
    "    model, mean, std, min, max, mov_acc, y_pred, y_true, _ = My_Model(X=X, y=y, type_model=type_model,f=f,s=s,t=t,l=l)\n",
    "    \n",
    "    path = r'SSI_CM\\\\' + name_clf + '\\\\' + name_clf + '_CM.csv'\n",
    "    with open(path, 'a', newline='') as file:\n",
    "        writer = csv.writer(file)\n",
    "        for idx in range(len(y_true)):\n",
    "            writer.writerow([y_true[idx],y_pred[idx][0]])\n",
    "    \n",
    "    #### TABLE 1 ####\n",
    "    result = []\n",
    "    mean = float(np.round(mean*100,3))\n",
    "    std = float(np.round(std*100,3))\n",
    "    min = float(np.round(min*100,3))\n",
    "    max = float(np.round(max*100,3))\n",
    "    \n",
    "    result.append(mean), result.append(std), result.append(min), result.append(max)\n",
    "    print(result)\n",
    "    \n",
    "    if(write_csv == 1):\n",
    "        with open(r'SSI\\\\'+name_clf+'_SSI_results.csv','a', newline='') as ssi_results:\n",
    "            writer = csv.writer(ssi_results)\n",
    "            writer.writerow(result)\n",
    "    \n",
    "        #### TABLE 3 ####\n",
    "        movs_per.append(mov_acc)\n",
    "        with open(r'SSI\\\\'+name_clf+'_SSI_Class_results.csv','a', newline='') as class_results:\n",
    "            writer = csv.writer(class_results)\n",
    "            writer.writerow(mov_acc)\n",
    "            \n",
    "    if subject < 10:\n",
    "        s = 'S0' + str(subject)\n",
    "    else:   \n",
    "        s = 'S'+str(subject)\n",
    "    title = name_clf + ' ' + s + ' Confusion Matrix'\n",
    "    plot_confusion_matrix(y_true, \n",
    "                        y_pred, \n",
    "                        classes=np.array(['1','2','3','4','5','6']), \n",
    "                        normalize=True, \n",
    "                        title=title)\n",
    "    path = r'SSI_CM\\\\'\n",
    "    path = path + name_clf  + '\\\\' + name_clf \n",
    "    path = path + '_' + s + '_Confusion_Matrix.png'\n",
    "    plot.savefig(path)\n",
    "    plot.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---------------------------------------------LOOP ONE------------------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "################### LOOP ONE ############################\n",
    "subject = 16\n",
    "if subject < 10:\n",
    "    s = 'S0' + str(subject)\n",
    "else:   \n",
    "    s = 'S'+str(subject)\n",
    "train_path = r'C:\\Users\\jmfig\\Desktop\\HoH_Controller\\SxxAll'\n",
    "test_path = r'C:\\Users\\jmfig\\Desktop\\HoH_Controller\\Sxx'\n",
    "X_train_file = train_path + '\\\\' + s + 'x_train.mat'\n",
    "y_train_file = train_path + '\\\\' + s + 'y_train.mat'\n",
    "X_test_file = test_path + '\\\\' + s + 'X_1.mat'\n",
    "y_test_file = test_path + '\\\\' + s + 'y.mat'\n",
    "\n",
    "''' Select a Model '''\n",
    "name_clf = 'LDA'\n",
    "# For MLP or NN\n",
    "f = 1\n",
    "s = 1\n",
    "t = 1\n",
    "l = 1\n",
    "\n",
    "if(name_clf == 'MLP1' or name_clf == 'NN1'):\n",
    "    f=100\n",
    "    s=70\n",
    "    t=30\n",
    "    l=6\n",
    "    if(name_clf == 'MLP1'):\n",
    "        type_model = 'MLP'\n",
    "    else:\n",
    "        type_model = 'NN'\n",
    "elif(name_clf == 'MLP2' or name_clf == 'NN2'):\n",
    "    f=100\n",
    "    s=100\n",
    "    t=100\n",
    "    l=6\n",
    "    if(name_clf == 'MLP2'):\n",
    "        type_model = 'MLP'\n",
    "    else:\n",
    "        type_model = 'NN'\n",
    "elif(name_clf == 'MLP3' or name_clf == 'NN3'):\n",
    "    f=150\n",
    "    s=130\n",
    "    t=130\n",
    "    l=6\n",
    "    if(name_clf == 'MLP3'):\n",
    "        type_model = 'MLP'\n",
    "    else:\n",
    "        type_model = 'NN'\n",
    "else:\n",
    "    type_model = name_clf\n",
    "\n",
    "''' Read Training Subjects Data and Labels'''\n",
    "X_subject = scipy.io.loadmat(X_train_file)\n",
    "y_subject = scipy.io.loadmat(y_train_file)\n",
    "X_subject = np.array(X_subject['x_train'])\n",
    "y_subject = np.array(y_subject['y_train'])\n",
    "X = X_subject\n",
    "y = np.reshape(y_subject,(y_subject.shape[0],))\n",
    "\n",
    "''' Feature Selection with PCA '''\n",
    "pca = PCA(n_components=36)\n",
    "pca.fit(X)\n",
    "X = pca.transform(X)\n",
    "''' Scaling '''\n",
    "scaling = StandardScaler(with_mean=True, with_std=True)\n",
    "scaling.fit(X)\n",
    "X = scaling.transform(X)\n",
    "''' Train Model '''\n",
    "print(\"------- ALL VS ONE (LOOP) -------\")\n",
    "print('Model:',type_model)\n",
    "model, mean, std, min, max, mov_acc, mov_acc_per_fold = My_Model(X=X, y=y, type_model=type_model,f=f,s=s,t=t,l=l)\n",
    "\n",
    "# Read Test Subjects Data\n",
    "X_test = scipy.io.loadmat(X_test_file)\n",
    "y_test = scipy.io.loadmat(y_test_file)\n",
    "X_test = np.array(X_test['X_1'])\n",
    "y_test = np.array(y_test['y'])\n",
    "y_test = np.reshape(y_test, (y_test.shape[0],))\n",
    "\n",
    "''' Predict (LOOP) '''\n",
    "score, y_pred = My_model_predict(model, X_test, y_test, type_model, pca, scaling)\n",
    "\n",
    "''' Get Confusion Matrix '''\n",
    "y_pred = np.array(y_pred)\n",
    "y_pred = np.reshape(y_pred, (y_pred.shape[0],))\n",
    "if subject < 10:\n",
    "    s = 'S0' + str(subject)\n",
    "else:   \n",
    "    s = 'S'+str(subject)\n",
    "title = name_clf + ' ' + s + ' vs All Confusion Matrix'\n",
    "plot_confusion_matrix(y_test, \n",
    "                    y_pred, \n",
    "                    classes=np.array(['1','2','3','4','5','6']), \n",
    "                    normalize=True, \n",
    "                    title=title)\n",
    "plot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read Test Subjects Data\n",
    "X_test = scipy.io.loadmat(X_test_file)\n",
    "y_test = scipy.io.loadmat(y_test_file)\n",
    "X_test = np.array(X_test['X_1'])\n",
    "y_test = np.array(y_test['y'])\n",
    "y_test = np.reshape(y_test, (y_test.shape[0],))\n",
    "\n",
    "''' Predict (LOOP) '''\n",
    "score, y_pred = My_model_predict(model, X_test, y_test, type_model, pca, scaling)\n",
    "\n",
    "''' Get Confusion Matrix '''\n",
    "y_pred = np.array(y_pred)\n",
    "y_pred = np.reshape(y_pred, (y_pred.shape[0],))\n",
    "if subject < 10:\n",
    "    s = 'S0' + str(subject)\n",
    "else:   \n",
    "    s = 'S'+str(subject)\n",
    "title = name_clf + ' ' + s + ' vs All Confusion Matrix'\n",
    "plot_confusion_matrix(y_test, \n",
    "                      y_pred, \n",
    "                      classes=np.array(['1','2','3','4','5','6']), \n",
    "                      normalize=True, \n",
    "                      title=title)\n",
    "path = r'C:\\Users\\jmfig\\Desktop\\HoH_Controller\\Confusion_Matrix'\n",
    "path = path + '\\\\' + s + '_vs_All_Confusion_Matrix.png'\n",
    "#plot.savefig(path)\n",
    "plot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "################### PARTIAL FIT and WARM START TESTING ############################\n",
    "''' Select a Model '''\n",
    "type_model = 'NN2'\n",
    "\n",
    "subject = 16\n",
    "if subject < 10:\n",
    "    s = 'S0' + str(subject)\n",
    "else:   \n",
    "    s = 'S'+str(subject)\n",
    "train_path = r'C:\\Users\\jmfig\\Desktop\\HoH_Controller\\SxxAll'\n",
    "test_path = r'C:\\Users\\jmfig\\Desktop\\HoH_Controller\\Sxx'\n",
    "X_train_file = train_path + '\\\\' + s + 'x_train.mat'\n",
    "y_train_file = train_path + '\\\\' + s + 'y_train.mat'\n",
    "X_test_file = test_path + '\\\\' + s + 'X_1.mat'\n",
    "y_test_file = test_path + '\\\\' + s + 'y.mat'\n",
    "\n",
    "if(type_model == 'MLP1' or type_model == 'NN1'):\n",
    "    f=50\n",
    "    s=30\n",
    "    t=10\n",
    "    l=6\n",
    "    if(type_model == 'MLP1'):\n",
    "        type_model = 'MLP'\n",
    "    else:\n",
    "        type_model = 'NN'\n",
    "elif(type_model == 'MLP2' or type_model == 'NN2'):\n",
    "    f=70\n",
    "    s=50\n",
    "    t=30\n",
    "    l=6\n",
    "    if(type_model == 'MLP2'):\n",
    "        type_model = 'MLP'\n",
    "    else:\n",
    "        type_model = 'NN'\n",
    "elif(type_model == 'MLP3' or type_model == 'NN3'):\n",
    "    f=100\n",
    "    s=70\n",
    "    t=30\n",
    "    l=6\n",
    "    if(type_model == 'MLP3'):\n",
    "        type_model = 'MLP'\n",
    "    else:\n",
    "        type_model = 'NN'\n",
    "\n",
    "''' Read Training Subjects Data and Labels'''\n",
    "X_subject = scipy.io.loadmat(X_train_file)\n",
    "y_subject = scipy.io.loadmat(y_train_file)\n",
    "X_subject = np.array(X_subject['x_train'])\n",
    "y_subject = np.array(y_subject['y_train'])\n",
    "X = X_subject\n",
    "y = np.reshape(y_subject,(y_subject.shape[0],))\n",
    "\n",
    "''' Feature Selection with PCA '''\n",
    "pca = PCA(n_components=36)\n",
    "pca.fit(X)\n",
    "X = pca.transform(X)\n",
    "''' Scaling '''\n",
    "scaling = StandardScaler(with_mean=True, with_std=True)\n",
    "scaling.fit(X)\n",
    "X = scaling.transform(X)\n",
    "''' Train Model '''\n",
    "print(\"------- ALL VS ONE (LOOP) -------\")\n",
    "model, _, _, _, _, _,_ = My_Model(X=X, y=y, type_model=type_model,f=f,s=s,t=t,l=l)\n",
    "\n",
    "# Read Test Subjects Data\n",
    "X_test = scipy.io.loadmat(X_test_file)\n",
    "y_test = scipy.io.loadmat(y_test_file)\n",
    "X_test = np.array(X_test['X_1'])\n",
    "y_test = np.array(y_test['y'])\n",
    "y_test = np.reshape(y_test, (y_test.shape[0],))\n",
    "\n",
    "''' Predict (LOOP) '''\n",
    "print('Model:',type_model)\n",
    "get_cm_one_run = 1\n",
    "score, y_pred = My_model_predict(model, X_test, y_test, type_model, pca, scaling, get_cm_one_run)\n",
    "\n",
    "''' Get Confusion Matrix '''\n",
    "y_pred = np.array(y_pred)\n",
    "y_pred = np.reshape(y_pred, (y_pred.shape[0],))\n",
    "plot_confusion_matrix(y_test, \n",
    "                      y_pred, \n",
    "                      classes=np.array(['1','2','3','4','5','6']), \n",
    "                      normalize=True, \n",
    "                      title=title)\n",
    "plot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "################### PARTIAL FIT TESTING ############################\n",
    "get_cm_one_run = 0\n",
    "pf_model = deepcopy(model)\n",
    "percent, pft_acc = PFvsWS('PF',pf_model, X_test, y_test, type_model, pca, scaling, get_cm_one_run)\n",
    "score = np.array(score)\n",
    "pft_acc = np.insert(pft_acc,0,score, axis=0)\n",
    "percent = np.insert(percent, 0, 0.0)\n",
    "# Plot percent vs accuracy\n",
    "plot.plot(percent, pft_acc)\n",
    "plot.title('Partial Fit Accuracy')\n",
    "plot.xlabel('Percent of Data')\n",
    "plot.ylabel('Accuracy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "################### WARM START TESTING ############################\n",
    "get_cm_one_run = 0\n",
    "ws_model = deepcopy(model)\n",
    "percent, ws_acc = PFvsWS('WS',ws_model, X_test, y_test, type_model, pca, scaling, get_cm_one_run)\n",
    "score = np.array(score)\n",
    "ws_acc = np.insert(ws_acc,0,score, axis=0)\n",
    "percent = np.insert(percent, 0, 0.0)\n",
    "# Plot percent vs accuracy\n",
    "plot.plot(percent, ws_acc)\n",
    "plot.title('Warm Start Accuracy')\n",
    "plot.xlabel('Percent of Data')\n",
    "plot.ylabel('Accuracy')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.13 64-bit (microsoft store)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "d12aceb7c7d1d4d1694daf67d70670324795e01857da6164fa0591aebabf1ea2"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}