Code for 12,13,14.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bebf6c3e",
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import division \n",
    "from pytesmo.metrics import nrmsd\n",
    "\n",
    "from math import sqrt\n",
    "import numpy as np\n",
    "import random\n",
    "\n",
    "\n",
    "import pandas as pd\n",
    "import time\n",
    "import impyute as impy\n",
    "import math\n",
    "\n",
    "from collections import defaultdict\n",
    "from chefboost import Chefboost as chef\n",
    "from openpyxl import load_workbook\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "42056c1c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def DMI(complete_data , df_incomplete_data):      \n",
    "    #df_incomplete_data=pd.DataFrame(data=X_incomplete, columns=X_incomplete.columns, index=X_incomplete.index)\n",
    "    \n",
    "    ######################################Step-1##############################################\n",
    "    \n",
    "    #Split DF into DI and DC subsets\n",
    "    \n",
    "    DI = df_incomplete_data[df_incomplete_data.isnull().any(axis=1)] #Xmis is the incomplete subset of X\n",
    "    index_list_DI = DI.index\n",
    "    DC = df_incomplete_data.dropna(inplace=False) #Xobs is the complete subset of X\n",
    "    L = 0\n",
    "    if len(DC) < 10:\n",
    "        print(\"1\")\n",
    "        randomNumbers = []\n",
    "        for i in range(1,11):\n",
    "            temp = random.randint(1,len(complete_data)-1)\n",
    "            if temp not in randomNumbers:\n",
    "                randomNumbers.append(temp)\n",
    "        #print(randomNumbers)\n",
    "        #df.iloc[[2, 3, 4]]\n",
    "        df_incomplete_data.iloc[randomNumbers] = complete_data.iloc[randomNumbers]\n",
    "        DI = df_incomplete_data[df_incomplete_data.isnull().any(axis=1)] #Xmis is the incomplete subset of X\n",
    "        index_list_DI = DI.index\n",
    "        DC = df_incomplete_data.dropna(inplace=False)\n",
    "    \n",
    "    \n",
    "       \n",
    "        \n",
    "\n",
    "    \n",
    "    ######################################Step-2##############################################\n",
    "    \n",
    "    #total no. of attributes in DI having missing values\n",
    "    M = DI.columns.get_indexer(DI.columns[DI.isnull().any()].tolist())\n",
    "    \n",
    "    all_leaves = []\n",
    "    L = 0\n",
    "    Ij = {}\n",
    "    ranges = []\n",
    "\n",
    "    \n",
    "    for Ai in range(len(M)):\n",
    "        DC_CPY = DC.copy()\n",
    "        index_list = DC_CPY.index\n",
    "        min_value = min(DC_CPY[DC_CPY.columns[M[Ai]]])\n",
    "        max_value = max(DC_CPY[DC_CPY.columns[M[Ai]]])\n",
    "        #print(min_value, max_value)\n",
    "        domain_size = max_value - min_value;\n",
    "        #NC = int(math.sqrt(domain_size))\n",
    "        NC = 1\n",
    "       # print(NC)\n",
    "        #max_range = max_value\n",
    "        #min_range = max_value*NC\n",
    "        #list_ranges = []\n",
    "       # l = [_ for _ in range(1,NC+1)]\n",
    "       # print(l)\n",
    "        #list_ranges.append([min_range,max_range])\n",
    "        #print(\"2*\")\n",
    "        #DC_CPY['Decision'] = ['' for _ in range(len(DC_CPY))]\n",
    "        #DC_CPY['Decision'] = str(pd.cut(DC.iloc[:,M[Ai]],  NC, right = False))\n",
    "        \n",
    "        min_range = min_value\n",
    "        max_range = min_value + (domain_size / NC)\n",
    "        ranges.append([min_range,max_range])\n",
    "        while NC - 1 != 0:\n",
    "            min_range = max_range\n",
    "            max_range = min_range + domain_size / NC\n",
    "            ranges.append([min_range,max_range])\n",
    "            NC -= 1\n",
    "            \n",
    "        #print(ranges)\n",
    "        \n",
    "   \n",
    "        '''while min_range <= max_range:\n",
    "            #print(\"3*\")\n",
    "            \n",
    "            max_range = min_range    \n",
    "            min_range = max_range * NC \n",
    "            list_ranges.append([min_range,max_range])\n",
    "            \n",
    "        list_ranges = list_ranges[::-1]\n",
    "        #print(\"List Ranges\",list_ranges) '''\n",
    "         \n",
    "        DC_CPY['Decision'] = ['' for _ in range(len(DC_CPY))]\n",
    "        for row in range(len(DC_CPY)):      \n",
    "            for rg in ranges:\n",
    "                if  rg[0] <= DC_CPY.iloc[row, M[Ai]] <= rg[1]:\n",
    "                    DC_CPY.iloc[row, len(DC_CPY.columns)-1] = str(rg)\n",
    "                    #print(DC_CPY.iloc[row, len(DC_CPY.columns)-1])\n",
    "                    break\n",
    "        #print(DC_CPY.head())\n",
    "       # DC_CPY['Decision'] = DC_CPY.columns[M[Ai]]\n",
    "        DC_CPY.drop(DC_CPY.columns[M[Ai]], axis=1, inplace=True)\n",
    "  \n",
    "        \n",
    "        DC_CPY_Temp = DC_CPY.copy()\n",
    "        \n",
    "        config = {'algorithm': 'C4.5'} #ID3, C4.5, CART, CHAID or Regression\n",
    "        model = chef.fit(DC_CPY, config)\n",
    "        chef.save_model(model, \"model.pkl\")\n",
    "\n",
    "        \n",
    "        modelnew = chef.load_model(\"model.pkl\")\n",
    "        leaves_one_missing_col = defaultdict(list)\n",
    "          \n",
    "        #print(\"4*\")\n",
    "        for row in range(len(DC_CPY_Temp)):\n",
    "            prediction = chef.predict(modelnew,DC_CPY_Temp.iloc[row,:-1].tolist())\n",
    "        \n",
    "            leaves_one_missing_col[prediction].append(index_list[row])\n",
    "            Ij[prediction] = False\n",
    "            \n",
    "    ######################################Step-3##############################################\n",
    "    \n",
    "        DI_temp = DI.drop(DI.columns[M[Ai]], axis=1, inplace=False)\n",
    "        \n",
    "        for row in range(len(DI)):\n",
    "            if pd.isnull(DI.iloc[row, M[Ai]]):\n",
    "                prediction = chef.predict(modelnew,DI_temp.iloc[row].tolist())\n",
    "                leaves_one_missing_col[prediction].append(index_list_DI[row])\n",
    "            \n",
    "        si = len(leaves_one_missing_col.keys())\n",
    "        \n",
    "        L = L + si\n",
    "                \n",
    "        all_leaves.append(leaves_one_missing_col)\n",
    "    print(\"Length of Leaves\",len(all_leaves))\n",
    "\n",
    "    ######################################Step-4##############################################\n",
    "    total = 0\n",
    "    count1 = 0\n",
    "    #print(\"all leaves\", len(all_leaves))\n",
    "    #print()\n",
    "    for tree_leaves in all_leaves:\n",
    "        for key in tree_leaves:\n",
    "            dj_indices = tree_leaves[key]\n",
    "            #print(len(dj_indices))\n",
    "            dj = df_incomplete_data.loc[dj_indices, :]\n",
    "            dj.sort_index(inplace=True)\n",
    "            if dj.isnull().sum().sum() != 0:\n",
    "                #print(dj.head())\n",
    "                dj_filled_EMI,itr = impy.em(dj.values)\n",
    "                #print(dj_filled_EMI)\n",
    "                df_dj_after_EMI=pd.DataFrame(data = dj_filled_EMI, columns=df_incomplete_data.columns, index=dj.index)  \n",
    "                #print(df_dj_after_EMI.head())\n",
    "                \n",
    "                df_incomplete_data = pd.concat([df_incomplete_data , df_dj_after_EMI])\n",
    "                \n",
    "                df_incomplete_data = df_incomplete_data.loc[~df_incomplete_data.index.duplicated(keep='last')]\n",
    "                \n",
    "                df_incomplete_data.sort_index(inplace=True)\n",
    "                total += itr\n",
    "            count1 += 1\n",
    "            \n",
    "           \n",
    "    #print(df_incomplete_data)  \n",
    "    #print(count1, total, delta)       \n",
    "    return df_incomplete_data, count1, total, NC;\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "##########################################################################################\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "53209618",
   "metadata": {},
   "outputs": [],
   "source": [
    "def nrms(all_dataset_imputed,df_complete_data):\n",
    "    numvar= len(df_complete_data.columns)\n",
    "    #sum_of_RMSE = 0\n",
    "    sum_of_NRMSE = 0\n",
    "    #sum_of_AE = 0\n",
    "    for i in range(0,numvar):\n",
    "        \n",
    "        l1_after_imputation = all_dataset_imputed.iloc[:,i].values     \n",
    "        l2_complete_data = df_complete_data.iloc[:,i].values     \n",
    "        nrms = nrmsd(l2_complete_data, l1_after_imputation) #Normalized root-mean-square deviation (nRMSD)\n",
    "        sum_of_NRMSE = sum_of_NRMSE + nrms\n",
    "        \n",
    "    average_NRMSE_All_Cols = sum_of_NRMSE / numvar\n",
    "    print(average_NRMSE_All_Cols)\n",
    "    return average_NRMSE_All_Cols\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "65c3203e",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.decomposition import PCA\n",
    "def fileName(indexFolder,cFilePath, fileName,count,col):\n",
    "    summaryName = 'Table-NRMS1.xlsx'\n",
    "    wb = load_workbook(summaryName)\n",
    "    ws1 = wb.active\n",
    "    percentageList = [10,20]\n",
    "    for j  in percentageList:\n",
    "        completeFileName = 'Data_' + str(14) + '.csv'\n",
    "        cFilePath = pd.read_csv(\"C:\\\\Users\\\\patel2k4\\\\Downloads\\\\\"+completeFileName)\n",
    "        col = cFilePath.columns\n",
    "        cFilePath.drop(columns = ['Labels'], axis = 1,  inplace = True)\n",
    "        df_complete_data=pd.DataFrame(data=cFilePath, columns=cFilePath.columns, index=cFilePath.index)\n",
    "        #col = cFilePath.columns\n",
    "        incompleteFileName = \"Data_\" + str(indexFolder) + \"_\" + fileName + \"_\" + str(j) + \"%.csv\"\n",
    "        df = pd.read_csv(\"C:\\\\Users\\\\patel2k4\\\\Downloads\\\\\" +  incompleteFileName, header = None)\n",
    "        print(df.shape)\n",
    "        df.columns = col\n",
    "        df1 = pd.DataFrame(df,columns = col)\n",
    "\n",
    "        df1.drop(columns = ['Labels'], axis = 1,  inplace = True)\n",
    "  \n",
    "        df.drop(columns = ['Labels'], axis = 1,  inplace = True)\n",
    "        df.index = [str(_) for _ in range(1,len(df)+1)]\n",
    "        cFilePath.index = [str(_) for _ in range(1,len(df)+1)]\n",
    "        col1 = df.columns\n",
    "\n",
    "        DIT = df.T\n",
    "        DCT = cFilePath.T\n",
    "        colList = []\n",
    "        for i in DIT.columns:\n",
    "            if DIT[i].isnull().values.any() == False:\n",
    "                colList.append(i)\n",
    "  \n",
    "        if len(colList) > 2:\n",
    "            l1 = [('PC' + str(i)) for i in range(1,21)]\n",
    "            redDI = DIT.loc[:,colList] \n",
    "            x = DIT[colList]\n",
    "            DIT.drop(columns = colList, axis = 1, inplace = True)\n",
    "            y = DCT[colList]\n",
    "            DCT.drop(columns = colList, axis = 1, inplace = True)\n",
    "            scalar = StandardScaler()\n",
    "            scaled_data = pd.DataFrame(scalar.fit_transform(redDI))\n",
    "            pca = PCA(n_components = 20)\n",
    "            pca.fit(scaled_data)\n",
    "            data_pca = pca.transform(scaled_data)\n",
    "            \n",
    "            data_pca = pd.DataFrame(data_pca,columns=[l1],index = col1 )\n",
    "            DIT[l1] = data_pca\n",
    "            DCT[l1] = data_pca\n",
    "            colDCT = DCT.columns\n",
    "               \n",
    "        transposeDIT = DIT.T\n",
    "        transposeDCT = DCT.T\n",
    "        #df.drop(columns = ['Labels'], axis = 1,  inplace = True)\n",
    "        start_time = time.time()\n",
    "        all_dataset_imputed, count1, total, nc = DMI(transposeDCT ,transposeDIT)\n",
    "        DIT = all_dataset_imputed.T \n",
    "        DCT = transposeDCT.T \n",
    "        df = DIT.T \n",
    "        #DCT.drop(columns = l1,axis = 1,inplace = True)\n",
    "        cFilePath = DCT.T\n",
    "        print(cFilePath.shape)\n",
    "      \n",
    "        \n",
    "    \n",
    "       \n",
    "    \n",
    "\n",
    "        timeFinal = (time.time() - start_time)\n",
    "        \n",
    "\n",
    "        nrmsValue = nrms( df,cFilePath)\n",
    "        \n",
    "        #df.drop(index = l1)\n",
    "\n",
    "        #new = pd.concat([df,x.T])\n",
    "        #rint(new.shape)\n",
    "        df.to_csv(\"C:\\\\Users\\\\patel2k4\\\\Downloads\\\\New folder\\\\\" + incompleteFileName, index = None)\n",
    "        ws1['B'+str(count)] = nrmsValue\n",
    "        ws1['C'+str(count)] = timeFinal\n",
    "        print(timeFinal,nrmsValue,incompleteFileName + \", Number of Horizontal Partitions: \" + str(count1) + \", EM Total iterations: \" + str(total)  + \", Convergence(%): 40%\")\n",
    "        ws1['D'+str(count)] = (\"Number of Horizontal Partitions: \" + str(count1) + \", EM iterations: \" + str(total)  + \", Convergence(%): 10%\" + \",NC :\" + str(nc))\n",
    "\n",
    "        count += 1\n",
    "        #print(\"iteration\",itr)\n",
    "\n",
    "\n",
    "        wb.save(summaryName) \n",
    "    return count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f592bb85",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#folderName = \"Data \" + str(3) \n",
    "completeFileName = 'Data_' + str(14) + '.csv'\n",
    "cFilePath = pd.read_csv(\"C:\\\\Users\\\\patel2k4\\\\Downloads\\\\\"+completeFileName)\n",
    "col = cFilePath.columns\n",
    "cFilePath.drop(columns = ['Labels'], axis = 1,  inplace = True)\n",
    "df_complete_data=pd.DataFrame(data=cFilePath, columns=cFilePath.columns, index=cFilePath.index)\n",
    "    \n",
    "count = 577\n",
    "count1 = fileName(14,cFilePath, \"AE\", count,col)\n",
    "count2 = fileName(6,cFilePath, \"AG\", count1,col)\n",
    "count3 = fileName(6,cFilePath, \"AL\", count2,col)\n",
    "count4 = fileName(6,cFilePath, \"AN\", count3,col)\n",
    "count5 = fileName(6,cFilePath, \"AW\", count4,col)\n",
    "count6 = fileName(6,cFilePath, \"C\", count5,col)\n",
    "count7 = fileName(6,cFilePath, \"NE\", count6,col)\n",
    "count8 = fileName(6,cFilePath, \"NG\", count7,col)\n",
    "count9 = fileName(6,cFilePath, \"NL\", count8,col)\n",
    "count10 = fileName(6,cFilePath, \"NN\", count9,col)\n",
    "count11 = fileName(14,cFilePath, \"NW\", count10,col)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b4a16637",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0967a551",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}