General model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f2c77d52",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "poly_data = pd.read_excel('resulting_dataset.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7a903866",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>polymer_name</th>\n",
       "      <th>Number of features</th>\n",
       "      <th>Features names</th>\n",
       "      <th>Thermal conductivity_value_median</th>\n",
       "      <th>Dynamic mechanical properties loss tangent_value_median</th>\n",
       "      <th>Dynamic compressive properties loss tangent_value_median</th>\n",
       "      <th>Thermal decomposition temperature_value_median</th>\n",
       "      <th>Compressive stress strength at break_value_median</th>\n",
       "      <th>Hansen parameter delta-h: hydrogen bonding_value_median</th>\n",
       "      <th>...</th>\n",
       "      <th>Crystallization kinetics r_value_variance</th>\n",
       "      <th>Contact angle_value_variance</th>\n",
       "      <th>Surface tension_value_variance</th>\n",
       "      <th>Interfacial tension_value_variance</th>\n",
       "      <th>Specific volume_value_variance</th>\n",
       "      <th>Tensile creep compliance_value_variance</th>\n",
       "      <th>Tensile creep rupture strength_value_variance</th>\n",
       "      <th>cu_formula</th>\n",
       "      <th>polymer_id</th>\n",
       "      <th>vector</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>polyethene</td>\n",
       "      <td>93</td>\n",
       "      <td>['Density', 'Specific volume', 'Refractive ind...</td>\n",
       "      <td>0.330</td>\n",
       "      <td>0.127</td>\n",
       "      <td>0.12</td>\n",
       "      <td>435.0</td>\n",
       "      <td>0.069</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.229120e+08</td>\n",
       "      <td>866.267205</td>\n",
       "      <td>122.313701</td>\n",
       "      <td>20.882573</td>\n",
       "      <td>12.574191</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>CH2</td>\n",
       "      <td>P010001</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>poly(prop-1-ene)</td>\n",
       "      <td>86</td>\n",
       "      <td>['Density', 'Specific volume', 'Refractive ind...</td>\n",
       "      <td>0.210</td>\n",
       "      <td>0.055</td>\n",
       "      <td>0.25</td>\n",
       "      <td>373.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>9.689055e+05</td>\n",
       "      <td>526.864086</td>\n",
       "      <td>200.976440</td>\n",
       "      <td>11.278686</td>\n",
       "      <td>0.002383</td>\n",
       "      <td>0.008107</td>\n",
       "      <td>NaN</td>\n",
       "      <td>C3H6</td>\n",
       "      <td>P010002</td>\n",
       "      <td>[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>poly(but-1-ene)</td>\n",
       "      <td>41</td>\n",
       "      <td>['Density', 'Specific volume', 'Refractive ind...</td>\n",
       "      <td>0.218</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>2.165257e+04</td>\n",
       "      <td>NaN</td>\n",
       "      <td>27.012677</td>\n",
       "      <td>0.962667</td>\n",
       "      <td>0.003568</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>C4H8</td>\n",
       "      <td>P010003</td>\n",
       "      <td>[1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>poly(pent-1-ene)</td>\n",
       "      <td>16</td>\n",
       "      <td>['Density', 'Specific volume', 'Glass transiti...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.004569</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>C5H10</td>\n",
       "      <td>P010004</td>\n",
       "      <td>[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>poly(3-methylbut-1-ene)</td>\n",
       "      <td>15</td>\n",
       "      <td>['Density', 'Specific volume', 'Glass transiti...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>452.5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000092</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>C5H10</td>\n",
       "      <td>P010006</td>\n",
       "      <td>[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18307</th>\n",
       "      <td>18307</td>\n",
       "      <td>poly{2,3-diphenyl-6-[6,7-diphenylnaphthalene-2...</td>\n",
       "      <td>5</td>\n",
       "      <td>['Glass transition temperature', 'Isothermal w...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>C57H34</td>\n",
       "      <td>P522075</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18308</th>\n",
       "      <td>18308</td>\n",
       "      <td>poly(1,1,2-trifluoro-2-{1,1,2,4,4,5,5-heptaflu...</td>\n",
       "      <td>1</td>\n",
       "      <td>['SMILES']</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>C7F13NaO5S</td>\n",
       "      <td>P522076</td>\n",
       "      <td>[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18309</th>\n",
       "      <td>18309</td>\n",
       "      <td>poly(naphthalene-4,5:8,1-tetrayl)</td>\n",
       "      <td>5</td>\n",
       "      <td>['Density', 'Specific volume', 'Electric condu...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>C10H4</td>\n",
       "      <td>P522077</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18310</th>\n",
       "      <td>18310</td>\n",
       "      <td>poly((phenylarsanediyl){1-[(phenylarsanediyl)e...</td>\n",
       "      <td>1</td>\n",
       "      <td>['SMILES']</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>C22H16As2</td>\n",
       "      <td>P522078</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18311</th>\n",
       "      <td>18311</td>\n",
       "      <td>poly((phenylarsanediyl){1-[(phenylarsanediyl)e...</td>\n",
       "      <td>1</td>\n",
       "      <td>['SMILES']</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>C22H16As2</td>\n",
       "      <td>P522079</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>18312 rows × 204 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Unnamed: 0                                       polymer_name  \\\n",
       "0               0                                         polyethene   \n",
       "1               1                                   poly(prop-1-ene)   \n",
       "2               2                                    poly(but-1-ene)   \n",
       "3               3                                   poly(pent-1-ene)   \n",
       "4               4                            poly(3-methylbut-1-ene)   \n",
       "...           ...                                                ...   \n",
       "18307       18307  poly{2,3-diphenyl-6-[6,7-diphenylnaphthalene-2...   \n",
       "18308       18308  poly(1,1,2-trifluoro-2-{1,1,2,4,4,5,5-heptaflu...   \n",
       "18309       18309                  poly(naphthalene-4,5:8,1-tetrayl)   \n",
       "18310       18310  poly((phenylarsanediyl){1-[(phenylarsanediyl)e...   \n",
       "18311       18311  poly((phenylarsanediyl){1-[(phenylarsanediyl)e...   \n",
       "\n",
       "       Number of features                                     Features names  \\\n",
       "0                      93  ['Density', 'Specific volume', 'Refractive ind...   \n",
       "1                      86  ['Density', 'Specific volume', 'Refractive ind...   \n",
       "2                      41  ['Density', 'Specific volume', 'Refractive ind...   \n",
       "3                      16  ['Density', 'Specific volume', 'Glass transiti...   \n",
       "4                      15  ['Density', 'Specific volume', 'Glass transiti...   \n",
       "...                   ...                                                ...   \n",
       "18307                   5  ['Glass transition temperature', 'Isothermal w...   \n",
       "18308                   1                                         ['SMILES']   \n",
       "18309                   5  ['Density', 'Specific volume', 'Electric condu...   \n",
       "18310                   1                                         ['SMILES']   \n",
       "18311                   1                                         ['SMILES']   \n",
       "\n",
       "       Thermal conductivity_value_median  \\\n",
       "0                                  0.330   \n",
       "1                                  0.210   \n",
       "2                                  0.218   \n",
       "3                                    NaN   \n",
       "4                                    NaN   \n",
       "...                                  ...   \n",
       "18307                                NaN   \n",
       "18308                                NaN   \n",
       "18309                                NaN   \n",
       "18310                                NaN   \n",
       "18311                                NaN   \n",
       "\n",
       "       Dynamic mechanical properties loss tangent_value_median  \\\n",
       "0                                                  0.127         \n",
       "1                                                  0.055         \n",
       "2                                                    NaN         \n",
       "3                                                    NaN         \n",
       "4                                                    NaN         \n",
       "...                                                  ...         \n",
       "18307                                                NaN         \n",
       "18308                                                NaN         \n",
       "18309                                                NaN         \n",
       "18310                                                NaN         \n",
       "18311                                                NaN         \n",
       "\n",
       "       Dynamic compressive properties loss tangent_value_median  \\\n",
       "0                                                   0.12          \n",
       "1                                                   0.25          \n",
       "2                                                    NaN          \n",
       "3                                                    NaN          \n",
       "4                                                    NaN          \n",
       "...                                                  ...          \n",
       "18307                                                NaN          \n",
       "18308                                                NaN          \n",
       "18309                                                NaN          \n",
       "18310                                                NaN          \n",
       "18311                                                NaN          \n",
       "\n",
       "       Thermal decomposition temperature_value_median  \\\n",
       "0                                               435.0   \n",
       "1                                               373.0   \n",
       "2                                                 NaN   \n",
       "3                                                 NaN   \n",
       "4                                               452.5   \n",
       "...                                               ...   \n",
       "18307                                             NaN   \n",
       "18308                                             NaN   \n",
       "18309                                             NaN   \n",
       "18310                                             NaN   \n",
       "18311                                             NaN   \n",
       "\n",
       "       Compressive stress strength at break_value_median  \\\n",
       "0                                                  0.069   \n",
       "1                                                    NaN   \n",
       "2                                                    NaN   \n",
       "3                                                    NaN   \n",
       "4                                                    NaN   \n",
       "...                                                  ...   \n",
       "18307                                                NaN   \n",
       "18308                                                NaN   \n",
       "18309                                                NaN   \n",
       "18310                                                NaN   \n",
       "18311                                                NaN   \n",
       "\n",
       "       Hansen parameter delta-h: hydrogen bonding_value_median  ...  \\\n",
       "0                                                    0.0        ...   \n",
       "1                                                    NaN        ...   \n",
       "2                                                    NaN        ...   \n",
       "3                                                    NaN        ...   \n",
       "4                                                    NaN        ...   \n",
       "...                                                  ...        ...   \n",
       "18307                                                NaN        ...   \n",
       "18308                                                NaN        ...   \n",
       "18309                                                NaN        ...   \n",
       "18310                                                NaN        ...   \n",
       "18311                                                NaN        ...   \n",
       "\n",
       "       Crystallization kinetics r_value_variance  \\\n",
       "0                                   1.229120e+08   \n",
       "1                                   9.689055e+05   \n",
       "2                                   2.165257e+04   \n",
       "3                                            NaN   \n",
       "4                                            NaN   \n",
       "...                                          ...   \n",
       "18307                                        NaN   \n",
       "18308                                        NaN   \n",
       "18309                                        NaN   \n",
       "18310                                        NaN   \n",
       "18311                                        NaN   \n",
       "\n",
       "       Contact angle_value_variance  Surface tension_value_variance  \\\n",
       "0                        866.267205                      122.313701   \n",
       "1                        526.864086                      200.976440   \n",
       "2                               NaN                       27.012677   \n",
       "3                               NaN                        0.000000   \n",
       "4                               NaN                             NaN   \n",
       "...                             ...                             ...   \n",
       "18307                           NaN                             NaN   \n",
       "18308                           NaN                             NaN   \n",
       "18309                           NaN                             NaN   \n",
       "18310                           NaN                             NaN   \n",
       "18311                           NaN                             NaN   \n",
       "\n",
       "       Interfacial tension_value_variance  Specific volume_value_variance  \\\n",
       "0                               20.882573                       12.574191   \n",
       "1                               11.278686                        0.002383   \n",
       "2                                0.962667                        0.003568   \n",
       "3                                0.000000                        0.004569   \n",
       "4                                     NaN                        0.000092   \n",
       "...                                   ...                             ...   \n",
       "18307                                 NaN                             NaN   \n",
       "18308                                 NaN                             NaN   \n",
       "18309                                 NaN                        0.000000   \n",
       "18310                                 NaN                             NaN   \n",
       "18311                                 NaN                             NaN   \n",
       "\n",
       "       Tensile creep compliance_value_variance  \\\n",
       "0                                     0.000000   \n",
       "1                                     0.008107   \n",
       "2                                          NaN   \n",
       "3                                          NaN   \n",
       "4                                          NaN   \n",
       "...                                        ...   \n",
       "18307                                      NaN   \n",
       "18308                                      NaN   \n",
       "18309                                      NaN   \n",
       "18310                                      NaN   \n",
       "18311                                      NaN   \n",
       "\n",
       "       Tensile creep rupture strength_value_variance  cu_formula  polymer_id  \\\n",
       "0                                                NaN         CH2     P010001   \n",
       "1                                                NaN        C3H6     P010002   \n",
       "2                                                NaN        C4H8     P010003   \n",
       "3                                                NaN       C5H10     P010004   \n",
       "4                                                NaN       C5H10     P010006   \n",
       "...                                              ...         ...         ...   \n",
       "18307                                            NaN      C57H34     P522075   \n",
       "18308                                            NaN  C7F13NaO5S     P522076   \n",
       "18309                                            NaN       C10H4     P522077   \n",
       "18310                                            NaN   C22H16As2     P522078   \n",
       "18311                                            NaN   C22H16As2     P522079   \n",
       "\n",
       "                                                  vector  \n",
       "0      [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  \n",
       "1      [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  \n",
       "2      [1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  \n",
       "3      [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  \n",
       "4      [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  \n",
       "...                                                  ...  \n",
       "18307  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  \n",
       "18308  [0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  \n",
       "18309  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  \n",
       "18310  [0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, ...  \n",
       "18311  [0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, ...  \n",
       "\n",
       "[18312 rows x 204 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "poly_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "bd99581e",
   "metadata": {},
   "outputs": [],
   "source": [
    "poly_data_cols_medians = [a for a in poly_data.columns if 'median' in a]\n",
    "poly_data_cols_vars = [a for a in poly_data.columns if 'variance' in a]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "94b78872",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>col_0</th>\n",
       "      <th>col_1</th>\n",
       "      <th>col_2</th>\n",
       "      <th>col_3</th>\n",
       "      <th>col_4</th>\n",
       "      <th>col_5</th>\n",
       "      <th>col_6</th>\n",
       "      <th>col_7</th>\n",
       "      <th>col_8</th>\n",
       "      <th>...</th>\n",
       "      <th>col_1014</th>\n",
       "      <th>col_1015</th>\n",
       "      <th>col_1016</th>\n",
       "      <th>col_1017</th>\n",
       "      <th>col_1018</th>\n",
       "      <th>col_1019</th>\n",
       "      <th>col_1020</th>\n",
       "      <th>col_1021</th>\n",
       "      <th>col_1022</th>\n",
       "      <th>col_1023</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18307</th>\n",
       "      <td>18307</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18308</th>\n",
       "      <td>18308</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18309</th>\n",
       "      <td>18309</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18310</th>\n",
       "      <td>18310</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18311</th>\n",
       "      <td>18311</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>18312 rows × 1025 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Unnamed: 0  col_0  col_1  col_2  col_3  col_4  col_5  col_6  col_7  \\\n",
       "0               0      0      0      0      0      0      0      0      0   \n",
       "1               1      0      1      0      0      0      0      0      0   \n",
       "2               2      1      1      0      0      0      0      0      0   \n",
       "3               3      0      1      0      0      0      0      0      0   \n",
       "4               4      0      1      0      0      0      0      0      0   \n",
       "...           ...    ...    ...    ...    ...    ...    ...    ...    ...   \n",
       "18307       18307      0      0      0      0      0      0      0      0   \n",
       "18308       18308      0      0      0      0      1      0      0      0   \n",
       "18309       18309      0      0      0      0      0      0      0      0   \n",
       "18310       18310      0      0      0      0      0      0      0      1   \n",
       "18311       18311      0      0      0      0      0      0      0      1   \n",
       "\n",
       "       col_8  ...  col_1014  col_1015  col_1016  col_1017  col_1018  col_1019  \\\n",
       "0          0  ...         0         0         0         0         0         0   \n",
       "1          0  ...         0         0         0         0         0         0   \n",
       "2          0  ...         0         0         0         0         0         0   \n",
       "3          0  ...         0         0         0         0         0         0   \n",
       "4          0  ...         0         0         0         0         0         0   \n",
       "...      ...  ...       ...       ...       ...       ...       ...       ...   \n",
       "18307      0  ...         0         0         0         0         0         0   \n",
       "18308      0  ...         0         0         0         0         0         0   \n",
       "18309      0  ...         0         0         0         0         0         0   \n",
       "18310      0  ...         0         0         0         0         0         0   \n",
       "18311      0  ...         0         0         0         1         0         0   \n",
       "\n",
       "       col_1020  col_1021  col_1022  col_1023  \n",
       "0             0         0         0         0  \n",
       "1             0         0         0         0  \n",
       "2             0         0         0         0  \n",
       "3             0         0         0         0  \n",
       "4             0         0         0         0  \n",
       "...         ...       ...       ...       ...  \n",
       "18307         0         0         0         0  \n",
       "18308         0         0         0         0  \n",
       "18309         0         0         0         0  \n",
       "18310         0         0         0         0  \n",
       "18311         0         0         0         0  \n",
       "\n",
       "[18312 rows x 1025 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_df = pd.read_csv('cols.csv')\n",
    "new_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "b329a2a3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "80"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "poly_data[f'{poly_data_cols_medians[0]}'].notna().sum()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "23cf0e67",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Thermal conductivity\n"
     ]
    }
   ],
   "source": [
    "column_name = 'Thermal conductivity_value_median'\n",
    "\n",
    "# Remove the suffix \"_value_median\"\n",
    "cleaned_column_name = poly_data_cols_medians[0].replace('_value_median', '')\n",
    "\n",
    "print(cleaned_column_name)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "6feb31dd",
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalized_mse(y_true, y_pred):\n",
    "    mse = mean_squared_error(y_true, y_pred)\n",
    "    var_y = np.var(y_true)\n",
    "    nmse = mse / var_y\n",
    "    return nmse\n",
    "\n",
    "def mean_percentage_error(y_true, y_pred):\n",
    "    y_true, y_pred = np.array(y_true), np.array(y_pred)\n",
    "    \n",
    "    # Avoid division by zero\n",
    "    non_zero_indices = y_true != 0\n",
    "    y_true_non_zero = y_true[non_zero_indices]\n",
    "    y_pred_non_zero = y_pred[non_zero_indices]\n",
    "    \n",
    "    # Calculate MPE\n",
    "    mpe = abs(np.mean((y_true_non_zero - y_pred_non_zero) / y_true_non_zero) * 100)\n",
    "    return mpe\n",
    "\n",
    "def root_mean_squared_error(y_true, y_pred):\n",
    "    mse = mean_squared_error(y_true, y_pred)\n",
    "    rmse = np.sqrt(mse)\n",
    "    return rmse\n",
    "\n",
    "def custom_metric(row):\n",
    "    if row['var'] > 0:\n",
    "        return 1 if abs(row['y_pred'] - row['y_test']) < row['var'] else 0\n",
    "    else:\n",
    "        return None\n",
    "\n",
    "mse_list = []\n",
    "r2_list = []\n",
    "rmse_list = []\n",
    "nmse_list = []\n",
    "mae_list = []\n",
    "mpe_list = []\n",
    "overall_list = []\n",
    "coefs_list = []\n",
    "data_list = []\n",
    "name_list = []\n",
    "\n",
    "for q in range(len(poly_data_cols_vars)):\n",
    "    if poly_data[f'{poly_data_cols_medians[q]}'].notna().sum() > 50:\n",
    "        \n",
    "        cleaned_column_name = poly_data_cols_medians[q].replace('_value_median', '')\n",
    "        data_size = poly_data[f'{poly_data_cols_medians[q]}'].notna().sum()\n",
    "        \n",
    "        new_df[f'{poly_data_cols_medians[q]}'] = poly_data[f'{poly_data_cols_medians[q]}']\n",
    "        new_df[f'{poly_data_cols_vars[q]}'] = poly_data[f'{poly_data_cols_vars[q]}']\n",
    "        #new_df.drop(['Unnamed: 0'], axis=1, inplace=True)\n",
    "        meta_df = new_df[new_df[f'{poly_data_cols_medians[q]}'].isna() == False]\n",
    "\n",
    "        cols = [c for c in meta_df.columns if 'col' in c]\n",
    "        X = meta_df[cols]\n",
    "        y = meta_df[f'{poly_data_cols_medians[q]}']\n",
    "\n",
    "        # Split the data into training and testing sets\n",
    "        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)\n",
    "\n",
    "        # Initialize the KNN Regressor model\n",
    "        model = KNeighborsRegressor(n_neighbors=5)  # 'n_neighbors' is the number of neighbors to consider, you can adjust it\n",
    "\n",
    "        # Train the model on the training data\n",
    "        model.fit(X_train, y_train)\n",
    "\n",
    "        # Make predictions on the test set\n",
    "        y_pred = model.predict(X_test)\n",
    "\n",
    "        #coefficients = pd.DataFrame({'Feature': X.columns, 'Coefficient': model.coef_})\n",
    "\n",
    "        mse = mean_squared_error(y_test, y_pred)\n",
    "        r2 = r2_score(y_test, y_pred)\n",
    "        rmse = root_mean_squared_error(y_test, y_pred)\n",
    "        nmse = normalized_mse(y_test, y_pred)\n",
    "        mae = mean_absolute_error(y_test, y_pred)\n",
    "        mpe = mean_percentage_error(y_test, y_pred)\n",
    "\n",
    "        indexes_to_show = y_test.index\n",
    "        y_var = meta_df.loc[indexes_to_show, f'{poly_data_cols_vars[q]}']\n",
    "\n",
    "        tst = list(y_test)\n",
    "        prd = list(y_pred)\n",
    "        ndf = pd.DataFrame({'y_test': tst, 'y_pred': prd, 'var': y_var})\n",
    "\n",
    "        # Apply the custom metric to each row\n",
    "        ndf['metric'] = ndf.apply(custom_metric, axis=1)\n",
    "\n",
    "        # Calculate the overall metric (excluding rows where var <= 0)\n",
    "        filtered_df = ndf.dropna(subset=['metric'])\n",
    "        overall = filtered_df['metric'].mean()\n",
    "        \n",
    "        data_list.append(data_size)\n",
    "        name_list.append(cleaned_column_name)\n",
    "        mse_list.append(mse)\n",
    "        r2_list.append(r2)\n",
    "        rmse_list.append(rmse)\n",
    "        nmse_list.append(nmse)\n",
    "        mae_list.append(mae)\n",
    "        mpe_list.append(mpe)\n",
    "        overall_list.append(overall)\n",
    "        #coefs_list.append(coefs)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "75a22677",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Data Size</th>\n",
       "      <th>Name</th>\n",
       "      <th>MSE</th>\n",
       "      <th>R2</th>\n",
       "      <th>RMSE</th>\n",
       "      <th>NMSE</th>\n",
       "      <th>MAE</th>\n",
       "      <th>MPE</th>\n",
       "      <th>Overall</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>80</td>\n",
       "      <td>Thermal conductivity</td>\n",
       "      <td>3.164134e-02</td>\n",
       "      <td>-3.871672</td>\n",
       "      <td>0.177880</td>\n",
       "      <td>4.871672</td>\n",
       "      <td>0.098837</td>\n",
       "      <td>77.144525</td>\n",
       "      <td>0.400000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>301</td>\n",
       "      <td>Dynamic mechanical properties loss tangent</td>\n",
       "      <td>2.816818e-01</td>\n",
       "      <td>0.294627</td>\n",
       "      <td>0.530737</td>\n",
       "      <td>0.705373</td>\n",
       "      <td>0.361926</td>\n",
       "      <td>193.757675</td>\n",
       "      <td>0.230769</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6325</td>\n",
       "      <td>Thermal decomposition temperature</td>\n",
       "      <td>3.774505e+03</td>\n",
       "      <td>0.668462</td>\n",
       "      <td>61.436999</td>\n",
       "      <td>0.331538</td>\n",
       "      <td>41.426355</td>\n",
       "      <td>6.547767</td>\n",
       "      <td>0.943953</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>59</td>\n",
       "      <td>Hansen parameter delta-h: hydrogen bonding</td>\n",
       "      <td>1.382237e+01</td>\n",
       "      <td>0.360865</td>\n",
       "      <td>3.717845</td>\n",
       "      <td>0.639135</td>\n",
       "      <td>2.950667</td>\n",
       "      <td>68.706544</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>83</td>\n",
       "      <td>Flexural modulus</td>\n",
       "      <td>1.476671e+01</td>\n",
       "      <td>-0.492477</td>\n",
       "      <td>3.842748</td>\n",
       "      <td>1.492477</td>\n",
       "      <td>2.320985</td>\n",
       "      <td>63.890962</td>\n",
       "      <td>0.428571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>74</td>\n",
       "      <td>Fiber tensile modulus</td>\n",
       "      <td>6.681067e+03</td>\n",
       "      <td>-3.804689</td>\n",
       "      <td>81.737797</td>\n",
       "      <td>4.804689</td>\n",
       "      <td>56.648125</td>\n",
       "      <td>112.655038</td>\n",
       "      <td>0.750000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>53</td>\n",
       "      <td>Crystallization kinetics r</td>\n",
       "      <td>1.353927e+08</td>\n",
       "      <td>-174.772928</td>\n",
       "      <td>11635.835576</td>\n",
       "      <td>175.772928</td>\n",
       "      <td>8937.743333</td>\n",
       "      <td>119457.763829</td>\n",
       "      <td>0.250000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>255</td>\n",
       "      <td>Contact angle</td>\n",
       "      <td>2.277655e+02</td>\n",
       "      <td>0.241732</td>\n",
       "      <td>15.091903</td>\n",
       "      <td>0.758268</td>\n",
       "      <td>12.310577</td>\n",
       "      <td>6.215002</td>\n",
       "      <td>0.769231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66</th>\n",
       "      <td>348</td>\n",
       "      <td>Surface tension</td>\n",
       "      <td>6.859090e+01</td>\n",
       "      <td>0.562093</td>\n",
       "      <td>8.281962</td>\n",
       "      <td>0.437907</td>\n",
       "      <td>5.614857</td>\n",
       "      <td>5.263455</td>\n",
       "      <td>0.812500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>1739</td>\n",
       "      <td>Specific volume</td>\n",
       "      <td>8.619929e-03</td>\n",
       "      <td>0.494610</td>\n",
       "      <td>0.092844</td>\n",
       "      <td>0.505390</td>\n",
       "      <td>0.061054</td>\n",
       "      <td>4.119176</td>\n",
       "      <td>0.015152</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>68 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    Data Size                                        Name           MSE  \\\n",
       "0          80                        Thermal conductivity  3.164134e-02   \n",
       "1         301  Dynamic mechanical properties loss tangent  2.816818e-01   \n",
       "2        6325           Thermal decomposition temperature  3.774505e+03   \n",
       "3          59  Hansen parameter delta-h: hydrogen bonding  1.382237e+01   \n",
       "4          83                            Flexural modulus  1.476671e+01   \n",
       "..        ...                                         ...           ...   \n",
       "63         74                       Fiber tensile modulus  6.681067e+03   \n",
       "64         53                  Crystallization kinetics r  1.353927e+08   \n",
       "65        255                               Contact angle  2.277655e+02   \n",
       "66        348                             Surface tension  6.859090e+01   \n",
       "67       1739                             Specific volume  8.619929e-03   \n",
       "\n",
       "            R2          RMSE        NMSE          MAE            MPE   Overall  \n",
       "0    -3.871672      0.177880    4.871672     0.098837      77.144525  0.400000  \n",
       "1     0.294627      0.530737    0.705373     0.361926     193.757675  0.230769  \n",
       "2     0.668462     61.436999    0.331538    41.426355       6.547767  0.943953  \n",
       "3     0.360865      3.717845    0.639135     2.950667      68.706544  0.000000  \n",
       "4    -0.492477      3.842748    1.492477     2.320985      63.890962  0.428571  \n",
       "..         ...           ...         ...          ...            ...       ...  \n",
       "63   -3.804689     81.737797    4.804689    56.648125     112.655038  0.750000  \n",
       "64 -174.772928  11635.835576  175.772928  8937.743333  119457.763829  0.250000  \n",
       "65    0.241732     15.091903    0.758268    12.310577       6.215002  0.769231  \n",
       "66    0.562093      8.281962    0.437907     5.614857       5.263455  0.812500  \n",
       "67    0.494610      0.092844    0.505390     0.061054       4.119176  0.015152  \n",
       "\n",
       "[68 rows x 9 columns]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res = pd.DataFrame({\n",
    "    'Data Size': data_list,\n",
    "    'Name': name_list,\n",
    "    'MSE': mse_list,\n",
    "    'R2': r2_list,\n",
    "    'RMSE': rmse_list,\n",
    "    'NMSE': nmse_list,\n",
    "    'MAE': mae_list,\n",
    "    'MPE': mpe_list,\n",
    "    'Overall': overall_list\n",
    "})\n",
    "\n",
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2cfce482",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from sklearn.linear_model import  LinearRegression, Lasso, ElasticNet\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.ensemble import BaggingRegressor, AdaBoostRegressor\n",
    "from xgboost import XGBRegressor\n",
    "from sklearn.ensemble import GradientBoostingRegressor\n",
    "from sklearn.svm import SVR\n",
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "def run_knn(X_train, X_test, y_train, y_test):\n",
    "    model = KNeighborsRegressor(n_neighbors=5)\n",
    "    model.fit(X_train, y_train)\n",
    "    y_pred = model.predict(X_test)\n",
    "    coefs = None  # KNN Regressor doesn't have coefficients\n",
    "    return y_pred, coefs\n",
    "\n",
    "def run_lasso(X_train, X_test, y_train, y_test):\n",
    "    model = Lasso()\n",
    "    model.fit(X_train, y_train)\n",
    "    y_pred = model.predict(X_test)\n",
    "    coefs = model.coef_\n",
    "    return y_pred, coefs\n",
    "\n",
    "def run_elastic_net(X_train, X_test, y_train, y_test):\n",
    "    model = ElasticNet()\n",
    "    model.fit(X_train, y_train)\n",
    "    y_pred = model.predict(X_test)\n",
    "    coefs = model.coef_\n",
    "    return y_pred, coefs\n",
    "\n",
    "def run_decision_tree(X_train, X_test, y_train, y_test):\n",
    "    model = DecisionTreeRegressor()\n",
    "    model.fit(X_train, y_train)\n",
    "    y_pred = model.predict(X_test)\n",
    "    coefs = None  # Decision Tree doesn't have coefficients\n",
    "    return y_pred, coefs\n",
    "\n",
    "def run_bagging(X_train, X_test, y_train, y_test):\n",
    "    model = BaggingRegressor()\n",
    "    model.fit(X_train, y_train)\n",
    "    y_pred = model.predict(X_test)\n",
    "    coefs = None  # Bagging doesn't have coefficients\n",
    "    return y_pred, coefs\n",
    "\n",
    "def run_adaboost(X_train, X_test, y_train, y_test):\n",
    "    model = AdaBoostRegressor()\n",
    "    model.fit(X_train, y_train)\n",
    "    y_pred = model.predict(X_test)\n",
    "    coefs = None  # AdaBoost doesn't have coefficients\n",
    "    return y_pred, coefs\n",
    "\n",
    "def run_xgboost(X_train, X_test, y_train, y_test):\n",
    "    model = XGBRegressor()\n",
    "    model.fit(X_train, y_train)\n",
    "    y_pred = model.predict(X_test)\n",
    "    coefs = None  # XGBoost doesn't have coefficients\n",
    "    return y_pred, coefs\n",
    "\n",
    "def run_svr(X_train, X_test, y_train, y_test):\n",
    "    model = SVR()\n",
    "    model.fit(X_train, y_train)\n",
    "    y_pred = model.predict(X_test)\n",
    "    coefs = None  # SVR doesn't have coefficients\n",
    "    return y_pred, coefs\n",
    "\n",
    "def run_gradient_boosting(X_train, X_test, y_train, y_test):\n",
    "    model = GradientBoostingRegressor()\n",
    "    model.fit(X_train, y_train)\n",
    "    y_pred = model.predict(X_test)\n",
    "    coefs = None  # GradientBoostingRegressor doesn't have coefficients\n",
    "    return y_pred, coefs\n",
    "\n",
    "def run_linear_regression(X_train, X_test, y_train, y_test):\n",
    "    model = LinearRegression()\n",
    "    model.fit(X_train, y_train)\n",
    "    y_pred = model.predict(X_test)\n",
    "    coefs = model.coef_\n",
    "    return y_pred, coefs\n",
    "\n",
    "def run_random_forest(X_train, X_test, y_train, y_test):\n",
    "    model = RandomForestRegressor()\n",
    "    model.fit(X_train, y_train)\n",
    "    y_pred = model.predict(X_test)\n",
    "    coefs = model.feature_importances_\n",
    "    return y_pred, coefs\n",
    "\n",
    "\n",
    "# List of models and their names\n",
    "models = [run_knn, run_lasso, run_elastic_net, run_decision_tree, run_bagging, run_adaboost,\n",
    "          run_xgboost, run_svr, run_gradient_boosting, run_linear_regression, run_random_forest]\n",
    "model_names = ['KNeighborsRegressor', 'Lasso', 'Elastic Net', 'Decision Tree', 'Bagging', 'AdaBoost', 'XGBoost',\n",
    "               'SVR', 'Gradient Boosting', 'Linear Regression', 'Random Forest']\n",
    "\n",
    "# List of models and their names\n",
    "models = [run_linear_regression, run_random_forest]\n",
    "model_names = ['Linear Regression', 'Random Forest']\n",
    "\n",
    "# Iterate over models\n",
    "for model, model_name in zip(models, model_names):\n",
    "    \n",
    "    # Specify the directory and file name\n",
    "    model_directory = f'{model_name}'\n",
    "    \n",
    "    # Create the directory if it doesn't exist\n",
    "    os.makedirs(model_directory, exist_ok=True)\n",
    "        \n",
    "    mse_list = []\n",
    "    r2_list = []\n",
    "    rmse_list = []\n",
    "    nmse_list = []\n",
    "    mae_list = []\n",
    "    mpe_list = []\n",
    "    overall_list = []\n",
    "    data_list = []\n",
    "    name_list = []\n",
    "\n",
    "    for q in range(len(poly_data_cols_vars)):\n",
    "        if poly_data[f'{poly_data_cols_medians[q]}'].notna().sum() > 50:\n",
    "        \n",
    "            cleaned_column_name = poly_data_cols_medians[q].replace('_value_median', '')\n",
    "            data_size = poly_data[f'{poly_data_cols_medians[q]}'].notna().sum()\n",
    "\n",
    "            new_df[f'{poly_data_cols_medians[q]}'] = poly_data[f'{poly_data_cols_medians[q]}']\n",
    "            new_df[f'{poly_data_cols_vars[q]}'] = poly_data[f'{poly_data_cols_vars[q]}']\n",
    "            #new_df.drop(['Unnamed: 0'], axis=1, inplace=True)\n",
    "            meta_df = new_df[new_df[f'{poly_data_cols_medians[q]}'].isna() == False]\n",
    "\n",
    "            cols = [c for c in meta_df.columns if 'col' in c]\n",
    "            X = meta_df[cols]\n",
    "            y = meta_df[f'{poly_data_cols_medians[q]}']\n",
    "\n",
    "            # Split the data into training and testing sets\n",
    "            X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)\n",
    "\n",
    "            # Train the model on the training data\n",
    "            y_pred, coefs = model(X_train, X_test, y_train, y_test)\n",
    "\n",
    "            mse = mean_squared_error(y_test, y_pred)\n",
    "            r2 = r2_score(y_test, y_pred)\n",
    "            rmse = root_mean_squared_error(y_test, y_pred)\n",
    "            nmse = normalized_mse(y_test, y_pred)\n",
    "            mae = mean_absolute_error(y_test, y_pred)\n",
    "            mpe = mean_percentage_error(y_test, y_pred)\n",
    "\n",
    "            indexes_to_show = y_test.index\n",
    "            y_var = meta_df.loc[indexes_to_show, f'{poly_data_cols_vars[q]}']\n",
    "\n",
    "            tst = list(y_test)\n",
    "            prd = list(y_pred)\n",
    "            ndf = pd.DataFrame({'y_test': tst, 'y_pred': prd, 'var': y_var})\n",
    "\n",
    "            # Apply the custom metric to each row\n",
    "            ndf['metric'] = ndf.apply(custom_metric, axis=1)\n",
    "\n",
    "            # Calculate the overall metric (excluding rows where var <= 0)\n",
    "            filtered_df = ndf.dropna(subset=['metric'])\n",
    "            overall = filtered_df['metric'].mean()\n",
    "\n",
    "            data_list.append(data_size)\n",
    "            name_list.append(cleaned_column_name)\n",
    "            mse_list.append(mse)\n",
    "            r2_list.append(r2)\n",
    "            rmse_list.append(rmse)\n",
    "            nmse_list.append(nmse)\n",
    "            mae_list.append(mae)\n",
    "            mpe_list.append(mpe)\n",
    "            overall_list.append(overall)\n",
    "            \n",
    "            if coefs is not None:\n",
    "                with open(f'{model_directory}/{cleaned_column_name}_{model_name}_coefficients.txt', 'w') as f:\n",
    "                    f.write(f'Coefficients for {cleaned_column_name} ({model_name}):\\n')\n",
    "                    for feature, coef in zip(X.columns, coefs):\n",
    "                        f.write(f'{feature}: {coef}\\n')\n",
    "                \n",
    "        # Create a DataFrame for the current model\n",
    "        res_model = pd.DataFrame({\n",
    "            'Data Size': data_list,\n",
    "            'Name': name_list,\n",
    "            'MSE': mse_list,\n",
    "            'R2': r2_list,\n",
    "            'RMSE': rmse_list,\n",
    "            'NMSE': nmse_list,\n",
    "            'MAE': mae_list,\n",
    "            'MPE': mpe_list,\n",
    "            'Overall': overall_list\n",
    "        })\n",
    "\n",
    "        # Add a column for the model name\n",
    "        #res_model['Model'] = model_name\n",
    "\n",
    "        # Save the results to an Excel file\n",
    "        with pd.ExcelWriter(f'{model_directory}/results_{model_name}.xlsx', engine='openpyxl', mode='w') as writer:\n",
    "            res_model.to_excel(writer, sheet_name='Results', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "4b5e32d6",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Users\\ivan\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_coordinate_descent.py:628: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations, check the scale of the features or consider increasing regularisation. Duality gap: 1.368e+35, tolerance: 1.325e+33\n",
      "  model = cd_fast.enet_coordinate_descent(\n",
      "D:\\Users\\ivan\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_coordinate_descent.py:628: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations, check the scale of the features or consider increasing regularisation. Duality gap: 2.920e+04, tolerance: 9.752e+02\n",
      "  model = cd_fast.enet_coordinate_descent(\n",
      "D:\\Users\\ivan\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_coordinate_descent.py:628: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations, check the scale of the features or consider increasing regularisation. Duality gap: 1.168e+07, tolerance: 2.399e+05\n",
      "  model = cd_fast.enet_coordinate_descent(\n",
      "D:\\Users\\ivan\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_coordinate_descent.py:628: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations, check the scale of the features or consider increasing regularisation. Duality gap: 1.125e+07, tolerance: 6.103e+05\n",
      "  model = cd_fast.enet_coordinate_descent(\n",
      "D:\\Users\\ivan\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_coordinate_descent.py:628: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations, check the scale of the features or consider increasing regularisation. Duality gap: 3.980e+36, tolerance: 1.325e+33\n",
      "  model = cd_fast.enet_coordinate_descent(\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import joblib\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "from sklearn.linear_model import Lasso, ElasticNet\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.ensemble import BaggingRegressor, AdaBoostRegressor, RandomForestRegressor, GradientBoostingRegressor\n",
    "from sklearn.svm import SVR\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from xgboost import XGBRegressor\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score\n",
    "\n",
    "def train_knn(X_train, y_train):\n",
    "    model = KNeighborsRegressor(n_neighbors=5)\n",
    "    model.fit(X_train, y_train)\n",
    "    return model\n",
    "\n",
    "def train_lasso(X_train, y_train):\n",
    "    model = Lasso()\n",
    "    model.fit(X_train, y_train)\n",
    "    return model\n",
    "\n",
    "def train_elastic_net(X_train, y_train):\n",
    "    model = ElasticNet()\n",
    "    model.fit(X_train, y_train)\n",
    "    return model\n",
    "\n",
    "def train_decision_tree(X_train, y_train):\n",
    "    model = DecisionTreeRegressor()\n",
    "    model.fit(X_train, y_train)\n",
    "    return model\n",
    "\n",
    "def train_bagging(X_train, y_train):\n",
    "    model = BaggingRegressor()\n",
    "    model.fit(X_train, y_train)\n",
    "    return model\n",
    "\n",
    "def train_adaboost(X_train, y_train):\n",
    "    model = AdaBoostRegressor()\n",
    "    model.fit(X_train, y_train)\n",
    "    return model\n",
    "\n",
    "def train_xgboost(X_train, y_train):\n",
    "    model = XGBRegressor()\n",
    "    model.fit(X_train, y_train)\n",
    "    return model\n",
    "\n",
    "def train_svr(X_train, y_train):\n",
    "    model = SVR()\n",
    "    model.fit(X_train, y_train)\n",
    "    return model\n",
    "\n",
    "def train_gradient_boosting(X_train, y_train):\n",
    "    model = GradientBoostingRegressor()\n",
    "    model.fit(X_train, y_train)\n",
    "    return model\n",
    "\n",
    "def train_linear_regression(X_train, y_train):\n",
    "    model = LinearRegression()\n",
    "    model.fit(X_train, y_train)\n",
    "    return model\n",
    "\n",
    "def train_random_forest(X_train, y_train):\n",
    "    model = RandomForestRegressor()\n",
    "    model.fit(X_train, y_train)\n",
    "    return model\n",
    "\n",
    "# List of models and their names\n",
    "models = [\n",
    "    train_knn,\n",
    "    train_lasso,\n",
    "    train_elastic_net,\n",
    "    train_decision_tree,\n",
    "    train_bagging,\n",
    "    train_adaboost,\n",
    "    train_xgboost,\n",
    "    train_svr,\n",
    "    train_gradient_boosting,\n",
    "    train_linear_regression,\n",
    "    train_random_forest\n",
    "]\n",
    "\n",
    "model_names = [\n",
    "    'KNeighborsRegressor',\n",
    "    'Lasso',\n",
    "    'Elastic Net',\n",
    "    'Decision Tree',\n",
    "    'Bagging',\n",
    "    'AdaBoost',\n",
    "    'XGBoost',\n",
    "    'SVR',\n",
    "    'Gradient Boosting',\n",
    "    'Linear Regression',\n",
    "    'Random Forest'\n",
    "]\n",
    "\n",
    "\n",
    "# Iterate over models\n",
    "for train_model, model_name in zip(models, model_names):\n",
    "    \n",
    "    # Specify the directory and file name\n",
    "    model_directory = f'{model_name}'\n",
    "    \n",
    "    # Create the directory if it doesn't exist\n",
    "    os.makedirs(model_directory, exist_ok=True)\n",
    "        \n",
    "\n",
    "    for q in range(len(poly_data_cols_vars)):\n",
    "        if poly_data[f'{poly_data_cols_medians[q]}'].notna().sum() > 50:\n",
    "        \n",
    "            cleaned_column_name = poly_data_cols_medians[q].replace('_value_median', '')\n",
    "            data_size = poly_data[f'{poly_data_cols_medians[q]}'].notna().sum()\n",
    "\n",
    "            new_df[f'{poly_data_cols_medians[q]}'] = poly_data[f'{poly_data_cols_medians[q]}']\n",
    "            new_df[f'{poly_data_cols_vars[q]}'] = poly_data[f'{poly_data_cols_vars[q]}']\n",
    "            meta_df = new_df[new_df[f'{poly_data_cols_medians[q]}'].isna() == False]\n",
    "\n",
    "            cols = [c for c in meta_df.columns if 'col' in c]\n",
    "            X = meta_df[cols]\n",
    "            y = meta_df[f'{poly_data_cols_medians[q]}']\n",
    "\n",
    "            # Split the data into training and testing sets\n",
    "            X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)\n",
    "\n",
    "            # Train the model on the training data\n",
    "            trained_model = train_model(X_train, y_train)\n",
    "\n",
    "            # Save the model using joblib\n",
    "            joblib.dump(trained_model, f'{model_directory}/{cleaned_column_name}_{model_name}_model.joblib')\n",
    "\n",
    "  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "fcb945e9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Charactristic</th>\n",
       "      <th>Regressor</th>\n",
       "      <th>Max $R^2$</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Thermal conductivity</td>\n",
       "      <td>Gradient Boosting</td>\n",
       "      <td>0.127081</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Dynamic mechanical properties loss tangent</td>\n",
       "      <td>Gradient Boosting</td>\n",
       "      <td>0.524970</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Thermal decomposition temperature</td>\n",
       "      <td>Random Forest</td>\n",
       "      <td>0.725618</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Hansen parameter delta-h: hydrogen bonding</td>\n",
       "      <td>AdaBoost</td>\n",
       "      <td>0.588484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Flexural modulus</td>\n",
       "      <td>Random Forest</td>\n",
       "      <td>0.477938</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>63</td>\n",
       "      <td>Fiber tensile modulus</td>\n",
       "      <td>SVR</td>\n",
       "      <td>-0.011367</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>64</td>\n",
       "      <td>Crystallization kinetics r</td>\n",
       "      <td>AdaBoost</td>\n",
       "      <td>-0.199474</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>65</td>\n",
       "      <td>Contact angle</td>\n",
       "      <td>KNeighborsRegressor</td>\n",
       "      <td>0.241732</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66</th>\n",
       "      <td>66</td>\n",
       "      <td>Surface tension</td>\n",
       "      <td>Bagging</td>\n",
       "      <td>0.593994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>67</td>\n",
       "      <td>Specific volume</td>\n",
       "      <td>XGBoost</td>\n",
       "      <td>0.712654</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>68 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    Unnamed: 0                               Charactristic  \\\n",
       "0            0                        Thermal conductivity   \n",
       "1            1  Dynamic mechanical properties loss tangent   \n",
       "2            2           Thermal decomposition temperature   \n",
       "3            3  Hansen parameter delta-h: hydrogen bonding   \n",
       "4            4                            Flexural modulus   \n",
       "..         ...                                         ...   \n",
       "63          63                       Fiber tensile modulus   \n",
       "64          64                  Crystallization kinetics r   \n",
       "65          65                               Contact angle   \n",
       "66          66                             Surface tension   \n",
       "67          67                             Specific volume   \n",
       "\n",
       "              Regressor  Max $R^2$  \n",
       "0     Gradient Boosting   0.127081  \n",
       "1     Gradient Boosting   0.524970  \n",
       "2         Random Forest   0.725618  \n",
       "3              AdaBoost   0.588484  \n",
       "4         Random Forest   0.477938  \n",
       "..                  ...        ...  \n",
       "63                  SVR  -0.011367  \n",
       "64             AdaBoost  -0.199474  \n",
       "65  KNeighborsRegressor   0.241732  \n",
       "66              Bagging   0.593994  \n",
       "67              XGBoost   0.712654  \n",
       "\n",
       "[68 rows x 4 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bests = pd.read_excel('max_r2.xlsx')\n",
    "bests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "76fd2d5d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\ivan\\AppData\\Local\\Temp\\ipykernel_2024\\2868588031.py:31: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  new_df[f'{characteristic_name}'] = list(y_null)\n",
      "C:\\Users\\ivan\\AppData\\Local\\Temp\\ipykernel_2024\\2868588031.py:25: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  new_df[f'{characteristic_name}'] = poly_data[f'{characteristic_name}_value_median']\n",
      "C:\\Users\\ivan\\AppData\\Local\\Temp\\ipykernel_2024\\2868588031.py:26: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  new_df[f'polymer_name'] = poly_data[f'polymer_name']\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import joblib\n",
    "\n",
    "# Define the folder paths\n",
    "saved_models_folder = 'saved models'\n",
    "null_predicts_folder = 'predicts'\n",
    "\n",
    "# Iterate over characteristics and regressors from the bests DataFrame\n",
    "for index, row in bests.iterrows():\n",
    "    characteristic_name = row['Charactristic']\n",
    "    regressor_name = row['Regressor']\n",
    "\n",
    "    # Define the path to the model\n",
    "    model_path = os.path.join(saved_models_folder, regressor_name, f'{characteristic_name}_{regressor_name}_model.joblib')\n",
    "\n",
    "    # Check if the model file exists\n",
    "    if os.path.exists(model_path):\n",
    "        # Load the model\n",
    "        model = joblib.load(model_path)\n",
    "\n",
    "        # Get X_null (assuming it's already defined)\n",
    "        cols = [c for c in new_df.columns if 'col' in c]\n",
    "\n",
    "        new_df[f'{characteristic_name}'] = poly_data[f'{characteristic_name}_value_median']\n",
    "        new_df[f'polymer_name'] = poly_data[f'polymer_name']\n",
    "        new_df = new_df[new_df[f'{characteristic_name}'].isna() == True]  # Keep only rows with NaN in the characteristic\n",
    "        X_null = new_df[cols]\n",
    "\n",
    "        y_null = model.predict(X_null)\n",
    "        new_df[f'{characteristic_name}'] = list(y_null)\n",
    "        meta_df = new_df[['polymer_name', f'{characteristic_name}']]\n",
    "\n",
    "        # Save the result DataFrame to an individual Excel file\n",
    "        result_path = os.path.join(null_predicts_folder, f'{characteristic_name}_{regressor_name}_null_predictions.xlsx')\n",
    "        meta_df.to_excel(result_path, index=False)\n",
    "    else:\n",
    "        print(f'Model not found for {characteristic_name} and {regressor_name}.')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "32e59e0e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Thermal conductivity shape: None - (18232, 1026)\n",
      "Thermal conductivity shape: result - (18232, 2)\n",
      "Dynamic mechanical properties loss tangent shape: None - (18011, 1026)\n",
      "Dynamic mechanical properties loss tangent shape: result - (18011, 2)\n",
      "Thermal decomposition temperature shape: None - (11987, 1026)\n",
      "Thermal decomposition temperature shape: result - (11987, 2)\n",
      "Hansen parameter delta-h: hydrogen bonding shape: None - (18253, 1026)\n",
      "Hansen parameter delta-h: hydrogen bonding shape: result - (18253, 2)\n",
      "Flexural modulus shape: None - (18229, 1026)\n",
      "Flexural modulus shape: result - (18229, 2)\n",
      "Vicat softening temperature shape: None - (18230, 1026)\n",
      "Vicat softening temperature shape: result - (18230, 2)\n",
      "Dynamic mechanical properties loss modulus shape: None - (18109, 1026)\n",
      "Dynamic mechanical properties loss modulus shape: result - (18109, 2)\n",
      "Pvt relation specific volume shape: None - (18256, 1026)\n",
      "Pvt relation specific volume shape: result - (18256, 2)\n",
      "Tensile modulus shape: None - (17209, 1026)\n",
      "Tensile modulus shape: result - (17209, 2)\n",
      "Heat of fusion shape: None - (17689, 1026)\n",
      "Heat of fusion shape: result - (17689, 2)\n",
      "Water vapor transmission shape: None - (18239, 1026)\n",
      "Water vapor transmission shape: result - (18239, 2)\n",
      "Lc phase transition temperature shape: None - (17351, 1026)\n",
      "Lc phase transition temperature shape: result - (17351, 2)\n",
      "Dynamic flexural properties storage modulus shape: None - (18234, 1026)\n",
      "Dynamic flexural properties storage modulus shape: result - (18234, 2)\n",
      "Thermal decomposition weight loss shape: None - (13076, 1026)\n",
      "Thermal decomposition weight loss shape: result - (13076, 2)\n",
      "Melting temperature shape: None - (14468, 1026)\n",
      "Melting temperature shape: result - (14468, 2)\n",
      "Hansen parameter delta p polar shape: None - (18253, 1026)\n",
      "Hansen parameter delta p polar shape: result - (18253, 2)\n",
      "Volume resistivity shape: None - (17369, 1026)\n",
      "Volume resistivity shape: result - (17369, 2)\n",
      "Crystallization kinetics k shape: None - (18253, 1026)\n",
      "Crystallization kinetics k shape: result - (18253, 2)\n",
      "Heat of fusion mol conversion shape: None - (18087, 1026)\n",
      "Heat of fusion mol conversion shape: result - (18087, 2)\n",
      "Elongation at yield shape: None - (18228, 1026)\n",
      "Elongation at yield shape: result - (18228, 2)\n",
      "Dynamic shear properties loss tangent shape: None - (18206, 1026)\n",
      "Dynamic shear properties loss tangent shape: result - (18206, 2)\n",
      "Dielectric loss factor shape: None - (18001, 1026)\n",
      "Dielectric loss factor shape: result - (18001, 2)\n",
      "Cohesive energy density shape: None - (17988, 1026)\n",
      "Cohesive energy density shape: result - (17988, 2)\n",
      "Glass transition temperature shape: None - (10220, 1026)\n",
      "Glass transition temperature shape: result - (10220, 2)\n",
      "Dynamic shear properties storage modulus shape: None - (18171, 1026)\n",
      "Dynamic shear properties storage modulus shape: result - (18171, 2)\n",
      "Crystallization kinetics n shape: None - (18241, 1026)\n",
      "Crystallization kinetics n shape: result - (18241, 2)\n",
      "Heat of crystallization shape: None - (18188, 1026)\n",
      "Heat of crystallization shape: result - (18188, 2)\n",
      "Density shape: None - (16573, 1026)\n",
      "Density shape: result - (16573, 2)\n",
      "Water absorption shape: None - (17588, 1026)\n",
      "Water absorption shape: result - (17588, 2)\n",
      "Electric conductivity shape: None - (17304, 1026)\n",
      "Electric conductivity shape: result - (17304, 2)\n",
      "Elongation at break shape: None - (17173, 1026)\n",
      "Elongation at break shape: result - (17173, 2)\n",
      "Dynamic shear properties loss modulus shape: None - (18220, 1026)\n",
      "Dynamic shear properties loss modulus shape: result - (18220, 2)\n",
      "Flexural stress strength at break shape: None - (18241, 1026)\n",
      "Flexural stress strength at break shape: result - (18241, 2)\n",
      "Tensile stress strength at break shape: None - (17159, 1026)\n",
      "Tensile stress strength at break shape: result - (17159, 2)\n",
      "Isothermal weight loss time shape: None - (18084, 1026)\n",
      "Isothermal weight loss time shape: result - (18084, 2)\n",
      "Intrinsic viscosity eta shape: None - (16334, 1026)\n",
      "Intrinsic viscosity eta shape: result - (16334, 2)\n",
      "Fiber tensile elongation at break shape: None - (18251, 1026)\n",
      "Fiber tensile elongation at break shape: result - (18251, 2)\n",
      "Dielectric loss tangent shape: None - (18046, 1026)\n",
      "Dielectric loss tangent shape: result - (18046, 2)\n",
      "Isothermal weight loss temperature shape: None - (18039, 1026)\n",
      "Isothermal weight loss temperature shape: result - (18039, 2)\n",
      "Solubility parameter shape: None - (17988, 1026)\n",
      "Solubility parameter shape: result - (17988, 2)\n",
      "Deflection temperature under load hdt shape: None - (18213, 1026)\n",
      "Deflection temperature under load hdt shape: result - (18213, 2)\n",
      "Specific heat capacity cp shape: None - (18098, 1026)\n",
      "Specific heat capacity cp shape: result - (18098, 2)\n",
      "Fiber tensile stress strength at break shape: None - (18221, 1026)\n",
      "Fiber tensile stress strength at break shape: result - (18221, 2)\n",
      "Brittleness temperature shape: None - (18231, 1026)\n",
      "Brittleness temperature shape: result - (18231, 2)\n",
      "Tensile stress strength at yield shape: None - (18045, 1026)\n",
      "Tensile stress strength at yield shape: result - (18045, 2)\n",
      "Dynamic mechanical properties storage modulus shape: None - (17903, 1026)\n",
      "Dynamic mechanical properties storage modulus shape: result - (17903, 2)\n",
      "Dynamic flexural properties loss tanget shape: None - (18239, 1026)\n",
      "Dynamic flexural properties loss tanget shape: result - (18239, 2)\n",
      "Refractive index shape: None - (17627, 1026)\n",
      "Refractive index shape: result - (17627, 2)\n",
      "Gas diffusion coefficient d shape: None - (17868, 1026)\n",
      "Gas diffusion coefficient d shape: result - (17868, 2)\n",
      "Oxygen index shape: None - (18136, 1026)\n",
      "Oxygen index shape: result - (18136, 2)\n",
      "Pvt relation pressure shape: None - (18259, 1026)\n",
      "Pvt relation pressure shape: result - (18259, 2)\n",
      "Izod impact shape: None - (18259, 1026)\n",
      "Izod impact shape: result - (18259, 2)\n",
      "Gas permeability coefficient p shape: None - (17595, 1026)\n",
      "Gas permeability coefficient p shape: result - (17595, 2)\n",
      "Thermal diffusivity shape: None - (18232, 1026)\n",
      "Thermal diffusivity shape: result - (18232, 2)\n",
      "Pvt relation temperature shape: None - (18255, 1026)\n",
      "Pvt relation temperature shape: result - (18255, 2)\n",
      "Radius of gyration shape: None - (18192, 1026)\n",
      "Radius of gyration shape: result - (18192, 2)\n",
      "Crystallization kinetics half time of crystallization shape: None - (18240, 1026)\n",
      "Crystallization kinetics half time of crystallization shape: result - (18240, 2)\n",
      "Crystallization temperature shape: None - (17855, 1026)\n",
      "Crystallization temperature shape: result - (17855, 2)\n",
      "Gas solubility coefficient s shape: None - (18050, 1026)\n",
      "Gas solubility coefficient s shape: result - (18050, 2)\n",
      "Second virial coefficient shape: None - (18211, 1026)\n",
      "Second virial coefficient shape: result - (18211, 2)\n",
      "Hansen parameter delta-d: dispersion component shape: None - (18252, 1026)\n",
      "Hansen parameter delta-d: dispersion component shape: result - (18252, 2)\n",
      "Softening temperature shape: None - (17535, 1026)\n",
      "Softening temperature shape: result - (17535, 2)\n",
      "Dielectric constant ac shape: None - (17549, 1026)\n",
      "Dielectric constant ac shape: result - (17549, 2)\n",
      "Fiber tensile modulus shape: None - (18238, 1026)\n",
      "Fiber tensile modulus shape: result - (18238, 2)\n",
      "Crystallization kinetics r shape: None - (18259, 1026)\n",
      "Crystallization kinetics r shape: result - (18259, 2)\n",
      "Contact angle shape: None - (18057, 1026)\n",
      "Contact angle shape: result - (18057, 2)\n",
      "Surface tension shape: None - (17964, 1026)\n",
      "Surface tension shape: result - (17964, 2)\n",
      "Specific volume shape: None - (16573, 1026)\n",
      "Specific volume shape: result - (16573, 2)\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import joblib\n",
    "\n",
    "# Define the folder paths\n",
    "saved_models_folder = 'saved models'\n",
    "null_predicts_folder = 'predicts2'\n",
    "\n",
    "\n",
    "# Iterate over characteristics and regressors from the bests DataFrame\n",
    "for index, row in bests.iterrows():\n",
    "    characteristic_name = row['Charactristic']\n",
    "    regressor_name = row['Regressor']\n",
    "\n",
    "    # Define the path to the model\n",
    "    model_path = os.path.join(saved_models_folder, regressor_name, f'{characteristic_name}_{regressor_name}_model.joblib')\n",
    "\n",
    "    # Check if the model file exists\n",
    "    if os.path.exists(model_path):\n",
    "        # Load the model\n",
    "        model = joblib.load(model_path)\n",
    "\n",
    "        # Extract columns with 'col' in the name from new_df\n",
    "        cols = [c for c in new_df.columns if 'col' in c]\n",
    "\n",
    "        # Create an empty DataFrame meta_df\n",
    "        meta_df = pd.DataFrame()\n",
    "\n",
    "        # Add columns to meta_df using data from poly_data\n",
    "        #characteristic_name = 'Softening temperature'\n",
    "        meta_df[f'polymer_name'] = poly_data['polymer_name']\n",
    "        meta_df[f'{characteristic_name}'] = poly_data[f'{characteristic_name}_value_median']\n",
    "\n",
    "        # Concatenate columns from new_df to meta_df\n",
    "        meta_df = pd.concat([meta_df, new_df[cols]], axis=1)\n",
    "\n",
    "        # Filter rows where 'Softening temperature' is NaN\n",
    "        final = meta_df[meta_df[f'{characteristic_name}'].isna() == True][[f'{characteristic_name}', 'polymer_name', *cols]]\n",
    "        #final.drop([f'{characteristic_name}'], axis=1)\n",
    "        \n",
    "        print(f'{characteristic_name} shape: None - {final.shape}')\n",
    "        X_null = final[cols]\n",
    "        \n",
    "        y_null = model.predict(X_null)\n",
    "        final[f'{characteristic_name}'] = list(y_null)\n",
    "        \n",
    "        final_df = final[['polymer_name', f'{characteristic_name}']]\n",
    "\n",
    "        # Save the result DataFrame to an individual Excel file\n",
    "        result_path = os.path.join(null_predicts_folder, f'{characteristic_name}_{regressor_name}_null_predictions.xlsx')\n",
    "        print(f'{characteristic_name} shape: result - {final_df.shape}')\n",
    "        final_df.to_excel(result_path, index=False)\n",
    "    else:\n",
    "        print(f'Model not found for {characteristic_name} and {regressor_name}.')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "499c7ec5",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Extract columns with 'col' in the name from new_df\n",
    "cols = [c for c in new_df.columns if 'col' in c]\n",
    "\n",
    "# Create an empty DataFrame meta_df\n",
    "meta_df = pd.DataFrame()\n",
    "\n",
    "# Add columns to meta_df using data from poly_data\n",
    "characteristic_name = 'Softening temperature'\n",
    "meta_df[f'polymer_name'] = poly_data['polymer_name']\n",
    "meta_df[f'{characteristic_name}'] = poly_data[f'{characteristic_name}_value_median']\n",
    "\n",
    "# Concatenate columns from new_df to meta_df\n",
    "meta_df = pd.concat([meta_df, new_df[cols]], axis=1)\n",
    "\n",
    "# Filter rows where 'Softening temperature' is NaN\n",
    "final = meta_df[meta_df[f'{characteristic_name}'].isna() == True][[f'{characteristic_name}', 'polymer_name', *cols]]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "f24976db",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>polymer_name</th>\n",
       "      <th>col_0</th>\n",
       "      <th>col_1</th>\n",
       "      <th>col_2</th>\n",
       "      <th>col_3</th>\n",
       "      <th>col_4</th>\n",
       "      <th>col_5</th>\n",
       "      <th>col_6</th>\n",
       "      <th>col_7</th>\n",
       "      <th>col_8</th>\n",
       "      <th>...</th>\n",
       "      <th>col_1014</th>\n",
       "      <th>col_1015</th>\n",
       "      <th>col_1016</th>\n",
       "      <th>col_1017</th>\n",
       "      <th>col_1018</th>\n",
       "      <th>col_1019</th>\n",
       "      <th>col_1020</th>\n",
       "      <th>col_1021</th>\n",
       "      <th>col_1022</th>\n",
       "      <th>col_1023</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>poly(but-1-ene)</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>poly(hept-1-ene)</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>poly(5-methylhept-1-ene)</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>poly(non-1-ene)</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>poly(dec-1-ene)</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18307</th>\n",
       "      <td>poly{2,3-diphenyl-6-[6,7-diphenylnaphthalene-2...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18308</th>\n",
       "      <td>poly(1,1,2-trifluoro-2-{1,1,2,4,4,5,5-heptaflu...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18309</th>\n",
       "      <td>poly(naphthalene-4,5:8,1-tetrayl)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18310</th>\n",
       "      <td>poly((phenylarsanediyl){1-[(phenylarsanediyl)e...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18311</th>\n",
       "      <td>poly((phenylarsanediyl){1-[(phenylarsanediyl)e...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>17535 rows × 1025 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            polymer_name  col_0  col_1  col_2  \\\n",
       "2                                        poly(but-1-ene)      1      1      0   \n",
       "7                                       poly(hept-1-ene)      0      1      0   \n",
       "9                               poly(5-methylhept-1-ene)      0      1      0   \n",
       "10                                       poly(non-1-ene)      0      1      0   \n",
       "11                                       poly(dec-1-ene)      0      1      0   \n",
       "...                                                  ...    ...    ...    ...   \n",
       "18307  poly{2,3-diphenyl-6-[6,7-diphenylnaphthalene-2...      0      0      0   \n",
       "18308  poly(1,1,2-trifluoro-2-{1,1,2,4,4,5,5-heptaflu...      0      0      0   \n",
       "18309                  poly(naphthalene-4,5:8,1-tetrayl)      0      0      0   \n",
       "18310  poly((phenylarsanediyl){1-[(phenylarsanediyl)e...      0      0      0   \n",
       "18311  poly((phenylarsanediyl){1-[(phenylarsanediyl)e...      0      0      0   \n",
       "\n",
       "       col_3  col_4  col_5  col_6  col_7  col_8  ...  col_1014  col_1015  \\\n",
       "2          0      0      0      0      0      0  ...         0         0   \n",
       "7          0      0      0      0      0      0  ...         0         0   \n",
       "9          0      0      0      0      0      0  ...         0         0   \n",
       "10         0      0      0      0      0      0  ...         0         0   \n",
       "11         0      0      0      0      0      0  ...         0         0   \n",
       "...      ...    ...    ...    ...    ...    ...  ...       ...       ...   \n",
       "18307      0      0      0      0      0      0  ...         0         0   \n",
       "18308      0      1      0      0      0      0  ...         0         0   \n",
       "18309      0      0      0      0      0      0  ...         0         0   \n",
       "18310      0      0      0      0      1      0  ...         0         0   \n",
       "18311      0      0      0      0      1      0  ...         0         0   \n",
       "\n",
       "       col_1016  col_1017  col_1018  col_1019  col_1020  col_1021  col_1022  \\\n",
       "2             0         0         0         0         0         0         0   \n",
       "7             0         0         0         0         0         0         0   \n",
       "9             0         0         0         0         0         0         0   \n",
       "10            0         0         0         0         0         0         0   \n",
       "11            0         0         0         0         0         0         0   \n",
       "...         ...       ...       ...       ...       ...       ...       ...   \n",
       "18307         0         0         0         0         0         0         0   \n",
       "18308         0         0         0         0         0         0         0   \n",
       "18309         0         0         0         0         0         0         0   \n",
       "18310         0         0         0         0         0         0         0   \n",
       "18311         0         1         0         0         0         0         0   \n",
       "\n",
       "       col_1023  \n",
       "2             0  \n",
       "7             0  \n",
       "9             0  \n",
       "10            0  \n",
       "11            0  \n",
       "...         ...  \n",
       "18307         0  \n",
       "18308         0  \n",
       "18309         0  \n",
       "18310         0  \n",
       "18311         0  \n",
       "\n",
       "[17535 rows x 1025 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final.drop([f'{characteristic_name}'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c37ca749",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}