4_optunaEvaluation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Optuna Evaluator"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this notebook, the results of the Optuna Study are evaluated, visualized in plots and the best hyperparameters are extracted from the hyperparameter tuning with the help of a weighted sum. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Notebook Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import optuna\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import plotly.graph_objects as go\n",
    "import plotly.express as px\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.inspection import permutation_importance\n",
    "from sklearn.ensemble import RandomForestRegressor\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import Optuna Study"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "## Change study_name to the name of the run_name of the hyperparameter tuning notebook\n",
    "study_name = \"A6000_OptunaRun_2048\"\n",
    "\n",
    "storage_name = f\"sqlite:///optuna/{study_name}.db\"\n",
    "study = optuna.load_study(study_name=study_name, storage=storage_name)\n",
    "\n",
    "# load study as dataframe\n",
    "study_results_df = study.trials_dataframe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>number</th>\n",
       "      <th>values_0</th>\n",
       "      <th>values_1</th>\n",
       "      <th>values_2</th>\n",
       "      <th>datetime_start</th>\n",
       "      <th>datetime_complete</th>\n",
       "      <th>duration</th>\n",
       "      <th>params_learning_rate</th>\n",
       "      <th>params_lora_alpha_value</th>\n",
       "      <th>params_lora_r_value</th>\n",
       "      <th>params_num_train_epochs</th>\n",
       "      <th>params_warmup_steps</th>\n",
       "      <th>system_attrs_nsga3:generation</th>\n",
       "      <th>state</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>105</th>\n",
       "      <td>105</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2024-04-27 12:15:45.376032</td>\n",
       "      <td>2024-04-27 13:32:43.300230</td>\n",
       "      <td>0 days 01:16:57.924198</td>\n",
       "      <td>0.000033</td>\n",
       "      <td>16</td>\n",
       "      <td>5</td>\n",
       "      <td>8</td>\n",
       "      <td>921</td>\n",
       "      <td>NaN</td>\n",
       "      <td>FAIL</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     number  values_0  values_1  values_2             datetime_start  \\\n",
       "105     105       NaN       NaN       NaN 2024-04-27 12:15:45.376032   \n",
       "\n",
       "             datetime_complete               duration  params_learning_rate  \\\n",
       "105 2024-04-27 13:32:43.300230 0 days 01:16:57.924198              0.000033   \n",
       "\n",
       "     params_lora_alpha_value  params_lora_r_value  params_num_train_epochs  \\\n",
       "105                       16                    5                        8   \n",
       "\n",
       "     params_warmup_steps  system_attrs_nsga3:generation state  \n",
       "105                  921                            NaN  FAIL  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## remove failed trial from study\n",
    "study_results_df.loc[study_results_df.number == 105]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>number</th>\n",
       "      <th>values_0</th>\n",
       "      <th>values_1</th>\n",
       "      <th>values_2</th>\n",
       "      <th>datetime_start</th>\n",
       "      <th>datetime_complete</th>\n",
       "      <th>params_learning_rate</th>\n",
       "      <th>params_lora_alpha_value</th>\n",
       "      <th>params_lora_r_value</th>\n",
       "      <th>params_num_train_epochs</th>\n",
       "      <th>params_warmup_steps</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.047838</td>\n",
       "      <td>0.988572</td>\n",
       "      <td>0.049686</td>\n",
       "      <td>2024-04-09 14:26:04.516440</td>\n",
       "      <td>2024-04-09 17:44:53.662875</td>\n",
       "      <td>0.000013</td>\n",
       "      <td>16</td>\n",
       "      <td>8</td>\n",
       "      <td>3</td>\n",
       "      <td>932</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.043519</td>\n",
       "      <td>0.989534</td>\n",
       "      <td>0.047869</td>\n",
       "      <td>2024-04-09 17:44:53.704285</td>\n",
       "      <td>2024-04-09 22:05:30.327186</td>\n",
       "      <td>0.000031</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>1067</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0.054714</td>\n",
       "      <td>0.987621</td>\n",
       "      <td>0.062322</td>\n",
       "      <td>2024-04-09 22:05:30.359980</td>\n",
       "      <td>2024-04-10 00:26:13.613859</td>\n",
       "      <td>0.000013</td>\n",
       "      <td>16</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>1033</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0.051484</td>\n",
       "      <td>0.987701</td>\n",
       "      <td>0.051460</td>\n",
       "      <td>2024-04-10 00:26:13.647323</td>\n",
       "      <td>2024-04-10 03:45:18.480474</td>\n",
       "      <td>0.000016</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>725</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0.055351</td>\n",
       "      <td>0.986182</td>\n",
       "      <td>0.051536</td>\n",
       "      <td>2024-04-10 03:45:18.521370</td>\n",
       "      <td>2024-04-10 06:06:12.163497</td>\n",
       "      <td>0.000021</td>\n",
       "      <td>5</td>\n",
       "      <td>11</td>\n",
       "      <td>2</td>\n",
       "      <td>492</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>131</th>\n",
       "      <td>131</td>\n",
       "      <td>0.038389</td>\n",
       "      <td>0.990733</td>\n",
       "      <td>0.048764</td>\n",
       "      <td>2024-05-06 06:27:25.063577</td>\n",
       "      <td>2024-05-06 13:46:27.471692</td>\n",
       "      <td>0.000031</td>\n",
       "      <td>11</td>\n",
       "      <td>9</td>\n",
       "      <td>7</td>\n",
       "      <td>678</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>132</th>\n",
       "      <td>132</td>\n",
       "      <td>0.037077</td>\n",
       "      <td>0.990888</td>\n",
       "      <td>0.049814</td>\n",
       "      <td>2024-05-06 13:46:27.504890</td>\n",
       "      <td>2024-05-06 20:03:11.129133</td>\n",
       "      <td>0.000025</td>\n",
       "      <td>29</td>\n",
       "      <td>9</td>\n",
       "      <td>6</td>\n",
       "      <td>560</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>133</th>\n",
       "      <td>133</td>\n",
       "      <td>0.032777</td>\n",
       "      <td>0.991232</td>\n",
       "      <td>0.054029</td>\n",
       "      <td>2024-05-06 20:03:11.173119</td>\n",
       "      <td>2024-05-07 03:21:55.932267</td>\n",
       "      <td>0.000029</td>\n",
       "      <td>32</td>\n",
       "      <td>9</td>\n",
       "      <td>7</td>\n",
       "      <td>654</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>134</th>\n",
       "      <td>134</td>\n",
       "      <td>0.034727</td>\n",
       "      <td>0.991043</td>\n",
       "      <td>0.050602</td>\n",
       "      <td>2024-05-07 03:21:55.974912</td>\n",
       "      <td>2024-05-07 10:41:11.549722</td>\n",
       "      <td>0.000025</td>\n",
       "      <td>32</td>\n",
       "      <td>9</td>\n",
       "      <td>7</td>\n",
       "      <td>656</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135</th>\n",
       "      <td>135</td>\n",
       "      <td>0.034687</td>\n",
       "      <td>0.990929</td>\n",
       "      <td>0.047858</td>\n",
       "      <td>2024-05-07 10:41:11.589210</td>\n",
       "      <td>2024-05-07 16:57:42.105109</td>\n",
       "      <td>0.000030</td>\n",
       "      <td>30</td>\n",
       "      <td>10</td>\n",
       "      <td>6</td>\n",
       "      <td>624</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>136 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     number  values_0  values_1  values_2             datetime_start  \\\n",
       "0         0  0.047838  0.988572  0.049686 2024-04-09 14:26:04.516440   \n",
       "1         1  0.043519  0.989534  0.047869 2024-04-09 17:44:53.704285   \n",
       "2         2  0.054714  0.987621  0.062322 2024-04-09 22:05:30.359980   \n",
       "3         3  0.051484  0.987701  0.051460 2024-04-10 00:26:13.647323   \n",
       "4         4  0.055351  0.986182  0.051536 2024-04-10 03:45:18.521370   \n",
       "..      ...       ...       ...       ...                        ...   \n",
       "131     131  0.038389  0.990733  0.048764 2024-05-06 06:27:25.063577   \n",
       "132     132  0.037077  0.990888  0.049814 2024-05-06 13:46:27.504890   \n",
       "133     133  0.032777  0.991232  0.054029 2024-05-06 20:03:11.173119   \n",
       "134     134  0.034727  0.991043  0.050602 2024-05-07 03:21:55.974912   \n",
       "135     135  0.034687  0.990929  0.047858 2024-05-07 10:41:11.589210   \n",
       "\n",
       "             datetime_complete  params_learning_rate  params_lora_alpha_value  \\\n",
       "0   2024-04-09 17:44:53.662875              0.000013                       16   \n",
       "1   2024-04-09 22:05:30.327186              0.000031                        4   \n",
       "2   2024-04-10 00:26:13.613859              0.000013                       16   \n",
       "3   2024-04-10 03:45:18.480474              0.000016                        7   \n",
       "4   2024-04-10 06:06:12.163497              0.000021                        5   \n",
       "..                         ...                   ...                      ...   \n",
       "131 2024-05-06 13:46:27.471692              0.000031                       11   \n",
       "132 2024-05-06 20:03:11.129133              0.000025                       29   \n",
       "133 2024-05-07 03:21:55.932267              0.000029                       32   \n",
       "134 2024-05-07 10:41:11.549722              0.000025                       32   \n",
       "135 2024-05-07 16:57:42.105109              0.000030                       30   \n",
       "\n",
       "     params_lora_r_value  params_num_train_epochs  params_warmup_steps  \n",
       "0                      8                        3                  932  \n",
       "1                      6                        4                 1067  \n",
       "2                      4                        2                 1033  \n",
       "3                      6                        3                  725  \n",
       "4                     11                        2                  492  \n",
       "..                   ...                      ...                  ...  \n",
       "131                    9                        7                  678  \n",
       "132                    9                        6                  560  \n",
       "133                    9                        7                  654  \n",
       "134                    9                        7                  656  \n",
       "135                   10                        6                  624  \n",
       "\n",
       "[136 rows x 11 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# study_results_df = study_results_df.drop(labels='datetime_start', axis=1)\n",
    "# study_results_df = study_results_df,drop(labels='datetime_complete', axis=1)\n",
    "study_results_df = study_results_df.drop(labels='duration', axis=1)\n",
    "study_results_df = study_results_df.drop(labels='state', axis=1)\n",
    "study_results_df = study_results_df.drop(labels='system_attrs_nsga3:generation', axis=1)\n",
    "\n",
    "study_results_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "find_best_df = study_results_df.drop(labels='datetime_start', axis=1)\n",
    "find_best_df = find_best_df.drop(labels='datetime_complete', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Export whole Optuna results\n",
    "# study_results_df.round(6).to_csv(f\"OptunaResults.csv\", index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get Top-Runs per Objective"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataframes = []\n",
    "round_decimals = 6\n",
    "top_number = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "## LOSS ascending\n",
    "# sort df after values_0\n",
    "study_results_df_loss = study_results_df.sort_values(by='values_0', ascending=True).head(top_number).round(round_decimals)\n",
    "study_results_df_loss.head()\n",
    "dataframes.append(study_results_df_loss)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "## BLEU descending\n",
    "# sort df after values_0\n",
    "study_results_df_bleu = study_results_df.sort_values(by='values_1', ascending=False).head(top_number).round(round_decimals)\n",
    "study_results_df_bleu.head()\n",
    "dataframes.append(study_results_df_bleu)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "## ROUGE descending\n",
    "# sort df after values_0\n",
    "study_results_df_rouge = study_results_df.sort_values(by='values_2', ascending=False).head(top_number).round(round_decimals)\n",
    "study_results_df_rouge.head(30)\n",
    "dataframes.append(study_results_df_rouge)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Export Dataframes to CSV\n",
    "# for id, df in enumerate(dataframes):\n",
    "#     df.T.to_csv(f\"transposed_top{top_number}_objective_{id}.csv\", index=True)\n",
    "#     df.T.to_csv(f\"transposed_top{top_number}_objective_{id}_float.csv\", float_format=f\"%.{round_decimals}f\", index=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>number</th>\n",
       "      <th>values_0</th>\n",
       "      <th>values_1</th>\n",
       "      <th>values_2</th>\n",
       "      <th>datetime_start</th>\n",
       "      <th>datetime_complete</th>\n",
       "      <th>params_learning_rate</th>\n",
       "      <th>params_lora_alpha_value</th>\n",
       "      <th>params_lora_r_value</th>\n",
       "      <th>params_num_train_epochs</th>\n",
       "      <th>params_warmup_steps</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0.054714</td>\n",
       "      <td>0.987621</td>\n",
       "      <td>0.062322</td>\n",
       "      <td>2024-04-09 22:05:30.359980</td>\n",
       "      <td>2024-04-10 00:26:13.613859</td>\n",
       "      <td>0.000013</td>\n",
       "      <td>16</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>1033</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130</th>\n",
       "      <td>130</td>\n",
       "      <td>0.038864</td>\n",
       "      <td>0.990470</td>\n",
       "      <td>0.061189</td>\n",
       "      <td>2024-05-06 00:09:35.306738</td>\n",
       "      <td>2024-05-06 06:27:25.035615</td>\n",
       "      <td>0.000030</td>\n",
       "      <td>11</td>\n",
       "      <td>9</td>\n",
       "      <td>6</td>\n",
       "      <td>675</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>42</td>\n",
       "      <td>0.044770</td>\n",
       "      <td>0.989132</td>\n",
       "      <td>0.059830</td>\n",
       "      <td>2024-04-15 08:07:16.840544</td>\n",
       "      <td>2024-04-15 10:28:49.325439</td>\n",
       "      <td>0.000031</td>\n",
       "      <td>12</td>\n",
       "      <td>11</td>\n",
       "      <td>2</td>\n",
       "      <td>852</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>88</td>\n",
       "      <td>0.047143</td>\n",
       "      <td>0.988618</td>\n",
       "      <td>0.059485</td>\n",
       "      <td>2024-04-22 16:10:44.243927</td>\n",
       "      <td>2024-04-22 18:32:04.552728</td>\n",
       "      <td>0.000028</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>593</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>39</td>\n",
       "      <td>0.037798</td>\n",
       "      <td>0.990522</td>\n",
       "      <td>0.058879</td>\n",
       "      <td>2024-04-14 20:05:56.711789</td>\n",
       "      <td>2024-04-15 01:24:41.843686</td>\n",
       "      <td>0.000035</td>\n",
       "      <td>13</td>\n",
       "      <td>8</td>\n",
       "      <td>5</td>\n",
       "      <td>1098</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     number  values_0  values_1  values_2             datetime_start  \\\n",
       "2         2  0.054714  0.987621  0.062322 2024-04-09 22:05:30.359980   \n",
       "130     130  0.038864  0.990470  0.061189 2024-05-06 00:09:35.306738   \n",
       "42       42  0.044770  0.989132  0.059830 2024-04-15 08:07:16.840544   \n",
       "88       88  0.047143  0.988618  0.059485 2024-04-22 16:10:44.243927   \n",
       "39       39  0.037798  0.990522  0.058879 2024-04-14 20:05:56.711789   \n",
       "\n",
       "             datetime_complete  params_learning_rate  params_lora_alpha_value  \\\n",
       "2   2024-04-10 00:26:13.613859              0.000013                       16   \n",
       "130 2024-05-06 06:27:25.035615              0.000030                       11   \n",
       "42  2024-04-15 10:28:49.325439              0.000031                       12   \n",
       "88  2024-04-22 18:32:04.552728              0.000028                       10   \n",
       "39  2024-04-15 01:24:41.843686              0.000035                       13   \n",
       "\n",
       "     params_lora_r_value  params_num_train_epochs  params_warmup_steps  \n",
       "2                      4                        2                 1033  \n",
       "130                    9                        6                  675  \n",
       "42                    11                        2                  852  \n",
       "88                     8                        2                  593  \n",
       "39                     8                        5                 1098  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataframes[2].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df\n",
    "find_best_df\n",
    "\n",
    "find_best_df_ws = find_best_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Weighted Sum Calculation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In order to define which of the hyperparameter tuning trials was the best in the multi-object optimization and thus define the optimal hyperparameters, a weighted sum was defined. The objectives BLEU and LOSS were used for the weighted sum, whereby the LOSS was defined as an inverse loss $\\widetilde{L}(x)$ with $1-loss$.\n",
    "\n",
    "Concluding to the Formular:\n",
    "\n",
    "$$w_{\\text{Loss}} =  w_{\\text{BLEU}} = 0.5$$\n",
    "$$f(x) = w_{\\text{Loss}} \\cdot \\widetilde{L}(x) +  w_{\\text{BLEU}} \\cdot B(x)$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>number</th>\n",
       "      <th>values_0</th>\n",
       "      <th>values_1</th>\n",
       "      <th>values_2</th>\n",
       "      <th>params_learning_rate</th>\n",
       "      <th>params_lora_alpha_value</th>\n",
       "      <th>params_lora_r_value</th>\n",
       "      <th>params_num_train_epochs</th>\n",
       "      <th>params_warmup_steps</th>\n",
       "      <th>inverted_loss</th>\n",
       "      <th>weighted_sum</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>116</td>\n",
       "      <td>0.031224</td>\n",
       "      <td>0.991329</td>\n",
       "      <td>0.046125</td>\n",
       "      <td>0.000034</td>\n",
       "      <td>30</td>\n",
       "      <td>10</td>\n",
       "      <td>6</td>\n",
       "      <td>448</td>\n",
       "      <td>0.968776</td>\n",
       "      <td>0.980053</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>127</td>\n",
       "      <td>0.031686</td>\n",
       "      <td>0.991554</td>\n",
       "      <td>0.046529</td>\n",
       "      <td>0.000042</td>\n",
       "      <td>22</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>1044</td>\n",
       "      <td>0.968314</td>\n",
       "      <td>0.979934</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>124</td>\n",
       "      <td>0.032367</td>\n",
       "      <td>0.991696</td>\n",
       "      <td>0.047422</td>\n",
       "      <td>0.000034</td>\n",
       "      <td>24</td>\n",
       "      <td>10</td>\n",
       "      <td>9</td>\n",
       "      <td>974</td>\n",
       "      <td>0.967633</td>\n",
       "      <td>0.979664</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>108</td>\n",
       "      <td>0.031922</td>\n",
       "      <td>0.991245</td>\n",
       "      <td>0.048796</td>\n",
       "      <td>0.000037</td>\n",
       "      <td>18</td>\n",
       "      <td>5</td>\n",
       "      <td>8</td>\n",
       "      <td>911</td>\n",
       "      <td>0.968078</td>\n",
       "      <td>0.979662</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>102</td>\n",
       "      <td>0.032023</td>\n",
       "      <td>0.991332</td>\n",
       "      <td>0.048186</td>\n",
       "      <td>0.000037</td>\n",
       "      <td>16</td>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>917</td>\n",
       "      <td>0.967977</td>\n",
       "      <td>0.979654</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   number  values_0  values_1  values_2  params_learning_rate  \\\n",
       "0     116  0.031224  0.991329  0.046125              0.000034   \n",
       "1     127  0.031686  0.991554  0.046529              0.000042   \n",
       "2     124  0.032367  0.991696  0.047422              0.000034   \n",
       "3     108  0.031922  0.991245  0.048796              0.000037   \n",
       "4     102  0.032023  0.991332  0.048186              0.000037   \n",
       "\n",
       "   params_lora_alpha_value  params_lora_r_value  params_num_train_epochs  \\\n",
       "0                       30                   10                        6   \n",
       "1                       22                    5                        6   \n",
       "2                       24                   10                        9   \n",
       "3                       18                    5                        8   \n",
       "4                       16                    6                        8   \n",
       "\n",
       "   params_warmup_steps  inverted_loss  weighted_sum  \n",
       "0                  448       0.968776      0.980053  \n",
       "1                 1044       0.968314      0.979934  \n",
       "2                  974       0.967633      0.979664  \n",
       "3                  911       0.968078      0.979662  \n",
       "4                  917       0.967977      0.979654  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# add inverted Loss\n",
    "find_best_df_ws['inverted_loss'] = 1 - find_best_df_ws['values_0']\n",
    "\n",
    "find_best_df_ws['weighted_sum'] = (0.5 * find_best_df_ws['inverted_loss']) + (0.5 * find_best_df_ws['values_1'])\n",
    "\n",
    "# Sort the DataFrame based on the 'weighted_sum' column in descending order\n",
    "find_best_df_ws_sorted = find_best_df_ws.sort_values(by='weighted_sum', ascending=False)\n",
    "\n",
    "# If you want to reset the index after sorting\n",
    "find_best_df_ws_sorted = find_best_df_ws_sorted.reset_index(drop=True)\n",
    "\n",
    "# Display the sorted DataFrame\n",
    "find_best_df_ws_sorted.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hyperparameter Importance"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The importance of the hyperparameters for the outcome of the objectives was determined using a random forest regressor. \n",
    "\n",
    "The importance provides information about which hyperparameters should possibly no longer be defined as hyperparameters in future versions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>number</th>\n",
       "      <th>values_0</th>\n",
       "      <th>values_1</th>\n",
       "      <th>values_2</th>\n",
       "      <th>datetime_start</th>\n",
       "      <th>datetime_complete</th>\n",
       "      <th>duration</th>\n",
       "      <th>params_learning_rate</th>\n",
       "      <th>params_lora_alpha_value</th>\n",
       "      <th>params_lora_r_value</th>\n",
       "      <th>params_num_train_epochs</th>\n",
       "      <th>params_warmup_steps</th>\n",
       "      <th>system_attrs_nsga3:generation</th>\n",
       "      <th>state</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.047838</td>\n",
       "      <td>0.988572</td>\n",
       "      <td>0.049686</td>\n",
       "      <td>2024-04-09 14:26:04.516440</td>\n",
       "      <td>2024-04-09 17:44:53.662875</td>\n",
       "      <td>0 days 03:18:49.146435</td>\n",
       "      <td>0.000013</td>\n",
       "      <td>16</td>\n",
       "      <td>8</td>\n",
       "      <td>3</td>\n",
       "      <td>932</td>\n",
       "      <td>0.0</td>\n",
       "      <td>COMPLETE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.043519</td>\n",
       "      <td>0.989534</td>\n",
       "      <td>0.047869</td>\n",
       "      <td>2024-04-09 17:44:53.704285</td>\n",
       "      <td>2024-04-09 22:05:30.327186</td>\n",
       "      <td>0 days 04:20:36.622901</td>\n",
       "      <td>0.000031</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>1067</td>\n",
       "      <td>0.0</td>\n",
       "      <td>COMPLETE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0.054714</td>\n",
       "      <td>0.987621</td>\n",
       "      <td>0.062322</td>\n",
       "      <td>2024-04-09 22:05:30.359980</td>\n",
       "      <td>2024-04-10 00:26:13.613859</td>\n",
       "      <td>0 days 02:20:43.253879</td>\n",
       "      <td>0.000013</td>\n",
       "      <td>16</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>1033</td>\n",
       "      <td>0.0</td>\n",
       "      <td>COMPLETE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0.051484</td>\n",
       "      <td>0.987701</td>\n",
       "      <td>0.051460</td>\n",
       "      <td>2024-04-10 00:26:13.647323</td>\n",
       "      <td>2024-04-10 03:45:18.480474</td>\n",
       "      <td>0 days 03:19:04.833151</td>\n",
       "      <td>0.000016</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>725</td>\n",
       "      <td>0.0</td>\n",
       "      <td>COMPLETE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0.055351</td>\n",
       "      <td>0.986182</td>\n",
       "      <td>0.051536</td>\n",
       "      <td>2024-04-10 03:45:18.521370</td>\n",
       "      <td>2024-04-10 06:06:12.163497</td>\n",
       "      <td>0 days 02:20:53.642127</td>\n",
       "      <td>0.000021</td>\n",
       "      <td>5</td>\n",
       "      <td>11</td>\n",
       "      <td>2</td>\n",
       "      <td>492</td>\n",
       "      <td>0.0</td>\n",
       "      <td>COMPLETE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>131</th>\n",
       "      <td>131</td>\n",
       "      <td>0.038389</td>\n",
       "      <td>0.990733</td>\n",
       "      <td>0.048764</td>\n",
       "      <td>2024-05-06 06:27:25.063577</td>\n",
       "      <td>2024-05-06 13:46:27.471692</td>\n",
       "      <td>0 days 07:19:02.408115</td>\n",
       "      <td>0.000031</td>\n",
       "      <td>11</td>\n",
       "      <td>9</td>\n",
       "      <td>7</td>\n",
       "      <td>678</td>\n",
       "      <td>NaN</td>\n",
       "      <td>COMPLETE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>132</th>\n",
       "      <td>132</td>\n",
       "      <td>0.037077</td>\n",
       "      <td>0.990888</td>\n",
       "      <td>0.049814</td>\n",
       "      <td>2024-05-06 13:46:27.504890</td>\n",
       "      <td>2024-05-06 20:03:11.129133</td>\n",
       "      <td>0 days 06:16:43.624243</td>\n",
       "      <td>0.000025</td>\n",
       "      <td>29</td>\n",
       "      <td>9</td>\n",
       "      <td>6</td>\n",
       "      <td>560</td>\n",
       "      <td>NaN</td>\n",
       "      <td>COMPLETE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>133</th>\n",
       "      <td>133</td>\n",
       "      <td>0.032777</td>\n",
       "      <td>0.991232</td>\n",
       "      <td>0.054029</td>\n",
       "      <td>2024-05-06 20:03:11.173119</td>\n",
       "      <td>2024-05-07 03:21:55.932267</td>\n",
       "      <td>0 days 07:18:44.759148</td>\n",
       "      <td>0.000029</td>\n",
       "      <td>32</td>\n",
       "      <td>9</td>\n",
       "      <td>7</td>\n",
       "      <td>654</td>\n",
       "      <td>NaN</td>\n",
       "      <td>COMPLETE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>134</th>\n",
       "      <td>134</td>\n",
       "      <td>0.034727</td>\n",
       "      <td>0.991043</td>\n",
       "      <td>0.050602</td>\n",
       "      <td>2024-05-07 03:21:55.974912</td>\n",
       "      <td>2024-05-07 10:41:11.549722</td>\n",
       "      <td>0 days 07:19:15.574810</td>\n",
       "      <td>0.000025</td>\n",
       "      <td>32</td>\n",
       "      <td>9</td>\n",
       "      <td>7</td>\n",
       "      <td>656</td>\n",
       "      <td>NaN</td>\n",
       "      <td>COMPLETE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135</th>\n",
       "      <td>135</td>\n",
       "      <td>0.034687</td>\n",
       "      <td>0.990929</td>\n",
       "      <td>0.047858</td>\n",
       "      <td>2024-05-07 10:41:11.589210</td>\n",
       "      <td>2024-05-07 16:57:42.105109</td>\n",
       "      <td>0 days 06:16:30.515899</td>\n",
       "      <td>0.000030</td>\n",
       "      <td>30</td>\n",
       "      <td>10</td>\n",
       "      <td>6</td>\n",
       "      <td>624</td>\n",
       "      <td>NaN</td>\n",
       "      <td>COMPLETE</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>135 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     number  values_0  values_1  values_2             datetime_start  \\\n",
       "0         0  0.047838  0.988572  0.049686 2024-04-09 14:26:04.516440   \n",
       "1         1  0.043519  0.989534  0.047869 2024-04-09 17:44:53.704285   \n",
       "2         2  0.054714  0.987621  0.062322 2024-04-09 22:05:30.359980   \n",
       "3         3  0.051484  0.987701  0.051460 2024-04-10 00:26:13.647323   \n",
       "4         4  0.055351  0.986182  0.051536 2024-04-10 03:45:18.521370   \n",
       "..      ...       ...       ...       ...                        ...   \n",
       "131     131  0.038389  0.990733  0.048764 2024-05-06 06:27:25.063577   \n",
       "132     132  0.037077  0.990888  0.049814 2024-05-06 13:46:27.504890   \n",
       "133     133  0.032777  0.991232  0.054029 2024-05-06 20:03:11.173119   \n",
       "134     134  0.034727  0.991043  0.050602 2024-05-07 03:21:55.974912   \n",
       "135     135  0.034687  0.990929  0.047858 2024-05-07 10:41:11.589210   \n",
       "\n",
       "             datetime_complete               duration  params_learning_rate  \\\n",
       "0   2024-04-09 17:44:53.662875 0 days 03:18:49.146435              0.000013   \n",
       "1   2024-04-09 22:05:30.327186 0 days 04:20:36.622901              0.000031   \n",
       "2   2024-04-10 00:26:13.613859 0 days 02:20:43.253879              0.000013   \n",
       "3   2024-04-10 03:45:18.480474 0 days 03:19:04.833151              0.000016   \n",
       "4   2024-04-10 06:06:12.163497 0 days 02:20:53.642127              0.000021   \n",
       "..                         ...                    ...                   ...   \n",
       "131 2024-05-06 13:46:27.471692 0 days 07:19:02.408115              0.000031   \n",
       "132 2024-05-06 20:03:11.129133 0 days 06:16:43.624243              0.000025   \n",
       "133 2024-05-07 03:21:55.932267 0 days 07:18:44.759148              0.000029   \n",
       "134 2024-05-07 10:41:11.549722 0 days 07:19:15.574810              0.000025   \n",
       "135 2024-05-07 16:57:42.105109 0 days 06:16:30.515899              0.000030   \n",
       "\n",
       "     params_lora_alpha_value  params_lora_r_value  params_num_train_epochs  \\\n",
       "0                         16                    8                        3   \n",
       "1                          4                    6                        4   \n",
       "2                         16                    4                        2   \n",
       "3                          7                    6                        3   \n",
       "4                          5                   11                        2   \n",
       "..                       ...                  ...                      ...   \n",
       "131                       11                    9                        7   \n",
       "132                       29                    9                        6   \n",
       "133                       32                    9                        7   \n",
       "134                       32                    9                        7   \n",
       "135                       30                   10                        6   \n",
       "\n",
       "     params_warmup_steps  system_attrs_nsga3:generation     state  \n",
       "0                    932                            0.0  COMPLETE  \n",
       "1                   1067                            0.0  COMPLETE  \n",
       "2                   1033                            0.0  COMPLETE  \n",
       "3                    725                            0.0  COMPLETE  \n",
       "4                    492                            0.0  COMPLETE  \n",
       "..                   ...                            ...       ...  \n",
       "131                  678                            NaN  COMPLETE  \n",
       "132                  560                            NaN  COMPLETE  \n",
       "133                  654                            NaN  COMPLETE  \n",
       "134                  656                            NaN  COMPLETE  \n",
       "135                  624                            NaN  COMPLETE  \n",
       "\n",
       "[135 rows x 14 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_imp_self = study.trials_dataframe()\n",
    "\n",
    "df_imp_self.drop(105, inplace=True)\n",
    "\n",
    "df_imp_self"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "config": {
        "plotlyServerURL": "https://plot.ly"
       },
       "data": [
        {
         "name": "Loss",
         "textposition": "auto",
         "texttemplate": "%{y:.2f}",
         "type": "bar",
         "x": [
          "learning_rate",
          "lora_alpha_value",
          "lora_r_value",
          "num_train_epochs",
          "warmup_steps"
         ],
         "y": [
          0.4803350543170682,
          0.08515628214514374,
          0.004108545901157112,
          0.8771306691877394,
          0.0036950947550209445
         ]
        },
        {
         "name": "BLEU",
         "textposition": "auto",
         "texttemplate": "%{y:.2f}",
         "type": "bar",
         "x": [
          "learning_rate",
          "lora_alpha_value",
          "lora_r_value",
          "num_train_epochs",
          "warmup_steps"
         ],
         "y": [
          0.3240187657494472,
          0.08639088658685437,
          0.005933680068393399,
          0.980668126417006,
          0.0033195325637178086
         ]
        },
        {
         "name": "ROUGE",
         "textposition": "auto",
         "texttemplate": "%{y:.2f}",
         "type": "bar",
         "x": [
          "learning_rate",
          "lora_alpha_value",
          "lora_r_value",
          "num_train_epochs",
          "warmup_steps"
         ],
         "y": [
          0.42145875653931303,
          0.4573817375663249,
          0.23816793300943728,
          0.5033550952529635,
          0.2393546895497236
         ]
        }
       ],
       "layout": {
        "barmode": "group",
        "legend": {
         "title": {
          "text": "Objectives"
         }
        },
        "template": {
         "data": {
          "bar": [
           {
            "error_x": {
             "color": "#2a3f5f"
            },
            "error_y": {
             "color": "#2a3f5f"
            },
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "bar"
           }
          ],
          "barpolar": [
           {
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "barpolar"
           }
          ],
          "carpet": [
           {
            "aaxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "baxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "type": "carpet"
           }
          ],
          "choropleth": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "choropleth"
           }
          ],
          "contour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "contour"
           }
          ],
          "contourcarpet": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "contourcarpet"
           }
          ],
          "heatmap": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmap"
           }
          ],
          "heatmapgl": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmapgl"
           }
          ],
          "histogram": [
           {
            "marker": {
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "histogram"
           }
          ],
          "histogram2d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2d"
           }
          ],
          "histogram2dcontour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2dcontour"
           }
          ],
          "mesh3d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "mesh3d"
           }
          ],
          "parcoords": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "parcoords"
           }
          ],
          "pie": [
           {
            "automargin": true,
            "type": "pie"
           }
          ],
          "scatter": [
           {
            "fillpattern": {
             "fillmode": "overlay",
             "size": 10,
             "solidity": 0.2
            },
            "type": "scatter"
           }
          ],
          "scatter3d": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter3d"
           }
          ],
          "scattercarpet": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattercarpet"
           }
          ],
          "scattergeo": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergeo"
           }
          ],
          "scattergl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergl"
           }
          ],
          "scattermapbox": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattermapbox"
           }
          ],
          "scatterpolar": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolar"
           }
          ],
          "scatterpolargl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolargl"
           }
          ],
          "scatterternary": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterternary"
           }
          ],
          "surface": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "surface"
           }
          ],
          "table": [
           {
            "cells": {
             "fill": {
              "color": "#EBF0F8"
             },
             "line": {
              "color": "white"
             }
            },
            "header": {
             "fill": {
              "color": "#C8D4E3"
             },
             "line": {
              "color": "white"
             }
            },
            "type": "table"
           }
          ]
         },
         "layout": {
          "annotationdefaults": {
           "arrowcolor": "#2a3f5f",
           "arrowhead": 0,
           "arrowwidth": 1
          },
          "autotypenumbers": "strict",
          "coloraxis": {
           "colorbar": {
            "outlinewidth": 0,
            "ticks": ""
           }
          },
          "colorscale": {
           "diverging": [
            [
             0,
             "#8e0152"
            ],
            [
             0.1,
             "#c51b7d"
            ],
            [
             0.2,
             "#de77ae"
            ],
            [
             0.3,
             "#f1b6da"
            ],
            [
             0.4,
             "#fde0ef"
            ],
            [
             0.5,
             "#f7f7f7"
            ],
            [
             0.6,
             "#e6f5d0"
            ],
            [
             0.7,
             "#b8e186"
            ],
            [
             0.8,
             "#7fbc41"
            ],
            [
             0.9,
             "#4d9221"
            ],
            [
             1,
             "#276419"
            ]
           ],
           "sequential": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ],
           "sequentialminus": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ]
          },
          "colorway": [
           "#636efa",
           "#EF553B",
           "#00cc96",
           "#ab63fa",
           "#FFA15A",
           "#19d3f3",
           "#FF6692",
           "#B6E880",
           "#FF97FF",
           "#FECB52"
          ],
          "font": {
           "color": "#2a3f5f"
          },
          "geo": {
           "bgcolor": "white",
           "lakecolor": "white",
           "landcolor": "#E5ECF6",
           "showlakes": true,
           "showland": true,
           "subunitcolor": "white"
          },
          "hoverlabel": {
           "align": "left"
          },
          "hovermode": "closest",
          "mapbox": {
           "style": "light"
          },
          "paper_bgcolor": "white",
          "plot_bgcolor": "#E5ECF6",
          "polar": {
           "angularaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "radialaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "scene": {
           "xaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "yaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "zaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           }
          },
          "shapedefaults": {
           "line": {
            "color": "#2a3f5f"
           }
          },
          "ternary": {
           "aaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "baxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "caxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "title": {
           "x": 0.05
          },
          "xaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          },
          "yaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          }
         }
        },
        "title": {
         "text": "Parameter Importance for Multiple Objectives"
        },
        "xaxis": {
         "title": {
          "text": "Hyperparameter"
         }
        },
        "yaxis": {
         "title": {
          "text": "Permutation Importance"
         }
        }
       }
      }
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df_imp_self = study.trials_dataframe()\n",
    "df_imp_self.drop(105, inplace=True)\n",
    "\n",
    "\n",
    "\n",
    "def plot_param_importance(df, objective_cols=['values_0'], objective_names=['VALUE_NAME']):\n",
    "    combined_importance_df = pd.DataFrame(columns=['Parameter'])\n",
    "    \n",
    "    for objective_col in objective_cols:\n",
    "        # Extract the relevant columns\n",
    "        param_cols = [col for col in df.columns if col.startswith('params_')]\n",
    "        X = df[param_cols]\n",
    "        y = df[objective_col]\n",
    "\n",
    "        # Train a Random Forest model\n",
    "        model = RandomForestRegressor(n_estimators=100, random_state=42)\n",
    "        model.fit(X, y)\n",
    "\n",
    "        # Calculate permutation importances\n",
    "        result = permutation_importance(model, X, y, n_repeats=10, random_state=42, n_jobs=-1)\n",
    "\n",
    "        # Create a DataFrame for plotting\n",
    "        importance_df = pd.DataFrame({\n",
    "            'Parameter': X.columns,\n",
    "            f'Importance_{objective_col}': result.importances_mean\n",
    "        })\n",
    "\n",
    "        # Sort by importance\n",
    "        importance_df = importance_df.sort_values(by='Parameter')\n",
    "\n",
    "        # Rename parameters\n",
    "        importance_df['Parameter'] = importance_df['Parameter'].replace({\n",
    "            'params_lora_r_value': 'lora_r_value',\n",
    "            'params_lora_alpha_value': 'lora_alpha_value',\n",
    "            'params_warmup_steps': 'warmup_steps',\n",
    "            'params_num_train_epochs': 'num_train_epochs',\n",
    "            'params_learning_rate': 'learning_rate'\n",
    "        })\n",
    "\n",
    "        # Merge with combined_importance_df\n",
    "        combined_importance_df = pd.merge(combined_importance_df, importance_df, on='Parameter', how='outer')\n",
    "\n",
    "    # Plot using Plotly\n",
    "    fig = go.Figure()\n",
    "\n",
    "    for id, objective_col in enumerate(objective_cols):\n",
    "        fig.add_trace(go.Bar(x=combined_importance_df['Parameter'], y=combined_importance_df[f'Importance_{objective_col}'], name=f'{objective_names[id]}'))\n",
    "\n",
    "    fig.update_layout(barmode='group', title='Parameter Importance for Multiple Objectives', xaxis_title='Hyperparameter', yaxis_title='Permutation Importance')\n",
    "    fig.update_layout(legend_title=\"Objectives\")\n",
    "    fig.update_traces(texttemplate='%{y:.2f}', textposition='auto')\n",
    "\n",
    "    ## Export Image\n",
    "    # fig.write_image(\"./plotly/ParameterImportanceforMultipleObjectives_800_400_5.png\", width=800, height=400, scale=5)\n",
    "    \n",
    "    \n",
    "    fig.show()\n",
    "# (Importance calculated using permutation importance with random forrest regressor)\n",
    "# Example usage\n",
    "# Assuming 'df' is your DataFrame loaded from the study\n",
    "# df = pd.read_csv('your_dataframe.csv')  # Load your DataFrame\n",
    "plot_param_importance(df_imp_self, objective_cols=['values_0', 'values_1', 'values_2'], objective_names=['Loss','BLEU','ROUGE'])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**More evaluations and overviews can be found in the Optuna Dashboard. See README.md for further instructions.**"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}