examples/hposuite_demo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Notebook with Minimal example to run hposuite"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_keys(['Ex_Functional_Bench', 'Ex_Surrogate_Bench', 'mfh3_bad', 'mfh6_bad', 'mfh3_good', 'mfh6_good', 'mfh3_moderate', 'mfh6_moderate', 'mfh3_terrible', 'mfh6_terrible'])\n",
      "dict_keys(['RandomSearch', 'RandomSearchWithPriors', 'Ex_MO_Opt', 'Ex_Multifidelity_Opt', 'Ex_Blackbox_Opt'])\n"
     ]
    }
   ],
   "source": [
    "# It is assumed that the datadir is hposuite/data\n",
    "\n",
    "from lib.benchmarks import BENCHMARKS\n",
    "from lib.optimizers import OPTIMIZERS\n",
    "\n",
    "print(BENCHMARKS.keys())\n",
    "print(OPTIMIZERS.keys())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Running Random Search on a Functional Benchmark (eg: ACKLEY)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "func_benchmark = BENCHMARKS[\"Ex_Functional_Bench\"]\n",
    "rs = OPTIMIZERS[\"RandomSearch\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Printing Ackley's description"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BenchmarkDescription(name='Ex_Functional_Bench', config_space=Configuration space object:\n",
       "  Hyperparameters:\n",
       "    x0, Type: UniformFloat, Range: [-32.768, 32.768], Default: 0.0\n",
       "    x1, Type: UniformFloat, Range: [-32.768, 32.768], Default: 0.0\n",
       ", load=functools.partial(<bound method FunctionalBenchmark.load of <hpoglue.benchmark.FunctionalBenchmark object at 0x6ffcb9dae110>>), metrics={'y': Measure(minimize=True, kind=<Kind.METRIC: 'metric'>, bounds=(0.0, inf))}, test_metrics=None, costs=None, fidelities=None, has_conditionals=False, is_tabular=False, env=Env(name='empty'), mem_req_mb=1024, predefined_points={'min': Config(config_id='min', values={'x0': 0.0, 'x1': 0.0}, description='This point yields a global optimum of y:0.0')}, extra={})"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "func_benchmark.desc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Run RandomSearch on Ackley using hposuite"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:hposuite.study:Generated 5 runs\n",
      "INFO:hposuite.study:Created study at /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_rs_func\n",
      "INFO:hposuite.study:Dumped experiments to /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_rs_func/dump_all.txt\n",
      "INFO:hposuite.study:Overwrite flag is set to True. Existing results will be overwritten!\n",
      "INFO:hposuite.study:Running 5 experiments sequentially\n",
      "INFO:hposuite.study:Running experiment 1/5\n",
      "INFO:hposuite.run:Overwriting optimizer=RandomSearch.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=383329928 in `state=<State.COMPLETE: 'COMPLETE'>` at /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_rs_func/optimizer=RandomSearch.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=383329928.\n",
      "WARNING:hposuite.run:Continuations are not supported for the optimizer: <class 'lib.optimizers.random_search.RandomSearch'>. Ignoring continuations=True.\n",
      "INFO:hpoglue._run:COMPLETED running optimizer=RandomSearch.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=383329928\n",
      "INFO:hposuite.run:Results dumped at /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_rs_func/optimizer=RandomSearch.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=383329928/optimizer=RandomSearch.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=383329928.parquet\n",
      "INFO:hposuite.study:Running experiment 2/5\n",
      "INFO:hposuite.run:Overwriting optimizer=RandomSearch.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=3324115917 in `state=<State.COMPLETE: 'COMPLETE'>` at /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_rs_func/optimizer=RandomSearch.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=3324115917.\n",
      "WARNING:hposuite.run:Continuations are not supported for the optimizer: <class 'lib.optimizers.random_search.RandomSearch'>. Ignoring continuations=True.\n",
      "INFO:hpoglue._run:COMPLETED running optimizer=RandomSearch.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=3324115917\n",
      "INFO:hposuite.run:Results dumped at /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_rs_func/optimizer=RandomSearch.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=3324115917/optimizer=RandomSearch.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=3324115917.parquet\n",
      "INFO:hposuite.study:Running experiment 3/5\n",
      "INFO:hposuite.run:Overwriting optimizer=RandomSearch.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=2811363265 in `state=<State.COMPLETE: 'COMPLETE'>` at /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_rs_func/optimizer=RandomSearch.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=2811363265.\n",
      "WARNING:hposuite.run:Continuations are not supported for the optimizer: <class 'lib.optimizers.random_search.RandomSearch'>. Ignoring continuations=True.\n",
      "INFO:hpoglue._run:COMPLETED running optimizer=RandomSearch.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=2811363265\n",
      "INFO:hposuite.run:Results dumped at /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_rs_func/optimizer=RandomSearch.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=2811363265/optimizer=RandomSearch.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=2811363265.parquet\n",
      "INFO:hposuite.study:Running experiment 4/5\n",
      "INFO:hposuite.run:Overwriting optimizer=RandomSearch.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=1884968545 in `state=<State.COMPLETE: 'COMPLETE'>` at /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_rs_func/optimizer=RandomSearch.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=1884968545.\n",
      "WARNING:hposuite.run:Continuations are not supported for the optimizer: <class 'lib.optimizers.random_search.RandomSearch'>. Ignoring continuations=True.\n",
      "INFO:hpoglue._run:COMPLETED running optimizer=RandomSearch.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=1884968545\n",
      "INFO:hposuite.run:Results dumped at /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_rs_func/optimizer=RandomSearch.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=1884968545/optimizer=RandomSearch.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=1884968545.parquet\n",
      "INFO:hposuite.study:Running experiment 5/5\n",
      "INFO:hposuite.run:Overwriting optimizer=RandomSearch.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=1859786276 in `state=<State.COMPLETE: 'COMPLETE'>` at /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_rs_func/optimizer=RandomSearch.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=1859786276.\n",
      "WARNING:hposuite.run:Continuations are not supported for the optimizer: <class 'lib.optimizers.random_search.RandomSearch'>. Ignoring continuations=True.\n",
      "INFO:hpoglue._run:COMPLETED running optimizer=RandomSearch.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=1859786276\n",
      "INFO:hposuite.run:Results dumped at /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_rs_func/optimizer=RandomSearch.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=1859786276/optimizer=RandomSearch.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=1859786276.parquet\n",
      "INFO:hposuite.study:Completed study with 5 runs\n"
     ]
    }
   ],
   "source": [
    "from hposuite import create_study\n",
    "\n",
    "study = create_study(\n",
    "    name=\"hposuite_demo_rs_func\",\n",
    "    output_dir=\"example-outputs\",\n",
    "    optimizers=rs,\n",
    "    benchmarks=func_benchmark,\n",
    "    num_seeds=5,\n",
    "    budget=100,\n",
    ")\n",
    "\n",
    "study.optimize(overwrite=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Fetching and Displaying the dataframe from the saved parquet files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>config.id</th>\n",
       "      <th>query.id</th>\n",
       "      <th>result.budget_cost</th>\n",
       "      <th>result.budget_used_total</th>\n",
       "      <th>query.fidelity.count</th>\n",
       "      <th>result.objective.1.value</th>\n",
       "      <th>result.continuations_cost.1</th>\n",
       "      <th>run.name</th>\n",
       "      <th>problem.name</th>\n",
       "      <th>problem.objective.count</th>\n",
       "      <th>...</th>\n",
       "      <th>problem.objective.1.min</th>\n",
       "      <th>problem.objective.1.max</th>\n",
       "      <th>problem.fidelity.count</th>\n",
       "      <th>problem.cost.count</th>\n",
       "      <th>problem.benchmark</th>\n",
       "      <th>problem.budget.kind</th>\n",
       "      <th>problem.budget.total</th>\n",
       "      <th>run.seed</th>\n",
       "      <th>run.opt.name</th>\n",
       "      <th>run.opt.hp_str</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>19.382502</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>optimizer=RandomSearch.benchmark=Ex_Functional...</td>\n",
       "      <td>optimizer=RandomSearch.benchmark=Ex_Functional...</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>inf</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Ex_Functional_Bench</td>\n",
       "      <td>TrialBudget</td>\n",
       "      <td>100</td>\n",
       "      <td>383329928</td>\n",
       "      <td>RandomSearch</td>\n",
       "      <td>default</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>18.551447</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>optimizer=RandomSearch.benchmark=Ex_Functional...</td>\n",
       "      <td>optimizer=RandomSearch.benchmark=Ex_Functional...</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>inf</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Ex_Functional_Bench</td>\n",
       "      <td>TrialBudget</td>\n",
       "      <td>100</td>\n",
       "      <td>383329928</td>\n",
       "      <td>RandomSearch</td>\n",
       "      <td>default</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>22.082396</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>optimizer=RandomSearch.benchmark=Ex_Functional...</td>\n",
       "      <td>optimizer=RandomSearch.benchmark=Ex_Functional...</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>inf</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Ex_Functional_Bench</td>\n",
       "      <td>TrialBudget</td>\n",
       "      <td>100</td>\n",
       "      <td>383329928</td>\n",
       "      <td>RandomSearch</td>\n",
       "      <td>default</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>21.5539</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>optimizer=RandomSearch.benchmark=Ex_Functional...</td>\n",
       "      <td>optimizer=RandomSearch.benchmark=Ex_Functional...</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>inf</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Ex_Functional_Bench</td>\n",
       "      <td>TrialBudget</td>\n",
       "      <td>100</td>\n",
       "      <td>383329928</td>\n",
       "      <td>RandomSearch</td>\n",
       "      <td>default</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>20.895702</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>optimizer=RandomSearch.benchmark=Ex_Functional...</td>\n",
       "      <td>optimizer=RandomSearch.benchmark=Ex_Functional...</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>inf</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Ex_Functional_Bench</td>\n",
       "      <td>TrialBudget</td>\n",
       "      <td>100</td>\n",
       "      <td>383329928</td>\n",
       "      <td>RandomSearch</td>\n",
       "      <td>default</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  config.id query.id  result.budget_cost  result.budget_used_total  \\\n",
       "0         1        1                   1                         1   \n",
       "1         2        2                   1                         2   \n",
       "2         3        3                   1                         3   \n",
       "3         4        4                   1                         4   \n",
       "4         5        5                   1                         5   \n",
       "\n",
       "   query.fidelity.count  result.objective.1.value  \\\n",
       "0                     0                 19.382502   \n",
       "1                     0                 18.551447   \n",
       "2                     0                 22.082396   \n",
       "3                     0                   21.5539   \n",
       "4                     0                 20.895702   \n",
       "\n",
       "   result.continuations_cost.1  \\\n",
       "0                         <NA>   \n",
       "1                         <NA>   \n",
       "2                         <NA>   \n",
       "3                         <NA>   \n",
       "4                         <NA>   \n",
       "\n",
       "                                            run.name  \\\n",
       "0  optimizer=RandomSearch.benchmark=Ex_Functional...   \n",
       "1  optimizer=RandomSearch.benchmark=Ex_Functional...   \n",
       "2  optimizer=RandomSearch.benchmark=Ex_Functional...   \n",
       "3  optimizer=RandomSearch.benchmark=Ex_Functional...   \n",
       "4  optimizer=RandomSearch.benchmark=Ex_Functional...   \n",
       "\n",
       "                                        problem.name  problem.objective.count  \\\n",
       "0  optimizer=RandomSearch.benchmark=Ex_Functional...                        1   \n",
       "1  optimizer=RandomSearch.benchmark=Ex_Functional...                        1   \n",
       "2  optimizer=RandomSearch.benchmark=Ex_Functional...                        1   \n",
       "3  optimizer=RandomSearch.benchmark=Ex_Functional...                        1   \n",
       "4  optimizer=RandomSearch.benchmark=Ex_Functional...                        1   \n",
       "\n",
       "   ... problem.objective.1.min  problem.objective.1.max  \\\n",
       "0  ...                       0                      inf   \n",
       "1  ...                       0                      inf   \n",
       "2  ...                       0                      inf   \n",
       "3  ...                       0                      inf   \n",
       "4  ...                       0                      inf   \n",
       "\n",
       "   problem.fidelity.count  problem.cost.count    problem.benchmark  \\\n",
       "0                       0                   0  Ex_Functional_Bench   \n",
       "1                       0                   0  Ex_Functional_Bench   \n",
       "2                       0                   0  Ex_Functional_Bench   \n",
       "3                       0                   0  Ex_Functional_Bench   \n",
       "4                       0                   0  Ex_Functional_Bench   \n",
       "\n",
       "   problem.budget.kind problem.budget.total   run.seed  run.opt.name  \\\n",
       "0          TrialBudget                  100  383329928  RandomSearch   \n",
       "1          TrialBudget                  100  383329928  RandomSearch   \n",
       "2          TrialBudget                  100  383329928  RandomSearch   \n",
       "3          TrialBudget                  100  383329928  RandomSearch   \n",
       "4          TrialBudget                  100  383329928  RandomSearch   \n",
       "\n",
       "   run.opt.hp_str  \n",
       "0         default  \n",
       "1         default  \n",
       "2         default  \n",
       "3         default  \n",
       "4         default  \n",
       "\n",
       "[5 rows x 22 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd  # type: ignore\n",
    "\n",
    "_df = pd.read_parquet(\"example-outputs/hposuite_demo_rs_func/\"\n",
    "                 \"optimizer=RandomSearch.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=383329928/\"\n",
    "                 \"optimizer=RandomSearch.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=383329928.parquet\"\n",
    ")\n",
    "\n",
    "_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Running a BlackBox Optimizer on a Synthetic Benchmark"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "SMAC provides the BlackBoxFacade which is used as an example of a BlackBox Optimizer. \\\n",
    "Multi-fidelity Hartmann 3D with the correlation \"good\" serves as an example of a Synethetic Multi-fidelity Benchmark."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "bb_optimizer = OPTIMIZERS[\"Ex_Blackbox_Opt\"]\n",
    "mf_syn_benchmark = BENCHMARKS[\"mfh3_good\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:hposuite.study:Generated 5 runs\n",
      "INFO:hposuite.study:Created study at /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_bb_mfsyn\n",
      "INFO:hposuite.study:Dumped experiments to /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_bb_mfsyn/dump_all.txt\n",
      "INFO:hposuite.study:Overwrite flag is set to True. Existing results will be overwritten!\n",
      "INFO:hposuite.study:Running 5 experiments sequentially\n",
      "INFO:hposuite.study:Running experiment 1/5\n",
      "INFO:hposuite.run:Overwriting optimizer=Ex_Blackbox_Opt.benchmark=mfh3_good.objectives=value.TrialBudget=20.seed=383329928 in `state=<State.PENDING: 'PENDING'>` at /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_bb_mfsyn/optimizer=Ex_Blackbox_Opt.benchmark=mfh3_good.objectives=value.TrialBudget=20.seed=383329928.\n",
      "WARNING:hposuite.run:Continuations are not supported for the optimizer: <class 'lib.optimizers.ex_blackbox_opt.Ex_Blackbox_Opt'>. Ignoring continuations=True.\n",
      "INFO:hpoglue._run:COMPLETED running optimizer=Ex_Blackbox_Opt.benchmark=mfh3_good.objectives=value.TrialBudget=20.seed=383329928\n",
      "INFO:hposuite.run:Results dumped at /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_bb_mfsyn/optimizer=Ex_Blackbox_Opt.benchmark=mfh3_good.objectives=value.TrialBudget=20.seed=383329928/optimizer=Ex_Blackbox_Opt.benchmark=mfh3_good.objectives=value.TrialBudget=20.seed=383329928.parquet\n",
      "INFO:hposuite.study:Running experiment 2/5\n",
      "INFO:hposuite.run:Overwriting optimizer=Ex_Blackbox_Opt.benchmark=mfh3_good.objectives=value.TrialBudget=20.seed=3324115917 in `state=<State.PENDING: 'PENDING'>` at /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_bb_mfsyn/optimizer=Ex_Blackbox_Opt.benchmark=mfh3_good.objectives=value.TrialBudget=20.seed=3324115917.\n",
      "WARNING:hposuite.run:Continuations are not supported for the optimizer: <class 'lib.optimizers.ex_blackbox_opt.Ex_Blackbox_Opt'>. Ignoring continuations=True.\n",
      "INFO:hpoglue._run:COMPLETED running optimizer=Ex_Blackbox_Opt.benchmark=mfh3_good.objectives=value.TrialBudget=20.seed=3324115917\n",
      "INFO:hposuite.run:Results dumped at /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_bb_mfsyn/optimizer=Ex_Blackbox_Opt.benchmark=mfh3_good.objectives=value.TrialBudget=20.seed=3324115917/optimizer=Ex_Blackbox_Opt.benchmark=mfh3_good.objectives=value.TrialBudget=20.seed=3324115917.parquet\n",
      "INFO:hposuite.study:Running experiment 3/5\n",
      "INFO:hposuite.run:Overwriting optimizer=Ex_Blackbox_Opt.benchmark=mfh3_good.objectives=value.TrialBudget=20.seed=2811363265 in `state=<State.PENDING: 'PENDING'>` at /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_bb_mfsyn/optimizer=Ex_Blackbox_Opt.benchmark=mfh3_good.objectives=value.TrialBudget=20.seed=2811363265.\n",
      "WARNING:hposuite.run:Continuations are not supported for the optimizer: <class 'lib.optimizers.ex_blackbox_opt.Ex_Blackbox_Opt'>. Ignoring continuations=True.\n",
      "INFO:hpoglue._run:COMPLETED running optimizer=Ex_Blackbox_Opt.benchmark=mfh3_good.objectives=value.TrialBudget=20.seed=2811363265\n",
      "INFO:hposuite.run:Results dumped at /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_bb_mfsyn/optimizer=Ex_Blackbox_Opt.benchmark=mfh3_good.objectives=value.TrialBudget=20.seed=2811363265/optimizer=Ex_Blackbox_Opt.benchmark=mfh3_good.objectives=value.TrialBudget=20.seed=2811363265.parquet\n",
      "INFO:hposuite.study:Running experiment 4/5\n",
      "INFO:hposuite.run:Overwriting optimizer=Ex_Blackbox_Opt.benchmark=mfh3_good.objectives=value.TrialBudget=20.seed=1884968545 in `state=<State.PENDING: 'PENDING'>` at /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_bb_mfsyn/optimizer=Ex_Blackbox_Opt.benchmark=mfh3_good.objectives=value.TrialBudget=20.seed=1884968545.\n",
      "WARNING:hposuite.run:Continuations are not supported for the optimizer: <class 'lib.optimizers.ex_blackbox_opt.Ex_Blackbox_Opt'>. Ignoring continuations=True.\n",
      "INFO:hpoglue._run:COMPLETED running optimizer=Ex_Blackbox_Opt.benchmark=mfh3_good.objectives=value.TrialBudget=20.seed=1884968545\n",
      "INFO:hposuite.run:Results dumped at /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_bb_mfsyn/optimizer=Ex_Blackbox_Opt.benchmark=mfh3_good.objectives=value.TrialBudget=20.seed=1884968545/optimizer=Ex_Blackbox_Opt.benchmark=mfh3_good.objectives=value.TrialBudget=20.seed=1884968545.parquet\n",
      "INFO:hposuite.study:Running experiment 5/5\n",
      "INFO:hposuite.run:Overwriting optimizer=Ex_Blackbox_Opt.benchmark=mfh3_good.objectives=value.TrialBudget=20.seed=1859786276 in `state=<State.PENDING: 'PENDING'>` at /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_bb_mfsyn/optimizer=Ex_Blackbox_Opt.benchmark=mfh3_good.objectives=value.TrialBudget=20.seed=1859786276.\n",
      "WARNING:hposuite.run:Continuations are not supported for the optimizer: <class 'lib.optimizers.ex_blackbox_opt.Ex_Blackbox_Opt'>. Ignoring continuations=True.\n",
      "INFO:hpoglue._run:COMPLETED running optimizer=Ex_Blackbox_Opt.benchmark=mfh3_good.objectives=value.TrialBudget=20.seed=1859786276\n",
      "INFO:hposuite.run:Results dumped at /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_bb_mfsyn/optimizer=Ex_Blackbox_Opt.benchmark=mfh3_good.objectives=value.TrialBudget=20.seed=1859786276/optimizer=Ex_Blackbox_Opt.benchmark=mfh3_good.objectives=value.TrialBudget=20.seed=1859786276.parquet\n",
      "INFO:hposuite.study:Completed study with 5 runs\n"
     ]
    }
   ],
   "source": [
    "from hposuite import create_study\n",
    "\n",
    "study = create_study(\n",
    "    name=\"hposuite_demo_bb_mfsyn\",\n",
    "    output_dir=\"example-outputs\",\n",
    "    optimizers=bb_optimizer,\n",
    "    benchmarks=mf_syn_benchmark,\n",
    "    num_seeds=5,\n",
    "    budget=20,\n",
    ")\n",
    "\n",
    "study.optimize(overwrite=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Running a MultiFidelity Optimizer on a Surrogate Benchmark"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We use a Multi-fidelity Optimizer like Hyperband from SMAC on the PD1 Cifar 100 Wide Resnet 2048 Benchmark which is a Surrogate Benchmark provided by the MF-Prior-Bench Package"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "sg_benchmark = BENCHMARKS[\"Ex_Surrogate_Bench\"]\n",
    "mf_optimizer = OPTIMIZERS[\"Ex_Multifidelity_Opt\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:hposuite.study:Generated 5 runs\n",
      "INFO:hposuite.study:Created study at /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_sg_mf\n",
      "INFO:hposuite.study:Dumped experiments to /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_sg_mf/dump_all.txt\n",
      "INFO:hposuite.study:Overwrite flag is set to True. Existing results will be overwritten!\n",
      "INFO:hposuite.study:Running 5 experiments sequentially\n",
      "INFO:hposuite.study:Running experiment 1/5\n",
      "INFO:hposuite.run:Overwriting optimizer=Ex_Multifidelity_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate.TrialBudget=100.seed=383329928 in `state=<State.PENDING: 'PENDING'>` at /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_sg_mf/optimizer=Ex_Multifidelity_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate.TrialBudget=100.seed=383329928.\n",
      "INFO:hpoglue._run:COMPLETED running optimizer=Ex_Multifidelity_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate.TrialBudget=100.seed=383329928\n",
      "INFO:hposuite.run:Results dumped at /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_sg_mf/optimizer=Ex_Multifidelity_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate.TrialBudget=100.seed=383329928/optimizer=Ex_Multifidelity_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate.TrialBudget=100.seed=383329928.parquet\n",
      "INFO:hposuite.study:Running experiment 2/5\n",
      "INFO:hposuite.run:Overwriting optimizer=Ex_Multifidelity_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate.TrialBudget=100.seed=3324115917 in `state=<State.PENDING: 'PENDING'>` at /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_sg_mf/optimizer=Ex_Multifidelity_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate.TrialBudget=100.seed=3324115917.\n",
      "INFO:hpoglue._run:COMPLETED running optimizer=Ex_Multifidelity_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate.TrialBudget=100.seed=3324115917\n",
      "INFO:hposuite.run:Results dumped at /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_sg_mf/optimizer=Ex_Multifidelity_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate.TrialBudget=100.seed=3324115917/optimizer=Ex_Multifidelity_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate.TrialBudget=100.seed=3324115917.parquet\n",
      "INFO:hposuite.study:Running experiment 3/5\n",
      "INFO:hposuite.run:Overwriting optimizer=Ex_Multifidelity_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate.TrialBudget=100.seed=2811363265 in `state=<State.PENDING: 'PENDING'>` at /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_sg_mf/optimizer=Ex_Multifidelity_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate.TrialBudget=100.seed=2811363265.\n",
      "INFO:hpoglue._run:COMPLETED running optimizer=Ex_Multifidelity_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate.TrialBudget=100.seed=2811363265\n",
      "INFO:hposuite.run:Results dumped at /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_sg_mf/optimizer=Ex_Multifidelity_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate.TrialBudget=100.seed=2811363265/optimizer=Ex_Multifidelity_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate.TrialBudget=100.seed=2811363265.parquet\n",
      "INFO:hposuite.study:Running experiment 4/5\n",
      "INFO:hposuite.run:Overwriting optimizer=Ex_Multifidelity_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate.TrialBudget=100.seed=1884968545 in `state=<State.PENDING: 'PENDING'>` at /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_sg_mf/optimizer=Ex_Multifidelity_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate.TrialBudget=100.seed=1884968545.\n",
      "INFO:hpoglue._run:COMPLETED running optimizer=Ex_Multifidelity_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate.TrialBudget=100.seed=1884968545\n",
      "INFO:hposuite.run:Results dumped at /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_sg_mf/optimizer=Ex_Multifidelity_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate.TrialBudget=100.seed=1884968545/optimizer=Ex_Multifidelity_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate.TrialBudget=100.seed=1884968545.parquet\n",
      "INFO:hposuite.study:Running experiment 5/5\n",
      "INFO:hposuite.run:Overwriting optimizer=Ex_Multifidelity_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate.TrialBudget=100.seed=1859786276 in `state=<State.PENDING: 'PENDING'>` at /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_sg_mf/optimizer=Ex_Multifidelity_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate.TrialBudget=100.seed=1859786276.\n",
      "INFO:hpoglue._run:COMPLETED running optimizer=Ex_Multifidelity_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate.TrialBudget=100.seed=1859786276\n",
      "INFO:hposuite.run:Results dumped at /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_sg_mf/optimizer=Ex_Multifidelity_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate.TrialBudget=100.seed=1859786276/optimizer=Ex_Multifidelity_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate.TrialBudget=100.seed=1859786276.parquet\n",
      "INFO:hposuite.study:Completed study with 5 runs\n"
     ]
    }
   ],
   "source": [
    "from hposuite import create_study\n",
    "\n",
    "study = create_study(\n",
    "    name=\"hposuite_demo_sg_mf\",\n",
    "    output_dir=\"example-outputs\",\n",
    "    optimizers=mf_optimizer,\n",
    "    benchmarks=sg_benchmark,\n",
    "    num_seeds=5,\n",
    "    budget=100,\n",
    ")\n",
    "\n",
    "study.optimize(overwrite=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Running a Multi-objective Optimizer on a Benchmark with Multiple Objectives"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Optuna's NSGA2 Sampler is used here as an example of a Multi-Objective Optimizer. \\\n",
    "PD1 Cifar 100 Wide Resnet 2048 from MF-Prior-Bench is the Benchmark used with \"valid_error_rate\" and \"train_cost\" as the objectives."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "mo_optimizer = OPTIMIZERS[\"Ex_MO_Opt\"]\n",
    "mo_benchmark = BENCHMARKS[\"Ex_Surrogate_Bench\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Printing the description of the Example Surrogate benchmark and its available list of objectives"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BenchmarkDescription(name='Ex_Surrogate_Bench', config_space=Configuration space object:\n",
      "  Hyperparameters:\n",
      "    lr_decay_factor, Type: UniformFloat, Range: [0.010093, 0.989012], Default: 0.4995525\n",
      "    lr_initial, Type: UniformFloat, Range: [1e-05, 9.779176], Default: 0.0098889716351, on log-scale\n",
      "    lr_power, Type: UniformFloat, Range: [0.100708, 1.999376], Default: 1.050042\n",
      "    opt_momentum, Type: UniformFloat, Range: [5.9e-05, 0.998993], Default: 0.0076772773169, on log-scale\n",
      ", load=functools.partial(<function _get_surrogate_benchmark at 0x6ffcb5326050>, benchmark_name='cifar100_wideresnet_2048', datadir=PosixPath('/home/soham/repos/hposuite/data/pd1')), metrics={'valid_error_rate': Measure(minimize=True, kind=<Kind.METRIC: 'metric'>, bounds=(0, 1))}, test_metrics={'test_error_rate': Measure(minimize=True, kind=<Kind.METRIC: 'metric'>, bounds=(0, 1))}, costs={'train_cost': Measure(minimize=True, kind=<Kind.COST: 'cost'>, bounds=(0, inf))}, fidelities={'epoch': RangeFidelity(kind=<class 'int'>, min=1, max=199, stepsize=1, supports_continuation=True)}, has_conditionals=False, is_tabular=False, env=Env(name='py310-mfpbench-1.9-pd1'), mem_req_mb=12288, predefined_points=None, extra={})\n",
      "dict_keys(['valid_error_rate']) dict_keys(['train_cost'])\n"
     ]
    }
   ],
   "source": [
    "print(mo_benchmark)\n",
    "print(mo_benchmark.metrics.keys(), mo_benchmark.costs.keys()) # We can use both the metrics and costs as objectives"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Running Optuna NSGA2 on the Surrogate Benchmark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:hposuite.study:Generated 5 runs\n",
      "INFO:hposuite.study:Created study at /home/soham/repos/hposuite/examples/../../hposuite-output/hposuite_demo_sg_mf\n",
      "INFO:hposuite.study:Dumped experiments to /home/soham/repos/hposuite/examples/../../hposuite-output/hposuite_demo_sg_mf/dump_all.txt\n",
      "INFO:hposuite.study:Overwrite flag is set to True. Existing results will be overwritten!\n",
      "INFO:hposuite.study:Running 5 experiments sequentially\n",
      "INFO:hposuite.study:Running experiment 1/5\n",
      "INFO:hposuite.run:Overwriting optimizer=Ex_MO_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate,train_cost.TrialBudget=100.seed=383329928 in `state=<State.PENDING: 'PENDING'>` at /home/soham/repos/hposuite-output/hposuite_demo_sg_mf/optimizer=Ex_MO_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate,train_cost.TrialBudget=100.seed=383329928.\n",
      "WARNING:hposuite.run:Continuations are not supported for the optimizer: <class 'lib.optimizers.ex_mo_opt.Ex_MO_Opt'>. Ignoring continuations=True.\n",
      "[I 2025-02-09 19:02:56,580] A new study created in memory with name: optimizer=Ex_MO_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate,train_cost.TrialBudget=100-383329928\n",
      "INFO:hpoglue._run:COMPLETED running optimizer=Ex_MO_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate,train_cost.TrialBudget=100.seed=383329928\n",
      "INFO:hposuite.run:Results dumped at /home/soham/repos/hposuite-output/hposuite_demo_sg_mf/optimizer=Ex_MO_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate,train_cost.TrialBudget=100.seed=383329928/optimizer=Ex_MO_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate,train_cost.TrialBudget=100.seed=383329928.parquet\n",
      "INFO:hposuite.study:Running experiment 2/5\n",
      "INFO:hposuite.run:Overwriting optimizer=Ex_MO_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate,train_cost.TrialBudget=100.seed=3324115917 in `state=<State.PENDING: 'PENDING'>` at /home/soham/repos/hposuite-output/hposuite_demo_sg_mf/optimizer=Ex_MO_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate,train_cost.TrialBudget=100.seed=3324115917.\n",
      "WARNING:hposuite.run:Continuations are not supported for the optimizer: <class 'lib.optimizers.ex_mo_opt.Ex_MO_Opt'>. Ignoring continuations=True.\n",
      "[I 2025-02-09 19:02:58,052] A new study created in memory with name: optimizer=Ex_MO_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate,train_cost.TrialBudget=100-3324115917\n",
      "INFO:hpoglue._run:COMPLETED running optimizer=Ex_MO_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate,train_cost.TrialBudget=100.seed=3324115917\n",
      "INFO:hposuite.run:Results dumped at /home/soham/repos/hposuite-output/hposuite_demo_sg_mf/optimizer=Ex_MO_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate,train_cost.TrialBudget=100.seed=3324115917/optimizer=Ex_MO_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate,train_cost.TrialBudget=100.seed=3324115917.parquet\n",
      "INFO:hposuite.study:Running experiment 3/5\n",
      "INFO:hposuite.run:Overwriting optimizer=Ex_MO_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate,train_cost.TrialBudget=100.seed=2811363265 in `state=<State.PENDING: 'PENDING'>` at /home/soham/repos/hposuite-output/hposuite_demo_sg_mf/optimizer=Ex_MO_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate,train_cost.TrialBudget=100.seed=2811363265.\n",
      "WARNING:hposuite.run:Continuations are not supported for the optimizer: <class 'lib.optimizers.ex_mo_opt.Ex_MO_Opt'>. Ignoring continuations=True.\n",
      "[I 2025-02-09 19:03:00,427] A new study created in memory with name: optimizer=Ex_MO_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate,train_cost.TrialBudget=100-2811363265\n",
      "INFO:hpoglue._run:COMPLETED running optimizer=Ex_MO_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate,train_cost.TrialBudget=100.seed=2811363265\n",
      "INFO:hposuite.run:Results dumped at /home/soham/repos/hposuite-output/hposuite_demo_sg_mf/optimizer=Ex_MO_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate,train_cost.TrialBudget=100.seed=2811363265/optimizer=Ex_MO_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate,train_cost.TrialBudget=100.seed=2811363265.parquet\n",
      "INFO:hposuite.study:Running experiment 4/5\n",
      "INFO:hposuite.run:Overwriting optimizer=Ex_MO_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate,train_cost.TrialBudget=100.seed=1884968545 in `state=<State.PENDING: 'PENDING'>` at /home/soham/repos/hposuite-output/hposuite_demo_sg_mf/optimizer=Ex_MO_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate,train_cost.TrialBudget=100.seed=1884968545.\n",
      "WARNING:hposuite.run:Continuations are not supported for the optimizer: <class 'lib.optimizers.ex_mo_opt.Ex_MO_Opt'>. Ignoring continuations=True.\n",
      "[I 2025-02-09 19:03:02,373] A new study created in memory with name: optimizer=Ex_MO_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate,train_cost.TrialBudget=100-1884968545\n",
      "INFO:hpoglue._run:COMPLETED running optimizer=Ex_MO_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate,train_cost.TrialBudget=100.seed=1884968545\n",
      "INFO:hposuite.run:Results dumped at /home/soham/repos/hposuite-output/hposuite_demo_sg_mf/optimizer=Ex_MO_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate,train_cost.TrialBudget=100.seed=1884968545/optimizer=Ex_MO_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate,train_cost.TrialBudget=100.seed=1884968545.parquet\n",
      "INFO:hposuite.study:Running experiment 5/5\n",
      "INFO:hposuite.run:Overwriting optimizer=Ex_MO_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate,train_cost.TrialBudget=100.seed=1859786276 in `state=<State.PENDING: 'PENDING'>` at /home/soham/repos/hposuite-output/hposuite_demo_sg_mf/optimizer=Ex_MO_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate,train_cost.TrialBudget=100.seed=1859786276.\n",
      "WARNING:hposuite.run:Continuations are not supported for the optimizer: <class 'lib.optimizers.ex_mo_opt.Ex_MO_Opt'>. Ignoring continuations=True.\n",
      "[I 2025-02-09 19:03:04,302] A new study created in memory with name: optimizer=Ex_MO_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate,train_cost.TrialBudget=100-1859786276\n",
      "INFO:hpoglue._run:COMPLETED running optimizer=Ex_MO_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate,train_cost.TrialBudget=100.seed=1859786276\n",
      "INFO:hposuite.run:Results dumped at /home/soham/repos/hposuite-output/hposuite_demo_sg_mf/optimizer=Ex_MO_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate,train_cost.TrialBudget=100.seed=1859786276/optimizer=Ex_MO_Opt.benchmark=Ex_Surrogate_Bench.objectives=valid_error_rate,train_cost.TrialBudget=100.seed=1859786276.parquet\n",
      "INFO:hposuite.study:Completed study with 5 runs\n"
     ]
    }
   ],
   "source": [
    "from hposuite import create_study\n",
    "\n",
    "study = create_study(\n",
    "    name=\"hposuite_demo_sg_mf\",\n",
    "    output_dir=\"../../hposuite-output\",\n",
    "    optimizers=mo_optimizer,\n",
    "    benchmarks=(\n",
    "        mo_benchmark,\n",
    "        {\n",
    "            \"objectives\": [\"valid_error_rate\", \"train_cost\"]\n",
    "        }\n",
    "    ),\n",
    "    num_seeds=5,\n",
    "    budget=100,\n",
    ")\n",
    "\n",
    "study.optimize(overwrite=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Running an Optimizer that supports Priors on a Benchmark with a Prior over its objective"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Optimizer: RandomSearch With Priors \\\n",
    "Benchmark: Ackley Functional Benchmark with the optimum as the Prior (perturbed by std=0.25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "rs_with_priors = OPTIMIZERS[\"RandomSearchWithPriors\"]\n",
    "func_benchmark = BENCHMARKS[\"Ex_Functional_Bench\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Print available predefined points in the Ackley Functional Benchmark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'min': Config(config_id='min', values={'x0': 0.0, 'x1': 0.0}, description='This point yields a global optimum of y:0.0')}\n"
     ]
    }
   ],
   "source": [
    "print(func_benchmark.desc.predefined_points)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Run RandomSearch With Priors on Ackley with hposuite"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:hposuite.study:Generated 5 runs\n",
      "INFO:hposuite.study:Created study at /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_priors\n",
      "INFO:hposuite.study:Dumped experiments to /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_priors/dump_all.txt\n",
      "INFO:hposuite.study:Overwrite flag is set to True. Existing results will be overwritten!\n",
      "INFO:hposuite.study:Running 5 experiments sequentially\n",
      "INFO:hposuite.study:Running experiment 1/5\n",
      "INFO:hposuite.run:Overwriting optimizer=RandomSearchWithPriors.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=383329928 in `state=<State.COMPLETE: 'COMPLETE'>` at /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_priors/optimizer=RandomSearchWithPriors.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=383329928.\n",
      "WARNING:hposuite.run:Continuations are not supported for the optimizer: <class 'lib.optimizers.random_search.RandomSearchWithPriors'>. Ignoring continuations=True.\n",
      "INFO:hpoglue._run:COMPLETED running optimizer=RandomSearchWithPriors.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=383329928\n",
      "INFO:hposuite.run:Results dumped at /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_priors/optimizer=RandomSearchWithPriors.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=383329928/optimizer=RandomSearchWithPriors.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=383329928.parquet\n",
      "INFO:hposuite.study:Running experiment 2/5\n",
      "INFO:hposuite.run:Overwriting optimizer=RandomSearchWithPriors.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=3324115917 in `state=<State.COMPLETE: 'COMPLETE'>` at /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_priors/optimizer=RandomSearchWithPriors.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=3324115917.\n",
      "WARNING:hposuite.run:Continuations are not supported for the optimizer: <class 'lib.optimizers.random_search.RandomSearchWithPriors'>. Ignoring continuations=True.\n",
      "INFO:hpoglue._run:COMPLETED running optimizer=RandomSearchWithPriors.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=3324115917\n",
      "INFO:hposuite.run:Results dumped at /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_priors/optimizer=RandomSearchWithPriors.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=3324115917/optimizer=RandomSearchWithPriors.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=3324115917.parquet\n",
      "INFO:hposuite.study:Running experiment 3/5\n",
      "INFO:hposuite.run:Overwriting optimizer=RandomSearchWithPriors.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=2811363265 in `state=<State.COMPLETE: 'COMPLETE'>` at /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_priors/optimizer=RandomSearchWithPriors.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=2811363265.\n",
      "WARNING:hposuite.run:Continuations are not supported for the optimizer: <class 'lib.optimizers.random_search.RandomSearchWithPriors'>. Ignoring continuations=True.\n",
      "INFO:hpoglue._run:COMPLETED running optimizer=RandomSearchWithPriors.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=2811363265\n",
      "INFO:hposuite.run:Results dumped at /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_priors/optimizer=RandomSearchWithPriors.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=2811363265/optimizer=RandomSearchWithPriors.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=2811363265.parquet\n",
      "INFO:hposuite.study:Running experiment 4/5\n",
      "INFO:hposuite.run:Overwriting optimizer=RandomSearchWithPriors.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=1884968545 in `state=<State.COMPLETE: 'COMPLETE'>` at /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_priors/optimizer=RandomSearchWithPriors.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=1884968545.\n",
      "WARNING:hposuite.run:Continuations are not supported for the optimizer: <class 'lib.optimizers.random_search.RandomSearchWithPriors'>. Ignoring continuations=True.\n",
      "INFO:hpoglue._run:COMPLETED running optimizer=RandomSearchWithPriors.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=1884968545\n",
      "INFO:hposuite.run:Results dumped at /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_priors/optimizer=RandomSearchWithPriors.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=1884968545/optimizer=RandomSearchWithPriors.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=1884968545.parquet\n",
      "INFO:hposuite.study:Running experiment 5/5\n",
      "INFO:hposuite.run:Overwriting optimizer=RandomSearchWithPriors.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=1859786276 in `state=<State.COMPLETE: 'COMPLETE'>` at /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_priors/optimizer=RandomSearchWithPriors.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=1859786276.\n",
      "WARNING:hposuite.run:Continuations are not supported for the optimizer: <class 'lib.optimizers.random_search.RandomSearchWithPriors'>. Ignoring continuations=True.\n",
      "INFO:hpoglue._run:COMPLETED running optimizer=RandomSearchWithPriors.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=1859786276\n",
      "INFO:hposuite.run:Results dumped at /home/soham/repos/hposuite/examples/example-outputs/hposuite_demo_priors/optimizer=RandomSearchWithPriors.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=1859786276/optimizer=RandomSearchWithPriors.benchmark=Ex_Functional_Bench.objectives=y.TrialBudget=100.seed=1859786276.parquet\n",
      "INFO:hposuite.study:Completed study with 5 runs\n"
     ]
    }
   ],
   "source": [
    "from hposuite import create_study\n",
    "\n",
    "study = create_study(\n",
    "    name=\"hposuite_demo_priors\",\n",
    "    output_dir=\"example-outputs\",\n",
    "    optimizers=rs_with_priors,\n",
    "    benchmarks=(\n",
    "        func_benchmark,\n",
    "        {\n",
    "            \"priors\": {\n",
    "                \"y\": func_benchmark.desc.predefined_points[\"min\"]\n",
    "            }\n",
    "        }\n",
    "    ),\n",
    "    num_seeds=5,\n",
    "    budget=100,\n",
    ")\n",
    "\n",
    "study.optimize(overwrite=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "hposuite_env",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}