examples/opt_bench_usage_examples.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# hposuite additional usage examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_keys(['yahpo-lcbench-3945', 'yahpo-lcbench-7593', 'yahpo-lcbench-34539', 'yahpo-lcbench-126025', 'yahpo-lcbench-126026', 'yahpo-lcbench-126029', 'yahpo-lcbench-146212', 'yahpo-lcbench-167104', 'yahpo-lcbench-167149', 'yahpo-lcbench-167152', 'yahpo-lcbench-167161', 'yahpo-lcbench-167168', 'yahpo-lcbench-167181', 'yahpo-lcbench-167184', 'yahpo-lcbench-167185', 'yahpo-lcbench-167190', 'yahpo-lcbench-167200', 'yahpo-lcbench-167201', 'yahpo-lcbench-168329', 'yahpo-lcbench-168330', 'yahpo-lcbench-168331', 'yahpo-lcbench-168335', 'yahpo-lcbench-168868', 'yahpo-lcbench-168908', 'yahpo-lcbench-168910', 'yahpo-lcbench-189354', 'yahpo-lcbench-189862', 'yahpo-lcbench-189865', 'yahpo-lcbench-189866', 'yahpo-lcbench-189873', 'yahpo-lcbench-189905', 'yahpo-lcbench-189906', 'yahpo-lcbench-189908', 'yahpo-lcbench-189909', 'mfh3_bad', 'mfh6_bad', 'mfh3_good', 'mfh6_good', 'mfh3_moderate', 'mfh6_moderate', 'mfh3_terrible', 'mfh6_terrible', 'pd1-cifar100-wide_resnet-2048', 'pd1-imagenet-resnet-512', 'pd1-lm1b-transformer-2048', 'pd1-translate_wmt-xformer_translate-64', 'pymoo-ackley', 'pymoo-griewank', 'pymoo-himmelblau', 'pymoo-rastrigin', 'pymoo-rosenbrock', 'pymoo-schwefel', 'pymoo-sphere', 'pymoo-zakharov', 'pymoo-kursawe', 'pymoo-zdt1', 'pymoo-zdt2', 'pymoo-zdt3', 'pymoo-zdt4', 'pymoo-zdt5', 'pymoo-zdt6', 'pymoo-omnitest', 'pymoo-sympart', 'pymoo-sympart_rotated', 'pymoo-dtlz1', 'pymoo-dtlz2', 'pymoo-dtlz3', 'pymoo-dtlz4', 'pymoo-dtlz5', 'pymoo-dtlz6', 'pymoo-dtlz7', 'ackley'])\n",
      "dict_keys(['RandomSearch', 'RandomSearchWithPriors', 'DEHB', 'Nevergrad', 'SMAC_BO', 'SMAC_Hyperband', 'SyneTune_BO', 'SyneTune_BOHB', 'Scikit_Optimize', 'Optuna'])\n"
     ]
    }
   ],
   "source": [
    "# It is assumed that the datadir is hposuite/data\n",
    "\n",
    "from hposuite.benchmarks import BENCHMARKS\n",
    "from hposuite.optimizers import OPTIMIZERS\n",
    "\n",
    "print(BENCHMARKS.keys())\n",
    "print(OPTIMIZERS.keys())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Running multiple different optimizers on multiple different benchmarks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following example runs RandomSearch and SMAC Blackbox Facade on Ackley Functional Benchmark and MF Hartmann 3D Synthetic Benchmark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rs = OPTIMIZERS[\"RandomSearch\"]\n",
    "bb = OPTIMIZERS[\"SMAC_BO\"]\n",
    "\n",
    "ackley = BENCHMARKS[\"ackley\"]\n",
    "mfh3g = BENCHMARKS[\"mfh3_good\"]\n",
    "\n",
    "from hposuite import create_study\n",
    "\n",
    "study = create_study(\n",
    "    name=\"hposuite-2opts-2benchs\",\n",
    "    output_dir=\"example-outputs\",\n",
    "    optimizers=[rs, bb],\n",
    "    benchmarks=[ackley, mfh3g],\n",
    "    num_seeds=3,\n",
    "    budget=10,\n",
    ")\n",
    "\n",
    "study.optimize(overwrite=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Running the same optimizer with different hyperparameters"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this example we use SMAC_Hyperband with eta=2 and eta=3 and run it on MF Hartmann 6D Benchmark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mf_hb_2 = (\n",
    "    OPTIMIZERS[\"SMAC_Hyperband\"],\n",
    "    {\"eta\": 2}\n",
    ")\n",
    "mf_hb_3 = (\n",
    "    OPTIMIZERS[\"SMAC_Hyperband\"],\n",
    "    {\"eta\": 3}\n",
    ")\n",
    "\n",
    "from hposuite import create_study\n",
    "\n",
    "study = create_study(\n",
    "    name=\"hposuite-1opt-2hps\",\n",
    "    output_dir=\"example-outputs\",\n",
    "    optimizers=[mf_hb_2, mf_hb_3],\n",
    "    benchmarks=\"mfh6_bad\",  # Benchmarks can also be specified as string names\n",
    "    num_seeds=3,\n",
    "    budget=10,\n",
    ")\n",
    "\n",
    "study.optimize(overwrite=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Running a single-objective Optimizer on the same Benchmark for different objectives in the same study"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For this example we select the YAHPO LCBench Surrogate Benchmark. \\\n",
    "Listing below the objectives (metrics and costs) available in the benchmark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_keys(['val_accuracy', 'val_cross_entropy', 'val_balanced_accuracy'])\n",
      "dict_keys(['time'])\n"
     ]
    }
   ],
   "source": [
    "lcbench_3945 = BENCHMARKS[\"yahpo-lcbench-3945\"]\n",
    "print(lcbench_3945.metrics.keys())\n",
    "print(lcbench_3945.costs.keys())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Running RandomSearch on the Benchmark with one objective at a time in a single study"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from hposuite import create_study\n",
    "\n",
    "study = create_study(\n",
    "    name=\"hposuite-1opt-1bench-2objs\",\n",
    "    output_dir=\"example-outputs\",\n",
    "    optimizers=\"RandomSearch\",\n",
    "    benchmarks=[\n",
    "        (lcbench_3945, {\"objectives\": \"val_cross_entropy\"}),\n",
    "        (lcbench_3945, {\"objectives\": \"val_accuracy\"}),\n",
    "    ],\n",
    "    num_seeds=3,\n",
    "    budget=10,\n",
    ")\n",
    "\n",
    "study.optimize(overwrite=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Running a Multi-objective Optimizer on the same benchmark multiple times with different pairs of objectives"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It is possible to run an MO Optimizer, say Optuna-NSGA2 on an MO benchmark like YAHPO LCBench taking different combinations of its objectives at a time, within the same study. \\\n",
    "For example: One run on the pair (val_cross_entropy, val_accuracy) and another on (val_cross_entropy, time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from hposuite import create_study\n",
    "\n",
    "study = create_study(\n",
    "    name=\"hposuite-1opt-1bench-mo-mix\",\n",
    "    output_dir=\"example-outputs\",\n",
    "    optimizers=\"RandomSearch\",\n",
    "    benchmarks=[\n",
    "        (lcbench_3945, {\"objectives\": [\"val_cross_entropy\", \"val_accuracy\"]}),\n",
    "        (lcbench_3945, {\"objectives\": [\"val_cross_entropy\", \"time\"]}),\n",
    "    ],\n",
    "    num_seeds=3,\n",
    "    budget=10,\n",
    ")\n",
    "\n",
    "study.optimize(overwrite=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Running a Prior-based Optimizer on a Benchmark with multiple objectives, with priors on different objectives at a time."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For this example, we use RandomSearch With Priors as the Optimizer and PYMOO SymPart Problem as the benchmark. \\\n",
    "The example below demonstrates how to optimize on a benchmark with priors over one of its multiple objectives at a time, in a single study"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "rsp = OPTIMIZERS[\"RandomSearchWithPriors\"]\n",
    "sympart = BENCHMARKS[\"pymoo-sympart\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Printing the Benchmark Description of the Pymoo ZDT1 Benchmark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BenchmarkDescription(name='pymoo-sympart', config_space=Configuration space object:\n",
       "  Hyperparameters:\n",
       "    x0, Type: UniformFloat, Range: [-100.0, 100.0], Default: 0.0\n",
       "    x1, Type: UniformFloat, Range: [-100.0, 100.0], Default: 0.0\n",
       ", load=functools.partial(<function _get_pymoo_problems at 0x7f5ad4b29750>, function_name='sympart'), metrics={'value1': Measure(minimize=True, kind=<Kind.METRIC: 'metric'>, bounds=(-inf, inf)), 'value2': Measure(minimize=True, kind=<Kind.METRIC: 'metric'>, bounds=(-inf, inf))}, test_metrics=None, costs=None, fidelities=None, has_conditionals=False, is_tabular=False, env=Env(name='py310-pymoo-0.6.1.3'), mem_req_mb=1024, predefined_points=None, extra={})"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sympart"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Running RandomSearchWithPriors on Pymoo Sympart with prior over one objective at a time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from hposuite import create_study\n",
    "\n",
    "study = create_study(\n",
    "    name=\"hposuite-ex-multiprior-so\",\n",
    "    output_dir=\"example-outputs\",\n",
    "    optimizers=rsp,\n",
    "    benchmarks=[\n",
    "        (sympart, {\n",
    "            \"objectives\": [\"value1\"],\n",
    "            \"priors\": {\n",
    "                \"value1\": {\n",
    "                    \"x0\": 50.0,\n",
    "                    \"x1\": 50.0,\n",
    "                }\n",
    "            }\n",
    "        }),\n",
    "        (sympart, {\n",
    "            \"objectives\": [\"value2\"],\n",
    "            \"priors\": {\n",
    "                \"value2\": {\n",
    "                    \"x0\": 20.0,\n",
    "                    \"x1\": 30.0,\n",
    "                }\n",
    "            }\n",
    "        }),\n",
    "    ],\n",
    "    num_seeds=3,\n",
    "    budget=10,\n",
    "    on_error=\"raise\",\n",
    ")\n",
    "\n",
    "study.optimize(overwrite=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Running a Prior-based Multi-objective Optimizer on a Multi-objective benchmark with priors over different objectives at a time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For this example, we again use RandomSearch With Priors as the Optimizer, with the hyperparameter mo-prior-sampling set to random, which picks which objective's prior to use at every iteration randomly. \\\n",
    "The PYMOO DTLZ1 Many Objective Problem is selected as the benchmark. \\\n",
    "The example below demonstrates how to optimize for Multiple Objectives on a benchmark with priors over one of a pair of its multiple objectives at a time, in a single study"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "rsp = OPTIMIZERS[\"RandomSearchWithPriors\"]\n",
    "dtlz1 = BENCHMARKS[\"pymoo-dtlz1\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's display, pymoo-dtlz1's description"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BenchmarkDescription(name='pymoo-dtlz1', config_space=Configuration space object:\n",
       "  Hyperparameters:\n",
       "    x0, Type: UniformFloat, Range: [0.0, 1.0], Default: 0.5\n",
       "    x1, Type: UniformFloat, Range: [0.0, 1.0], Default: 0.5\n",
       "    x2, Type: UniformFloat, Range: [0.0, 1.0], Default: 0.5\n",
       "    x3, Type: UniformFloat, Range: [0.0, 1.0], Default: 0.5\n",
       "    x4, Type: UniformFloat, Range: [0.0, 1.0], Default: 0.5\n",
       "    x5, Type: UniformFloat, Range: [0.0, 1.0], Default: 0.5\n",
       "    x6, Type: UniformFloat, Range: [0.0, 1.0], Default: 0.5\n",
       ", load=functools.partial(<function _get_pymoo_problems at 0x7b5d66bf1750>, function_name='dtlz1'), metrics={'value1': Measure(minimize=True, kind=<Kind.METRIC: 'metric'>, bounds=(-inf, inf)), 'value2': Measure(minimize=True, kind=<Kind.METRIC: 'metric'>, bounds=(-inf, inf)), 'value3': Measure(minimize=True, kind=<Kind.METRIC: 'metric'>, bounds=(-inf, inf))}, test_metrics=None, costs=None, fidelities=None, has_conditionals=False, is_tabular=False, env=Env(name='py310-pymoo-0.6.1.3'), mem_req_mb=1024, predefined_points=None, extra={})"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dtlz1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Generating a random prior for the pymoo-dtlz1 benchmark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'value1': {'x0': 0.19419993333048924,\n",
      "            'x1': 0.6614779339684947,\n",
      "            'x2': 0.544900305283605,\n",
      "            'x3': 0.26440861964677975,\n",
      "            'x4': 0.41235220818493157,\n",
      "            'x5': 0.864046574915138,\n",
      "            'x6': 0.3311259829012908},\n",
      " 'value2': {'x0': 0.9798940978253005,\n",
      "            'x1': 0.8443932203723541,\n",
      "            'x2': 0.6604842848181971,\n",
      "            'x3': 0.3554239597245046,\n",
      "            'x4': 0.7319828518310841,\n",
      "            'x5': 0.6029296320892957,\n",
      "            'x6': 0.01626536051098082},\n",
      " 'value3': {'x0': 0.3481083542516228,\n",
      "            'x1': 0.30891397333194337,\n",
      "            'x2': 0.815010110519983,\n",
      "            'x3': 0.08701246224184911,\n",
      "            'x4': 0.3681382403794782,\n",
      "            'x5': 0.6872534330433554,\n",
      "            'x6': 0.019535086267388646}}\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "dtlz1_priors = {\n",
    "    f\"value{i}\": {\n",
    "        var.name: np.random.uniform(var.lower, var.upper) for var in list(dtlz1.config_space.values())\n",
    "    }\n",
    "    for i in range(1, 4)\n",
    "}\n",
    "\n",
    "from pprint import pprint\n",
    "pprint(dtlz1_priors)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Running RandomSearch With Priors on DTLZ1 optimizing for two objectives and taking priors over one of the objectives at a time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from hposuite import create_study\n",
    "\n",
    "study = create_study(\n",
    "    name=\"hposuite-ex-multiprior-priors\",\n",
    "    output_dir=\"example-outputs\",\n",
    "    optimizers=rsp,\n",
    "    benchmarks=[\n",
    "        (\n",
    "            dtlz1, {\n",
    "                \"objectives\": [\"value1\", \"value2\"],\n",
    "                \"priors\": {\n",
    "                    \"value1\": dtlz1_priors[\"value1\"]\n",
    "                }\n",
    "            }\n",
    "        ),\n",
    "        (\n",
    "            dtlz1, {\n",
    "                \"objectives\": [\"value2\", \"value3\"],\n",
    "                \"priors\": {\n",
    "                    \"value2\": dtlz1_priors[\"value2\"]\n",
    "                }\n",
    "            }\n",
    "        ),\n",
    "        (\n",
    "            dtlz1, {\n",
    "                \"objectives\": [\"value3\", \"value1\"],\n",
    "                \"priors\": {\n",
    "                    \"value3\": dtlz1_priors[\"value3\"]\n",
    "                }\n",
    "            }\n",
    "        ),\n",
    "    ],\n",
    "    num_seeds=3,\n",
    "    budget=10,\n",
    "    on_error=\"raise\",\n",
    ")\n",
    "\n",
    "study.optimize(overwrite=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "hposuite_env",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}