exp_3_hyperparamter_investiagtion_number_of_context_samples.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-01 22:47:51.288029: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Number of context samples: 1,2, 5, 10, 15, 20, 30, 40, 50, 60, 80, 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start meta-training -------------------- \n",
      "\n",
      "Iter 0/20000 - Time 22.43 sec - Train-Loss: -2955.20532\n",
      "\n",
      "Iter 1000/20000 - Time 14.88 sec - Train-Loss: -291.36035\n",
      "\n",
      "Iter 2000/20000 - Time 15.32 sec - Train-Loss: -298.40445\n",
      "\n",
      "Iter 3000/20000 - Time 16.42 sec - Train-Loss: -510.00452\n",
      "\n",
      "Iter 4000/20000 - Time 16.21 sec - Train-Loss: -257.42609\n",
      "\n",
      "Iter 5000/20000 - Time 14.90 sec - Train-Loss: -232.34467\n",
      "\n",
      "Iter 6000/20000 - Time 15.30 sec - Train-Loss: -157.19856\n",
      "\n",
      "Iter 7000/20000 - Time 14.93 sec - Train-Loss: -89.49222\n",
      "\n",
      "Iter 8000/20000 - Time 14.88 sec - Train-Loss: -42.02010\n",
      "\n",
      "Iter 9000/20000 - Time 14.91 sec - Train-Loss: -28.42173\n",
      "\tStart meta-test posterior inference in 2 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\n",
      "Iter 10000/20000 - Time 15.22 sec - Train-Loss: -13.63249- Val-avg_ll: 0.141 +- 0.083- Val-avg_rmse: 0.204 +- 0.015- Val-cal_err: 0.066 +- 0.032\n",
      "\n",
      "Iter 11000/20000 - Time 30.49 sec - Train-Loss: -12.00340\n",
      "\n",
      "Iter 12000/20000 - Time 19.94 sec - Train-Loss: -11.01156\n",
      "\n",
      "Iter 13000/20000 - Time 16.55 sec - Train-Loss: -9.88858\n",
      "\n",
      "Iter 14000/20000 - Time 16.52 sec - Train-Loss: -8.73724\n",
      "\n",
      "Iter 15000/20000 - Time 15.72 sec - Train-Loss: -6.79111\n",
      "\n",
      "Iter 16000/20000 - Time 15.19 sec - Train-Loss: -7.21167\n",
      "\n",
      "Iter 17000/20000 - Time 15.21 sec - Train-Loss: -5.08829\n",
      "\n",
      "Iter 18000/20000 - Time 15.09 sec - Train-Loss: -6.29340\n",
      "\n",
      "Iter 19000/20000 - Time 15.12 sec - Train-Loss: -5.38306\n",
      "\tStart meta-test posterior inference in 4 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\tMeta-Test batch #3 consisting of 5 tasks----\n",
      "\tMeta-Test batch #4 consisting of 5 tasks----\n",
      "avg_ll: 0.2096 +- 0.0851\n",
      "avg_rmse: 0.1981 +- 0.0177\n",
      "cal_err: 0.0399 +- 0.0194\n"
     ]
    }
   ],
   "source": [
    "tf.get_logger().setLevel('ERROR')\n",
    "\n",
    "from pacoh_nn.datasets.regression_datasets import provide_data\n",
    "\n",
    "meta_train_data, meta_test_data, _ = provide_data(dataset='cauchy_20', n_samples = 40)\n",
    "\n",
    "from pacoh_nn.pacoh_nn_regression import PACOH_NN_Regression\n",
    "pacoh_model = PACOH_NN_Regression(meta_train_data, prior_weight=0.184, bandwidth=480.,\n",
    "                                  hyper_prior_likelihood_log_var_mean_mean=-1.0,\n",
    "                                  hyper_prior_log_var_mean=-1.74, hyper_prior_nn_std=0.12,\n",
    "                                  hyper_prior_weight=1e-5, lr=1.5e-3, learn_likelihood=True,\n",
    "                                  random_seed=28, num_iter_meta_train=20000)\n",
    "\n",
    "pacoh_model.meta_fit(meta_test_data[:10], eval_period=10000, log_period=1000, plot_period=5000)\n",
    "\n",
    "eval_metrics_mean, eval_metrics_std = pacoh_model.meta_eval_datasets(meta_test_data)\n",
    "for key in eval_metrics_mean:\n",
    "    print(\"%s: %.4f +- %.4f\" % (key, eval_metrics_mean[key], eval_metrics_std[key]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start meta-training -------------------- \n",
      "\n",
      "Iter 0/20000 - Time 28.50 sec - Train-Loss: -9661.45020\n",
      "\n",
      "Iter 1000/20000 - Time 24.91 sec - Train-Loss: -312.95370\n",
      "\n",
      "Iter 2000/20000 - Time 19.71 sec - Train-Loss: -303.06601\n",
      "\n",
      "Iter 3000/20000 - Time 19.13 sec - Train-Loss: -515.22015\n",
      "\n",
      "Iter 4000/20000 - Time 19.61 sec - Train-Loss: -268.86929\n",
      "\n",
      "Iter 5000/20000 - Time 18.37 sec - Train-Loss: -247.08301\n",
      "\n",
      "Iter 6000/20000 - Time 20.74 sec - Train-Loss: -162.67421\n",
      "\n",
      "Iter 7000/20000 - Time 15.32 sec - Train-Loss: -118.15692\n",
      "\n",
      "Iter 8000/20000 - Time 18.88 sec - Train-Loss: -38.40429\n",
      "\n",
      "Iter 9000/20000 - Time 21.76 sec - Train-Loss: -28.07271\n",
      "\tStart meta-test posterior inference in 2 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\n",
      "Iter 10000/20000 - Time 18.29 sec - Train-Loss: -15.52446- Val-avg_ll: -1.148 +- 0.682- Val-avg_rmse: 0.274 +- 0.038- Val-cal_err: 0.135 +- 0.046\n",
      "\n",
      "Iter 11000/20000 - Time 30.77 sec - Train-Loss: -13.68072\n",
      "\n",
      "Iter 12000/20000 - Time 17.29 sec - Train-Loss: -11.07443\n",
      "\n",
      "Iter 13000/20000 - Time 19.31 sec - Train-Loss: -9.93320\n",
      "\n",
      "Iter 14000/20000 - Time 20.32 sec - Train-Loss: -8.90051\n",
      "\n",
      "Iter 15000/20000 - Time 19.34 sec - Train-Loss: -6.86962\n",
      "\n",
      "Iter 16000/20000 - Time 23.52 sec - Train-Loss: -6.28824\n",
      "\n",
      "Iter 17000/20000 - Time 19.81 sec - Train-Loss: -4.74267\n",
      "\n",
      "Iter 18000/20000 - Time 16.39 sec - Train-Loss: -4.73911\n",
      "\n",
      "Iter 19000/20000 - Time 20.57 sec - Train-Loss: -3.83733\n",
      "\tStart meta-test posterior inference in 4 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\tMeta-Test batch #3 consisting of 5 tasks----\n",
      "\tMeta-Test batch #4 consisting of 5 tasks----\n",
      "avg_ll: -0.2083 +- 0.1771\n",
      "avg_rmse: 0.2654 +- 0.0253\n",
      "cal_err: 0.0697 +- 0.0226\n"
     ]
    }
   ],
   "source": [
    "tf.get_logger().setLevel('ERROR')\n",
    "\n",
    "from pacoh_nn.datasets.regression_datasets import provide_data\n",
    "\n",
    "meta_train_data, meta_test_data, _ = provide_data(dataset='cauchy_20', n_samples = 5)\n",
    "\n",
    "from pacoh_nn.pacoh_nn_regression import PACOH_NN_Regression\n",
    "pacoh_model = PACOH_NN_Regression(meta_train_data, prior_weight=0.184, bandwidth=480.,\n",
    "                                  hyper_prior_likelihood_log_var_mean_mean=-1.0,\n",
    "                                  hyper_prior_log_var_mean=-1.74, hyper_prior_nn_std=0.12,\n",
    "                                  hyper_prior_weight=1e-5, lr=1.5e-3, learn_likelihood=True,\n",
    "                                  random_seed=28, num_iter_meta_train=20000)\n",
    "\n",
    "pacoh_model.meta_fit(meta_test_data[:10], eval_period=10000, log_period=1000, plot_period=5000)\n",
    "\n",
    "eval_metrics_mean, eval_metrics_std = pacoh_model.meta_eval_datasets(meta_test_data)\n",
    "for key in eval_metrics_mean:\n",
    "    print(\"%s: %.4f +- %.4f\" % (key, eval_metrics_mean[key], eval_metrics_std[key]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start meta-training -------------------- \n",
      "\n",
      "Iter 0/20000 - Time 46.63 sec - Train-Loss: -6224.47998\n",
      "\n",
      "Iter 1000/20000 - Time 16.80 sec - Train-Loss: -296.82806\n",
      "\n",
      "Iter 2000/20000 - Time 15.42 sec - Train-Loss: -296.96631\n",
      "\n",
      "Iter 3000/20000 - Time 15.20 sec - Train-Loss: -401.99216\n",
      "\n",
      "Iter 4000/20000 - Time 15.28 sec - Train-Loss: -248.58430\n",
      "\n",
      "Iter 5000/20000 - Time 14.93 sec - Train-Loss: -236.51128\n",
      "\n",
      "Iter 6000/20000 - Time 15.30 sec - Train-Loss: -134.54558\n",
      "\n",
      "Iter 7000/20000 - Time 16.49 sec - Train-Loss: -88.81742\n",
      "\n",
      "Iter 8000/20000 - Time 16.12 sec - Train-Loss: -32.36059\n",
      "\n",
      "Iter 9000/20000 - Time 15.24 sec - Train-Loss: -26.07649\n",
      "\tStart meta-test posterior inference in 2 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\n",
      "Iter 10000/20000 - Time 15.03 sec - Train-Loss: -14.84618- Val-avg_ll: -0.101 +- 0.187- Val-avg_rmse: 0.217 +- 0.016- Val-cal_err: 0.106 +- 0.037\n",
      "\n",
      "Iter 11000/20000 - Time 30.18 sec - Train-Loss: -12.98969\n",
      "\n",
      "Iter 12000/20000 - Time 22.99 sec - Train-Loss: -8.70574\n",
      "\n",
      "Iter 13000/20000 - Time 24.66 sec - Train-Loss: -8.76796\n",
      "\n",
      "Iter 14000/20000 - Time 16.49 sec - Train-Loss: -7.69474\n",
      "\n",
      "Iter 15000/20000 - Time 16.48 sec - Train-Loss: -6.70475\n",
      "\n",
      "Iter 16000/20000 - Time 16.40 sec - Train-Loss: -6.83723\n",
      "\n",
      "Iter 17000/20000 - Time 15.55 sec - Train-Loss: -4.86428\n",
      "\n",
      "Iter 18000/20000 - Time 15.12 sec - Train-Loss: -5.14440\n",
      "\n",
      "Iter 19000/20000 - Time 15.06 sec - Train-Loss: -4.73534\n",
      "\tStart meta-test posterior inference in 4 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\tMeta-Test batch #3 consisting of 5 tasks----\n",
      "\tMeta-Test batch #4 consisting of 5 tasks----\n",
      "avg_ll: 0.1142 +- 0.0909\n",
      "avg_rmse: 0.2123 +- 0.0177\n",
      "cal_err: 0.0458 +- 0.0394\n"
     ]
    }
   ],
   "source": [
    "tf.get_logger().setLevel('ERROR')\n",
    "\n",
    "from pacoh_nn.datasets.regression_datasets import provide_data\n",
    "\n",
    "meta_train_data, meta_test_data, _ = provide_data(dataset='cauchy_20', n_samples = 10)\n",
    "\n",
    "from pacoh_nn.pacoh_nn_regression import PACOH_NN_Regression\n",
    "pacoh_model = PACOH_NN_Regression(meta_train_data, prior_weight=0.184, bandwidth=480.,\n",
    "                                  hyper_prior_likelihood_log_var_mean_mean=-1.0,\n",
    "                                  hyper_prior_log_var_mean=-1.74, hyper_prior_nn_std=0.12,\n",
    "                                  hyper_prior_weight=1e-5, lr=1.5e-3, learn_likelihood=True,\n",
    "                                  random_seed=28, num_iter_meta_train=20000)\n",
    "\n",
    "pacoh_model.meta_fit(meta_test_data[:10], eval_period=10000, log_period=1000, plot_period=5000)\n",
    "\n",
    "eval_metrics_mean, eval_metrics_std = pacoh_model.meta_eval_datasets(meta_test_data)\n",
    "for key in eval_metrics_mean:\n",
    "    print(\"%s: %.4f +- %.4f\" % (key, eval_metrics_mean[key], eval_metrics_std[key]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start meta-training -------------------- \n",
      "\n",
      "Iter 0/20000 - Time 43.04 sec - Train-Loss: -5369.98779\n",
      "\n",
      "Iter 1000/20000 - Time 19.69 sec - Train-Loss: -308.59323\n",
      "\n",
      "Iter 2000/20000 - Time 16.97 sec - Train-Loss: -294.59036\n",
      "\n",
      "Iter 3000/20000 - Time 15.88 sec - Train-Loss: -423.54395\n",
      "\n",
      "Iter 4000/20000 - Time 15.60 sec - Train-Loss: -249.71324\n",
      "\n",
      "Iter 5000/20000 - Time 15.50 sec - Train-Loss: -254.67412\n",
      "\n",
      "Iter 6000/20000 - Time 15.55 sec - Train-Loss: -168.08333\n",
      "\n",
      "Iter 7000/20000 - Time 17.50 sec - Train-Loss: -91.29592\n",
      "\n",
      "Iter 8000/20000 - Time 17.37 sec - Train-Loss: -31.53190\n",
      "\n",
      "Iter 9000/20000 - Time 16.65 sec - Train-Loss: -20.13051\n",
      "\tStart meta-test posterior inference in 2 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\n",
      "Iter 10000/20000 - Time 16.02 sec - Train-Loss: -16.12270- Val-avg_ll: 0.252 +- 0.080- Val-avg_rmse: 0.187 +- 0.017- Val-cal_err: 0.055 +- 0.016\n",
      "\n",
      "Iter 11000/20000 - Time 25.58 sec - Train-Loss: -10.89875\n",
      "\n",
      "Iter 12000/20000 - Time 15.76 sec - Train-Loss: -9.27435\n",
      "\n",
      "Iter 13000/20000 - Time 15.81 sec - Train-Loss: -9.71636\n",
      "\n",
      "Iter 14000/20000 - Time 15.73 sec - Train-Loss: -6.79653\n",
      "\n",
      "Iter 15000/20000 - Time 15.69 sec - Train-Loss: -6.61995\n",
      "\n",
      "Iter 16000/20000 - Time 16.35 sec - Train-Loss: -6.91327\n",
      "\n",
      "Iter 17000/20000 - Time 17.54 sec - Train-Loss: -5.53860\n",
      "\n",
      "Iter 18000/20000 - Time 16.50 sec - Train-Loss: -4.38246\n",
      "\n",
      "Iter 19000/20000 - Time 16.00 sec - Train-Loss: -4.54746\n",
      "\tStart meta-test posterior inference in 4 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\tMeta-Test batch #3 consisting of 5 tasks----\n",
      "\tMeta-Test batch #4 consisting of 5 tasks----\n",
      "avg_ll: 0.3165 +- 0.0759\n",
      "avg_rmse: 0.1785 +- 0.0156\n",
      "cal_err: 0.0496 +- 0.0288\n"
     ]
    }
   ],
   "source": [
    "tf.get_logger().setLevel('ERROR')\n",
    "\n",
    "from pacoh_nn.datasets.regression_datasets import provide_data\n",
    "\n",
    "meta_train_data, meta_test_data, _ = provide_data(dataset='cauchy_20', n_samples = 80)\n",
    "\n",
    "from pacoh_nn.pacoh_nn_regression import PACOH_NN_Regression\n",
    "pacoh_model = PACOH_NN_Regression(meta_train_data, prior_weight=0.184, bandwidth=480.,\n",
    "                                  hyper_prior_likelihood_log_var_mean_mean=-1.0,\n",
    "                                  hyper_prior_log_var_mean=-1.74, hyper_prior_nn_std=0.12,\n",
    "                                  hyper_prior_weight=1e-5, lr=1.5e-3, learn_likelihood=True,\n",
    "                                  random_seed=28, num_iter_meta_train=20000)\n",
    "\n",
    "pacoh_model.meta_fit(meta_test_data[:10], eval_period=10000, log_period=1000, plot_period=5000)\n",
    "\n",
    "eval_metrics_mean, eval_metrics_std = pacoh_model.meta_eval_datasets(meta_test_data)\n",
    "for key in eval_metrics_mean:\n",
    "    print(\"%s: %.4f +- %.4f\" % (key, eval_metrics_mean[key], eval_metrics_std[key]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "Start meta-training -------------------- \n",
      "\n",
      "Iter 0/20000 - Time 30.72 sec - Train-Loss: -5722.87646\n",
      "\n",
      "Iter 1000/20000 - Time 15.92 sec - Train-Loss: -309.99338\n",
      "\n",
      "Iter 2000/20000 - Time 15.50 sec - Train-Loss: -284.68814\n",
      "\n",
      "Iter 3000/20000 - Time 15.32 sec - Train-Loss: -498.37878\n",
      "\n",
      "Iter 4000/20000 - Time 15.23 sec - Train-Loss: -224.98041\n",
      "\n",
      "Iter 5000/20000 - Time 17.83 sec - Train-Loss: -149.14165\n",
      "\n",
      "Iter 6000/20000 - Time 21.55 sec - Train-Loss: -98.89811\n",
      "\n",
      "Iter 7000/20000 - Time 23.93 sec - Train-Loss: -45.54832\n",
      "\n",
      "Iter 8000/20000 - Time 19.80 sec - Train-Loss: -22.44049\n",
      "\n",
      "Iter 9000/20000 - Time 15.45 sec - Train-Loss: -18.24091\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "\tStart meta-test posterior inference in 2 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\n",
      "Iter 10000/20000 - Time 15.40 sec - Train-Loss: -16.53016- Val-avg_ll: -1.369 +- 0.843- Val-avg_rmse: 0.324 +- 0.045- Val-cal_err: 0.153 +- 0.102\n",
      "\n",
      "Iter 11000/20000 - Time 27.18 sec - Train-Loss: -13.77258\n",
      "\n",
      "Iter 12000/20000 - Time 15.69 sec - Train-Loss: -11.54720\n",
      "\n",
      "Iter 13000/20000 - Time 15.70 sec - Train-Loss: -11.53994\n",
      "\n",
      "Iter 14000/20000 - Time 15.31 sec - Train-Loss: -8.63615\n",
      "\n",
      "Iter 15000/20000 - Time 15.42 sec - Train-Loss: -6.91055\n",
      "\n",
      "Iter 16000/20000 - Time 15.29 sec - Train-Loss: -7.51274\n",
      "\n",
      "Iter 17000/20000 - Time 15.35 sec - Train-Loss: -5.07543\n",
      "\n",
      "Iter 18000/20000 - Time 15.27 sec - Train-Loss: -4.27170\n",
      "\n",
      "Iter 19000/20000 - Time 16.05 sec - Train-Loss: -4.96172\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `2`\n",
      "\tStart meta-test posterior inference in 4 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\tMeta-Test batch #3 consisting of 5 tasks----\n",
      "\tMeta-Test batch #4 consisting of 5 tasks----\n",
      "avg_ll: -0.3956 +- 0.2367\n",
      "avg_rmse: 0.2994 +- 0.0366\n",
      "cal_err: 0.1377 +- 0.0650\n"
     ]
    }
   ],
   "source": [
    "tf.get_logger().setLevel('ERROR')\n",
    "\n",
    "from pacoh_nn.datasets.regression_datasets import provide_data\n",
    "\n",
    "meta_train_data, meta_test_data, _ = provide_data(dataset='cauchy_20', n_samples = 2)\n",
    "\n",
    "from pacoh_nn.pacoh_nn_regression import PACOH_NN_Regression\n",
    "pacoh_model = PACOH_NN_Regression(meta_train_data, prior_weight=0.184, bandwidth=480.,\n",
    "                                  hyper_prior_likelihood_log_var_mean_mean=-1.0,\n",
    "                                  hyper_prior_log_var_mean=-1.74, hyper_prior_nn_std=0.12,\n",
    "                                  hyper_prior_weight=1e-5, lr=1.5e-3, learn_likelihood=True,\n",
    "                                  random_seed=28, num_iter_meta_train=20000)\n",
    "\n",
    "pacoh_model.meta_fit(meta_test_data[:10], eval_period=10000, log_period=1000, plot_period=5000)\n",
    "\n",
    "eval_metrics_mean, eval_metrics_std = pacoh_model.meta_eval_datasets(meta_test_data)\n",
    "for key in eval_metrics_mean:\n",
    "    print(\"%s: %.4f +- %.4f\" % (key, eval_metrics_mean[key], eval_metrics_std[key]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start meta-training -------------------- \n",
      "\n",
      "Iter 0/20000 - Time 34.04 sec - Train-Loss: -4388.93604\n",
      "\n",
      "Iter 1000/20000 - Time 15.83 sec - Train-Loss: -306.98764\n",
      "\n",
      "Iter 2000/20000 - Time 15.29 sec - Train-Loss: -291.33847\n",
      "\n",
      "Iter 3000/20000 - Time 19.48 sec - Train-Loss: -470.93790\n",
      "\n",
      "Iter 4000/20000 - Time 17.99 sec - Train-Loss: -245.43079\n",
      "\n",
      "Iter 5000/20000 - Time 15.82 sec - Train-Loss: -234.92070\n",
      "\n",
      "Iter 6000/20000 - Time 15.18 sec - Train-Loss: -137.69014\n",
      "\n",
      "Iter 7000/20000 - Time 15.03 sec - Train-Loss: -84.36649\n",
      "\n",
      "Iter 8000/20000 - Time 15.04 sec - Train-Loss: -30.08511\n",
      "\n",
      "Iter 9000/20000 - Time 15.42 sec - Train-Loss: -19.09468\n",
      "\tStart meta-test posterior inference in 2 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\n",
      "Iter 10000/20000 - Time 15.69 sec - Train-Loss: -13.77795- Val-avg_ll: 0.152 +- 0.101- Val-avg_rmse: 0.202 +- 0.014- Val-cal_err: 0.061 +- 0.037\n",
      "\n",
      "Iter 11000/20000 - Time 24.42 sec - Train-Loss: -11.60813\n",
      "\n",
      "Iter 12000/20000 - Time 15.31 sec - Train-Loss: -11.64622\n",
      "\n",
      "Iter 13000/20000 - Time 16.50 sec - Train-Loss: -9.33721\n",
      "\n",
      "Iter 14000/20000 - Time 17.34 sec - Train-Loss: -7.87783\n",
      "\n",
      "Iter 15000/20000 - Time 15.68 sec - Train-Loss: -6.41316\n",
      "\n",
      "Iter 16000/20000 - Time 15.18 sec - Train-Loss: -6.01440\n",
      "\n",
      "Iter 17000/20000 - Time 15.19 sec - Train-Loss: -5.91379\n",
      "\n",
      "Iter 18000/20000 - Time 15.17 sec - Train-Loss: -5.21844\n",
      "\n",
      "Iter 19000/20000 - Time 15.51 sec - Train-Loss: -5.43933\n",
      "\tStart meta-test posterior inference in 4 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\tMeta-Test batch #3 consisting of 5 tasks----\n",
      "\tMeta-Test batch #4 consisting of 5 tasks----\n",
      "avg_ll: 0.1888 +- 0.1045\n",
      "avg_rmse: 0.1998 +- 0.0196\n",
      "cal_err: 0.0571 +- 0.0282\n"
     ]
    }
   ],
   "source": [
    "tf.get_logger().setLevel('ERROR')\n",
    "\n",
    "from pacoh_nn.datasets.regression_datasets import provide_data\n",
    "\n",
    "meta_train_data, meta_test_data, _ = provide_data(dataset='cauchy_20', n_samples = 30)\n",
    "\n",
    "from pacoh_nn.pacoh_nn_regression import PACOH_NN_Regression\n",
    "pacoh_model = PACOH_NN_Regression(meta_train_data, prior_weight=0.184, bandwidth=480.,\n",
    "                                  hyper_prior_likelihood_log_var_mean_mean=-1.0,\n",
    "                                  hyper_prior_log_var_mean=-1.74, hyper_prior_nn_std=0.12,\n",
    "                                  hyper_prior_weight=1e-5, lr=1.5e-3, learn_likelihood=True,\n",
    "                                  random_seed=28, num_iter_meta_train=20000)\n",
    "\n",
    "pacoh_model.meta_fit(meta_test_data[:10], eval_period=10000, log_period=1000, plot_period=5000)\n",
    "\n",
    "eval_metrics_mean, eval_metrics_std = pacoh_model.meta_eval_datasets(meta_test_data)\n",
    "for key in eval_metrics_mean:\n",
    "    print(\"%s: %.4f +- %.4f\" % (key, eval_metrics_mean[key], eval_metrics_std[key]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start meta-training -------------------- \n",
      "\n",
      "Iter 0/20000 - Time 27.91 sec - Train-Loss: -3372.07153\n",
      "\n",
      "Iter 1000/20000 - Time 16.69 sec - Train-Loss: -291.00070\n",
      "\n",
      "Iter 2000/20000 - Time 15.98 sec - Train-Loss: -292.75684\n",
      "\n",
      "Iter 3000/20000 - Time 15.28 sec - Train-Loss: -419.66513\n",
      "\n",
      "Iter 4000/20000 - Time 15.65 sec - Train-Loss: -257.01187\n",
      "\n",
      "Iter 5000/20000 - Time 19.44 sec - Train-Loss: -218.27399\n",
      "\n",
      "Iter 6000/20000 - Time 16.42 sec - Train-Loss: -142.88303\n",
      "\n",
      "Iter 7000/20000 - Time 15.47 sec - Train-Loss: -105.43353\n",
      "\n",
      "Iter 8000/20000 - Time 17.76 sec - Train-Loss: -49.36441\n",
      "\n",
      "Iter 9000/20000 - Time 16.53 sec - Train-Loss: -22.30229\n",
      "\tStart meta-test posterior inference in 2 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\n",
      "Iter 10000/20000 - Time 15.79 sec - Train-Loss: -13.48880- Val-avg_ll: 0.260 +- 0.136- Val-avg_rmse: 0.187 +- 0.022- Val-cal_err: 0.043 +- 0.020\n",
      "\n",
      "Iter 11000/20000 - Time 24.96 sec - Train-Loss: -11.93734\n",
      "\n",
      "Iter 12000/20000 - Time 16.08 sec - Train-Loss: -9.28812\n",
      "\n",
      "Iter 13000/20000 - Time 16.07 sec - Train-Loss: -9.58772\n",
      "\n",
      "Iter 14000/20000 - Time 16.69 sec - Train-Loss: -7.37932\n",
      "\n",
      "Iter 15000/20000 - Time 17.50 sec - Train-Loss: -7.13390\n",
      "\n",
      "Iter 16000/20000 - Time 19.43 sec - Train-Loss: -6.79008\n",
      "\n",
      "Iter 17000/20000 - Time 17.31 sec - Train-Loss: -5.72781\n",
      "\n",
      "Iter 18000/20000 - Time 19.20 sec - Train-Loss: -4.82783\n",
      "\n",
      "Iter 19000/20000 - Time 20.99 sec - Train-Loss: -6.50581\n",
      "\tStart meta-test posterior inference in 4 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\tMeta-Test batch #3 consisting of 5 tasks----\n",
      "\tMeta-Test batch #4 consisting of 5 tasks----\n",
      "avg_ll: 0.3109 +- 0.1149\n",
      "avg_rmse: 0.1777 +- 0.0220\n",
      "cal_err: 0.0386 +- 0.0183\n"
     ]
    }
   ],
   "source": [
    "tf.get_logger().setLevel('ERROR')\n",
    "\n",
    "from pacoh_nn.datasets.regression_datasets import provide_data\n",
    "\n",
    "meta_train_data, meta_test_data, _ = provide_data(dataset='cauchy_20', n_samples = 60)\n",
    "\n",
    "from pacoh_nn.pacoh_nn_regression import PACOH_NN_Regression\n",
    "pacoh_model = PACOH_NN_Regression(meta_train_data, prior_weight=0.184, bandwidth=480.,\n",
    "                                  hyper_prior_likelihood_log_var_mean_mean=-1.0,\n",
    "                                  hyper_prior_log_var_mean=-1.74, hyper_prior_nn_std=0.12,\n",
    "                                  hyper_prior_weight=1e-5, lr=1.5e-3, learn_likelihood=True,\n",
    "                                  random_seed=28, num_iter_meta_train=20000)\n",
    "\n",
    "pacoh_model.meta_fit(meta_test_data[:10], eval_period=10000, log_period=1000, plot_period=5000)\n",
    "\n",
    "eval_metrics_mean, eval_metrics_std = pacoh_model.meta_eval_datasets(meta_test_data)\n",
    "for key in eval_metrics_mean:\n",
    "    print(\"%s: %.4f +- %.4f\" % (key, eval_metrics_mean[key], eval_metrics_std[key]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start meta-training -------------------- \n",
      "\n",
      "Iter 0/20000 - Time 34.17 sec - Train-Loss: -7766.59375\n",
      "\n",
      "Iter 1000/20000 - Time 16.58 sec - Train-Loss: -315.27258\n",
      "\n",
      "Iter 2000/20000 - Time 16.10 sec - Train-Loss: -298.42432\n",
      "\n",
      "Iter 3000/20000 - Time 15.36 sec - Train-Loss: -387.34042\n",
      "\n",
      "Iter 4000/20000 - Time 15.11 sec - Train-Loss: -260.26794\n",
      "\n",
      "Iter 5000/20000 - Time 15.25 sec - Train-Loss: -203.97554\n",
      "\n",
      "Iter 6000/20000 - Time 15.11 sec - Train-Loss: -134.70589\n",
      "\n",
      "Iter 7000/20000 - Time 15.82 sec - Train-Loss: -85.14509\n",
      "\n",
      "Iter 8000/20000 - Time 16.55 sec - Train-Loss: -33.54728\n",
      "\n",
      "Iter 9000/20000 - Time 16.39 sec - Train-Loss: -25.69529\n",
      "\tStart meta-test posterior inference in 2 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\n",
      "Iter 10000/20000 - Time 15.24 sec - Train-Loss: -13.55351- Val-avg_ll: -0.144 +- 0.265- Val-avg_rmse: 0.231 +- 0.026- Val-cal_err: 0.076 +- 0.037\n",
      "\n",
      "Iter 11000/20000 - Time 25.18 sec - Train-Loss: -12.02034\n",
      "\n",
      "Iter 12000/20000 - Time 15.58 sec - Train-Loss: -10.11500\n",
      "\n",
      "Iter 13000/20000 - Time 15.82 sec - Train-Loss: -10.46142\n",
      "\n",
      "Iter 14000/20000 - Time 15.61 sec - Train-Loss: -8.74646\n",
      "\n",
      "Iter 15000/20000 - Time 15.66 sec - Train-Loss: -7.03988\n",
      "\n",
      "Iter 16000/20000 - Time 15.40 sec - Train-Loss: -6.44552\n",
      "\n",
      "Iter 17000/20000 - Time 16.26 sec - Train-Loss: -4.91511\n",
      "\n",
      "Iter 18000/20000 - Time 16.56 sec - Train-Loss: -4.75845\n",
      "\n",
      "Iter 19000/20000 - Time 15.95 sec - Train-Loss: -4.73403\n",
      "\tStart meta-test posterior inference in 4 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\tMeta-Test batch #3 consisting of 5 tasks----\n",
      "\tMeta-Test batch #4 consisting of 5 tasks----\n",
      "avg_ll: 0.0590 +- 0.1138\n",
      "avg_rmse: 0.2246 +- 0.0228\n",
      "cal_err: 0.0714 +- 0.0453\n"
     ]
    }
   ],
   "source": [
    "tf.get_logger().setLevel('ERROR')\n",
    "\n",
    "from pacoh_nn.datasets.regression_datasets import provide_data\n",
    "\n",
    "meta_train_data, meta_test_data, _ = provide_data(dataset='cauchy_20', n_samples = 15)\n",
    "\n",
    "from pacoh_nn.pacoh_nn_regression import PACOH_NN_Regression\n",
    "pacoh_model = PACOH_NN_Regression(meta_train_data, prior_weight=0.184, bandwidth=480.,\n",
    "                                  hyper_prior_likelihood_log_var_mean_mean=-1.0,\n",
    "                                  hyper_prior_log_var_mean=-1.74, hyper_prior_nn_std=0.12,\n",
    "                                  hyper_prior_weight=1e-5, lr=1.5e-3, learn_likelihood=True,\n",
    "                                  random_seed=28, num_iter_meta_train=20000)\n",
    "\n",
    "pacoh_model.meta_fit(meta_test_data[:10], eval_period=10000, log_period=1000, plot_period=5000)\n",
    "\n",
    "eval_metrics_mean, eval_metrics_std = pacoh_model.meta_eval_datasets(meta_test_data)\n",
    "for key in eval_metrics_mean:\n",
    "    print(\"%s: %.4f +- %.4f\" % (key, eval_metrics_mean[key], eval_metrics_std[key]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start meta-training -------------------- \n",
      "\n",
      "Iter 0/20000 - Time 26.60 sec - Train-Loss: -2819.89478\n",
      "\n",
      "Iter 1000/20000 - Time 15.37 sec - Train-Loss: -330.34238\n",
      "\n",
      "Iter 2000/20000 - Time 15.34 sec - Train-Loss: -300.46216\n",
      "\n",
      "Iter 3000/20000 - Time 15.37 sec - Train-Loss: -393.04745\n",
      "\n",
      "Iter 4000/20000 - Time 15.38 sec - Train-Loss: -252.56081\n",
      "\n",
      "Iter 5000/20000 - Time 15.91 sec - Train-Loss: -256.84921\n",
      "\n",
      "Iter 6000/20000 - Time 15.48 sec - Train-Loss: -149.62935\n",
      "\n",
      "Iter 7000/20000 - Time 15.29 sec - Train-Loss: -103.08224\n",
      "\n",
      "Iter 8000/20000 - Time 15.42 sec - Train-Loss: -35.62235\n",
      "\n",
      "Iter 9000/20000 - Time 16.72 sec - Train-Loss: -23.22264\n",
      "\tStart meta-test posterior inference in 2 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\n",
      "Iter 10000/20000 - Time 16.61 sec - Train-Loss: -15.01887- Val-avg_ll: 0.166 +- 0.087- Val-avg_rmse: 0.200 +- 0.013- Val-cal_err: 0.049 +- 0.032\n",
      "\n",
      "Iter 11000/20000 - Time 24.02 sec - Train-Loss: -11.11533\n",
      "\n",
      "Iter 12000/20000 - Time 17.61 sec - Train-Loss: -11.14481\n",
      "\n",
      "Iter 13000/20000 - Time 16.32 sec - Train-Loss: -9.75591\n",
      "\n",
      "Iter 14000/20000 - Time 15.53 sec - Train-Loss: -8.16839\n",
      "\n",
      "Iter 15000/20000 - Time 15.42 sec - Train-Loss: -6.60986\n",
      "\n",
      "Iter 16000/20000 - Time 15.43 sec - Train-Loss: -5.77873\n",
      "\n",
      "Iter 17000/20000 - Time 15.33 sec - Train-Loss: -5.81433\n",
      "\n",
      "Iter 18000/20000 - Time 15.70 sec - Train-Loss: -5.61976\n",
      "\n",
      "Iter 19000/20000 - Time 15.69 sec - Train-Loss: -6.60725\n",
      "\tStart meta-test posterior inference in 4 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\tMeta-Test batch #3 consisting of 5 tasks----\n",
      "\tMeta-Test batch #4 consisting of 5 tasks----\n",
      "avg_ll: 0.2490 +- 0.0813\n",
      "avg_rmse: 0.1857 +- 0.0171\n",
      "cal_err: 0.0477 +- 0.0243\n"
     ]
    }
   ],
   "source": [
    "tf.get_logger().setLevel('ERROR')\n",
    "\n",
    "from pacoh_nn.datasets.regression_datasets import provide_data\n",
    "\n",
    "meta_train_data, meta_test_data, _ = provide_data(dataset='cauchy_20', n_samples = 50)\n",
    "\n",
    "from pacoh_nn.pacoh_nn_regression import PACOH_NN_Regression\n",
    "pacoh_model = PACOH_NN_Regression(meta_train_data, prior_weight=0.184, bandwidth=480.,\n",
    "                                  hyper_prior_likelihood_log_var_mean_mean=-1.0,\n",
    "                                  hyper_prior_log_var_mean=-1.74, hyper_prior_nn_std=0.12,\n",
    "                                  hyper_prior_weight=1e-5, lr=1.5e-3, learn_likelihood=True,\n",
    "                                  random_seed=28, num_iter_meta_train=20000)\n",
    "\n",
    "pacoh_model.meta_fit(meta_test_data[:10], eval_period=10000, log_period=1000, plot_period=5000)\n",
    "\n",
    "eval_metrics_mean, eval_metrics_std = pacoh_model.meta_eval_datasets(meta_test_data)\n",
    "for key in eval_metrics_mean:\n",
    "    print(\"%s: %.4f +- %.4f\" % (key, eval_metrics_mean[key], eval_metrics_std[key]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start meta-training -------------------- \n",
      "\n",
      "Iter 0/20000 - Time 32.42 sec - Train-Loss: -5461.99414\n",
      "\n",
      "Iter 1000/20000 - Time 17.00 sec - Train-Loss: -291.55627\n",
      "\n",
      "Iter 2000/20000 - Time 15.63 sec - Train-Loss: -294.72195\n",
      "\n",
      "Iter 3000/20000 - Time 16.94 sec - Train-Loss: -441.15787\n",
      "\n",
      "Iter 4000/20000 - Time 16.36 sec - Train-Loss: -253.43150\n",
      "\n",
      "Iter 5000/20000 - Time 15.57 sec - Train-Loss: -228.54820\n",
      "\n",
      "Iter 6000/20000 - Time 15.46 sec - Train-Loss: -127.64603\n",
      "\n",
      "Iter 7000/20000 - Time 15.26 sec - Train-Loss: -82.68899\n",
      "\n",
      "Iter 8000/20000 - Time 15.75 sec - Train-Loss: -37.06221\n",
      "\n",
      "Iter 9000/20000 - Time 16.10 sec - Train-Loss: -17.80065\n",
      "\tStart meta-test posterior inference in 2 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\n",
      "Iter 10000/20000 - Time 15.36 sec - Train-Loss: -13.61953- Val-avg_ll: 0.345 +- 0.080- Val-avg_rmse: 0.168 +- 0.014- Val-cal_err: 0.049 +- 0.022\n",
      "\n",
      "Iter 11000/20000 - Time 24.76 sec - Train-Loss: -9.70923\n",
      "\n",
      "Iter 12000/20000 - Time 15.76 sec - Train-Loss: -9.40251\n",
      "\n",
      "Iter 13000/20000 - Time 15.51 sec - Train-Loss: -8.78774\n",
      "\n",
      "Iter 14000/20000 - Time 15.61 sec - Train-Loss: -6.45897\n",
      "\n",
      "Iter 15000/20000 - Time 17.00 sec - Train-Loss: -5.61505\n",
      "\n",
      "Iter 16000/20000 - Time 16.66 sec - Train-Loss: -6.16734\n",
      "\n",
      "Iter 17000/20000 - Time 15.73 sec - Train-Loss: -6.15768\n",
      "\n",
      "Iter 18000/20000 - Time 15.59 sec - Train-Loss: -4.56017\n",
      "\n",
      "Iter 19000/20000 - Time 15.52 sec - Train-Loss: -4.50011\n",
      "\tStart meta-test posterior inference in 4 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\tMeta-Test batch #3 consisting of 5 tasks----\n",
      "\tMeta-Test batch #4 consisting of 5 tasks----\n",
      "avg_ll: 0.4358 +- 0.0794\n",
      "avg_rmse: 0.1537 +- 0.0147\n",
      "cal_err: 0.0465 +- 0.0183\n"
     ]
    }
   ],
   "source": [
    "tf.get_logger().setLevel('ERROR')\n",
    "\n",
    "from pacoh_nn.datasets.regression_datasets import provide_data\n",
    "\n",
    "meta_train_data, meta_test_data, _ = provide_data(dataset='cauchy_20', n_samples = 100)\n",
    "\n",
    "from pacoh_nn.pacoh_nn_regression import PACOH_NN_Regression\n",
    "pacoh_model = PACOH_NN_Regression(meta_train_data, prior_weight=0.184, bandwidth=480.,\n",
    "                                  hyper_prior_likelihood_log_var_mean_mean=-1.0,\n",
    "                                  hyper_prior_log_var_mean=-1.74, hyper_prior_nn_std=0.12,\n",
    "                                  hyper_prior_weight=1e-5, lr=1.5e-3, learn_likelihood=True,\n",
    "                                  random_seed=28, num_iter_meta_train=20000)\n",
    "\n",
    "pacoh_model.meta_fit(meta_test_data[:10], eval_period=10000, log_period=1000, plot_period=5000)\n",
    "\n",
    "eval_metrics_mean, eval_metrics_std = pacoh_model.meta_eval_datasets(meta_test_data)\n",
    "for key in eval_metrics_mean:\n",
    "    print(\"%s: %.4f +- %.4f\" % (key, eval_metrics_mean[key], eval_metrics_std[key]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "Start meta-training -------------------- \n",
      "\n",
      "Iter 0/20000 - Time 26.72 sec - Train-Loss: -1413.88464\n",
      "\n",
      "Iter 1000/20000 - Time 15.41 sec - Train-Loss: -334.18805\n",
      "\n",
      "Iter 2000/20000 - Time 15.60 sec - Train-Loss: -271.20828\n",
      "\n",
      "Iter 3000/20000 - Time 15.16 sec - Train-Loss: -308.17081\n",
      "\n",
      "Iter 4000/20000 - Time 15.32 sec - Train-Loss: -212.03194\n",
      "\n",
      "Iter 5000/20000 - Time 15.38 sec - Train-Loss: -156.38701\n",
      "\n",
      "Iter 6000/20000 - Time 17.56 sec - Train-Loss: -98.49091\n",
      "\n",
      "Iter 7000/20000 - Time 16.64 sec - Train-Loss: -52.56311\n",
      "\n",
      "Iter 8000/20000 - Time 16.14 sec - Train-Loss: -21.07015\n",
      "\n",
      "Iter 9000/20000 - Time 15.30 sec - Train-Loss: -22.60230\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "\tStart meta-test posterior inference in 2 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\n",
      "Iter 10000/20000 - Time 15.22 sec - Train-Loss: -15.46674- Val-avg_ll: -12.700 +- 14.202- Val-avg_rmse: 0.498 +- 0.147- Val-cal_err: 0.191 +- 0.098\n",
      "\n",
      "Iter 11000/20000 - Time 29.46 sec - Train-Loss: -13.00710\n",
      "\n",
      "Iter 12000/20000 - Time 18.04 sec - Train-Loss: -13.06448\n",
      "\n",
      "Iter 13000/20000 - Time 16.95 sec - Train-Loss: -11.86900\n",
      "\n",
      "Iter 14000/20000 - Time 17.23 sec - Train-Loss: -12.38401\n",
      "\n",
      "Iter 15000/20000 - Time 17.69 sec - Train-Loss: -9.34153\n",
      "\n",
      "Iter 16000/20000 - Time 18.26 sec - Train-Loss: -8.25531\n",
      "\n",
      "Iter 17000/20000 - Time 17.22 sec - Train-Loss: -5.55534\n",
      "\n",
      "Iter 18000/20000 - Time 16.08 sec - Train-Loss: -7.62802\n",
      "\n",
      "Iter 19000/20000 - Time 17.24 sec - Train-Loss: -4.35637\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "NOTE: The requested batch size `5` is bigger the number of training samples `1`\n",
      "\tStart meta-test posterior inference in 4 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\tMeta-Test batch #3 consisting of 5 tasks----\n",
      "\tMeta-Test batch #4 consisting of 5 tasks----\n",
      "avg_ll: -4.9725 +- 3.8543\n",
      "avg_rmse: 0.5101 +- 0.1292\n",
      "cal_err: 0.2005 +- 0.0672\n"
     ]
    }
   ],
   "source": [
    "tf.get_logger().setLevel('ERROR')\n",
    "\n",
    "from pacoh_nn.datasets.regression_datasets import provide_data\n",
    "\n",
    "meta_train_data, meta_test_data, _ = provide_data(dataset='cauchy_20', n_samples = 1)\n",
    "\n",
    "from pacoh_nn.pacoh_nn_regression import PACOH_NN_Regression\n",
    "pacoh_model = PACOH_NN_Regression(meta_train_data, prior_weight=0.184, bandwidth=480.,\n",
    "                                  hyper_prior_likelihood_log_var_mean_mean=-1.0,\n",
    "                                  hyper_prior_log_var_mean=-1.74, hyper_prior_nn_std=0.12,\n",
    "                                  hyper_prior_weight=1e-5, lr=1.5e-3, learn_likelihood=True,\n",
    "                                  random_seed=28, num_iter_meta_train=20000)\n",
    "\n",
    "pacoh_model.meta_fit(meta_test_data[:10], eval_period=10000, log_period=1000, plot_period=5000)\n",
    "\n",
    "eval_metrics_mean, eval_metrics_std = pacoh_model.meta_eval_datasets(meta_test_data)\n",
    "for key in eval_metrics_mean:\n",
    "    print(\"%s: %.4f +- %.4f\" % (key, eval_metrics_mean[key], eval_metrics_std[key]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "hyper_invest",
   "language": "python",
   "name": "hyper_invest"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}