TuningParameters.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import TimeSeriesSplit\n",
    "from sklearn.neural_network import MLPRegressor\n",
    "from sklearn.ensemble import GradientBoostingRegressor\n",
    "from sklearn.svm import SVR\n",
    "\n",
    "import pickle\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import  mean_absolute_error\n",
    "from sklearn.metrics import make_scorer\n",
    "\n",
    "\n",
    "from imp import reload\n",
    "import MyPrepaData\n",
    "reload(MyPrepaData)\n",
    "from MyEvalCriteria import mae,rmse,corr,KGE,ia,BHV\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['stl3', 'd2m', 'tcwv', 'tcw', 'stl2', 'mn2t', 'tsn', 'stl4'], dtype='object', name=1)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "micQ1 = pd.read_excel('result.xls',index_col=1,header=None)\n",
    "reanalysis = micQ1.index[:8]\n",
    "reanalysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 1 folds for each of 40000 candidates, totalling 40000 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  26 tasks      | elapsed:    3.9s\n",
      "[Parallel(n_jobs=-1)]: Done 332 tasks      | elapsed:    6.3s\n",
      "[Parallel(n_jobs=-1)]: Done 1332 tasks      | elapsed:   14.3s\n",
      "[Parallel(n_jobs=-1)]: Done 2732 tasks      | elapsed:   26.3s\n",
      "[Parallel(n_jobs=-1)]: Done 4532 tasks      | elapsed:   45.1s\n",
      "[Parallel(n_jobs=-1)]: Done 6732 tasks      | elapsed:  1.3min\n",
      "[Parallel(n_jobs=-1)]: Done 8102 tasks      | elapsed:  1.7min\n",
      "[Parallel(n_jobs=-1)]: Done 9602 tasks      | elapsed:  2.3min\n",
      "[Parallel(n_jobs=-1)]: Done 11302 tasks      | elapsed:  3.0min\n",
      "[Parallel(n_jobs=-1)]: Done 13202 tasks      | elapsed:  3.8min\n",
      "[Parallel(n_jobs=-1)]: Done 15302 tasks      | elapsed:  4.9min\n",
      "[Parallel(n_jobs=-1)]: Done 17602 tasks      | elapsed:  6.1min\n",
      "[Parallel(n_jobs=-1)]: Done 19997 tasks      | elapsed:  7.5min\n",
      "[Parallel(n_jobs=-1)]: Done 21347 tasks      | elapsed:  8.2min\n",
      "[Parallel(n_jobs=-1)]: Done 22797 tasks      | elapsed:  9.2min\n",
      "[Parallel(n_jobs=-1)]: Done 24347 tasks      | elapsed: 10.2min\n",
      "[Parallel(n_jobs=-1)]: Done 25997 tasks      | elapsed: 11.2min\n",
      "[Parallel(n_jobs=-1)]: Done 27747 tasks      | elapsed: 12.6min\n",
      "[Parallel(n_jobs=-1)]: Done 29597 tasks      | elapsed: 13.7min\n",
      "[Parallel(n_jobs=-1)]: Done 31547 tasks      | elapsed: 15.3min\n",
      "[Parallel(n_jobs=-1)]: Done 33597 tasks      | elapsed: 16.6min\n",
      "[Parallel(n_jobs=-1)]: Done 35747 tasks      | elapsed: 18.6min\n",
      "[Parallel(n_jobs=-1)]: Done 37997 tasks      | elapsed: 20.1min\n",
      "[Parallel(n_jobs=-1)]: Done 40000 out of 40000 | elapsed: 22.1min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 22min 4s\n",
      "{'max_depth': 3, 'max_leaf_nodes': 8, 'min_samples_leaf': 6, 'min_samples_split': 16} -125.5868285156934\n",
      "Fitting 1 folds for each of 40000 candidates, totalling 40000 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  32 tasks      | elapsed:    0.6s\n",
      "[Parallel(n_jobs=-1)]: Done 632 tasks      | elapsed:    5.8s\n",
      "[Parallel(n_jobs=-1)]: Done 1632 tasks      | elapsed:   14.6s\n",
      "[Parallel(n_jobs=-1)]: Done 3032 tasks      | elapsed:   27.4s\n",
      "[Parallel(n_jobs=-1)]: Done 4832 tasks      | elapsed:   49.8s\n",
      "[Parallel(n_jobs=-1)]: Done 7032 tasks      | elapsed:  1.4min\n",
      "[Parallel(n_jobs=-1)]: Done 9010 tasks      | elapsed:  2.0min\n",
      "[Parallel(n_jobs=-1)]: Done 10510 tasks      | elapsed:  2.6min\n",
      "[Parallel(n_jobs=-1)]: Done 12210 tasks      | elapsed:  3.2min\n",
      "[Parallel(n_jobs=-1)]: Done 14110 tasks      | elapsed:  4.1min\n",
      "[Parallel(n_jobs=-1)]: Done 16210 tasks      | elapsed:  5.1min\n",
      "[Parallel(n_jobs=-1)]: Done 18202 tasks      | elapsed:  6.2min\n",
      "[Parallel(n_jobs=-1)]: Done 19452 tasks      | elapsed:  7.0min\n",
      "[Parallel(n_jobs=-1)]: Done 20802 tasks      | elapsed:  7.6min\n",
      "[Parallel(n_jobs=-1)]: Done 22252 tasks      | elapsed:  8.6min\n",
      "[Parallel(n_jobs=-1)]: Done 23802 tasks      | elapsed:  9.8min\n",
      "[Parallel(n_jobs=-1)]: Done 25452 tasks      | elapsed: 10.7min\n",
      "[Parallel(n_jobs=-1)]: Done 27202 tasks      | elapsed: 12.0min\n",
      "[Parallel(n_jobs=-1)]: Done 29052 tasks      | elapsed: 13.2min\n",
      "[Parallel(n_jobs=-1)]: Done 31002 tasks      | elapsed: 15.0min\n",
      "[Parallel(n_jobs=-1)]: Done 33052 tasks      | elapsed: 16.4min\n",
      "[Parallel(n_jobs=-1)]: Done 35202 tasks      | elapsed: 18.4min\n",
      "[Parallel(n_jobs=-1)]: Done 37452 tasks      | elapsed: 20.0min\n",
      "[Parallel(n_jobs=-1)]: Done 39802 tasks      | elapsed: 22.3min\n",
      "[Parallel(n_jobs=-1)]: Done 40000 out of 40000 | elapsed: 22.5min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 22min 30s\n",
      "{'max_depth': 5, 'max_leaf_nodes': 10, 'min_samples_leaf': 6, 'min_samples_split': 16} -153.84379710965814\n",
      "Fitting 1 folds for each of 40000 candidates, totalling 40000 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  28 tasks      | elapsed:    0.4s\n",
      "[Parallel(n_jobs=-1)]: Done 328 tasks      | elapsed:    3.5s\n",
      "[Parallel(n_jobs=-1)]: Done 828 tasks      | elapsed:    8.6s\n",
      "[Parallel(n_jobs=-1)]: Done 1528 tasks      | elapsed:   15.6s\n",
      "[Parallel(n_jobs=-1)]: Done 2428 tasks      | elapsed:   24.4s\n",
      "[Parallel(n_jobs=-1)]: Done 3528 tasks      | elapsed:   35.2s\n",
      "[Parallel(n_jobs=-1)]: Done 4828 tasks      | elapsed:   53.2s\n",
      "[Parallel(n_jobs=-1)]: Done 6328 tasks      | elapsed:  1.3min\n",
      "[Parallel(n_jobs=-1)]: Done 8028 tasks      | elapsed:  1.8min\n",
      "[Parallel(n_jobs=-1)]: Done 9928 tasks      | elapsed:  2.5min\n",
      "[Parallel(n_jobs=-1)]: Done 12028 tasks      | elapsed:  3.3min\n",
      "[Parallel(n_jobs=-1)]: Done 14328 tasks      | elapsed:  4.4min\n",
      "[Parallel(n_jobs=-1)]: Done 16828 tasks      | elapsed:  5.6min\n",
      "[Parallel(n_jobs=-1)]: Done 18703 tasks      | elapsed:  6.8min\n",
      "[Parallel(n_jobs=-1)]: Done 20153 tasks      | elapsed:  7.7min\n",
      "[Parallel(n_jobs=-1)]: Done 21703 tasks      | elapsed:  8.6min\n",
      "[Parallel(n_jobs=-1)]: Done 23353 tasks      | elapsed:  9.9min\n",
      "[Parallel(n_jobs=-1)]: Done 25103 tasks      | elapsed: 10.9min\n",
      "[Parallel(n_jobs=-1)]: Done 26953 tasks      | elapsed: 12.4min\n",
      "[Parallel(n_jobs=-1)]: Done 28903 tasks      | elapsed: 13.7min\n",
      "[Parallel(n_jobs=-1)]: Done 30953 tasks      | elapsed: 15.5min\n",
      "[Parallel(n_jobs=-1)]: Done 33103 tasks      | elapsed: 17.1min\n",
      "[Parallel(n_jobs=-1)]: Done 35353 tasks      | elapsed: 19.2min\n",
      "[Parallel(n_jobs=-1)]: Done 37703 tasks      | elapsed: 20.9min\n",
      "[Parallel(n_jobs=-1)]: Done 40000 out of 40000 | elapsed: 23.3min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 23min 20s\n",
      "{'max_depth': 5, 'max_leaf_nodes': 14, 'min_samples_leaf': 6, 'min_samples_split': 2} -177.20394494435644\n",
      "Fitting 1 folds for each of 40000 candidates, totalling 40000 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  32 tasks      | elapsed:    0.6s\n",
      "[Parallel(n_jobs=-1)]: Done 632 tasks      | elapsed:    7.1s\n",
      "[Parallel(n_jobs=-1)]: Done 1632 tasks      | elapsed:   17.8s\n",
      "[Parallel(n_jobs=-1)]: Done 3032 tasks      | elapsed:   32.8s\n",
      "[Parallel(n_jobs=-1)]: Done 4510 tasks      | elapsed:   53.2s\n",
      "[Parallel(n_jobs=-1)]: Done 5610 tasks      | elapsed:  1.3min\n",
      "[Parallel(n_jobs=-1)]: Done 6910 tasks      | elapsed:  1.7min\n",
      "[Parallel(n_jobs=-1)]: Done 8410 tasks      | elapsed:  2.1min\n",
      "[Parallel(n_jobs=-1)]: Done 10110 tasks      | elapsed:  2.9min\n",
      "[Parallel(n_jobs=-1)]: Done 11312 tasks      | elapsed:  3.5min\n",
      "[Parallel(n_jobs=-1)]: Done 12362 tasks      | elapsed:  4.0min\n",
      "[Parallel(n_jobs=-1)]: Done 13512 tasks      | elapsed:  4.6min\n",
      "[Parallel(n_jobs=-1)]: Done 14762 tasks      | elapsed:  5.3min\n",
      "[Parallel(n_jobs=-1)]: Done 16112 tasks      | elapsed:  5.9min\n",
      "[Parallel(n_jobs=-1)]: Done 17562 tasks      | elapsed:  6.8min\n",
      "[Parallel(n_jobs=-1)]: Done 19112 tasks      | elapsed:  7.9min\n",
      "[Parallel(n_jobs=-1)]: Done 20762 tasks      | elapsed:  8.9min\n",
      "[Parallel(n_jobs=-1)]: Done 22512 tasks      | elapsed: 10.1min\n",
      "[Parallel(n_jobs=-1)]: Done 24362 tasks      | elapsed: 11.3min\n",
      "[Parallel(n_jobs=-1)]: Done 26312 tasks      | elapsed: 12.7min\n",
      "[Parallel(n_jobs=-1)]: Done 28362 tasks      | elapsed: 14.4min\n",
      "[Parallel(n_jobs=-1)]: Done 30512 tasks      | elapsed: 16.1min\n",
      "[Parallel(n_jobs=-1)]: Done 32762 tasks      | elapsed: 17.8min\n",
      "[Parallel(n_jobs=-1)]: Done 35112 tasks      | elapsed: 19.8min\n",
      "[Parallel(n_jobs=-1)]: Done 37562 tasks      | elapsed: 21.5min\n",
      "[Parallel(n_jobs=-1)]: Done 40000 out of 40000 | elapsed: 23.7min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 23min 40s\n",
      "{'max_depth': 4, 'max_leaf_nodes': 6, 'min_samples_leaf': 11, 'min_samples_split': 34} -192.72587895502693\n",
      "Fitting 1 folds for each of 40000 candidates, totalling 40000 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  28 tasks      | elapsed:    0.4s\n",
      "[Parallel(n_jobs=-1)]: Done 328 tasks      | elapsed:    2.9s\n",
      "[Parallel(n_jobs=-1)]: Done 828 tasks      | elapsed:    7.2s\n",
      "[Parallel(n_jobs=-1)]: Done 1528 tasks      | elapsed:   13.4s\n",
      "[Parallel(n_jobs=-1)]: Done 2428 tasks      | elapsed:   21.2s\n",
      "[Parallel(n_jobs=-1)]: Done 3528 tasks      | elapsed:   30.7s\n",
      "[Parallel(n_jobs=-1)]: Done 4828 tasks      | elapsed:   47.0s\n",
      "[Parallel(n_jobs=-1)]: Done 6328 tasks      | elapsed:  1.2min\n",
      "[Parallel(n_jobs=-1)]: Done 8028 tasks      | elapsed:  1.6min\n",
      "[Parallel(n_jobs=-1)]: Done 9928 tasks      | elapsed:  2.2min\n",
      "[Parallel(n_jobs=-1)]: Done 12028 tasks      | elapsed:  3.0min\n",
      "[Parallel(n_jobs=-1)]: Done 14328 tasks      | elapsed:  4.0min\n",
      "[Parallel(n_jobs=-1)]: Done 16828 tasks      | elapsed:  5.1min\n",
      "[Parallel(n_jobs=-1)]: Done 18317 tasks      | elapsed:  6.0min\n",
      "[Parallel(n_jobs=-1)]: Done 19767 tasks      | elapsed:  6.9min\n",
      "[Parallel(n_jobs=-1)]: Done 21317 tasks      | elapsed:  7.7min\n",
      "[Parallel(n_jobs=-1)]: Done 22967 tasks      | elapsed:  8.8min\n",
      "[Parallel(n_jobs=-1)]: Done 24717 tasks      | elapsed:  9.8min\n",
      "[Parallel(n_jobs=-1)]: Done 26567 tasks      | elapsed: 11.1min\n",
      "[Parallel(n_jobs=-1)]: Done 28517 tasks      | elapsed: 12.4min\n",
      "[Parallel(n_jobs=-1)]: Done 30567 tasks      | elapsed: 13.9min\n",
      "[Parallel(n_jobs=-1)]: Done 32717 tasks      | elapsed: 15.6min\n",
      "[Parallel(n_jobs=-1)]: Done 34967 tasks      | elapsed: 17.8min\n",
      "[Parallel(n_jobs=-1)]: Done 37317 tasks      | elapsed: 19.8min\n",
      "[Parallel(n_jobs=-1)]: Done 39767 tasks      | elapsed: 22.3min\n",
      "[Parallel(n_jobs=-1)]: Done 40000 out of 40000 | elapsed: 22.6min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 22min 35s\n",
      "{'max_depth': 4, 'max_leaf_nodes': 6, 'min_samples_leaf': 1, 'min_samples_split': 32} -199.83383552869208\n",
      "Fitting 1 folds for each of 40000 candidates, totalling 40000 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  32 tasks      | elapsed:    0.6s\n",
      "[Parallel(n_jobs=-1)]: Done 632 tasks      | elapsed:    6.8s\n",
      "[Parallel(n_jobs=-1)]: Done 1632 tasks      | elapsed:   17.3s\n",
      "[Parallel(n_jobs=-1)]: Done 3032 tasks      | elapsed:   32.3s\n",
      "[Parallel(n_jobs=-1)]: Done 4832 tasks      | elapsed:   56.3s\n",
      "[Parallel(n_jobs=-1)]: Done 5968 tasks      | elapsed:  1.3min\n",
      "[Parallel(n_jobs=-1)]: Done 7268 tasks      | elapsed:  1.7min\n",
      "[Parallel(n_jobs=-1)]: Done 8768 tasks      | elapsed:  2.2min\n",
      "[Parallel(n_jobs=-1)]: Done 10468 tasks      | elapsed:  3.0min\n",
      "[Parallel(n_jobs=-1)]: Done 12368 tasks      | elapsed:  3.9min\n",
      "[Parallel(n_jobs=-1)]: Done 13989 tasks      | elapsed:  4.8min\n",
      "[Parallel(n_jobs=-1)]: Done 15139 tasks      | elapsed:  5.5min\n",
      "[Parallel(n_jobs=-1)]: Done 16389 tasks      | elapsed:  6.1min\n",
      "[Parallel(n_jobs=-1)]: Done 17739 tasks      | elapsed:  6.8min\n",
      "[Parallel(n_jobs=-1)]: Done 19189 tasks      | elapsed:  7.7min\n",
      "[Parallel(n_jobs=-1)]: Done 20739 tasks      | elapsed:  8.7min\n",
      "[Parallel(n_jobs=-1)]: Done 22389 tasks      | elapsed:  9.9min\n",
      "[Parallel(n_jobs=-1)]: Done 24139 tasks      | elapsed: 11.2min\n",
      "[Parallel(n_jobs=-1)]: Done 25989 tasks      | elapsed: 12.3min\n",
      "[Parallel(n_jobs=-1)]: Done 27939 tasks      | elapsed: 14.0min\n",
      "[Parallel(n_jobs=-1)]: Done 29989 tasks      | elapsed: 15.2min\n",
      "[Parallel(n_jobs=-1)]: Done 32139 tasks      | elapsed: 16.9min\n",
      "[Parallel(n_jobs=-1)]: Done 34389 tasks      | elapsed: 18.4min\n",
      "[Parallel(n_jobs=-1)]: Done 36739 tasks      | elapsed: 20.3min\n",
      "[Parallel(n_jobs=-1)]: Done 39189 tasks      | elapsed: 22.4min\n",
      "[Parallel(n_jobs=-1)]: Done 40000 out of 40000 | elapsed: 23.2min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 23min 14s\n",
      "{'max_depth': 3, 'max_leaf_nodes': 4, 'min_samples_leaf': 1, 'min_samples_split': 36} -215.28527354262673\n",
      "Fitting 1 folds for each of 40000 candidates, totalling 40000 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  32 tasks      | elapsed:    0.5s\n",
      "[Parallel(n_jobs=-1)]: Done 632 tasks      | elapsed:    5.9s\n",
      "[Parallel(n_jobs=-1)]: Done 1632 tasks      | elapsed:   15.1s\n",
      "[Parallel(n_jobs=-1)]: Done 3032 tasks      | elapsed:   29.1s\n",
      "[Parallel(n_jobs=-1)]: Done 4832 tasks      | elapsed:   48.7s\n",
      "[Parallel(n_jobs=-1)]: Done 7032 tasks      | elapsed:  1.4min\n",
      "[Parallel(n_jobs=-1)]: Done 9080 tasks      | elapsed:  2.0min\n",
      "[Parallel(n_jobs=-1)]: Done 10580 tasks      | elapsed:  2.5min\n",
      "[Parallel(n_jobs=-1)]: Done 12280 tasks      | elapsed:  3.1min\n",
      "[Parallel(n_jobs=-1)]: Done 14180 tasks      | elapsed:  3.9min\n",
      "[Parallel(n_jobs=-1)]: Done 16280 tasks      | elapsed:  4.8min\n",
      "[Parallel(n_jobs=-1)]: Done 18580 tasks      | elapsed:  6.1min\n",
      "[Parallel(n_jobs=-1)]: Done 21080 tasks      | elapsed:  7.4min\n",
      "[Parallel(n_jobs=-1)]: Done 23028 tasks      | elapsed:  8.6min\n",
      "[Parallel(n_jobs=-1)]: Done 24478 tasks      | elapsed:  9.4min\n",
      "[Parallel(n_jobs=-1)]: Done 26028 tasks      | elapsed: 10.4min\n",
      "[Parallel(n_jobs=-1)]: Done 27678 tasks      | elapsed: 11.9min\n",
      "[Parallel(n_jobs=-1)]: Done 29428 tasks      | elapsed: 13.0min\n",
      "[Parallel(n_jobs=-1)]: Done 31278 tasks      | elapsed: 14.6min\n",
      "[Parallel(n_jobs=-1)]: Done 33228 tasks      | elapsed: 15.9min\n",
      "[Parallel(n_jobs=-1)]: Done 35278 tasks      | elapsed: 17.8min\n",
      "[Parallel(n_jobs=-1)]: Done 37428 tasks      | elapsed: 19.2min\n",
      "[Parallel(n_jobs=-1)]: Done 39678 tasks      | elapsed: 21.4min\n",
      "[Parallel(n_jobs=-1)]: Done 40000 out of 40000 | elapsed: 21.7min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 21min 41s\n",
      "{'max_depth': 3, 'max_leaf_nodes': 8, 'min_samples_leaf': 46, 'min_samples_split': 2} -224.0534559724497\n",
      "Fitting 1 folds for each of 40000 candidates, totalling 40000 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  32 tasks      | elapsed:    0.5s\n",
      "[Parallel(n_jobs=-1)]: Done 632 tasks      | elapsed:    5.9s\n",
      "[Parallel(n_jobs=-1)]: Done 1632 tasks      | elapsed:   15.2s\n",
      "[Parallel(n_jobs=-1)]: Done 3032 tasks      | elapsed:   28.6s\n",
      "[Parallel(n_jobs=-1)]: Done 4832 tasks      | elapsed:   50.6s\n",
      "[Parallel(n_jobs=-1)]: Done 7032 tasks      | elapsed:  1.4min\n",
      "[Parallel(n_jobs=-1)]: Done 8986 tasks      | elapsed:  2.0min\n",
      "[Parallel(n_jobs=-1)]: Done 10486 tasks      | elapsed:  2.6min\n",
      "[Parallel(n_jobs=-1)]: Done 12186 tasks      | elapsed:  3.3min\n",
      "[Parallel(n_jobs=-1)]: Done 14086 tasks      | elapsed:  4.2min\n",
      "[Parallel(n_jobs=-1)]: Done 16186 tasks      | elapsed:  5.2min\n",
      "[Parallel(n_jobs=-1)]: Done 17598 tasks      | elapsed:  6.0min\n",
      "[Parallel(n_jobs=-1)]: Done 18848 tasks      | elapsed:  6.8min\n",
      "[Parallel(n_jobs=-1)]: Done 20198 tasks      | elapsed:  7.6min\n",
      "[Parallel(n_jobs=-1)]: Done 21648 tasks      | elapsed:  8.5min\n",
      "[Parallel(n_jobs=-1)]: Done 23198 tasks      | elapsed:  9.6min\n",
      "[Parallel(n_jobs=-1)]: Done 24848 tasks      | elapsed: 10.5min\n",
      "[Parallel(n_jobs=-1)]: Done 26598 tasks      | elapsed: 11.8min\n",
      "[Parallel(n_jobs=-1)]: Done 28448 tasks      | elapsed: 13.1min\n",
      "[Parallel(n_jobs=-1)]: Done 30398 tasks      | elapsed: 14.5min\n",
      "[Parallel(n_jobs=-1)]: Done 32448 tasks      | elapsed: 16.1min\n",
      "[Parallel(n_jobs=-1)]: Done 34598 tasks      | elapsed: 17.6min\n",
      "[Parallel(n_jobs=-1)]: Done 36848 tasks      | elapsed: 19.1min\n",
      "[Parallel(n_jobs=-1)]: Done 39198 tasks      | elapsed: 21.1min\n",
      "[Parallel(n_jobs=-1)]: Done 40000 out of 40000 | elapsed: 21.9min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 21min 55s\n",
      "{'max_depth': 2, 'max_leaf_nodes': 4, 'min_samples_leaf': 46, 'min_samples_split': 2} -233.67178740900977\n",
      "Fitting 1 folds for each of 40000 candidates, totalling 40000 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  32 tasks      | elapsed:    0.5s\n",
      "[Parallel(n_jobs=-1)]: Done 632 tasks      | elapsed:    6.0s\n",
      "[Parallel(n_jobs=-1)]: Done 1632 tasks      | elapsed:   15.3s\n",
      "[Parallel(n_jobs=-1)]: Done 3032 tasks      | elapsed:   28.0s\n",
      "[Parallel(n_jobs=-1)]: Done 4832 tasks      | elapsed:   49.4s\n",
      "[Parallel(n_jobs=-1)]: Done 7032 tasks      | elapsed:  1.4min\n",
      "[Parallel(n_jobs=-1)]: Done 9010 tasks      | elapsed:  2.0min\n",
      "[Parallel(n_jobs=-1)]: Done 10510 tasks      | elapsed:  2.6min\n",
      "[Parallel(n_jobs=-1)]: Done 12210 tasks      | elapsed:  3.2min\n",
      "[Parallel(n_jobs=-1)]: Done 14110 tasks      | elapsed:  4.1min\n",
      "[Parallel(n_jobs=-1)]: Done 16210 tasks      | elapsed:  5.1min\n",
      "[Parallel(n_jobs=-1)]: Done 18094 tasks      | elapsed:  6.2min\n",
      "[Parallel(n_jobs=-1)]: Done 19344 tasks      | elapsed:  6.9min\n",
      "[Parallel(n_jobs=-1)]: Done 20694 tasks      | elapsed:  7.6min\n",
      "[Parallel(n_jobs=-1)]: Done 22144 tasks      | elapsed:  8.6min\n",
      "[Parallel(n_jobs=-1)]: Done 23694 tasks      | elapsed:  9.8min\n",
      "[Parallel(n_jobs=-1)]: Done 25344 tasks      | elapsed: 10.7min\n",
      "[Parallel(n_jobs=-1)]: Done 27094 tasks      | elapsed: 12.1min\n",
      "[Parallel(n_jobs=-1)]: Done 28944 tasks      | elapsed: 13.3min\n",
      "[Parallel(n_jobs=-1)]: Done 30894 tasks      | elapsed: 14.9min\n",
      "[Parallel(n_jobs=-1)]: Done 32944 tasks      | elapsed: 16.3min\n",
      "[Parallel(n_jobs=-1)]: Done 35094 tasks      | elapsed: 18.1min\n",
      "[Parallel(n_jobs=-1)]: Done 37344 tasks      | elapsed: 19.7min\n",
      "[Parallel(n_jobs=-1)]: Done 39694 tasks      | elapsed: 22.1min\n",
      "[Parallel(n_jobs=-1)]: Done 40000 out of 40000 | elapsed: 22.5min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 22min 30s\n",
      "{'max_depth': 3, 'max_leaf_nodes': 4, 'min_samples_leaf': 11, 'min_samples_split': 38} -237.3601081906223\n",
      "Fitting 1 folds for each of 40000 candidates, totalling 40000 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  28 tasks      | elapsed:    0.4s\n",
      "[Parallel(n_jobs=-1)]: Done 328 tasks      | elapsed:    3.4s\n",
      "[Parallel(n_jobs=-1)]: Done 828 tasks      | elapsed:    8.2s\n",
      "[Parallel(n_jobs=-1)]: Done 1528 tasks      | elapsed:   14.8s\n",
      "[Parallel(n_jobs=-1)]: Done 2428 tasks      | elapsed:   23.4s\n",
      "[Parallel(n_jobs=-1)]: Done 3528 tasks      | elapsed:   34.3s\n",
      "[Parallel(n_jobs=-1)]: Done 4828 tasks      | elapsed:   53.9s\n",
      "[Parallel(n_jobs=-1)]: Done 6328 tasks      | elapsed:  1.3min\n",
      "[Parallel(n_jobs=-1)]: Done 8028 tasks      | elapsed:  1.8min\n",
      "[Parallel(n_jobs=-1)]: Done 9928 tasks      | elapsed:  2.4min\n",
      "[Parallel(n_jobs=-1)]: Done 12028 tasks      | elapsed:  3.2min\n",
      "[Parallel(n_jobs=-1)]: Done 14328 tasks      | elapsed:  4.2min\n",
      "[Parallel(n_jobs=-1)]: Done 16828 tasks      | elapsed:  5.3min\n",
      "[Parallel(n_jobs=-1)]: Done 19528 tasks      | elapsed:  6.9min\n",
      "[Parallel(n_jobs=-1)]: Done 22156 tasks      | elapsed:  8.3min\n",
      "[Parallel(n_jobs=-1)]: Done 23706 tasks      | elapsed:  9.3min\n",
      "[Parallel(n_jobs=-1)]: Done 25356 tasks      | elapsed: 10.2min\n",
      "[Parallel(n_jobs=-1)]: Done 27106 tasks      | elapsed: 11.4min\n",
      "[Parallel(n_jobs=-1)]: Done 28956 tasks      | elapsed: 12.5min\n",
      "[Parallel(n_jobs=-1)]: Done 30906 tasks      | elapsed: 13.9min\n",
      "[Parallel(n_jobs=-1)]: Done 32956 tasks      | elapsed: 15.2min\n",
      "[Parallel(n_jobs=-1)]: Done 35106 tasks      | elapsed: 16.9min\n",
      "[Parallel(n_jobs=-1)]: Done 37356 tasks      | elapsed: 18.4min\n",
      "[Parallel(n_jobs=-1)]: Done 39706 tasks      | elapsed: 20.5min\n",
      "[Parallel(n_jobs=-1)]: Done 40000 out of 40000 | elapsed: 20.8min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 20min 47s\n",
      "{'max_depth': 2, 'max_leaf_nodes': 4, 'min_samples_leaf': 1, 'min_samples_split': 8} -243.7606744990642\n"
     ]
    }
   ],
   "source": [
    "param_grid = {'max_depth': range(1,11),\n",
    "              'min_samples_leaf': range(1,51,5), \n",
    "              'max_leaf_nodes':range(2,41,2),\n",
    "              'min_samples_split':range(2,41,2),\n",
    "              }\n",
    "for leadtime in range(1,11):\n",
    "    est = GradientBoostingRegressor(learning_rate=0.1, random_state=10, n_estimators=100)\n",
    "    \n",
    "    X,y = MyPrepaData.Preparation(data=data,Qlag=[1,4],Rlag=range(1,7),reanalysis=reanalysis,isreanalysis=True,leadtime=leadtime)\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=730,shuffle=False)\n",
    "    length = X_train.shape[0]\n",
    "    tscv = [([_ for _ in range(length-731)],[_ for _ in range(length-731,length)]) for i in range(1)]\n",
    "    %time gs_cv = GridSearchCV(est, param_grid, scoring=make_scorer(mae,greater_is_better=False),\\\n",
    "                               cv = tscv ,n_jobs=-1,verbose=1).fit(X_train.values,y_train.values)\n",
    "    with open('gbrt_para%d.pickle'%leadtime, 'wb') as f:\n",
    "        pickle.dump(gs_cv,f)\n",
    "    print(gs_cv.best_params_,gs_cv.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "125.03499043174489\n",
      "153.6544116802135\n",
      "176.00736790599356\n",
      "192.43293110861043\n",
      "199.14175120968176\n",
      "216.68489037431712\n",
      "224.3698242892771\n",
      "233.79444037541683\n",
      "238.0307035125584\n",
      "243.56934104822867\n"
     ]
    }
   ],
   "source": [
    "optpara = []\n",
    "for leadtime in range(1,11):\n",
    "    bestpara = []\n",
    "    bestscore = []\n",
    "    with open('gbrt_para%d.pickle'%(leadtime), 'rb') as f:\n",
    "        result = pickle.load(f)\n",
    "        bestpara.append(result.best_params_)\n",
    "        bestscore.append(result.best_score_)\n",
    "#     print(-max(bestscore))\n",
    "    optpara.append(bestpara[bestscore.index(max(bestscore))])\n",
    "with open('gbrt_allpara.pickle', 'wb') as f:\n",
    "            pickle.dump(optpara,f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 265,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 1 folds for each of 40 candidates, totalling 40 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=12)]: Using backend LokyBackend with 12 concurrent workers.\n",
      "[Parallel(n_jobs=12)]: Done  40 out of  40 | elapsed:   19.3s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 26.2 s\n",
      "{'learning_rate': 0.01, 'n_estimators': 3800} -124.93798489156994\n",
      "Fitting 1 folds for each of 40 candidates, totalling 40 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=12)]: Using backend LokyBackend with 12 concurrent workers.\n",
      "[Parallel(n_jobs=12)]: Done  40 out of  40 | elapsed:   26.7s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 32.8 s\n",
      "{'learning_rate': 0.01, 'n_estimators': 2700} -153.9928194072676\n",
      "Fitting 1 folds for each of 40 candidates, totalling 40 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=12)]: Using backend LokyBackend with 12 concurrent workers.\n",
      "[Parallel(n_jobs=12)]: Done  40 out of  40 | elapsed:   33.2s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 36.8 s\n",
      "{'learning_rate': 0.01, 'n_estimators': 1300} -179.28208422067976\n",
      "Fitting 1 folds for each of 40 candidates, totalling 40 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=12)]: Using backend LokyBackend with 12 concurrent workers.\n",
      "[Parallel(n_jobs=12)]: Done  40 out of  40 | elapsed:   26.1s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 28 s\n",
      "{'learning_rate': 0.01, 'n_estimators': 900} -195.93959587276495\n",
      "Fitting 1 folds for each of 40 candidates, totalling 40 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=12)]: Using backend LokyBackend with 12 concurrent workers.\n",
      "[Parallel(n_jobs=12)]: Done  40 out of  40 | elapsed:   36.7s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 39.7 s\n",
      "{'learning_rate': 0.01, 'n_estimators': 1000} -203.10591384437276\n",
      "Fitting 1 folds for each of 40 candidates, totalling 40 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=12)]: Using backend LokyBackend with 12 concurrent workers.\n",
      "[Parallel(n_jobs=12)]: Done  40 out of  40 | elapsed:   18.7s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 19.9 s\n",
      "{'learning_rate': 0.01, 'n_estimators': 700} -216.72023264854346\n",
      "Fitting 1 folds for each of 40 candidates, totalling 40 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=12)]: Using backend LokyBackend with 12 concurrent workers.\n",
      "[Parallel(n_jobs=12)]: Done  17 out of  40 | elapsed:    4.1s remaining:    5.6s\n",
      "[Parallel(n_jobs=12)]: Done  40 out of  40 | elapsed:   14.2s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 15.7 s\n",
      "{'learning_rate': 0.01, 'n_estimators': 1400} -225.83967245258893\n",
      "Fitting 1 folds for each of 40 candidates, totalling 40 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=12)]: Using backend LokyBackend with 12 concurrent workers.\n",
      "[Parallel(n_jobs=12)]: Done  17 out of  40 | elapsed:    3.9s remaining:    5.3s\n",
      "[Parallel(n_jobs=12)]: Done  40 out of  40 | elapsed:   13.9s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 16 s\n",
      "{'learning_rate': 0.01, 'n_estimators': 2000} -233.95618476312077\n",
      "Fitting 1 folds for each of 40 candidates, totalling 40 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=12)]: Using backend LokyBackend with 12 concurrent workers.\n",
      "[Parallel(n_jobs=12)]: Done  40 out of  40 | elapsed:   20.7s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 22.7 s\n",
      "{'learning_rate': 0.01, 'n_estimators': 1300} -239.5760740285998\n",
      "Fitting 1 folds for each of 40 candidates, totalling 40 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=12)]: Using backend LokyBackend with 12 concurrent workers.\n",
      "[Parallel(n_jobs=12)]: Done  17 out of  40 | elapsed:    4.3s remaining:    5.9s\n",
      "[Parallel(n_jobs=12)]: Done  40 out of  40 | elapsed:   16.8s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 18.6 s\n",
      "{'learning_rate': 0.01, 'n_estimators': 1200} -243.97890594655084\n"
     ]
    }
   ],
   "source": [
    "with open('gbrt_allpara.pickle', 'rb') as f:\n",
    "    optpara = pickle.load(f)\n",
    "min_samples_split= [_['min_samples_split'] for _ in optpara]\n",
    "min_samples_leaf = [_['min_samples_leaf'] for _ in optpara]\n",
    "max_depth = [_['max_depth'] for _ in optpara]\n",
    "max_leaf_nodes = [_['max_leaf_nodes'] for _ in optpara]\n",
    "\n",
    "for leadtime in range(1,11):\n",
    "    param_grid = {'n_estimators': range(100,4001,100),\n",
    "          'learning_rate': [0.01], \n",
    "          }\n",
    "    est = GradientBoostingRegressor(min_samples_split= min_samples_split[leadtime-1],\n",
    "                                          max_depth = max_depth[leadtime-1],max_leaf_nodes=max_leaf_nodes[leadtime-1],\n",
    "                                          min_samples_leaf=min_samples_leaf[leadtime-1],random_state=10)\n",
    "    X,y = Preparation(data=data,Qlag=[1,4],Rlag=range(1,7),reanalysis=reanalysis,isreanalysis=True,leadtime=leadtime)\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=730,shuffle=False)\n",
    "    length = X_train.shape[0]\n",
    "    tscv = [([_ for _ in range(length-731)],[_ for _ in range(length-731,length)]) for i in range(1)]\n",
    "    %time gs_cv = GridSearchCV(est, param_grid, scoring=make_scorer(mae,greater_is_better=False),\\\n",
    "                               cv = tscv ,n_jobs=12,verbose=1).fit(X_train.values,y_train.values)\n",
    "    with open('gbrt_para_learn_est%d.pickle'%(leadtime), 'wb') as f:\n",
    "        pickle.dump(gs_cv,f)\n",
    "    print(gs_cv.best_params_,gs_cv.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 266,
   "metadata": {},
   "outputs": [],
   "source": [
    "optpara = []\n",
    "for leadtime in range(1,11):\n",
    "    with open('gbrt_para_learn_est%d.pickle'%(leadtime), 'rb') as f:\n",
    "        result = pickle.load(f)\n",
    "        optpara.append(result.best_params_)\n",
    "with open('gbrt_allpara_learn_est.pickle', 'wb') as f:\n",
    "    pickle.dump(optpara,f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# GBRT没有再分析数据优化参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 267,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 1 folds for each of 40000 candidates, totalling 40000 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=8)]: Done  34 tasks      | elapsed:    1.8s\n",
      "[Parallel(n_jobs=8)]: Done 1307 tasks      | elapsed:    8.0s\n",
      "[Parallel(n_jobs=8)]: Done 3807 tasks      | elapsed:   22.0s\n",
      "[Parallel(n_jobs=8)]: Done 7147 tasks      | elapsed:   50.7s\n",
      "[Parallel(n_jobs=8)]: Done 9397 tasks      | elapsed:  1.3min\n",
      "[Parallel(n_jobs=8)]: Done 12147 tasks      | elapsed:  1.9min\n",
      "[Parallel(n_jobs=8)]: Done 14719 tasks      | elapsed:  2.6min\n",
      "[Parallel(n_jobs=8)]: Done 16219 tasks      | elapsed:  3.0min\n",
      "[Parallel(n_jobs=8)]: Done 17919 tasks      | elapsed:  3.6min\n",
      "[Parallel(n_jobs=8)]: Done 19819 tasks      | elapsed:  4.4min\n",
      "[Parallel(n_jobs=8)]: Done 21919 tasks      | elapsed:  5.1min\n",
      "[Parallel(n_jobs=8)]: Done 24219 tasks      | elapsed:  6.0min\n",
      "[Parallel(n_jobs=8)]: Done 26719 tasks      | elapsed:  6.9min\n",
      "[Parallel(n_jobs=8)]: Done 29419 tasks      | elapsed:  7.9min\n",
      "[Parallel(n_jobs=8)]: Done 32319 tasks      | elapsed:  9.2min\n",
      "[Parallel(n_jobs=8)]: Done 35419 tasks      | elapsed: 10.6min\n",
      "[Parallel(n_jobs=8)]: Done 38719 tasks      | elapsed: 12.1min\n",
      "[Parallel(n_jobs=8)]: Done 39985 out of 40000 | elapsed: 12.8min remaining:    0.2s\n",
      "[Parallel(n_jobs=8)]: Done 40000 out of 40000 | elapsed: 12.8min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 12min 46s\n",
      "{'max_depth': 3, 'max_leaf_nodes': 8, 'min_samples_leaf': 6, 'min_samples_split': 18} -130.0282592082776\n",
      "Fitting 1 folds for each of 40000 candidates, totalling 40000 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=8)]: Done 160 tasks      | elapsed:    1.0s\n",
      "[Parallel(n_jobs=8)]: Done 1360 tasks      | elapsed:    8.0s\n",
      "[Parallel(n_jobs=8)]: Done 3360 tasks      | elapsed:   20.1s\n",
      "[Parallel(n_jobs=8)]: Done 6160 tasks      | elapsed:   44.8s\n",
      "[Parallel(n_jobs=8)]: Done 9184 tasks      | elapsed:  1.3min\n",
      "[Parallel(n_jobs=8)]: Done 11384 tasks      | elapsed:  1.8min\n",
      "[Parallel(n_jobs=8)]: Done 13984 tasks      | elapsed:  2.5min\n",
      "[Parallel(n_jobs=8)]: Done 16984 tasks      | elapsed:  3.3min\n",
      "[Parallel(n_jobs=8)]: Done 19674 tasks      | elapsed:  4.3min\n",
      "[Parallel(n_jobs=8)]: Done 21574 tasks      | elapsed:  4.8min\n",
      "[Parallel(n_jobs=8)]: Done 23674 tasks      | elapsed:  5.6min\n",
      "[Parallel(n_jobs=8)]: Done 25974 tasks      | elapsed:  6.4min\n",
      "[Parallel(n_jobs=8)]: Done 28474 tasks      | elapsed:  7.3min\n",
      "[Parallel(n_jobs=8)]: Done 31174 tasks      | elapsed:  8.4min\n",
      "[Parallel(n_jobs=8)]: Done 34074 tasks      | elapsed:  9.5min\n",
      "[Parallel(n_jobs=8)]: Done 37174 tasks      | elapsed: 10.8min\n",
      "[Parallel(n_jobs=8)]: Done 40000 out of 40000 | elapsed: 12.2min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 12min 11s\n",
      "{'max_depth': 2, 'max_leaf_nodes': 4, 'min_samples_leaf': 31, 'min_samples_split': 2} -162.98585138712258\n",
      "Fitting 1 folds for each of 40000 candidates, totalling 40000 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=8)]: Done 160 tasks      | elapsed:    1.0s\n",
      "[Parallel(n_jobs=8)]: Done 1360 tasks      | elapsed:    8.2s\n",
      "[Parallel(n_jobs=8)]: Done 3360 tasks      | elapsed:   19.8s\n",
      "[Parallel(n_jobs=8)]: Done 6160 tasks      | elapsed:   44.7s\n",
      "[Parallel(n_jobs=8)]: Done 9760 tasks      | elapsed:  1.4min\n",
      "[Parallel(n_jobs=8)]: Done 12008 tasks      | elapsed:  1.9min\n",
      "[Parallel(n_jobs=8)]: Done 14608 tasks      | elapsed:  2.5min\n",
      "[Parallel(n_jobs=8)]: Done 17368 tasks      | elapsed:  3.3min\n",
      "[Parallel(n_jobs=8)]: Done 19068 tasks      | elapsed:  3.9min\n",
      "[Parallel(n_jobs=8)]: Done 20968 tasks      | elapsed:  4.4min\n",
      "[Parallel(n_jobs=8)]: Done 23068 tasks      | elapsed:  5.2min\n",
      "[Parallel(n_jobs=8)]: Done 25368 tasks      | elapsed:  6.0min\n",
      "[Parallel(n_jobs=8)]: Done 27868 tasks      | elapsed:  7.1min\n",
      "[Parallel(n_jobs=8)]: Done 30568 tasks      | elapsed:  8.1min\n",
      "[Parallel(n_jobs=8)]: Done 33468 tasks      | elapsed:  9.3min\n",
      "[Parallel(n_jobs=8)]: Done 36568 tasks      | elapsed: 10.6min\n",
      "[Parallel(n_jobs=8)]: Done 39868 tasks      | elapsed: 12.2min\n",
      "[Parallel(n_jobs=8)]: Done 40000 out of 40000 | elapsed: 12.2min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 12min 15s\n",
      "{'max_depth': 2, 'max_leaf_nodes': 4, 'min_samples_leaf': 1, 'min_samples_split': 16} -193.26480859759943\n",
      "Fitting 1 folds for each of 40000 candidates, totalling 40000 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=8)]: Done 160 tasks      | elapsed:    1.1s\n",
      "[Parallel(n_jobs=8)]: Done 1360 tasks      | elapsed:    8.7s\n",
      "[Parallel(n_jobs=8)]: Done 3360 tasks      | elapsed:   22.0s\n",
      "[Parallel(n_jobs=8)]: Done 6160 tasks      | elapsed:   49.0s\n",
      "[Parallel(n_jobs=8)]: Done 9760 tasks      | elapsed:  1.5min\n",
      "[Parallel(n_jobs=8)]: Done 13636 tasks      | elapsed:  2.4min\n",
      "[Parallel(n_jobs=8)]: Done 16236 tasks      | elapsed:  3.1min\n",
      "[Parallel(n_jobs=8)]: Done 19236 tasks      | elapsed:  4.1min\n",
      "[Parallel(n_jobs=8)]: Done 22636 tasks      | elapsed:  5.3min\n",
      "[Parallel(n_jobs=8)]: Done 24884 tasks      | elapsed:  6.1min\n",
      "[Parallel(n_jobs=8)]: Done 26984 tasks      | elapsed:  7.0min\n",
      "[Parallel(n_jobs=8)]: Done 29284 tasks      | elapsed:  7.9min\n",
      "[Parallel(n_jobs=8)]: Done 31784 tasks      | elapsed:  9.0min\n",
      "[Parallel(n_jobs=8)]: Done 34484 tasks      | elapsed: 10.0min\n",
      "[Parallel(n_jobs=8)]: Done 37384 tasks      | elapsed: 11.1min\n",
      "[Parallel(n_jobs=8)]: Done 40000 out of 40000 | elapsed: 12.4min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 12min 26s\n",
      "{'max_depth': 2, 'max_leaf_nodes': 4, 'min_samples_leaf': 1, 'min_samples_split': 16} -214.44765281508435\n",
      "Fitting 1 folds for each of 40000 candidates, totalling 40000 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=8)]: Done 160 tasks      | elapsed:    1.1s\n",
      "[Parallel(n_jobs=8)]: Done 1360 tasks      | elapsed:    8.1s\n",
      "[Parallel(n_jobs=8)]: Done 3360 tasks      | elapsed:   21.0s\n",
      "[Parallel(n_jobs=8)]: Done 6160 tasks      | elapsed:   45.8s\n",
      "[Parallel(n_jobs=8)]: Done 9760 tasks      | elapsed:  1.4min\n",
      "[Parallel(n_jobs=8)]: Done 13484 tasks      | elapsed:  2.3min\n",
      "[Parallel(n_jobs=8)]: Done 16084 tasks      | elapsed:  3.0min\n",
      "[Parallel(n_jobs=8)]: Done 19084 tasks      | elapsed:  3.9min\n",
      "[Parallel(n_jobs=8)]: Done 22484 tasks      | elapsed:  5.0min\n",
      "[Parallel(n_jobs=8)]: Done 26284 tasks      | elapsed:  6.3min\n",
      "[Parallel(n_jobs=8)]: Done 30484 tasks      | elapsed:  7.9min\n",
      "[Parallel(n_jobs=8)]: Done 33402 tasks      | elapsed:  9.0min\n",
      "[Parallel(n_jobs=8)]: Done 35902 tasks      | elapsed: 10.2min\n",
      "[Parallel(n_jobs=8)]: Done 38602 tasks      | elapsed: 11.3min\n",
      "[Parallel(n_jobs=8)]: Done 40000 out of 40000 | elapsed: 12.0min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 12min 3s\n",
      "{'max_depth': 3, 'max_leaf_nodes': 4, 'min_samples_leaf': 1, 'min_samples_split': 24} -229.57528700685756\n",
      "Fitting 1 folds for each of 40000 candidates, totalling 40000 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=8)]: Done 196 tasks      | elapsed:    1.5s\n",
      "[Parallel(n_jobs=8)]: Done 1696 tasks      | elapsed:   10.4s\n",
      "[Parallel(n_jobs=8)]: Done 4196 tasks      | elapsed:   26.6s\n",
      "[Parallel(n_jobs=8)]: Done 7696 tasks      | elapsed:   59.1s\n",
      "[Parallel(n_jobs=8)]: Done 10471 tasks      | elapsed:  1.6min\n",
      "[Parallel(n_jobs=8)]: Done 13221 tasks      | elapsed:  2.2min\n",
      "[Parallel(n_jobs=8)]: Done 14686 tasks      | elapsed:  2.6min\n",
      "[Parallel(n_jobs=8)]: Done 16186 tasks      | elapsed:  3.0min\n",
      "[Parallel(n_jobs=8)]: Done 17886 tasks      | elapsed:  3.5min\n",
      "[Parallel(n_jobs=8)]: Done 19786 tasks      | elapsed:  4.2min\n",
      "[Parallel(n_jobs=8)]: Done 21886 tasks      | elapsed:  4.9min\n",
      "[Parallel(n_jobs=8)]: Done 24186 tasks      | elapsed:  5.8min\n",
      "[Parallel(n_jobs=8)]: Done 26686 tasks      | elapsed:  6.7min\n",
      "[Parallel(n_jobs=8)]: Done 29386 tasks      | elapsed:  7.7min\n",
      "[Parallel(n_jobs=8)]: Done 32286 tasks      | elapsed:  8.9min\n",
      "[Parallel(n_jobs=8)]: Done 35386 tasks      | elapsed: 10.2min\n",
      "[Parallel(n_jobs=8)]: Done 38686 tasks      | elapsed: 11.6min\n",
      "[Parallel(n_jobs=8)]: Done 40000 out of 40000 | elapsed: 12.3min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 12min 17s\n",
      "{'max_depth': 1, 'max_leaf_nodes': 2, 'min_samples_leaf': 31, 'min_samples_split': 2} -243.6838633210695\n",
      "Fitting 1 folds for each of 40000 candidates, totalling 40000 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=8)]: Done 196 tasks      | elapsed:    1.3s\n",
      "[Parallel(n_jobs=8)]: Done 1696 tasks      | elapsed:    9.6s\n",
      "[Parallel(n_jobs=8)]: Done 4196 tasks      | elapsed:   25.4s\n",
      "[Parallel(n_jobs=8)]: Done 6326 tasks      | elapsed:   49.7s\n",
      "[Parallel(n_jobs=8)]: Done 8576 tasks      | elapsed:  1.4min\n",
      "[Parallel(n_jobs=8)]: Done 11326 tasks      | elapsed:  2.2min\n",
      "[Parallel(n_jobs=8)]: Done 14561 tasks      | elapsed:  3.1min\n",
      "[Parallel(n_jobs=8)]: Done 16061 tasks      | elapsed:  3.5min\n",
      "[Parallel(n_jobs=8)]: Done 17761 tasks      | elapsed:  4.0min\n",
      "[Parallel(n_jobs=8)]: Done 19661 tasks      | elapsed:  4.7min\n",
      "[Parallel(n_jobs=8)]: Done 21761 tasks      | elapsed:  5.4min\n",
      "[Parallel(n_jobs=8)]: Done 24061 tasks      | elapsed:  6.2min\n",
      "[Parallel(n_jobs=8)]: Done 26561 tasks      | elapsed:  7.1min\n",
      "[Parallel(n_jobs=8)]: Done 29261 tasks      | elapsed:  8.1min\n",
      "[Parallel(n_jobs=8)]: Done 31650 tasks      | elapsed:  9.6min\n",
      "[Parallel(n_jobs=8)]: Done 33200 tasks      | elapsed: 10.3min\n",
      "[Parallel(n_jobs=8)]: Done 34850 tasks      | elapsed: 11.3min\n",
      "[Parallel(n_jobs=8)]: Done 37168 tasks      | elapsed: 12.4min\n",
      "[Parallel(n_jobs=8)]: Done 40000 out of 40000 | elapsed: 13.8min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 13min 51s\n",
      "{'max_depth': 3, 'max_leaf_nodes': 4, 'min_samples_leaf': 6, 'min_samples_split': 16} -247.08168977529832\n",
      "Fitting 1 folds for each of 40000 candidates, totalling 40000 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=8)]: Done 160 tasks      | elapsed:    1.1s\n",
      "[Parallel(n_jobs=8)]: Done 1360 tasks      | elapsed:    8.0s\n",
      "[Parallel(n_jobs=8)]: Done 3360 tasks      | elapsed:   19.7s\n",
      "[Parallel(n_jobs=8)]: Done 6160 tasks      | elapsed:   44.1s\n",
      "[Parallel(n_jobs=8)]: Done 9448 tasks      | elapsed:  1.3min\n",
      "[Parallel(n_jobs=8)]: Done 11648 tasks      | elapsed:  1.8min\n",
      "[Parallel(n_jobs=8)]: Done 14248 tasks      | elapsed:  2.4min\n",
      "[Parallel(n_jobs=8)]: Done 17248 tasks      | elapsed:  3.2min\n",
      "[Parallel(n_jobs=8)]: Done 20648 tasks      | elapsed:  4.3min\n",
      "[Parallel(n_jobs=8)]: Done 23572 tasks      | elapsed:  5.4min\n",
      "[Parallel(n_jobs=8)]: Done 25672 tasks      | elapsed:  6.2min\n",
      "[Parallel(n_jobs=8)]: Done 27972 tasks      | elapsed:  7.2min\n",
      "[Parallel(n_jobs=8)]: Done 30472 tasks      | elapsed:  8.1min\n",
      "[Parallel(n_jobs=8)]: Done 33172 tasks      | elapsed:  9.3min\n",
      "[Parallel(n_jobs=8)]: Done 36072 tasks      | elapsed: 10.7min\n",
      "[Parallel(n_jobs=8)]: Done 39172 tasks      | elapsed: 12.1min\n",
      "[Parallel(n_jobs=8)]: Done 40000 out of 40000 | elapsed: 12.6min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 12min 35s\n",
      "{'max_depth': 1, 'max_leaf_nodes': 2, 'min_samples_leaf': 1, 'min_samples_split': 2} -256.81280271446735\n",
      "Fitting 1 folds for each of 40000 candidates, totalling 40000 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=8)]: Done 124 tasks      | elapsed:    0.9s\n",
      "[Parallel(n_jobs=8)]: Done 1024 tasks      | elapsed:    7.1s\n",
      "[Parallel(n_jobs=8)]: Done 2524 tasks      | elapsed:   17.7s\n",
      "[Parallel(n_jobs=8)]: Done 4624 tasks      | elapsed:   35.9s\n",
      "[Parallel(n_jobs=8)]: Done 7324 tasks      | elapsed:  1.0min\n",
      "[Parallel(n_jobs=8)]: Done 10624 tasks      | elapsed:  1.7min\n",
      "[Parallel(n_jobs=8)]: Done 14524 tasks      | elapsed:  2.6min\n",
      "[Parallel(n_jobs=8)]: Done 18895 tasks      | elapsed:  3.8min\n",
      "[Parallel(n_jobs=8)]: Done 21445 tasks      | elapsed:  4.6min\n",
      "[Parallel(n_jobs=8)]: Done 24295 tasks      | elapsed:  5.6min\n",
      "[Parallel(n_jobs=8)]: Done 27445 tasks      | elapsed:  6.8min\n",
      "[Parallel(n_jobs=8)]: Done 30895 tasks      | elapsed:  8.1min\n",
      "[Parallel(n_jobs=8)]: Done 34645 tasks      | elapsed:  9.6min\n",
      "[Parallel(n_jobs=8)]: Done 38695 tasks      | elapsed: 11.2min\n",
      "[Parallel(n_jobs=8)]: Done 39985 out of 40000 | elapsed: 11.9min remaining:    0.2s\n",
      "[Parallel(n_jobs=8)]: Done 40000 out of 40000 | elapsed: 11.9min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 11min 53s\n",
      "{'max_depth': 1, 'max_leaf_nodes': 2, 'min_samples_leaf': 6, 'min_samples_split': 2} -261.98115226359465\n",
      "Fitting 1 folds for each of 40000 candidates, totalling 40000 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=8)]: Done 160 tasks      | elapsed:    1.1s\n",
      "[Parallel(n_jobs=8)]: Done 1360 tasks      | elapsed:    7.8s\n",
      "[Parallel(n_jobs=8)]: Done 3360 tasks      | elapsed:   19.3s\n",
      "[Parallel(n_jobs=8)]: Done 6160 tasks      | elapsed:   42.5s\n",
      "[Parallel(n_jobs=8)]: Done 9760 tasks      | elapsed:  1.4min\n",
      "[Parallel(n_jobs=8)]: Done 12572 tasks      | elapsed:  2.0min\n",
      "[Parallel(n_jobs=8)]: Done 15172 tasks      | elapsed:  2.7min\n",
      "[Parallel(n_jobs=8)]: Done 18172 tasks      | elapsed:  3.6min\n",
      "[Parallel(n_jobs=8)]: Done 21572 tasks      | elapsed:  4.7min\n",
      "[Parallel(n_jobs=8)]: Done 23656 tasks      | elapsed:  5.5min\n",
      "[Parallel(n_jobs=8)]: Done 25756 tasks      | elapsed:  6.2min\n",
      "[Parallel(n_jobs=8)]: Done 28056 tasks      | elapsed:  7.2min\n",
      "[Parallel(n_jobs=8)]: Done 30556 tasks      | elapsed:  8.2min\n",
      "[Parallel(n_jobs=8)]: Done 33256 tasks      | elapsed:  9.4min\n",
      "[Parallel(n_jobs=8)]: Done 36156 tasks      | elapsed: 11.1min\n",
      "[Parallel(n_jobs=8)]: Done 39256 tasks      | elapsed: 12.8min\n",
      "[Parallel(n_jobs=8)]: Done 40000 out of 40000 | elapsed: 13.2min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 13min 15s\n",
      "{'max_depth': 1, 'max_leaf_nodes': 2, 'min_samples_leaf': 1, 'min_samples_split': 2} -266.05603290029245\n"
     ]
    }
   ],
   "source": [
    "param_grid = {'max_depth': range(1,11),\n",
    "              'min_samples_leaf': range(1,51,5), \n",
    "              'max_leaf_nodes':range(2,41,2),\n",
    "              'min_samples_split':range(2,41,2),\n",
    "              }\n",
    "est = GradientBoostingRegressor(n_estimators=100,learning_rate=0.1,random_state=10)\n",
    "for leadtime in range(1,11):\n",
    "    X,y = MyPrepaData.Preparation(data=data,Qlag=[1,4],Rlag=range(1,7),reanalysis=reanalysis,isreanalysis=False,leadtime=leadtime)\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=730,shuffle=False)\n",
    "    length = X_train.shape[0]\n",
    "    tscv = [([_ for _ in range(length-731)],[_ for _ in range(length-731,length)]) for i in range(1)]\n",
    "    %time gs_cv = GridSearchCV(est, param_grid, scoring=make_scorer(mae,greater_is_better=False),\\\n",
    "                               cv = tscv ,n_jobs=8,verbose=1).fit(X_train.values,y_train.values)\n",
    "    with open('gbrt_para_nore%d.pickle'%leadtime, 'wb') as f:\n",
    "        pickle.dump(gs_cv,f)\n",
    "    print(gs_cv.best_params_,gs_cv.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 268,
   "metadata": {},
   "outputs": [],
   "source": [
    "optpara = []\n",
    "for leadtime in range(1,11):\n",
    "    with open('gbrt_para_nore%d.pickle'%(leadtime), 'rb') as f:\n",
    "        result = pickle.load(f)\n",
    "        optpara.append(result.best_params_)\n",
    "with open('gbrt_allpara_nore.pickle', 'wb') as f:\n",
    "    pickle.dump(optpara,f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 269,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 40 candidates, totalling 200 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=6)]: Using backend LokyBackend with 6 concurrent workers.\n",
      "[Parallel(n_jobs=6)]: Done  62 tasks      | elapsed:    7.5s\n",
      "[Parallel(n_jobs=6)]: Done 200 out of 200 | elapsed:   56.7s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 57.7 s\n",
      "{'learning_rate': 0.01, 'n_estimators': 1100} -130.89342848930474\n",
      "Fitting 5 folds for each of 40 candidates, totalling 200 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=6)]: Using backend LokyBackend with 6 concurrent workers.\n",
      "[Parallel(n_jobs=6)]: Done 132 tasks      | elapsed:   20.8s\n",
      "[Parallel(n_jobs=6)]: Done 200 out of 200 | elapsed:   44.0s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 44.5 s\n",
      "{'learning_rate': 0.01, 'n_estimators': 900} -163.20544759775686\n",
      "Fitting 5 folds for each of 40 candidates, totalling 200 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=6)]: Using backend LokyBackend with 6 concurrent workers.\n",
      "[Parallel(n_jobs=6)]: Done 126 tasks      | elapsed:   19.7s\n",
      "[Parallel(n_jobs=6)]: Done 200 out of 200 | elapsed:   48.9s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 49.6 s\n",
      "{'learning_rate': 0.01, 'n_estimators': 1200} -193.75425316438844\n",
      "Fitting 5 folds for each of 40 candidates, totalling 200 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=6)]: Using backend LokyBackend with 6 concurrent workers.\n",
      "[Parallel(n_jobs=6)]: Done 132 tasks      | elapsed:   23.4s\n",
      "[Parallel(n_jobs=6)]: Done 200 out of 200 | elapsed:   45.2s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 45.6 s\n",
      "{'learning_rate': 0.01, 'n_estimators': 700} -214.4519502097778\n",
      "Fitting 5 folds for each of 40 candidates, totalling 200 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=6)]: Using backend LokyBackend with 6 concurrent workers.\n",
      "[Parallel(n_jobs=6)]: Done 100 tasks      | elapsed:   14.7s\n",
      "[Parallel(n_jobs=6)]: Done 200 out of 200 | elapsed:   54.6s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 55.4 s\n",
      "{'learning_rate': 0.01, 'n_estimators': 700} -229.6923012968204\n",
      "Fitting 5 folds for each of 40 candidates, totalling 200 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=6)]: Using backend LokyBackend with 6 concurrent workers.\n",
      "[Parallel(n_jobs=6)]: Done 137 tasks      | elapsed:   14.4s\n",
      "[Parallel(n_jobs=6)]: Done 200 out of 200 | elapsed:   29.7s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 30.3 s\n",
      "{'learning_rate': 0.01, 'n_estimators': 1200} -243.00476840482557\n",
      "Fitting 5 folds for each of 40 candidates, totalling 200 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=6)]: Using backend LokyBackend with 6 concurrent workers.\n",
      "[Parallel(n_jobs=6)]: Done  61 tasks      | elapsed:    6.5s\n",
      "[Parallel(n_jobs=6)]: Done 200 out of 200 | elapsed:  1.0min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 1min 3s\n",
      "{'learning_rate': 0.01, 'n_estimators': 600} -247.11546006892985\n",
      "Fitting 5 folds for each of 40 candidates, totalling 200 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=6)]: Using backend LokyBackend with 6 concurrent workers.\n",
      "[Parallel(n_jobs=6)]: Done 134 tasks      | elapsed:   15.9s\n",
      "[Parallel(n_jobs=6)]: Done 200 out of 200 | elapsed:   31.5s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 32.2 s\n",
      "{'learning_rate': 0.01, 'n_estimators': 1100} -256.3562121220269\n",
      "Fitting 5 folds for each of 40 candidates, totalling 200 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=6)]: Using backend LokyBackend with 6 concurrent workers.\n",
      "[Parallel(n_jobs=6)]: Done 106 tasks      | elapsed:   11.0s\n",
      "[Parallel(n_jobs=6)]: Done 200 out of 200 | elapsed:   33.5s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 33.9 s\n",
      "{'learning_rate': 0.01, 'n_estimators': 900} -261.23785895318986\n",
      "Fitting 5 folds for each of 40 candidates, totalling 200 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=6)]: Using backend LokyBackend with 6 concurrent workers.\n",
      "[Parallel(n_jobs=6)]: Done 172 tasks      | elapsed:   18.5s\n",
      "[Parallel(n_jobs=6)]: Done 189 out of 200 | elapsed:   21.4s remaining:    1.2s\n",
      "[Parallel(n_jobs=6)]: Done 200 out of 200 | elapsed:   23.1s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 23.4 s\n",
      "{'learning_rate': 0.01, 'n_estimators': 900} -266.5111025281714\n"
     ]
    }
   ],
   "source": [
    "with open('gbrt_allpara_nore.pickle', 'rb') as f:\n",
    "    optpara = pickle.load(f)\n",
    "min_samples_split= [_['min_samples_split'] for _ in optpara]\n",
    "min_samples_leaf = [_['min_samples_leaf'] for _ in optpara]\n",
    "max_depth = [_['max_depth'] for _ in optpara]\n",
    "max_leaf_nodes = [_['max_leaf_nodes'] for _ in optpara]\n",
    "\n",
    "for leadtime in range(1,11):\n",
    "    param_grid = {'n_estimators':range(100,4001,100),\n",
    "          'learning_rate': [0.01],\n",
    "          }\n",
    "    est = GradientBoostingRegressor(min_samples_split= min_samples_split[leadtime-1],\n",
    "                                          max_depth = max_depth[leadtime-1],max_leaf_nodes=max_leaf_nodes[leadtime-1],\n",
    "                                          min_samples_leaf=min_samples_leaf[leadtime-1],random_state=10)\n",
    "    X,y = Preparation(data=data,Qlag=[1,4],Rlag=range(1,7),reanalysis=reanalysis,isreanalysis=False,leadtime=leadtime)\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=730,shuffle=False)\n",
    "    length = X_train.shape[0]\n",
    "    tscv = [([_ for _ in range(length-731)],[_ for _ in range(length-731,length)]) for i in range(5)]\n",
    "    %time gs_cv = GridSearchCV(est, param_grid, scoring=make_scorer(mae,greater_is_better=False),\\\n",
    "                               cv = tscv ,n_jobs=6,verbose=1).fit(X_train.values,y_train.values)\n",
    "    with open('gbrt_para_learn_est%d_nore.pickle'%(leadtime), 'wb') as f:\n",
    "        pickle.dump(gs_cv,f)\n",
    "    print(gs_cv.best_params_,gs_cv.best_score_)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 270,
   "metadata": {},
   "outputs": [],
   "source": [
    "optpara = []\n",
    "for leadtime in range(1,11):\n",
    "    with open('gbrt_para_learn_est%d_nore.pickle'%(leadtime), 'rb') as f:\n",
    "        result = pickle.load(f)\n",
    "        optpara.append(result.best_params_)\n",
    "with open('gbrt_allpara_learn_est_nore.pickle', 'wb') as f:\n",
    "    pickle.dump(optpara,f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GBRT-n_estimators:3800,2700,1300,900,1000,700,1400,2000,1300,1200,\n",
      "GBRT-max_leaf_nodes:7,9,13,7,15,4,5,4,4,17,\n",
      "GBRT-min_samples_leaf:2,7,2,4,2,1,10,10,8,1,\n",
      "GBRT-max_depth:4,6,8,5,9,9,2,2,7,2,\n",
      "GBRT-min_samples_split:18,15,12,13,8,3,19,3,19,8,\n",
      "**************************************************\n",
      "GBRT_nore-n_estimators:1100,900,1200,700,700,1200,600,1100,900,900,\n",
      "GBRT_nore-max_leaf_nodes:8,4,4,4,4,2,4,2,2,2,\n",
      "GBRT_nore-min_samples_leaf:6,31,1,1,1,31,6,1,6,1,\n",
      "GBRT_nore-max_depth:3,2,2,2,3,1,3,1,1,1,\n",
      "GBRT_nore-min_samples_split:18,2,16,16,24,2,16,2,2,2,\n",
      "**************************************************\n"
     ]
    }
   ],
   "source": [
    "# show parameters of GBRT and GBRTMIC\n",
    "names = locals()\n",
    "for label in ['','_nore']:\n",
    "    n_estimators = []\n",
    "    max_leaf_nodes = []\n",
    "    min_samples_leaf = []\n",
    "    max_depth = []\n",
    "    min_samples_split = []\n",
    "    with open('gbrt_allpara_learn_est%s.pickle'%label, 'rb') as f:\n",
    "        result = pickle.load(f)\n",
    "        for r in result:\n",
    "            n_estimators.append(r['n_estimators'])\n",
    "    print('GBRT%s-n_estimators:'%label,end='')\n",
    "    for n in n_estimators:\n",
    "        print(n,end=',')\n",
    "    print()\n",
    "\n",
    "    with open('gbrt_allpara%s.pickle'%label, 'rb') as f:\n",
    "        result = pickle.load(f)\n",
    "        for r in result:\n",
    "            for na in ['max_leaf_nodes','min_samples_leaf','max_depth','min_samples_split']:\n",
    "                names[na].append(r[na])\n",
    "        for na in ['max_leaf_nodes','min_samples_leaf','max_depth','min_samples_split']:\n",
    "            print('GBRT%s-%s:'%(label,na),end='')\n",
    "            for m in names[na]:\n",
    "                print(m,end=',')\n",
    "            print()\n",
    "    print('*'*50)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SVR 优化模型参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 1 folds for each of 8000 candidates, totalling 8000 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=12)]: Using backend LokyBackend with 12 concurrent workers.\n",
      "[Parallel(n_jobs=12)]: Done  28 tasks      | elapsed:    1.2s\n",
      "[Parallel(n_jobs=12)]: Done 254 tasks      | elapsed:    6.1s\n",
      "[Parallel(n_jobs=12)]: Done 504 tasks      | elapsed:   11.0s\n",
      "[Parallel(n_jobs=12)]: Done 854 tasks      | elapsed:   18.8s\n",
      "[Parallel(n_jobs=12)]: Done 1304 tasks      | elapsed:   31.2s\n",
      "[Parallel(n_jobs=12)]: Done 1854 tasks      | elapsed:   48.9s\n",
      "[Parallel(n_jobs=12)]: Done 2504 tasks      | elapsed:  1.2min\n",
      "[Parallel(n_jobs=12)]: Done 3254 tasks      | elapsed:  1.8min\n",
      "[Parallel(n_jobs=12)]: Done 4104 tasks      | elapsed:  2.5min\n",
      "[Parallel(n_jobs=12)]: Done 5054 tasks      | elapsed:  3.4min\n",
      "[Parallel(n_jobs=12)]: Done 6104 tasks      | elapsed:  4.5min\n",
      "[Parallel(n_jobs=12)]: Done 7254 tasks      | elapsed:  5.9min\n",
      "[Parallel(n_jobs=12)]: Done 8000 out of 8000 | elapsed:  6.8min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 6min 50s\n",
      "{'C': 6.2105263157894735, 'epsilon': 0.0068736842105263166, 'gamma': 0.03226315789473684} -0.02144091343069382\n",
      "Fitting 1 folds for each of 8000 candidates, totalling 8000 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=12)]: Using backend LokyBackend with 12 concurrent workers.\n",
      "[Parallel(n_jobs=12)]: Done  28 tasks      | elapsed:    0.8s\n",
      "[Parallel(n_jobs=12)]: Done 328 tasks      | elapsed:    5.6s\n",
      "[Parallel(n_jobs=12)]: Done 828 tasks      | elapsed:   15.8s\n",
      "[Parallel(n_jobs=12)]: Done 1528 tasks      | elapsed:   34.1s\n",
      "[Parallel(n_jobs=12)]: Done 2252 tasks      | elapsed:   58.8s\n",
      "[Parallel(n_jobs=12)]: Done 2802 tasks      | elapsed:  1.3min\n",
      "[Parallel(n_jobs=12)]: Done 3452 tasks      | elapsed:  1.8min\n",
      "[Parallel(n_jobs=12)]: Done 4202 tasks      | elapsed:  2.4min\n",
      "[Parallel(n_jobs=12)]: Done 5052 tasks      | elapsed:  3.2min\n",
      "[Parallel(n_jobs=12)]: Done 6002 tasks      | elapsed:  4.1min\n",
      "[Parallel(n_jobs=12)]: Done 7052 tasks      | elapsed:  5.3min\n",
      "[Parallel(n_jobs=12)]: Done 8000 out of 8000 | elapsed:  6.4min finished\n",
      "[Parallel(n_jobs=12)]: Using backend LokyBackend with 12 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 6min 24s\n",
      "{'C': 1.0, 'epsilon': 0.008436842105263158, 'gamma': 0.05831578947368421} -0.02638720020340654\n",
      "Fitting 1 folds for each of 8000 candidates, totalling 8000 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=12)]: Done  28 tasks      | elapsed:    0.8s\n",
      "[Parallel(n_jobs=12)]: Done 328 tasks      | elapsed:    5.4s\n",
      "[Parallel(n_jobs=12)]: Done 828 tasks      | elapsed:   15.2s\n",
      "[Parallel(n_jobs=12)]: Done 1528 tasks      | elapsed:   33.5s\n",
      "[Parallel(n_jobs=12)]: Done 2263 tasks      | elapsed:   57.2s\n",
      "[Parallel(n_jobs=12)]: Done 2813 tasks      | elapsed:  1.3min\n",
      "[Parallel(n_jobs=12)]: Done 3463 tasks      | elapsed:  1.7min\n",
      "[Parallel(n_jobs=12)]: Done 4213 tasks      | elapsed:  2.3min\n",
      "[Parallel(n_jobs=12)]: Done 5063 tasks      | elapsed:  3.1min\n",
      "[Parallel(n_jobs=12)]: Done 6013 tasks      | elapsed:  3.9min\n",
      "[Parallel(n_jobs=12)]: Done 7063 tasks      | elapsed:  5.1min\n",
      "[Parallel(n_jobs=12)]: Done 8000 out of 8000 | elapsed:  6.1min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 6min 8s\n",
      "{'C': 1.0, 'epsilon': 0.0016631578947368423, 'gamma': 0.08436842105263158} -0.029990157384471746\n",
      "Fitting 1 folds for each of 8000 candidates, totalling 8000 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=12)]: Using backend LokyBackend with 12 concurrent workers.\n",
      "[Parallel(n_jobs=12)]: Done  28 tasks      | elapsed:    0.8s\n",
      "[Parallel(n_jobs=12)]: Done 328 tasks      | elapsed:    5.6s\n",
      "[Parallel(n_jobs=12)]: Done 828 tasks      | elapsed:   15.8s\n",
      "[Parallel(n_jobs=12)]: Done 1528 tasks      | elapsed:   34.0s\n",
      "[Parallel(n_jobs=12)]: Done 2260 tasks      | elapsed:   58.0s\n",
      "[Parallel(n_jobs=12)]: Done 2810 tasks      | elapsed:  1.3min\n",
      "[Parallel(n_jobs=12)]: Done 3460 tasks      | elapsed:  1.7min\n",
      "[Parallel(n_jobs=12)]: Done 4210 tasks      | elapsed:  2.3min\n",
      "[Parallel(n_jobs=12)]: Done 5060 tasks      | elapsed:  3.0min\n",
      "[Parallel(n_jobs=12)]: Done 6010 tasks      | elapsed:  3.9min\n",
      "[Parallel(n_jobs=12)]: Done 7060 tasks      | elapsed:  5.1min\n",
      "[Parallel(n_jobs=12)]: Done 8000 out of 8000 | elapsed:  6.1min finished\n",
      "[Parallel(n_jobs=12)]: Using backend LokyBackend with 12 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 6min 6s\n",
      "{'C': 1.0, 'epsilon': 0.00791578947368421, 'gamma': 0.02705263157894737} -0.032044313559217295\n",
      "Fitting 1 folds for each of 8000 candidates, totalling 8000 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=12)]: Done  28 tasks      | elapsed:    0.8s\n",
      "[Parallel(n_jobs=12)]: Done 328 tasks      | elapsed:    5.6s\n",
      "[Parallel(n_jobs=12)]: Done 828 tasks      | elapsed:   15.6s\n",
      "[Parallel(n_jobs=12)]: Done 1528 tasks      | elapsed:   33.6s\n",
      "[Parallel(n_jobs=12)]: Done 2270 tasks      | elapsed:   57.1s\n",
      "[Parallel(n_jobs=12)]: Done 2820 tasks      | elapsed:  1.3min\n",
      "[Parallel(n_jobs=12)]: Done 3470 tasks      | elapsed:  1.7min\n",
      "[Parallel(n_jobs=12)]: Done 4220 tasks      | elapsed:  2.3min\n",
      "[Parallel(n_jobs=12)]: Done 5070 tasks      | elapsed:  3.0min\n",
      "[Parallel(n_jobs=12)]: Done 6020 tasks      | elapsed:  3.8min\n",
      "[Parallel(n_jobs=12)]: Done 7070 tasks      | elapsed:  4.9min\n",
      "[Parallel(n_jobs=12)]: Done 8000 out of 8000 | elapsed:  5.9min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 5min 54s\n",
      "{'C': 11.421052631578947, 'epsilon': 0.0016631578947368423, 'gamma': 0.006210526315789474} -0.03378306834129117\n",
      "Fitting 1 folds for each of 8000 candidates, totalling 8000 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=12)]: Using backend LokyBackend with 12 concurrent workers.\n",
      "[Parallel(n_jobs=12)]: Done  28 tasks      | elapsed:    0.8s\n",
      "[Parallel(n_jobs=12)]: Done 328 tasks      | elapsed:    5.6s\n",
      "[Parallel(n_jobs=12)]: Done 828 tasks      | elapsed:   15.7s\n",
      "[Parallel(n_jobs=12)]: Done 1528 tasks      | elapsed:   33.4s\n",
      "[Parallel(n_jobs=12)]: Done 2428 tasks      | elapsed:  1.0min\n",
      "[Parallel(n_jobs=12)]: Done 3008 tasks      | elapsed:  1.4min\n",
      "[Parallel(n_jobs=12)]: Done 3658 tasks      | elapsed:  1.8min\n",
      "[Parallel(n_jobs=12)]: Done 4408 tasks      | elapsed:  2.4min\n",
      "[Parallel(n_jobs=12)]: Done 5258 tasks      | elapsed:  3.1min\n",
      "[Parallel(n_jobs=12)]: Done 6208 tasks      | elapsed:  4.0min\n",
      "[Parallel(n_jobs=12)]: Done 7258 tasks      | elapsed:  5.1min\n",
      "[Parallel(n_jobs=12)]: Done 8000 out of 8000 | elapsed:  5.9min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 5min 53s\n",
      "{'C': 1.0, 'epsilon': 0.0001, 'gamma': 0.021842105263157895} -0.03484717007379201\n",
      "Fitting 1 folds for each of 8000 candidates, totalling 8000 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=12)]: Using backend LokyBackend with 12 concurrent workers.\n",
      "[Parallel(n_jobs=12)]: Done  28 tasks      | elapsed:    0.8s\n",
      "[Parallel(n_jobs=12)]: Done 328 tasks      | elapsed:    5.6s\n",
      "[Parallel(n_jobs=12)]: Done 828 tasks      | elapsed:   15.5s\n",
      "[Parallel(n_jobs=12)]: Done 1528 tasks      | elapsed:   33.8s\n",
      "[Parallel(n_jobs=12)]: Done 2428 tasks      | elapsed:  1.0min\n",
      "[Parallel(n_jobs=12)]: Done 3010 tasks      | elapsed:  1.4min\n",
      "[Parallel(n_jobs=12)]: Done 3660 tasks      | elapsed:  1.8min\n",
      "[Parallel(n_jobs=12)]: Done 4410 tasks      | elapsed:  2.4min\n",
      "[Parallel(n_jobs=12)]: Done 5260 tasks      | elapsed:  3.1min\n",
      "[Parallel(n_jobs=12)]: Done 6210 tasks      | elapsed:  4.0min\n",
      "[Parallel(n_jobs=12)]: Done 7260 tasks      | elapsed:  5.0min\n",
      "[Parallel(n_jobs=12)]: Done 8000 out of 8000 | elapsed:  5.8min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 5min 48s\n",
      "{'C': 1.0, 'epsilon': 0.0021842105263157894, 'gamma': 0.03747368421052632} -0.03556332827967928\n",
      "Fitting 1 folds for each of 8000 candidates, totalling 8000 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=12)]: Using backend LokyBackend with 12 concurrent workers.\n",
      "[Parallel(n_jobs=12)]: Done  28 tasks      | elapsed:    0.8s\n",
      "[Parallel(n_jobs=12)]: Done 328 tasks      | elapsed:    5.7s\n",
      "[Parallel(n_jobs=12)]: Done 828 tasks      | elapsed:   15.8s\n",
      "[Parallel(n_jobs=12)]: Done 1528 tasks      | elapsed:   33.2s\n",
      "[Parallel(n_jobs=12)]: Done 2428 tasks      | elapsed:  1.0min\n",
      "[Parallel(n_jobs=12)]: Done 3010 tasks      | elapsed:  1.4min\n",
      "[Parallel(n_jobs=12)]: Done 3660 tasks      | elapsed:  1.8min\n",
      "[Parallel(n_jobs=12)]: Done 4410 tasks      | elapsed:  2.4min\n",
      "[Parallel(n_jobs=12)]: Done 5260 tasks      | elapsed:  3.1min\n",
      "[Parallel(n_jobs=12)]: Done 6210 tasks      | elapsed:  3.9min\n",
      "[Parallel(n_jobs=12)]: Done 7260 tasks      | elapsed:  4.9min\n",
      "[Parallel(n_jobs=12)]: Done 8000 out of 8000 | elapsed:  5.7min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 5min 43s\n",
      "{'C': 6.2105263157894735, 'epsilon': 0.0006210526315789474, 'gamma': 0.01663157894736842} -0.03593469652284178\n",
      "Fitting 1 folds for each of 8000 candidates, totalling 8000 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=12)]: Using backend LokyBackend with 12 concurrent workers.\n",
      "[Parallel(n_jobs=12)]: Done  28 tasks      | elapsed:    0.8s\n",
      "[Parallel(n_jobs=12)]: Done 328 tasks      | elapsed:    5.5s\n",
      "[Parallel(n_jobs=12)]: Done 828 tasks      | elapsed:   15.3s\n",
      "[Parallel(n_jobs=12)]: Done 1528 tasks      | elapsed:   33.1s\n",
      "[Parallel(n_jobs=12)]: Done 2428 tasks      | elapsed:  1.0min\n",
      "[Parallel(n_jobs=12)]: Done 3210 tasks      | elapsed:  1.5min\n",
      "[Parallel(n_jobs=12)]: Done 3860 tasks      | elapsed:  1.9min\n",
      "[Parallel(n_jobs=12)]: Done 4610 tasks      | elapsed:  2.5min\n",
      "[Parallel(n_jobs=12)]: Done 5460 tasks      | elapsed:  3.2min\n",
      "[Parallel(n_jobs=12)]: Done 6410 tasks      | elapsed:  4.0min\n",
      "[Parallel(n_jobs=12)]: Done 7460 tasks      | elapsed:  5.1min\n",
      "[Parallel(n_jobs=12)]: Done 8000 out of 8000 | elapsed:  5.6min finished\n",
      "[Parallel(n_jobs=12)]: Using backend LokyBackend with 12 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 5min 36s\n",
      "{'C': 1.0, 'epsilon': 0.004789473684210527, 'gamma': 0.06873684210526315} -0.03693100181806052\n",
      "Fitting 1 folds for each of 8000 candidates, totalling 8000 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=12)]: Done  28 tasks      | elapsed:    0.8s\n",
      "[Parallel(n_jobs=12)]: Done 328 tasks      | elapsed:    5.6s\n",
      "[Parallel(n_jobs=12)]: Done 828 tasks      | elapsed:   15.5s\n",
      "[Parallel(n_jobs=12)]: Done 1528 tasks      | elapsed:   32.8s\n",
      "[Parallel(n_jobs=12)]: Done 2428 tasks      | elapsed:  1.0min\n",
      "[Parallel(n_jobs=12)]: Done 3010 tasks      | elapsed:  1.4min\n",
      "[Parallel(n_jobs=12)]: Done 3660 tasks      | elapsed:  1.8min\n",
      "[Parallel(n_jobs=12)]: Done 4410 tasks      | elapsed:  2.3min\n",
      "[Parallel(n_jobs=12)]: Done 5260 tasks      | elapsed:  3.0min\n",
      "[Parallel(n_jobs=12)]: Done 6210 tasks      | elapsed:  3.9min\n",
      "[Parallel(n_jobs=12)]: Done 7260 tasks      | elapsed:  4.9min\n",
      "[Parallel(n_jobs=12)]: Done 8000 out of 8000 | elapsed:  5.7min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 5min 40s\n",
      "{'C': 6.2105263157894735, 'epsilon': 0.004268421052631579, 'gamma': 0.01663157894736842} -0.03765432962147253\n"
     ]
    }
   ],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "param_grid = {'C': np.linspace(1,100,20),\n",
    "              'gamma': np.linspace(0.001,0.1,20), \n",
    "              'epsilon':np.linspace(0.0001,0.01,20),\n",
    "              }\n",
    "for leadtime in range(1,11):\n",
    "    X,y = Preparation(data=data,Qlag=[1,4],Rlag=range(1,7),reanalysis=reanalysis,isreanalysis=True,leadtime=leadtime)\n",
    "    scalemodely = MinMaxScaler().fit(np.array(y).reshape(-1, 1)) \n",
    "    scalemodelx = MinMaxScaler().fit(X)\n",
    "    y_scale = scalemodely.transform(np.array(y).reshape(-1, 1))\n",
    "    X_scale = scalemodelx.transform(X)\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X_scale,y_scale,test_size=730,shuffle=False)\n",
    "    length = X_train.shape[0]\n",
    "    tscv = [([_ for _ in range(length-731)],[_ for _ in range(length-731,length)]) for i in range(1)]\n",
    "    est = SVR() \n",
    "    %time gs_cv = GridSearchCV(est, param_grid, scoring=make_scorer(mae,greater_is_better=False),\\\n",
    "                               cv = tscv ,n_jobs=12,verbose=1).fit(X_train,y_train)\n",
    "    with open('svr_para%d.pickle'%(leadtime), 'wb') as f:\n",
    "        pickle.dump(gs_cv,f)\n",
    "    print(gs_cv.best_params_,gs_cv.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>C</td>\n",
       "      <td>6.2105</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>11.4211</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>6.2105</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>6.2105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>epsilon</td>\n",
       "      <td>0.0069</td>\n",
       "      <td>0.0084</td>\n",
       "      <td>0.0017</td>\n",
       "      <td>0.0079</td>\n",
       "      <td>0.0017</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>0.0022</td>\n",
       "      <td>0.0006</td>\n",
       "      <td>0.0048</td>\n",
       "      <td>0.0043</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>gamma</td>\n",
       "      <td>0.0323</td>\n",
       "      <td>0.0583</td>\n",
       "      <td>0.0844</td>\n",
       "      <td>0.0271</td>\n",
       "      <td>0.0062</td>\n",
       "      <td>0.0218</td>\n",
       "      <td>0.0375</td>\n",
       "      <td>0.0166</td>\n",
       "      <td>0.0687</td>\n",
       "      <td>0.0166</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              0       1       2       3        4       5       6       7  \\\n",
       "C        6.2105  1.0000  1.0000  1.0000  11.4211  1.0000  1.0000  6.2105   \n",
       "epsilon  0.0069  0.0084  0.0017  0.0079   0.0017  0.0001  0.0022  0.0006   \n",
       "gamma    0.0323  0.0583  0.0844  0.0271   0.0062  0.0218  0.0375  0.0166   \n",
       "\n",
       "              8       9  \n",
       "C        1.0000  6.2105  \n",
       "epsilon  0.0048  0.0043  \n",
       "gamma    0.0687  0.0166  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# show parameters of SVR\n",
    "C = []\n",
    "epsilon = []\n",
    "gamma = []\n",
    "for leadtime in range(1,11):\n",
    "    with open('svr_para%d.pickle'%(leadtime), 'rb') as f:\n",
    "        gs_cv = pickle.load(f)\n",
    "#         print(gs_cv.best_params_, gs_cv.best_score_)\n",
    "        C.append(gs_cv.best_params_['C'])\n",
    "        epsilon.append(gs_cv.best_params_['epsilon'])\n",
    "        gamma.append(gs_cv.best_params_['gamma'])\n",
    "dfpara = pd.DataFrame({'C':C,'epsilon':epsilon,'gamma':gamma})\n",
    "dfpara.applymap(lambda x: '%.4f'%x).T"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ANN参数率定"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neural_network import MLPRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=12)]: Using backend LokyBackend with 12 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 50 folds for each of 76 candidates, totalling 3800 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=12)]: Done  36 tasks      | elapsed:    0.8s\n",
      "[Parallel(n_jobs=12)]: Done 798 tasks      | elapsed:    9.9s\n",
      "[Parallel(n_jobs=12)]: Done 1548 tasks      | elapsed:   23.4s\n",
      "[Parallel(n_jobs=12)]: Done 1954 tasks      | elapsed:   36.8s\n",
      "[Parallel(n_jobs=12)]: Done 2404 tasks      | elapsed:   44.1s\n",
      "[Parallel(n_jobs=12)]: Done 2954 tasks      | elapsed:   56.9s\n",
      "[Parallel(n_jobs=12)]: Done 3604 tasks      | elapsed:  1.5min\n",
      "[Parallel(n_jobs=12)]: Done 3800 out of 3800 | elapsed:  1.8min finished\n",
      "[Parallel(n_jobs=12)]: Using backend LokyBackend with 12 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 1min 47s\n",
      "{'activation': 'tanh', 'hidden_layer_sizes': (5,)} -0.02206226524030604\n",
      "Fitting 50 folds for each of 76 candidates, totalling 3800 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=12)]: Done  32 tasks      | elapsed:    0.6s\n",
      "[Parallel(n_jobs=12)]: Done 632 tasks      | elapsed:    6.9s\n",
      "[Parallel(n_jobs=12)]: Done 1466 tasks      | elapsed:   22.6s\n",
      "[Parallel(n_jobs=12)]: Done 1816 tasks      | elapsed:   41.2s\n",
      "[Parallel(n_jobs=12)]: Done 2266 tasks      | elapsed:   55.5s\n",
      "[Parallel(n_jobs=12)]: Done 2816 tasks      | elapsed:  1.2min\n",
      "[Parallel(n_jobs=12)]: Done 3466 tasks      | elapsed:  2.0min\n",
      "[Parallel(n_jobs=12)]: Done 3800 out of 3800 | elapsed:  3.3min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 3min 16s\n",
      "{'activation': 'tanh', 'hidden_layer_sizes': (3,)} -0.02774835074111245\n",
      "Fitting 50 folds for each of 76 candidates, totalling 3800 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=12)]: Using backend LokyBackend with 12 concurrent workers.\n",
      "[Parallel(n_jobs=12)]: Done  36 tasks      | elapsed:    0.9s\n",
      "[Parallel(n_jobs=12)]: Done 750 tasks      | elapsed:   11.6s\n",
      "[Parallel(n_jobs=12)]: Done 1370 tasks      | elapsed:   26.5s\n",
      "[Parallel(n_jobs=12)]: Done 1720 tasks      | elapsed:   46.8s\n",
      "[Parallel(n_jobs=12)]: Done 2170 tasks      | elapsed:  1.2min\n",
      "[Parallel(n_jobs=12)]: Done 2720 tasks      | elapsed:  1.6min\n",
      "[Parallel(n_jobs=12)]: Done 3370 tasks      | elapsed:  2.6min\n",
      "[Parallel(n_jobs=12)]: Done 3800 out of 3800 | elapsed:  4.8min finished\n",
      "[Parallel(n_jobs=12)]: Using backend LokyBackend with 12 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 4min 50s\n",
      "{'activation': 'logistic', 'hidden_layer_sizes': (2,)} -0.03139300817208598\n",
      "Fitting 50 folds for each of 76 candidates, totalling 3800 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=12)]: Done  36 tasks      | elapsed:    1.0s\n",
      "[Parallel(n_jobs=12)]: Done 663 tasks      | elapsed:   11.7s\n",
      "[Parallel(n_jobs=12)]: Done 1037 tasks      | elapsed:   21.4s\n",
      "[Parallel(n_jobs=12)]: Done 1387 tasks      | elapsed:   32.8s\n",
      "[Parallel(n_jobs=12)]: Done 1837 tasks      | elapsed:  1.1min\n",
      "[Parallel(n_jobs=12)]: Done 2387 tasks      | elapsed:  1.7min\n",
      "[Parallel(n_jobs=12)]: Done 3037 tasks      | elapsed:  2.7min\n",
      "[Parallel(n_jobs=12)]: Done 3800 out of 3800 | elapsed:  6.3min finished\n",
      "[Parallel(n_jobs=12)]: Using backend LokyBackend with 12 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 6min 19s\n",
      "{'activation': 'logistic', 'hidden_layer_sizes': (2,)} -0.034360577212713905\n",
      "Fitting 50 folds for each of 76 candidates, totalling 3800 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=12)]: Done  28 tasks      | elapsed:    0.7s\n",
      "[Parallel(n_jobs=12)]: Done 328 tasks      | elapsed:    5.9s\n",
      "[Parallel(n_jobs=12)]: Done 828 tasks      | elapsed:   21.5s\n",
      "[Parallel(n_jobs=12)]: Done 1202 tasks      | elapsed:   31.7s\n",
      "[Parallel(n_jobs=12)]: Done 1652 tasks      | elapsed:   59.2s\n",
      "[Parallel(n_jobs=12)]: Done 2202 tasks      | elapsed:  1.8min\n",
      "[Parallel(n_jobs=12)]: Done 2852 tasks      | elapsed:  3.9min\n",
      "[Parallel(n_jobs=12)]: Done 3602 tasks      | elapsed:  6.7min\n",
      "[Parallel(n_jobs=12)]: Done 3800 out of 3800 | elapsed:  8.5min finished\n",
      "[Parallel(n_jobs=12)]: Using backend LokyBackend with 12 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 8min 28s\n",
      "{'activation': 'logistic', 'hidden_layer_sizes': (2,)} -0.03601524192005468\n",
      "Fitting 50 folds for each of 76 candidates, totalling 3800 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=12)]: Done  28 tasks      | elapsed:    0.7s\n",
      "[Parallel(n_jobs=12)]: Done 328 tasks      | elapsed:    5.7s\n",
      "[Parallel(n_jobs=12)]: Done 828 tasks      | elapsed:   19.2s\n",
      "[Parallel(n_jobs=12)]: Done 1401 tasks      | elapsed:   38.1s\n",
      "[Parallel(n_jobs=12)]: Done 1851 tasks      | elapsed:  1.3min\n",
      "[Parallel(n_jobs=12)]: Done 2401 tasks      | elapsed:  2.4min\n",
      "[Parallel(n_jobs=12)]: Done 3051 tasks      | elapsed:  4.4min\n",
      "[Parallel(n_jobs=12)]: Done 3800 out of 3800 | elapsed:  8.2min finished\n",
      "[Parallel(n_jobs=12)]: Using backend LokyBackend with 12 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 8min 11s\n",
      "{'activation': 'logistic', 'hidden_layer_sizes': (2,)} -0.038458861248601345\n",
      "Fitting 50 folds for each of 76 candidates, totalling 3800 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=12)]: Done  32 tasks      | elapsed:    0.7s\n",
      "[Parallel(n_jobs=12)]: Done 632 tasks      | elapsed:   10.4s\n",
      "[Parallel(n_jobs=12)]: Done 1192 tasks      | elapsed:   21.8s\n",
      "[Parallel(n_jobs=12)]: Done 1631 tasks      | elapsed:   45.3s\n",
      "[Parallel(n_jobs=12)]: Done 2081 tasks      | elapsed:  1.4min\n",
      "[Parallel(n_jobs=12)]: Done 2631 tasks      | elapsed:  3.1min\n",
      "[Parallel(n_jobs=12)]: Done 3281 tasks      | elapsed:  5.2min\n",
      "[Parallel(n_jobs=12)]: Done 3800 out of 3800 | elapsed:  8.4min finished\n",
      "[Parallel(n_jobs=12)]: Using backend LokyBackend with 12 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 8min 26s\n",
      "{'activation': 'logistic', 'hidden_layer_sizes': (2,)} -0.039964740360519606\n",
      "Fitting 50 folds for each of 76 candidates, totalling 3800 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=12)]: Done  36 tasks      | elapsed:    0.7s\n",
      "[Parallel(n_jobs=12)]: Done 747 tasks      | elapsed:   10.8s\n",
      "[Parallel(n_jobs=12)]: Done 1337 tasks      | elapsed:   25.6s\n",
      "[Parallel(n_jobs=12)]: Done 1687 tasks      | elapsed:   48.4s\n",
      "[Parallel(n_jobs=12)]: Done 2137 tasks      | elapsed:  1.5min\n",
      "[Parallel(n_jobs=12)]: Done 2687 tasks      | elapsed:  3.6min\n",
      "[Parallel(n_jobs=12)]: Done 3337 tasks      | elapsed:  5.8min\n",
      "[Parallel(n_jobs=12)]: Done 3800 out of 3800 | elapsed:  8.8min finished\n",
      "[Parallel(n_jobs=12)]: Using backend LokyBackend with 12 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 8min 49s\n",
      "{'activation': 'logistic', 'hidden_layer_sizes': (2,)} -0.04038546211359115\n",
      "Fitting 50 folds for each of 76 candidates, totalling 3800 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=12)]: Done  36 tasks      | elapsed:    0.9s\n",
      "[Parallel(n_jobs=12)]: Done 738 tasks      | elapsed:   11.3s\n",
      "[Parallel(n_jobs=12)]: Done 1304 tasks      | elapsed:   25.1s\n",
      "[Parallel(n_jobs=12)]: Done 1654 tasks      | elapsed:   51.6s\n",
      "[Parallel(n_jobs=12)]: Done 2104 tasks      | elapsed:  1.7min\n",
      "[Parallel(n_jobs=12)]: Done 2654 tasks      | elapsed:  4.0min\n",
      "[Parallel(n_jobs=12)]: Done 3304 tasks      | elapsed:  6.5min\n",
      "[Parallel(n_jobs=12)]: Done 3800 out of 3800 | elapsed: 10.6min finished\n",
      "[Parallel(n_jobs=12)]: Using backend LokyBackend with 12 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 10min 37s\n",
      "{'activation': 'logistic', 'hidden_layer_sizes': (2,)} -0.04168372037740585\n",
      "Fitting 50 folds for each of 76 candidates, totalling 3800 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=12)]: Done  36 tasks      | elapsed:    1.2s\n",
      "[Parallel(n_jobs=12)]: Done 612 tasks      | elapsed:   11.1s\n",
      "[Parallel(n_jobs=12)]: Done 1076 tasks      | elapsed:   21.8s\n",
      "[Parallel(n_jobs=12)]: Done 1426 tasks      | elapsed:   36.2s\n",
      "[Parallel(n_jobs=12)]: Done 1876 tasks      | elapsed:  1.4min\n",
      "[Parallel(n_jobs=12)]: Done 2426 tasks      | elapsed:  3.0min\n",
      "[Parallel(n_jobs=12)]: Done 3076 tasks      | elapsed:  5.9min\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 10min 50s\n",
      "{'activation': 'tanh', 'hidden_layer_sizes': (2,)} -0.04236489193413391\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=12)]: Done 3800 out of 3800 | elapsed: 10.8min finished\n"
     ]
    }
   ],
   "source": [
    "param_grid = {'hidden_layer_sizes': [(_,) for _ in range(2,21,1)],\n",
    "              'activation': ['identity','relu','logistic','tanh']\n",
    "              }\n",
    "for leadtime in range(1,11):\n",
    "    X,y = Preparation(data=data,Qlag=[1,4],Rlag=range(1,7),reanalysis=reanalysis,isreanalysis=True,leadtime=leadtime)\n",
    "    scalemodely = MinMaxScaler().fit(np.array(y).reshape(-1, 1)) \n",
    "    scalemodelx = MinMaxScaler().fit(X)\n",
    "    y_scale = scalemodely.transform(np.array(y).reshape(-1, 1))\n",
    "    X_scale = scalemodelx.transform(X)\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X_scale,y_scale,test_size=730,shuffle=False)\n",
    "    length = X_train.shape[0]\n",
    "    tscv = [([_ for _ in range(length-731)],[_ for _ in range(length-731,length)]) for i in range(itera)]\n",
    "    est = MLPRegressor(solver='lbfgs', alpha=0.0001,batch_size='auto', \n",
    "                             learning_rate='constant',learning_rate_init=0.001, power_t=0.5,max_iter=3000, shuffle=True, \n",
    "                             random_state=None,tol=1e-5, verbose=False, warm_start=False,momentum=0.9, \n",
    "                             nesterovs_momentum=True,early_stopping=True, validation_fraction=0.1,beta_1=0.9, \n",
    "                             beta_2=0.999, epsilon=1e-08,n_iter_no_change=10) \n",
    "    %time gs_cv = GridSearchCV(est, param_grid, scoring=make_scorer(mae,greater_is_better=False),\\\n",
    "                               cv = tscv ,n_jobs=12,verbose=1).fit(X_train,y_train)\n",
    "    with open('ann_para%d.pickle'%(leadtime), 'wb') as f:\n",
    "        pickle.dump(gs_cv,f)\n",
    "    print(gs_cv.best_params_,gs_cv.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "names = locals()\n",
    "for leadtime in range(1,11):\n",
    "    with open('ann_para%d.pickle'%(leadtime), 'rb') as f:\n",
    "        gs_cv = pickle.load(f)\n",
    "    para_result = pd.DataFrame.from_dict(gs_cv.cv_results_)\n",
    "    col = ['param_activation', 'param_hidden_layer_sizes']+['split%d_test_score'%__ for __ in range(itera)]\n",
    "    para_result = para_result[col]\n",
    "    \n",
    "    para_result.param_hidden_layer_sizes = para_result.param_hidden_layer_sizes.apply(lambda x:int(x[0]))\n",
    "    para_result = para_result.set_index(['param_activation', 'param_hidden_layer_sizes'])\n",
    "    para_result_b = para_result['split0_test_score']\n",
    "    for i in para_result.columns[1:]:\n",
    "        para_result_b = pd.concat([para_result_b,para_result[i]],axis=0)\n",
    "    para_result_b.name = 'MAE'\n",
    "    para_result_b = para_result_b.reset_index()\n",
    "    para_result_b.MAE = para_result_b.MAE.apply(lambda x:-x)\n",
    "    names['para_result%d'%leadtime] = para_result_b"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 绘图数据（无缩放） "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 / 10\n",
      "2 / 10\n",
      "3 / 10\n",
      "4 / 10\n",
      "5 / 10\n",
      "6 / 10\n",
      "7 / 10\n",
      "8 / 10\n",
      "9 / 10\n",
      "10 / 10\n",
      "Wall time: 4h 19min 22s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "idx = pd.IndexSlice\n",
    "itera = 50\n",
    "for leadtime in range(1,11):\n",
    "    X, y =  Preparation(data=data,Qlag=[1,4],Rlag=range(1,7),reanalysis=reanalysis,isreanalysis=True,leadtime=leadtime)\n",
    "    scalemodely = MinMaxScaler().fit(np.array(y).reshape(-1, 1)) \n",
    "    scalemodelx = MinMaxScaler().fit(X)\n",
    "    y_scale = scalemodely.transform(np.array(y).reshape(-1, 1))\n",
    "    X_scale = scalemodelx.transform(X)\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X_scale,y_scale,test_size=730,shuffle=False)\n",
    "    X_train_val, X_test_val, y_train_val, y_test_val = train_test_split(X_train,y_train,test_size=731,shuffle=False)\n",
    "    para = pd.DataFrame(columns=pd.MultiIndex.from_product([range(2,21,1),['identity','relu','logistic','tanh'],['train_mae','test_mae']]),index = range(itera))\n",
    "    for node,activation in pd.MultiIndex.from_product([range(2,21,1),['identity','relu','logistic','tanh']]):\n",
    "        model = MLPRegressor(hidden_layer_sizes=(node,), activation=activation, solver='lbfgs', alpha=0.0001,batch_size='auto', \n",
    "                             learning_rate='constant',learning_rate_init=0.001, power_t=0.5,max_iter=3000, shuffle=True, \n",
    "                             random_state=None,tol=1e-5, verbose=False, warm_start=False,momentum=0.9, \n",
    "                             nesterovs_momentum=True,early_stopping=True, validation_fraction=0.1,beta_1=0.9, \n",
    "                             beta_2=0.999, epsilon=1e-08,n_iter_no_change=10)\n",
    "        train_mae = []\n",
    "        test_mae = []\n",
    "        for i in range(itera): # 优选模型参数时，不能使用测试集\n",
    "            model.fit(X_train_val,y_train_val.ravel())\n",
    "            y_fit = model.predict(X_train_val)\n",
    "            y_fit_val = model.predict(X_test_val)\n",
    "            y_train_val_,y_test_val_,y_fit_,y_fit_val_ = scalemodely.inverse_transform(y_train_val.reshape(-1, 1)),\\\n",
    "            scalemodely.inverse_transform(y_test_val.reshape(-1, 1)),\\\n",
    "            scalemodely.inverse_transform(y_fit.reshape(-1, 1)),scalemodely.inverse_transform(y_fit_val.reshape(-1, 1))\n",
    "            para.loc[i,idx[[node],activation,:]] = [mae(y_train_val_,y_fit_),mae(y_test_val_,y_fit_val_)]           \n",
    "    para.to_csv('leadtime%d_lbfgs_self.csv'%leadtime)\n",
    "    print(leadtime,'/','10')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('4', 'tanh'),\n",
       " ('2', 'tanh'),\n",
       " ('3', 'logistic'),\n",
       " ('2', 'logistic'),\n",
       " ('2', 'logistic'),\n",
       " ('2', 'logistic'),\n",
       " ('2', 'logistic'),\n",
       " ('2', 'logistic'),\n",
       " ('2', 'tanh'),\n",
       " ('2', 'tanh')]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 查看最优化参数\n",
    "ANNpara = []\n",
    "for leadtime in range(1,11):\n",
    "    idx = pd.IndexSlice\n",
    "    para = pd.read_csv('leadtime%d_lbfgs_self.csv' % leadtime, index_col=0, header=[0, 1, 2])\n",
    "    temp = para.loc[:,idx[:,:,'test_mae']].median().idxmin()\n",
    "    ANNpara.append((temp[0],temp[1]))\n",
    "ANNpara"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "           MAE Activation function  Number of nodes\n",
      "0   252.626762            logistic                2\n",
      "1   237.673118            logistic                2\n",
      "2   251.717471            logistic                2\n",
      "3   275.693547            logistic                2\n",
      "4   237.532637            logistic                2\n",
      "..         ...                 ...              ...\n",
      "43  266.161492                relu               20\n",
      "44  252.380132                relu               20\n",
      "45  311.959613                relu               20\n",
      "46  301.189295                relu               20\n",
      "47  330.105039                relu               20\n",
      "\n",
      "[3648 rows x 3 columns]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZUAAACiCAYAAACJU1CaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOydZ3gc1dmw7zNlq1ar3i0ZGxsbtwAGTCghQAK26QQIJaa3hJ76JiHtTSMheRMDIaHZlFBDaAabhBI+asA0F9yLZMlWb7vaPnO+H0eWtZZky0hGxXNf115azezMPCv7zDNPF1JKHBwcHBwcBgNtqAVwcHBwcBg9OErFwcHBwWHQcJSKg4ODg8Og4SgVBwcHB4dBw1EqDg4ODg6DhqNUHBwcHBwGDUepODg4ODgMGsZgn1AIsT9wMpAHrAZqAD+wApgF5AL3SCkTg31tB4eRhhDCC/wYiAFbgTqgGYgDRcBYKeWdQyehg8OeIfZG8aMQYjowAzhISnmzEGIBasH8ATgWSEgpF+10zJXAlQB+v/+QSZMmDbpcDg598cEHHzRKKfM/7+sKIQ4HrgAqgQ4p5R+FEPcDcSnlNUKIm4F/SSlX7HScs14chpS+1sygWyqd1AI/B9Z3/h4G3EASsIDozgdIKe8G7gaYOXOmXLp06V4SzcGhJ0KIyqG4rpTyv8B/hRCvAf/s3BwGPJ3vnfXiMCzpa83sDffXCYALeBjIFEIcA7wAbADmAJnAkoFep6qqipKSEgzDoK2tDYBgMDjQ0zo4fK4IIU5GKZDfAvsJIeag1o4hhJgNRKSUGwZ6nQcffJB4PM4VV1yBZVnU1tZSWlo60NM6OPRg0JWKlPLlXexeN9DzJ+vq2NTcTE1tLeXl5YBSJo8//jhnnnkmpmkO9BIODp8bO7uB9wbr1q1jzJgxbNigdJOu62zatIlUKkVFRcXevrzDPsaIy/6ympv5y51/wefz8fTTT3PttdcCUF5ezhNPPDHE0jk4DD/uuusupJSsXLmSefPm8d///pcvfvGL3HbbbUMtmsMoZMQpFZlM0tjYQHl5OVlZWdTU1ACQm5vL5s2bh1Y4B4dhSGNjI5qmMWXKFL797W/z+uuvo2lal9vYwWEw2VuB+r2GtCxK8vN54403qKqqQghBfX09W7duZdq0aUMtnoPDsKO0tJTs7GzWrVuHx+PhvPPOI5lMUlxcPNSiOYxCRpxSQUpuuuQS1rS3c84553RtXrx4MRdeeOEQCubgMDz53ve+xyeffMKtt97ate2ll17i+uuvH0KpHEYrI0+pAEHD4OCDD8ayLHRdJxQKcd5556Hr+lCL5uAw7MjOzk5bL7Ztc9hhh5GdnT3UojmMQkakUpEdETIzM7t+DwQCQyiNg8Pwp/t60TTNUSgOe40RF6gHsONxZCo11GI4ODg4OOzEiLRUsFLIRAJhjEzxHRwcdlBXV0dzc/NQizFk5OTkUFhYONRiDBoj9q4sEwnw+YZaDAcHhwHS3NzMxIkT98mYqGVZrF27dlQplRHp/pKAnXCaHDs4jBb2RYUCo/N7j0ilIgwTu6NjqMVwcHBwcNiJEapUDOyOyFCL4eDg4OCwEyNTqZgmdtRRKg4ODg7DjRGpVDAM7FgMadtDLYmDg8MQ0RZN8s6GJn70z+W8s6GJtmhy0M795JNPsmDBgj73NzU18eabb/a6b926daxatWrQZBlpjNjsL4FqLinc7qEWxcHBYRD5+fMr+XRr+y4/U5bt5cQpRVz50AcA/P29Ku6edwhLltdS09pjphkAB5Zk8tNTpvRLhvz8/F023MzNzeWoo45K2/bcc88xZcoUJkyY0K9rjFZGrFKBzrRiR6k4OOxz7Jfn57XV9Wnb/rO6nv3y/X0qlT0lFovx5ptvsmjRIn76059y6623UlxczOrVq/n617/OSy+9xLRp0wgGg0SjURYtWoRt23z00UeEQiHKyspwu9189NFH3HDDDYMi00hg5CsVB4cRjhBiGnAj8DRwFPAcEAeKgLFSyjuHULzPnf5aE+9saOLR97d0/X7qF0qZNS6X644bHEvht7/9LdXV1SxfvpyVK1cSDoeZOnUqc+fOJZVKkUgkOOCAA7jtttu4/PLLKSkpYerUqVRXV9Pa2srrr7/OL3/5Sw4//PBBkWekMDJjKgBCw44OzhOJg8NQIYSYCHQAbuAQIAdYA1wupXwBcAshpvZy3JVCiKVCiKUNDQ2fq8zDhQNLMnnsyllccHg5j105i8nFmbs/qJ/U1dVxxRVXsHDhQgzDYMaMGSSTSV599VXef/996urqaGho4O2332bevHmsWrWKkpIS3nrrra59BxxwADfddBPvvPPOoMk1Ehi5lorp1Ko4jArOBULAJOB/UMrlAmB7VZwF9Hh6klLeDdwNMHPmTPm5SDrMCHpNZo3LZda43EE/97nnnpv2e2trK2PGjKGsrAwhBIcffniaBXLsscf2ep5vfOMbgy7bcGfEKhVhGNgRJ63YYWQjpfxfACHEUcAZQDXwJLBUCDEbiEgpNwyhiA5AVlYW3/nOd4ZajBHByFUqpond3oaUEiHEUIvj4DAgpJRf22nTtiERxMFhgIzYmIrQNLAlJAcvN93BwcHBYWCMWKUCIAXYCUepODg4fL787ne/45VXXhlqMYYlI1qpIEEmnbRiB4d9lkjnHJaOJkgMXoz1ww8/5PXXX+9zf0FBAVLuk/kRu2XExlQAEAI7Hh9qKQaV1tZWtm3bd93pHo+HsrIyTNMcalEchorFP4Da5bv/3HE/hpVPwQFz4OWfw8l/hHA9vH1H758vmgazf9svEZYsWYLP58OyLJYsWcJNN93Eddddx8EHH8z48eMB+OSTT1i4cCF33XWXM9K8GyNaqahuxaMrrbixsZGxY8fi9XqHWpTPHSklTU1NVFdXs99++w21OKMCOx4H20Ybjf+f1ixWCuWhM+D0u8D0wfv3D8qpS0pKCIfD+Hw+Nm/eTHFxMR6Ph/PPP5/77ruP8ePHM2PGDEKhEPX19Y5S6cagKxUhxJeArwLlwF+BU4A3gJXALCAXuEdKOWC/lRiFtSrJZBKPxzPUYgwJQghyc3PZV4v59gZ2eztWKIR73LihFqX/9NOaoKMJHj4TvrYQnr9eWS5n3QPerAGLUFRUxHvvvceaNWtIJpNUVlbS3NxMbW0tdXV1+P1+3G43jY2N1NXVdVkvDnvHUnlbSvm6EOJW4GCgAPgUuAL4A3AsSuks6n6QEOJK4EqA8vLyfl1ImCZyFM5V2ZdTpPfl7763sFpahlqEvYPphVNvh/wDoKMeJp08KAoF4KSTTuKkk04C4KKLLgLgxRdfBGDWrFldnzvvvPMG5XqjiUFXKlLKZGdbifullGuEENmovkZuIMkgVggLXcdOJZGWhRiFYzkdHAYDKxRWHb1HW5zK5YOccWC4Yfq5g6ZQHAbG3nB/XQgcBrQJIcIot9ejKGUyB8gElgzmNWUigRiNPmMHh0FAxmPY0Sj6aFMqAO4M9dNRKMOGvWGpPAw83MfudYN9PejsVryPK5W2aJJPt7az6JOtnDyjhANLMgl6B+cm8uSTTxIOh7nkkkt63d/U1MSqVat6zJcANbAolUoxefLkQZHFYc+RiSRWOIyeOXgNFx0c+mJEZ39tZzS3wHcGFjkMGJcLq7kFSkqGWhKHfYARpVSStbVofj/C68VuD6mNutNY0hlY5LArNI8bq7Vl1PXJa4m1EEmlr32f4SPbkz3gc2+3sCdMmMCZZ57Jc88916/jnn32WU477bQBX38kM2KUSqq5mdiaNVRfdTXlCxdgFBYiDBM7Gh11acXdcQYWOQwYIUBKZDSK8PmGWprdcut7t7K6efVuP3fjITdy4YsXpm17eM7D3Pyfm3v9/KScSXz/sO/3S4buFnbmbtyGLS0tPPvss1x88cWcdtppxONx7rvvPr75zW/261qjjRHTpsWORKm+6moAqi6+hI2z5yBTSYRhYI3CtOI9xRlY5LA79nWLfk/4xz/+wV133cUrr7xCZWUl0WiUBx98kD//+c+sXr2a0047jaeffpqbb76ZVatW8dprr1FZWcn5559PU1MTzz77LFVVVZx55pnYts38+fOH+it9bowYS0XzeSlfcD9Vl1zKmHvvwSwtRRgmGAZ2ODzqTPs9xRlY5LBLDJNUWxtGXt5QS7Jb+mtN1IRremzL8+ax4KQFA5YhLy+PG264gZqaGhYsWMCKFStIJBLccMMNtLS0EAgEOOOMM3jqqacoKSlhzJgxVFRUoGkaJSUl5OfnU15ezimnnMLvf/97LrvssgHLNFIYMUrFyMnBLi0FQM/Oxu60TpQikSqt2O0eQgn3HZyBRYNLtxn1bwN1QDODPKNe83iwmpvBqfzuF3V1dVx77bX89re/JRKJUFJSwk9+8hO2bt3KVVddxdatW9m2bRt1dXV4vV42btxIe3s7tbW1xONxOjo6qK+v56KLLuK2224jbwQo88FixCgVAC0jg4qHHsIOd6D5/V3bBZ0ZYI5ScRhhdJtR7wGCUsr7hBD3A3Ep5TVCiJuFEFOllCt2Om6POlAI08QOhdTDl8s1+F9kCPAZPpactaTHtsFgZ+scYPHixV3vX331VQD+/e9/A/DII48A8PLLLwPw9NNPU1lZya9//WuuuuqqQZFppDCilIqRk8OWq67GCoUove22tH2jOa3YYVSzfUb9icD26HQYpWRgUGfUS+xIBH2UKJVsTzbZDDzTa29RUVHBj3/846EW43NnxATqt+MqLydZU4O0rB0bhYYdHZzUWYc9xxlY9NmRUv6vlPJPqC4TDUKIOaji4YWDPqNe07HC4UE5lYNDX4woSwXAHDMGkkmS27bhKivr3Dj6uhV/JiLN4MtR3VtNr+qNNAh8+OGHhEIhvvSlL/W63xlYNHCklBfu/lOf4by2jau0FOFyY0ejpFpadqwbB4e9QL+UihDiBOB0wA80AYuklP/Zi3L1iat8DADJysquxSGMUVwA6QwsGpEMhzWTrK0lvnYtW668qqu2CyGQto3QRpyTYkhJpVKcc845PPHEExjGiHsW/1zZ7V9HCFEOLJNSvtxtW4UQIkdK2bxXpesFs7QUNI1EVRX+I49U8pgmdmjXrUxGPc7AomHDcFkzdjTKlitVkLjqYtW3bdwLi1QRZLdEl5FMqrkZOxJF83kxcnL22nUMw9htEaSDYrdKRUpZBSCEOAJoBCZLKfvXs2AvIEwTs6SERGXljm2aBpY9qjJbunAGFo04hsua0YNBSm+/nZrrrkuzVOxIJC17crhR++tfE1+1+4r6vG9egx2LUX3NNym76y/o2dnIaJTGv9zV6+fdkydR9MMf9kuG2267jY5Ol3pWVlZX65XW1lauvPJK7rjjDr73ve/x8MN99c7dd9kTOy4IXA48tpdk6TfmmDHE169P2yYFambEaFMq/cUZWDQcGdI1Y+TkYBYWqPeFhdgdEaxwGJmbg5GfPxQiDSrC66X6kksBqL5GtUSpeOzRQTl3Xl4eUkqys7O5/PLLaekcdJaVlYXf76fEac7ZJ3uiVLKB+1CzUv69d8TpH2Z5OZF33sHu6FavIsFOJIb1E9hexRlYNBwZ8jUjDJPyBx9A2jYAmtutiiCHMf21JlLNzVQ89BCV3/gGFQ89hFFcjObzUvHQgwOWoa6ujoyMDJ566qmugsftlrhhGDz//POEw2FSqZQTY9mJ/sRUvgpI1JjgSlTV75Di6iz2SmzZgmfSJLVRCOx9vVbFGVg0LBhOa0YmE1TNu4icyy4jc/ZshGlihULqAWyEW/VGTg52RJUSGMXFuMpKB+3c3/++ahXzrW99q2vbokVqAvqCBaoNzCmnnDJo1xtN7DYFREr5LynlvwFDCHGCEOK4z0GuXWKOURlgaXEVw3DSih2GBcNpzZglJeh5ecSWd88glKNmrWg+L+NffhnNt28P6RtO7NJSEULM63ybiWobcc/eF2n36Lm5CJ+PZFVV1zZhmsjRmlbsMGIYjmvGM3kSkQ8+RFoWQtcRumrCSvbwrUbvL0ZODuy9pC+Hz8DuLJVCYJKU8g5g2ESmhBC4KirSLZV9vACyJdZCTbgm7dUSaxmUc69bt45Vq1aRSqU49dRT+33cs88+OyjXH2EMuzXjnjQJGYmQ2KAK84XbTWqYx1UcRi67i6m8LqV8r/P9h3tbmD3BVV5O+I03ulreC13HTiS6nsZGC87AohHHsFsz7s64Y3T5ctwTJyLcbuzW1lG3Vhx2IhmDVAyEpl6a3vm+8+f2AljbBmmBtNXL3v7eAlcA9D1LRNilpdJtcSClfB5ACDEsjE1XRQUyEsFqbEzb7jSWHHycgUX9ZziuGT0QwDV2bFdcRQiBRA6rfnn7apufQf/etgUdjVD3KYS2QfMmiLZA80aofBs2vQEbX4MNr8D6l9Vrwyuw8T+w6f/B5jdh2ycQaYLGdepnYs/CCv1WQUKIR4EYkAcMedqDWVEBqGB995x7mUiAd/QE7ZyBRSOX4bRmPNOm0b54MXY8juZ2IyTYHRH0jIyhFAsA0zSJxWJ4R9G67S+xWAzTNAd2Eikh3g6hWmirATsF2RXw7p0w6VRYdCOc8ifw50P9SnBlqPeBEqVwPJnq2IbV6jzjj+s89mR45BxV/5Yzbkd26W7YE7vmQeBT1OCgIacrrbiyEt/MmWqjdCyVvYEzsOgzM2zWjGf6dNqff5746tV4Z8wAtxurpaWrOHIoycvLY/PmzUMtxpBRXFy85wfZNiQjEG2F1kqIh6BhDWz9CCrfguJpcPjV8OApcNodyoJ56FSlNAAueBJW/AMmfFV14Jj7B+Xm+vdPYNPrnceeBmfdqwqqjf7PqtoTpXIpsAooAP67B8ftFTSvF6OgIC1Yj66P3saSu8EZWDQsGTZrxjN5MhgG0WXL8M6YoYogWwYnkWOgZGVlkZXl1Fb1iZSQjKpXrA0ijcoaySiC6v9C/iTV++/9u9Vnyw6Fr/4anr4KzroPFt0EX/4hXPYSrH8VEh3Qvg0mnQIPngpn3A2BYqWUzrpXHf/ERer9optV9/M9KKjeE6XyJrACZcoPC8yKih5pxftqBpgzsGhYMmzWjObx4J44cUdcxTCwUskud9g+w/Yn/O036EQYNBNcftXqSDfVSzNBd+0IZu9M91iIEHsuQyIEHc3QlaEpOs/T+dNwKzdVKqpkaatRFkhrJbRXw5E3wX//0mlp3Agn/x9MO0t9L9MLmgGnzodgGXz5RzDxRPV9yw9X1wgUwWMXdFM6P4IDZqtzJzrglPmQWQLH/WiPWz71p6Jek1Lanb+WoZ66dt7e/fNfAr4KlAMvATWo9t8rgFlALnCPlHLAfipXRQXRDz7oqg4WponVsW9aKg7Dhz1ZM0KI/YGTUYpnMSr28gawkkFeL95p02h94gmsUAg9EFCtjSKRka1UUgn11A6oJgak3/ClrTKgoq0qfpAIqY+ZXsgsVQrEHVA33OYN3Y6V6uYu9G6nloCtPmP61E031gq+PGC7IvCB4e2pjFKJzrhHnerNp7sgo0CNp8gdB43rofp99ZlEBA69FN6/BybOhld+odxTmUXw2i/BXwBr/6Vu9g+ephRDzjhoWg+puFIKkWbIm6CC9pNPVt8/FQcrCXZSfe9T/typdH4IE7+q/j6GVzVSDBQoBTXtHDWjaQ/oj6VSKoQYJ6X8M4AQwhRCHINKl+xtjNzbUsrXhRC3AgdLKR8WQiwAjgD+AByLUjqL9kjSXnCVl4Ntk6yuxj1uHBiqqGt7mrGDwxDR7zUjpVwvhHgVmA4cjFJAnwJXMMjrxTNtGjz+OLEVK/AfcYRKw29vH5lFkPEQtFZB+9addnSue9OrbvrRVnVTtDp1sisDwnUQLIV371AuoGe+qQLZ3myoWapurBkFkDNBfTZQCO010LBOWTZWHCaepILZk09TlsIpf1bKadvHKl3XHQBfrlIe4VolRzKqguG1y2HmZfDe35Sl8dQVSmnkjoNnr1Vyxlph5qWq8/gZf1WB9cZ1cPGiHYH2NEvjh8rS8GSp7+sOKMWxneBOg9mkVN/F9MGM8wa1tVN/Wt9vEUK4hRB/QD011QC/k1L2OpdUSpkUQkwFHgG2F06EATeQpI+Z20KIK4ErAco7g/C7w9WZAZasqsI9bpyqV7FtUo2NmKOgC+twxRlYtGv2dM0AtcB3pJTzhBDZwI3shfXi3n9/hNdLbPlypVQ8HlLNLV3raEQQbYWWzUqpZI2B7P3A9KiajHhIWSLRFrXv3TuV0lh0oxpa11oFT1+prJf9j+8MRp/aLZB9Rnoge7t76ckfqJu+YcILt6j9Ve+o4x84WR0vLXjqMhWbyBkPRVOVUtm2DLLL1c//dxukImB4IKscJp8KD5wCZ96j5I21wkXPqxt9D/fUD5ULq2mj+q5CVy6q7LE7XFQ7K45dIYRSPDDovQL7dUeQUq4Hvt2fzwohLkR1ZW0FGjuf0F4ANgBzUO0rlux8nJTybuBugJkzZ/YredsoKkK4XGnBei0zk/jateiZmSPbrP8MOAOLhg/9XTOdEyJdwLNCiBuB9cCjKGUyKOtlu80uDAPPlClEly1Tv7tc2K0tyFQKMZwfDqRUiqJpvVIqpgeyKuCd23dMOp37BxVofrazyDZNadypLIZVz8FB31Bur2lfg39cmn7TvvwV2PKusijiHd1u+t0C2Wf8TV2/+AsqmH3mvfDCzXDsD+C4W+Afl8CaFzuV0l1KKT17rVJqZYdC83oonKqsqJ2VxoQTVT1JKrHDPZVVviPeYfqh7BDQ3UoxJSMqzXf614dVE9lB/58kpXwY6GtyzbrBvJbQdcyysh7tWhCC+Pr1eA48cMS7wZyBRaOb7tMhe2HA60Xz+9NCDN5p04guXUqqoQEjPx/ZGVfRh+NDgpTQ0aCUSTys4hVuP6x8lpZpZxI54mq1/+x7QHfhE5A96xrw5sCUM+Cpy9MD0Sf+WmVObT/3qfN33LQnnqRu0kXT1P4e7qUfwaS5Kh5iJSERVZZC90B4MgrfeAYiTbTk7kck7xr1+bPvAcOLLwOydQPiIVqEJHLuQhVQv/Lfar/hIbv04M5kAZdy2bkDMKMPpTFMu5L3d0b9OUA16inqLCll73eqIcBVUUHko4/StumZQVINDaQaGjALhj4P//PAGVg0vBgua0bz+dADGV1ZXp7p0wGILltG4PjjEaZB9JNPEKaJlpGB5verl9uNcLkQbvfnP89+u2XSuEa5tFwZSqGsfBqWPYGMtxM64krmPndG2mFLTl9E9nSV/t7i9ve8aSPJjraAbdPizSJiZkG8BSbPBmnhc3nIbqwBBC0ZeT2PlxbZ0XYwPbR4MogICakwTJ6NtC1chkm8fhmRVBxX3n6c+tzpafI9fco/eLP+fVKuDKamxnDm82ely3/WErL9hTs2bK8N6UVptMRaiKTSk5J8ho9sT/aA9+/u2N3RX0tlK5AppawXQqzo5zGfC2ZFBfZrr2G1tqJ3y3XXg1k73GAezxBKODCcgUUjlmGzZoziYhLr1oPbjVlWhp6VRWz5cgLHH48eUBaKtCzsaAwrFEYmE8pl1pni6p02NW1t9YeWWAuRpLoxyc6sLK/hJdOlrteeaCeaSg8VeQ0vmRJoWk8oUEA0WALSxkaSSISJT/oqt9e9xgpXEX+M96yx2Rqt59I3vkeRK4vvHvkzzluc3gvvkdkP873V99OeivL9WT/ioiUXpe1feOJCbvrgF2hC4w/H/pGLl1ycvv+khdyw9BZsaTP/y/O55KVL0vbff+L9XPrhr9T74gN7/k0S7fzw03vV/sLpPfY3Rht5YMUDTMqdxOFFh3f93SxpYdkWtrR5vfp16iJ1nDLuFM5/8fy04/8+5+/89O2fYmoml0y9hPNeSJ/E+vDsh7nyX1cStaL84ou/6PH9HzjpAc5ddC6/Pfq3PfYtOWtJv0sW+nsHmAtYQogS4EBUyuOwoKuyvqoKb7f/+MI0QdOJr9+AZ8rId4PtDmdg0bBj2KwZIzubeOd7IQSeadOILluWliUpdB3RS5sUKxQiWV+PnpWFlJKknSRhJYikIsRSMSQSKSWWtEjZKZZsWkJtpJbzJp3HBS9ekHauhScu5LrXrkNKye3H3d7jprzwxAV879Ub0DWdW7/8Z+YtmZe2//4T72e9P8BkXwl+veeDokczKTAD1CVaido9M7DjdpKqaAN+048uejbSNHWTA3ImIaXE0HreGk3NZEruFNUlXe854Mxv+pl34DzcmpuAK9Bjf9Ad5JbDb0HTNLLcPZW0ZVs8tuYxJFIpqJcu7fH9//Thn9CExoljT+xxfMpOsbxxOSk7xXmTeo72tqWNjU3ADKBrPb+/LnSK/EW9/m32hP4qlU+B7VWG1oCuOFCEAHuHk9jVrQeYd3q69tczM7GaGknV12MWFjLacQYWDSuGzZrRPB70YCZ2NIrm9eKZPp2ON94gWVXVZ+ZXR3GQqEhg2x6ktJCtmxBC8GnTp1SHqzmy5Ei+/sLX0465/8T7uXfFvQgEX5v4tR7ndOkuDi86HE1oeIyeSsHUDPYPlCMiTeg9S+DIcWXy08mX4tdcZOg9k3CC7mx+cMxvaE+0k+PpmaiS48nhliNuwW/4CbqDPfZnu7P59VG/xpIWCaunUspyZ/GTWT9BIEjJVI/9AVeAK6ZdgRCC9kR7j/0+w8fpE05HIKiL1PW8viebPx37J7Z2bMVv9hyLHnQFufXoWwmYAbLdPa2GbE82vz/m91jS6v37e3O48/g7MTWzh3tr+/G/Oeo3vX63PaG/2V8PbX8vhJg0oCsOED0YJFlbB52ZXXowiJ6VlVZZ3x0tM0h87TrlBhvlDeucgUXDh+G0ZgDMomLia9eA14t3mgpGR5ctS1MqcStGNBmlNd6Gu3ACp72Y7vNfeNJCvv+GsloPKjioxzWCriB/PPaPZLoye73p+XUXZwUPBDuJr5en4YDm4tbKtWQ1rqfmxJ7Hu3UPh3kKcWkmLWYGS85cnFbN7jN8lAcrkFL22mDVpbv4YskX0YTW635N08j3qVKE3vbrmk5pRmmf+zWhkeVRFogt7V7bJm23cPymv9f9R485mkgyQlu8rcf5PYaHI0uPxK27aQ/RdhIAACAASURBVIo29fr9ZhbN7FM+QzMo8qs2dLFwrM/v19uxe8KedCk+FZgHrB3QFQeInp1Nojr9S5vl5ek9wLohTBMMQ2WDTZ066t1gDsOH4bJmAPTsLNVwVUqMvDyMkhJiy5dhzP4y7bF2WmItxDtdRltTjRxg96x5MDWTCydfyNjgWHI9uT32G5qBV/fiMTy9ulfcmsmRuVMRuklNL+4rV6yNYKgO64vXqwLCnRDSwpU3CTJLyDZcfXr4hRB93rS1zvPurlfeQPfvrm3SrvYH3UHCyZ4lTbqmd1lYPnPvyT/QPoL9zf66C/gI+IOU8p1+n30voPn97KwWXBUVhF56qc+hQ3ogQKqxkWRdHa6ing1jpW1jRyJYra3IWAzX+PGO8nEYEMNpzQBoLhdadhYyFsPymIgDJxB54x22NqzCMN2ksHknupLX2pdSmajl/ulH9jhHpiuTuePmErfivVxBxSSOKDkCXdNpiTSw5PRFqHYnGkiJDw1hKmXiE4ban4qpFNpEBF8igjj37+ieTHzoLDljEVgplYElBD7TD96eyqw3BnJTH4z9A2VvKq3d7R/od+uv++saIcRRwBQhRENnYdeQoLlcaD5vV78v6BzYlUiQrK3FVdp7gFrPyiKxbh1GMIjm9WLH49ihEKnGJlJNjUjLBk2AZWOWlCB8g9Ph97OwL7eZGS3DmobTmtlOIjdA/SeraPIkKfjSEYiXXydXyyQ+eTxxmWBGvJDX3/mQS/NPJdfoGXOwpEVrrJWAO4DP9LH4zMUIRFd1pc/wKQsl0UF2rIPsd++AA+bCyz9TxX8tlfDidyHeTvbX/072un+p4sBXf6UK/QLFqleVtMluWEu2EJA7UW3vxfLZW1i2JJGySdl25zqUGJqG29A+t3U53BvE7op+u7+klG8CbwohrkHl3g8Zel4eyZqt0KlUtg/sSlZW9qlUhGGAYRJbuRIAKxIFJMLtQcsIdFk4VkszViiMNkRKZV8eWCSlpKmpCc8ITgHvznBYM7a0aY42Ux2uJhRvwZ1ow+XLInHw/tgCnnj0Fv5xtHIJPT3nSX4zRmX4uSyN5+Y+hezsq6hJ8Go+yksOx9R2MVQqGYOaD1WTxwknwkOnw2l/URXgnzwGYw4Dd6ZSMNsr1s+6T7Ub2T4XJBWDYDnk7NfnHA/LloRjSSJJi0giRcrq7P9Ip7xC4DY03IZOwrKwJWgINE25xwSg6wJTE3TELdqi6lzRhEXSsinKdFMfSrBkRS0nTS2iIOCiLhTHa+j43QZ+t0HAa+A1dbymvsfKpiOeoq49RtKykRJ8Lp0Mj4nX1PGYn5/y2hv01/31C2Bz569DnkalB4Mkt2zp+t1VVgaaRqKyEv8Xv9j3cYGAao1vmn22MREuN1Zjw5ANL9rXBxZ5PB7Kyvagh9EwZbismeZYM8salylfvMtgkbacl6v+y58PuIOKu+ZzQyDIbF8bl713M4bQiUlJJBWhbeW7ZLmCFGQU4Df8WO0hzCIDM3MXCiWVgG0fqeaLWRWw5Ac72ph8+Ueqlcr2ivbeKtYnfEUpo5KDdvSl6oZtS0KxFPWhGLaU1LXHWbKiltlTiygMemgKJTpTnCE/4KKmJcriLqXgpr49jkQ9vBQG3TR0UxpFQQ+plE17NEl7NEnKklz+4FIAHnmvioWXHIqpCTRNEI6nMHRBVU2k6/r7F2SQk+HCbezaogrFkmxpjiAEXdefPa2Iggw3Gxs7ECiFGPAa5Ge4CfpcuHQNl/E5F6AOgP5aKjGgGDUfYsjR/enpdsI0MUtK+gzWd0fz90zVSzuX10uquaXP+MzexhlYNGoY8jXTEmshmoxSllFGzIrh0T0cM/1Uqt+uZpIoJi5b2XbBRZTdM5/nZ91HnuVjayJErieHAl8BHmOHtax5vaQaG1U38N6wUlC7DFq2wMs/halnK5dW9tgdbVDioR2fT8a6tUnpbL3uzlIt3Ls9pUspCcVTNLTH2dYWI2Xb+EydjoTFJQvfB9RN/555h/Cf1fU0dSQZm+tj5ticNKWw4OJDqW6J0B5NkR9wISHt+L994xDueHUdy2vaOeOgUtw73cRfWllLNGHz0spajpmQx9dmlnHFgx90Hf/QZYfhatEIuE1Ks7xkeo00a6M9lqSysYPGcAJDCOKWnSbfPfMO4ZMtbRiaoCjoYZo/yIaGDpas2MBJU4sozfLiMTWCPhd+155bRp8n/Y2p/FoIUQQchQo+DinC5ULz+9MGDLkqKoivG3hrMaFpSClVP6RAz6clB4f+MBzWTFu8jVOeSS9KfebUp/l28Cz0eIqGa65Xn7tC/dSee5rJgfF4vD37gAnTxA61d9W6pGHbUP+pcmm9+guIhaB4OvhzlUtr0lyINiulYiVUR+BIs1I4tctVny5/flrcJJa0aI0kCMVStEQShKIp1teH+e+mZgoye7rEXl1dj6lrvLuxiUyvwcur0utA/vWpUgrPfFzTq9L4z5p6zjp4DF+aGGdGWRCPS+ex93d4Q06aWszG+jBJy2a/PD+vrKpPO37x8m1cdtR+1LcnqA9FcRs6++X5CXhMtjRH2NoWJRRNsWRlLX63jrFT65tXV9cTS9o8/VENEwoyuPGEiXzrkQ8BpXQevPQwfC6ddXVhDF0jP+CiMODB5zaQUqJpAk0IdG3olc2e9NQoAc4G3mIYKBYjL4/EluquehWzvJyOt95SQ4cGGg/RBFZbm6NUHAbKkK4ZrZe0XF0zIDcb6TIpe3Ah1fMupvTuv6H7fKQ2b6H+hz/Ef9RRZHz5y/gOPhiZSiIME6utDYTACoXSlYqUqkdX03r4z6/U0Km5v1cDosL10NGkfhpe5dLyBNWURcOtXt0sE9uWtEaT1LREMHRBQyjB4hXb+PIBBWxu6uA3S1bjd+uceVApx00qTLvpz51eQkGGm7MPGYPb0KgPxdL2z5lWjC4EX51SiNfU0QS9Hh9P2Ugp8bl1HrrsMJYsr+WkaUWUBL3k+FzMHJvT6/mPmpDPDY99TEmWl69MLuSgiizW1oVZsqKWr04ppD2a4uYnPkYTggsPL+e4yT3lz/GZnDSlCK9LZ/GKbWn/botXbMNn6jRHkhxcnk1ehsmymrYu911+wEVdu8rKMzRBYaab2vY4i5cr91px0EtrJL2g021oBL0mQoDHNPCYGh5Tx9QH5mrrb0zlbqAeuBo1H2LI0TMz1RNPJ12V9VVVeCYNrNZM8/pINTSoWI2Dw2dgOKyZ3mpFbGxa/BB/dzGTp38ZAPeYMVjtIVINDXi/8AVCr7xC5pw5xNeuofpb11K+cAFGYSFaIAOre5NW21at2uvXwP/7HbRUwUm/gaLpamZJRyPk7q8skl1kb0USyr21pSVKyrbJcBk0hJJc+oByDz363hbunTeTP53zBcpzfZi6Robb4OHLDmfx8m2c1BmTaI0kMXSBLSWlWT4evlztnz2tmNKgl3A8RcCjYkIZbmPH/qlFFGS42Nq2oxdZwjIoz/Hx3RMPoDmSoK49RjxldyYCCIqDnrTze02dwkwPr69tYFNjBzd7J3LNwzssjbsuOJgrjh7HEeNyyc1wp8k/e1oRRZlu2qMpCjM9+Fw6c6cV8+h7O5TOVw4s4p8fVrNkRS0ratrIzZjIN/++4/z3XjST9zY2U9ceozjo6eH+u//iQ6ls6qAjbuExNL5QnkVzR4JH39vC7KlF5HVTSm5DIz/DTbbfRYbbwG3uWRigv5bKO8BG1HS6cQxx9hfsiI1sT7/t3q5lwErF7cZqbkImEghXzx4/Dg79YNitGQCB4MD9DsNsXY1uuhi3+EWEYSJ0He+MGXhnzCCnowM9kMHmM1VFfdXFqkfXuMUv7og3Sku5vFq3wFv/p2aNfOUXUDYzXaHsFCPpTiiWZFNjB03hOIam4dI13ljXRKQzA6s7r62p5+ovjacpnKAtmqQxHCfH5+Kmr0wgYdmE4xZuUyOetAjHLVoiCfwugwtnlROOW1Q1q9ENQiil09QRx+fSuXBWBZZto2mCKSVBvC6VzdXdjZTldzEuP4N4SmWHtUeTNEcSeAyNC2eVE0nY6EJw4wkTufKYcTR1JHh9TUOa/G+ua+TqY8fTGkkSS1o0dcTJ9Jhcd/z+dCRShGMWKcsmYdm0RhMUZrrTLKXioIerjhnPN2ZVEElY/Gtlbdr5X1lVh21L/vVpHSfPKO7h/vt3N/ffhIIMirO8ae61ey+aSW1bjPwMN2OyvWxti3HfW5s4ZXoJU0qDBL27SNDYif7GVBZ0+/X1fp99L6JadQfUjd/tRs/Lo/TOOzDzC9CDQWWuDwCJwOrowHCUisNnYDisGZ/h48UzXiRhJ9CFjqEZ+E0/2Z5sYoWFJKprenXx6n4/eiCT8oULqLr4Esr+cieu/fZDGCYgsZvr0eNbwZetJiCe+Es11CpQpCYg7kahRBMWlc0dtEWS5PhdFGZ6eO6TbTy5dAtNHQlmTy3i3EPH7BTTKGJrW5RkyqY46CU/003AbfQZsLZsSdKySdkyLc2Ybu+3Z1pp/YxDuA0dt6GT5XNRnuvHtiXhRIr69li3JAKD8Xl+Mj1GmvyzpxfTHE7QEo3jdxkcWJxJboa71xiIbUssKRmXJzmkPJvWSIKtbTHCcdWTK+g1mTO9mEd7cd9dOGtsn+4/TcBxkwoozHTzzMfpXUleWVVHNGGzcmsbN31lhxX06HtbeOzKWcwa139je0T3KTfycklUVoHbjZGVhVVfz+Zzzuky17t8wZ8BYZqkWlowRuL8bocRgxBif+BkIA9YjRo97AdWALNQrrN7pJQ9OxzuhmxPNgFXAEtauHdqwGjm55Patq2PI8Fqa8PobMKqZWQQW70G15gxEA9hrf4P+tQj4J07VPHiSz9WmVzeHGW95E5QNSY73fATKZvqlgiVTRFKsjwkLJv739rEMRPyCXoNCjI93HTCRKaXBQl4TOVeWqbcW+U5PjI9Jples1/BaF0Tvbr/BhNNE0omj8nYXD9t0STVrVG2tccozHQr99aKbZw0tYhcv0nCknxhTHZnHKPv76BpAg0BneL73AYl2T7iKYuOuEVzR5yAx0izZHL9JlvboggEEtnD0ikMeGiJJCgOevC7e7rX5k4vIZG0+eqBhbyxNt3Kev6TrXukVMRwrGCeOXOmXLp06W4/Z7W1EV22DD07B83vY+PsOWn7xy1+EbujZzfO/iBTKWQ8hu+ww4Z1+p7D4CCE+EBKOXOIrj0dmAEcJKW8WQixADWP5Q/AsUBCSrlop2O6z6g/pLIf6fTdkbZN5N13Ef6MPlPn9WAQmYgT37iJ9uefI/fck7EbNiJMDz7XJhh7JPz9bFVrsv/xUPORip/spFBSlk1te4xNDR3YUpLjc9MQjnXFTAAWXnIoxZkeEpa6H7VFk7gNwfiCDAIec8DB48+TWNKiIRSnOZzANAUeQ6cg00OGe3Cf4eNJC8uWpGwbywZbys46HDXDRhcCIZRlGElYJFI2SdsmkbTJ8pnUheLdlI6bjriF321QH4px4X3vdV2nL0ulrzUzoi0VFVcRKq5imF3meun8+ZilJZ3m+mdDGAZ2KIGMRoe0ZYvDPkEt8HN2xF3CgBtIotrmR3c+YE9m1PeG0DT0oiJStXV9jhLebuW3/fMpOt56i6xjJqJn52Es+wuc8StloXSfsX7g6coF1i2bq72ugU2hFG3SwKNrvLy6gXhS3dy68+9P67j6S+MJtcdojyUpDHgYX5CBZw+DxMMBj6kzJsdHWbaXpCX3WuFivwPofZTmTSiUHL5fLh3xFA3hOLGURXssRUGmq8vKmTujmMnFezZqekQrFWEYaJkBZDyeZq6bhQVsPutr5F59NYETThjQNYayZYvD6EcIcQLgAh4GMoUQxwAvABuAOUAmsKTvM3x2zLw8ktW7aXOeipJ52ATCr/2H8AfryCt7BqPtE+yWrWjdixcPmJOmUBpCMVZvbWfxilpmH5iPLW2ufuxjWqNJ5k4r5uyZZekxh2nF1LZFSdrKRZTjH/mxTCEELmP4ejm2y+cyXGT7XVAYIJa0CMdTeE2Di48oJz/Ts0dBehjhSgXAyM0lvnkzmseDMEyVzaIbeA86iKa770bLyMA/a9ZnOvdQt2xxGP1IKV/exe6BV/PuAi0jA800kKmU6o23M1YS6tfgKs3HPWEsoZdfoXD2FuIVZyCjfjzeHOXyOvB0yCwGVHpwTUuEbW0x5i3orFh/fwt/ueBgjto/l6Mm5DOlJJiW0rs95uAxdcZk+zBGkKtrNCGlxIzHyGxvw9fQgNXainfqVPD33n+tL0b8v56emYnojAtZbW3YHRGs9nbyv/1t3PvvT8Of/kR0xWcbEd69ZYuDw2hDaBpGcTFWSwt2dCcPm7ShZSPYSTA95EwMkwpZtFrHYY2dg9XYhKxbCdkVECginrJYXx/mvU3NxJM2i5enp7y+ua6B7x+UxZQilW0WiiXJcBlccHg5eX4XY3L87JeXsU8pFCklMjWwKYsDxU4kSDU3E1u3jsi77xL58APiGzagZ2VhFhUhk8k9PueIt1TS4irdgoOax0PB//wPtbfcQv2tt1L085/33beoEz0YTKsgdlq2OIx2XGPGoAcCJLZswWpqBNNE82cgwrUQbQFPJubq+/D4l9EQ3I+WZTGKT9CxIy3YgcMR2ftR3x5jXV2YSNLizXWNLN/SwhVf2j8t5XXOlAKiVWuJu9sJF6pO4gUBN8VBL1m+XWdDjUasUIj4ps3Yba1oWVmYhYXowWBX26ldYcdiWO3tWI2NWB0dgEBoorNNs6ZGeAiB6GwFI20bFcm3kLZUDwyWjZQ2pFJIBOg6VlMT8VWr8B9zNPG1a6m57nrKF9yPTFloPm+fTXh3ZsQrFaHraFlBZCyG2KknkR4IUHjLLWz70Y+o+9WvKP7lLzGLi3s9jxbIIFFVyZbLLk9LSU61tjgtWxxGLULXMXJzMXJzsTs6SNbVkdy0BiJ1eA85Gj56CH32j7C2biBgL6PlH4tJrF9D8ujTWdaRw6JnVnLi1CKSKZvvPPkJoViKmVmCUj3Bh9cdQqojiuHz0tiRoC4GvqZq9i8Ikr9/xW47+g4lMpnE7ujAiscxs7MHrQjajkRIVFaSqqvHGFOGnuEH3SC+Zg1SqnuWUVSo5j51xnKlZWGHw6RaWkg1NGBHowghMMvLMTxehGGQampSs4hsG2xASvT8fNXo0zBINTSC0BCGegC3k1HMwgJkIoHd3k719Tdgt7cDEDj+eLZcp/rBVV1yKQDjX36536PKR4WtaeTmYsd7zlzevq/olltASoRpIFwuhK7R9uyzNN51F9t+/GOqLr2UVF0dWy67HFAVxBtnz0Gmkl0tWxwcRjua34+7rAj/GAPfrGPQP74bfeLR8NKP0cZ+gewLzke4DNqXbmCtyOH8+97nkfequOj+90hYNscfkM8fDg3w0ylu9vOCsWkDTafOwajaRJmR4tAZ+zF5cgXBui1o7QMrTh5spGVhhUIkqquJfPwxHe++S6qlBRkOk9i6lcSWLcjEHpcKdWHH48Q2bKBj6QdElq9AeNwkNmxk45y5JDZuwMjLw1VRjrQsEuvXE1m6lI733ye6YiUd77xD9JNPSNZsxWoPEf3oY4RpEl+7jo1z5hBbu0ZZIR0d1N96Kw1/+hNYKRKbOs+/fj1CgN3expbLL6fqgguQHR0kNm5i85lnYYXDlN9zD2P/8SRld92FUVhI+UJVu1v+4AOMf/llNF//5zuNeEsFtvcB6zur0iwtpeLvD5PYvJlNp55K6fz5BL5yAnY4TMPtd+A79FCE282Yu//Gliuvouxvf8VVXo4wTKdli8O+Q2f7eqFpiI8fUIWNfz8becY9SG8x9oqXyJg1k9Db7/Ovo9K7zry5roFvH15I60d1JNxBtHiU6svVQ1rNpeppVyXR6GiBTOIrV6IddLB6Uh8CpGVhR2NYHWGshkaslhb1pK8JNI8Xs6KC5JYtOzwX+fkktm5FGAZmYSHC7F9G1PaJtInKKhI11YReXEzkvfcof+hBaq5X1kD1Nd8E1A28/ve34TvsMHyHHopZWAjJFMI0aFqwkOjHH5PsnLWUceQXqbnuOgBqrlU/Kx59pCsTVugG1Vdfo87/TTV4bezjj+E/5hi0jAzMstKuNjzbjx+3+EV0Ccn6eoz8fADMoiJcZb0PPuyLQS9+FEIYwO+BO4Ai4BTgDWAl/awQ7m/x43akbdPx1ttowWCXH3Fn+lMcuf0z5Q8+QHLrNtz77w9AqrkZ77SpTnX9KGYoix8Hyp6ul16REupXQctm1cvr+J+oOpQvXgeLbkIe+z/YFV+h+u1Pifz4BzSfdxkXRCd3Hf7wZYeRveFTmhOQsWkN46eMAwQ1115L2Z13oAUCYNnYySRGTo5KDLBSeA86KC2OIG0bmUh0vexoFFwudLcb4XZ3ehr23G1mJxLISAQrFCLV0oLd1o5EIiQIjwfcbqz6eqIff0z044/Jv+EGqubNSzvHuBdfJFlTDZqGa+xYpVy6Zc3JVAo7FkfGY2oYoNCQiTgymaT2l78iunQpwucjc+5cci+/jNS2bVRdfAnlCxeg5+Rih0Ns/e73SNbUUDp/PkLXqP7WtZTOn4+eFUQmk4RffQ3vQQfhnTGDZE111/E7dxDRg8H+719wP0ZBAWgayeoatECG+j2VQvP7MXJ7r6b/3IofpZQpIUQbarLnwUAB8ClwBTsqhL8K7KpCeI+uKTQNPTtb+Rr7GMPbvTiy+x9558+Me/EFEpWVNN55JyW/+x3CNJ2WLQ6jn7YaaFwLb/4Rti2DaGvnEK0K5Jd/iBz3FbasrOXeZj+TCyYydckzPHj3uby0roXZ04rJT4TpSKYYu20T4Xtup76sjDF33gGAa9w4Wh55lNbHHwfDIGfePDKOPx47HCa2ciVGbi5WKIwdjSBjcUAqJdcZQJa2RfcwvnC70Xx+9EDGjriDLbsCz3RmVUkpIZnEam/HjsZUCY3QEB4PWlYWemYAGY1ihUJsve46UvVqRopRpOptxtx7D1suv4LS2+erm3DtNpK1dbgnTCCxcSPJykqM4mJ1jnAYGVeTJQVglpWRrKlSx8+fT/4N16uC6kQS3e/HamvvsgaM/AKscBjhclP65z+TqKnBLC1h81lfA+iyaMYtfpHgGWcA6daEkZ+vxoBsD853jijYXgph5OeTqK5Wf7vOv6HV3IxZWqL2FxejZ2QgPF7loRngcMK90qZFCPEz4GEp5XohRDZwI6qu8+fAcUBYSvlKX8d/lievxLZtJDZsQM/q+8a/3RLZXfuWyAcfUP+b3xA8+2yyzz3XadmyD7BPWyrRVlj/impf37hWTWrc/wTw5ULTemTu/qxudfHrx5fyRgt8I7KW8/91N1x5LdpRR2HrLrJbtqGvX0vj/NtxjR9P4Y9+hKu0NC2bMrltG01//SuxlSvxTJ1Kye9uBQnoOqmGBvXUb6gmkVJK5dZOpRAuE6utvUtcmUohk0mM/DywLDVKfPtNVWzPghLqpmlZCMMkWVtLfONGEhs2EN+wgZx530AmktRcfz2lt8/HyM9HaDrJ+nrMoiKg2/3ixRdofvjvtC9ahN3ejnvyZIJnnknGl46BZBIMg9iq1SQ2bSK+di3xdeso+sktVM27KO3PPG7xi1ht7djhENg2rvHjlfy6TrKqCjuZBNHpfisqSrc0CgpB10hUViGQyoVVXIxMJhFuNwihSh9SFlgpbMtC83iQyaSyBHVdPSAbhnrpOlY4jB2J7lFmV3c+N0tFCKGjZnIXCyFORrWeeBTVbmKvVQjrgUyVOrcr2bYXR+6mfYvvkEPwH300bf/8J/7DD8c1dix2e9xp2eIw+kjGYON/4NX/hbZq1b5+7FFgxaG1imTOBJY2mvzwmVVsaoHzS+BrBxyK9eHTyH8tJvvASQTdOtEVy2n8299wT55M4f/8D5rX26OZq1lcTOFPf0r4lVfwTJ9GfMMGaq67nrK/3oWelY3dEab257/ADocp+vnPSG3bpm768+ejZ2dhh1UgWsvIoOA73yaxeTPVV19D2V//ipGbg0wkaHnscRCC7HPPVfuvupqyO+5Aywxg5ORQ97OfYRQUoGdlU/n1rwNQc90OS8DUNBVvicXQMvyMe2ERaDrB008nc+5cwq+8Qttzz2EWFpJYt065p26/HSMnG6FpNP31r7jGj0foBmPuuYctV1zRqRSU1SAjHbgqKjAKCtLcfmZJiaqxC7WTqq8nsXkTZkmnJZGfT6qhHj0/H++0qeh+/6DEd42cnH5ndO0JI7qhZHekbdPx9ttomX3HVfYEq72dmhtvxMjLo/g3v8Fub8M98QCnun6Usk9aKh2N0LQOELDyGcibCKUHkYq20+opY3VHgBdW1HL0hHyqWyIIBIdqITqqKgl++jH6Px6l8Gc/I7FpEy0PPIDnC1+g4Lvf7bpZSsvCDrUjAc3nR+t2I9S8HjbOPTlNnIpHHqH+//4PPTOTvKuvZvPZZ6ftH/v449T9/vdK6dzyYyovuDBtf/mDD7LliitASsbce08PS2G/Z58h1diEHgz2jDlsjyls2QKahpGbi56Tg+Z2k2xoILWtFpCqAacQaB43m844M/38zz+HHYl2uY+6LJ0XFmG1tKgU4NzcfrmXZDKpss9iMTSfDz03d9h5SfpaM6NGqQDEVq7E6hiEccKddLz9/9s78/ioqrOPf5/ZZ7JMdhKyEAigRRBls+DK21KgIiAo2E9RULS1trXQ7bXSVl+XtvbtixJRqcgq1ldRrAoS1Nra+lG7WIv6WqkIBEhIyL5MMtud8/5xB8gyw5ZJIXC+n898MpnfPOece+c+97n3nvOc8zY1S5eSPm8eKZMmYU1OwnXeeQkpW3N6cdYFFV8tvPkLGDIF3rgXddUywq50qnZupyKSgc87hIUdZhFef9M40tx26prbGXhwNxkEwQhjS0snXF9Pw4YNZC9efHhUlOHzxpObKwAAFq9JREFUoQJ+nMXFYLEQrqoi0tZmPqJye7D365e4juYT1FUoRCQQwJ6dxa4rpzFoy2Yi7X6sWZlYk1OwJHm6ncBVMEioro7Qvn2odj/24gGdOtpt/fqZj7H2V5iP/DCXOEcps7M7O/u0Cwo95Yycpbgr1qxswnU7IEFBxTN+PJ5x42h89lncY8aYI1IMo8cdWRrNKSXkp62+EveQychTszFmPUEgdRAPbHyTP1U7GDkwGaet88qB2z6qYuElAxlYkkVScSqhgzUEy/dQPm8eBY89Su7dd5sDWurqMBobsaan4Rp+3uEVWh0FBUTa2wk3NhKuqiawezeOQx3FOTmEa+vMiiLmGvFGSwv2nCMd0aHKzmu/GE1N2HP7HdGrqsFiOdynEvH7Dz9ysmXnENq33+xEF3O0ly0zA0tSMiWvvYrF48FZcvT1QsThwJGXhz03l0hTE6GKik4d5aH9FViSk7DnRjPj3W7E7T7jAsnxcEbdqUR8Ptreew9rRuKWBA83NFC5aBH2wkKyFi0iafQonV2fYBr8DbSFOw+c8Ng8pLv+faPtzpo7lVA7fLKFSGo+lrI7aBv7TTzbvocxcQn/SJ/MT8r28+UReYzI9zJ/zZE1NZ66+SIuGphxeG6uYPlePps8uVPRg7ZsJnTgAI7Bg7Hn5Bz1MXQkEMBobCTS1obYbIenRcJiBasFrFasLjcqFEQczugcWZ3PVWKzmQmJdjuRdj9EDDAMVNgAFcHqTTPvGpxOM+fM4zYnnj3OHJNjEa6tNdvvNju6z7aLzT5/p3LoxHMoCCoULqsLp818fhsIB/BH/Khh+ebyoRYLLmUn6cDxZe768rz4JRwt2cQZsWIJt+KZdx2tv15FzY5/0DiyCGlpRBA8dg8ZruPr6erpifNY9j0pP5aty+rCbXMTioTwh/20h9sxlIFVrFjFitvuJtN1fM956/31+II+wipMREUOv97c/ya+kI/JxZOZu3luJ5uts7bidXqxyLH7x06HoNQnCLbBB8+gXvspgQtvwnrlMvaofEouuxN/8RTSrVn813Qv/pBBVkp0TY2Pqph2fh7D+ns7TfZoSUmmaO1a9i5YYD7+ycpGnE48Y8ZgcbmO2RSL04klulQFAIWFvbHFvYotK+tUN+G0pM8ElbZwG1Oen9LpsyenPsk979yDQvHT8T/lhq2dk5Wem/I02w+8y0DvQJIHDSFojZgTrinzFttiKD795B0q/NVcmPMl5m79Sif79VPWcefHP8KfHaR07AjyBg2j9UANVaqJW99ezNppG3hy5+MUpRYxZsDFZIQcWPxBIk4HB51+IirCu5XvUtNew6QBk/DVVJJj8XIw0sTCv3yXjVdt5Hd7f0emK5OStJLD4UwpRURFEIQPaj+gNdTK6H6jufblzh2XG6/ayPOfPo9VrFzc/2Jmvzy7k7511lZSHalYLVYa/A04WwKIP4ByOWlwhXGIg+aWGsTlorZqV6e2bZi6gbvevoumYBO/vOyX3LTtpk5lb5i6gQf+8gAlaSVMLp5MTtBlbrvLQZW9jaARZN3/rWN3825+OPaHLChb0Ml+9eTVPPz+wwBckn9Jt9+7pq2Gh/7+EKNyRjGxcCKGMjoFMLfVjcPqoKatBoVixoszum17IgLuGUOwDd7fgPHaXTSQwkdp0/BVJXHHy3/n0Wsm4fKnolQAm9VCsTeJVLedof1SuXRIdszibBkZRKKZ1rbsbCxer9kJfRY+7tF0ps8ElXjYLOa4dqH7wdys2llUuQwqYW3RGhZsubGTvnryar696xfm+xGXdrMXhFxHFi6Lk6Kf3QOf7KRl1nyGlpby5sjVGB/W8IXvrCNkA++y5YSNCJXfvp3c5aWkZXixhCPUPHg/PjfkL76CYB3U3TqfAb8uZduYVXgahe+9eDcBG/xi0lIQIdvqpc5o4jt/WMxDl/6KZb9fgjUC54y/jw0XLiXT5qUu3MSPXl2Mb8c/ee7thwAYcUkGT45aSpbFS2OoiaUbF1GzaSMr3ltPWtDGLbeuoL2+noO33U7Oo6W4071YQxHeuu8mZt75BOk+qPrmfEoeLeX1C1fhbLKTVx0kN7cIj83DqnFLOwUdIxLm3Y9f5bMGg+tmXUzbgZ3UfuN2MleU4sjxkpnRjzf2vUFeUh52S/fHDV6nl3sm3IPT4iTLld2tfAReL3+dbXu2MSR9SLegtmbyGm7cduPh37ErB3wHWFC2gGx3NndetIT22gOdyi+bXUY6pz6oiMjlmMnARcAKTmIGimMS9BH562qM391LhZHOjcYS7vN48bpSmD6yP1Z3GsWZHtI8jhNattfi8Rzuk4iXda05++gzQSUSo+8nyZbCwmG34bRbSHN2P0G4ramMdM8jEG7EZul+S+7AzaX+6RQ4k8l0dHcKu+HBXj4TS3I6dmWhPDqe/VCGa9Gm59h07gQynQGG5uSw95o5AFR9K6o/tYHxH7tIDgaw3xrkwK3m5/VfN/9616/jV6vMtVryz5No2fMZUlrKtrGrMaqaeOSxQ3oSNJv6wNJSNl62GqOxiQdXRvURHmgy9aLSUh6esQajsYmvvmpOtGm7yaDyNrPeg7cdaV9+vRebw8XeBQs7a+vXcfuDuzGseyl8rI5wCKq/OZ/Bj5Ty1nmrkI8aeXyZeZ6zf7GdA98w7eqi25jx5Hr+a3U2LUX5eOw7WDNmKVk286T+7T8uJrK7mrq3fLjbasiaMYDkKqj9xnwGrijltVGrcDd6GPn+Ddj7BXE1h7sFHcGB2/dFcpKySLalmbp4qQ838dNXFmP7678o+aONzPB+igoChOvh4G3zyX+ilJc/v4r0diskd/vJTwVvK6XeFJEH6I0ZKILtNP31Gdyv38NuI4efp/yYu8+3Q9BHQf/+3D0j56TXfzfzHHoh0UHTp+kzQYUYiY0WbHx9pY9heSmUXt/9TiUYtPHW34dzzah8rKr7xHUWSeKV3RMYkpPMnBi7IqBsvFGXxg2DCwg57KQ+UUrzzbeT+kQpoWwvkbR09l06i9xz+2NkZ3XTjbQ8fn7tXVw2vICFBf3wriyl6ZbbSXu8lEimF8ORxvIJ1zM6L5m5RQXsnWmOez8ctH77AmtGTaUkJcD1xUWUT5/ZSR/w0m95cOx1jB+UxZwBReyd0VkvemUzW+deywXFLoqKC7q1j/Q8BlyTBQW5nTQj04tyZbB7XDHZhhWb10vFXDNRrPqb0bqfe5bGUVlk9bNCbjapK0tpjm6bpHuJBIV+rfsZ8O4ucr5yNTkVZsA7t7SUNy5cjVHbxMWP3m/+Dv9xAbXRoFQbDUop69dxx9aVAOSPGh3dLtP+3c+twvjwIGuXlyFAfv9xoEx9cGkpz1xhBtxFr5UDYLvxSEBtvtn8K6/1ygq9J4xSKiQiw4HVSqkdHWag6PEa9S3+INaGfSTvLoMbt+A00llU/g/IHs7AQSUnvEysRnM89Jmg4vPb2Tr9t4jvILgzwOrAXr+H5zN/DenDUW3zeXn6yzT5I3hdVsIRsIqLpXNGkp/mwuPo5pd4HFbunXEe2SlO2gMRXpq+mWa/cdjeYXXzxPzRZLjtNDsDZBYNpRlILxpKbYpBe6iFBy74DJvxEc3OOYSzvQCEsr1c9e5Cts54icfHVyKuVlrtGaQPGEoT4C2O2geCLJnlwS4ByEjrdtKXtBR+OXQVAP6UH1Cwfi37b1hAwfq12HL7IVaDx0uWYjjSCKXPomD9GvbfcCMF69dgy81FnHa+K8ugHCoiU81A0qF9ZZNWMyj0KY3BCjKKhtAMZBQNoT5VEaz5lKmD3kFQ+HO8MeseN2w3OFM44PIRzjHLDvQ7VPYazp9WB0E/oYGF7LnKnLPoUMAr/u1zFF5eh9UZwZYUoGjtKvYuWEjRmpXm7Kz1n1F4eR1GwIKtKJ/ymbM72Q94fiOZ5/pRVhe2/rmUz57TufyXX6B4Wit2h49AUW63fatcJ7ZEam8hIvOAcUCTiLRiPvZKyAwUlqZ9eN5/nPC4r2Hb+gOKriold9REHMmZWCy670PTO/SZoFLksOB59ibax96Ge9sc1BV3oLLOIdP4AOuuN2H8FfDBq6jBX0Jeuh81bSmqpZqCHb9BknNpvnwRZVOfRtk9iL8B3Bm4Bb7SvApqfURSv4rj/acJDZ6K/aV7UFcuhfpdFG7+DhJsg68+S1gyKSl7GUvLboosyVDzT/jjTwAwzrkEZ/pAZNtmlMtF2YyX8NTsIP1Pd5gbkLMxtv3f7gKgYcSEzid2LwRaKvBcdBtGxkCUw4Yz1Ryy6ExS2Cq2Yim6CEbMweqrQb23HMvnzD4GZ1Y6tqQIlr8+ApffAZ5M3Ml5pBNEXt2Kcghl0zbicXqxzl1HWmohat8nDNr6ItaGT+hfuQc558uwYAsE27C7bVg61G2vLEOGToHrN4LVjseTjSsS6FJ2GrYFm1BYiTjTGLDyYcpv+TYDHl+GLb8IS1Iy9rlLzEn+cguwW808BXuqDXv161D0eaxXfwsMg1B6WueglpeL2G2k37EMbC4kO7t7+cleLDevg7CfQLKDjMISc98WllCfHCTgtnI6TLijlNoAbIgj92iN+k3/Mrh28JW4fjMT3/QnaHIW0z/V25MiNZpj0mfyVNrbWnE27qLKOYDcT5+Gc6/EEmqDtnrw1dDuzMIhBtb104jMXAH9L8Cy6RZz9lV/E5RcARfdCk9dCzOWQ/bn4MmrIdACNieRoVORC+chT81GzXgMlT8Ky6s/MbOlHElQMBZyh8O6q2DWShgwAXa+DmIDuwsKxsCz82HC7bB5EUy8EwZNhPK3IdwOKXmQlA3rpkXtL4ZPXzVHo9ncUDiW8IFyIp7+WJp3YWvfA8NngdUBVgftIQN7SyOhINhVMxavF0tSFhhBMEIQCRPe+88j9v49MPwa095iM5cQrd8F2efAe2vh3Gngja6TEGyDup2xtagerq8jEhYsRhO2tLRu+tHs29tasdbU8NnkL1Oy5QVsmclYkg9Nd6PACBJuaOxQvhfcmZhL2JnlB/eX89nM6yl5fg2O1u1w/lxwpx3RD1Sb5b/yAo7MlCP1KwVBnzndeRgsRnP39nNm5qm0NVTjeXYOvrHfImnbd4lMXIJl5HVH9ptG0wPi+oxS6rR7jR49WsXE32L+bWvo/HkkolTrQaVWXKbUh88r9bMCpd55TKmmSqWq/6lU5XalGspNffszUf1Rper3KFXxD6Wq/k+pporO9u8+plRLtVKtNUr56sz3nfQV5ufhoFLhkFIBn1lPyG9qjfuVMgyljLCpt9ZE7Tcdse+4HbHsj3f7E2F/NC0BeqjmoArs269CNdUnbh/wqdCBvSqwb58K7fko5rb1qHylFPA3dRoc+yfziucvbb4WZVRsVxW1jcp45zFlNOyLvW80mpMgns/0mTuVY3IcV9u9qgMEWsGZbE4l3vVqsKf2x0NP7U9nennbzsQ7FeDMPiY0p5SzYkLJYzpQb+s9bZ/mlHHGBhWNppeI5zM9nyP+dMIZTTyId8Lubb2n7dNoNJo+zpkVVDQajUZzStFBRaPRaDQJQwcVjUaj0SQMHVQ0Go1GkzB0UNFoNBpNwtBBRaPRaDQJQwcVjUaj0SQMHVQ0Go1GkzB0UNFoNBpNwtBBRaPRaDQJQwcVjUaj0SSMhC/SJSI24L+B5cB4oAJIAj4CPg9kAiuVUsFE163R9DVE5HLMNeiLgG1of9H0cRIeVJRSYRFpAjzAKKXUBhFZgxlg/ge4AtOJNne0E5GvAV+L/tsqIjviVJEF1B6lCVo/ef10bltv6wOOYtebvK2UelNEHqB3/AX0b671k9OPZRvbZ2ItstLTF3A3MBL47+j/DwO/AlKAGcAXelD2URdT0vrJ66dz2/4d+ql6AcN7y1/0b6713jo24r164/GXFegHpAIfi8hlwBbgM+DL0c/LEl2vRtMXEZF5wDigEajV/qLp6/TG4y8D+Eb03z91kT9NdH0aTV9GKbUB2BBH1v6i6XP0xdFfj2u91/TTuW3/Dv1MRf/mWj8Z/aT85bRcTlij0Wg0fZO+eKei0Wg0mtMUHVQ0Go1GkzD6VFARkctF5H4ReTKOPlhEFonIfXH0ESKy6ijlu6Pl/0REpItmFZE5InJeHNurReTnIvKiiBTE0L8iIpNEZGEc+2tFZLKILImh2UTkQREpEZF5IjJRRKbF0bO67p8u+rdEZI2IfDWO/iURWSUiY2Pp0f/vF5EvxrG/WER+ISJXxtGHiMgUEekXR79fRO4TkZUxtB+LyFgRmRHH9vsiMruLfviYibXvzmR66i/R78T1maP5S1Q/JT5zNvtLDP2EfCYR/tKnggpmotgSoDKWqJTaCbwBfNJVE5GhgA9wHqX88zGHQ+9U3Tubvg9YgQlx6n5BKfUj4EWl1P4YX6nDTBbaHqfuq4HfAakiktul7DDQMaH098DsGLoopWoBiWNvAVYA3wPyY9kDvwfew8zs7qaLyBeAPXHKF2AUkAN8HEe/GzNLvDBO+5cAy4AXY9g6on9jlZ0ODFJKPQ/M6tC8jsdMt313hnPS/gLH5TNH8xc4RT5zNvtLDPsT9Zke+0ufCipKqZCIDAdWH+VrVZgZyF2ZC0wHzhWRwhg6Sqk/K6VuJrYTDFVKPQ18XkQyY9lHD+yYDhxlM3CjiMTa7w9F2zgEqIljbwGM6PvWo9QTDxW1nwA8GOc7BqaT3BJDcwEXAROJvY9RSj2M6YQLYsjpwAFgI/CfR2nnZMwpS7ryN8wTxX/E0BqB7WLmfezu0J5Dx8xv6Nm+63P00F/gGD5zDH+BU+8zZ7u/wAn6TCL8JeF5Kr2JdEgUE5FHlFLVXfQvYkbmF7vaKqXujX5njFJqX5zyp2EeCJtjyC+IyFXAh0qpujhNnIT5Y8QiBxhGl6uWDlQC/YE7o7k+HdsVL6G0q54nInVAjogkK6Vau+rAYqAa8wrwkRj6xGgby2KUn66U+pmILAD2x9DzovtwJ/B0DN0FHAQujqPnRW2tSqlQDG0k5pVpexzbP2NePd/fQY+XXHjG0xN/gWP7zDH8BU6Rz5zN/hJDPyGfSYS/6CHFGo1Go0kYferxl0aj0WhOb3RQ0Wg0Gk3C0EFFo9FoNAlDBxWNRqPRJAwdVPoAIvLD6Hj3k7EdJSIzJU4CWYzvZ4hIvFlzNZrTHu0vpxYdVP7NiMhcEfmNmNnGOSJy+3GYHaRLgtYJsADYopSKO5NAR5RS9SdZj0aTcLS/9D36VJ7KGcKfMVf0ewy4FUiL5gvMAUqB+4D1wDTgHcz1yhuBi0TkVuAHmEvNZmCuV3M35jjyzUqpKhGZgpmsNA34OTAYcxz6eyLiAp4B1gKXRusbAwwEXsEcn74DM0N5GOaKhLmYiVWjAJdSak0v7ReNJhbaX/oY+k7l1PAH4H+B5UB59LNKwA+0KqU2YSalrQEOzZv0Z+BJzAN5mFJqObAXc7qFtUqpquj3rlNKvRUtdwjmVdt2AKWUH2hRSr2AmVg2C/gXZvLaVGCiUuptoBm4Evg75jQVVZiZwfp40ZwK/oD2lz7DWbnRp5gSYKBS6g1gE1CMebDmYs4vlCsi/TEP4kIgm+g8PNH3ZcA5IvIzzKuvLDrMCwRsjmbwhoEPMTNnBwKIiBPoLyJ50c+3ROu+AHgOaBWR6zAzkbdhOvHXMbNy/wIkJ3xvaDRHR/tLH0Nn1Gs0Go0mYeg7FY1Go9EkDB1UNBqNRpMwdFDRaDQaTcLQQUWj0Wg0CUMHFY1Go9EkDB1UNBqNRpMwdFDRaDQaTcL4f6ZuS4piN4dRAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 466.56x162 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "# sns.set(style=\"whitegrid\")\n",
    "from matplotlib import rcParams\n",
    "rcParams.update({'font.size': 7})\n",
    "rcParams['font.sans-serif'] = 'Times New Roman'\n",
    "paralegend = {'title':None,'frameon':None,'handlelength':3,'handletextpad':1.5,'borderpad':0.8}\n",
    "# 读取数据\n",
    "def readdata(leadtime):\n",
    "    para = pd.read_csv('leadtime%d_lbfgs_self.csv' % leadtime, index_col=0, header=[0, 1, 2])\n",
    "\n",
    "    idx = pd.IndexSlice\n",
    "    logistic_test_mae = para.loc[:, idx[:, ['logistic'], 'test_mae']]\n",
    "    tanh_test_mae = para.loc[:, idx[:, ['tanh'], 'test_mae']]\n",
    "    identity_test_mae = para.loc[:, idx[:, ['identity'], 'test_mae']]\n",
    "    relu_test_mae = para.loc[:, idx[:, ['relu'], 'test_mae']]\n",
    "    logistic_test_mae.columns = range(2, 21, 1)\n",
    "    tanh_test_mae.columns = range(2, 21, 1)\n",
    "    identity_test_mae.columns = range(2, 21, 1)\n",
    "    relu_test_mae.columns = range(2, 21, 1)\n",
    "    df = None\n",
    "    for i in logistic_test_mae:\n",
    "        df_temp = pd.DataFrame(logistic_test_mae[i].iloc[:-2].values, columns=['MAE'])\n",
    "        df_temp['Activation function'] = ['logistic' for k in range(df_temp.shape[0])]\n",
    "        df_temp['Number of nodes'] = [i for k in range(df_temp.shape[0])]\n",
    "\n",
    "        df_temp1 = pd.DataFrame(tanh_test_mae[i].iloc[:-2].values, columns=['MAE'])\n",
    "        df_temp1['Activation function'] = ['tanh' for k in range(df_temp1.shape[0])]\n",
    "        df_temp1['Number of nodes'] = [i for k in range(df_temp1.shape[0])]\n",
    "\n",
    "        df_temp2 = pd.DataFrame(identity_test_mae[i].iloc[:-2].values, columns=['MAE'])\n",
    "\n",
    "        df_temp2['Activation function'] = ['identity' for k in range(df_temp2.shape[0])]\n",
    "        df_temp2['Number of nodes'] = [i for k in range(df_temp2.shape[0])]\n",
    "\n",
    "        df_temp3 = pd.DataFrame(relu_test_mae[i].iloc[:-2].values, columns=['MAE'])\n",
    "        df_temp3['Activation function'] = ['relu' for k in range(df_temp3.shape[0])]\n",
    "        df_temp3['Number of nodes'] = [i for k in range(df_temp3.shape[0])]\n",
    "        if df is None:\n",
    "            df = pd.concat([df_temp, df_temp1, df_temp2, df_temp3], axis=0)\n",
    "        else:\n",
    "            df = pd.concat([df, df_temp, df_temp1, df_temp2, df_temp3], axis=0)\n",
    "    df['MAE'] = df['MAE'].astype('float')\n",
    "    return df\n",
    "\n",
    "df = readdata(10)\n",
    "df2 = readdata(1)\n",
    "print(df)\n",
    "'''\n",
    "plt.figure(figsize=(7.48,16.25))\n",
    "for leadtime in range(1,11):\n",
    "    plt.subplot(5, 2, leadtime)\n",
    "    #图（a）\n",
    "    sns.lineplot(x=\"Number of nodes\", y=\"MAE\", hue=\"Activation function\", style=\"Activation function\",\n",
    "                       data=readdata(leadtime),markers=True,markersize=5, dashes=False)\n",
    "    plt.legend(['logistic','tanh','identity','relu'],loc=(0.4,0.4),**paralegend)\n",
    "    # plt.text(x=0.05, y=0.95, s='(a)', transform=ax1.transAxes)\n",
    "    # ax1.set_ylim(100,300)\n",
    "    # ax1.set_yticks(np.linspace(100,300,5))\n",
    "    # plt.xticks(range(2, 21, 1))\n",
    "    plt.ylabel('MAE (m$^3$/s)',labelpad=0.1)\n",
    "\n",
    "plt.savefig('fig\\ANN_para.tif', dpi=500,bbox_inches = 'tight')\n",
    "plt.show()\n",
    "\n",
    "'''\n",
    "\n",
    "\n",
    "plt.figure(figsize=(6.48,2.25))\n",
    "ax1 = plt.subplot(1, 2, 1)\n",
    "#图（a）\n",
    "ax1 = sns.lineplot(x=\"Number of nodes\", y=\"MAE\", hue=\"Activation function\", style=\"Activation function\", data=df2,\n",
    "                   markers=True,markersize=5, dashes=False)\n",
    "plt.legend(['logistic','tanh','identity','relu'],loc=(0.4,0.4),**paralegend)\n",
    "plt.text(x=0.05, y=0.95, s='(a)', transform=ax1.transAxes)\n",
    "ax1.set_ylim(100,300)\n",
    "ax1.set_yticks(np.linspace(100,300,5))\n",
    "plt.xticks(range(2, 21, 1))\n",
    "plt.ylabel('MAE (m$^3$/s)',labelpad=0.1)\n",
    "#图（b）\n",
    "ax2 = plt.subplot(1, 2, 2)\n",
    "ax2 = sns.lineplot(x=\"Number of nodes\", y=\"MAE\", hue=\"Activation function\", style=\"Activation function\", data=df,\n",
    "                   markers=True,markersize=5, dashes=False)\n",
    "# print(df[df['Activation function']=='identity'].head())\n",
    "ax2.set_ylim(250,500)\n",
    "ax2.set_yticks(np.linspace(200,500,7))\n",
    "plt.legend(['logistic','tanh','identity','relu'],loc=(0.3,0.54),**paralegend)\n",
    "plt.text(x=0.05, y=0.95, s='(b)', transform=ax2.transAxes)\n",
    "plt.xticks(range(2, 21, 1))\n",
    "plt.ylabel('MAE (m$^3$/s)',labelpad=0.1)\n",
    "\n",
    "plt.subplots_adjust(wspace =0.3)#调整子图间距\n",
    "plt.savefig('fig\\ANN_para.tif', dpi=500,bbox_inches = 'tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 参数绘图"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>param_C</th>\n",
       "      <th>param_epsilon</th>\n",
       "      <th>param_gamma</th>\n",
       "      <th>split0_test_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>0.001</td>\n",
       "      <td>-0.036400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>0.00621053</td>\n",
       "      <td>-0.024505</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>0.0114211</td>\n",
       "      <td>-0.023215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>0.0166316</td>\n",
       "      <td>-0.022594</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>0.0218421</td>\n",
       "      <td>-0.022237</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7995</td>\n",
       "      <td>100</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.0791579</td>\n",
       "      <td>-0.022452</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7996</td>\n",
       "      <td>100</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.0843684</td>\n",
       "      <td>-0.022665</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7997</td>\n",
       "      <td>100</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.0895789</td>\n",
       "      <td>-0.022667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7998</td>\n",
       "      <td>100</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.0947895</td>\n",
       "      <td>-0.022908</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7999</td>\n",
       "      <td>100</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.1</td>\n",
       "      <td>-0.023076</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8000 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     param_C param_epsilon param_gamma  split0_test_score\n",
       "0          1        0.0001       0.001          -0.036400\n",
       "1          1        0.0001  0.00621053          -0.024505\n",
       "2          1        0.0001   0.0114211          -0.023215\n",
       "3          1        0.0001   0.0166316          -0.022594\n",
       "4          1        0.0001   0.0218421          -0.022237\n",
       "...      ...           ...         ...                ...\n",
       "7995     100          0.01   0.0791579          -0.022452\n",
       "7996     100          0.01   0.0843684          -0.022665\n",
       "7997     100          0.01   0.0895789          -0.022667\n",
       "7998     100          0.01   0.0947895          -0.022908\n",
       "7999     100          0.01         0.1          -0.023076\n",
       "\n",
       "[8000 rows x 4 columns]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pickle\n",
    "names = locals()\n",
    "leadtime = 1\n",
    "with open('svr_para%d.pickle'%(leadtime), 'rb') as f:\n",
    "    gs_cv = pickle.load(f)\n",
    "para_result = pd.DataFrame.from_dict(gs_cv.cv_results_)\n",
    "para_result\n",
    "col = ['param_C', 'param_epsilon', 'param_gamma', 'split0_test_score']\n",
    "para_result = para_result[col]\n",
    "para_result\n",
    "# para_result.param_hidden_layer_sizes = para_result.param_hidden_layer_sizes.apply(lambda x:int(x[0]))\n",
    "# para_result = para_result.set_index(['param_activation', 'param_hidden_layer_sizes'])\n",
    "# para_result_b = para_result['split0_test_score']\n",
    "# for i in para_result.columns[1:]:\n",
    "#     para_result_b = pd.concat([para_result_b,para_result[i]],axis=0)\n",
    "# para_result_b.name = 'MAE'\n",
    "# para_result_b = para_result_b.reset_index()\n",
    "# para_result_b.MAE = para_result_b.MAE.apply(lambda x:-x)\n",
    "# names['para_result%d'%leadtime] = para_result_b\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "param_C, param_epsilon = np.meshgrid(para_result['param_C'].drop_duplicates(),para_result['param_epsilon'].drop_duplicates())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "param_gamma,param_gamma = np.meshgrid(para_result['param_gamma'].drop_duplicates(),para_result['param_gamma'].drop_duplicates())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.001, 0.006210526315789474, 0.011421052631578946,\n",
       "        0.01663157894736842, 0.021842105263157895, 0.02705263157894737,\n",
       "        0.03226315789473684, 0.03747368421052632, 0.04268421052631579,\n",
       "        0.04789473684210526, 0.05310526315789474, 0.05831578947368421,\n",
       "        0.06352631578947368, 0.06873684210526315, 0.07394736842105264,\n",
       "        0.07915789473684211, 0.08436842105263158, 0.08957894736842105,\n",
       "        0.09478947368421052, 0.1],\n",
       "       [0.001, 0.006210526315789474, 0.011421052631578946,\n",
       "        0.01663157894736842, 0.021842105263157895, 0.02705263157894737,\n",
       "        0.03226315789473684, 0.03747368421052632, 0.04268421052631579,\n",
       "        0.04789473684210526, 0.05310526315789474, 0.05831578947368421,\n",
       "        0.06352631578947368, 0.06873684210526315, 0.07394736842105264,\n",
       "        0.07915789473684211, 0.08436842105263158, 0.08957894736842105,\n",
       "        0.09478947368421052, 0.1],\n",
       "       [0.001, 0.006210526315789474, 0.011421052631578946,\n",
       "        0.01663157894736842, 0.021842105263157895, 0.02705263157894737,\n",
       "        0.03226315789473684, 0.03747368421052632, 0.04268421052631579,\n",
       "        0.04789473684210526, 0.05310526315789474, 0.05831578947368421,\n",
       "        0.06352631578947368, 0.06873684210526315, 0.07394736842105264,\n",
       "        0.07915789473684211, 0.08436842105263158, 0.08957894736842105,\n",
       "        0.09478947368421052, 0.1],\n",
       "       [0.001, 0.006210526315789474, 0.011421052631578946,\n",
       "        0.01663157894736842, 0.021842105263157895, 0.02705263157894737,\n",
       "        0.03226315789473684, 0.03747368421052632, 0.04268421052631579,\n",
       "        0.04789473684210526, 0.05310526315789474, 0.05831578947368421,\n",
       "        0.06352631578947368, 0.06873684210526315, 0.07394736842105264,\n",
       "        0.07915789473684211, 0.08436842105263158, 0.08957894736842105,\n",
       "        0.09478947368421052, 0.1],\n",
       "       [0.001, 0.006210526315789474, 0.011421052631578946,\n",
       "        0.01663157894736842, 0.021842105263157895, 0.02705263157894737,\n",
       "        0.03226315789473684, 0.03747368421052632, 0.04268421052631579,\n",
       "        0.04789473684210526, 0.05310526315789474, 0.05831578947368421,\n",
       "        0.06352631578947368, 0.06873684210526315, 0.07394736842105264,\n",
       "        0.07915789473684211, 0.08436842105263158, 0.08957894736842105,\n",
       "        0.09478947368421052, 0.1],\n",
       "       [0.001, 0.006210526315789474, 0.011421052631578946,\n",
       "        0.01663157894736842, 0.021842105263157895, 0.02705263157894737,\n",
       "        0.03226315789473684, 0.03747368421052632, 0.04268421052631579,\n",
       "        0.04789473684210526, 0.05310526315789474, 0.05831578947368421,\n",
       "        0.06352631578947368, 0.06873684210526315, 0.07394736842105264,\n",
       "        0.07915789473684211, 0.08436842105263158, 0.08957894736842105,\n",
       "        0.09478947368421052, 0.1],\n",
       "       [0.001, 0.006210526315789474, 0.011421052631578946,\n",
       "        0.01663157894736842, 0.021842105263157895, 0.02705263157894737,\n",
       "        0.03226315789473684, 0.03747368421052632, 0.04268421052631579,\n",
       "        0.04789473684210526, 0.05310526315789474, 0.05831578947368421,\n",
       "        0.06352631578947368, 0.06873684210526315, 0.07394736842105264,\n",
       "        0.07915789473684211, 0.08436842105263158, 0.08957894736842105,\n",
       "        0.09478947368421052, 0.1],\n",
       "       [0.001, 0.006210526315789474, 0.011421052631578946,\n",
       "        0.01663157894736842, 0.021842105263157895, 0.02705263157894737,\n",
       "        0.03226315789473684, 0.03747368421052632, 0.04268421052631579,\n",
       "        0.04789473684210526, 0.05310526315789474, 0.05831578947368421,\n",
       "        0.06352631578947368, 0.06873684210526315, 0.07394736842105264,\n",
       "        0.07915789473684211, 0.08436842105263158, 0.08957894736842105,\n",
       "        0.09478947368421052, 0.1],\n",
       "       [0.001, 0.006210526315789474, 0.011421052631578946,\n",
       "        0.01663157894736842, 0.021842105263157895, 0.02705263157894737,\n",
       "        0.03226315789473684, 0.03747368421052632, 0.04268421052631579,\n",
       "        0.04789473684210526, 0.05310526315789474, 0.05831578947368421,\n",
       "        0.06352631578947368, 0.06873684210526315, 0.07394736842105264,\n",
       "        0.07915789473684211, 0.08436842105263158, 0.08957894736842105,\n",
       "        0.09478947368421052, 0.1],\n",
       "       [0.001, 0.006210526315789474, 0.011421052631578946,\n",
       "        0.01663157894736842, 0.021842105263157895, 0.02705263157894737,\n",
       "        0.03226315789473684, 0.03747368421052632, 0.04268421052631579,\n",
       "        0.04789473684210526, 0.05310526315789474, 0.05831578947368421,\n",
       "        0.06352631578947368, 0.06873684210526315, 0.07394736842105264,\n",
       "        0.07915789473684211, 0.08436842105263158, 0.08957894736842105,\n",
       "        0.09478947368421052, 0.1],\n",
       "       [0.001, 0.006210526315789474, 0.011421052631578946,\n",
       "        0.01663157894736842, 0.021842105263157895, 0.02705263157894737,\n",
       "        0.03226315789473684, 0.03747368421052632, 0.04268421052631579,\n",
       "        0.04789473684210526, 0.05310526315789474, 0.05831578947368421,\n",
       "        0.06352631578947368, 0.06873684210526315, 0.07394736842105264,\n",
       "        0.07915789473684211, 0.08436842105263158, 0.08957894736842105,\n",
       "        0.09478947368421052, 0.1],\n",
       "       [0.001, 0.006210526315789474, 0.011421052631578946,\n",
       "        0.01663157894736842, 0.021842105263157895, 0.02705263157894737,\n",
       "        0.03226315789473684, 0.03747368421052632, 0.04268421052631579,\n",
       "        0.04789473684210526, 0.05310526315789474, 0.05831578947368421,\n",
       "        0.06352631578947368, 0.06873684210526315, 0.07394736842105264,\n",
       "        0.07915789473684211, 0.08436842105263158, 0.08957894736842105,\n",
       "        0.09478947368421052, 0.1],\n",
       "       [0.001, 0.006210526315789474, 0.011421052631578946,\n",
       "        0.01663157894736842, 0.021842105263157895, 0.02705263157894737,\n",
       "        0.03226315789473684, 0.03747368421052632, 0.04268421052631579,\n",
       "        0.04789473684210526, 0.05310526315789474, 0.05831578947368421,\n",
       "        0.06352631578947368, 0.06873684210526315, 0.07394736842105264,\n",
       "        0.07915789473684211, 0.08436842105263158, 0.08957894736842105,\n",
       "        0.09478947368421052, 0.1],\n",
       "       [0.001, 0.006210526315789474, 0.011421052631578946,\n",
       "        0.01663157894736842, 0.021842105263157895, 0.02705263157894737,\n",
       "        0.03226315789473684, 0.03747368421052632, 0.04268421052631579,\n",
       "        0.04789473684210526, 0.05310526315789474, 0.05831578947368421,\n",
       "        0.06352631578947368, 0.06873684210526315, 0.07394736842105264,\n",
       "        0.07915789473684211, 0.08436842105263158, 0.08957894736842105,\n",
       "        0.09478947368421052, 0.1],\n",
       "       [0.001, 0.006210526315789474, 0.011421052631578946,\n",
       "        0.01663157894736842, 0.021842105263157895, 0.02705263157894737,\n",
       "        0.03226315789473684, 0.03747368421052632, 0.04268421052631579,\n",
       "        0.04789473684210526, 0.05310526315789474, 0.05831578947368421,\n",
       "        0.06352631578947368, 0.06873684210526315, 0.07394736842105264,\n",
       "        0.07915789473684211, 0.08436842105263158, 0.08957894736842105,\n",
       "        0.09478947368421052, 0.1],\n",
       "       [0.001, 0.006210526315789474, 0.011421052631578946,\n",
       "        0.01663157894736842, 0.021842105263157895, 0.02705263157894737,\n",
       "        0.03226315789473684, 0.03747368421052632, 0.04268421052631579,\n",
       "        0.04789473684210526, 0.05310526315789474, 0.05831578947368421,\n",
       "        0.06352631578947368, 0.06873684210526315, 0.07394736842105264,\n",
       "        0.07915789473684211, 0.08436842105263158, 0.08957894736842105,\n",
       "        0.09478947368421052, 0.1],\n",
       "       [0.001, 0.006210526315789474, 0.011421052631578946,\n",
       "        0.01663157894736842, 0.021842105263157895, 0.02705263157894737,\n",
       "        0.03226315789473684, 0.03747368421052632, 0.04268421052631579,\n",
       "        0.04789473684210526, 0.05310526315789474, 0.05831578947368421,\n",
       "        0.06352631578947368, 0.06873684210526315, 0.07394736842105264,\n",
       "        0.07915789473684211, 0.08436842105263158, 0.08957894736842105,\n",
       "        0.09478947368421052, 0.1],\n",
       "       [0.001, 0.006210526315789474, 0.011421052631578946,\n",
       "        0.01663157894736842, 0.021842105263157895, 0.02705263157894737,\n",
       "        0.03226315789473684, 0.03747368421052632, 0.04268421052631579,\n",
       "        0.04789473684210526, 0.05310526315789474, 0.05831578947368421,\n",
       "        0.06352631578947368, 0.06873684210526315, 0.07394736842105264,\n",
       "        0.07915789473684211, 0.08436842105263158, 0.08957894736842105,\n",
       "        0.09478947368421052, 0.1],\n",
       "       [0.001, 0.006210526315789474, 0.011421052631578946,\n",
       "        0.01663157894736842, 0.021842105263157895, 0.02705263157894737,\n",
       "        0.03226315789473684, 0.03747368421052632, 0.04268421052631579,\n",
       "        0.04789473684210526, 0.05310526315789474, 0.05831578947368421,\n",
       "        0.06352631578947368, 0.06873684210526315, 0.07394736842105264,\n",
       "        0.07915789473684211, 0.08436842105263158, 0.08957894736842105,\n",
       "        0.09478947368421052, 0.1],\n",
       "       [0.001, 0.006210526315789474, 0.011421052631578946,\n",
       "        0.01663157894736842, 0.021842105263157895, 0.02705263157894737,\n",
       "        0.03226315789473684, 0.03747368421052632, 0.04268421052631579,\n",
       "        0.04789473684210526, 0.05310526315789474, 0.05831578947368421,\n",
       "        0.06352631578947368, 0.06873684210526315, 0.07394736842105264,\n",
       "        0.07915789473684211, 0.08436842105263158, 0.08957894736842105,\n",
       "        0.09478947368421052, 0.1]], dtype=object)"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_gamma.T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0            0.001\n",
       "1       0.00621053\n",
       "2        0.0114211\n",
       "3        0.0166316\n",
       "4        0.0218421\n",
       "           ...    \n",
       "7995     0.0791579\n",
       "7996     0.0843684\n",
       "7997     0.0895789\n",
       "7998     0.0947895\n",
       "7999           0.1\n",
       "Name: param_gamma, Length: 8000, dtype: object"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "para_result['param_gamma']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import numpy as np\n",
    "# import matplotlib.pyplot as plt\n",
    "# from mpl_toolkits.mplot3d import Axes3D\n",
    "# from matplotlib import cm\n",
    "\n",
    "# create some fake data\n",
    "# x = y = np.arange(-4.0, 4.0, 0.02)\n",
    "# here are the x,y and respective z values\n",
    "# X, Y = np.meshgrid(x, y)\n",
    "# Z = np.sinc(np.sqrt(X*X+Y*Y))\n",
    "# this is the value to use for the color\n",
    "# V = np.sin(Y)\n",
    "# print(X.shape)\n",
    "# print(Z.shape)\n",
    "# create the figure, add a 3d axis, set the viewing angle\n",
    "# fig = plt.figure()\n",
    "# ax = fig.add_subplot(111, projection='3d')\n",
    "# ax.view_init(45,60)\n",
    "\n",
    "# here we create the surface plot, but pass V through a colormap\n",
    "# to create a different color for each patch\n",
    "# ax.plot_surface(X, Y, Z, facecolors=cm.Oranges(V))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}