PCA2S_TimeSeries.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "PCA2S.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/aliabdollahy/PCA2S/blob/master/PCA2S_TimeSeries.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OT539_lBCLYV",
        "colab_type": "code",
        "outputId": "adb2c9d5-e2e3-47ed-95eb-c36d52444193",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 425
        }
      },
      "source": [
        "%ls\n",
        "!pip3 install mxnet\n",
        "!pip3 install pandas==0.22 \n",
        "!pip3 install bert\n",
        "#!pip3 install bayesopt\n",
        "!pip3 install bayesian-optimization\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "%cd drive/My Drive/"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[0m\u001b[01;34mdrive\u001b[0m/  \u001b[01;34msample_data\u001b[0m/\n",
            "Requirement already satisfied: mxnet in /usr/local/lib/python3.6/dist-packages (1.5.1.post0)\n",
            "Requirement already satisfied: requests<3,>=2.20.0 in /usr/local/lib/python3.6/dist-packages (from mxnet) (2.21.0)\n",
            "Requirement already satisfied: numpy<2.0.0,>1.16.0 in /usr/local/lib/python3.6/dist-packages (from mxnet) (1.17.5)\n",
            "Requirement already satisfied: graphviz<0.9.0,>=0.8.1 in /usr/local/lib/python3.6/dist-packages (from mxnet) (0.8.4)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.20.0->mxnet) (3.0.4)\n",
            "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.20.0->mxnet) (2.8)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.20.0->mxnet) (2019.11.28)\n",
            "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.20.0->mxnet) (1.24.3)\n",
            "Requirement already satisfied: pandas==0.22 in /usr/local/lib/python3.6/dist-packages (0.22.0)\n",
            "Requirement already satisfied: numpy>=1.9.0 in /usr/local/lib/python3.6/dist-packages (from pandas==0.22) (1.17.5)\n",
            "Requirement already satisfied: pytz>=2011k in /usr/local/lib/python3.6/dist-packages (from pandas==0.22) (2018.9)\n",
            "Requirement already satisfied: python-dateutil>=2 in /usr/local/lib/python3.6/dist-packages (from pandas==0.22) (2.6.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.6/dist-packages (from python-dateutil>=2->pandas==0.22) (1.12.0)\n",
            "Requirement already satisfied: bert in /usr/local/lib/python3.6/dist-packages (2.2.0)\n",
            "Requirement already satisfied: erlastic in /usr/local/lib/python3.6/dist-packages (from bert) (2.0.0)\n",
            "Requirement already satisfied: bayesian-optimization in /usr/local/lib/python3.6/dist-packages (1.0.1)\n",
            "Requirement already satisfied: numpy>=1.9.0 in /usr/local/lib/python3.6/dist-packages (from bayesian-optimization) (1.17.5)\n",
            "Requirement already satisfied: scikit-learn>=0.18.0 in /usr/local/lib/python3.6/dist-packages (from bayesian-optimization) (0.22.1)\n",
            "Requirement already satisfied: scipy>=0.14.0 in /usr/local/lib/python3.6/dist-packages (from bayesian-optimization) (1.4.1)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn>=0.18.0->bayesian-optimization) (0.14.1)\n",
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n",
            "/content/drive/My Drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nZVqKGmC-RjX",
        "colab_type": "code",
        "outputId": "e0985655-fa32-4760-8309-3000dc7e2779",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "import time\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "from mxnet import nd, autograd, gluon\n",
        "from mxnet.gluon import nn, rnn\n",
        "import mxnet as mx\n",
        "import datetime\n",
        "import seaborn as sns\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "from sklearn.decomposition import PCA\n",
        "\n",
        "import math\n",
        "\n",
        "from sklearn.preprocessing import MinMaxScaler\n",
        "from sklearn.metrics import mean_squared_error\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "\n",
        "import xgboost as xgb\n",
        "from sklearn.metrics import accuracy_score\n",
        "import warnings\n",
        "\n",
        "warnings.filterwarnings(\"ignore\")\n",
        "\n",
        "context = mx.cpu()\n",
        "model_ctx = mx.cpu()\n",
        "mx.random.seed(1719)\n",
        "\n",
        "\n",
        "dataset_ex_df = pd.read_excel('dataset.xlsx')\n",
        "dataset_ex_df.columns = ['index', 'load']\n",
        "\n",
        "\n",
        "\n",
        "# plt.figure(figsize=(14, 5), dpi=100)\n",
        "# plt.plot(dataset_ex_df['Date'], dataset_ex_df['load'], label='Goldman Sachs stock')\n",
        "# plt.vlines(datetime.date(2016, 4, 20), 0, 900, linestyles='--', colors='gray', label='Train/Test data cut-off')\n",
        "# plt.xlabel('Date')\n",
        "# plt.ylabel('USD')\n",
        "# plt.title('Figure 2: Goldman Sachs stock price')\n",
        "# plt.legend()\n",
        "# plt.show()\t\n",
        "\n",
        "\n",
        "num_training_days = int(dataset_ex_df.shape[0] * .7)\n",
        "print('Number of training points: {}. Number of test points: {}.'.format(num_training_days, \\\n",
        "                                                                     dataset_ex_df.shape[0] - num_training_days))\n",
        "\n",
        "\n",
        "def get_technical_indicators(dataset):\n",
        "    # Create 7 and 21 days Moving Average\n",
        "    dataset['ma7'] = dataset.rolling(window=7).mean()\n",
        "    dataset['ma21'] = dataset['load'].rolling(window=21).mean()\n",
        "\n",
        "    # Create MACD\n",
        "    dataset['26ema'] = pd.ewma(dataset['load'], span=26)\n",
        "    dataset['12ema'] = pd.ewma(dataset['load'], span=12)\n",
        "    dataset['MACD'] = (dataset['12ema'] - dataset['26ema'])\n",
        "\n",
        "    # Create Bollinger Bands\n",
        "    dataset['20sd'] = pd.stats.moments.rolling_std(dataset['load'], 20)\n",
        "    dataset['upper_band'] = dataset['ma21'] + (dataset['20sd'] * 2)\n",
        "    dataset['lower_band'] = dataset['ma21'] - (dataset['20sd'] * 2)\n",
        "\n",
        "    # Create Exponential moving average\n",
        "    dataset['ema'] = dataset['load'].ewm(com=0.5).mean()\n",
        "\n",
        "    # Create Momentum\n",
        "    dataset['momentum'] = dataset['load'] - 1\n",
        "    dataset['log_momentum'] = np.log(dataset['load'] - 1)\n",
        "\n",
        "    return dataset\n",
        "\n",
        "\n",
        "dataset_TI_df = get_technical_indicators(dataset_ex_df[['load']])\n",
        "dataset_TI_df.head()\n",
        "\n",
        "\n",
        "def plot_technical_indicators(dataset, last_days):\n",
        "    plt.figure(figsize=(16, 10), dpi=100)\n",
        "    shape_0 = dataset.shape[0]\n",
        "    xmacd_ = shape_0 - last_days\n",
        "\n",
        "    dataset = dataset.iloc[-last_days:, :]\n",
        "    x_ = range(3, dataset.shape[0])\n",
        "    x_ = list(dataset.index)\n",
        "\n",
        "    # Plot first subplot\n",
        "    plt.subplot(2, 1, 1)\n",
        "    plt.plot(dataset['ma7'], label='MA 7', color='g', linestyle='--')\n",
        "    plt.plot(dataset['load'], label='real load', color='b')\n",
        "    plt.plot(dataset['ma21'], label='MA 21', color='r', linestyle='--')\n",
        "    plt.plot(dataset['upper_band'], label='Upper Band', color='c')\n",
        "    plt.plot(dataset['lower_band'], label='Lower Band', color='c')\n",
        "    plt.fill_between(x_, dataset['lower_band'], dataset['upper_band'], alpha=0.35)\n",
        "    plt.title('Technical indicators for VM(s) loads.'.format(last_days))\n",
        "    plt.ylabel('Load')\n",
        "    plt.legend()\n",
        "\n",
        "    # Plot second subplot\n",
        "    plt.subplot(2, 1, 2)\n",
        "    plt.title('MACD')\n",
        "    plt.plot(dataset['MACD'], label='MACD', linestyle='-.')\n",
        "    plt.hlines(15, xmacd_, shape_0, colors='g', linestyles='--')\n",
        "    plt.hlines(-15, xmacd_, shape_0, colors='g', linestyles='--')\n",
        "    plt.plot(dataset['log_momentum'], label='Momentum', color='b', linestyle='-')\n",
        "\n",
        "    plt.legend()\n",
        "    # plt.show()\n",
        "\n",
        "\n",
        "# plot_technical_indicators(dataset_TI_df, 400)\n",
        "\n",
        "# just import bert\n",
        "import bert\n",
        "\n",
        "data_FT = dataset_ex_df[['index', 'load']]\n",
        "\n",
        "from statsmodels.tsa.arima_model import ARIMA\n",
        "from pandas import DataFrame\n",
        "from pandas import datetime\n",
        "\n",
        "series = data_FT['load']\n",
        "model = ARIMA(series, order=(5, 1, 0))\n",
        "model_fit = model.fit(disp=0)\n",
        "print(model_fit.summary())\n",
        "\n",
        "close_fft = np.fft.fft(np.asarray(data_FT['load'].tolist()))\n",
        "fft_df = pd.DataFrame({'fft': close_fft})\n",
        "fft_df['absolute'] = fft_df['fft'].apply(lambda x: np.abs(x))\n",
        "fft_df['angle'] = fft_df['fft'].apply(lambda x: np.angle(x))\n",
        "\n",
        "plt.figure(figsize=(14, 7), dpi=100)\n",
        "fft_list = np.asarray(fft_df['fft'].tolist())\n",
        "for num_ in [3, 6, 9, 100]:\n",
        "    fft_list_m10 = np.copy(fft_list);\n",
        "    fft_list_m10[num_:-num_] = 0\n",
        "    plt.plot(np.fft.ifft(fft_list_m10), label='Fourier transform with {} components'.format(num_))\n",
        "plt.plot(data_FT['load'], label='Real')\n",
        "plt.xlabel('Days')\n",
        "plt.ylabel('USD')\n",
        "plt.title('Figure 3: Goldman Sachs (close) stock prices & Fourier transforms')\n",
        "plt.legend()\n",
        "# plt.show()\n",
        "\n",
        "\n",
        "from collections import deque\n",
        "\n",
        "items = deque(np.asarray(fft_df['absolute'].tolist()))\n",
        "items.rotate(int(np.floor(len(fft_df) / 2)))\n",
        "plt.figure(figsize=(10, 7), dpi=80)\n",
        "plt.stem(items)\n",
        "plt.title('Figure 4: Components of Fourier transforms')\n",
        "# plt.show()\n",
        "\n",
        "\n",
        "\n",
        "from pandas.tools.plotting import autocorrelation_plot\n",
        "\n",
        "autocorrelation_plot(series)\n",
        "plt.figure(figsize=(10, 7), dpi=80)\n",
        "# plt.show()\n",
        "\n",
        "from pandas import read_csv\n",
        "from pandas import datetime\n",
        "from statsmodels.tsa.arima_model import ARIMA\n",
        "from sklearn.metrics import mean_squared_error\n",
        "\n",
        "X = series.values\n",
        "size = int(len(X) * 0.66)\n",
        "train, test = X[0:size], X[size:len(X)]\n",
        "history = [x for x in train]\n",
        "predictions = list()\n",
        "for t in range(len(test)):\n",
        "    model = ARIMA(history, order=(5, 1, 0))\n",
        "    model_fit = model.fit(disp=0)\n",
        "    output = model_fit.forecast()\n",
        "    yhat = output[0]\n",
        "    predictions.append(yhat)\n",
        "    obs = test[t]\n",
        "    history.append(obs)\n",
        "\n",
        "error = mean_squared_error(test, predictions)\n",
        "print('Test MSE: %.3f' % error)\n",
        "\n",
        "# Plot the predicted (from ARIMA) and real prices\n",
        "\n",
        "plt.figure(figsize=(12, 6), dpi=100)\n",
        "plt.plot(test, label='Real')\n",
        "plt.plot(predictions, color='red', label='Predicted')\n",
        "plt.xlabel('Days')\n",
        "plt.ylabel('USD')\n",
        "plt.title('Figure 5: ARIMA model on GS stock')\n",
        "plt.legend()\n",
        "# plt.show()\n",
        "\n",
        "print('Total dataset has {} samples, and {} features.'.format(dataset_TI_df.shape[0],\n",
        "                                                              dataset_TI_df.shape[1]))\n",
        "\n",
        "\n",
        "def get_feature_importance_data(data_income):\n",
        "    data = data_income.copy()\n",
        "    y = data['load']\n",
        "    X = data.iloc[:, 1:]\n",
        "\n",
        "    train_samples = int(X.shape[0] * 0.65)\n",
        "\n",
        "    X_train = X.iloc[:train_samples]\n",
        "    X_test = X.iloc[train_samples:]\n",
        "\n",
        "    y_train = y.iloc[:train_samples]\n",
        "    y_test = y.iloc[train_samples:]\n",
        "\n",
        "    return (X_train, y_train), (X_test, y_test)\n",
        "\n",
        "\n",
        "(X_train_FI, y_train_FI), (X_test_FI, y_test_FI) = get_feature_importance_data(dataset_TI_df)\n",
        "\n",
        "regressor = xgb.XGBRegressor(gamma=0.0, n_estimators=150, base_score=0.7, colsample_bytree=1, learning_rate=0.05)\n",
        "xgbModel = regressor.fit(X_train_FI, y_train_FI,\n",
        "                         eval_set=[(X_train_FI, y_train_FI), (X_test_FI, y_test_FI)],\n",
        "                         verbose=False)\n",
        "eval_result = regressor.evals_result()\n",
        "\n",
        "training_rounds = range(len(eval_result['validation_0']['rmse']))\n",
        "plt.scatter(x=training_rounds, y=eval_result['validation_0']['rmse'], label='Training Error')\n",
        "plt.scatter(x=training_rounds, y=eval_result['validation_1']['rmse'], label='Validation Error')\n",
        "plt.xlabel('Iterations')\n",
        "plt.ylabel('RMSE')\n",
        "plt.title('Training Vs Validation Error')\n",
        "plt.legend()\n",
        "# plt.show()\n",
        "\n",
        "fig = plt.figure(figsize=(8, 8))\n",
        "plt.xticks(rotation='vertical')\n",
        "plt.bar([i for i in range(len(xgbModel.feature_importances_))], xgbModel.feature_importances_.tolist(),\n",
        "        tick_label=X_test_FI.columns)\n",
        "plt.title('Figure 6: Feature importance of the technical indicators.')\n",
        "\n",
        "\n",
        "plt.show()\n",
        "\n",
        "def gelu(x):\n",
        "    return 0.5 * x * (1 + math.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * math.pow(x, 3))))\n",
        "\n",
        "\n",
        "def relu(x):\n",
        "    return max(x, 0)\n",
        "\n",
        "\n",
        "def lrelu(x):\n",
        "    return max(0.01 * x, x)\n",
        "\n",
        "\n",
        "plt.figure(figsize=(15, 5))\n",
        "plt.subplots_adjust(left=None, bottom=None, right=None, top=None, wspace=.5, hspace=None)\n",
        "\n",
        "ranges_ = (-10, 3, .25)\n",
        "\n",
        "plt.subplot(1, 2, 1)\n",
        "plt.plot([i for i in np.arange(*ranges_)], [relu(i) for i in np.arange(*ranges_)], label='ReLU', marker='.')\n",
        "plt.plot([i for i in np.arange(*ranges_)], [gelu(i) for i in np.arange(*ranges_)], label='GELU')\n",
        "plt.hlines(0, -10, 3, colors='gray', linestyles='--', label='0')\n",
        "plt.title('Figure 7: GELU as an activation function for autoencoders')\n",
        "plt.ylabel('f(x) for GELU and ReLU')\n",
        "plt.xlabel('x')\n",
        "plt.legend()\n",
        "\n",
        "plt.subplot(1, 2, 2)\n",
        "plt.plot([i for i in np.arange(*ranges_)], [lrelu(i) for i in np.arange(*ranges_)], label='Leaky ReLU')\n",
        "plt.hlines(0, -10, 3, colors='gray', linestyles='--', label='0')\n",
        "plt.ylabel('f(x) for Leaky ReLU')\n",
        "plt.xlabel('x')\n",
        "plt.title('Figure 8: LeakyReLU')\n",
        "plt.legend()\n",
        "\n",
        "# plt.show()\n",
        "\n",
        "batch_size = 64\n",
        "n_batches = dataset_TI_df.shape[0] / batch_size\n",
        "VAE_data = dataset_TI_df.values\n",
        "\n",
        "\n",
        "train_iter = mx.io.NDArrayIter(data={'data': VAE_data[:num_training_days, :-1]},\n",
        "                               label={'label': VAE_data[:num_training_days, -1]}, batch_size=batch_size)\n",
        "test_iter = mx.io.NDArrayIter(data={'data': VAE_data[num_training_days:, :-1]},\n",
        "                              label={'label': VAE_data[num_training_days:, -1]}, batch_size=batch_size)\n",
        "\n",
        "model_ctx = mx.cpu()\n",
        "\n",
        "\n",
        "class VAE(gluon.HybridBlock):\n",
        "    def __init__(self, n_hidden=400, n_latent=2, n_layers=1, n_output=784,\n",
        "                 batch_size=100, act_type='relu', **kwargs):\n",
        "        self.soft_zero = 1e-10\n",
        "        self.n_latent = n_latent\n",
        "        self.batch_size = batch_size\n",
        "        self.output = None\n",
        "        self.mu = None\n",
        "        super(VAE, self).__init__(**kwargs)\n",
        "\n",
        "        with self.name_scope():\n",
        "            self.encoder = nn.HybridSequential(prefix='encoder')\n",
        "\n",
        "            for i in range(n_layers):\n",
        "                self.encoder.add(nn.Dense(n_hidden, activation=act_type))\n",
        "            self.encoder.add(nn.Dense(n_latent * 2, activation=None))\n",
        "\n",
        "            self.decoder = nn.HybridSequential(prefix='decoder')\n",
        "            for i in range(n_layers):\n",
        "                self.decoder.add(nn.Dense(n_hidden, activation=act_type))\n",
        "            self.decoder.add(nn.Dense(n_output, activation='sigmoid'))\n",
        "\n",
        "    def hybrid_forward(self, F, x):\n",
        "        h = self.encoder(x)\n",
        "        # print(h)\n",
        "        mu_lv = F.split(h, axis=1, num_outputs=2)\n",
        "        mu = mu_lv[0]\n",
        "        lv = mu_lv[1]\n",
        "        self.mu = mu\n",
        "\n",
        "        eps = F.random_normal(loc=0, scale=1, shape=(self.batch_size, self.n_latent), ctx=model_ctx)\n",
        "        z = mu + F.exp(0.5 * lv) * eps\n",
        "        y = self.decoder(z)\n",
        "        self.output = y\n",
        "\n",
        "        KL = 0.5 * F.sum(1 + lv - mu * mu - F.exp(lv), axis=1)\n",
        "        logloss = F.sum(x * F.log(y + self.soft_zero) + (1 - x) * F.log(1 - y + self.soft_zero), axis=1)\n",
        "        loss = -logloss - KL\n",
        "\n",
        "        return loss\n",
        "      \n",
        "    def state_info(self, batch_size=0):\n",
        "        return [{'shape': (2, batch_size, 2), '__layout__': 'LNC'}]\n",
        "\n",
        "    def begin_state(self, batch_size=0, func=mx.ndarray.zeros, **kwargs):\n",
        "        states = []\n",
        "        for i, info in enumerate(self.state_info(batch_size)):\n",
        "            if info is not None:\n",
        "                info.update(kwargs)\n",
        "            else:\n",
        "                info = kwargs\n",
        "            states.append(func(name='%sh0_%d'%('myprefix', i), **info))\n",
        "        return states\n",
        "\n",
        "\n",
        "n_hidden = 400  # neurons in each layer\n",
        "n_latent = 2\n",
        "n_layers = 3  # num of dense layers in encoder and decoder respectively\n",
        "n_output = VAE_data.shape[1] - 1\n",
        "\n",
        "net = VAE(n_hidden=n_hidden, n_latent=n_latent, n_layers=n_layers, n_output=n_output, batch_size=batch_size,\n",
        "          # act_type='gelu'\n",
        "          )\n",
        "net.collect_params().initialize(mx.init.Xavier(), ctx=mx.cpu())\n",
        "net.hybridize()\n",
        "trainer = gluon.Trainer(net.collect_params(), 'adam', {'learning_rate': .01})\n",
        "print(net)\n",
        "\n",
        "n_epoch = 10\n",
        "print_period = n_epoch // 10\n",
        "start = time.time()\n",
        "\n",
        "training_loss = []\n",
        "validation_loss = []\n",
        "\n",
        "\n",
        "for epoch in range(n_epoch):\n",
        "    epoch_loss = 0\n",
        "    epoch_val_loss = 0\n",
        "\n",
        "    train_iter.reset()\n",
        "    test_iter.reset()\n",
        "\n",
        "    n_batch_train = 0\n",
        "\n",
        "    for batch in train_iter:\n",
        "        n_batch_train += 1\n",
        "        data = batch.data[0].as_in_context(mx.cpu())\n",
        "        print(data)\n",
        "        with autograd.record():\n",
        "            loss = net(data)\n",
        "        loss.backward()\n",
        "        trainer.step(data.shape[0])\n",
        "        epoch_loss += nd.mean(loss).asscalar()\n",
        "\n",
        "    n_batch_val = 0\n",
        "    for batch in test_iter:\n",
        "        n_batch_val += 1\n",
        "        data = batch.data[0].as_in_context(mx.cpu())\n",
        "        print(data)\n",
        "        loss = net(data)\n",
        "        epoch_val_loss += nd.mean(loss).asscalar()\n",
        "\n",
        "    epoch_loss /= n_batch_train\n",
        "    epoch_val_loss /= n_batch_val\n",
        "\n",
        "    training_loss.append(epoch_loss)\n",
        "    validation_loss.append(epoch_val_loss)\n",
        "    \n",
        "    if epoch % max(print_period, 1) == 0:\n",
        "        print(epoch_loss)\n",
        "        print('Epoch {}, Training loss {:.2f}, Validation loss {:.2f}'.\\\n",
        "              format(epoch, epoch_loss, epoch_val_loss))\n",
        "\n",
        "end = time.time()\n",
        "print('Training completed in {} seconds.'.format(int(end - start)))\n",
        "\n",
        "dataset_TI_df['Date'] = dataset_ex_df['Date']\n",
        "vae_added_df = mx.nd.array(dataset_TI_df.iloc[:, :-1].values)\n",
        "print('The shape of the newly created (from the autoencoder) features is {}.'.format(vae_added_df.shape))\n",
        "\n",
        "# pca = PCA(n_components=.8)\n",
        "# x_pca = StandardScaler().fit_transform(vae_added_df)\n",
        "# principalComponents = pca.fit_transform(x_pca)\n",
        "\n",
        "gan_num_features = dataset_TI_df.shape[1]\n",
        "sequence_length = 17\n",
        "\n",
        "\n",
        "class RNNModel(gluon.Block):\n",
        "    def __init__(self, num_embed, num_hidden, num_layers, bidirectional=False,\n",
        "                 sequence_length=sequence_length, **kwargs):\n",
        "        super(RNNModel, self).__init__(**kwargs)\n",
        "        self.num_hidden = num_hidden\n",
        "        with self.name_scope():\n",
        "            self.rnn = rnn.LSTM(num_hidden, num_layers, input_size=num_embed,\n",
        "                                bidirectional=bidirectional, layout='TNC')\n",
        "\n",
        "            self.decoder = nn.Dense(1, in_units=num_hidden)\n",
        "\n",
        "    def forward(self, inputs, hidden):\n",
        "        output, hidden = self.rnn(inputs, hidden)\n",
        "        decoded = self.decoder(output.reshape((-1, self.num_hidden)))\n",
        "        return decoded, hidden\n",
        "\n",
        "    def begin_state(self, *args, **kwargs):\n",
        "        return self.rnn.begin_state(*args, **kwargs)\n",
        "\n",
        "\n",
        "lstm_model = RNNModel(num_embed=gan_num_features, num_hidden=500, num_layers=1)\n",
        "lstm_model.collect_params().initialize(mx.init.Xavier(), ctx=mx.cpu())\n",
        "trainer = gluon.Trainer(lstm_model.collect_params(), 'adam', {'learning_rate': .01})\n",
        "loss = gluon.loss.L1Loss()\n",
        "\n",
        "print(lstm_model)\n",
        "\n",
        "\n",
        "class TriangularSchedule():\n",
        "    def __init__(self, min_lr, max_lr, cycle_length, inc_fraction=0.5):\n",
        "        self.min_lr = min_lr\n",
        "        self.max_lr = max_lr\n",
        "        self.cycle_length = cycle_length\n",
        "        self.inc_fraction = inc_fraction\n",
        "\n",
        "    def __call__(self, iteration):\n",
        "        if iteration <= self.cycle_length * self.inc_fraction:\n",
        "            unit_cycle = iteration * 1 / (self.cycle_length * self.inc_fraction)\n",
        "        elif iteration <= self.cycle_length:\n",
        "            unit_cycle = (self.cycle_length - iteration) * 1 / (self.cycle_length * (1 - self.inc_fraction))\n",
        "        else:\n",
        "            unit_cycle = 0\n",
        "        adjusted_cycle = (unit_cycle * (self.max_lr - self.min_lr)) + self.min_lr\n",
        "        return adjusted_cycle\n",
        "\n",
        "\n",
        "class CyclicalSchedule():\n",
        "    def __init__(self, schedule_class, cycle_length, cycle_length_decay=1, cycle_magnitude_decay=1, **kwargs):\n",
        "        self.schedule_class = schedule_class\n",
        "        self.length = cycle_length\n",
        "        self.length_decay = cycle_length_decay\n",
        "        self.magnitude_decay = cycle_magnitude_decay\n",
        "        self.kwargs = kwargs\n",
        "\n",
        "    def __call__(self, iteration):\n",
        "        cycle_idx = 0\n",
        "        cycle_length = self.length\n",
        "        idx = self.length\n",
        "        while idx <= iteration:\n",
        "            cycle_length = math.ceil(cycle_length * self.length_decay)\n",
        "            cycle_idx += 1\n",
        "            idx += cycle_length\n",
        "        cycle_offset = iteration - idx + cycle_length\n",
        "\n",
        "        schedule = self.schedule_class(cycle_length=cycle_length, **self.kwargs)\n",
        "        return schedule(cycle_offset) * self.magnitude_decay ** cycle_idx\n",
        "\n",
        "\n",
        "schedule = CyclicalSchedule(TriangularSchedule, min_lr=0.5, max_lr=2, cycle_length=500)\n",
        "iterations = 1500\n",
        "\n",
        "plt.plot([i + 1 for i in range(iterations)], [schedule(i) for i in range(iterations)])\n",
        "plt.title('Learning rate for each epoch')\n",
        "plt.xlabel(\"Epoch\")\n",
        "plt.ylabel(\"Learning Rate\")\n",
        "# plt.show()\n",
        "\n",
        "num_fc = 512\n",
        "\n",
        "# ... other parts of the GAN\n",
        "\n",
        "cnn_net = gluon.nn.Sequential()\n",
        "with net.name_scope():\n",
        "    # Add the 1D Convolutional layers\n",
        "    cnn_net.add(gluon.nn.Conv1D(32, kernel_size=5, strides=2))\n",
        "    cnn_net.add(nn.LeakyReLU(0.01))\n",
        "    cnn_net.add(gluon.nn.Conv1D(64, kernel_size=5, strides=2))\n",
        "    cnn_net.add(nn.LeakyReLU(0.01))\n",
        "    cnn_net.add(nn.BatchNorm())\n",
        "    cnn_net.add(gluon.nn.Conv1D(128, kernel_size=5, strides=2))\n",
        "    cnn_net.add(nn.LeakyReLU(0.01))\n",
        "    cnn_net.add(nn.BatchNorm())\n",
        "\n",
        "    # Add the two Fully Connected layers\n",
        "    cnn_net.add(nn.Dense(220, use_bias=False), nn.BatchNorm(), nn.LeakyReLU(0.01))\n",
        "    cnn_net.add(nn.Dense(220, use_bias=False), nn.Activation(activation='relu'))\n",
        "    cnn_net.add(nn.Dense(1))\n",
        "\n",
        "# ... other parts of the GAN\n",
        "\n",
        "\n",
        "print(cnn_net)\n",
        "\n",
        "\n",
        "\n",
        "# Initialize the optimizer\n",
        "from bayes_opt import BayesianOptimization\n",
        "from bayes_opt import UtilityFunction\n",
        "\n",
        "utility = UtilityFunction(kind=\"ucb\", kappa=2.5, xi=0.0)\n",
        "\n",
        "\n",
        "\n",
        "# def train(net, ctx, batch_size=64, epochs=10, learning_rate=0.01, wd=0.001):\n",
        "#     train_data = gluon.data.DataLoader(\n",
        "#         trainIterator, batch_size, shuffle=True)\n",
        "#     validation_data = gluon.data.DataLoader(\n",
        "#         validationIterator, batch_size)\n",
        "#     net.collect_params().reset_ctx(ctx)\n",
        "#     net.hybridize()\n",
        "#\n",
        "#     loss = gluon.loss.SoftmaxCrossEntropyLoss()\n",
        "#     trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': learning_rate, 'wd': wd})\n",
        "#\n",
        "#     train_util(net, train_data, validation_data,\n",
        "#                loss, trainer, ctx, epochs, batch_size)\n",
        "#\n",
        "# train(cnn_net, dataset_TI_df)\n",
        "\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of training points: 100800. Number of test points: 43200.\n",
            "                             ARIMA Model Results                              \n",
            "==============================================================================\n",
            "Dep. Variable:                 D.load   No. Observations:               143999\n",
            "Model:                 ARIMA(5, 1, 0)   Log Likelihood              408811.716\n",
            "Method:                       css-mle   S.D. of innovations              0.014\n",
            "Date:                Wed, 12 Feb 2020   AIC                        -817609.431\n",
            "Time:                        13:56:17   BIC                        -817540.288\n",
            "Sample:                             1   HQIC                       -817588.786\n",
            "                                                                              \n",
            "================================================================================\n",
            "                   coef    std err          z      P>|z|      [0.025      0.975]\n",
            "--------------------------------------------------------------------------------\n",
            "const         1.509e-06   6.73e-05      0.022      0.982      -0.000       0.000\n",
            "ar.L1.D.load     0.2732      0.003    103.715      0.000       0.268       0.278\n",
            "ar.L2.D.load     0.1382      0.003     50.627      0.000       0.133       0.144\n",
            "ar.L3.D.load     0.0603      0.003     21.916      0.000       0.055       0.066\n",
            "ar.L4.D.load     0.0051      0.003      1.868      0.062      -0.000       0.010\n",
            "ar.L5.D.load    -0.0308      0.003    -11.703      0.000      -0.036      -0.026\n",
            "                                    Roots                                    \n",
            "=============================================================================\n",
            "                  Real          Imaginary           Modulus         Frequency\n",
            "-----------------------------------------------------------------------------\n",
            "AR.1            1.8148           -0.4435j            1.8682           -0.0381\n",
            "AR.2            1.8148           +0.4435j            1.8682            0.0381\n",
            "AR.3           -0.6533           -1.9702j            2.0757           -0.3010\n",
            "AR.4           -0.6533           +1.9702j            2.0757            0.3010\n",
            "AR.5           -2.1574           -0.0000j            2.1574           -0.5000\n",
            "-----------------------------------------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}