mobi_tft_no_bioinfo.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-06-24T02:14:23.984150Z",
     "start_time": "2021-06-24T02:14:23.691048Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "from tqdm.notebook import tqdm\n",
    "from matplotlib import pyplot as plt\n",
    "from datetime import datetime\n",
    "\n",
    "repo_dir = os.getcwd() + '/repo'\n",
    "tft_dir = os.path.join(repo_dir, 'tft')\n",
    "os.chdir(tft_dir)\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-06-24T02:14:25.045803Z",
     "start_time": "2021-06-24T02:14:24.292529Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " carriage_services.csv\t\t     'outputs_mobiact(v2)(undersampling)'\r\n",
      " data_formatters\t\t     'outputs_mobiact(v2)_wo_pinfo'\r\n",
      " data_formatters.zip\t\t      outputs_mobiact_wo_age\r\n",
      " dlr_preprocessed\t\t      outputs_mobiact_wo_gender\r\n",
      " dlr_tft_results\t\t      outputs_mobiact_wo_known\r\n",
      " expt_settings\t\t\t      outputs_mobiact_wo_known_no_bioinfo\r\n",
      " expt_settings.zip\t\t      outputs_mobiact_wo_known_no_bioinfo2\r\n",
      " libs\t\t\t\t      outputs_notchFall\r\n",
      " libs.zip\t\t\t     'outputs_notchFall(oversampling)'\r\n",
      " mobiact_dataset\t\t      outputs_notchFall_swa\r\n",
      " MobiAct_Dataset_v2.0.rar\t     'outputs_notchFall(undersampling)'\r\n",
      " MobiAct_Dataset_v2-Copy1.0.rar       outputs_pm2.5\r\n",
      " mobiact_preprocessed\t\t      outputs_smartFall\r\n",
      " MobiAct_preprocessed.zip\t     'outputs_smartFall(oversampling)'\r\n",
      " mobi_tft_wo_known_no_bioinfo\t      outputs_smartFall_swa\r\n",
      " mobi_tft_wo_known_results\t     'outputs_smartFall(undersampling)'\r\n",
      " notch_dataset\t\t\t      outputs_stock\r\n",
      " output_folder\t\t\t      outputs_stock_v2\r\n",
      " outputs\t\t\t      outputs_stock_v3\r\n",
      " outputs_carriage_services\t      PRSA_data_2010.1.1-2014.12.31.csv\r\n",
      " outputs_carriage_services_v2\t      __pycache__\r\n",
      " outputs_carriage_services_w_open     README.md\r\n",
      " outputs_dlr\t\t\t      repo\r\n",
      " outputs_mobiact\t\t      requirements.txt\r\n",
      " outputs_mobiact2\t\t      run.sh\r\n",
      "'outputs_mobiact(v2)'\t\t      script_download_data.py\r\n",
      " outputs_mobiact_v2_2\t\t      script_hyperparam_opt.py\r\n",
      "'outputs_mobiact(v2)(oversampling)'   script_train_fixed_params.py\r\n",
      "'outputs_mobiact(v2)_swa'\t      smartFall_dataset\r\n",
      "'outputs_mobiact(v2)_swa2'\t      stock.csv\r\n"
     ]
    }
   ],
   "source": [
    "!ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-06-24T02:14:25.793647Z",
     "start_time": "2021-06-24T02:14:25.788269Z"
    }
   },
   "outputs": [],
   "source": [
    "output_folder = os.path.join(os.getcwd(), 'outputs_mobiact_wo_known_no_bioinfo2')   # Root folder to save experiment outputs\n",
    "force_download = False \n",
    "if not os.path.exists(output_folder):\n",
    "    os.makedirs(output_folder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-06-24T02:14:26.874635Z",
     "start_time": "2021-06-24T02:14:26.852091Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====================================================================\n",
      "\n",
      "===========================MobiActDataset===========================\n",
      "\n",
      "The NEW version of the MobiAct dataset includes:\n",
      "\n",
      " \tFour different types of falls performed by 66 participants\n",
      "\n",
      " \tEleven different types of ADLs performed by 19 participants and nine types of ADLs performed by 59 participants (plus one activity \"LYI\" which results from the inactivity period after a fall by 66 participants)\n",
      "\n",
      " \tFive sub-scenarios which construct one scenario of daily living, which consists of a sequence of 50 activities and performed by 19 participants.\n",
      "\n",
      "\n",
      "\n",
      "The new released version of the MobiAct dataset includes:\n",
      "\n",
      "  The raw recorded data in txt format, separated by each activity\n",
      "\n",
      "  The annotated data in csv format, separated by each activity\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Filename format:\n",
      "\n",
      "<ADL OR FALL OR SCENARIO_CODE>_<SENSOR_CODE>_<SUBJECT_ID>_<TRIAL_NO>.txt\n",
      "\n",
      "\n",
      "\n",
      "examples:\n",
      "\n",
      "1 -->\tWAL_acc_5_1.txt\n",
      "\n",
      "2 -->\tSTD_ori_9_5.txt\n",
      "\n",
      "3 -->\tFKL_gyro_3_2.txt\n",
      "\n",
      "4 -->\tSRH_acc_1_1.txt\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Subjects:\n",
      "\n",
      "+------+---------+-----------+-------+----------+----------+----------+\n",
      "\n",
      "|  ID  |  Name   |  Surname  |  Age  |  Height  |  Weight  |  Gender  |\n",
      "\n",
      "+------+---------+-----------+-------+----------+----------+----------+\n",
      "\n",
      "|    1 | sub1    | sub1      |    32 |      180 |       85 |  M       |   \n",
      "\n",
      "|    2 | sub2    | sub2      |    26 |      169 |       64 |  M       |\n",
      "\n",
      "|    3 | sub3    | sub3      |    26 |      164 |       55 |  F       |  \n",
      "\n",
      "|    4 | sub4    | sub4      |    32 |      186 |       93 |  M       |   \n",
      "\n",
      "|    5 | sub5    | sub5      |    36 |      160 |       50 |  F       |    \n",
      "\n",
      "|    6 | sub6    | sub6      |    22 |      172 |       62 |  F       |    \n",
      "\n",
      "|    7 | sub7    | sub7      |    25 |      189 |       80 |  M       |   \n",
      "\n",
      "|    8 | sub8    | sub8      |    22 |      183 |       93 |  M       |    \n",
      "\n",
      "|    9 | sub9    | sub9      |    30 |      177 |      102 |  M       |   \n",
      "\n",
      "|   10 | sub10   | sub10     |    26 |      170 |       90 |  F       |  \n",
      "\n",
      "|   11 | sub11   | sub11     |    26 |      168 |       80 |  F       |  \n",
      "\n",
      "|   12 | sub12   | sub12     |    29 |      178 |       83 |  M       |   \n",
      "\n",
      "|   13 | sub13   | sub13     |    24 |      177 |       62 |  M       | \n",
      "\n",
      "|   14 | sub14   | sub14     |    24 |      178 |       85 |  M       |   \n",
      "\n",
      "|   15 | sub15   | sub15     |    25 |      173 |       82 |  M       | \n",
      "\n",
      "|   16 | sub16   | sub16     |    27 |      172 |       56 |  F       |   \n",
      "\n",
      "|   17 | sub17   | sub17     |    25 |      173 |       67 |  M       |   \n",
      "\n",
      "|   18 | sub18   | sub18     |    25 |      176 |       73 |  M       |   \n",
      "\n",
      "|   19 | sub19   | sub19     |    25 |      161 |       63 |  F       |   \n",
      "\n",
      "|   20 | sub20   | sub20     |    26 |      178 |       71 |  M       |   \n",
      "\n",
      "|   21 | sub21   | sub21     |    25 |      180 |       70 |  M       |   \n",
      "\n",
      "|   22 | sub22   | sub22     |    22 |      187 |       90 |  M       |   \n",
      "\n",
      "|   23 | sub23   | sub23     |    23 |      171 |       64 |  M       |   \n",
      "\n",
      "|   24 | sub24   | sub24     |    21 |      193 |      112 |  M       |   \n",
      "\n",
      "|   25 | sub25   | sub25     |    22 |      170 |       62 |  F       |    \n",
      "\n",
      "|   26 | sub26   | sub26     |    25 |      163 |       60 |  F       |   \n",
      "\n",
      "|   27 | sub27   | sub27     |    25 |      180 |       82 |  M       |  \n",
      "\n",
      "|   28 | sub28   | sub28     |    23 |      178 |       70 |  Ì       |   \n",
      "\n",
      "|   29 | sub29   | sub29     |    27 |      186 |      103 |  M       |    \n",
      "\n",
      "|   30 | sub30   | sub30     |    47 |      172 |       90 |  M       |   \n",
      "\n",
      "|   31 | sub31   | sub31     |    27 |      170 |       75 |  M       |   \n",
      "\n",
      "|   32 | sub32   | sub32     |    25 |      190 |       77 |  M       |   \n",
      "\n",
      "|   33 | sub33   | sub33     |    27 |      171 |       70 |  M       |   \n",
      "\n",
      "|   34 | sub34   | sub34     |    24 |      175 |       85 |  Ì       |    \n",
      "\n",
      "|   35 | sub35   | sub35     |    23 |      181 |       76 |  M       |    \n",
      "\n",
      "|   36 | sub36   | sub36     |    22 |      164 |       62 |  F       |   \n",
      "\n",
      "|   37 | sub37   | sub37     |    25 |      172 |       63 |  M       |  \n",
      "\n",
      "|   38 | sub38   | sub38     |    21 |      170 |       88 |  F       |    \n",
      "\n",
      "|   39 | sub39   | sub39     |    26 |      174 |       79 |  M       |    \n",
      "\n",
      "|   40 | sub40   | sub40     |    23 |      178 |       95 |  M       |    \n",
      "\n",
      "|   41 | sub41   | sub41     |    20 |      172 |       67 |  F       |  \n",
      "\n",
      "|   42 | sub42   | sub42     |    22 |      173 |       73 |  M       |   \n",
      "\n",
      "|   43 | sub43   | sub43     |    24 |      179 |       80 |  M       |   \n",
      "\n",
      "|   44 | sub44   | sub44     |    25 |      176 |       80 |  M       |  \n",
      "\n",
      "|   45 | sub45   | sub45     |    26 |      175 |       92 |  M       |   \n",
      "\n",
      "|   46 | sub46   | sub46     |    23 |      175 |       68 |  F       |   \n",
      "\n",
      "|   47 | sub47   | sub47     |    21 |      180 |       85 |  M       |  \n",
      "\n",
      "|   48 | sub48   | sub48     |    22 |      180 |       80 |  M       |   \n",
      "\n",
      "|   49 | sub49   | sub49     |    23 |      178 |       75 |  M       |    \n",
      "\n",
      "|   50 | sub50   | sub50     |    23 |      165 |       50 |  F       |   \n",
      "\n",
      "|   51 | sub51   | sub51     |    23 |      171 |       70 |  M       |    \n",
      "\n",
      "|   52 | sub52   | sub52     |    20 |      179 |       79 |  M       |  \n",
      "\n",
      "|   53 | sub53   | sub53     |    27 |      186 |      120 |  M       |   \n",
      "\n",
      "|   54 | sub54   | sub54     |    27 |      164 |       55 |  F       |    \n",
      "\n",
      "|   55 | sub55   | sub55     |    28 |      178 |       78 |  M       |    \n",
      "\n",
      "|   56 | sub56   | sub56     |    29 |      170 |       75 |  M       |  \n",
      "\n",
      "|   57 | sub57   | sub57     |    21 |      187 |       70 |  Ì       |\n",
      "\n",
      "|   58 | sub58   | sub58     |    21 |      158 |       58 |  F       |  \n",
      "\n",
      "|   59 | sub59   | sub59     |    26 |      175 |       70 |  M       |   \n",
      "\n",
      "|   60 | sub60   | sub60     |    24 |      183 |      107 |  M       |    \n",
      "\n",
      "|   61 | sub61   | sub61     |    24 |      170 |       70 |  M       |   \n",
      "\n",
      "|   62 | sub62   | sub62     |    20 |      180 |       70 |  M       |    \n",
      "\n",
      "|   63 | sub63   | sub63     |    24 |      187 |       85 |  M       |  \n",
      "\n",
      "|   64 | sub64   | sub64     |    26 |      181 |       70 |  M       |   \n",
      "\n",
      "|   65 | sub65   | sub65     |    40 |      170 |      100 |  M       |    \n",
      "\n",
      "|   66 | sub66   | sub66     |    20 |      193 |       83 |  M       |    \n",
      "\n",
      "|   67 | sub67   | sub67     |    23 |      180 |       67 |  M       |  \n",
      "\n",
      "+------+---------+-----------+-------+----------+----------+----------+\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Activities of Daily Living:\n",
      "\n",
      "+----+-------+----------------------------+--------+----------+---------------------------------------------------+\n",
      "\n",
      "| No.| Label | Activity                   | Trials | Duration | Description                                       |\n",
      "\n",
      "+----+-------+----------------------------+--------+----------+---------------------------------------------------+\n",
      "\n",
      "| 1  | STD   | Standing                   | 1      | 5min     | Standing with subtle movements                    |\n",
      "\n",
      "| 2  | WAL   | Walking                    | 1      | 5min     | Normal walking                                    |\n",
      "\n",
      "| 3  | JOG   | Jogging                    | 3      | 30s      | Jogging                                           |\n",
      "\n",
      "| 4  | JUM   | Jumping                    | 3      | 30s      | Continuous jumping                                |\n",
      "\n",
      "| 5  | STU   | Stairs up                  | 6      | 10s      | Stairs up (10 stairs)                             |\n",
      "\n",
      "| 6  | STN   | Stairs down                | 6      | 10s      | Stairs down (10 stairs)                           |\n",
      "\n",
      "| 7  | SCH   | Stand to sit(sit on chair) | 6      | 6s       | Transition from standing to sitting               |\n",
      "\n",
      "| 8  | SIT   | Sitting on chair           | 1      | 1min     | Sitting on a chair with subtle movements          |\n",
      "\n",
      "| 9  | CHU   | Sit to stand(chair up)     | 6      | 6s       | Transition from sitting to standing               |\n",
      "\n",
      "| 10 | CSI   | Car-step in                | 6      | 6s       | Step in a car                                     |\n",
      "\n",
      "| 11 | CSO   | Car-step out               | 6      | 6s       | Step out a car                                    |\n",
      "\n",
      "| 12 | LYI   | Lying                      | 12     | -        | Activity taken from the lying period after a fall |\n",
      "\n",
      "+----+-------+----------------------------+--------+----------+---------------------------------------------------+\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Falls:\n",
      "\n",
      "+----+-------+--------------------+--------+----------+---------------------------------------------------------+\n",
      "\n",
      "| No.| Label | Activity           | Trials | Duration | Description                                             |\n",
      "\n",
      "+----+-------+--------------------+--------+----------+---------------------------------------------------------+\n",
      "\n",
      "| 10 | FOL   | Forward-lying      | 3      | 10s      | Fall Forward from standing, use of hands to dampen fall |\n",
      "\n",
      "| 11 | FKL   | Front-knees-lying  | 3      | 10s      | Fall forward from standing, first impact on knees       |\n",
      "\n",
      "| 12 | BSC   | Back-sitting-chair | 3      | 10s      | Fall backward while trying to sit on a chair            |\n",
      "\n",
      "| 13 | SDL   | Sideward-lying     | 3      | 10s      | Fall sidewards from standing, bending legs              |\n",
      "\n",
      "+----+-------+--------------------+--------+----------+---------------------------------------------------------+\n",
      "\n",
      "\n",
      "\n",
      "Scenarios:\n",
      "\n",
      "+---------------------------------------------------------------------------------------------+\n",
      "\n",
      "| 1st Scenario of Leaving the Home (SLH), Total duration 125 at max (1 trial/participant)   |\n",
      "\n",
      "+----+-------+--------------------+-----------------------------------------------------------+\n",
      "\n",
      "| No.| Label | Activity           | Description                                               |\n",
      "\n",
      "+----+-------+--------------------+-----------------------------------------------------------+\n",
      "\n",
      "| 1  | STD   | Standing           |  The recording starts with the participant standing       |\n",
      "\n",
      "| 2  | WAL   | Walking            |  outside the door and locking the door. Then walks        |\n",
      "\n",
      "| 3  | STN   | Stairs down        |  and descent stairs to leave his home. Following, he      |\n",
      "\n",
      "| 4  | WAL   | Walking            |  riches the parking area where he stands in front of the  |\n",
      "\n",
      "| 5  | STD   | Standing           |  car, unlocks the lock of the car, opens the door and     |\n",
      "\n",
      "| 6  | CSI   | Car-step in        |  gets in the car. He remains sited for some seconds,      |\n",
      "\n",
      "| 7  | SIT   | Sitting on chair   |  then he gets out of the car, closes the door and stands  |\n",
      "\n",
      "| 8  | CSO   | Car-step out       |  in front of the door to lock the car.                    |\n",
      "\n",
      "| 9  | STD   | Standing           |                                                           |\n",
      "\n",
      "+-----+------+--------------------+-----------------------------------------------------------+\n",
      "\n",
      "| 2nd Scenario of Being at work (SBW), Total duration 185 at max (1 trial/participant)      |\n",
      "\n",
      "+----+-------+--------------------+-----------------------------------------------------------+\n",
      "\n",
      "| No.| Label | Activity           | Description                                               |\n",
      "\n",
      "+----+-------+--------------------+-----------------------------------------------------------+\n",
      "\n",
      "| 1  | STD   | Standing           |  The recording starts with the participant standing       |\n",
      "\n",
      "| 2  | WAL   | Walking            |  outside the cars door. Then walks from the parking       |\n",
      "\n",
      "| 3  | STU   | Stairs up          |  area to his work building. He walks and ascent stairs    |\n",
      "\n",
      "| 4  | WAL   | Walking            |  till he riches his office where he stops in front of the |\n",
      "\n",
      "| 5  | STD   | Standing           |  door. Once he finds the keys he opens the door, gets     |\n",
      "\n",
      "| 6  | WAL   | Walking            |  in his office and walks to his chair, where he sits.     |\n",
      "\n",
      "| 7  | SCH   | Stand to sit       |                                                           |\n",
      "\n",
      "| 8  | SIT   | Sitting on chair   |                                                           |\n",
      "\n",
      "+-----+------+--------------------+-----------------------------------------------------------+\n",
      "\n",
      "| 3rd Scenario of Leaving work (SLW), Total duration 185 at max (1 trial/participant)       |\n",
      "\n",
      "+----+-------+--------------------+-----------------------------------------------------------+\n",
      "\n",
      "| No.| Label | Activity           | Description                                               |\n",
      "\n",
      "+----+-------+--------------------+-----------------------------------------------------------+\n",
      "\n",
      "| 1  | SIT   | Sitting on chair   |  The recording starts with the participant sitting in the |\n",
      "\n",
      "| 2  | CHU   | Sit to stand       |  chair in his office area. Then he gets up from the       |\n",
      "\n",
      "| 3  | WAL   | Walking            |  chair, walks to the door and stands outside the office   |\n",
      "\n",
      "| 4  | STD   | Standing           |  door. Once he find the keys, he lock the door and        |\n",
      "\n",
      "| 5  | WAL   | Walking            |  walks and descent stairs till he riches the parking      |\n",
      "\n",
      "| 6  | STN   | Stairs down        |  area. He walks to his car and stands in front of the     |\n",
      "\n",
      "| 7  | WAL   | Walking            |  car, unlocks the lock of the car, opens the door and     |\n",
      "\n",
      "| 8  | STD   | Standing           |  gets in the car. He remains sited for some seconds,      |\n",
      "\n",
      "| 9  | CSI   | Car-step in        |  then he gets out of the car, closes the door and stands  |\n",
      "\n",
      "| 10 | SIT   | Sitting on chair   |  in front of the door to lock the car.                    |\n",
      "\n",
      "| 11 | CSO   | Car-step out       |                                                           |\n",
      "\n",
      "| 12 | STD   | Standing           |                                                           |\n",
      "\n",
      "+----+-------+--------------------+-----------------------------------------------------------+\n",
      "\n",
      "| 4th Scenario of Being Exercise (SBE), Total duration 125 at max (1 trial/participant)     |\n",
      "\n",
      "+----+-------+--------------------+-----------------------------------------------------------+\n",
      "\n",
      "| No.| Label | Activity           | Description                                               |\n",
      "\n",
      "+----+-------+--------------------+-----------------------------------------------------------+\n",
      "\n",
      "| 1  | STD   | Standing           |  The recording starts with the participant standing in    |\n",
      "\n",
      "| 2  | WAL   | Walking            |  front of the car. He starts his exercise by walking,     |\n",
      "\n",
      "| 3  | JOG   | Jogging            |  then starts jogging from some seconds and once again     |\n",
      "\n",
      "| 4  | WAL   | Walking            |  walking. Then he stops for some seconds to get a         |\n",
      "\n",
      "| 5  | STD   | Standing           |  breath and he starts jumping and once more he            |\n",
      "\n",
      "| 6  | JUM   | Jumping            |  standing to relax a little. Finally he walks till his    |\n",
      "\n",
      "| 7  | STD   | Standing           |  car and stands outside the door.                         |\n",
      "\n",
      "| 8  | WAL   | Walking            |                                                           |\n",
      "\n",
      "| 9  | STD   | Standing           |                                                           |\n",
      "\n",
      "+----+-------+--------------------+-----------------------------------------------------------+\n",
      "\n",
      "| 5th Scenario of Returning at Home (SRH), Total duration 155 at max (1 trial/participant)  |\n",
      "\n",
      "+----+-------+--------------------+-----------------------------------------------------------+\n",
      "\n",
      "| No.| Label | Activity           | Description                                               |\n",
      "\n",
      "+----+-------+--------------------+-----------------------------------------------------------+\n",
      "\n",
      "| 1  | STD   | Standing           |  The recording starts with the participant standing       |\n",
      "\n",
      "| 2  | CSI   | Car-step in        |  outside the cars door. He unlocks the lock of the car,   |\n",
      "\n",
      "| 3  | SIT   | Sitting on chair   |  opens the door and gets in the car. He remains sited     |\n",
      "\n",
      "| 4  | CSO   | Car-step out       |  for some seconds, then he gets out of the car, closes    |\n",
      "\n",
      "| 5  | STD   | Standing           |  the door and stands in front of the door to lock the     |\n",
      "\n",
      "| 6  | WAL   | Walking            |  car.  Then walks from the parking area to his home.      |\n",
      "\n",
      "| 7  | STU   | Stairs up          |  He walks and ascent stairs till riches his home door,    |\n",
      "\n",
      "| 8  |  WAL  | Walking            |  where he stands to finds the keys. Then he opens the     |\n",
      "\n",
      "| 9  | STD   | Standing           |  door, gets in his home, walks till a chair and sits.     |\n",
      "\n",
      "| 10 | WAL   | Walking            |                                                           |\n",
      "\n",
      "| 11 | SCH   | Stand to sit       |                                                           |\n",
      "\n",
      "+----+-------+--------------------+-----------------------------------------------------------+\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Sensors:\n",
      "\n",
      "+------+---------------+----------------------------------------------------+--------------------------------------------------------------+\n",
      "\n",
      "| Code | Type          | Values                                             | Description                                                  |\n",
      "\n",
      "+------+---------------+----------------------------------------------------+--------------------------------------------------------------+\n",
      "\n",
      "| acc  | accelerometer | timestamp(ns),x,y,z(m/s^2)                         | Acceleration force along the x y z axes (including gravity). |\n",
      "\n",
      "| gyro | gyroscope     | timestamp(ns),x,y,z(rad/s)                         | Rate of rotation around the x y z axes (Angular velocity).   |\n",
      "\n",
      "| ori  | orientation   | timestamp(ns),Azimuth,Pitch,Roll(degrees)          | Angle around the z x y axes.                                 |\n",
      "\n",
      "+------+---------------+----------------------------------------------------+--------------------------------------------------------------+\n",
      "\n",
      "\n",
      "\n",
      "Related work: \n",
      "\n",
      "  Chatzaki C., Pediaditis M., Vavoulas G., Tsiknakis M. (2017) \n",
      "\n",
      "   Human Daily Activity and Fall Recognition Using a Smartphones Acceleration Sensor. \n",
      "\n",
      "   In: Rocker C., O'Donoghue J., Ziefle M., Helfert M., Molloy W. (eds) \n",
      "\n",
      "   Information and Communication Technologies for Ageing Well and e-Health. ICT4AWE 2016.\n",
      "\n",
      "   Communications in Computer and Information Science, vol 736, pp 100-118.\n",
      "\n",
      "   Springer, Cham, DOI 10.1007/978-3-319-62704-5_7\n",
      "\n",
      " \n",
      "\n",
      "   Chatzaki, C., Pediaditis, M., Vavoulas, G. and Tsiknakis, M., \n",
      "\n",
      "    \"Estimating normal and abnormal activities using smartphones\",\n",
      "\n",
      "    In Proceedings of the 13th International Conference on Wearable Micro and Nano Technologies for Personalised Health (pHealth),\n",
      "\n",
      "\tv.224, pp 195-200,29-31 May 2016, Heraklion, Crete, Greece,  DOI:10.3233/978-1-61499-653-8-195\n",
      "\n",
      "\n",
      "\n",
      "   Chatzaki Charikleia, \n",
      "\n",
      "\t\"Estimating human activity patterns in dynamic environments based on smart, wearable sensors : a feasibility study\",\n",
      "\n",
      "\t M.Sc. Thesis , Dept. Informatics Enginnering,Heraklion, Crete, Greece, 2016\n",
      "\n",
      "\t\n",
      "\n",
      "   Vavoulas, G., Chatzaki, C., Malliotakis, T., Pediaditis, M. and Tsiknakis, M., \n",
      "\n",
      "    \"The MobiAct Dataset: Recognition of Activities of Daily Living using Smartphones\",\n",
      "\n",
      "    In Proceedings of the International Conference on Information and Communication Technologies for Ageing Well and e-Health (ICT4AWE 2016),\n",
      "\n",
      "    vol. 1, pp 143-151,ISBN: 978-989-758-180-9, DOI: 10.5220/0005792401430151\n",
      "\n",
      "\n",
      "\n",
      "   G. Vavoulas, M. Pediaditis, C. Chatzaki, E. G. Spanakis, M. Tsiknakis, \n",
      "\n",
      "    \"The MobiFall Dataset: Fall Detection and Classification with a Smartphone\", \n",
      "\n",
      "    invited publication for the International Journal of Monitoring and Surveillance Technologies Research,\n",
      "\n",
      "    pp 44-56, 2014, DOI:10.4018/ijmstr.2014010103\n",
      "\n",
      "\t\n",
      "\n",
      "   G. Vavoulas, M. Pediaditis, E. Spanakis, M. Tsiknakis,\n",
      "\n",
      "    \"The MobiFall Dataset: An Initial Evaluation of Fall Detection Algorithms Using Smartphones\",\n",
      "\n",
      "\t6th IEEE International Symposium on Monitoring & Surveillance Research (ISMSR): Healthcare-Bioinformatics, \n",
      "\n",
      "\tChania, Greece, 2013, DOI:10.1109/BIBE.2013.6701629\n",
      "\n",
      "\t\n",
      "\n",
      "====================================================================\n",
      "\n"
     ]
    }
   ],
   "source": [
    "file = open('mobiact_dataset/Readme.txt', 'r', encoding='latin1')\n",
    "strings = file.readlines()\n",
    "file.close()\n",
    "for s in strings:\n",
    "    print(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-06-24T02:14:39.436453Z",
     "start_time": "2021-06-24T02:14:39.424523Z"
    }
   },
   "outputs": [],
   "source": [
    "person_list = []\n",
    "for s in strings:\n",
    "    if 'sub' in s and '|' in s:\n",
    "        temp = s.split('|')\n",
    "        temp = [x.strip() for x in temp]\n",
    "        if len(temp) == 9:\n",
    "            person_list.append(temp[3:-1])\n",
    "\n",
    "activity_list = []\n",
    "for s in strings:\n",
    "    if '|' in s:\n",
    "        temp = s.split('|')\n",
    "        temp = [x.strip() for x in temp]\n",
    "        if len(temp) == 8:\n",
    "            activity_list.append(temp[1:-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-06-24T02:14:39.668550Z",
     "start_time": "2021-06-24T02:14:39.649324Z"
    }
   },
   "outputs": [],
   "source": [
    "falls = ['FOL', 'FKL', 'BSC', 'SDL']\n",
    "\n",
    "columns = ['name', 'age', 'height', 'weight', 'gender']\n",
    "person_info = pd.DataFrame(person_list, columns=columns)\n",
    "\n",
    "activity_info = pd.DataFrame(activity_list)\n",
    "activity_info.columns = activity_info.iloc[0]\n",
    "activity_info = activity_info.drop(0)\n",
    "activity_info = activity_info.drop(13)\n",
    "activity_info = activity_info.reset_index(drop=True)\n",
    "# print(activity_info.columns)\n",
    "index = activity_info['No.']\n",
    "activity_info = activity_info.drop(['No.'], axis=1)\n",
    "activity_info.index = index\n",
    "activity_info['label_encoded'] = list(range(len(activity_info)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-06-24T02:14:40.913716Z",
     "start_time": "2021-06-24T02:14:40.885464Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['FKL_20_3_annotated.csv', 'FKL_17_3_annotated.csv', 'SCH_22_6_annotated.csv']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_dir = 'mobiact_dataset/'\n",
    "file_list = os.listdir(data_dir)\n",
    "file_list = [file for file in file_list if file.endswith('.csv')]\n",
    "file_list[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-06-24T02:15:22.084305Z",
     "start_time": "2021-06-24T02:14:41.517533Z"
    }
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv('mobiact_preprocessed/train.csv', index_col=0)\n",
    "valid = pd.read_csv('mobiact_preprocessed/valid.csv', index_col=0)\n",
    "test = pd.read_csv('mobiact_preprocessed/test.csv', index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-06-24T02:15:28.495300Z",
     "start_time": "2021-06-24T02:15:22.869137Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "encoder = LabelEncoder()\n",
    "encoder.fit(activity_info['Label'])\n",
    "\n",
    "train_encoded = encoder.transform(train['label'])\n",
    "train['label_encoded'] = train_encoded\n",
    "\n",
    "valid_encoded = encoder.transform(valid['label'])\n",
    "valid['label_encoded'] = valid_encoded\n",
    "\n",
    "test_encoded = encoder.transform(test['label'])\n",
    "test['label_encoded'] = test_encoded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-06-24T02:15:29.331635Z",
     "start_time": "2021-06-24T02:15:29.295066Z"
    }
   },
   "outputs": [],
   "source": [
    "train['known'] = 0\n",
    "valid['known'] = 0\n",
    "test['known'] = 0\n",
    "\n",
    "train['circum'] = 0\n",
    "valid['circum'] = 0\n",
    "test['circum'] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-06-24T02:16:15.742111Z",
     "start_time": "2021-06-24T02:16:15.715891Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>timestamp</th>\n",
       "      <th>rel_time</th>\n",
       "      <th>acc_x</th>\n",
       "      <th>acc_y</th>\n",
       "      <th>acc_z</th>\n",
       "      <th>gyro_x</th>\n",
       "      <th>gyro_y</th>\n",
       "      <th>gyro_z</th>\n",
       "      <th>azimuth</th>\n",
       "      <th>pitch</th>\n",
       "      <th>...</th>\n",
       "      <th>label</th>\n",
       "      <th>person_id</th>\n",
       "      <th>person_age</th>\n",
       "      <th>person_height</th>\n",
       "      <th>person_weight</th>\n",
       "      <th>person_gender</th>\n",
       "      <th>trial_num</th>\n",
       "      <th>label_encoded</th>\n",
       "      <th>known</th>\n",
       "      <th>circum</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4122885402000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.238091</td>\n",
       "      <td>8.132329</td>\n",
       "      <td>4.245767</td>\n",
       "      <td>0.049480</td>\n",
       "      <td>-0.007636</td>\n",
       "      <td>-0.013744</td>\n",
       "      <td>273.73670</td>\n",
       "      <td>-67.593300</td>\n",
       "      <td>...</td>\n",
       "      <td>SIT</td>\n",
       "      <td>6</td>\n",
       "      <td>22</td>\n",
       "      <td>172</td>\n",
       "      <td>62</td>\n",
       "      <td>F</td>\n",
       "      <td>4</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4122890370000</td>\n",
       "      <td>0.004968</td>\n",
       "      <td>3.256741</td>\n",
       "      <td>8.139323</td>\n",
       "      <td>4.259755</td>\n",
       "      <td>0.045204</td>\n",
       "      <td>-0.008552</td>\n",
       "      <td>-0.003360</td>\n",
       "      <td>274.05545</td>\n",
       "      <td>-67.167404</td>\n",
       "      <td>...</td>\n",
       "      <td>SIT</td>\n",
       "      <td>6</td>\n",
       "      <td>22</td>\n",
       "      <td>172</td>\n",
       "      <td>62</td>\n",
       "      <td>F</td>\n",
       "      <td>4</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4122895406000</td>\n",
       "      <td>0.010004</td>\n",
       "      <td>3.275647</td>\n",
       "      <td>8.146413</td>\n",
       "      <td>4.273935</td>\n",
       "      <td>0.040012</td>\n",
       "      <td>-0.004887</td>\n",
       "      <td>-0.009163</td>\n",
       "      <td>274.36743</td>\n",
       "      <td>-66.759420</td>\n",
       "      <td>...</td>\n",
       "      <td>SIT</td>\n",
       "      <td>6</td>\n",
       "      <td>22</td>\n",
       "      <td>172</td>\n",
       "      <td>62</td>\n",
       "      <td>F</td>\n",
       "      <td>4</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4122900447000</td>\n",
       "      <td>0.015045</td>\n",
       "      <td>3.294929</td>\n",
       "      <td>8.154904</td>\n",
       "      <td>4.283354</td>\n",
       "      <td>0.032681</td>\n",
       "      <td>-0.011301</td>\n",
       "      <td>-0.018631</td>\n",
       "      <td>274.72174</td>\n",
       "      <td>-66.325455</td>\n",
       "      <td>...</td>\n",
       "      <td>SIT</td>\n",
       "      <td>6</td>\n",
       "      <td>22</td>\n",
       "      <td>172</td>\n",
       "      <td>62</td>\n",
       "      <td>F</td>\n",
       "      <td>4</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4122905384000</td>\n",
       "      <td>0.019982</td>\n",
       "      <td>3.314145</td>\n",
       "      <td>8.164513</td>\n",
       "      <td>4.288158</td>\n",
       "      <td>0.029322</td>\n",
       "      <td>-0.002443</td>\n",
       "      <td>-0.014966</td>\n",
       "      <td>275.06073</td>\n",
       "      <td>-65.903100</td>\n",
       "      <td>...</td>\n",
       "      <td>SIT</td>\n",
       "      <td>6</td>\n",
       "      <td>22</td>\n",
       "      <td>172</td>\n",
       "      <td>62</td>\n",
       "      <td>F</td>\n",
       "      <td>4</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       timestamp  rel_time     acc_x     acc_y     acc_z    gyro_x    gyro_y  \\\n",
       "0  4122885402000  0.000000  3.238091  8.132329  4.245767  0.049480 -0.007636   \n",
       "1  4122890370000  0.004968  3.256741  8.139323  4.259755  0.045204 -0.008552   \n",
       "2  4122895406000  0.010004  3.275647  8.146413  4.273935  0.040012 -0.004887   \n",
       "3  4122900447000  0.015045  3.294929  8.154904  4.283354  0.032681 -0.011301   \n",
       "4  4122905384000  0.019982  3.314145  8.164513  4.288158  0.029322 -0.002443   \n",
       "\n",
       "     gyro_z    azimuth      pitch  ...  label person_id  person_age  \\\n",
       "0 -0.013744  273.73670 -67.593300  ...    SIT         6          22   \n",
       "1 -0.003360  274.05545 -67.167404  ...    SIT         6          22   \n",
       "2 -0.009163  274.36743 -66.759420  ...    SIT         6          22   \n",
       "3 -0.018631  274.72174 -66.325455  ...    SIT         6          22   \n",
       "4 -0.014966  275.06073 -65.903100  ...    SIT         6          22   \n",
       "\n",
       "   person_height  person_weight  person_gender trial_num  label_encoded  \\\n",
       "0            172             62              F         4             11   \n",
       "1            172             62              F         4             11   \n",
       "2            172             62              F         4             11   \n",
       "3            172             62              F         4             11   \n",
       "4            172             62              F         4             11   \n",
       "\n",
       "   known  circum  \n",
       "0      0       0  \n",
       "1      0       0  \n",
       "2      0       0  \n",
       "3      0       0  \n",
       "4      0       0  \n",
       "\n",
       "[5 rows x 21 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Formatter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-06-24T02:17:02.143872Z",
     "start_time": "2021-06-24T02:17:02.137583Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Available data types:\n",
      "DataTypes.REAL_VALUED\n",
      "DataTypes.CATEGORICAL\n",
      "DataTypes.DATE\n",
      "\n",
      "Avaialbe input types:\n",
      "InputTypes.TARGET\n",
      "InputTypes.OBSERVED_INPUT\n",
      "InputTypes.KNOWN_INPUT\n",
      "InputTypes.STATIC_INPUT\n",
      "InputTypes.ID\n",
      "InputTypes.TIME\n"
     ]
    }
   ],
   "source": [
    "from data_formatters.base import GenericDataFormatter, DataTypes, InputTypes\n",
    "\n",
    "# View avialable inputs and data types.\n",
    "print(\"Available data types:\")\n",
    "for option in DataTypes:\n",
    "    print(option)\n",
    "\n",
    "print()\n",
    "print(\"Avaialbe input types:\")\n",
    "for option in InputTypes:\n",
    "    print(option)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-06-24T02:17:50.191780Z",
     "start_time": "2021-06-24T02:17:48.561319Z"
    }
   },
   "outputs": [],
   "source": [
    "from libs import utils \n",
    "import sklearn.preprocessing\n",
    "class MobiActFormatter(GenericDataFormatter):\n",
    "    _column_definition = [\n",
    "        ('person_id', DataTypes.CATEGORICAL, InputTypes.ID),\n",
    "        ('rel_time', DataTypes.REAL_VALUED, InputTypes.TIME),\n",
    "        ('known', DataTypes.CATEGORICAL, InputTypes.KNOWN_INPUT),\n",
    "        ('circum', DataTypes.CATEGORICAL, InputTypes.STATIC_INPUT),\n",
    "        ('acc_x', DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),\n",
    "        ('acc_y', DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),\n",
    "        ('acc_z', DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),\n",
    "        ('gyro_x', DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),\n",
    "        ('gyro_y', DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),\n",
    "        ('gyro_z', DataTypes.REAL_VALUED, InputTypes.OBSERVED_INPUT),\n",
    "        ('label_encoded', DataTypes.REAL_VALUED, InputTypes.TARGET)\n",
    "    ]\n",
    "    \n",
    "    def __init__(self):\n",
    "        \"\"\"initializes formatter\"\"\"\n",
    "        \n",
    "        self.identifiers = None\n",
    "        self._real_scalers = None\n",
    "        self._cat_scalers = None\n",
    "        self._target_scaler = None\n",
    "        self._num_classes_per_cat_input = None\n",
    "    \n",
    "    def split_data(self, train, valid, test):\n",
    "        \"\"\"\n",
    "        split data frame into training-validation-test data frames\n",
    "        \n",
    "        \"\"\"\n",
    "        print('Formatting train-valid-test splits.')\n",
    "        \n",
    "        train_data = train\n",
    "\n",
    "        val_data = valid\n",
    "        \n",
    "        test_data = test\n",
    "        \n",
    "        self.set_scalers(train_data)\n",
    "        \n",
    "        return (self.transform_inputs(data) for data in [train_data, val_data, test_data])\n",
    "    \n",
    "    def set_scalers(self, df):\n",
    "        \"\"\"\n",
    "        Calibrates scalers using the data supplied.\n",
    "        (제공된 데이터를 사용해서 scaler 교정)\n",
    "\n",
    "        Args:\n",
    "          df: Data to use to calibrate scalers.\n",
    "        \"\"\"\n",
    "        print('Setting scalers with training data...')\n",
    "        \n",
    "        column_definitions = self.get_column_definition()\n",
    "\n",
    "        id_column = utils.get_single_col_by_input_type(InputTypes.ID,\n",
    "                                                       column_definitions)\n",
    "        target_column = utils.get_single_col_by_input_type(InputTypes.TARGET,\n",
    "                                                           column_definitions)\n",
    "        # extract identifiers in case required\n",
    "        self.identifiers = list(df[id_column].unique())\n",
    "        \n",
    "        # Format real scalers\n",
    "        real_inputs = utils.extract_cols_from_data_type(\n",
    "            DataTypes.REAL_VALUED, column_definitions,\n",
    "            {InputTypes.ID, InputTypes.TIME})\n",
    "        \n",
    "        data = df[real_inputs].values\n",
    "        self._real_scalers = sklearn.preprocessing.StandardScaler().fit(data)\n",
    "        \n",
    "        # target데이터를 prediction에 사용\n",
    "        self._target_scaler = sklearn.preprocessing.StandardScaler().fit(df[[target_column]].values)\n",
    "        \n",
    "        categorical_inputs = utils.extract_cols_from_data_type(\n",
    "            DataTypes.CATEGORICAL, column_definitions,\n",
    "            {InputTypes.ID, InputTypes.TIME})\n",
    "        \n",
    "        categorical_scalers = {}\n",
    "        num_classes = []\n",
    "        for col in categorical_inputs:\n",
    "            srs = df[col].apply(str)\n",
    "            categorical_scalers[col] = sklearn.preprocessing.LabelEncoder().fit(srs.values)\n",
    "            num_classes.append(srs.nunique())\n",
    "        \n",
    "        # set categorical scaler outputs\n",
    "        self._cat_scalers = categorical_scalers\n",
    "        self._num_classes_per_cat_input = num_classes\n",
    "        \n",
    "    def transform_inputs(self, df):\n",
    "        \"\"\"\n",
    "        performs feature transformations.\n",
    "        \n",
    "        feature engineering, preprocessing and normalization을 포함\n",
    "        \n",
    "        Args:\n",
    "            df - data frame to transform.\n",
    "            \n",
    "        Returns:\n",
    "            Transformed data frame.\n",
    "        \"\"\"\n",
    "        output = df.copy()\n",
    "        \n",
    "        if self._real_scalers is None and self._cat_scalers is None:\n",
    "            raise ValueError('Scalers have not been set!')\n",
    "        \n",
    "        column_definitions = self.get_column_definition()\n",
    "        real_inputs = utils.extract_cols_from_data_type(\n",
    "            DataTypes.REAL_VALUED, column_definitions,\n",
    "            {InputTypes.ID, InputTypes.TIME}\n",
    "        )\n",
    "        categorical_inputs = utils.extract_cols_from_data_type(\n",
    "            DataTypes.CATEGORICAL, column_definitions,\n",
    "            {InputTypes.ID, InputTypes.TIME})\n",
    "        output[real_inputs] = self._real_scalers.transform(df[real_inputs].values)\n",
    "        for col in categorical_inputs:\n",
    "            string_df = df[col].apply(str)\n",
    "            output[col] = self._cat_scalers[col].transform(string_df)\n",
    "        return output\n",
    "    \n",
    "    def format_predictions(self, predictions):\n",
    "        output = predictions.copy()\n",
    "        \n",
    "        column_names = predictions.columns\n",
    "        for col in column_names:\n",
    "            if col not in {'forecast_time', 'identifier'}:\n",
    "                output[col] = self._target_scaler.inverse_transform(predictions[col])\n",
    "                \n",
    "        return output\n",
    "    \n",
    "    def get_fixed_params(self):\n",
    "        fixed_params = {\n",
    "            'total_time_steps': 87,     # Total width of the Temporal Fusion Decoder\n",
    "            'num_encoder_steps': 43,    # Length of LSTM decoder (ie. # historical inputs)\n",
    "            'num_epochs': 100,            # Max number of epochs for training\n",
    "            'early_stopping_patience': 5, # Early stopping threshold for # iterations with no loss improvement\n",
    "            'multiprocessing_workers': 5  # Number of multi-processing workers\n",
    "        }\n",
    "        \n",
    "        return fixed_params\n",
    "    \n",
    "    def get_default_model_params(self):\n",
    "        model_params = {\n",
    "            'dropout_rate': 0.3,\n",
    "            'hidden_layer_size': 160,\n",
    "            'learning_rate': 0.01,\n",
    "            'minibatch_size': 64,\n",
    "            'max_gradient_norm': 0.01,\n",
    "            'num_heads': 4,\n",
    "            'stack_size': 1\n",
    "        }\n",
    "\n",
    "        return model_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-06-24T02:19:46.425654Z",
     "start_time": "2021-06-24T02:18:36.566713Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Formatting train-valid-test splits.\n",
      "Setting scalers with training data...\n"
     ]
    }
   ],
   "source": [
    "data_formatter = MobiActFormatter()\n",
    "train, valid, test = data_formatter.split_data(train, valid, test)\n",
    "train_samples, valid_samples = data_formatter.get_num_samples_for_calibration(\n",
    "  )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-06-24T02:20:24.707054Z",
     "start_time": "2021-06-24T02:20:24.691528Z"
    }
   },
   "outputs": [],
   "source": [
    "import libs.hyperparam_opt\n",
    "import libs.tft_model\n",
    "import libs.utils as util\n",
    "\n",
    "ModelClass = libs.tft_model.TemporalFusionTransformer\n",
    "data_params = data_formatter.get_experiment_params()\n",
    "param_ranges = ModelClass.get_hyperparm_choices()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-06-24T02:21:02.982320Z",
     "start_time": "2021-06-24T02:21:02.978615Z"
    }
   },
   "outputs": [],
   "source": [
    "model_params = {'dropout_rate': 0.3,      # Dropout discard rate\n",
    "                'hidden_layer_size': 320, # Internal state size of TFT\n",
    "                'learning_rate': 0.001,   # ADAM initial learning rate\n",
    "                'minibatch_size': 512,    # Minibatch size for training\n",
    "                'max_gradient_norm': 100.,# Max norm for gradient clipping\n",
    "                'num_heads': 4,           # Number of heads for multi-head attention\n",
    "                'stack_size': 1           # Number of stacks (default 1 for interpretability)\n",
    "               }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-06-24T02:26:09.508659Z",
     "start_time": "2021-06-24T02:26:09.503147Z"
    }
   },
   "outputs": [],
   "source": [
    "model_folder = os.path.join(output_folder, 'saved_models', 'MobiAct_wo_known_no_bioinfo2', 'fixed')\n",
    "model_params['model_folder'] = model_folder\n",
    "\n",
    "model_params.update(data_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-06-24T02:22:19.541903Z",
     "start_time": "2021-06-24T02:22:19.537269Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** Loading hyperparm manager ***\n"
     ]
    }
   ],
   "source": [
    "HyperparamOptManager = libs.hyperparam_opt.HyperparamOptManager\n",
    "print(\"*** Loading hyperparm manager ***\")\n",
    "opt_manager = HyperparamOptManager(param_ranges, data_params, model_folder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-06-21T07:37:12.063513Z",
     "start_time": "2021-06-21T07:37:12.046194Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading results from /workspace/inkyung/TFT/repo/tft/outputs_mobiact_wo_known_no_bioinfo2/saved_models/MobiAct_wo_known_no_bioinfo/fixed\n",
      "Loaded results from previous training\n"
     ]
    }
   ],
   "source": [
    "success = opt_manager.load_results()\n",
    "restart_opt = None\n",
    "if success and not restart_opt:\n",
    "    print(\"Loaded results from previous training\")\n",
    "else:\n",
    "    print(\"Creating new hyperparameter optimisation\")\n",
    "    opt_manager.clear()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-06-28T05:19:43.923628Z",
     "start_time": "2021-06-28T05:19:43.917398Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Selecting GPU ID=3\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from libs.tft_model import TemporalFusionTransformer\n",
    "\n",
    "# Specify GPU usage\n",
    "tf_config = utils.get_default_tensorflow_config(tf_device=\"gpu\", gpu_id='3')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-06-15T19:08:02.262702Z",
     "start_time": "2021-06-14T16:18:53.720201Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Device mapping:\n",
      "/job:localhost/replica:0/task:0/device:XLA_CPU:0 -> device: XLA_CPU device\n",
      "/job:localhost/replica:0/task:0/device:XLA_GPU:0 -> device: XLA_GPU device\n",
      "/job:localhost/replica:0/task:0/device:GPU:0 -> device: 0, name: Tesla V100-DGXS-32GB, pci bus id: 0000:0f:00.0, compute capability: 7.0\n",
      "\n",
      "Resetting temp folder...\n",
      "*** TemporalFusionTransformer params ***\n",
      "# dropout_rate = 0.3\n",
      "# hidden_layer_size = 320\n",
      "# learning_rate = 0.001\n",
      "# minibatch_size = 512\n",
      "# max_gradient_norm = 100.0\n",
      "# num_heads = 4\n",
      "# stack_size = 1\n",
      "# model_folder = /workspace/inkyung/TFT/repo/tft/outputs_mobiact_wo_known_no_bioinfo2/saved_models/MobiAct_wo_known_no_bioinfo/fixed\n",
      "# total_time_steps = 87\n",
      "# num_encoder_steps = 43\n",
      "# num_epochs = 100\n",
      "# early_stopping_patience = 5\n",
      "# multiprocessing_workers = 5\n",
      "# column_definition = [('person_id', <DataTypes.REAL_VALUED: 0>, <InputTypes.ID: 4>), ('rel_time', <DataTypes.DATE: 2>, <InputTypes.TIME: 5>), ('known', <DataTypes.REAL_VALUED: 0>, <InputTypes.KNOWN_INPUT: 2>), ('acc_x', <DataTypes.REAL_VALUED: 0>, <InputTypes.OBSERVED_INPUT: 1>), ('acc_y', <DataTypes.REAL_VALUED: 0>, <InputTypes.OBSERVED_INPUT: 1>), ('acc_z', <DataTypes.REAL_VALUED: 0>, <InputTypes.OBSERVED_INPUT: 1>), ('gyro_x', <DataTypes.REAL_VALUED: 0>, <InputTypes.OBSERVED_INPUT: 1>), ('gyro_y', <DataTypes.REAL_VALUED: 0>, <InputTypes.OBSERVED_INPUT: 1>), ('gyro_z', <DataTypes.REAL_VALUED: 0>, <InputTypes.OBSERVED_INPUT: 1>), ('label_encoded', <DataTypes.REAL_VALUED: 0>, <InputTypes.TARGET: 0>), ('circum', <DataTypes.CATEGORICAL: 1>, <InputTypes.STATIC_INPUT: 3>)]\n",
      "# input_size = 9\n",
      "# output_size = 1\n",
      "# category_counts = [1]\n",
      "# input_obs_loc = [7]\n",
      "# static_input_loc = [8]\n",
      "# known_regular_inputs = [0]\n",
      "# known_categorical_inputs = [0]\n",
      "Tensor(\"TemporalFusionTransformer/stack:0\", shape=(None, 1, 320), dtype=float32)\n",
      "WARNING:tensorflow:Layer lstm will not use cuDNN kernel since it doesn't meet the cuDNN kernel criteria. It will use generic GPU kernel as fallback when running on GPU\n",
      "WARNING:tensorflow:Layer lstm_1 will not use cuDNN kernel since it doesn't meet the cuDNN kernel criteria. It will use generic GPU kernel as fallback when running on GPU\n",
      "Model: \"functional_1\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            [(None, 87, 9)]      0                                            \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 87, 1)]      0           input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 87)]         0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "sequential (Sequential)         (None, 87, 320)      320         tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 320)]        0           sequential[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 1, 320)]     0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "flatten (Flatten)               (None, 320)          0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "dense_9 (Dense)                 (None, 320)          102720      flatten[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 1, 320)]     0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "activation (Activation)         (None, 320)          0           dense_9[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_13 (Dense)                (None, 1, 320)       102720      tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "dense_10 (Dense)                (None, 320)          102720      activation[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_2 (Activation)       (None, 1, 320)       0           dense_13[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dropout (Dropout)               (None, 320)          0           dense_10[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_14 (Dense)                (None, 1, 320)       102720      activation_2[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_11 (Dense)                (None, 1)            321         dropout[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_12 (Dense)                (None, 1)            321         dropout[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, 1, 320)       0           dense_14[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_8 (Dense)                 (None, 1)            321         flatten[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "multiply (Multiply)             (None, 1)            0           dense_11[0][0]                   \n",
      "                                                                 dense_12[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_15 (Dense)                (None, 1, 320)       102720      dropout_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_16 (Dense)                (None, 1, 320)       102720      dropout_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add (Add)                       (None, 1)            0           dense_8[0][0]                    \n",
      "                                                                 multiply[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "multiply_1 (Multiply)           (None, 1, 320)       0           dense_15[0][0]                   \n",
      "                                                                 dense_16[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization (LayerNorma (None, 1)            2           add[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "add_1 (Add)                     (None, 1, 320)       0           tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 multiply_1[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_1 (Activation)       (None, 1)            0           layer_normalization[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_1 (LayerNor (None, 1, 320)       640         add_1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 1, 1)]       0           activation_1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 1, 320)]     0           layer_normalization_1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "multiply_2 (Multiply)           (None, 1, 320)       0           tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 320)]        0           multiply_2[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_17 (Dense)                (None, 320)          102720      tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "activation_3 (Activation)       (None, 320)          0           dense_17[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 87, 8)]      0           input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_18 (Dense)                (None, 320)          102720      activation_3[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 87, 1)]      0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 87, 1)]      0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 87, 1)]      0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 87, 1)]      0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 87, 1)]      0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 87, 1)]      0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 87, 1)]      0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 87, 1)]      0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)             (None, 320)          0           dense_18[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_1 (TimeDistrib (None, 87, 320)      640         tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_2 (TimeDistrib (None, 87, 320)      640         tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_3 (TimeDistrib (None, 87, 320)      640         tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_4 (TimeDistrib (None, 87, 320)      640         tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_5 (TimeDistrib (None, 87, 320)      640         tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_6 (TimeDistrib (None, 87, 320)      640         tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_7 (TimeDistrib (None, 87, 320)      640         tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "time_distributed (TimeDistribut (None, 87, 320)      640         tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "dense_19 (Dense)                (None, 320)          102720      dropout_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_20 (Dense)                (None, 320)          102720      dropout_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 87, 320, 6)] 0           time_distributed_1[0][0]         \n",
      "                                                                 time_distributed_2[0][0]         \n",
      "                                                                 time_distributed_3[0][0]         \n",
      "                                                                 time_distributed_4[0][0]         \n",
      "                                                                 time_distributed_5[0][0]         \n",
      "                                                                 time_distributed_6[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 87, 320, 1)] 0           time_distributed_7[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 87, 320, 1)] 0           time_distributed[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "multiply_3 (Multiply)           (None, 320)          0           dense_19[0][0]                   \n",
      "                                                                 dense_20[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 43, 320, 6)] 0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 43, 320, 1)] 0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 43, 320, 1)] 0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "add_2 (Add)                     (None, 320)          0           tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 multiply_3[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 43, 320, 8)] 0           tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_2 (LayerNor (None, 320)          640         add_2[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 43, 2560)]   0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 1, 320)]     0           layer_normalization_2[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 44, 320, 1)] 0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_9 (TimeDistrib (None, 43, 320)      819520      tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_10 (TimeDistri (None, 1, 320)       102400      tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 44, 320)]    0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 1, 320)]     0           layer_normalization_2[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 43, 320)]    0           time_distributed_9[0][0]         \n",
      "                                                                 time_distributed_10[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 43, 320)]    0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 43, 320)]    0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 43, 320)]    0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 43, 320)]    0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 43, 320)]    0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 43, 320)]    0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 43, 320)]    0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 43, 320)]    0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_47 (TimeDistri (None, 44, 320)      102720      tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_48 (TimeDistri (None, 1, 320)       102400      tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "activation_7 (Activation)       (None, 43, 320)      0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_14 (TimeDistri (None, 43, 320)      102720      tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_18 (TimeDistri (None, 43, 320)      102720      tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_22 (TimeDistri (None, 43, 320)      102720      tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_26 (TimeDistri (None, 43, 320)      102720      tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_30 (TimeDistri (None, 43, 320)      102720      tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_34 (TimeDistri (None, 43, 320)      102720      tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_38 (TimeDistri (None, 43, 320)      102720      tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_42 (TimeDistri (None, 43, 320)      102720      tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 44, 320)]    0           time_distributed_47[0][0]        \n",
      "                                                                 time_distributed_48[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 44, 320)]    0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_11 (TimeDistri (None, 43, 320)      102720      activation_7[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "activation_9 (Activation)       (None, 43, 320)      0           time_distributed_14[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "activation_10 (Activation)      (None, 43, 320)      0           time_distributed_18[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "activation_11 (Activation)      (None, 43, 320)      0           time_distributed_22[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "activation_12 (Activation)      (None, 43, 320)      0           time_distributed_26[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "activation_13 (Activation)      (None, 43, 320)      0           time_distributed_30[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "activation_14 (Activation)      (None, 43, 320)      0           time_distributed_34[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "activation_15 (Activation)      (None, 43, 320)      0           time_distributed_38[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "activation_16 (Activation)      (None, 43, 320)      0           time_distributed_42[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "activation_17 (Activation)      (None, 44, 320)      0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_52 (TimeDistri (None, 44, 320)      102720      tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "dropout_6 (Dropout)             (None, 43, 320)      0           time_distributed_11[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_15 (TimeDistri (None, 43, 320)      102720      activation_9[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_19 (TimeDistri (None, 43, 320)      102720      activation_10[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_23 (TimeDistri (None, 43, 320)      102720      activation_11[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_27 (TimeDistri (None, 43, 320)      102720      activation_12[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_31 (TimeDistri (None, 43, 320)      102720      activation_13[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_35 (TimeDistri (None, 43, 320)      102720      activation_14[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_39 (TimeDistri (None, 43, 320)      102720      activation_15[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_43 (TimeDistri (None, 43, 320)      102720      activation_16[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_49 (TimeDistri (None, 44, 320)      102720      activation_17[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_19 (Activation)      (None, 44, 320)      0           time_distributed_52[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_12 (TimeDistri (None, 43, 8)        2568        dropout_6[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_13 (TimeDistri (None, 43, 8)        2568        dropout_6[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_7 (Dropout)             (None, 43, 320)      0           time_distributed_15[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dropout_8 (Dropout)             (None, 43, 320)      0           time_distributed_19[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dropout_9 (Dropout)             (None, 43, 320)      0           time_distributed_23[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dropout_10 (Dropout)            (None, 43, 320)      0           time_distributed_27[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dropout_11 (Dropout)            (None, 43, 320)      0           time_distributed_31[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dropout_12 (Dropout)            (None, 43, 320)      0           time_distributed_35[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dropout_13 (Dropout)            (None, 43, 320)      0           time_distributed_39[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dropout_14 (Dropout)            (None, 43, 320)      0           time_distributed_43[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dense_25 (Dense)                (None, 320)          102720      tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "dense_29 (Dense)                (None, 320)          102720      tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "dropout_15 (Dropout)            (None, 44, 320)      0           time_distributed_49[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_53 (TimeDistri (None, 44, 320)      102720      activation_19[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_8 (TimeDistrib (None, 43, 8)        20488       tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "multiply_7 (Multiply)           (None, 43, 8)        0           time_distributed_12[0][0]        \n",
      "                                                                 time_distributed_13[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_16 (TimeDistri (None, 43, 320)      102720      dropout_7[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_17 (TimeDistri (None, 43, 320)      102720      dropout_7[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_20 (TimeDistri (None, 43, 320)      102720      dropout_8[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_21 (TimeDistri (None, 43, 320)      102720      dropout_8[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_24 (TimeDistri (None, 43, 320)      102720      dropout_9[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_25 (TimeDistri (None, 43, 320)      102720      dropout_9[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_28 (TimeDistri (None, 43, 320)      102720      dropout_10[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_29 (TimeDistri (None, 43, 320)      102720      dropout_10[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_32 (TimeDistri (None, 43, 320)      102720      dropout_11[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_33 (TimeDistri (None, 43, 320)      102720      dropout_11[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_36 (TimeDistri (None, 43, 320)      102720      dropout_12[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_37 (TimeDistri (None, 43, 320)      102720      dropout_12[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_40 (TimeDistri (None, 43, 320)      102720      dropout_13[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_41 (TimeDistri (None, 43, 320)      102720      dropout_13[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_44 (TimeDistri (None, 43, 320)      102720      dropout_14[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_45 (TimeDistri (None, 43, 320)      102720      dropout_14[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_5 (Activation)       (None, 320)          0           dense_25[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_6 (Activation)       (None, 320)          0           dense_29[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_50 (TimeDistri (None, 44, 1)        321         dropout_15[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_51 (TimeDistri (None, 44, 1)        321         dropout_15[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dropout_16 (Dropout)            (None, 44, 320)      0           time_distributed_53[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "add_6 (Add)                     (None, 43, 8)        0           time_distributed_8[0][0]         \n",
      "                                                                 multiply_7[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "multiply_8 (Multiply)           (None, 43, 320)      0           time_distributed_16[0][0]        \n",
      "                                                                 time_distributed_17[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "multiply_9 (Multiply)           (None, 43, 320)      0           time_distributed_20[0][0]        \n",
      "                                                                 time_distributed_21[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "multiply_10 (Multiply)          (None, 43, 320)      0           time_distributed_24[0][0]        \n",
      "                                                                 time_distributed_25[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "multiply_11 (Multiply)          (None, 43, 320)      0           time_distributed_28[0][0]        \n",
      "                                                                 time_distributed_29[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "multiply_12 (Multiply)          (None, 43, 320)      0           time_distributed_32[0][0]        \n",
      "                                                                 time_distributed_33[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "multiply_13 (Multiply)          (None, 43, 320)      0           time_distributed_36[0][0]        \n",
      "                                                                 time_distributed_37[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "multiply_14 (Multiply)          (None, 43, 320)      0           time_distributed_40[0][0]        \n",
      "                                                                 time_distributed_41[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "multiply_15 (Multiply)          (None, 43, 320)      0           time_distributed_44[0][0]        \n",
      "                                                                 time_distributed_45[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dense_26 (Dense)                (None, 320)          102720      activation_5[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_30 (Dense)                (None, 320)          102720      activation_6[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_46 (TimeDistri (None, 44, 1)        321         tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "multiply_17 (Multiply)          (None, 44, 1)        0           time_distributed_50[0][0]        \n",
      "                                                                 time_distributed_51[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_54 (TimeDistri (None, 44, 320)      102720      dropout_16[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_55 (TimeDistri (None, 44, 320)      102720      dropout_16[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_6 (LayerNor (None, 43, 8)        16          add_6[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "add_7 (Add)                     (None, 43, 320)      0           tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 multiply_8[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "add_8 (Add)                     (None, 43, 320)      0           tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 multiply_9[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "add_9 (Add)                     (None, 43, 320)      0           tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 multiply_10[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "add_10 (Add)                    (None, 43, 320)      0           tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 multiply_11[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "add_11 (Add)                    (None, 43, 320)      0           tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 multiply_12[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "add_12 (Add)                    (None, 43, 320)      0           tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 multiply_13[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "add_13 (Add)                    (None, 43, 320)      0           tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 multiply_14[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "add_14 (Add)                    (None, 43, 320)      0           tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 multiply_15[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dropout_4 (Dropout)             (None, 320)          0           dense_26[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dropout_5 (Dropout)             (None, 320)          0           dense_30[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "add_15 (Add)                    (None, 44, 1)        0           time_distributed_46[0][0]        \n",
      "                                                                 multiply_17[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "multiply_18 (Multiply)          (None, 44, 320)      0           time_distributed_54[0][0]        \n",
      "                                                                 time_distributed_55[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "activation_8 (Activation)       (None, 43, 8)        0           layer_normalization_6[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_7 (LayerNor (None, 43, 320)      640         add_7[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_8 (LayerNor (None, 43, 320)      640         add_8[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_9 (LayerNor (None, 43, 320)      640         add_9[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_10 (LayerNo (None, 43, 320)      640         add_10[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_11 (LayerNo (None, 43, 320)      640         add_11[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_12 (LayerNo (None, 43, 320)      640         add_12[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_13 (LayerNo (None, 43, 320)      640         add_13[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_14 (LayerNo (None, 43, 320)      640         add_14[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "dense_27 (Dense)                (None, 320)          102720      dropout_4[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_28 (Dense)                (None, 320)          102720      dropout_4[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_31 (Dense)                (None, 320)          102720      dropout_5[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_32 (Dense)                (None, 320)          102720      dropout_5[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_15 (LayerNo (None, 44, 1)        2           add_15[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "add_16 (Add)                    (None, 44, 320)      0           tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 multiply_18[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 43, 1, 8)]   0           activation_8[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 43, 320, 8)] 0           layer_normalization_7[0][0]      \n",
      "                                                                 layer_normalization_8[0][0]      \n",
      "                                                                 layer_normalization_9[0][0]      \n",
      "                                                                 layer_normalization_10[0][0]     \n",
      "                                                                 layer_normalization_11[0][0]     \n",
      "                                                                 layer_normalization_12[0][0]     \n",
      "                                                                 layer_normalization_13[0][0]     \n",
      "                                                                 layer_normalization_14[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "multiply_5 (Multiply)           (None, 320)          0           dense_27[0][0]                   \n",
      "                                                                 dense_28[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "multiply_6 (Multiply)           (None, 320)          0           dense_31[0][0]                   \n",
      "                                                                 dense_32[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_18 (Activation)      (None, 44, 1)        0           layer_normalization_15[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_16 (LayerNo (None, 44, 320)      640         add_16[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "multiply_16 (Multiply)          (None, 43, 320, 8)   0           tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "add_4 (Add)                     (None, 320)          0           tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 multiply_5[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "add_5 (Add)                     (None, 320)          0           tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 multiply_6[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 44, 1, 1)]   0           activation_18[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 44, 320, 1)] 0           layer_normalization_16[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 43, 320)]    0           multiply_16[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_4 (LayerNor (None, 320)          640         add_4[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_5 (LayerNor (None, 320)          640         add_5[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "multiply_19 (Multiply)          (None, 44, 320, 1)   0           tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "dense_21 (Dense)                (None, 320)          102720      tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "lstm (LSTM)                     [(None, 43, 320), (N 820480      tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 layer_normalization_4[0][0]      \n",
      "                                                                 layer_normalization_5[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 44, 320)]    0           multiply_19[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "activation_4 (Activation)       (None, 320)          0           dense_21[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lstm_1 (LSTM)                   (None, 44, 320)      820480      tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 lstm[0][1]                       \n",
      "                                                                 lstm[0][2]                       \n",
      "__________________________________________________________________________________________________\n",
      "dense_22 (Dense)                (None, 320)          102720      activation_4[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 87, 320)]    0           lstm[0][0]                       \n",
      "                                                                 lstm_1[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "dropout_3 (Dropout)             (None, 320)          0           dense_22[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dropout_17 (Dropout)            (None, 87, 320)      0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "dense_23 (Dense)                (None, 320)          102720      dropout_3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_24 (Dense)                (None, 320)          102720      dropout_3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_56 (TimeDistri (None, 87, 320)      102720      dropout_17[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_57 (TimeDistri (None, 87, 320)      102720      dropout_17[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "multiply_4 (Multiply)           (None, 320)          0           dense_23[0][0]                   \n",
      "                                                                 dense_24[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "multiply_20 (Multiply)          (None, 87, 320)      0           time_distributed_56[0][0]        \n",
      "                                                                 time_distributed_57[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 87, 320)]    0           tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "add_3 (Add)                     (None, 320)          0           tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 multiply_4[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "add_17 (Add)                    (None, 87, 320)      0           multiply_20[0][0]                \n",
      "                                                                 tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_3 (LayerNor (None, 320)          640         add_3[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_17 (LayerNo (None, 87, 320)      640         add_17[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 1, 320)]     0           layer_normalization_3[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_58 (TimeDistri (None, 87, 320)      102720      layer_normalization_17[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_59 (TimeDistri (None, 1, 320)       102400      tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 87, 320)]    0           time_distributed_58[0][0]        \n",
      "                                                                 time_distributed_59[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "activation_20 (Activation)      (None, 87, 320)      0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_60 (TimeDistri (None, 87, 320)      102720      activation_20[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_18 (Dropout)            (None, 87, 320)      0           time_distributed_60[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_61 (TimeDistri (None, 87, 320)      102720      dropout_18[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_62 (TimeDistri (None, 87, 320)      102720      dropout_18[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "multiply_21 (Multiply)          (None, 87, 320)      0           time_distributed_61[0][0]        \n",
      "                                                                 time_distributed_62[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "add_18 (Add)                    (None, 87, 320)      0           layer_normalization_17[0][0]     \n",
      "                                                                 multiply_21[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_18 (LayerNo (None, 87, 320)      640         add_18[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(3,)]               0           layer_normalization_18[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [()]                 0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(3,)]               0           layer_normalization_18[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [()]                 0           tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(1,)]               0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(1,)]               0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(2,)]               0           tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, None)]       0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, None, None)] 0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "dense_89 (Dense)                (None, 87, 80)       25600       layer_normalization_18[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "dense_90 (Dense)                (None, 87, 80)       25600       layer_normalization_18[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, None, None)] 0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "dense_91 (Dense)                (None, 87, 80)       25600       layer_normalization_18[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "dense_92 (Dense)                (None, 87, 80)       25600       layer_normalization_18[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "dense_93 (Dense)                (None, 87, 80)       25600       layer_normalization_18[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "dense_94 (Dense)                (None, 87, 80)       25600       layer_normalization_18[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "dense_95 (Dense)                (None, 87, 80)       25600       layer_normalization_18[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "dense_96 (Dense)                (None, 87, 80)       25600       layer_normalization_18[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "lambda (Lambda)                 (None, 87, 87)       0           dense_89[0][0]                   \n",
      "                                                                 dense_90[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_1 (Lambda)               (None, None, None)   0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "lambda_3 (Lambda)               (None, 87, 87)       0           dense_91[0][0]                   \n",
      "                                                                 dense_92[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_4 (Lambda)               (None, None, None)   0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "lambda_6 (Lambda)               (None, 87, 87)       0           dense_93[0][0]                   \n",
      "                                                                 dense_94[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_7 (Lambda)               (None, None, None)   0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "lambda_9 (Lambda)               (None, 87, 87)       0           dense_95[0][0]                   \n",
      "                                                                 dense_96[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_10 (Lambda)              (None, None, None)   0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "add_19 (Add)                    (None, 87, 87)       0           lambda[0][0]                     \n",
      "                                                                 lambda_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "add_20 (Add)                    (None, 87, 87)       0           lambda_3[0][0]                   \n",
      "                                                                 lambda_4[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "add_21 (Add)                    (None, 87, 87)       0           lambda_6[0][0]                   \n",
      "                                                                 lambda_7[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "add_22 (Add)                    (None, 87, 87)       0           lambda_9[0][0]                   \n",
      "                                                                 lambda_10[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_21 (Activation)      (None, 87, 87)       0           add_19[0][0]                     \n",
      "                                                                 add_20[0][0]                     \n",
      "                                                                 add_21[0][0]                     \n",
      "                                                                 add_22[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "dropout_19 (Dropout)            (None, 87, 87)       0           activation_21[0][0]              \n",
      "                                                                 activation_21[1][0]              \n",
      "                                                                 activation_21[2][0]              \n",
      "                                                                 activation_21[3][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_88 (Dense)                (None, 87, 80)       25600       layer_normalization_18[0][0]     \n",
      "                                                                 layer_normalization_18[0][0]     \n",
      "                                                                 layer_normalization_18[0][0]     \n",
      "                                                                 layer_normalization_18[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "lambda_2 (Lambda)               (None, 87, 80)       0           dropout_19[0][0]                 \n",
      "                                                                 dense_88[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_5 (Lambda)               (None, 87, 80)       0           dropout_19[1][0]                 \n",
      "                                                                 dense_88[1][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_8 (Lambda)               (None, 87, 80)       0           dropout_19[2][0]                 \n",
      "                                                                 dense_88[2][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_11 (Lambda)              (None, 87, 80)       0           dropout_19[3][0]                 \n",
      "                                                                 dense_88[3][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dropout_20 (Dropout)            (None, 87, 80)       0           lambda_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dropout_21 (Dropout)            (None, 87, 80)       0           lambda_5[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dropout_22 (Dropout)            (None, 87, 80)       0           lambda_8[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dropout_23 (Dropout)            (None, 87, 80)       0           lambda_11[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(4, None, 87, 80)]  0           dropout_20[0][0]                 \n",
      "                                                                 dropout_21[0][0]                 \n",
      "                                                                 dropout_22[0][0]                 \n",
      "                                                                 dropout_23[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 87, 80)]     0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "dense_97 (Dense)                (None, 87, 320)      25600       tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "dropout_24 (Dropout)            (None, 87, 320)      0           dense_97[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dropout_25 (Dropout)            (None, 87, 320)      0           dropout_24[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_63 (TimeDistri (None, 87, 320)      102720      dropout_25[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_64 (TimeDistri (None, 87, 320)      102720      dropout_25[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "multiply_22 (Multiply)          (None, 87, 320)      0           time_distributed_63[0][0]        \n",
      "                                                                 time_distributed_64[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "add_23 (Add)                    (None, 87, 320)      0           multiply_22[0][0]                \n",
      "                                                                 layer_normalization_18[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_19 (LayerNo (None, 87, 320)      640         add_23[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_65 (TimeDistri (None, 87, 320)      102720      layer_normalization_19[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_22 (Activation)      (None, 87, 320)      0           time_distributed_65[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_66 (TimeDistri (None, 87, 320)      102720      activation_22[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_26 (Dropout)            (None, 87, 320)      0           time_distributed_66[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_67 (TimeDistri (None, 87, 320)      102720      dropout_26[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_68 (TimeDistri (None, 87, 320)      102720      dropout_26[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "multiply_23 (Multiply)          (None, 87, 320)      0           time_distributed_67[0][0]        \n",
      "                                                                 time_distributed_68[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "add_24 (Add)                    (None, 87, 320)      0           layer_normalization_19[0][0]     \n",
      "                                                                 multiply_23[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_20 (LayerNo (None, 87, 320)      640         add_24[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_69 (TimeDistri (None, 87, 320)      102720      layer_normalization_20[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_70 (TimeDistri (None, 87, 320)      102720      layer_normalization_20[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "multiply_24 (Multiply)          (None, 87, 320)      0           time_distributed_69[0][0]        \n",
      "                                                                 time_distributed_70[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "add_25 (Add)                    (None, 87, 320)      0           multiply_24[0][0]                \n",
      "                                                                 layer_normalization_17[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_21 (LayerNo (None, 87, 320)      640         add_25[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 44, 320)]    0           layer_normalization_21[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_71 (TimeDistri (None, 44, 3)        963         tf_op_layer_TemporalFusionTransfo\n",
      "==================================================================================================\n",
      "Total params: 10,773,813\n",
      "Trainable params: 10,773,813\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "None\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cached data \"train\" updated\n",
      "Cached data \"valid\" updated\n",
      "*** Fitting TemporalFusionTransformer ***\n",
      "Getting batched_data\n",
      "Using cached training data\n",
      "Using cached validation data\n",
      "Using keras standard fit\n",
      "Train on 11188446 samples, validate on 834023 samples\n",
      "Epoch 1/100\n",
      "11188446/11188446 [==============================] - ETA: 0s - loss: 0.0451WARNING:tensorflow:From /opt/conda/lib/python3.8/site-packages/tensorflow/python/keras/engine/training_v1.py:2048: Model.state_updates (from tensorflow.python.keras.engine.training) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This property should not be used in TensorFlow 2.0, as updates are applied automatically.\n",
      "11188446/11188446 [==============================] - 11951s 1ms/sample - loss: 0.0451 - val_loss: 0.0262\n",
      "Epoch 2/100\n",
      "11188446/11188446 [==============================] - 11943s 1ms/sample - loss: 0.0284 - val_loss: 0.0251\n",
      "Epoch 3/100\n",
      "11188446/11188446 [==============================] - 11937s 1ms/sample - loss: 0.0324 - val_loss: 0.0239\n",
      "Epoch 4/100\n",
      "11188446/11188446 [==============================] - 11936s 1ms/sample - loss: 0.0252 - val_loss: 0.0264\n",
      "Epoch 5/100\n",
      "11188446/11188446 [==============================] - 11934s 1ms/sample - loss: 0.0266 - val_loss: 0.0239\n",
      "Epoch 6/100\n",
      "11188446/11188446 [==============================] - 11937s 1ms/sample - loss: 0.0257 - val_loss: 0.0238\n",
      "Epoch 7/100\n",
      "11188446/11188446 [==============================] - 11935s 1ms/sample - loss: 0.0329 - val_loss: 0.0251\n",
      "Epoch 8/100\n",
      "11188446/11188446 [==============================] - 11919s 1ms/sample - loss: 0.0248 - val_loss: 0.0241\n",
      "Cannot load from /workspace/inkyung/TFT/repo/tft/outputs_mobiact_wo_known_no_bioinfo2/saved_models/MobiAct_wo_known_no_bioinfo/fixed/tmp, skipping ...\n",
      "Using cached validation data\n",
      "Optimal model found, updating\n",
      "WARNING:tensorflow:From /workspace/inkyung/TFT/repo/tft/libs/tft_model.py:1373: The name tf.keras.backend.get_session is deprecated. Please use tf.compat.v1.keras.backend.get_session instead.\n",
      "\n",
      "Model saved to: /workspace/inkyung/TFT/repo/tft/outputs_mobiact_wo_known_no_bioinfo2/saved_models/MobiAct_wo_known_no_bioinfo/fixed/TemporalFusionTransformer.ckpt\n",
      "Model saved to: /workspace/inkyung/TFT/repo/tft/outputs_mobiact_wo_known_no_bioinfo2/saved_models/MobiAct_wo_known_no_bioinfo/fixed/TemporalFusionTransformer.ckpt\n"
     ]
    }
   ],
   "source": [
    "tf.compat.v1.get_default_graph()\n",
    "with tf.Graph().as_default(), tf.compat.v1.Session(config=tf_config) as sess:\n",
    "    tf.compat.v1.keras.backend.set_session(sess)\n",
    "    params = opt_manager.get_next_parameters()\n",
    "    # Create a TFT model\n",
    "    model = TemporalFusionTransformer(model_params,\n",
    "                                    use_cudnn=True) # Run model on GPU using CuDNNLSTM cells\n",
    "    \n",
    "    # Sample data into minibatches for training\n",
    "    if not model.training_data_cached():\n",
    "        model.cache_batched_data(train, \"train\", num_samples=train_samples)\n",
    "        model.cache_batched_data(valid, \"valid\", num_samples=valid_samples)\n",
    "\n",
    "    # Train and save model\n",
    "    model.fit()\n",
    "    \n",
    "    val_loss = model.evaluate()\n",
    "    if np.allclose(val_loss, 0.) or np.isnan(val_loss):\n",
    "        # Set all invalid losses to infintiy.\n",
    "        # N.b. val_loss only becomes 0. when the weights are nan.\n",
    "        print(\"Skipping bad configuration....\")\n",
    "        val_loss = np.inf\n",
    "    opt_manager.update_score(params, val_loss, model)\n",
    "    tf.compat.v1.keras.backend.set_session(sess)\n",
    "    model.save(model_folder)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evalution "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-06-28T05:20:12.118546Z",
     "start_time": "2021-06-28T05:19:49.761808Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Device mapping:\n",
      "/job:localhost/replica:0/task:0/device:XLA_CPU:0 -> device: XLA_CPU device\n",
      "/job:localhost/replica:0/task:0/device:XLA_GPU:0 -> device: XLA_GPU device\n",
      "/job:localhost/replica:0/task:0/device:GPU:0 -> device: 0, name: Tesla V100-DGXS-32GB, pci bus id: 0000:0f:00.0, compute capability: 7.0\n",
      "\n",
      "Resetting temp folder...\n",
      "*** TemporalFusionTransformer params ***\n",
      "# dropout_rate = 0.3\n",
      "# hidden_layer_size = 320\n",
      "# learning_rate = 0.001\n",
      "# minibatch_size = 512\n",
      "# max_gradient_norm = 100.0\n",
      "# num_heads = 4\n",
      "# stack_size = 1\n",
      "# model_folder = /workspace/inkyung/TFT/repo/tft/outputs_mobiact_wo_known_no_bioinfo2/saved_models/MobiAct_wo_known_no_bioinfo2/fixed\n",
      "# total_time_steps = 87\n",
      "# num_encoder_steps = 43\n",
      "# num_epochs = 100\n",
      "# early_stopping_patience = 5\n",
      "# multiprocessing_workers = 5\n",
      "# column_definition = [('person_id', <DataTypes.CATEGORICAL: 1>, <InputTypes.ID: 4>), ('rel_time', <DataTypes.REAL_VALUED: 0>, <InputTypes.TIME: 5>), ('acc_x', <DataTypes.REAL_VALUED: 0>, <InputTypes.OBSERVED_INPUT: 1>), ('acc_y', <DataTypes.REAL_VALUED: 0>, <InputTypes.OBSERVED_INPUT: 1>), ('acc_z', <DataTypes.REAL_VALUED: 0>, <InputTypes.OBSERVED_INPUT: 1>), ('gyro_x', <DataTypes.REAL_VALUED: 0>, <InputTypes.OBSERVED_INPUT: 1>), ('gyro_y', <DataTypes.REAL_VALUED: 0>, <InputTypes.OBSERVED_INPUT: 1>), ('gyro_z', <DataTypes.REAL_VALUED: 0>, <InputTypes.OBSERVED_INPUT: 1>), ('label_encoded', <DataTypes.REAL_VALUED: 0>, <InputTypes.TARGET: 0>), ('known', <DataTypes.CATEGORICAL: 1>, <InputTypes.KNOWN_INPUT: 2>), ('circum', <DataTypes.CATEGORICAL: 1>, <InputTypes.STATIC_INPUT: 3>)]\n",
      "# input_size = 9\n",
      "# output_size = 1\n",
      "# category_counts = [1, 1]\n",
      "# input_obs_loc = [6]\n",
      "# static_input_loc = [8]\n",
      "# known_regular_inputs = []\n",
      "# known_categorical_inputs = [0, 1]\n",
      "Tensor(\"TemporalFusionTransformer/stack:0\", shape=(None, 1, 320), dtype=float32)\n",
      "WARNING:tensorflow:Layer lstm will not use cuDNN kernel since it doesn't meet the cuDNN kernel criteria. It will use generic GPU kernel as fallback when running on GPU\n",
      "WARNING:tensorflow:Layer lstm_1 will not use cuDNN kernel since it doesn't meet the cuDNN kernel criteria. It will use generic GPU kernel as fallback when running on GPU\n",
      "Model: \"functional_1\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            [(None, 87, 9)]      0                                            \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 87, 2)]      0           input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 87)]         0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "sequential_1 (Sequential)       (None, 87, 320)      320         tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 320)]        0           sequential_1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 1, 320)]     0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "flatten (Flatten)               (None, 320)          0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "dense_8 (Dense)                 (None, 320)          102720      flatten[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 1, 320)]     0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "activation (Activation)         (None, 320)          0           dense_8[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_12 (Dense)                (None, 1, 320)       102720      tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "dense_9 (Dense)                 (None, 320)          102720      activation[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_2 (Activation)       (None, 1, 320)       0           dense_12[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dropout (Dropout)               (None, 320)          0           dense_9[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_13 (Dense)                (None, 1, 320)       102720      activation_2[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_10 (Dense)                (None, 1)            321         dropout[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_11 (Dense)                (None, 1)            321         dropout[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, 1, 320)       0           dense_13[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_7 (Dense)                 (None, 1)            321         flatten[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "multiply (Multiply)             (None, 1)            0           dense_10[0][0]                   \n",
      "                                                                 dense_11[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_14 (Dense)                (None, 1, 320)       102720      dropout_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_15 (Dense)                (None, 1, 320)       102720      dropout_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add (Add)                       (None, 1)            0           dense_7[0][0]                    \n",
      "                                                                 multiply[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "multiply_1 (Multiply)           (None, 1, 320)       0           dense_14[0][0]                   \n",
      "                                                                 dense_15[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization (LayerNorma (None, 1)            2           add[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "add_1 (Add)                     (None, 1, 320)       0           tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 multiply_1[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_1 (Activation)       (None, 1)            0           layer_normalization[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_1 (LayerNor (None, 1, 320)       640         add_1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 1, 1)]       0           activation_1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 1, 320)]     0           layer_normalization_1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "multiply_2 (Multiply)           (None, 1, 320)       0           tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 320)]        0           multiply_2[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_16 (Dense)                (None, 320)          102720      tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "activation_3 (Activation)       (None, 320)          0           dense_16[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 87, 7)]      0           input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_17 (Dense)                (None, 320)          102720      activation_3[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 87, 1)]      0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 87, 1)]      0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 87, 1)]      0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 87, 1)]      0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 87, 1)]      0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 87, 1)]      0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 87)]         0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 87, 1)]      0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)             (None, 320)          0           dense_17[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_1 (TimeDistrib (None, 87, 320)      640         tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_2 (TimeDistrib (None, 87, 320)      640         tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_3 (TimeDistrib (None, 87, 320)      640         tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_4 (TimeDistrib (None, 87, 320)      640         tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_5 (TimeDistrib (None, 87, 320)      640         tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_6 (TimeDistrib (None, 87, 320)      640         tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "sequential (Sequential)         (None, 87, 320)      320         tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "time_distributed (TimeDistribut (None, 87, 320)      640         tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "dense_18 (Dense)                (None, 320)          102720      dropout_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_19 (Dense)                (None, 320)          102720      dropout_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 87, 320, 6)] 0           time_distributed_1[0][0]         \n",
      "                                                                 time_distributed_2[0][0]         \n",
      "                                                                 time_distributed_3[0][0]         \n",
      "                                                                 time_distributed_4[0][0]         \n",
      "                                                                 time_distributed_5[0][0]         \n",
      "                                                                 time_distributed_6[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 87, 320, 1)] 0           sequential[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 87, 320, 1)] 0           time_distributed[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "multiply_3 (Multiply)           (None, 320)          0           dense_18[0][0]                   \n",
      "                                                                 dense_19[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 43, 320, 6)] 0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 43, 320, 1)] 0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 43, 320, 1)] 0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "add_2 (Add)                     (None, 320)          0           tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 multiply_3[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 43, 320, 8)] 0           tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_2 (LayerNor (None, 320)          640         add_2[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 43, 2560)]   0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 1, 320)]     0           layer_normalization_2[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 44, 320, 1)] 0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_8 (TimeDistrib (None, 43, 320)      819520      tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_9 (TimeDistrib (None, 1, 320)       102400      tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 44, 320)]    0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 1, 320)]     0           layer_normalization_2[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 43, 320)]    0           time_distributed_8[0][0]         \n",
      "                                                                 time_distributed_9[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 43, 320)]    0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 43, 320)]    0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 43, 320)]    0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 43, 320)]    0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 43, 320)]    0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 43, 320)]    0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 43, 320)]    0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 43, 320)]    0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_46 (TimeDistri (None, 44, 320)      102720      tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_47 (TimeDistri (None, 1, 320)       102400      tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "activation_7 (Activation)       (None, 43, 320)      0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_13 (TimeDistri (None, 43, 320)      102720      tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_17 (TimeDistri (None, 43, 320)      102720      tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_21 (TimeDistri (None, 43, 320)      102720      tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_25 (TimeDistri (None, 43, 320)      102720      tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_29 (TimeDistri (None, 43, 320)      102720      tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_33 (TimeDistri (None, 43, 320)      102720      tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_37 (TimeDistri (None, 43, 320)      102720      tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_41 (TimeDistri (None, 43, 320)      102720      tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 44, 320)]    0           time_distributed_46[0][0]        \n",
      "                                                                 time_distributed_47[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 44, 320)]    0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_10 (TimeDistri (None, 43, 320)      102720      activation_7[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "activation_9 (Activation)       (None, 43, 320)      0           time_distributed_13[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "activation_10 (Activation)      (None, 43, 320)      0           time_distributed_17[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "activation_11 (Activation)      (None, 43, 320)      0           time_distributed_21[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "activation_12 (Activation)      (None, 43, 320)      0           time_distributed_25[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "activation_13 (Activation)      (None, 43, 320)      0           time_distributed_29[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "activation_14 (Activation)      (None, 43, 320)      0           time_distributed_33[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "activation_15 (Activation)      (None, 43, 320)      0           time_distributed_37[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "activation_16 (Activation)      (None, 43, 320)      0           time_distributed_41[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "activation_17 (Activation)      (None, 44, 320)      0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_51 (TimeDistri (None, 44, 320)      102720      tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "dropout_6 (Dropout)             (None, 43, 320)      0           time_distributed_10[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_14 (TimeDistri (None, 43, 320)      102720      activation_9[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_18 (TimeDistri (None, 43, 320)      102720      activation_10[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_22 (TimeDistri (None, 43, 320)      102720      activation_11[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_26 (TimeDistri (None, 43, 320)      102720      activation_12[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_30 (TimeDistri (None, 43, 320)      102720      activation_13[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_34 (TimeDistri (None, 43, 320)      102720      activation_14[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_38 (TimeDistri (None, 43, 320)      102720      activation_15[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_42 (TimeDistri (None, 43, 320)      102720      activation_16[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_48 (TimeDistri (None, 44, 320)      102720      activation_17[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_19 (Activation)      (None, 44, 320)      0           time_distributed_51[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_11 (TimeDistri (None, 43, 8)        2568        dropout_6[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_12 (TimeDistri (None, 43, 8)        2568        dropout_6[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_7 (Dropout)             (None, 43, 320)      0           time_distributed_14[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dropout_8 (Dropout)             (None, 43, 320)      0           time_distributed_18[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dropout_9 (Dropout)             (None, 43, 320)      0           time_distributed_22[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dropout_10 (Dropout)            (None, 43, 320)      0           time_distributed_26[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dropout_11 (Dropout)            (None, 43, 320)      0           time_distributed_30[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dropout_12 (Dropout)            (None, 43, 320)      0           time_distributed_34[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dropout_13 (Dropout)            (None, 43, 320)      0           time_distributed_38[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dropout_14 (Dropout)            (None, 43, 320)      0           time_distributed_42[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dense_24 (Dense)                (None, 320)          102720      tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "dense_28 (Dense)                (None, 320)          102720      tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "dropout_15 (Dropout)            (None, 44, 320)      0           time_distributed_48[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_52 (TimeDistri (None, 44, 320)      102720      activation_19[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_7 (TimeDistrib (None, 43, 8)        20488       tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "multiply_7 (Multiply)           (None, 43, 8)        0           time_distributed_11[0][0]        \n",
      "                                                                 time_distributed_12[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_15 (TimeDistri (None, 43, 320)      102720      dropout_7[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_16 (TimeDistri (None, 43, 320)      102720      dropout_7[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_19 (TimeDistri (None, 43, 320)      102720      dropout_8[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_20 (TimeDistri (None, 43, 320)      102720      dropout_8[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_23 (TimeDistri (None, 43, 320)      102720      dropout_9[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_24 (TimeDistri (None, 43, 320)      102720      dropout_9[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_27 (TimeDistri (None, 43, 320)      102720      dropout_10[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_28 (TimeDistri (None, 43, 320)      102720      dropout_10[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_31 (TimeDistri (None, 43, 320)      102720      dropout_11[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_32 (TimeDistri (None, 43, 320)      102720      dropout_11[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_35 (TimeDistri (None, 43, 320)      102720      dropout_12[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_36 (TimeDistri (None, 43, 320)      102720      dropout_12[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_39 (TimeDistri (None, 43, 320)      102720      dropout_13[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_40 (TimeDistri (None, 43, 320)      102720      dropout_13[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_43 (TimeDistri (None, 43, 320)      102720      dropout_14[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_44 (TimeDistri (None, 43, 320)      102720      dropout_14[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_5 (Activation)       (None, 320)          0           dense_24[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_6 (Activation)       (None, 320)          0           dense_28[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_49 (TimeDistri (None, 44, 1)        321         dropout_15[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_50 (TimeDistri (None, 44, 1)        321         dropout_15[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dropout_16 (Dropout)            (None, 44, 320)      0           time_distributed_52[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "add_6 (Add)                     (None, 43, 8)        0           time_distributed_7[0][0]         \n",
      "                                                                 multiply_7[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "multiply_8 (Multiply)           (None, 43, 320)      0           time_distributed_15[0][0]        \n",
      "                                                                 time_distributed_16[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "multiply_9 (Multiply)           (None, 43, 320)      0           time_distributed_19[0][0]        \n",
      "                                                                 time_distributed_20[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "multiply_10 (Multiply)          (None, 43, 320)      0           time_distributed_23[0][0]        \n",
      "                                                                 time_distributed_24[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "multiply_11 (Multiply)          (None, 43, 320)      0           time_distributed_27[0][0]        \n",
      "                                                                 time_distributed_28[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "multiply_12 (Multiply)          (None, 43, 320)      0           time_distributed_31[0][0]        \n",
      "                                                                 time_distributed_32[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "multiply_13 (Multiply)          (None, 43, 320)      0           time_distributed_35[0][0]        \n",
      "                                                                 time_distributed_36[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "multiply_14 (Multiply)          (None, 43, 320)      0           time_distributed_39[0][0]        \n",
      "                                                                 time_distributed_40[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "multiply_15 (Multiply)          (None, 43, 320)      0           time_distributed_43[0][0]        \n",
      "                                                                 time_distributed_44[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dense_25 (Dense)                (None, 320)          102720      activation_5[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_29 (Dense)                (None, 320)          102720      activation_6[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_45 (TimeDistri (None, 44, 1)        321         tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "multiply_17 (Multiply)          (None, 44, 1)        0           time_distributed_49[0][0]        \n",
      "                                                                 time_distributed_50[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_53 (TimeDistri (None, 44, 320)      102720      dropout_16[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_54 (TimeDistri (None, 44, 320)      102720      dropout_16[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_6 (LayerNor (None, 43, 8)        16          add_6[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "add_7 (Add)                     (None, 43, 320)      0           tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 multiply_8[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "add_8 (Add)                     (None, 43, 320)      0           tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 multiply_9[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "add_9 (Add)                     (None, 43, 320)      0           tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 multiply_10[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "add_10 (Add)                    (None, 43, 320)      0           tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 multiply_11[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "add_11 (Add)                    (None, 43, 320)      0           tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 multiply_12[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "add_12 (Add)                    (None, 43, 320)      0           tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 multiply_13[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "add_13 (Add)                    (None, 43, 320)      0           tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 multiply_14[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "add_14 (Add)                    (None, 43, 320)      0           tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 multiply_15[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dropout_4 (Dropout)             (None, 320)          0           dense_25[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dropout_5 (Dropout)             (None, 320)          0           dense_29[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "add_15 (Add)                    (None, 44, 1)        0           time_distributed_45[0][0]        \n",
      "                                                                 multiply_17[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "multiply_18 (Multiply)          (None, 44, 320)      0           time_distributed_53[0][0]        \n",
      "                                                                 time_distributed_54[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "activation_8 (Activation)       (None, 43, 8)        0           layer_normalization_6[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_7 (LayerNor (None, 43, 320)      640         add_7[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_8 (LayerNor (None, 43, 320)      640         add_8[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_9 (LayerNor (None, 43, 320)      640         add_9[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_10 (LayerNo (None, 43, 320)      640         add_10[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_11 (LayerNo (None, 43, 320)      640         add_11[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_12 (LayerNo (None, 43, 320)      640         add_12[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_13 (LayerNo (None, 43, 320)      640         add_13[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_14 (LayerNo (None, 43, 320)      640         add_14[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "dense_26 (Dense)                (None, 320)          102720      dropout_4[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_27 (Dense)                (None, 320)          102720      dropout_4[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_30 (Dense)                (None, 320)          102720      dropout_5[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_31 (Dense)                (None, 320)          102720      dropout_5[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_15 (LayerNo (None, 44, 1)        2           add_15[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "add_16 (Add)                    (None, 44, 320)      0           tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 multiply_18[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 43, 1, 8)]   0           activation_8[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 43, 320, 8)] 0           layer_normalization_7[0][0]      \n",
      "                                                                 layer_normalization_8[0][0]      \n",
      "                                                                 layer_normalization_9[0][0]      \n",
      "                                                                 layer_normalization_10[0][0]     \n",
      "                                                                 layer_normalization_11[0][0]     \n",
      "                                                                 layer_normalization_12[0][0]     \n",
      "                                                                 layer_normalization_13[0][0]     \n",
      "                                                                 layer_normalization_14[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "multiply_5 (Multiply)           (None, 320)          0           dense_26[0][0]                   \n",
      "                                                                 dense_27[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "multiply_6 (Multiply)           (None, 320)          0           dense_30[0][0]                   \n",
      "                                                                 dense_31[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_18 (Activation)      (None, 44, 1)        0           layer_normalization_15[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_16 (LayerNo (None, 44, 320)      640         add_16[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "multiply_16 (Multiply)          (None, 43, 320, 8)   0           tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "add_4 (Add)                     (None, 320)          0           tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 multiply_5[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "add_5 (Add)                     (None, 320)          0           tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 multiply_6[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 44, 1, 1)]   0           activation_18[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 44, 320, 1)] 0           layer_normalization_16[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 43, 320)]    0           multiply_16[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_4 (LayerNor (None, 320)          640         add_4[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_5 (LayerNor (None, 320)          640         add_5[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "multiply_19 (Multiply)          (None, 44, 320, 1)   0           tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "dense_20 (Dense)                (None, 320)          102720      tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "lstm (LSTM)                     [(None, 43, 320), (N 820480      tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 layer_normalization_4[0][0]      \n",
      "                                                                 layer_normalization_5[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 44, 320)]    0           multiply_19[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "activation_4 (Activation)       (None, 320)          0           dense_20[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lstm_1 (LSTM)                   (None, 44, 320)      820480      tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 lstm[0][1]                       \n",
      "                                                                 lstm[0][2]                       \n",
      "__________________________________________________________________________________________________\n",
      "dense_21 (Dense)                (None, 320)          102720      activation_4[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 87, 320)]    0           lstm[0][0]                       \n",
      "                                                                 lstm_1[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "dropout_3 (Dropout)             (None, 320)          0           dense_21[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dropout_17 (Dropout)            (None, 87, 320)      0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "dense_22 (Dense)                (None, 320)          102720      dropout_3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_23 (Dense)                (None, 320)          102720      dropout_3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_55 (TimeDistri (None, 87, 320)      102720      dropout_17[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_56 (TimeDistri (None, 87, 320)      102720      dropout_17[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "multiply_4 (Multiply)           (None, 320)          0           dense_22[0][0]                   \n",
      "                                                                 dense_23[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "multiply_20 (Multiply)          (None, 87, 320)      0           time_distributed_55[0][0]        \n",
      "                                                                 time_distributed_56[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 87, 320)]    0           tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "add_3 (Add)                     (None, 320)          0           tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 multiply_4[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "add_17 (Add)                    (None, 87, 320)      0           multiply_20[0][0]                \n",
      "                                                                 tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_3 (LayerNor (None, 320)          640         add_3[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_17 (LayerNo (None, 87, 320)      640         add_17[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 1, 320)]     0           layer_normalization_3[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_57 (TimeDistri (None, 87, 320)      102720      layer_normalization_17[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_58 (TimeDistri (None, 1, 320)       102400      tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 87, 320)]    0           time_distributed_57[0][0]        \n",
      "                                                                 time_distributed_58[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "activation_20 (Activation)      (None, 87, 320)      0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_59 (TimeDistri (None, 87, 320)      102720      activation_20[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_18 (Dropout)            (None, 87, 320)      0           time_distributed_59[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_60 (TimeDistri (None, 87, 320)      102720      dropout_18[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_61 (TimeDistri (None, 87, 320)      102720      dropout_18[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "multiply_21 (Multiply)          (None, 87, 320)      0           time_distributed_60[0][0]        \n",
      "                                                                 time_distributed_61[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "add_18 (Add)                    (None, 87, 320)      0           layer_normalization_17[0][0]     \n",
      "                                                                 multiply_21[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_18 (LayerNo (None, 87, 320)      640         add_18[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(3,)]               0           layer_normalization_18[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [()]                 0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(3,)]               0           layer_normalization_18[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [()]                 0           tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(1,)]               0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(1,)]               0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(2,)]               0           tf_op_layer_TemporalFusionTransfo\n",
      "                                                                 tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, None)]       0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, None, None)] 0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "dense_88 (Dense)                (None, 87, 80)       25600       layer_normalization_18[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "dense_89 (Dense)                (None, 87, 80)       25600       layer_normalization_18[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, None, None)] 0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "dense_90 (Dense)                (None, 87, 80)       25600       layer_normalization_18[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "dense_91 (Dense)                (None, 87, 80)       25600       layer_normalization_18[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "dense_92 (Dense)                (None, 87, 80)       25600       layer_normalization_18[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "dense_93 (Dense)                (None, 87, 80)       25600       layer_normalization_18[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "dense_94 (Dense)                (None, 87, 80)       25600       layer_normalization_18[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "dense_95 (Dense)                (None, 87, 80)       25600       layer_normalization_18[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "lambda (Lambda)                 (None, 87, 87)       0           dense_88[0][0]                   \n",
      "                                                                 dense_89[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_1 (Lambda)               (None, None, None)   0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "lambda_3 (Lambda)               (None, 87, 87)       0           dense_90[0][0]                   \n",
      "                                                                 dense_91[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_4 (Lambda)               (None, None, None)   0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "lambda_6 (Lambda)               (None, 87, 87)       0           dense_92[0][0]                   \n",
      "                                                                 dense_93[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_7 (Lambda)               (None, None, None)   0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "lambda_9 (Lambda)               (None, 87, 87)       0           dense_94[0][0]                   \n",
      "                                                                 dense_95[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_10 (Lambda)              (None, None, None)   0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "add_19 (Add)                    (None, 87, 87)       0           lambda[0][0]                     \n",
      "                                                                 lambda_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "add_20 (Add)                    (None, 87, 87)       0           lambda_3[0][0]                   \n",
      "                                                                 lambda_4[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "add_21 (Add)                    (None, 87, 87)       0           lambda_6[0][0]                   \n",
      "                                                                 lambda_7[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "add_22 (Add)                    (None, 87, 87)       0           lambda_9[0][0]                   \n",
      "                                                                 lambda_10[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_21 (Activation)      (None, 87, 87)       0           add_19[0][0]                     \n",
      "                                                                 add_20[0][0]                     \n",
      "                                                                 add_21[0][0]                     \n",
      "                                                                 add_22[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "dropout_19 (Dropout)            (None, 87, 87)       0           activation_21[0][0]              \n",
      "                                                                 activation_21[1][0]              \n",
      "                                                                 activation_21[2][0]              \n",
      "                                                                 activation_21[3][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_87 (Dense)                (None, 87, 80)       25600       layer_normalization_18[0][0]     \n",
      "                                                                 layer_normalization_18[0][0]     \n",
      "                                                                 layer_normalization_18[0][0]     \n",
      "                                                                 layer_normalization_18[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "lambda_2 (Lambda)               (None, 87, 80)       0           dropout_19[0][0]                 \n",
      "                                                                 dense_87[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_5 (Lambda)               (None, 87, 80)       0           dropout_19[1][0]                 \n",
      "                                                                 dense_87[1][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_8 (Lambda)               (None, 87, 80)       0           dropout_19[2][0]                 \n",
      "                                                                 dense_87[2][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_11 (Lambda)              (None, 87, 80)       0           dropout_19[3][0]                 \n",
      "                                                                 dense_87[3][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dropout_20 (Dropout)            (None, 87, 80)       0           lambda_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dropout_21 (Dropout)            (None, 87, 80)       0           lambda_5[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dropout_22 (Dropout)            (None, 87, 80)       0           lambda_8[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dropout_23 (Dropout)            (None, 87, 80)       0           lambda_11[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(4, None, 87, 80)]  0           dropout_20[0][0]                 \n",
      "                                                                 dropout_21[0][0]                 \n",
      "                                                                 dropout_22[0][0]                 \n",
      "                                                                 dropout_23[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 87, 80)]     0           tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "dense_96 (Dense)                (None, 87, 320)      25600       tf_op_layer_TemporalFusionTransfo\n",
      "__________________________________________________________________________________________________\n",
      "dropout_24 (Dropout)            (None, 87, 320)      0           dense_96[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dropout_25 (Dropout)            (None, 87, 320)      0           dropout_24[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_62 (TimeDistri (None, 87, 320)      102720      dropout_25[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_63 (TimeDistri (None, 87, 320)      102720      dropout_25[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "multiply_22 (Multiply)          (None, 87, 320)      0           time_distributed_62[0][0]        \n",
      "                                                                 time_distributed_63[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "add_23 (Add)                    (None, 87, 320)      0           multiply_22[0][0]                \n",
      "                                                                 layer_normalization_18[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_19 (LayerNo (None, 87, 320)      640         add_23[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_64 (TimeDistri (None, 87, 320)      102720      layer_normalization_19[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_22 (Activation)      (None, 87, 320)      0           time_distributed_64[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_65 (TimeDistri (None, 87, 320)      102720      activation_22[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_26 (Dropout)            (None, 87, 320)      0           time_distributed_65[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_66 (TimeDistri (None, 87, 320)      102720      dropout_26[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_67 (TimeDistri (None, 87, 320)      102720      dropout_26[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "multiply_23 (Multiply)          (None, 87, 320)      0           time_distributed_66[0][0]        \n",
      "                                                                 time_distributed_67[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "add_24 (Add)                    (None, 87, 320)      0           layer_normalization_19[0][0]     \n",
      "                                                                 multiply_23[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_20 (LayerNo (None, 87, 320)      640         add_24[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_68 (TimeDistri (None, 87, 320)      102720      layer_normalization_20[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_69 (TimeDistri (None, 87, 320)      102720      layer_normalization_20[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "multiply_24 (Multiply)          (None, 87, 320)      0           time_distributed_68[0][0]        \n",
      "                                                                 time_distributed_69[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "add_25 (Add)                    (None, 87, 320)      0           multiply_24[0][0]                \n",
      "                                                                 layer_normalization_17[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_21 (LayerNo (None, 87, 320)      640         add_25[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_TemporalFusionTrans [(None, 44, 320)]    0           layer_normalization_21[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_70 (TimeDistri (None, 44, 3)        963         tf_op_layer_TemporalFusionTransfo\n",
      "==================================================================================================\n",
      "Total params: 10,773,493\n",
      "Trainable params: 10,773,493\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "None\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /workspace/inkyung/TFT/repo/tft/libs/tft_model.py:1396: The name tf.keras.backend.get_session is deprecated. Please use tf.compat.v1.keras.backend.get_session instead.\n",
      "\n",
      "Loading model from /workspace/inkyung/TFT/repo/tft/outputs_mobiact_wo_known_no_bioinfo2/saved_models/MobiAct_wo_known_no_bioinfo2/fixed/TemporalFusionTransformer.ckpt\n",
      "Unsuccessful TensorSliceReader constructor: Failed to find any matching files for /workspace/inkyung/TFT/repo/tft/outputs_mobiact_wo_known_no_bioinfo2/saved_models/MobiAct_wo_known_no_bioinfo2/fixed/TemporalFusionTransformer.ckpt\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "The passed save_path is not a valid checkpoint: /workspace/inkyung/TFT/repo/tft/outputs_mobiact_wo_known_no_bioinfo2/saved_models/MobiAct_wo_known_no_bioinfo2/fixed/TemporalFusionTransformer.ckpt",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-23-e0810a52c6e4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      5\u001b[0m     model = TemporalFusionTransformer(model_params,\n\u001b[1;32m      6\u001b[0m                                       use_cudnn=True)\n\u001b[0;32m----> 7\u001b[0;31m     \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_folder\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m     \u001b[0;31m# Make forecasts\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/workspace/inkyung/TFT/repo/tft/libs/tft_model.py\u001b[0m in \u001b[0;36mload\u001b[0;34m(self, model_folder, use_keras_loadings)\u001b[0m\n\u001b[1;32m   1393\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1394\u001b[0m       \u001b[0;31m# Loads tensorflow graph for optimal models.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1395\u001b[0;31m       utils.load(\n\u001b[0m\u001b[1;32m   1396\u001b[0m           \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mv1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkeras\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1397\u001b[0m           \u001b[0mmodel_folder\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/workspace/inkyung/TFT/repo/tft/libs/utils.py\u001b[0m in \u001b[0;36mload\u001b[0;34m(tf_session, model_folder, cp_name, scope, verbose)\u001b[0m\n\u001b[1;32m    209\u001b[0m     \u001b[0msaver\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mv1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSaver\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvar_list\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvar_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_to_keep\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m100000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    210\u001b[0m   \u001b[0;31m# Load\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 211\u001b[0;31m   \u001b[0msaver\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrestore\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtf_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mload_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    212\u001b[0m   \u001b[0mall_vars\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mv\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mv1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_default_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_graph_def\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnode\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    213\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/tensorflow/python/training/saver.py\u001b[0m in \u001b[0;36mrestore\u001b[0;34m(self, sess, save_path)\u001b[0m\n\u001b[1;32m   1288\u001b[0m     \u001b[0mcheckpoint_prefix\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msave_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1289\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mcheckpoint_management\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcheckpoint_exists_internal\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcheckpoint_prefix\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1290\u001b[0;31m       raise ValueError(\"The passed save_path is not a valid checkpoint: \" +\n\u001b[0m\u001b[1;32m   1291\u001b[0m                        checkpoint_prefix)\n\u001b[1;32m   1292\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: The passed save_path is not a valid checkpoint: /workspace/inkyung/TFT/repo/tft/outputs_mobiact_wo_known_no_bioinfo2/saved_models/MobiAct_wo_known_no_bioinfo2/fixed/TemporalFusionTransformer.ckpt"
     ]
    }
   ],
   "source": [
    "tf.compat.v1.reset_default_graph()\n",
    "with tf.Graph().as_default(), tf.compat.v1.Session(config=tf_config) as sess:\n",
    "    tf.compat.v1.keras.backend.set_session(sess)\n",
    "    # Create a new model & load weights\n",
    "    model = TemporalFusionTransformer(model_params,\n",
    "                                      use_cudnn=True)\n",
    "    model.load(model_folder)\n",
    "    \n",
    "    # Make forecasts\n",
    "    output_map = model.predict(test, return_targets=True)\n",
    "    \n",
    "    targets = data_formatter.format_predictions(output_map[\"targets\"])\n",
    "    # Format predictions\n",
    "    \n",
    "    p50_forecast = data_formatter.format_predictions(output_map[\"p50\"])\n",
    "    p90_forecast = data_formatter.format_predictions(output_map[\"p90\"])\n",
    "\n",
    "    def extract_numerical_data(data):\n",
    "        \"\"\"Strips out forecast time and identifier columns.\"\"\"\n",
    "        return data[[\n",
    "            col for col in data.columns\n",
    "            if col not in {\"forecast_time\", \"identifier\"}\n",
    "        ]]\n",
    "    # Compute Losses\n",
    "    p50_loss = utils.numpy_normalised_quantile_loss(\n",
    "        extract_numerical_data(targets), extract_numerical_data(p50_forecast),\n",
    "        0.5)\n",
    "    p90_loss = utils.numpy_normalised_quantile_loss(\n",
    "        extract_numerical_data(targets), extract_numerical_data(p90_forecast),\n",
    "        0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-06-28T05:18:35.596921Z",
     "start_time": "2021-06-28T05:18:38.201Z"
    }
   },
   "outputs": [],
   "source": [
    "import datetime as dte\n",
    "print(\"Hyperparam optimisation completed @ {}\".format(dte.datetime.now()))\n",
    "# print(\"Best validation loss = {}\".format(val_loss))\n",
    "# print(\"Params:\")\n",
    "\n",
    "# for k in best_params:\n",
    "#     print(k, \" = \", best_params[k])\n",
    "#     print()\n",
    "print(\"Normalised quantile losses: P50={}, P90={}\".format(p50_loss.mean(), p90_loss.mean()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-06-21T07:54:56.972234Z",
     "start_time": "2021-06-21T07:54:56.961087Z"
    }
   },
   "outputs": [],
   "source": [
    "import itertools\n",
    "def plot_confusion_matrix(cm, target_names=None, cmap=None, normalize=True, labels=True, title='Confusion matrix'):\n",
    "    accuracy = np.trace(cm) / float(np.sum(cm))\n",
    "    misclass = 1 - accuracy\n",
    "\n",
    "    if cmap is None:\n",
    "        cmap = plt.get_cmap('Blues')\n",
    "\n",
    "    if normalize:\n",
    "        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
    "        \n",
    "    plt.figure(figsize=(16, 12))\n",
    "    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "#     plt.title(title)\n",
    "    plt.colorbar()\n",
    "\n",
    "    thresh = cm.max() / 1.5 if normalize else cm.max() / 2\n",
    "    \n",
    "    if target_names is not None:\n",
    "        tick_marks = np.arange(len(target_names))\n",
    "        plt.xticks(tick_marks, target_names, )\n",
    "        plt.yticks(tick_marks, target_names)\n",
    "    \n",
    "    if labels:\n",
    "        for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
    "            if normalize:\n",
    "                plt.text(j, i, \"{:0.2f}\".format(cm[i, j]),\n",
    "                         horizontalalignment=\"center\",\n",
    "                         color=\"white\" if cm[i, j] > thresh else \"black\",\n",
    "                         fontsize='xx-large')\n",
    "            else:\n",
    "                plt.text(j, i, \"{:,}\".format(cm[i, j]),\n",
    "                         horizontalalignment=\"center\",\n",
    "                         color=\"white\" if cm[i, j] > thresh else \"black\",\n",
    "                         fontsize='xx-large')\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('True label', fontsize=30)\n",
    "    plt.xlabel('Predicted label\\naccuracy={:0.2f}; misclass={:0.2f}'.format(accuracy, misclass), fontsize=30)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-06-21T07:55:02.456773Z",
     "start_time": "2021-06-21T07:55:02.423712Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import accuracy_score, f1_score, recall_score, precision_score\n",
    "from sklearn.metrics import classification_report\n",
    "\n",
    "def getScores(forecast, target):\n",
    "    predicted, transformed_target = list(), list()\n",
    "\n",
    "    for i in range(len(forecast)):\n",
    "        value = forecast.iloc[i].values\n",
    "        for v in value:\n",
    "            if v < 16.5 and v >= 15.5:\n",
    "                predicted.append(16.0)\n",
    "            elif v < 15.5 and v >= 14.5:\n",
    "                predicted.append(15.0)\n",
    "            elif v < 14.5 and v >= 13.5:\n",
    "                predicted.append(14.0)\n",
    "            elif v < 13.5 and v >= 12.5:\n",
    "                predicted.append(13.0)\n",
    "            elif v < 12.5 and v >= 11.5:\n",
    "                predicted.append(12.0)\n",
    "            elif v < 11.5 and v >= 10.5:\n",
    "                predicted.append(11.0)\n",
    "            elif v < 10.5 and v >= 9.5:\n",
    "                predicted.append(10.0)\n",
    "            elif v < 9.5 and v >= 8.5:\n",
    "                predicted.append(9.0)\n",
    "            elif v < 8.5 and v >= 7.5:\n",
    "                predicted.append(8.0)\n",
    "            elif v < 7.5 and v >= 6.5:\n",
    "                predicted.append(7.0)\n",
    "            elif v < 6.5 and v >= 5.5:\n",
    "                predicted.append(6.0)\n",
    "            elif v < 5.5 and v >= 4.5:\n",
    "                predicted.append(5.0)\n",
    "            elif v < 4.5 and v >= 3.5:\n",
    "                predicted.append(4.0)\n",
    "            elif v < 3.5 and v >= 2.5:\n",
    "                predicted.append(3.0)\n",
    "            elif v < 2.5 and v >= 1.5:\n",
    "                predicted.append(2.0)\n",
    "            elif v < 1.5 and v >= 0.5:\n",
    "                predicted.append(1.0)\n",
    "            elif v < 0.5 and v >= -.5:\n",
    "                predicted.append(0.0)\n",
    "                \n",
    "    for i in range(len(target)):\n",
    "        value = target.iloc[i].values\n",
    "        for v in value:\n",
    "            if v < 16.5 and v >= 15.5:\n",
    "                transformed_target.append(16.0)\n",
    "            elif v < 15.5 and v >= 14.5:\n",
    "                transformed_target.append(15.0)\n",
    "            elif v < 14.5 and v >= 13.5:\n",
    "                transformed_target.append(14.0)\n",
    "            elif v < 13.5 and v >= 12.5:\n",
    "                transformed_target.append(13.0)\n",
    "            elif v < 12.5 and v >= 11.5:\n",
    "                transformed_target.append(12.0)\n",
    "            elif v < 11.5 and v >= 10.5:\n",
    "                transformed_target.append(11.0)\n",
    "            elif v < 10.5 and v >= 9.5:\n",
    "                transformed_target.append(10.0)\n",
    "            elif v < 9.5 and v >= 8.5:\n",
    "                transformed_target.append(9.0)\n",
    "            elif v < 8.5 and v >= 7.5:\n",
    "                transformed_target.append(8.0)\n",
    "            elif v < 7.5 and v >= 6.5:\n",
    "                transformed_target.append(7.0)\n",
    "            elif v < 6.5 and v >= 5.5:\n",
    "                transformed_target.append(6.0)\n",
    "            elif v < 5.5 and v >= 4.5:\n",
    "                transformed_target.append(5.0)\n",
    "            elif v < 4.5 and v >= 3.5:\n",
    "                transformed_target.append(4.0)\n",
    "            elif v < 3.5 and v >= 2.5:\n",
    "                transformed_target.append(3.0)\n",
    "            elif v < 2.5 and v >= 1.5:\n",
    "                transformed_target.append(2.0)\n",
    "            elif v < 1.5 and v >= 0.5:\n",
    "                transformed_target.append(1.0)\n",
    "            elif v < 0.5 and v >= -.5:\n",
    "                transformed_target.append(0.0)\n",
    "            else:\n",
    "                print(v)\n",
    "    \n",
    "    recall = recall_score(transformed_target, predicted, average='macro')\n",
    "    precision = precision_score(transformed_target, predicted, average='macro')\n",
    "    f1 = f1_score(transformed_target, predicted, average='macro')\n",
    "    acc = accuracy_score(transformed_target, predicted)\n",
    "    print(classification_report(transformed_target, predicted))\n",
    "    return acc, precision, recall, f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-06-21T07:55:07.959020Z",
     "start_time": "2021-06-21T07:55:07.912602Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[54 55 56 57 58 59 60 61 62 63 64 65 66 67]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>forecast_time</th>\n",
       "      <th>identifier</th>\n",
       "      <th>t+0</th>\n",
       "      <th>t+1</th>\n",
       "      <th>t+2</th>\n",
       "      <th>t+3</th>\n",
       "      <th>t+4</th>\n",
       "      <th>t+5</th>\n",
       "      <th>t+6</th>\n",
       "      <th>t+7</th>\n",
       "      <th>...</th>\n",
       "      <th>t+34</th>\n",
       "      <th>t+35</th>\n",
       "      <th>t+36</th>\n",
       "      <th>t+37</th>\n",
       "      <th>t+38</th>\n",
       "      <th>t+39</th>\n",
       "      <th>t+40</th>\n",
       "      <th>t+41</th>\n",
       "      <th>t+42</th>\n",
       "      <th>t+43</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.209990</td>\n",
       "      <td>54</td>\n",
       "      <td>11.999058</td>\n",
       "      <td>11.999049</td>\n",
       "      <td>11.999048</td>\n",
       "      <td>11.999082</td>\n",
       "      <td>11.999079</td>\n",
       "      <td>11.999078</td>\n",
       "      <td>11.999076</td>\n",
       "      <td>11.999075</td>\n",
       "      <td>...</td>\n",
       "      <td>11.999051</td>\n",
       "      <td>11.999051</td>\n",
       "      <td>11.99905</td>\n",
       "      <td>11.99905</td>\n",
       "      <td>11.99905</td>\n",
       "      <td>11.99905</td>\n",
       "      <td>11.999049</td>\n",
       "      <td>11.999049</td>\n",
       "      <td>11.999049</td>\n",
       "      <td>11.999049</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.215368</td>\n",
       "      <td>54</td>\n",
       "      <td>11.999058</td>\n",
       "      <td>11.999049</td>\n",
       "      <td>11.999048</td>\n",
       "      <td>11.999082</td>\n",
       "      <td>11.999079</td>\n",
       "      <td>11.999078</td>\n",
       "      <td>11.999076</td>\n",
       "      <td>11.999075</td>\n",
       "      <td>...</td>\n",
       "      <td>11.999051</td>\n",
       "      <td>11.999051</td>\n",
       "      <td>11.99905</td>\n",
       "      <td>11.99905</td>\n",
       "      <td>11.99905</td>\n",
       "      <td>11.99905</td>\n",
       "      <td>11.999049</td>\n",
       "      <td>11.999049</td>\n",
       "      <td>11.999049</td>\n",
       "      <td>11.999049</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.220053</td>\n",
       "      <td>54</td>\n",
       "      <td>11.999058</td>\n",
       "      <td>11.999049</td>\n",
       "      <td>11.999048</td>\n",
       "      <td>11.999082</td>\n",
       "      <td>11.999079</td>\n",
       "      <td>11.999078</td>\n",
       "      <td>11.999076</td>\n",
       "      <td>11.999075</td>\n",
       "      <td>...</td>\n",
       "      <td>11.999051</td>\n",
       "      <td>11.999051</td>\n",
       "      <td>11.99905</td>\n",
       "      <td>11.99905</td>\n",
       "      <td>11.99905</td>\n",
       "      <td>11.99905</td>\n",
       "      <td>11.999049</td>\n",
       "      <td>11.999049</td>\n",
       "      <td>11.999049</td>\n",
       "      <td>11.999049</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.225053</td>\n",
       "      <td>54</td>\n",
       "      <td>11.999058</td>\n",
       "      <td>11.999049</td>\n",
       "      <td>11.999048</td>\n",
       "      <td>11.999082</td>\n",
       "      <td>11.999079</td>\n",
       "      <td>11.999078</td>\n",
       "      <td>11.999076</td>\n",
       "      <td>11.999075</td>\n",
       "      <td>...</td>\n",
       "      <td>11.999051</td>\n",
       "      <td>11.999051</td>\n",
       "      <td>11.99905</td>\n",
       "      <td>11.99905</td>\n",
       "      <td>11.99905</td>\n",
       "      <td>11.99905</td>\n",
       "      <td>11.999049</td>\n",
       "      <td>11.999049</td>\n",
       "      <td>11.999049</td>\n",
       "      <td>11.999049</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.230049</td>\n",
       "      <td>54</td>\n",
       "      <td>11.999059</td>\n",
       "      <td>11.999049</td>\n",
       "      <td>11.999048</td>\n",
       "      <td>11.999082</td>\n",
       "      <td>11.999079</td>\n",
       "      <td>11.999078</td>\n",
       "      <td>11.999076</td>\n",
       "      <td>11.999075</td>\n",
       "      <td>...</td>\n",
       "      <td>11.999051</td>\n",
       "      <td>11.999051</td>\n",
       "      <td>11.99905</td>\n",
       "      <td>11.99905</td>\n",
       "      <td>11.99905</td>\n",
       "      <td>11.99905</td>\n",
       "      <td>11.999049</td>\n",
       "      <td>11.999049</td>\n",
       "      <td>11.999049</td>\n",
       "      <td>11.999049</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 46 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   forecast_time  identifier        t+0        t+1        t+2        t+3  \\\n",
       "0       0.209990          54  11.999058  11.999049  11.999048  11.999082   \n",
       "1       0.215368          54  11.999058  11.999049  11.999048  11.999082   \n",
       "2       0.220053          54  11.999058  11.999049  11.999048  11.999082   \n",
       "3       0.225053          54  11.999058  11.999049  11.999048  11.999082   \n",
       "4       0.230049          54  11.999059  11.999049  11.999048  11.999082   \n",
       "\n",
       "         t+4        t+5        t+6        t+7  ...       t+34       t+35  \\\n",
       "0  11.999079  11.999078  11.999076  11.999075  ...  11.999051  11.999051   \n",
       "1  11.999079  11.999078  11.999076  11.999075  ...  11.999051  11.999051   \n",
       "2  11.999079  11.999078  11.999076  11.999075  ...  11.999051  11.999051   \n",
       "3  11.999079  11.999078  11.999076  11.999075  ...  11.999051  11.999051   \n",
       "4  11.999079  11.999078  11.999076  11.999075  ...  11.999051  11.999051   \n",
       "\n",
       "       t+36      t+37      t+38      t+39       t+40       t+41       t+42  \\\n",
       "0  11.99905  11.99905  11.99905  11.99905  11.999049  11.999049  11.999049   \n",
       "1  11.99905  11.99905  11.99905  11.99905  11.999049  11.999049  11.999049   \n",
       "2  11.99905  11.99905  11.99905  11.99905  11.999049  11.999049  11.999049   \n",
       "3  11.99905  11.99905  11.99905  11.99905  11.999049  11.999049  11.999049   \n",
       "4  11.99905  11.99905  11.99905  11.99905  11.999049  11.999049  11.999049   \n",
       "\n",
       "        t+43  \n",
       "0  11.999049  \n",
       "1  11.999049  \n",
       "2  11.999049  \n",
       "3  11.999049  \n",
       "4  11.999049  \n",
       "\n",
       "[5 rows x 46 columns]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(p50_forecast['identifier'].unique())\n",
    "p50_forecast.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-06-21T10:42:43.923120Z",
     "start_time": "2021-06-21T07:55:13.415982Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0       0.95      0.95      0.95     64900\n",
      "         2.0       0.97      0.97      0.97    172700\n",
      "         3.0       0.97      0.97      0.97    190476\n",
      "         4.0       0.95      0.95      0.95     62612\n",
      "         5.0       0.95      0.95      0.95     55044\n",
      "         6.0       1.00      1.00      1.00    766392\n",
      "         7.0       1.00      1.00      1.00    767580\n",
      "         8.0       0.98      0.98      0.98    695992\n",
      "         9.0       0.95      0.95      0.95    108900\n",
      "        10.0       0.95      0.95      0.95     58828\n",
      "        11.0       0.93      0.93      0.93    273108\n",
      "        12.0       0.98      0.98      0.98   3142700\n",
      "        13.0       0.98      0.98      0.98    363836\n",
      "        14.0       0.99      0.99      0.99    420464\n",
      "        15.0       1.00      1.00      1.00   2626668\n",
      "\n",
      "    accuracy                           0.99   9770200\n",
      "   macro avg       0.97      0.97      0.97   9770200\n",
      "weighted avg       0.99      0.99      0.99   9770200\n",
      "\n",
      "Done  54\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0       0.96      0.96      0.96     68508\n",
      "         2.0       0.95      0.95      0.95    129272\n",
      "         3.0       0.96      0.96      0.96    149072\n",
      "         4.0       0.94      0.94      0.94     51216\n",
      "         5.0       0.94      0.94      0.94     47564\n",
      "         6.0       1.00      1.00      1.00    775324\n",
      "         7.0       1.00      1.00      1.00    772464\n",
      "         8.0       0.98      0.98      0.98    694760\n",
      "         9.0       0.94      0.94      0.94    103796\n",
      "        10.0       0.95      0.95      0.95     60148\n",
      "        11.0       0.94      0.94      0.94    318032\n",
      "        12.0       0.99      0.99      0.99   3283060\n",
      "        13.0       0.98      0.98      0.98    338096\n",
      "        14.0       0.98      0.98      0.98    345224\n",
      "        15.0       1.00      1.00      1.00   2629132\n",
      "\n",
      "    accuracy                           0.99   9765668\n",
      "   macro avg       0.97      0.97      0.97   9765668\n",
      "weighted avg       0.99      0.99      0.99   9765668\n",
      "\n",
      "Done  55\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0       0.96      0.96      0.96     72864\n",
      "         2.0       0.96      0.96      0.96    165176\n",
      "         3.0       0.97      0.97      0.97    183920\n",
      "         4.0       0.96      0.96      0.96     66176\n",
      "         5.0       0.95      0.95      0.95     61776\n",
      "         6.0       1.00      1.00      1.00    783156\n",
      "         7.0       1.00      1.00      1.00    766304\n",
      "         8.0       0.98      0.98      0.98    632192\n",
      "         9.0       0.94      0.94      0.94     92752\n",
      "        10.0       0.95      0.95      0.95     61204\n",
      "        11.0       0.94      0.94      0.94    292644\n",
      "        12.0       0.99      0.99      0.99   3250236\n",
      "        13.0       0.98      0.98      0.98    335016\n",
      "        14.0       0.98      0.98      0.98    363352\n",
      "        15.0       1.00      1.00      1.00   2632212\n",
      "\n",
      "    accuracy                           0.99   9758980\n",
      "   macro avg       0.97      0.97      0.97   9758980\n",
      "weighted avg       0.99      0.99      0.99   9758980\n",
      "\n",
      "Done  56\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0       0.96      0.96      0.96     73700\n",
      "         2.0       0.97      0.97      0.97    191136\n",
      "         3.0       0.97      0.97      0.97    198220\n",
      "         4.0       0.95      0.95      0.95     57640\n",
      "         5.0       0.94      0.94      0.94     45936\n",
      "         6.0       1.00      1.00      1.00    761200\n",
      "         7.0       1.00      1.00      1.00    775148\n",
      "         8.0       0.98      0.98      0.98    681516\n",
      "         9.0       0.94      0.94      0.94    100496\n",
      "        10.0       0.94      0.94      0.94     53724\n",
      "        11.0       0.94      0.94      0.94    274560\n",
      "        12.0       0.99      0.99      0.99   3107500\n",
      "        13.0       0.98      0.98      0.98    387816\n",
      "        14.0       0.99      0.99      0.99    440836\n",
      "        15.0       1.00      1.00      1.00   2629880\n",
      "\n",
      "    accuracy                           0.99   9779308\n",
      "   macro avg       0.97      0.97      0.97   9779308\n",
      "weighted avg       0.99      0.99      0.99   9779308\n",
      "\n",
      "Done  57\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0       0.96      0.96      0.96     68596\n",
      "         1.0       0.93      0.93      0.93    103840\n",
      "         2.0       0.97      0.97      0.97    346764\n",
      "         3.0       0.97      0.97      0.97    322872\n",
      "         4.0       0.95      0.95      0.95     56408\n",
      "         5.0       0.94      0.94      0.94     46772\n",
      "         6.0       1.00      1.00      1.00    988372\n",
      "         7.0       1.00      1.00      1.00    838860\n",
      "         8.0       0.98      0.98      0.98    704440\n",
      "         9.0       0.93      0.93      0.93    121352\n",
      "        10.0       0.95      0.95      0.95     57860\n",
      "        11.0       0.98      0.98      0.98   1905222\n",
      "        12.0       0.99      0.99      0.99   4712730\n",
      "        13.0       0.99      0.99      0.99    531124\n",
      "        14.0       0.99      0.99      0.99    591448\n",
      "        15.0       1.00      1.00      1.00   5772008\n",
      "\n",
      "    accuracy                           0.99  17168668\n",
      "   macro avg       0.97      0.97      0.97  17168668\n",
      "weighted avg       0.99      0.99      0.99  17168668\n",
      "\n",
      "Done  58\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0       0.96      0.96      0.96     83292\n",
      "         1.0       0.94      0.94      0.94    123024\n",
      "         2.0       0.97      0.97      0.97    278124\n",
      "         3.0       0.96      0.96      0.96    242440\n",
      "         4.0       0.94      0.94      0.94     50248\n",
      "         5.0       0.92      0.92      0.92     39028\n",
      "         6.0       1.00      1.00      1.00    945340\n",
      "         7.0       1.00      1.00      1.00    856988\n",
      "         8.0       0.98      0.98      0.98    703824\n",
      "         9.0       0.93      0.93      0.93    119108\n",
      "        10.0       0.95      0.95      0.95     57376\n",
      "        11.0       0.99      0.99      0.99   2137146\n",
      "        12.0       0.99      0.99      0.99   5048318\n",
      "        13.0       0.98      0.98      0.98    526856\n",
      "        14.0       0.99      0.99      0.99    572484\n",
      "        15.0       1.00      1.00      1.00   5375568\n",
      "\n",
      "    accuracy                           0.99  17159164\n",
      "   macro avg       0.97      0.97      0.97  17159164\n",
      "weighted avg       0.99      0.99      0.99  17159164\n",
      "\n",
      "Done  59\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0       0.96      0.96      0.96     70752\n",
      "         1.0       0.94      0.94      0.94    112376\n",
      "         2.0       0.97      0.97      0.97    262328\n",
      "         3.0       0.96      0.96      0.96    212564\n",
      "         4.0       0.94      0.94      0.94     49896\n",
      "         5.0       0.94      0.94      0.94     49280\n",
      "         6.0       1.00      1.00      1.00   1009976\n",
      "         7.0       1.00      1.00      1.00    851180\n",
      "         8.0       0.98      0.98      0.98    687192\n",
      "         9.0       0.95      0.95      0.95    152636\n",
      "        10.0       0.95      0.95      0.95     61556\n",
      "        11.0       0.99      0.99      0.99   2022262\n",
      "        12.0       0.99      0.99      0.99   5015362\n",
      "        13.0       0.99      0.99      0.99    533280\n",
      "        14.0       0.99      0.99      0.99    597080\n",
      "        15.0       1.00      1.00      1.00   5473116\n",
      "\n",
      "    accuracy                           0.99  17160836\n",
      "   macro avg       0.97      0.97      0.97  17160836\n",
      "weighted avg       0.99      0.99      0.99  17160836\n",
      "\n",
      "Done  60\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0       0.95      0.95      0.95     61072\n",
      "         1.0       0.94      0.94      0.94    118536\n",
      "         2.0       0.97      0.97      0.97    269896\n",
      "         3.0       0.96      0.96      0.96    240504\n",
      "         4.0       0.94      0.94      0.94     47652\n",
      "         5.0       0.91      0.91      0.91     34584\n",
      "         6.0       1.00      1.00      1.00   1008480\n",
      "         7.0       1.00      1.00      1.00    859012\n",
      "         8.0       0.98      0.98      0.98    749716\n",
      "         9.0       0.94      0.94      0.94    138908\n",
      "        10.0       0.95      0.95      0.95     54252\n",
      "        11.0       0.99      0.99      0.99   1980726\n",
      "        12.0       0.99      0.99      0.99   4755454\n",
      "        13.0       0.99      0.99      0.99    562892\n",
      "        14.0       0.99      0.99      0.99    611424\n",
      "        15.0       1.00      1.00      1.00   5679256\n",
      "\n",
      "    accuracy                           0.99  17172364\n",
      "   macro avg       0.97      0.97      0.97  17172364\n",
      "weighted avg       0.99      0.99      0.99  17172364\n",
      "\n",
      "Done  61\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0       0.94      0.94      0.94     52888\n",
      "         1.0       0.93      0.93      0.93    104500\n",
      "         2.0       0.96      0.96      0.96    214544\n",
      "         3.0       0.96      0.96      0.96    236808\n",
      "         4.0       0.94      0.94      0.94     51524\n",
      "         5.0       0.93      0.93      0.93     42196\n",
      "         6.0       1.00      1.00      1.00    993036\n",
      "         7.0       1.00      1.00      1.00    880440\n",
      "         8.0       0.98      0.98      0.98    740124\n",
      "         9.0       0.94      0.94      0.94    132748\n",
      "        10.0       0.94      0.94      0.94     52360\n",
      "        11.0       0.99      0.99      0.99   2262678\n",
      "        12.0       0.99      0.99      0.99   4940210\n",
      "        13.0       0.99      0.99      0.99    549384\n",
      "        14.0       0.99      0.99      0.99    595496\n",
      "        15.0       1.00      1.00      1.00   5310272\n",
      "\n",
      "    accuracy                           0.99  17159208\n",
      "   macro avg       0.97      0.97      0.97  17159208\n",
      "weighted avg       0.99      0.99      0.99  17159208\n",
      "\n",
      "Done  63\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0       0.96      0.96      0.96     82280\n",
      "         1.0       0.93      0.93      0.93    103312\n",
      "         2.0       0.97      0.97      0.97    259952\n",
      "         3.0       0.97      0.97      0.97    272096\n",
      "         4.0       0.95      0.95      0.95     58916\n",
      "         5.0       0.92      0.92      0.92     37224\n",
      "         6.0       1.00      1.00      1.00    951544\n",
      "         7.0       1.00      1.00      1.00    841104\n",
      "         8.0       0.98      0.98      0.98    711568\n",
      "         9.0       0.94      0.94      0.94    138864\n",
      "        10.0       0.95      0.95      0.95     61336\n",
      "        11.0       0.99      0.99      0.99   2242922\n",
      "        12.0       0.99      0.99      0.99   4884814\n",
      "        13.0       0.98      0.98      0.98    519112\n",
      "        14.0       0.99      0.99      0.99    574772\n",
      "        15.0       1.00      1.00      1.00   5387052\n",
      "\n",
      "    accuracy                           0.99  17126868\n",
      "   macro avg       0.97      0.97      0.97  17126868\n",
      "weighted avg       0.99      0.99      0.99  17126868\n",
      "\n",
      "Done  64\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0       0.96      0.96      0.96     79112\n",
      "         1.0       0.94      0.94      0.94    113036\n",
      "         2.0       0.97      0.97      0.97    319572\n",
      "         3.0       0.97      0.97      0.97    275616\n",
      "         4.0       0.95      0.95      0.95     57772\n",
      "         5.0       0.93      0.93      0.93     43076\n",
      "         6.0       1.00      1.00      1.00    991232\n",
      "         7.0       1.00      1.00      1.00    873708\n",
      "         8.0       0.98      0.98      0.98    664840\n",
      "         9.0       0.94      0.94      0.94    131252\n",
      "        10.0       0.96      0.96      0.96     68332\n",
      "        11.0       0.99      0.99      0.99   2312442\n",
      "        12.0       0.99      0.99      0.99   4855598\n",
      "        13.0       0.98      0.98      0.98    476476\n",
      "        14.0       0.98      0.98      0.98    513700\n",
      "        15.0       1.00      1.00      1.00   5379132\n",
      "\n",
      "    accuracy                           0.99  17154896\n",
      "   macro avg       0.97      0.97      0.97  17154896\n",
      "weighted avg       0.99      0.99      0.99  17154896\n",
      "\n",
      "Done  65\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0       0.96      0.96      0.96     81180\n",
      "         1.0       0.93      0.93      0.93    104324\n",
      "         2.0       0.97      0.97      0.97    282568\n",
      "         3.0       0.97      0.97      0.97    261668\n",
      "         4.0       0.93      0.93      0.93     40392\n",
      "         5.0       0.92      0.92      0.92     39556\n",
      "         6.0       1.00      1.00      1.00    977636\n",
      "         7.0       1.00      1.00      1.00    891132\n",
      "         8.0       0.98      0.98      0.98    710556\n",
      "         9.0       0.95      0.95      0.95    151580\n",
      "        10.0       0.95      0.95      0.95     54076\n",
      "        11.0       0.99      0.99      0.99   2372634\n",
      "        12.0       0.99      0.99      0.99   5130862\n",
      "        13.0       0.98      0.98      0.98    393756\n",
      "        14.0       0.98      0.98      0.98    499356\n",
      "        15.0       1.00      1.00      1.00   5173564\n",
      "\n",
      "    accuracy                           0.99  17164840\n",
      "   macro avg       0.97      0.97      0.97  17164840\n",
      "weighted avg       0.99      0.99      0.99  17164840\n",
      "\n",
      "Done  66\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0       0.96      0.96      0.96     80960\n",
      "         1.0       0.93      0.93      0.93     94864\n",
      "         2.0       0.97      0.97      0.97    288596\n",
      "         3.0       0.97      0.97      0.97    294184\n",
      "         4.0       0.94      0.94      0.94     48048\n",
      "         5.0       0.92      0.92      0.92     38500\n",
      "         6.0       1.00      1.00      1.00    904200\n",
      "         7.0       1.00      1.00      1.00    862532\n",
      "         8.0       0.98      0.98      0.98    700040\n",
      "         9.0       0.94      0.94      0.94    142164\n",
      "        10.0       0.96      0.96      0.96     72380\n",
      "        11.0       0.99      0.99      0.99   2134110\n",
      "        12.0       0.99      0.99      0.99   4592610\n",
      "        13.0       0.99      0.99      0.99    552992\n",
      "        14.0       0.99      0.99      0.99    648032\n",
      "        15.0       1.00      1.00      1.00   5703280\n",
      "\n",
      "    accuracy                           0.99  17157492\n",
      "   macro avg       0.97      0.97      0.97  17157492\n",
      "weighted avg       0.99      0.99      0.99  17157492\n",
      "\n",
      "Done  67\n"
     ]
    }
   ],
   "source": [
    "test_per_list = p50_forecast['identifier'].unique()\n",
    "save_dir = 'mobi_tft_wo_known_no_bioinfo/'\n",
    "total_acc, total_precision, total_recall, total_f1 = 0., 0., 0., 0.\n",
    "\n",
    "for num in test_per_list:\n",
    "    forecast = p50_forecast[p50_forecast['identifier'] == num]\n",
    "    target = targets[targets['identifier'] == num]\n",
    "\n",
    "    forecast = forecast.drop(['forecast_time', 'identifier'], axis=1)\n",
    "    target = target.drop(['forecast_time', 'identifier'], axis=1)\n",
    "    \n",
    "    acc, precision, recall, f1 = getScores(forecast, target)\n",
    "    \n",
    "    result = pd.DataFrame([acc, precision, recall, f1])\n",
    "    result = result.T\n",
    "    result.columns = ['Accuracy', 'Precision', 'Recall', 'F1 Score']\n",
    "    result.to_csv(save_dir + str(num) + 'result.csv')\n",
    "    \n",
    "    total_acc += acc\n",
    "    total_precision += precision\n",
    "    total_recall += recall\n",
    "    total_f1 += f1\n",
    "    \n",
    "    print('Done ', num)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-06-21T10:42:49.299891Z",
     "start_time": "2021-06-21T10:42:49.295446Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average Accuracy:  0.9883476832222431\n",
      "Average Precision:  0.9690480408325184\n",
      "Average Recall:  0.9690590630089096\n",
      "Average F1 Score:  0.969053548255309\n"
     ]
    }
   ],
   "source": [
    "print('Average Accuracy: ', total_acc / len(test_per_list))\n",
    "print('Average Precision: ', total_precision / len(test_per_list))\n",
    "print('Average Recall: ', total_recall / len(test_per_list))\n",
    "print('Average F1 Score: ', total_f1 / len(test_per_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": false,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}