Transformer Project.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "35202a3b-61f2-4613-b78c-936c20d33d3e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\qz199\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\torch\\nn\\modules\\transformer.py:286: UserWarning: enable_nested_tensor is True, but self.use_nested_tensor is False because encoder_layer.self_attn.batch_first was not True(use batch_first for better inference performance)\n",
      "  warnings.warn(f\"enable_nested_tensor is True, but self.use_nested_tensor is False because {why_not_sparsity_fast_path}\")\n",
      "C:\\Users\\qz199\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\torch\\nn\\modules\\loss.py:535: UserWarning: Using a target size (torch.Size([64])) that is different to the input size (torch.Size([64, 1])). This will likely lead to incorrect results due to broadcasting. Please ensure they have the same size.\n",
      "  return F.mse_loss(input, target, reduction=self.reduction)\n",
      "C:\\Users\\qz199\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\torch\\nn\\modules\\loss.py:535: UserWarning: Using a target size (torch.Size([51])) that is different to the input size (torch.Size([51, 1])). This will likely lead to incorrect results due to broadcasting. Please ensure they have the same size.\n",
      "  return F.mse_loss(input, target, reduction=self.reduction)\n",
      "C:\\Users\\qz199\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\torch\\nn\\modules\\loss.py:535: UserWarning: Using a target size (torch.Size([29])) that is different to the input size (torch.Size([29, 1])). This will likely lead to incorrect results due to broadcasting. Please ensure they have the same size.\n",
      "  return F.mse_loss(input, target, reduction=self.reduction)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1, Train Loss: 5.5283, Test Loss: 0.1181\n",
      "Epoch 2, Train Loss: 0.0513, Test Loss: 0.0172\n",
      "Epoch 3, Train Loss: 0.0144, Test Loss: 0.0087\n",
      "Epoch 4, Train Loss: 0.0118, Test Loss: 0.0035\n",
      "Epoch 5, Train Loss: 0.0088, Test Loss: 0.0021\n",
      "Epoch 6, Train Loss: 0.0077, Test Loss: 0.0021\n",
      "Epoch 7, Train Loss: 0.0076, Test Loss: 0.0025\n",
      "Epoch 8, Train Loss: 0.0076, Test Loss: 0.0019\n",
      "Epoch 9, Train Loss: 0.0086, Test Loss: 0.0043\n",
      "Epoch 10, Train Loss: 0.0074, Test Loss: 0.0023\n",
      "Epoch 11, Train Loss: 0.0067, Test Loss: 0.0034\n",
      "Epoch 12, Train Loss: 0.0070, Test Loss: 0.0021\n",
      "Epoch 13, Train Loss: 0.0066, Test Loss: 0.0019\n",
      "Epoch 14, Train Loss: 0.0066, Test Loss: 0.0021\n",
      "Epoch 15, Train Loss: 0.0074, Test Loss: 0.0057\n",
      "Epoch 16, Train Loss: 0.0077, Test Loss: 0.0028\n",
      "Epoch 17, Train Loss: 0.0070, Test Loss: 0.0020\n",
      "Epoch 18, Train Loss: 0.0059, Test Loss: 0.0029\n",
      "Epoch 19, Train Loss: 0.0059, Test Loss: 0.0044\n",
      "Epoch 20, Train Loss: 0.0064, Test Loss: 0.0022\n",
      "Epoch 21, Train Loss: 0.0074, Test Loss: 0.0083\n",
      "Epoch 22, Train Loss: 0.0089, Test Loss: 0.0021\n",
      "Epoch 23, Train Loss: 0.0060, Test Loss: 0.0019\n",
      "Epoch 24, Train Loss: 0.0067, Test Loss: 0.0046\n",
      "Epoch 25, Train Loss: 0.0069, Test Loss: 0.0034\n",
      "Epoch 26, Train Loss: 0.0085, Test Loss: 0.0042\n",
      "Epoch 27, Train Loss: 0.0071, Test Loss: 0.0058\n",
      "Epoch 28, Train Loss: 0.0061, Test Loss: 0.0024\n",
      "Epoch 29, Train Loss: 0.0051, Test Loss: 0.0022\n",
      "Epoch 30, Train Loss: 0.0050, Test Loss: 0.0041\n",
      "Epoch 31, Train Loss: 0.0059, Test Loss: 0.0028\n",
      "Epoch 32, Train Loss: 0.0055, Test Loss: 0.0019\n",
      "Epoch 33, Train Loss: 0.0049, Test Loss: 0.0022\n",
      "Epoch 34, Train Loss: 0.0048, Test Loss: 0.0022\n",
      "Epoch 35, Train Loss: 0.0050, Test Loss: 0.0020\n",
      "Epoch 36, Train Loss: 0.0043, Test Loss: 0.0020\n",
      "Epoch 37, Train Loss: 0.0046, Test Loss: 0.0026\n",
      "Epoch 38, Train Loss: 0.0043, Test Loss: 0.0019\n",
      "Epoch 39, Train Loss: 0.0043, Test Loss: 0.0019\n",
      "Epoch 40, Train Loss: 0.0042, Test Loss: 0.0025\n",
      "Epoch 41, Train Loss: 0.0046, Test Loss: 0.0019\n",
      "Epoch 42, Train Loss: 0.0043, Test Loss: 0.0032\n",
      "Epoch 43, Train Loss: 0.0044, Test Loss: 0.0019\n",
      "Epoch 44, Train Loss: 0.0041, Test Loss: 0.0020\n",
      "Epoch 45, Train Loss: 0.0054, Test Loss: 0.0034\n",
      "Epoch 46, Train Loss: 0.0050, Test Loss: 0.0034\n",
      "Epoch 47, Train Loss: 0.0046, Test Loss: 0.0032\n",
      "Epoch 48, Train Loss: 0.0050, Test Loss: 0.0018\n",
      "Epoch 49, Train Loss: 0.0043, Test Loss: 0.0018\n",
      "Epoch 50, Train Loss: 0.0040, Test Loss: 0.0022\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAioAAAHHCAYAAACRAnNyAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA+8ElEQVR4nO3deXxU5f3+/+vMJBmSkI01UCCAsiuoqJQiH6xBgSoKuKClStRKFVAo1W/lZ4WAtSAupW6IG9S6sCmWWkGBCioqqygK4lK2FjBSJGFLSGbu3x+TGTJmnUnmnAm+njzmkZkzZ86858yQXHPf97mPZYwxAgAAiEEupwsAAACoDEEFAADELIIKAACIWQQVAAAQswgqAAAgZhFUAABAzCKoAACAmEVQAQAAMYugAgAAYhZBBUCdatu2rXJycpwuA8ApgqACxKC5c+fKsixt2LDB6VLqncLCQv35z39Wr169lJaWpgYNGqhjx44aO3asvvzyS6fLAxCmOKcLAHBq2b59u1wuZ74DHThwQAMHDtTGjRt12WWX6Ze//KUaNmyo7du3a968eXr66ad14sQJR2oDEBmCCoBKlZSUyOfzKSEhocaP8Xg8Uayoajk5Ofr444+1aNEiXXnllSH33Xfffbrnnnvq5Hki2S8AIkPXD1CP/fe//9VNN92k5s2by+PxqFu3bnr++edD1jlx4oQmTZqknj17Ki0tTcnJyerbt6/eeeedkPV27twpy7L00EMPaebMmTrttNPk8Xi0detW5ebmyrIsff3118rJyVF6errS0tJ044036tixYyHb+eEYlUA31po1azRhwgQ1bdpUycnJGjp0qL777ruQx/p8PuXm5qply5ZKSkrSz3/+c23durVG417Wrl2rf/7zn7r55pvLhRTJH6Aeeuih4O0LL7xQF154Ybn1cnJy1LZt22r3y8cff6y4uDhNmTKl3Da2b98uy7L0+OOPB5cdOnRI48ePV+vWreXxeHT66afrgQcekM/nC3nsvHnz1LNnT6WkpCg1NVVnnnmm/vKXv1T52oFTGS0qQD317bff6qc//aksy9LYsWPVtGlTLV26VDfffLMKCgo0fvx4SVJBQYGeffZZXXfddbrlllt0+PBhPffccxowYIDWrVuns846K2S7c+bMUWFhoUaNGiWPx6NGjRoF77vmmmvUrl07TZs2TZs2bdKzzz6rZs2a6YEHHqi23ttvv10ZGRmaPHmydu7cqZkzZ2rs2LGaP39+cJ2JEydqxowZGjx4sAYMGKBPPvlEAwYMUGFhYbXbX7JkiSTp+uuvr8HeC98P90uLFi3Ur18/LViwQJMnTw5Zd/78+XK73br66qslSceOHVO/fv303//+V7/5zW/Upk0bffDBB5o4caL27dunmTNnSpKWL1+u6667TtnZ2cF9um3bNq1Zs0bjxo2LyusCYp4BEHPmzJljJJn169dXus7NN99sWrRoYQ4cOBCy/NprrzVpaWnm2LFjxhhjSkpKTFFRUcg633//vWnevLm56aabgst27NhhJJnU1FSTl5cXsv7kyZONpJD1jTFm6NChpnHjxiHLsrKyzMiRI8u9lv79+xufzxdc/tvf/ta43W5z6NAhY4wx+/fvN3FxcWbIkCEh28vNzTWSQrZZkaFDhxpJ5vvvv69yvYB+/fqZfv36lVs+cuRIk5WVFbxd1X6ZPXu2kWS2bNkSsrxr167moosuCt6+7777THJysvnyyy9D1rv77ruN2+02u3fvNsYYM27cOJOammpKSkpq9BqAHwO6foB6yBijV199VYMHD5YxRgcOHAheBgwYoPz8fG3atEmS5Ha7g2MpfD6fDh48qJKSEp177rnBdcq68sor1bRp0wqf99Zbbw253bdvX/3vf/9TQUFBtTWPGjVKlmWFPNbr9WrXrl2SpJUrV6qkpESjR48Oedztt99e7bYlBWtISUmp0frhqmi/DBs2THFxcSGtQp999pm2bt2q4cOHB5ctXLhQffv2VUZGRsh71b9/f3m9Xr377ruSpPT0dB09elTLly+PymsA6iOCClAPfffddzp06JCefvppNW3aNORy4403SpLy8vKC6//1r39V9+7d1aBBAzVu3FhNmzbVP//5T+Xn55fbdrt27Sp93jZt2oTczsjIkCR9//331dZc3WMDgeX0008PWa9Ro0bBdauSmpoqSTp8+HC160aiov3SpEkTZWdna8GCBcFl8+fPV1xcnIYNGxZc9tVXX2nZsmXl3qv+/ftLOvlejR49Wh07dtSgQYPUqlUr3XTTTVq2bFlUXg9QXzBGBaiHAgMwf/WrX2nkyJEVrtO9e3dJ0osvvqicnBwNGTJEd911l5o1aya3261p06bpm2++Kfe4xMTESp/X7XZXuNwYU23NtXlsTXTu3FmStGXLFvXt27fa9S3LqvC5vV5vhetXtl+uvfZa3Xjjjdq8ebPOOussLViwQNnZ2WrSpElwHZ/Pp4svvlj/7//9vwq30bFjR0lSs2bNtHnzZr311ltaunSpli5dqjlz5uiGG27QX//612pfE3AqIqgA9VDTpk2VkpIir9cb/FZemUWLFql9+/Z67bXXQrpefjgA1GlZWVmSpK+//jqk9eJ///tfjVpsBg8erGnTpunFF1+sUVDJyMjQv//973LLAy07NTVkyBD95je/CXb/fPnll5o4cWLIOqeddpqOHDlS7XslSQkJCRo8eLAGDx4sn8+n0aNHa/bs2br33nvLtTYBPwZ0/QD1kNvt1pVXXqlXX31Vn332Wbn7yx72G2jJKNt6sHbtWn344YfRLzQM2dnZiouL06xZs0KWlz3Etyq9e/fWwIED9eyzz+r1118vd/+JEyd05513Bm+fdtpp+uKLL0L21SeffKI1a9aEVXd6eroGDBigBQsWaN68eUpISNCQIUNC1rnmmmv04Ycf6q233ir3+EOHDqmkpESSP5SV5XK5gi1jRUVFYdUFnCpoUQFi2PPPP1/hGIVx48Zp+vTpeuedd9SrVy/dcsst6tq1qw4ePKhNmzZpxYoVOnjwoCTpsssu02uvvaahQ4fq0ksv1Y4dO/TUU0+pa9euOnLkiN0vqVLNmzfXuHHj9PDDD+vyyy/XwIED9cknn2jp0qVq0qRJSGtQZV544QVdcsklGjZsmAYPHqzs7GwlJyfrq6++0rx587Rv377gXCo33XSTHnnkEQ0YMEA333yz8vLy9NRTT6lbt241Ghxc1vDhw/WrX/1KTz75pAYMGKD09PSQ+++66y4tWbJEl112mXJyctSzZ08dPXpUW7Zs0aJFi7Rz5041adJEv/71r3Xw4EFddNFFatWqlXbt2qXHHntMZ511lrp06RJWTcCpgqACxLAfti4E5OTkqFWrVlq3bp2mTp2q1157TU8++aQaN26sbt26hcxrkpOTo/3792v27Nl666231LVrV7344otauHChVq1aZdMrqZkHHnhASUlJeuaZZ7RixQr17t1bb7/9ti644AI1aNCg2sc3bdpUH3zwgZ588knNnz9f99xzj06cOKGsrCxdfvnlIXORdOnSRS+88IImTZqkCRMmqGvXrvrb3/6ml19+Oez9cvnllysxMVGHDx8OOdonICkpSatXr9af/vQnLVy4UC+88IJSU1PVsWNHTZkyRWlpaZL8Y46efvppPfnkkzp06JAyMzM1fPhw5ebmOnZaAsBplqmrkWwAEAWHDh1SRkaG/vjHP9bZFPgA6g8iOoCYcfz48XLLArO2VjTdPYBTH10/AGLG/PnzNXfuXP3iF79Qw4YN9f777+uVV17RJZdcoj59+jhdHgAHEFQAxIzu3bsrLi5OM2bMUEFBQXCA7R//+EenSwPgEMaoAACAmMUYFQAAELMIKgAAIGbV6zEqPp9Pe/fuVUpKSo0mgwIAAM4zxujw4cNq2bJltXME1eugsnfvXrVu3drpMgAAQAT27NmjVq1aVblOvQ4qKSkpkvwvNHCKdwAAENsKCgrUunXr4N/xqtTroBLo7klNTSWoAABQz9Rk2AaDaQEAQMwiqAAAgJhFUAEAADGrXo9RAQCcWrxer4qLi50uA7UUHx8vt9tdJ9siqAAAHGeM0f79+3Xo0CGnS0EdSU9PV2ZmZq3nOSOoAAAcFwgpzZo1U1JSEpN41mPGGB07dkx5eXmSpBYtWtRqewQVAICjvF5vMKQ0btzY6XJQBxITEyVJeXl5atasWa26gRhMCwBwVGBMSlJSksOVoC4F3s/ajjkiqAAAYgLdPaeWuno/CSoAACBmEVQAAIghbdu21cyZM50uI2YQVAAAiIBlWVVecnNzI9ru+vXrNWrUqFrVduGFF2r8+PG12kas4KifChw/4dXBYycU57LUPLWB0+UAAGLQvn37gtfnz5+vSZMmafv27cFlDRs2DF43xsjr9Sourvo/u02bNq3bQus5WlQqsOzzfeoz/V/63YJPnC4FABCjMjMzg5e0tDRZlhW8/cUXXyglJUVLly5Vz5495fF49P777+ubb77RFVdcoebNm6thw4Y677zztGLFipDt/rDrx7IsPfvssxo6dKiSkpLUoUMHLVmypFa1v/rqq+rWrZs8Ho/atm2rhx9+OOT+J598Uh06dFCDBg3UvHlzXXXVVcH7Fi1apDPPPFOJiYlq3Lix+vfvr6NHj9aqnqrQolKBhNLjvU+U+ByuBAB+nIwxOl7stf15E+PddXr00d13362HHnpI7du3V0ZGhvbs2aNf/OIXuv/+++XxePTCCy9o8ODB2r59u9q0aVPpdqZMmaIZM2bowQcf1GOPPaYRI0Zo165datSoUdg1bdy4Uddcc41yc3M1fPhwffDBBxo9erQaN26snJwcbdiwQXfccYf+9re/6Wc/+5kOHjyo9957T5K/Fem6667TjBkzNHToUB0+fFjvvfeejDER76PqEFQqkBDnb2gq8hJUAMAJx4u96jrpLdufd+vUAUpKqLs/jVOnTtXFF18cvN2oUSP16NEjePu+++7T4sWLtWTJEo0dO7bS7eTk5Oi6666TJP3pT3/So48+qnXr1mngwIFh1/TII48oOztb9957rySpY8eO2rp1qx588EHl5ORo9+7dSk5O1mWXXaaUlBRlZWXp7LPPluQPKiUlJRo2bJiysrIkSWeeeWbYNYSDrp8KBIIKLSoAgNo499xzQ24fOXJEd955p7p06aL09HQ1bNhQ27Zt0+7du6vcTvfu3YPXk5OTlZqaGpyiPlzbtm1Tnz59Qpb16dNHX331lbxery6++GJlZWWpffv2uv766/XSSy/p2LFjkqQePXooOztbZ555pq6++mo988wz+v777yOqo6ZoUamAJxhU7G92BAD4u2C2Th3gyPPWpeTk5JDbd955p5YvX66HHnpIp59+uhITE3XVVVfpxIkTVW4nPj4+5LZlWfL5ovNlOiUlRZs2bdKqVav09ttva9KkScrNzdX69euVnp6u5cuX64MPPtDbb7+txx57TPfcc4/Wrl2rdu3aRaUeWlQqEGxRoesHABxhWZaSEuJsv0R7dtw1a9YoJydHQ4cO1ZlnnqnMzEzt3Lkzqs/5Q126dNGaNWvK1dWxY8fgOXni4uLUv39/zZgxQ59++ql27typf/3rX5L8702fPn00ZcoUffzxx0pISNDixYujVi8tKhVIcNP1AwCoex06dNBrr72mwYMHy7Is3XvvvVFrGfnuu++0efPmkGUtWrTQ7373O5133nm67777NHz4cH344Yd6/PHH9eSTT0qS3njjDf373//W//3f/ykjI0NvvvmmfD6fOnXqpLVr12rlypW65JJL1KxZM61du1bfffedunTpEpXXIBFUKuRhjAoAIAoeeeQR3XTTTfrZz36mJk2a6Pe//70KCgqi8lwvv/yyXn755ZBl9913n/7whz9owYIFmjRpku677z61aNFCU6dOVU5OjiQpPT1dr732mnJzc1VYWKgOHTrolVdeUbdu3bRt2za9++67mjlzpgoKCpSVlaWHH35YgwYNisprkCTLRPOYoigrKChQWlqa8vPzlZqaWmfb3fW/o+r34ColJ7j1+dTwR1QDAGqusLBQO3bsULt27dSgAZNsniqqel/D+fvNGJUKMEYFAIDYQFCpQGCMSrHXyOertw1OAADUewSVCgRaVCRaVQAAcBJBpQIEFQAAYgNBpQKBrh+JI38AAHASQaUClmUFw0oRQQUAAMcQVCrB+X4AAHAeQaUSBBUAAJxHUKkE0+gDAOA8gkolPPGBSd84gzIAAE4hqFSCwbQAgKpYllXlJTc3t1bbfv311+tsvfqMkxJWgjEqAICq7Nu3L3h9/vz5mjRpkrZv3x5c1rBhQyfKOuXQolIJggoAoCqZmZnBS1pamizLClk2b948denSRQ0aNFDnzp315JNPBh974sQJjR07Vi1atFCDBg2UlZWladOmSZLatm0rSRo6dKgsywreDpfP59PUqVPVqlUreTwenXXWWVq2bFmNajDGKDc3V23atJHH41HLli11xx13RLajaokWlUoEB9MyMy0A2M8YqfiY/c8bnyRZVq0389JLL2nSpEl6/PHHdfbZZ+vjjz/WLbfcouTkZI0cOVKPPvqolixZogULFqhNmzbas2eP9uzZI0lav369mjVrpjlz5mjgwIFyu90R1fCXv/xFDz/8sGbPnq2zzz5bzz//vC6//HJ9/vnn6tChQ5U1vPrqq/rzn/+sefPmqVu3btq/f78++eSTWu+XSBBUKkGLCgA4qPiY9KeW9j/v/7dXSkiu9WYmT56shx9+WMOGDZMktWvXTlu3btXs2bM1cuRI7d69Wx06dNAFF1wgy7KUlZUVfGzTpk0lSenp6crMzIy4hoceeki///3vde2110qSHnjgAb3zzjuaOXOmnnjiiSpr2L17tzIzM9W/f3/Fx8erTZs2Ov/88yOupTbo+qmEh6ACAIjA0aNH9c033+jmm29Ww4YNg5c//vGP+uabbyRJOTk52rx5szp16qQ77rhDb7/9dp3WUFBQoL1796pPnz4hy/v06aNt27ZVW8PVV1+t48ePq3379rrlllu0ePFilZSU1GmNNUWLSiWCLSp0/QCA/eKT/K0bTjxvLR05ckSS9Mwzz6hXr14h9wW6cc455xzt2LFDS5cu1YoVK3TNNdeof//+WrRoUa2fv6aqqqF169bavn27VqxYoeXLl2v06NF68MEHtXr1asXHx9tWo0RQqRQTvgGAgyyrTrpgnNC8eXO1bNlS//73vzVixIhK10tNTdXw4cM1fPhwXXXVVRo4cKAOHjyoRo0aKT4+Xt5azOOVmpqqli1bas2aNerXr19w+Zo1a0K6cKqqITExUYMHD9bgwYM1ZswYde7cWVu2bNE555wTcV2RIKhUItCiwjwqAIBwTZkyRXfccYfS0tI0cOBAFRUVacOGDfr+++81YcIEPfLII2rRooXOPvtsuVwuLVy4UJmZmUpPT5fkP/Jn5cqV6tOnjzwejzIyMip9rh07dmjz5s0hyzp06KC77rpLkydP1mmnnaazzjpLc+bM0ebNm/XSSy9JUpU1zJ07V16vV7169VJSUpJefPFFJSYmhoxjsQtBpRIEFQBApH79618rKSlJDz74oO666y4lJyfrzDPP1Pjx4yVJKSkpmjFjhr766iu53W6dd955evPNN+Vy+f/2PPzww5owYYKeeeYZ/eQnP9HOnTsrfa4JEyaUW/bee+/pjjvuUH5+vn73u98pLy9PXbt21ZIlS9ShQ4dqa0hPT9f06dM1YcIEeb1enXnmmfrHP/6hxo0b1/m+qo5ljDG2P2up3NxcTZkyJWRZp06d9MUXX9To8QUFBUpLS1N+fr5SU1PrtLap/9iq59fs0K39TtPdgzrX6bYBACcVFhZqx44dateunRo0aOB0OagjVb2v4fz9drxFpVu3blqxYkXwdlyc4yVJ4vBkAABigeOpIC4urlbHiUdL8PBkTkoIAIBjHJ9H5auvvlLLli3Vvn17jRgxQrt376503aKiIhUUFIRcooUWFQAAnOdoUOnVq5fmzp2rZcuWadasWdqxY4f69u2rw4cPV7j+tGnTlJaWFry0bt06arUx4RsAAM5zNKgMGjRIV199tbp3764BAwbozTff1KFDh7RgwYIK1584caLy8/ODl8A5CaKBCd8AwF4OHtuBKKir99PxMSplpaenq2PHjvr6668rvN/j8cjj8dhSCxO+AYA9AjOdHjt2TImJiQ5Xg7py7Jj/pJK1nck2poLKkSNH9M033+j66693uhTmUQEAm7jdbqWnpysvL0+SlJSUJKsOzmAMZxhjdOzYMeXl5Sk9PT3isz8HOBpU7rzzTg0ePFhZWVnau3evJk+eLLfbreuuu87JsiQxmBYA7BQ4+jMQVlD/1fbszwGOBpX//Oc/uu666/S///1PTZs21QUXXKCPPvooeIprJwW7fhijAgBRZ1mWWrRooWbNmqm4uNjpclBL8fHxtW5JCXA0qMybN8/Jp68SLSoAYD+3211nf+BwanB8HpVYRVABAMB5BJVKeBhMCwCA4wgqlUgobXqkRQUAAOcQVCrBhG8AADiPoFIJxqgAAOA8gkolONcPAADOI6hUomzXD+efAADAGQSVSgSCisQ4FQAAnEJQqURgZlqJ7h8AAJxCUKkEQQUAAOcRVCrhclmKd/vP3knXDwAAziCoVCF4YkJaVAAAcARBpQrMpQIAgLMIKlVI4Hw/AAA4iqBSBabRBwDAWQSVKgTGqBQVE1QAAHACQaUKCXGlZ1CmRQUAAEcQVKrAYFoAAJxFUKkCJyYEAMBZBJUqBIOK1+twJQAA/DgRVKrAhG8AADiLoFIFxqgAAOAsgkoVmPANAABnEVSqEOz64fBkAAAcQVCpAl0/AAA4i6BSBYIKAADOIqhUgaACAICzCCpV8DBGBQAARxFUqhA86oeTEgIA4AiCShWCXT+0qAAA4AiCShWYmRYAAGcRVKqQEOeWxIRvAAA4haBSBQ9dPwAAOIqgUoWThydz9mQAAJxAUKkC86gAAOAsgkoVOOoHAABnEVSq4OGoHwAAHEVQqQJdPwAAOIugUgWCCgAAziKoVIExKgAAOIugUoXAzLRM+AYAgDMIKlWg6wcAAGcRVKoQPHtyiU/GGIerAQDgx4egUgWP2x28XuwlqAAAYDeCShUCLSoSA2oBAHACQaUKIUGFcSoAANiOoFIFt8tSnMuSRFABAMAJBJVqcOQPAADOIahU4+Skb16HKwEA4MeHoFINJn0DAMA5MRNUpk+fLsuyNH78eKdLCUHXDwAAzomJoLJ+/XrNnj1b3bt3d7qUcggqAAA4x/GgcuTIEY0YMULPPPOMMjIynC6nnEDXD/OoAABgP8eDypgxY3TppZeqf//+1a5bVFSkgoKCkEu0eWhRAQDAMXFOPvm8efO0adMmrV+/vkbrT5s2TVOmTIlyVaHo+gEAwDmOtajs2bNH48aN00svvaQGDRrU6DETJ05Ufn5+8LJnz54oV1n28GSCCgAAdnOsRWXjxo3Ky8vTOeecE1zm9Xr17rvv6vHHH1dRUZHcZU4KKEkej0cej8fWOoOHJxcTVAAAsJtjQSU7O1tbtmwJWXbjjTeqc+fO+v3vf18upDgl0KJSRIsKAAC2cyyopKSk6IwzzghZlpycrMaNG5db7qSEOH9gYowKAAD2c/yon1jHUT8AADjH0aN+fmjVqlVOl1AOR/0AAOAcWlSqcXLCN05KCACA3Qgq1aDrBwAA5xBUqkHXDwAAziGoVINz/QAA4ByCSjWC86jQogIAgO0IKtWg6wcAAOcQVKpBUAEAwDkElWowRgUAAOcQVKoRHKPCSQkBALAdQaUawXlUaFEBAMB2BJVqMEYFAADnEFSq4eHsyQAAOIagUo0Eun4AAHAMQaUawaN+aFEBAMB2BJVqMDMtAADOIahU4+RgWq/DlQAA8ONDUKkGE74BAOAcgko1PByeDACAYwgq1Qh0/fiMVEKrCgAAtiKoVCMQVCS6fwAAsBtBpRqBMSoS3T8AANiNoFKNOLdLLst/naACAIC9CCo1wFwqAAA4g6BSA4HuH4IKAAD2IqjUQAInJgQAwBEElRrwcGJCAAAcQVCpASZ9AwDAGQSVGkggqAAA4AiCSg0Eg4qXExMCAGAngkoNBE9MSIsKAAC2IqjUAPOoAADgDIJKDTBGBQAAZxBUaiDY9cPhyQAA2IqgUgO0qAAA4AyCSg0QVAAAcAZBpQaY8A0AAGcQVGqAkxICAOAMgkoNJHCuHwAAHEFQqQEPZ08GAMARBJUaYMI3AACcQVCpAY76AQDAGQSVGmDCNwAAnEFQqYGTLSqcPRkAADsRVGqArh8AAJxBUKkBD4cnAwDgCIJKDQTHqNCiAgCArQgqNUDXDwAAziCo1ADzqAAA4AyCSg1weDIAAM5wNKjMmjVL3bt3V2pqqlJTU9W7d28tXbrUyZIqFGxRKSaoAABgJ0eDSqtWrTR9+nRt3LhRGzZs0EUXXaQrrrhCn3/+uZNllcNJCQEAcEack08+ePDgkNv333+/Zs2apY8++kjdunVzqKryPAymBQDAEY4GlbK8Xq8WLlyoo0ePqnfv3k6XE4KzJwMA4AzHg8qWLVvUu3dvFRYWqmHDhlq8eLG6du1a4bpFRUUqKioK3i4oKLClRrp+AABwhuNH/XTq1EmbN2/W2rVrddttt2nkyJHaunVrhetOmzZNaWlpwUvr1q1tqTFw1I/XZ+T1GVueEwAASJYxJqb+8vbv31+nnXaaZs+eXe6+ilpUWrdurfz8fKWmpkatpqNFJeo2+S1J0rapA5WY4I7acwEAcKorKChQWlpajf5+R9T1s2fPHlmWpVatWkmS1q1bp5dfflldu3bVqFGjItlkkM/nCwkjZXk8Hnk8nlptPxKBrh/JP06FoAIAgD0i6vr55S9/qXfeeUeStH//fl188cVat26d7rnnHk2dOrXG25k4caLeffdd7dy5U1u2bNHEiRO1atUqjRgxIpKyoibOZcmy/NeLvF5niwEA4EckoqDy2Wef6fzzz5ckLViwQGeccYY++OADvfTSS5o7d26Nt5OXl6cbbrhBnTp1UnZ2ttavX6+33npLF198cSRlRY1lWZyYEAAAB0TU9VNcXBzsglmxYoUuv/xySVLnzp21b9++Gm/nueeei+TpHZEQ51JRiY+gAgCAjSJqUenWrZueeuopvffee1q+fLkGDhwoSdq7d68aN25cpwXGCg+HKAMAYLuIgsoDDzyg2bNn68ILL9R1112nHj16SJKWLFkS7BI61dD1AwCA/SLq+rnwwgt14MABFRQUKCMjI7h81KhRSkpKqrPiYkkC0+gDAGC7iFpUjh8/rqKiomBI2bVrl2bOnKnt27erWbNmdVpgrAieQZmgAgCAbSIKKldccYVeeOEFSdKhQ4fUq1cvPfzwwxoyZIhmzZpVpwXGClpUAACwX0RBZdOmTerbt68kadGiRWrevLl27dqlF154QY8++midFhgrAicmpEUFAAD7RBRUjh07ppSUFEnS22+/rWHDhsnlcumnP/2pdu3aVacFxorgYFqO+gEAwDYRBZXTTz9dr7/+uvbs2aO33npLl1xyiST/BG7RPOeOk+j6AQDAfhEFlUmTJunOO+9U27Ztdf7556t3796S/K0rZ599dp0WGCsIKgAA2C+iw5OvuuoqXXDBBdq3b19wDhVJys7O1tChQ+usuFhyMqhwrh8AAOwSUVCRpMzMTGVmZuo///mPJKlVq1an7GRvkuRhjAoAALaLqOvH5/Np6tSpSktLU1ZWlrKyspSenq777rtPPt+p+Yecrh8AAOwXUYvKPffco+eee07Tp09Xnz59JEnvv/++cnNzVVhYqPvvv79Oi4wFBBUAAOwXUVD561//qmeffTZ41mRJ6t69u37yk59o9OjRp2ZQKe36KaLrBwAA20TU9XPw4EF17ty53PLOnTvr4MGDtS4qFtGiAgCA/SIKKj169NDjjz9ebvnjjz+u7t2717qoWERQAQDAfhF1/cyYMUOXXnqpVqxYEZxD5cMPP9SePXv05ptv1mmBsYKgAgCA/SJqUenXr5++/PJLDR06VIcOHdKhQ4c0bNgwff755/rb3/5W1zXGhOAYFYIKAAC2iXgelZYtW5YbNPvJJ5/oueee09NPP13rwmKNhxYVAABsF1GLyo9R4OzJTPgGAIB9CCo1xBgVAADsR1CpIYIKAAD2C2uMyrBhw6q8/9ChQ7WpJaYx4RsAAPYLK6ikpaVVe/8NN9xQq4JiFS0qAADYL6ygMmfOnGjVEfNOBhWvw5UAAPDjwRiVGgoGFbp+AACwDUGlhgJjVOj6AQDAPgSVGmLCNwAA7EdQqSEG0wIAYD+CSg0xRgUAAPsRVGooMEal2Gvk8xmHqwEA4MeBoFJDgRYViVYVAADsQlCpocBJCSWpiHEqAADYgqBSQ/FuK3idAbUAANiDoFJDlmUxoBYAAJsRVMLgYdI3AABsRVAJA3OpAABgL4JKGAgqAADYi6AShpNjVDiDMgAAdiCohCEw6RuHJwMAYA+CShjo+gEAwF4ElTAQVAAAsBdBJQyBrh/mUQEAwB4ElTDQogIAgL0IKmHwxDGYFgAAOxFUwkCLCgAA9iKohCFwBmWCCgAA9iCohIHBtAAA2IugEoYExqgAAGArgkoYGKMCAIC9HA0q06ZN03nnnaeUlBQ1a9ZMQ4YM0fbt250sqUoEFQAA7OVoUFm9erXGjBmjjz76SMuXL1dxcbEuueQSHT161MmyKnVyjAonJQQAwA5xTj75smXLQm7PnTtXzZo108aNG/V///d/DlVVOVpUAACwl6NB5Yfy8/MlSY0aNarw/qKiIhUVFQVvFxQU2FJXgIegAgCArWJmMK3P59P48ePVp08fnXHGGRWuM23aNKWlpQUvrVu3trXGYIsKhycDAGCLmAkqY8aM0WeffaZ58+ZVus7EiROVn58fvOzZs8fGCsuMUaFFBQAAW8RE18/YsWP1xhtv6N1331WrVq0qXc/j8cjj8dhYWSjmUQEAwF6OBhVjjG6//XYtXrxYq1atUrt27Zwsp1oMpgUAwF6OBpUxY8bo5Zdf1t///nelpKRo//79kqS0tDQlJiY6WVqFAl0/tKgAAGAPR8eozJo1S/n5+brwwgvVokWL4GX+/PlOllUpTzwnJQQAwE6Od/3UJ5yUEAAAe8XMUT/1AWNUAACwF0ElDEz4BgCAvQgqYWDCNwAA7EVQCQMTvgEAYC+CShgYowIAgL0IKmEo2/VT345YAgCgPiKohCEQVCTGqQAAYAeCShgCY1Qkun8AALADQSUMBBUAAOxFUAmDy2Up3m1JousHAAA7EFTCxCHKAADYh6ASpsCAWs6gDABA9BFUwuSJ4wzKAADYhaASJlpUAACwD0ElTMxOCwCAfQgqYQoOpuWoHwAAoo6gEiZaVAAAsA9BJUwEFQAA7ENQCZMneGJCr8OVAABw6iOohIkJ3wAAsA9BJUx0/QAAYB+CSpiYRwUAAPsQVMLE4ckAANiHoBImun4AALAPQSVMdP0AAGAfgkqYOCkhAAD2IaiEia4fAADsQ1AJk4egAgCAbQgqYeKoHwAA7ENQCRNdPwAA2IegEiaO+gEAwD4ElTDR9QMAgH0IKmE62fXD2ZMBAIg2gkqYGKMCAIB9CCphCgYVun4AAIg6gkqYPG5aVAAAsAtBJUx0/QAAYB+CSpgIKgAA2IegEibmUQEAwD4ElTBx9mQAAOxDUAlTsEWFo34AAIg6gkqYEsoc9WOMcbgaAABObQSVMAVaVCSp2EtQAQAgmggqYfKUCSpM+gYAQHQRVMIU6PqRGFALAEC0EVTC5HJZinNZkggqAABEG0ElAkz6BgCAPQgqETh5YkKvw5UAAHBqI6hEIDBOhdlpAQCILoJKBOj6AQDAHo4GlXfffVeDBw9Wy5YtZVmWXn/9dSfLqTGCCgAA9nA0qBw9elQ9evTQE0884WQZYQvOTss8KgAARFWck08+aNAgDRo0yMkSIuKJ95+YsKiYoAIAQDQ5GlTCVVRUpKKiouDtgoICR+rw0KICAIAt6tVg2mnTpiktLS14ad26tSN1MEYFAAB71KugMnHiROXn5wcve/bscaQOggoAAPaoV10/Ho9HHo/H6TJOzqNC1w8AAFFVr1pUYgUtKgAA2MPRFpUjR47o66+/Dt7esWOHNm/erEaNGqlNmzYOVlY1ggoAAPZwNKhs2LBBP//5z4O3J0yYIEkaOXKk5s6d61BV1SOoAABgD0eDyoUXXihjjJMlROTkhG+clBAAgGhijEoEPLSoAABgC4JKBOj6AQDAHgSVCHCuHwAA7EFQiUCgRaWIFhUAAKKKoBIBD0EFAABbEFQikBDnP3syY1QAAIgugkoEGEwLAIA9CCoRIKgAAGAPgkoEOOoHAAB7EFQiwIRvAADYg6ASAbp+AACwB0ElAsGgQtcPAABRRVCJQHCMCi0qAABEFUElAsxMCwCAPQgqETg5RsXrcCUAAJzaCCoR4PBkAADsQVCJAIcnAwBgD4JKBAJdPz4jldCqAgBA1BBUIuApPSmhxIBaAACiiaASgUCLikT3DwAA0URQiYDbZcntsiQxoBYAgGgiqESISd8AAIg+gkqEmPQNAIDoI6hEiBMTAgAQfQSVCDHpGwAA0UdQiRCTvgEAEH0ElQjR9QMAQPQRVCIUDCpeTkwIAEC0EFQixOHJAABEH0ElQhyeDABA9BFUIsQYFQAAoo+gEqHgUT8cngwAQNQQVCKUUHoG5aJiggoAANFCUIkQE74BABB9BJUIMUYFAIDoI6hEiJlpAQCIPoJKhBIYTAsAQNQRVCLEhG8AAEQfQSVCTPgGAED0EVQixGBaAACij6ASIQ5PBgAg+ggqETrZosLZkwEAiBaCSoTo+gEAIPoIKhHiXD8AAEQfQSVCHJ4MAED0EVQq8+kCqbiw0rs98QQVAACijaBSka1/l167RZrdV/rPhgpXSXCXnj2ZoAIAQNQQVCoSnyQ1bC4d+FJ67mJp+aRyrSsMpgUAIPoIKhXpcLE0+iOp+3DJ+KQ1fynXusLMtAAARB9BpTJJjaRhT0vXvlJh6woTvgEAEH0xEVSeeOIJtW3bVg0aNFCvXr20bt06p0s6qfMvKmxdSfnfZknSgSNFunLWB3pg2RdatT1PhwuLna0XAIBTiGWMMU4WMH/+fN1www166qmn1KtXL82cOVMLFy7U9u3b1axZsyofW1BQoLS0NOXn5ys1NTX6xX7xpvTGeOnItzKWS696hureQ5fquBoEV3FZ0hk/SdP5bRupV/vGOq9thtKTEqJfGwAA9UQ4f78dDyq9evXSeeedp8cff1yS5PP51Lp1a91+++26++67q3ys7UFFko4dlJbdLX06P7jI60rQcStJh3yJ+t7r0WGTpMNK0hEl6rBJkuVJVonLo2KXRyVWgkpcDVTi8sjrSpDX7b9u3AmyXG5Z7ji5XS65XG653XFyu11yud1yuUqvWy7JsuSyJFfpT8uSLMuSS5KrtI3MkiRjJBlZCrzFRlaZd9tY/jWNf23/o6zA2qXLvCWS8coyJZLPK5fPf1u+ElnGK8t4JcstudyS5ZZxuSUrTnK5ZLni/LddbrmMkWUZuYyRSz5Jxl+v5ZNlfHJZpZVaLhlT+tNyySdLslwycslYVumj/NVZlr/G0h/+igM3pLKvIvj6T171yeU7IctbLMtX7P9pSoLLXKZYlq9ExnLJqzj5LLe8Vpx8rjh5FSdjueUrvW3k8m/ZVfr+SJLlkmW5Sgt1yypdy+VS6ev233Zb/vfEsnxySbKMz79fZfz71pjS2/79JGPkc7nlk0s+yy2jOPks/3Wf5ZZPbvksS66SIrm9x+UqKVRcyTG5vIVyewsV5z1W+vO4ShSnYneiTlgenXA10AnLfyly+W8XySPjilec25LbcsntsuR2uxTndinOZcntcinOXfoZ9JbIMsWyfF5ZvpLSz0dx6Wem2N8aWfr++eQqc93yfwZLbxtj5DM+GZ9PxpjSnz75fEZGPsnnkyUjt3xyWz7594R/S6V7RW75Sj93cfJZ8TLuBBl3vHxWvOROkHHFS+44GVecJCv4/8j/nvnfH8tlyf+vtErjPfnelP5U2euSvHIHX5OxLP9P45K3zGdasiSXJSvwWbZc/uuln3HLsoL/X12W8ddX+j/UKr39g4946ee8goXBT70JrlX+l33p+6dAaf4bgf9frsAWjCn972NkjP//r0zpJWRz/tdiKfCLyf9eW6W/Zyqq/YfVWiZQ6cntW8HrJvh6jAn8vrKCr81XWoP/FRhZxvj/rxuvFPifanyl75vv5ONLf98E3gf/bcv/O63Mvj35Ppzc55YlyRgZ4/V/Xn1eGa9Xxvg/u/KVyPj894e8V4H9XGbrllXmduD32w9v+4uQT25Jlv9/gOUu/T9klXkdpZ9hy/+M/r8NVvB9Duw1K7CvSl+HVfo+yJIs478/LT1d7dudVt2bF5Zw/n7H1ekzh+nEiRPauHGjJk6cGFzmcrnUv39/ffjhh+XWLyoqUlFRUfB2QUGBLXWGCIxd6TpEevNOqeC/cvtOqKFOqKEOqVVFnWne0gsAAPXMhpRstf/da449v6NB5cCBA/J6vWrevHnI8ubNm+uLL74ot/60adM0ZcoUu8qrWudfSB0HSkUFpZfDUmHpzzLLjhZ8r2NH8qXi47K8RVJxoawS/3WrpFAub6Fc3iJZ3hOl39J8wcSvMt+ug8vKKf8dqWwLSbA9xbLK3FeaoEu/pZy852Tri1X6TcZrxZ38th5y3e1P8XIFv1W65G9hcZV+03QZr//7s/HKyJKvNOWbQPuCVeZ6mYpdxqfS76P+7wvGyJJXrjKvv6LvhmXvK/O9ILg0ZP9YlkqsOHkV7/9pxctb9qcrXl65/d/QjVdulchlSuQ2JXIZr9zyyl16++R+LG0fKP3GGXh1lvGVftOxyi4NvheBVgVf6bek0lddur9cpd+YSn9KwZr8rQfe4H52l74HLvlUbHlKW0oSdcLlUbGrgX+Zq4H/usujOONVgilSgimUxxQq3lekBF+h4n2FwZ8uU1L+c1ZBI6xX7tL955ZX/p++0uv+z4o/wfvfW3Py/TWBVxVoabOC70/gm66/xUUqbesI7gtfaYubL7CPgq/eJUu+0veqRHGmuPS9Kva/l6ZYcaZErjLfHqyQ1xT6+gLb91kueQPtNlZw75e2hPi/vbvKVGOdfCfLvG7/Oy75WxcD3/ID++OHn9cffo7LfruvKavcVit4nVW0q4f8hrBCf1uU3foPf49YpS0aZX+v1MQPfhOVPm+Z61KZZz5ZhWVCHlH6/ybkXQheD/yUFGgDC753wevB96yq37EnBZ7P36ZXuhWrzHMFW3+qeO1lnir0eU3I8sDnyBX8/Vj2f0Fpi1EN93n5Lf/wc2cpzpNUo21Fi6NBJVwTJ07UhAkTgrcLCgrUunVr5wpyuaTEdP+lEsmlFwAA6qOzHH5+R4NKkyZN5Ha79e2334Ys//bbb5WZmVlufY/HI4/HY1d5AADAYY4enpyQkKCePXtq5cqVwWU+n08rV65U7969HawMAADEAse7fiZMmKCRI0fq3HPP1fnnn6+ZM2fq6NGjuvHGG50uDQAAOMzxoDJ8+HB99913mjRpkvbv36+zzjpLy5YtKzfAFgAA/Pg4Po9KbTgyjwoAAKiVcP5+x8QU+gAAABUhqAAAgJhFUAEAADGLoAIAAGIWQQUAAMQsggoAAIhZBBUAABCzCCoAACBmEVQAAEDMcnwK/doITKpbUFDgcCUAAKCmAn+3azI5fr0OKocPH5YktW7d2uFKAABAuA4fPqy0tLQq16nX5/rx+Xzau3evUlJSZFlWnW67oKBArVu31p49eziPkA3Y3/Zif9uL/W0v9re9ItnfxhgdPnxYLVu2lMtV9SiUet2i4nK51KpVq6g+R2pqKh90G7G/7cX+thf7217sb3uFu7+ra0kJYDAtAACIWQQVAAAQswgqlfB4PJo8ebI8Ho/TpfwosL/txf62F/vbXuxve0V7f9frwbQAAODURosKAACIWQQVAAAQswgqAAAgZhFUAABAzCKoVOCJJ55Q27Zt1aBBA/Xq1Uvr1q1zuqRTwrvvvqvBgwerZcuWsixLr7/+esj9xhhNmjRJLVq0UGJiovr376+vvvrKmWJPAdOmTdN5552nlJQUNWvWTEOGDNH27dtD1iksLNSYMWPUuHFjNWzYUFdeeaW+/fZbhyqu32bNmqXu3bsHJ73q3bu3li5dGryffR1d06dPl2VZGj9+fHAZ+7zu5ObmyrKskEvnzp2D90dzXxNUfmD+/PmaMGGCJk+erE2bNqlHjx4aMGCA8vLynC6t3jt69Kh69OihJ554osL7Z8yYoUcffVRPPfWU1q5dq+TkZA0YMECFhYU2V3pqWL16tcaMGaOPPvpIy5cvV3FxsS655BIdPXo0uM5vf/tb/eMf/9DChQu1evVq7d27V8OGDXOw6vqrVatWmj59ujZu3KgNGzbooosu0hVXXKHPP/9cEvs6mtavX6/Zs2ere/fuIcvZ53WrW7du2rdvX/Dy/vvvB++L6r42CHH++eebMWPGBG97vV7TsmVLM23aNAerOvVIMosXLw7e9vl8JjMz0zz44IPBZYcOHTIej8e88sorDlR46snLyzOSzOrVq40x/v0bHx9vFi5cGFxn27ZtRpL58MMPnSrzlJKRkWGeffZZ9nUUHT582HTo0MEsX77c9OvXz4wbN84Yw+e7rk2ePNn06NGjwvuiva9pUSnjxIkT2rhxo/r37x9c5nK51L9/f3344YcOVnbq27Fjh/bv3x+y79PS0tSrVy/2fR3Jz8+XJDVq1EiStHHjRhUXF4fs886dO6tNmzbs81ryer2aN2+ejh49qt69e7Ovo2jMmDG69NJLQ/atxOc7Gr766iu1bNlS7du314gRI7R7925J0d/X9fqkhHXtwIED8nq9at68ecjy5s2b64svvnCoqh+H/fv3S1KF+z5wHyLn8/k0fvx49enTR2eccYYk/z5PSEhQenp6yLrs88ht2bJFvXv3VmFhoRo2bKjFixera9eu2rx5M/s6CubNm6dNmzZp/fr15e7j8123evXqpblz56pTp07at2+fpkyZor59++qzzz6L+r4mqAA/AmPGjNFnn30W0qeMutepUydt3rxZ+fn5WrRokUaOHKnVq1c7XdYpac+ePRo3bpyWL1+uBg0aOF3OKW/QoEHB6927d1evXr2UlZWlBQsWKDExMarPTddPGU2aNJHb7S43Uvnbb79VZmamQ1X9OAT2L/u+7o0dO1ZvvPGG3nnnHbVq1Sq4PDMzUydOnNChQ4dC1mefRy4hIUGnn366evbsqWnTpqlHjx76y1/+wr6Ogo0bNyovL0/nnHOO4uLiFBcXp9WrV+vRRx9VXFycmjdvzj6PovT0dHXs2FFff/111D/fBJUyEhIS1LNnT61cuTK4zOfzaeXKlerdu7eDlZ362rVrp8zMzJB9X1BQoLVr17LvI2SM0dixY7V48WL961//Urt27ULu79mzp+Lj40P2+fbt27V79272eR3x+XwqKipiX0dBdna2tmzZos2bNwcv5557rkaMGBG8zj6PniNHjuibb75RixYtov/5rvVw3FPMvHnzjMfjMXPnzjVbt241o0aNMunp6Wb//v1Ol1bvHT582Hz88cfm448/NpLMI488Yj7++GOza9cuY4wx06dPN+np6ebvf/+7+fTTT80VV1xh2rVrZ44fP+5w5fXTbbfdZtLS0syqVavMvn37gpdjx44F17n11ltNmzZtzL/+9S+zYcMG07t3b9O7d28Hq66/7r77brN69WqzY8cO8+mnn5q7777bWJZl3n77bWMM+9oOZY/6MYZ9Xpd+97vfmVWrVpkdO3aYNWvWmP79+5smTZqYvLw8Y0x09zVBpQKPPfaYadOmjUlISDDnn3+++eijj5wu6ZTwzjvvGEnlLiNHjjTG+A9Rvvfee03z5s2Nx+Mx2dnZZvv27c4WXY9VtK8lmTlz5gTXOX78uBk9erTJyMgwSUlJZujQoWbfvn3OFV2P3XTTTSYrK8skJCSYpk2bmuzs7GBIMYZ9bYcfBhX2ed0ZPny4adGihUlISDA/+clPzPDhw83XX38dvD+a+9oyxpjat8sAAADUPcaoAACAmEVQAQAAMYugAgAAYhZBBQAAxCyCCgAAiFkEFQAAELMIKgAAIGYRVADUe5Zl6fXXX3e6DABRQFABUCs5OTmyLKvcZeDAgU6XBuAUEOd0AQDqv4EDB2rOnDkhyzwej0PVADiV0KICoNY8Ho8yMzNDLhkZGZL83TKzZs3SoEGDlJiYqPbt22vRokUhj9+yZYsuuugiJSYmqnHjxho1apSOHDkSss7zzz+vbt26yePxqEWLFho7dmzI/QcOHNDQoUOVlJSkDh06aMmSJcH7vv/+e40YMUJNmzZVYmKiOnToUC5YAYhNBBUAUXfvvffqyiuv1CeffKIRI0bo2muv1bZt2yRJR48e1YABA5SRkaH169dr4cKFWrFiRUgQmTVrlsaMGaNRo0Zpy5YtWrJkiU4//fSQ55gyZYquueYaffrpp/rFL36hESNG6ODBg8Hn37p1q5YuXapt27Zp1qxZatKkiX07AEDk6uTUhgB+tEaOHGncbrdJTk4Oudx///3GGP9ZnG+99daQx/Tq1cvcdtttxhhjnn76aZORkWGOHDkSvP+f//yncblcZv/+/cYYY1q2bGnuueeeSmuQZP7whz8Ebx85csRIMkuXLjXGGDN48GBz44031s0LBmArxqgAqLWf//znmjVrVsiyRo0aBa/37t075L7evXtr8+bNkqRt27apR48eSk5ODt7fp08f+Xw+bd++XZZlae/evcrOzq6yhu7duwevJycnKzU1VXl5eZKk2267TVdeeaU2bdqkSy65REOGDNHPfvaziF4rAHsRVADUWnJycrmumLqSmJhYo/Xi4+NDbluWJZ/PJ0kaNGiQdu3apTfffFPLly9Xdna2xowZo4ceeqjO6wVQtxijAiDqPvroo3K3u3TpIknq0qWLPvnkEx09ejR4/5o1a+RyudSpUyelpKSobdu2WrlyZa1qaNq0qUaOHKkXX3xRM2fO1NNPP12r7QGwBy0qAGqtqKhI+/fvD1kWFxcXHLC6cOFCnXvuubrgggv00ksvad26dXruueckSSNGjNDkyZM1cuRI5ebm6rvvvtPtt9+u66+/Xs2bN5ck5ebm6tZbb1WzZs00aNAgHT58WGvWrNHtt99eo/omTZqknj17qlu3bioqKtIbb7wRDEoAYhtBBUCtLVu2TC1atAhZ1qlTJ33xxReS/EfkzJs3T6NHj1aLFi30yiuvqGvXrpKkpKQkvfXWWxo3bpzOO+88JSUl6corr9QjjzwS3NbIkSNVWFioP//5z7rzzjvVpEkTXXXVVTWuLyEhQRMnTtTOnTuVmJiovn37at68eXXwygFEm2WMMU4XAeDUZVmWFi9erCFDhjhdCoB6iDEqAAAgZhFUAABAzGKMCoCooncZQG3QogIAAGIWQQUAAMQsggoAAIhZBBUAABCzCCoAACBmEVQAAEDMIqgAAICYRVABAAAxi6ACAABi1v8PO1wOLV+wSNsAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from torch.utils.data import TensorDataset, DataLoader\n",
    "import torch.optim as optim\n",
    "import pandas as pd\n",
    "\n",
    "data = pd.read_excel('return.xlsx')\n",
    "\n",
    "# Dropping the date column and preparing features and target\n",
    "X = data.drop(columns=['Equity', 'Return'])\n",
    "y = data['Return']\n",
    "\n",
    "# Splitting the dataset into training and test sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "\n",
    "# Standardizing the features\n",
    "scaler = StandardScaler()\n",
    "X_train_scaled = scaler.fit_transform(X_train)\n",
    "X_test_scaled = scaler.transform(X_test)\n",
    "\n",
    "# Converting to PyTorch tensors\n",
    "X_train_tensor = torch.tensor(X_train_scaled, dtype=torch.float32)\n",
    "y_train_tensor = torch.tensor(y_train.values, dtype=torch.float32)\n",
    "X_test_tensor = torch.tensor(X_test_scaled, dtype=torch.float32)\n",
    "y_test_tensor = torch.tensor(y_test.values, dtype=torch.float32)\n",
    "\n",
    "# Creating dataloaders\n",
    "train_dataset = TensorDataset(X_train_tensor, y_train_tensor)\n",
    "test_dataset = TensorDataset(X_test_tensor, y_test_tensor)\n",
    "\n",
    "batch_size = 64\n",
    "train_loader = DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True)\n",
    "test_loader = DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=False)\n",
    "\n",
    "# Defining a simple Transformer model\n",
    "class TransformerModel(nn.Module):\n",
    "    def __init__(self, input_dim, num_heads, dim_feedforward, num_layers, output_dim):\n",
    "        super(TransformerModel, self).__init__()\n",
    "        self.linear_in = nn.Linear(input_dim, dim_feedforward)\n",
    "        encoder_layer = nn.TransformerEncoderLayer(d_model=dim_feedforward, nhead=num_heads)\n",
    "        self.transformer_encoder = nn.TransformerEncoder(encoder_layer, num_layers=num_layers)\n",
    "        self.linear_out = nn.Linear(dim_feedforward, output_dim)\n",
    "\n",
    "    def forward(self, src):\n",
    "        src = self.linear_in(src)\n",
    "        output = self.transformer_encoder(src)\n",
    "        output = self.linear_out(output)\n",
    "        return output\n",
    "\n",
    "# Model parameters\n",
    "input_dim = X_train.shape[1]\n",
    "num_heads = 2\n",
    "dim_feedforward = 512\n",
    "num_layers = 2\n",
    "output_dim = 1\n",
    "\n",
    "# Model, loss, and optimizer\n",
    "model = TransformerModel(input_dim, num_heads, dim_feedforward, num_layers, output_dim)\n",
    "criterion = nn.MSELoss()\n",
    "optimizer = optim.Adam(model.parameters(), lr=0.001)\n",
    "\n",
    "import torch\n",
    "\n",
    "def train(model, criterion, optimizer, train_loader, test_loader, epochs):\n",
    "    model.train()\n",
    "    train_losses = []\n",
    "    test_losses = []\n",
    "    y_predicts = []  # Store predictions for each epoch\n",
    "    \n",
    "    for epoch in range(epochs):\n",
    "        total_loss = 0\n",
    "        for X_batch, y_batch in train_loader:\n",
    "            optimizer.zero_grad()\n",
    "            output = model(X_batch)\n",
    "            loss = criterion(output, y_batch)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            total_loss += loss.item()\n",
    "        train_losses.append(total_loss / len(train_loader))\n",
    "\n",
    "        # Validation loss and predictions\n",
    "        model.eval()\n",
    "        total_loss_test = 0\n",
    "        all_predictions = []\n",
    "        with torch.no_grad():\n",
    "            for X_batch, y_batch in test_loader:\n",
    "                output = model(X_batch)\n",
    "                loss = criterion(output, y_batch)\n",
    "                total_loss_test += loss.item()\n",
    "                all_predictions.extend(output.detach().cpu().numpy())\n",
    "                \n",
    "        test_losses.append(total_loss_test / len(test_loader))\n",
    "        y_predicts.append(all_predictions)  # Store predictions for this epoch\n",
    "        \n",
    "        model.train()\n",
    "        \n",
    "        print(f'Epoch {epoch+1}, Train Loss: {train_losses[-1]:.4f}, Test Loss: {test_losses[-1]:.4f}')\n",
    "        \n",
    "    return train_losses, test_losses, y_predicts\n",
    "\n",
    "\n",
    "# Training the model\n",
    "epochs = 50\n",
    "train_losses, test_losses,another_value = train(model, criterion, optimizer, train_loader, test_loader, epochs)\n",
    "\n",
    "# Plotting the learning curves\n",
    "plt.plot(train_losses, label='Train Loss')\n",
    "plt.plot(test_losses, label='Test Loss')\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('Loss')\n",
    "plt.title('Learning Curves')\n",
    "plt.legend()\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4c60a745-987b-40b3-92da-77d203c97288",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test, y_predictor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b9e0dc55-aaaf-484d-9029-88fb80b21be2",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestRegressor\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "# Using previously prepared training data\n",
    "\n",
    "# Training random forest model\n",
    "rf = RandomForestRegressor(n_estimators=100, random_state=42)\n",
    "rf.fit(X_train, y_train)\n",
    "\n",
    "# Ranking feature by importance\n",
    "feature_importances = rf.feature_importances_\n",
    "sorted_idx = np.argsort(feature_importances)[::-1]\n",
    "\n",
    "# Visualizing importance\n",
    "plt.figure(figsize=(8, 6))\n",
    "sns.barplot(x=feature_importances[sorted_idx], y=X.columns[sorted_idx])\n",
    "plt.title('Feature Importance')\n",
    "plt.xlabel('Importance')\n",
    "plt.ylabel('Features')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eea6e15d-3382-4f87-921a-bbc5baba5b7e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Calculating correlation between features\n",
    "correlation_matrix = X.corr()\n",
    "\n",
    "# Seaborn heatmap\n",
    "plt.figure(figsize=(8, 6))  \n",
    "sns.heatmap(correlation_matrix, annot=False, cmap='coolwarm', fmt=\".2f\")\n",
    "plt.title('Last Encoder Layer Self-Attention Heatmap')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bddf47e3-d1c1-4f65-b658-e9958eb4ad2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.metrics import r2_score\n",
    "from sklearn.inspection import permutation_importance\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "\n",
    "# Loading data\n",
    "data = pd.read_excel('AAPL_with_return.xlsx')\n",
    "X = data.drop(columns=['AAPL UW Equity', 'Return'])\n",
    "y = data['Return']\n",
    "\n",
    "# Train/test split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "\n",
    "# Training\n",
    "model = RandomForestRegressor(random_state=42)\n",
    "model.fit(X_train, y_train)\n",
    "\n",
    "# Using R-squared as benchmark, random forest model as example\n",
    "base_score = model.score(X_test, y_test)\n",
    "print(f\"Base R² score: {base_score:.3f}\")\n",
    "\n",
    "# Scoring and ranking\n",
    "results = permutation_importance(model, X_test, y_test, n_repeats=10, random_state=42, n_jobs=-1)\n",
    "\n",
    "# Visualize\n",
    "feature_names = X.columns\n",
    "forest_importances = pd.Series(results.importances_mean, index=feature_names)\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "forest_importances.plot.bar(yerr=results.importances_std, ax=ax)\n",
    "ax.set_title(\"Feature importances using permutation on full model\")\n",
    "ax.set_ylabel(\"Mean accuracy decrease\")\n",
    "fig.tight_layout()\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ab638845-e871-4458-835d-66b6700c690c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "import pandas as pd\n",
    "model = LinearRegression()\n",
    "model.fit(X_train, y_train)\n",
    "base_score = model.score(X_test, y_test)\n",
    "\n",
    "# Saving R-squared changes for each feature\n",
    "r_squared_changes = {}\n",
    "\n",
    "for feature in X.columns:\n",
    "    X_train_temp = X_train.drop(columns=[feature])\n",
    "    X_test_temp = X_test.drop(columns=[feature])\n",
    "    \n",
    "    model_temp = LinearRegression()\n",
    "    model_temp.fit(X_train_temp, y_train)\n",
    "    score_temp = model_temp.score(X_test_temp, y_test)\n",
    "    \n",
    "    # Calculate difference in R-squared after feature is removed \n",
    "    r_squared_changes[feature] = base_score - score_temp\n",
    "\n",
    "# Printing\n",
    "for feature, change in r_squared_changes.items():\n",
    "    print(f\"{feature}: R^2 change = {change:.3f}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}