ocr/2.im2latex.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget http://baidudeeplearning.bj.bcebos.com/image_contest_level_1.tar.gz\n",
    "# !tar -zxf image_contest_level_1.tar.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "from skimage.transform import resize as imresize\n",
    "import cv2\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ['CUDA_VISIBLE_DEVICES'] = '1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "100000"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "directory = 'image_contest_level_1/'\n",
    "images = ['%d.png'%(d) for d in range(100000)]\n",
    "with open(directory+'labels.txt','r') as fopen:\n",
    "    labels = [i.split()[0] for i in list(filter(None,fopen.read().split('\\n')))]\n",
    "len(images)\n",
    "len(labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "charset = list(set(''.join(labels)))\n",
    "num_classes = len(charset) + 2\n",
    "encode_maps = {}\n",
    "decode_maps = {}\n",
    "for i, char in enumerate(charset, 3):\n",
    "    encode_maps[char] = i\n",
    "    decode_maps[i] = char\n",
    "    \n",
    "SPACE_INDEX = 0\n",
    "SPACE_TOKEN = '<PAD>'\n",
    "encode_maps[SPACE_TOKEN] = SPACE_INDEX\n",
    "decode_maps[SPACE_INDEX] = SPACE_TOKEN\n",
    "\n",
    "GO_INDEX = 1\n",
    "GO_TOKEN = '<GO>'\n",
    "encode_maps[GO_TOKEN] = GO_INDEX\n",
    "decode_maps[GO_INDEX] = GO_TOKEN\n",
    "\n",
    "EOS_INDEX = 2\n",
    "EOS_TOKEN = '<EOS>'\n",
    "encode_maps[EOS_TOKEN] = EOS_INDEX\n",
    "decode_maps[EOS_INDEX] = EOS_TOKEN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "GO = 1\n",
    "PAD = 0\n",
    "EOS = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "image_height = 60\n",
    "image_width = 240\n",
    "image_channel = 1\n",
    "max_stepsize = 128\n",
    "num_hidden = 256\n",
    "epoch = 20\n",
    "batch_size = 128\n",
    "initial_learning_rate = 1e-3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100000/100000 [02:42<00:00, 614.11it/s]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "X, Y = [], []\n",
    "for i in tqdm(range(len(images))):\n",
    "    img = images[i]\n",
    "    X.append(imresize(cv2.imread(directory+img, 0).astype(np.float32)/255., (image_height,image_width)))\n",
    "    Y.append([encode_maps[c] for c in labels[i]] + [2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "train_X, test_X, train_Y, test_Y = train_test_split(X, Y, test_size = 0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['5', '-', '5', '*', '1', '<EOS>']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[decode_maps[c] for c in Y[-1]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7f40178e3fd0>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAB2CAYAAADRN8iWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO2da3CU15nnf6db3bq0Wt2tGxISQlzEXYANJgaMb0nsGF8AJ7YZx05qJhPPVM1UzdTO1k5m58t82K3a/bAzO1u1mxpnZmJ7M14ncZw4ZuxccHA5wRiwAwEcIRACGYHQvaXuFuqL+t0Pr87D+woJC9Ct4fyqKFp9e0+ffvt5n/M/z0VZloXBYDAYcg/PbA/AYDAYDDeGMeAGg8GQoxgDbjAYDDmKMeAGg8GQoxgDbjAYDDmKMeAGg8GQo9yUAVdKfUkp1ayUalFKfWuqBmUwGAyGz0bdaBy4UsoLnAK+CLQDh4E/sCzr91M3PIPBYDBMxM144JuAFsuyWi3LSgGvATumZlgGg8Fg+CzybuK1NcB5x9/twOeu9YLy8nKrvr7+Jg5pMEw9qVSK1tZWAObNm0ckEpnlERkMbj7++OMey7Iqxt5/MwZ8UiilXgBeAKirq+Ojjz6a7kMaDBOSzWaJxWJ0dHQAsH//fvbs2cOSJUsAqKysZOvWrQDs2LGDYDCIx2P2+g2zi1Kqbbz7b+bMvAAscPxdO3qfC8uyXrQsa6NlWRsrKq66gBgMBoPhBrkZD/ww0KCUWoRtuHcDz07JqAyGaebQoUOu/zWbNm1i06ZNszEkg+G6uWEDbllWRin158DPAS/wr5ZlfTJlIzMYphFtpAsKCjhx4gQPPPAAAIsXL6asrAyA4uLiWRufwTAZbkoDtyzrbeDtKRqLwWAwGK6Dad/ENBjmEh6Ph1AohNfrBexNyy984QvyeHFxMX6/f7aGZ5hm9CY2wMjIiNzv9XpzcsPaGPAcIh6PMzw87LovGAwCkJ+fPxtDylmMPHL7kUqliEajtLe3A/DBBx/IY1u2bKG2tpZwOAyQMxdxY8DnKMlkEkC8BYBLly5x8uRJmpqaAFi5cqXotcFgkJKSEkKhEADhcNgYdYMB2/EB6O3t5dSpU7z66qsARKNRcYD6+/t57rnnxIDnCrm1XjAYDAaDYDzwOYr2wD/99FPeftveJ87Ly6Orq4vDhw8D8Oqrr1JQUABAKBSiurqaRx99FICqqip5zOPxoJQS2SAUChEKhcT78Hg8V3n8mUwGsLXBwsJCGZfP58Pn88nrDIZc4ezZs+zZs4dEIiH3LVq0CIBHH32UQCAwW0O7YYwBn6OkUinAXtr19PQAcPDgQS5evCjGtr+/X57n8/koKSnhww8/BGwJRRtwr9eLz+eTFPHGxka2bt1KSUkJ4NaDOzo62L9/P/PmzQNsY683e6qqqli4cCFVVVUA8nqDIRdYtGgRO3fuZN++fQCsWbOGtWvXAlBdXZ2Tm5i5NVqDwWAwCMYDn6NoT8Dj8TA4OAjYmzFDQ0PiEQcCAfLy7K8wnU4TjUbp6+sDoK2tDaWUvIfTAz99+jS/+tWvREIJBALiVVdWVnLgwAHZqU+lUuJpl5aW8uUvf5n7779f3lcvO/WxDIa5hl5h+v1+gsEg69atk8d0tEmuRiUZAz5H0QYxLy9PjLS+3/m3vm1ZFplMRoy7Uuoqo3r58mXAjmbp7OyU+4uKilwXgmw2K+GKQ0NDfPrppwBEIhEKCwvRNeQfe+wxkWmcY7oZksmkK/IGkGPk6o/MMDfw+/2UlpbO9jCmlJw34M7A/IGBATE8Ho/HFVaXayF1eqMwEomwYIFdM6ylpYWBgQExxF6v16XZOY1oNpuV2+PpepZliSEeGhpyvU5veoLtnev3Gh4epqmpicrKSgBWrFghxywrK7uu2Fm9STo2rn1oaIi2Nrvw2jvvvMOyZctYs2YNADU1NfJ95ppWaTBMB+ZXYDAYDDnKLeGBR6NRAN5++20+/vhjeeypp56ioaEBsLXdXFqCa9mgurqae+65B7B17fPnz4smrpRyedrae9ZcS5ceT2IBJMVcv69lWa73jEajEsaolOKP//iPAdtTvxEPvL+/n+bmZvm86XSa5uZmwM6U++CDD6Q+93PPPecKfTQYbndy2oBblsXw8DC9vb2AbeA++cQuiNjd3U1zczPPPPMMAPfff39OLcG1NFFcXOwK98vLy3PFaGuUUiK7gK1ra4Oq30u/bmhoiEQi4ZoDLTGNvcglEgm5QHq9Xrxe77iGP5FIyPtrnJtHY3GOpbe3l3PnzgHQ3NzM739vt1Vta2ujvLxcvt++vj7mz5/v+kwGw+3M3LZiBoPBYJiQnHRj0uk0YHt93d3dnDlzBoDOzk7Z0IzH47S1tfH9738fsL23r371q+IVznUPfDz0xqOWNzKZjHiygUCAuro67r33XsCWOvQ8VVVVsXr1aqmh0tLSwuDgIAMDA4A9N/o9y8vL2b59O11dXQC0t7fL82KxGKFQiDvvvBOAhx56SFY0XV1dNDc3c+LECQAeeOABFi9eDNgbnGM9ex0tE4/H6ejo4OTJkwD8/ve/l+9zcHCQ0tLSOf1d6TmOxWJy7iUSCVKplEuG0iucsTVr5vJnM8x9ctKAa6N18eJFDhw4wJ49ewA4d+6cK002FotJPPPp06c5ffq0/JAqKipyIjLF4/FIgZ3S0lIikYho4JcvXxYZJRAIUF1dzbJlywA7y0y3sNOyx+bNmwE78iMWi4lhfu+99yRUsKqqio0bN1JdXS1j0McbHh6mp6dH9OpwOCxGa8+ePRw7dkxknIGBAXbt2gUgBbec6HH7/X4uXrwokSddXV2SaaqjbPRzxzN22kjGYrFxy4NO9LqpQo/14sWLvPPOO4B9gXSGQvb29sq5dscdd/D4449TU1MD2N9pLpyHhrlJzhhwHfKWSCTEOzx16hTHjx8X/bS7u1tSy8H+EesQuaNHj+L1evmTP/kTwPaEcuGH4/V6JXb1S1/6Eh6Phx/84AeA/Xk1JSUlRCIR8XqXLFlCeXk5MLHh0wZ85cqVYojGC7/Ucz88PMzIyIjM8cWLF3nzzTcBu84EIIk9jz32mNSZGA+ti1dWVvLMM8/IMQYHB2UsyWSSbDYrGviZM2fkAlFWVkZ+fr5czNvb28ctDwr2hWY6yoPG43EuXLDbwDY1NXHgwAHAPi+7urrEmchms7KP0drayvHjx/nmN78JwPLlyyUsM5c22Q1zA7N+MxgMhhwlZzxwvTzu7+9n//79AOzbt4/f/va3Uu93rD7sjNiIRqOcOXPGFWaoPbSSkpI5W8jG4/FICnxFRQWVlZUijUSjUfm8Pp/P5S1ns1l5bLzP5Xxf/f9E6KgTXZVQe+CVlZU8/vjjwJXi+Fu2bAH4zOL4+r6KigoKCgokWqi8vJzXX38dsDNGk8kkv/vd7wDby16+fDkA27Zto76+XlYRr776qkTLBINBqe8MTFuN53g8ztGjRwF47bXXRP/v7e3l8uXLMv9KKVkJdnd3MzIywre//W0Ann32WSlNYDxww/XymQZcKbUAeAWYB1jAi5Zl/aNSqhT4PlAPnAOetiyrf7oGqn8M0WhUfihHjhyhvb3dVTfEeRuuhJt5PB56enp46aWXANvg6A2/Rx55hEAgMCcNOFy5eOlUeefmmCaRSNDW1iYblaWlpRQVFQFTXzVQG9/y8nKRoerr64Er2vZkL4i6xZmWicrKykQz1zKENn59fX3y/g0NDQwNDXHkyBGAq0qEzkR50Hg8LiUJLl68KBcQy7JcJXidIY8+n++qDFqD4UaZzFmUAf7KsqxVwN3AnymlVgHfAt61LKsBeHf0b4PBYDDMEJ/pgVuW1QF0jN6OKaWagBpgB3D/6NNeBt4D/no6BplMJmXDrqOjg0uXLgF25IFSyuV1O5sWjIyMSJiXThjRHlMikZAoheXLl+P3+8Xzm0v98HSRKrDlo76+PlfCjPbOE4kE7e3tHDx4ELAzOHUkyXTV7dbe83TjlIWcq4758+fLd/iTn/yEO+64A7DrnWcyGanMeP78eXlNKBSasho5Y+vNOBOrxq4G9bgLCwupq6tj9+7dAKxfv95IJ4Yb5ro0cKVUPXAHcBCYN2rcAS5hSyzTQjKZlN3+H/7whxIznMlkXEtRr9crP4bFixezbds2iSk+deoU3d3dopdnMhmRYt544w2+/vWvT5g5GI/HXdEtmpnoYK6zTeFKBqXWfZ0GJJ1OMzg4yMWLFwG4cOGCzFkmkyEajcpn0GVgx+ufaVmWq3myc44DgQCFhYUuQzUZkskkg4ODUoTLsiyZt2AwSCAQkLkvLS0VOaW0tJS+vj6JShkZGZFxKaUoKCiQC1gkEpGL17lz52hsbJS9kp6eHtHx58+fzzPPPMOKFSsAW4O/UQMaDAZd0o+WUPr6+kin0xIx4yxbUFRUREVFhTgL5eXlxoAbbphJG3ClVDHwI+AvLcsadKZTW5ZlKaWsCV73AvACQF1d3XUPMB6P097eLtruyZMnxUglk0mXMfH7/RI6t3z5clavXi06d09PD4cOHWLv3r2AHfamvfEzZ87Q0tIinqpSSvTjWCxGR0cHx44dA+DEiRM88MADgH2RGC9JZSpRSslnTKfTJJNJ0VTHeqTOEL9EIiEXul//+tecPn1a5m14eJjy8nJpv1ZdXS0GfGRkhEQiwalTp+TvlStXAra3mJ+fP6EBtyxLjj8wMOBaHbS1tfHrX/9anqs183Xr1lFWViarivr6enbu3AnYcx+Px8WAW5YlWndTU5Or3dvZs2dpaWkB4NChQ7zxxhsyzkQiIQa0oqKCwcFBnn32WQDuvffemzLgd911F2Cfe9/97ncBe5N3bElc5yrCYJgqJrWTopTyYRvvf7Ms643RuzuVUtWjj1cDXeO91rKsFy3L2mhZ1kYdPWEwGAyGm2cyUSgK+BegybKsv3c89FPg68B/G/3/zWkZIbaG+e677wJ29qFe0o8tquT3+yUkrqamhoqKCqmlHQ6H6e7udoW26SV9R0cHb775pix5CwsLJfKhvb2dN998Uzxwn88nEsauXbvGzTKcavS4qqqqpHPOeDj18o6ODpGLzpw5w7Fjx2TFkU6nCQQCEgIXiUTEA08mkwwPD8sKZP78+fJ50+k0dXV1ssopKSlxdeRxNoJobW2V0MLu7m4KCgo4dOgQYFda1MlYbW1tJBIJNm7cCODKANXJVtqbdcpJ0WiUpqYmSfI5f/689A4dGBiQMFI9f9obj8Vi9Pb2yp5KLBaTRJrrpbi4WDIqU6mUhFBms1laWlpk/g3Xj67zPza7FiYf4XQ7MBkJZSvwPHBcKXV09L7/jG24f6CU+gbQBjw9HQMcGhqipKSEz33uc4CdEj92GeoM09IbnB9++CHBYNClu2rjBLa2q2/39PTQ0tIiIWl5eXliwN566y1X95pAIMBjjz0GcM1Mw6lCKSXGtaioiKKiIjGaHo9HTnC9aaaN7b59++SxVCrF5cuX5e90Ok1/f78YmPb2drkYWpZFXl6eHLO9vV0uXj/72c9oaGgQ2WDbtm1SHTAcDuP1euUYfX19tLa2AnDgwAHa2tpEqqioqHB1B+rr65OSB0uWLJGLsG4fp8emlBKJRu9n6I3KS5cuuQy9M4zP6/XKOXP58mWGhoZkc9spQd0M+fn50iC3qamJM2fOuBwNfRydqzBVx73V0N9vNBodN7sWruQYzKVgg9liMlEovwEmKiz9+akdjsFgMBgmy5zPxPR6vfT29koGZTqdvko60d7V4OCgeNWDg4OcPHlSNk4bGxtpaGhgyZIlgL05p5fxqVSKoaEh/v3f/x2Ad999V6SRrVu3kpeXJ9mPDz/88LTX2JgIHTLpbFbs9OTS6bR4pPF4XLxM7VHrsebn57uaPzjfZ2RkBI/HI68dHh4WryiTyXD69Gmp193S0sLXvvY1wJadnFE5paWl8r309fXR29srRbH6+vqkdorP58Pj8cjmZCaTkbnv7OxkeHjY9X3r7/fkyZOutnDJZNLl8Xq9Xldkjd4I1d7vVHnAesXR29vLW2+9BdiSlXPpD1dqxEQiEcLhsESv5EI9npkiHo+LJHbq1Klxs2vBbuwxXdm1ucacN+Aa/YNLpVKi8+ofo7N6nTY88XicRCIhkReWZVFfX89TTz0F2IbCKQ1cuHBBNNRsNisnSzwe5+mnn5Z055qamlntCqM7zMPVewA6BBDsz+d83BmXPLYpg9Ogjf1MzuV/Op1mYGBA4qqbmpo4ffo0YOvhtbW1MjafzyfGvKCgQCQpsL9DfVHw+XwUFRVJyKP+vsC+8Ogmy3ps2oBrKUKfC07ZS4/b2V1I3/Z6veTl5U3Zd6fPvd7eXrl49vX1kUgkXNE6Wj5at24dTz75pGjnxoC70Rf2PXv2jJtdC0x7hm0uMecNeFFREWvXrhXD4Axx6+zsZHBwUB7LZrNimLxeL0VFRWIompubee+996QaodMbX7VqFS+99JIYo+7ubrnya89b/9BuNHElHo+LRhuLxYhGo2J8/H6/hDCGQiFXXPZn4dRWndqvM9lHG6yJWqxN1F7N+XqwjVAmk5GL5JkzZ/jRj34kz8vPz5f5cWrXegXgNJrOi4lzvMPDw7KB7PF4KCwslPK1SikZi/68zi5Ek8GyLEZGRqbMA3d+Rn0B8fv9rgQzfXEDW+NfvHixnFems5Abva+0c+dO9u3bJw2t165dKxvcZhPzCmYWDAaDIUeZ85f/wsJC5s2bJ1fcb37zm/z4xz8G7OiGvLw8VxMB7XFrD0hrkYODg5w7d06895KSEgkfa2xs5A//8A/53ve+B8D7778vy3Zd1W+yHpuWMHSHFq37xuNxV7PeaDQqkRiFhYXSfPmpp55ixYoVE3rgzoqL6XRaPp/X68Xn87mqEzpfY1nWhI0RnDKFfo4z8sMpPzjnNBaLyarl0KFDRCIR6dYz1qt3es9OOWes568/h/N143GtVcNYjTubzcqYp1oDd674nPOrpRqw90p0Qtl9991HOBw2HuQ4OPdQgsEg69atk8f8fr/JWB2HOW/AwV6a6/jndDrN3XffDcDSpUsJBoMSX9ze3i6bIDqG1KmZdnR08OKLLwK2sX/wwQcBWyZZuHChpFd/+umn0iRi7BJXx6cC0pZML/kBud3c3MwHH3zgavGmjbmWULR+mp+fL+Neu3YtJSUlYgycJ61e/msJY7wLy3iGwbIs0um0zEUkEqG2tlb+TqfTImFkMhmy2axcCJ2bmNrwOkMOtdR07tw5Tp48KVJQb2+vzIUe93gZnM4LBdg/1LE1Ra5lqJ2VGZ3hpTouXb+Hlpby8/OlsxHg0s2vl3g8Lnsluk4N2HPm/F6c1RZLS0tnpH5MruLcBDd8NsYNMBgMhhxlTnng2tMbm8Hm9/vFUyotLZUmAslkkkQiIREig4ODkmHX1tbG3r17xZPu7e2lt7dXdrZff/118Z527dpFMBjk4YcfBuyMR123o6uri8uXL8vueFdXl4zvww8/5OTJkyKFwJW61IODgy4JxVnHe2wiklJKvOF9+/ZRXV0tGaRjcXrHztC58TbD9HH0pp3evQ+FQlRUVHDffffJc7W8c/jwYWprayUqpKuryxWqZ1mWK4xRH+PYsWN0dXWJpLJt2zaJ1gkGg/h8Ppk3v98vntZYr9wpr+i5uZYHrqURPTZ9v7OeTXFxsXjc9fX1PP300zQ2NspjN8rQ0BDHjx8H7GqIOhkplUrh9/tdXrcOjTQRFIapZM4Y8Hg87kqv1nHfjY2NrFu3zlUmdiKy2aws2xsbG1m1ahU//OEPAfjNb34jxaDArtZ3+PBhwF5Gb9++XWSa4eFhyWjMz8/nww8/5O233wZs46PfIx6PXyWhaMZm3KXTaTFWRUVFFBcXSyxrWVmZxKvv2LFDomPGEggEKC0tlVT2np4el0zhZOwFI5PJiBHbtGkTn//85yW64/3335cLne4C/5WvfAWwjdTPfvYzwJaohoaGXNKEngs9Dh1imEql2LZtm3ze7u5ueY4zQga4KqzPKatcKzpmbAcmbcx11Iue73A4zKZNmwB44oknWL16tcyhnoMbwZkJ2t/fL3KSLlWgI0927tzJ6tWrAVwZogbDzTKnDLguB/raa6+JdjwyMkJpaamc+NcKIXLWA9dlT/UPNxaL8dFHH0msd19fn3iLkUiEjRs3ikFdvny5eG//9E//RGtrKx0dduXcgYGBCeOws9msy+srKiqScesUdbA19+3bt0sscDgclgtTSUkJJSUl4yYIBYNBNmzYIMb6u9/9rqwM9CrAabS1p55OpwmHw3KRWLp0KaFQSFYZly5dkrGVl5fz7LPPSshWKpWS0K5//ud/prW11VWeYLxNSD2n2uusqKhg3rx5shpxxqjr+i3a+FqWJd9vfn7+pIy4/rx6b8Dr9RIIBOSCtXbtWjZs2ADYHnhlZeW0x18XFBSIBx6JRGQsZiPOMJUYDdxgMBhylDnjgQMuuUFHZfzqV79CKSUhf5PtXenxeAiHwxKK1NnZySeffCLeXDqdFk9S9110JqzoJfauXbsYHh4WD9zZLDgYDFJSUiJeVSAQEN13xYoVbNmyxaV56uW67gijPf7JeoPFxcWunpGVlZWSuai1d+cc6tuBQIDKykopPFVRUUF1dbXsJeiVD9gFg5wNiWOxmOwVLF68mEuXLokn7azyp+UUvQfR09Mjq5H8/HzKysrkdc7VQSwWc9URd4Y0Xit8c2RkxCUTZTIZOV5FRQUrV64UGcjv9zNvnt1vJBKJ3FTyjD6ejiTSc+NsMB0IBAiHwzKHpaWlJuPSMC3MGQPu8Xhkqb59+3Zefvll4MrGpNYaq6qqJvUDVEpRWFgoBjQQCBAIBFzhavoH19PTQ09Pj1w0CgsLXWGL69evF2Pf1dXlaha8YsUKCWt0GnPdums60u71uFOplEt6cKaWFxQUyEVo6dKlrFy5knvuuQewM09LS0vltU7N3ev1umQqv98vhkgbJW2I0+m0PE+HKupqkN///velMUNjYyP19fVSK2RoaEj0cB1uqL/f3t5e1ya2c8N37Ians6KkZVky15s3b+b5559n8eLFgG1AndmcN/NdOOvuvP/++7z//vvA1QZ8xYoVsklcWFgon0kpZWQUw5RhJBSDwWDIUeaMB+71emWD8dChQ65KenDF84lGo65MPe0xwuS8XP1+zpoenZ2d7NmzR96noqJCvOxwOMwTTzwhbdScESfFxcXiacPM1GhwZkaC+zOn02n5TPPmzWPz5s2AHXmxatUqV/LKZJf0Tvnh3nvvJRqNilyjvWY9Lmc1xL6+PvGOi4uLaWhoYPny5QCu/pjRaJT9+/dL+7fz589LNFJXVxfZbNa1aez8vHl5eeJZZ7NZ2QiurKyksrJS2rZNpXzhPA/7+vpEQhkcHHQlFSWTSYnIOXTokGSo6poes1kQzXDrMGcMOFzRbzOZjBiJZDJJR0eHRIx0dXVJf0y/3y+aLVy7vKvX68Xv97tihXUIXF9fn6uPYTweF53ZGUEwF3DGF9fW1oo2r8vHOtO3dZTLkiVLWLBgwXUXftLH01LM+vXr6ezsFON74sQJMdj6wuIsUatv60a+TvSFZmhoiBUrVrgacfz0pz8FrmS9OtP89W19PKckpj/7wMAAZ8+eFcNdVlYmBnOq5Yvxel0ODQ1x5MgRiRGvra0VQz88PMyOHTumdAyG25c5Y8A9Ho8kr9x3332SWBKNRvnd734nSRJKKdG1S0tL6e/v57nnngO4Zo1g3fDXGWOsjX0oFMLv90/bj3wq8fv9Ygy3bNniCs3LZDISKrhkyRLxeG9mZeDsCFReXs7dd98tF9fz589LvLyuBOkMAdQGdbwwQH0x0XOuVzZLliyRuiG6y44zIWns2JyrEe3xHz16lLa2NlatWgXYeyq6i9DNfLepVMq1wujr6xu3bVoikSCTybjORx3Oqv83GKYCs34zGAyGHGXOeODFxcXiTfX394ue2dfXR3d3t3iaJSUl4tnpIu+TSU8eW51wZGREPPDKykoef/xxVq5cCcztIvt+v1/knfr6eunDuHLlStauXStyRygUcqWy34zWqj3o4uJiCZ3UfztXNGPT4CdDNptlcHCQ/fv3A3bGrJbIhoaGUEq5aoVrz11X/HOOUa8MdGcXrY9funRJHhsr5VwPyWRSSgz85Cc/4fjx467wTS2nFBUVUV5eLsXRnn/+eVkNlZWVzekVniG3mLQBV0p5gY+AC5ZlPaaUWgS8BpQBHwPPW5aVupnB6B/XXXfdJT9iHUaof8SBQIAvfvGLADz44IMsWbJkwg2hsdXqnLU7ksmkGP6KigrKyspEW57LPzBnZb2lS5dKqJwu16ovfFO5OebshjQ0NCR7Bbr7u36OzqoEOzTTGRPurMLnHFs2myUajfLpp58CdtPqtrY2AKm86Iz11jKJTpfXRtqp7+vysXosznjxzyKZTIo+r9HHSCaTV0ko2oB7PB45n8rLy7nnnnvYvXs3YGvgeh/FNOI1TCXX8yv/C6DJ8fd/B/7BsqylQD/wjakcmMFgMBiuzaQ8cKVULfAo8F+B/6DsdfODwLOjT3kZ+Dvg2zc6EJ/PJx7w8uXL2bVrF2B7dgcPHpTNskQiITv6PT09hMNh8Zh8Pp94XbFYjIGBAVpbWwG7/delS5dEQikuLpbjae97LksnGufmq5ZLphtn8kpzc7Osjnp6emS+nRmu+rGf//zn8lhlZeWEK6VsNisRQcPDwy5veaws47w9UZamrouiVwrOSoWfhTPrtqenh7q6Ogkp1c2ZwU44cjbY9nq9smm5c+dOHnzwQVkdhUIhEy5omBYmK6H8T+A/AcHRv8uAqGVZuqxcO1Bzs4PRhqmsrEx64Z04cYL29nYxvMPDw5LRd/DgQRobG6UxQyQSkdTyjz/+mN7eXlcZWKdsUlZWJtEOu3fvZsGCBTlhwGcDZ3f55uZmiRDq7++Xx/R3p7+nRCIhskhLSwutra0iRZSVlcnztcw1nkwytreks7lDfn6+PK5fN9a4TyYlfyyxWIyWlhbAbtqcSCRE8z99+jTvvPMOYFez1BcdPTa9N6HLHcyl8NBIJXYAAA7pSURBVFPDrclnGnCl1GNAl2VZHyul7r/eAyilXgBeACTE7RrPBa7UzgD44he/iGVZ/OAHPwDshsPaA+/t7eXixYucOXMGsDVa/aNKpVK0t7dLjQ2tgesqezt37hQtvaGhwXhJ10Abv+HhYfr7+8ULHRoaclVbBFy1UbQx/+STT3jllVck3HPZsmXy/WrtWj9XdwTS7+lsxVZYWCjesH6dXgE4vWx9Hml9OhqNypgrKiquuambSqUkkejgwYPs27dPaoen02kx7j09PYyMjMhFPxwOi8GORCJzeh/FcOswGYu1FXhCKXUOe9PyQeAfgbBSSl8AaoEL473YsqwXLcvaaFnWxpuJADAYDAaDm8/0wC3L+hvgbwBGPfD/aFnWV5VSPwS+gm3Uvw68OVWD8ng84s0sXLiQhoYGKbhkWZYrq62zs1Nkk7HRDbqetPN9tWfkbIyQy0tdnUgSjUaJx+Pyt7MyYigUcoX8XS/ODFnd3WcinJmRWg+PxWJ0dHRIhuWXv/xl0YtTqZSrX2g8HpdVU15enqtwmdfrle/sjjvuYGRkhI8++giwPXDtjesQQ/2eBw4ckLmoqqq6qqJlNpuVPZbu7m5JgT958iTJZFJWeGNfA1ciVOrq6njyyScBO13+ZnptGgyT5WbiwP8aeE0p9V+AI8C/TM2Q3BQXF7Np0yb5wb/00kuSVt/T00MikRCjkUqlxLj4fD6XfupMoYfr00VzgTNnzvDWW29JSnplZSUbN24E4IEHHqCoqGjcpsKfhbNErO46ow1lUVHRVd3l9THy8vJcF8uamhqeeOIJwI7f18a9v7+fw4cPu8r1jn1P53vri61ezWmDruOz4UqlSZ3defr0aSlHvHr1arxer7w+Pz9fQhkBfvnLX7Jv3z7Avpg4a9+M1eMDgYBUrVy8eLFsWlZVVRkJxTAjXJcBtyzrPeC90dutgMkLNhgMhllizmRiTkRxcTF+v182y77xjW/wve99D7B7OWYyGVlmFxQUuBruKqVcHrdSylU/+1ZAf55wOMyKFSukOUNbW5t8xqqqKkZGRmQVU1BQMOmmBslkUiSqN954g+bmZpE4nMWknHMNtgeuveN7772Xr3zlK7KJHYlExLPt6+sjGo2KB+xszAD2xqWOWCkoKBBPuqKigmw2Kx55OBx2ba4WFRWJzBGPxzl69Chgnxd/+qd/KpEl+fn5jIyMyPF1n1OwV3g+n8+V/TkWffyVK1eKfNXd3W3qfhtmhDlvwMFdEW/x4sWsX78esJffp0+flh/O5cuXZXmvK9c5Y4qdMdTOYku3AsPDw/T09EgIXk9PjxiltrY2nnjiCZm3uro6QqHQNSUVPW+JRILOzk7ADsW8cOGCZCM6Gwfr1zi7FWkJZ9u2bdTV1cl36PV65TsbGhqira3N1a1HG8pMJoPP55OuSlu3bpXIkmXLllFZWUlDQwNgf/e6JO3g4CCpVEoM79DQkBj3jo4Ozp49K4bX4/G4+lfW19ezbNkyOX4qlZLqi2MzNAHRx1955RWWLl0K2OV7t2zZYgy4YdrJGQumDUVpaal0etmwYQNHjhzh9ddfB+yu6foHrutD6x9RcXExFRUVYsQaGhpkYyuX0ckqTU1N7N2719ViTW/2ai9XG80vfOELBINBl/HVnvzly5dJJBIu46/Lora3t4sxg6tLFeTl5bk2n7UhrKuro6SkxOXBOqsRFhUVidft9/vlOwT7e9M685o1ayQ/ID8/n0wmI4Z3x44dEtJ37NgxBgcH5fP5fD5ZwWUyGfbv3y/vOX/+fGm/B3blQn3haW1tZe/evZLa76w+GI/HSSQS4q13dHTInC1evJiamhoZjwlRNUwX5qwyGAyGHCVnPHDtwTgbLJSVlREIBGR53NraKsvcoqIiYrEYW7duBZBKfdrrLikpmVQVw7mOlkxisRiJREI8ZGd1vP7+flpbWzlx4gRg68V9fX3ioQYCAfEW0+k0ra2tHDlyBIAjR45I1M/g4KBIDmNJJpMopUQm2b17t3iytbW15OXlucIY9fFqamp48sknRRPXES9wZR9DH6+0tFQ8bq1da8/6wQcflI5O8XicCxcuyFw4ZbSCggK2bt0qDZ71cXShrVAoJN55fX09GzdulOSd3/zmN/zyl78EbFlmZGTE5Vlr73zfvn3k5eWJXj8TnZoMtyc5Y8DHo6CggIULF/JHf/RHgG1gdDhad3c3a9asEYMSDodvSU1Sf6by8nIikYhkOGodGGz5aXh4mHfffReA48ePi3HRj+uG0o2NjbS2tvLb3/4WsKUZrYF7vV58Pp9r70BLL36/n1AoJN2RFi1aJA06xouz13sR2ijr5xQUFMg+hu6yo+WWsRcBr9crF+Hq6moef/xxwDaYP/7xj6UOzsjIiIw5EAhIqrtz/sYbW3l5OeXl5fKcVColslRTUxO9vb2i3esu9WCfe11dXbJXMH/+/Ftqv8UwdzBugcFgMOQoOe0W6OgU7WUnk0lJptDo5fet6gFp73D9+vWk02m+853vAPZmpPaOPR6Pqx1YNBqlpaVFvNlAICARHHv37iUvL0+8yVgsJh6prgzoTIpyRp2sXLmSe+65B7A94uvpwelM3tHvmU6nr2rcMNHrIpGInAfl5eWUlZXJyiEvL08iRDZu3Mi8efOuq3CZ3mDdsGGDvM/Fixf5+c9/zqFDhwB7E1PLd5FIhNLSUtkYNfKJYbq4paxafn7+lFUU1EYkFou5ejF6vd451VHcKaFUVFRIB3lnRM7Yhs7gHvvly5dFS7cs66p0eW1Aday3s8qfljvy8/Opqalhw4YNgB17rvXpa6HLG+g5dcbyj4yMoJSSC4Gz+uB4aEM7b948Ghoa5PibN2+WcS1YsIBQKHRd54kzkknPb3V1NfPnz5cuTr/4xS/EgD/00ENs3rxZ9PobyYA1GCbDLWXAp4pUKiUeaHt7Ox988IE8tmXLFtF5w+HwnOmwkpeXR1FRkasxs9Mo68p+4C7LOhZn9b+x6DhvZxcebTSrq6upr6+Xao/OFnnXQqek61jvCxcuSCikTsjROvdnXTC1wV63bp14ymAbdmcbuJtBj0HH0T/88MOAfV44j3er7rkY5haz70IaDAaD4YYwHrgDHQbW29vLqVOnAHj11VeJRqPi2fb390tda61xzgV8Ph/V1dXS3CIYDErY4NmzZ12auA5/G1ssCq5462O74oAtBThT20tKSiRZ55FHHuGuu+66bt1XKUUwGJQwvtraWvHGBwYGWLFiBQsXLgTs+b7W+zqljpmguLhYjqVXHgbDTGIM+DicPXuWPXv2AFeaAugwu0cffXROxo8XFhYyf/58HnroIcBuS3fgwAEA3n77bY4cOSIXKG2ItcTh1Lx1F3gdguiMdQ6Hw9TW1rJ69WrALul65513AnY8d1lZ2aR0bydKKVeVwUWLFsm47rjjDlcK/tgysAbD7Y75NRgMBkOOYjzwcVi0aJHUW9m3bx9r1qxh7dq1gL1UnktRKBqv1+ta0vt8PqmT0tbWRk9Pj2wOajnFGV2jozL0ezjDL7VHHIlEWLRokdQjaWxslM3CYDB4Q6GaSiny8/Ml/FNvEOtj62qUBoPhaowBd6CNn9/vFyOt9VhtRHIlsqCoqEikjnA4zKpVqyRmua2tzVVZL5vNivRRU1PDQw89JA0PfD6fXKh8Pp+r008wGBRDfzMXM5/PN6f2EwyGXMEY8HHw+/0Sw5urOGtnh0IhampqJMnGWTNFow14KBSipKRENhWnKq7eYDBMPXNHAzAYDAbDdWE88FsYLWsUFhZSWFgoVfYMBsOtgfHADQaDIUcxBtxgMBhyFGPADQaDIUdRM9mdXSnVDSSAnhk7aG5QjpmT8TDzMj5mXsbnVp6XhZZlVYy9c0YNOIBS6iPLsjbO6EHnOGZOxsfMy/iYeRmf23FejIRiMBgMOYox4AaDwZCjzIYBf3EWjjnXMXMyPmZexsfMy/jcdvMy4xq4wWAwGKYGI6EYDAZDjjJjBlwp9SWlVLNSqkUp9a2ZOu5cRCl1Til1XCl1VCn10eh9pUqpXyqlTo/+H5ntcU43Sql/VUp1KaVOOO4bdx6Uzf8aPX+OKaXunL2RTx8TzMnfKaUujJ4vR5VS2x2P/c3onDQrpR6enVFPP0qpBUqpfUqp3yulPlFK/cXo/bf1+TIjBlwp5QX+N/AIsAr4A6XUqpk49hzmAcuy1jvCnr4FvGtZVgPw7ujftzovAV8ac99E8/AI0DD67wXg2zM0xpnmJa6eE4B/GD1f1luW9TbA6G9oN7B69DX/Z/S3diuSAf7KsqxVwN3An41+/tv6fJkpD3wT0GJZVqtlWSngNWDHDB07V9gBvDx6+2Vg5yyOZUawLOt9oG/M3RPNww7gFcvmQyCslLrlGlFOMCcTsQN4zbKspGVZZ4EW7N/aLYdlWR2WZf129HYMaAJquM3Pl5ky4DXAecff7aP33a5YwC+UUh8rpV4YvW+eZVkdo7cvAfNmZ2izzkTzcLufQ38+KgX8q0Neuy3nRClVD9wBHOQ2P1/MJubscI9lWXdiL/P+TCl1r/NByw4Nuu3Dg8w8CN8GlgDrgQ7gf8zucGYPpVQx8CPgLy3LGnQ+djueLzNlwC8ACxx/147ed1tiWdaF0f+7gB9jL3s79RJv9P+u2RvhrDLRPNy255BlWZ2WZY1YlpUFvsMVmeS2mhOllA/beP+bZVlvjN59W58vM2XADwMNSqlFSik/9sbLT2fo2HMKpVRAKRXUt4GHgBPY8/H10ad9HXhzdkY460w0Dz8FvjYaXXA3MOBYOt/SjNFud2GfL2DPyW6lVL5SahH2ht2hmR7fTKCUUsC/AE2WZf2946Hb+3yxLGtG/gHbgVPAGeBvZ+q4c+0fsBj43ei/T/RcAGXYu+ingb1A6WyPdQbm4v9hSwJpbI3yGxPNA6CwI5nOAMeBjbM9/hmck/87+pmPYRumasfz/3Z0TpqBR2Z7/NM4L/dgyyPHgKOj/7bf7ueLycQ0GAyGHMVsYhoMBkOOYgy4wWAw5CjGgBsMBkOOYgy4wWAw5CjGgBsMBkOOYgy4wWAw5CjGgBsMBkOOYgy4wWAw5Cj/H7JTbNVzJyMCAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(X[-1], cmap = 'gray')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://github.com/guillaumegenthial/im2latex/blob/master/model/components/attention_mechanism.py\n",
    "\n",
    "class AttentionMechanism(object):\n",
    "    \"\"\"Class to compute attention over an image\"\"\"\n",
    "\n",
    "    def __init__(self, img, dim_e, tiles=1):\n",
    "        \"\"\"Stores the image under the right shape.\n",
    "        We loose the H, W dimensions and merge them into a single\n",
    "        dimension that corresponds to \"regions\" of the image.\n",
    "        Args:\n",
    "            img: (tf.Tensor) image\n",
    "            dim_e: (int) dimension of the intermediary vector used to\n",
    "                compute attention\n",
    "            tiles: (int) default 1, input to context h may have size\n",
    "                    (tile * batch_size, ...)\n",
    "        \"\"\"\n",
    "        if len(img.shape) == 3:\n",
    "            self._img = img\n",
    "        elif len(img.shape) == 4:\n",
    "            N    = tf.shape(img)[0]\n",
    "            H, W = tf.shape(img)[1], tf.shape(img)[2] # image\n",
    "            C    = img.shape[3].value                 # channels\n",
    "            self._img = tf.reshape(img, shape=[N, H*W, C])\n",
    "        else:\n",
    "            print(\"Image shape not supported\")\n",
    "            raise NotImplementedError\n",
    "\n",
    "        # dimensions\n",
    "        self._n_regions  = tf.shape(self._img)[1]\n",
    "        self._n_channels = self._img.shape[2].value\n",
    "        self._dim_e      = dim_e\n",
    "        self._tiles      = tiles\n",
    "        self._scope_name = \"att_mechanism\"\n",
    "\n",
    "        # attention vector over the image\n",
    "        self._att_img = tf.layers.dense(\n",
    "            inputs=self._img,\n",
    "            units=self._dim_e,\n",
    "            use_bias=False,\n",
    "            name=\"att_img\")\n",
    "\n",
    "\n",
    "    def context(self, h):\n",
    "        \"\"\"Computes attention\n",
    "        Args:\n",
    "            h: (batch_size, num_units) hidden state\n",
    "        Returns:\n",
    "            c: (batch_size, channels) context vector\n",
    "        \"\"\"\n",
    "        with tf.variable_scope(self._scope_name):\n",
    "            if self._tiles > 1:\n",
    "                att_img = tf.expand_dims(self._att_img, axis=1)\n",
    "                att_img = tf.tile(att_img, multiples=[1, self._tiles, 1, 1])\n",
    "                att_img = tf.reshape(att_img, shape=[-1, self._n_regions,\n",
    "                        self._dim_e])\n",
    "                img = tf.expand_dims(self._img, axis=1)\n",
    "                img = tf.tile(img, multiples=[1, self._tiles, 1, 1])\n",
    "                img = tf.reshape(img, shape=[-1, self._n_regions,\n",
    "                        self._n_channels])\n",
    "            else:\n",
    "                att_img = self._att_img\n",
    "                img     = self._img\n",
    "\n",
    "            # computes attention over the hidden vector\n",
    "            att_h = tf.layers.dense(inputs=h, units=self._dim_e, use_bias=False)\n",
    "\n",
    "            # sums the two contributions\n",
    "            att_h = tf.expand_dims(att_h, axis=1)\n",
    "            att = tf.tanh(att_img + att_h)\n",
    "\n",
    "            # computes scalar product with beta vector\n",
    "            # works faster with a matmul than with a * and a tf.reduce_sum\n",
    "            att_beta = tf.get_variable(\"att_beta\", shape=[self._dim_e, 1],\n",
    "                    dtype=tf.float32)\n",
    "            att_flat = tf.reshape(att, shape=[-1, self._dim_e])\n",
    "            e = tf.matmul(att_flat, att_beta)\n",
    "            e = tf.reshape(e, shape=[-1, self._n_regions])\n",
    "\n",
    "            # compute weights\n",
    "            a = tf.nn.softmax(e)\n",
    "            a = tf.expand_dims(a, axis=-1)\n",
    "            c = tf.reduce_sum(a * img, axis=1)\n",
    "\n",
    "            return c\n",
    "\n",
    "\n",
    "    def initial_cell_state(self, cell):\n",
    "        \"\"\"Returns initial state of a cell computed from the image\n",
    "        Assumes cell.state_type is an instance of named_tuple.\n",
    "        Ex: LSTMStateTuple\n",
    "        Args:\n",
    "            cell: (instance of RNNCell) must define _state_size\n",
    "        \"\"\"\n",
    "        _states_0 = []\n",
    "        for hidden_name in cell._state_size._fields:\n",
    "            hidden_dim = getattr(cell._state_size, hidden_name)\n",
    "            h = self.initial_state(hidden_name, hidden_dim)\n",
    "            _states_0.append(h)\n",
    "\n",
    "        initial_state_cell = type(cell.state_size)(*_states_0)\n",
    "\n",
    "        return initial_state_cell\n",
    "\n",
    "\n",
    "    def initial_state(self, name, dim):\n",
    "        \"\"\"Returns initial state of dimension specified by dim\"\"\"\n",
    "        with tf.variable_scope(self._scope_name):\n",
    "            img_mean = tf.reduce_mean(self._img, axis=1)\n",
    "            W = tf.get_variable(\"W_{}_0\".format(name), shape=[self._n_channels,\n",
    "                    dim])\n",
    "            b = tf.get_variable(\"b_{}_0\".format(name), shape=[dim])\n",
    "            h = tf.tanh(tf.matmul(img_mean, W) + b)\n",
    "\n",
    "            return h"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://github.com/guillaumegenthial/im2latex/blob/master/model/components/attention_cell.py\n",
    "\n",
    "import collections\n",
    "from tensorflow.contrib.rnn import RNNCell, LSTMStateTuple\n",
    "\n",
    "\n",
    "AttentionState = collections.namedtuple(\"AttentionState\", (\"cell_state\", \"o\"))\n",
    "\n",
    "\n",
    "class AttentionCell(RNNCell):\n",
    "    def __init__(self, cell, attention_mechanism, dropout, dim_e,\n",
    "                 dim_o, num_units,\n",
    "        num_proj, dtype=tf.float32):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            cell: (RNNCell)\n",
    "            attention_mechanism: (AttentionMechanism)\n",
    "            dropout: (tf.float)\n",
    "            attn_cell_config: (dict) hyper params\n",
    "        \"\"\"\n",
    "        # variables and tensors\n",
    "        self._cell                = cell\n",
    "        self._attention_mechanism = attention_mechanism\n",
    "        self._dropout             = dropout\n",
    "\n",
    "        # hyperparameters and shapes\n",
    "        self._n_channels     = self._attention_mechanism._n_channels\n",
    "        self._dim_e          = dim_e\n",
    "        self._dim_o          = dim_o\n",
    "        self._num_units      = num_units\n",
    "        self._num_proj       = num_proj\n",
    "        self._dtype          = dtype\n",
    "\n",
    "        # for RNNCell\n",
    "        self._state_size = AttentionState(self._cell._state_size, self._dim_o)\n",
    "\n",
    "\n",
    "    @property\n",
    "    def state_size(self):\n",
    "        return self._state_size\n",
    "\n",
    "\n",
    "    @property\n",
    "    def output_size(self):\n",
    "        return self._num_proj\n",
    "\n",
    "\n",
    "    @property\n",
    "    def output_dtype(self):\n",
    "        return self._dtype\n",
    "\n",
    "\n",
    "    def initial_state(self):\n",
    "        \"\"\"Returns initial state for the lstm\"\"\"\n",
    "        initial_cell_state = self._attention_mechanism.initial_cell_state(self._cell)\n",
    "        initial_o          = self._attention_mechanism.initial_state(\"o\", self._dim_o)\n",
    "\n",
    "        return AttentionState(initial_cell_state, initial_o)\n",
    "\n",
    "\n",
    "    def step(self, embedding, attn_cell_state):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            embedding: shape = (batch_size, dim_embeddings) embeddings\n",
    "                from previous time step\n",
    "            attn_cell_state: (AttentionState) state from previous time step\n",
    "        \"\"\"\n",
    "        prev_cell_state, o = attn_cell_state\n",
    "\n",
    "        scope = tf.get_variable_scope()\n",
    "        with tf.variable_scope(scope):\n",
    "            # compute new h\n",
    "            x                     = tf.concat([embedding, o], axis=-1)\n",
    "            new_h, new_cell_state = self._cell.__call__(x, prev_cell_state)\n",
    "            new_h = tf.nn.dropout(new_h, self._dropout)\n",
    "\n",
    "            # compute attention\n",
    "            c = self._attention_mechanism.context(new_h)\n",
    "\n",
    "            # compute o\n",
    "            o_W_c = tf.get_variable(\"o_W_c\", dtype=tf.float32,\n",
    "                    shape=(self._n_channels, self._dim_o))\n",
    "            o_W_h = tf.get_variable(\"o_W_h\", dtype=tf.float32,\n",
    "                    shape=(self._num_units, self._dim_o))\n",
    "\n",
    "            new_o = tf.tanh(tf.matmul(new_h, o_W_h) + tf.matmul(c, o_W_c))\n",
    "            new_o = tf.nn.dropout(new_o, self._dropout)\n",
    "\n",
    "            y_W_o = tf.get_variable(\"y_W_o\", dtype=tf.float32,\n",
    "                    shape=(self._dim_o, self._num_proj))\n",
    "            logits = tf.matmul(new_o, y_W_o)\n",
    "\n",
    "            # new Attn cell state\n",
    "            new_state = AttentionState(new_cell_state, new_o)\n",
    "\n",
    "            return logits, new_state\n",
    "\n",
    "\n",
    "    def __call__(self, inputs, state):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            inputs: the embedding of the previous word for training only\n",
    "            state: (AttentionState) (h, o) where h is the hidden state and\n",
    "                o is the vector used to make the prediction of\n",
    "                the previous word\n",
    "        \"\"\"\n",
    "        new_output, new_state = self.step(inputs, state)\n",
    "\n",
    "        return (new_output, new_state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import division\n",
    "import math\n",
    "import numpy as np\n",
    "from six.moves import xrange\n",
    "import tensorflow as tf\n",
    "\n",
    "\n",
    "# taken from https://github.com/tensorflow/tensor2tensor/blob/37465a1759e278e8f073cd04cd9b4fe377d3c740/tensor2tensor/layers/common_attention.py\n",
    "\n",
    "# taken from https://raw.githubusercontent.com/guillaumegenthial/im2latex/master/model/components/positional.py\n",
    "\n",
    "def add_timing_signal_nd(x, min_timescale=1.0, max_timescale=1.0e4):\n",
    "    \"\"\"Adds a bunch of sinusoids of different frequencies to a Tensor.\n",
    "\n",
    "    Each channel of the input Tensor is incremented by a sinusoid of a difft\n",
    "    frequency and phase in one of the positional dimensions.\n",
    "\n",
    "    This allows attention to learn to use absolute and relative positions.\n",
    "    Timing signals should be added to some precursors of both the query and the\n",
    "    memory inputs to attention.\n",
    "\n",
    "    The use of relative position is possible because sin(a+b) and cos(a+b) can\n",
    "    be experessed in terms of b, sin(a) and cos(a).\n",
    "\n",
    "    x is a Tensor with n \"positional\" dimensions, e.g. one dimension for a\n",
    "    sequence or two dimensions for an image\n",
    "\n",
    "    We use a geometric sequence of timescales starting with\n",
    "    min_timescale and ending with max_timescale.  The number of different\n",
    "    timescales is equal to channels // (n * 2). For each timescale, we\n",
    "    generate the two sinusoidal signals sin(timestep/timescale) and\n",
    "    cos(timestep/timescale).  All of these sinusoids are concatenated in\n",
    "    the channels dimension.\n",
    "\n",
    "    Args:\n",
    "        x: a Tensor with shape [batch, d1 ... dn, channels]\n",
    "        min_timescale: a float\n",
    "        max_timescale: a float\n",
    "\n",
    "    Returns:\n",
    "        a Tensor the same shape as x.\n",
    "\n",
    "    \"\"\"\n",
    "    static_shape = x.get_shape().as_list()\n",
    "    num_dims = len(static_shape) - 2\n",
    "    channels = tf.shape(x)[-1]\n",
    "    num_timescales = channels // (num_dims * 2)\n",
    "    log_timescale_increment = (\n",
    "            math.log(float(max_timescale) / float(min_timescale)) /\n",
    "            (tf.to_float(num_timescales) - 1))\n",
    "    inv_timescales = min_timescale * tf.exp(\n",
    "            tf.to_float(tf.range(num_timescales)) * -log_timescale_increment)\n",
    "    for dim in xrange(num_dims):\n",
    "        length = tf.shape(x)[dim + 1]\n",
    "        position = tf.to_float(tf.range(length))\n",
    "        scaled_time = tf.expand_dims(position, 1) * tf.expand_dims(\n",
    "                inv_timescales, 0)\n",
    "        signal = tf.concat([tf.sin(scaled_time), tf.cos(scaled_time)], axis=1)\n",
    "        prepad = dim * 2 * num_timescales\n",
    "        postpad = channels - (dim + 1) * 2 * num_timescales\n",
    "        signal = tf.pad(signal, [[0, 0], [prepad, postpad]])\n",
    "        for _ in xrange(1 + dim):\n",
    "            signal = tf.expand_dims(signal, 0)\n",
    "        for _ in xrange(num_dims - 1 - dim):\n",
    "            signal = tf.expand_dims(signal, -2)\n",
    "        x += signal\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "attention_size = 256\n",
    "size_layer = 256\n",
    "embedded_size = 256\n",
    "beam_width = 15\n",
    "learning_rate = 1e-4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# CNN part I took from https://github.com/guillaumegenthial/im2latex/blob/master/model/encoder.py\n",
    "# I use tf.contrib.seq2seq as decoder part\n",
    "\n",
    "class Model:\n",
    "    def __init__(self):\n",
    "        self.X = tf.placeholder(tf.float32, shape=(None, 60, 240, 1))\n",
    "        self.Y = tf.placeholder(tf.int32, [None, None])\n",
    "        self.Y_seq_len = tf.count_nonzero(self.Y, 1, dtype=tf.int32)\n",
    "        batch_size = tf.shape(self.X)[0]\n",
    "        x_len = tf.shape(self.X)[2] // 2\n",
    "        main = tf.strided_slice(self.Y, [0, 0], [batch_size, -1], [1, 1])\n",
    "        decoder_input = tf.concat([tf.fill([batch_size, 1], GO), main], 1)\n",
    "        \n",
    "        decoder_embeddings = tf.Variable(tf.random_uniform([len(encode_maps), embedded_size], -1, 1))\n",
    "        \n",
    "        img = self.X\n",
    "        \n",
    "        out = tf.layers.conv2d(img, 64, 3, 1, \"SAME\",\n",
    "                activation=tf.nn.relu)\n",
    "        out = tf.layers.max_pooling2d(out, 2, 2, \"SAME\")\n",
    "\n",
    "        out = tf.layers.conv2d(out, 128, 3, 1, \"SAME\",\n",
    "                activation=tf.nn.relu)\n",
    "        out = tf.layers.max_pooling2d(out, 2, 2, \"SAME\")\n",
    "\n",
    "        out = tf.layers.conv2d(out, 256, 3, 1, \"SAME\",\n",
    "                activation=tf.nn.relu)\n",
    "\n",
    "        out = tf.layers.conv2d(out, 256, 3, 1, \"SAME\",\n",
    "                activation=tf.nn.relu)\n",
    "        out = tf.layers.max_pooling2d(out, (2, 1), (2, 1), \"SAME\")\n",
    "        out = tf.layers.conv2d(out, 512, 3, 1, \"SAME\",\n",
    "                activation=tf.nn.relu)\n",
    "        out = tf.layers.max_pooling2d(out, (1, 2), (1, 2), \"SAME\")\n",
    "        out = tf.layers.conv2d(out, 512, 3, 1, \"VALID\",\n",
    "                activation=tf.nn.relu)\n",
    "        img = add_timing_signal_nd(out)\n",
    "        print(img)\n",
    "        \n",
    "        with tf.variable_scope(\"attn_cell\", reuse=False):\n",
    "            attn_meca = AttentionMechanism(img, attention_size)\n",
    "            recu_cell = tf.nn.rnn_cell.LSTMCell(size_layer)\n",
    "            attn_cell = AttentionCell(recu_cell, attn_meca, 1.0,\n",
    "                        attention_size, attention_size, size_layer, len(encode_maps))\n",
    "\n",
    "            encoder_state = attn_cell.initial_state()\n",
    "\n",
    "            training_helper = tf.contrib.seq2seq.ScheduledEmbeddingTrainingHelper(\n",
    "                    inputs = tf.nn.embedding_lookup(decoder_embeddings, decoder_input),\n",
    "                    sequence_length = self.Y_seq_len,\n",
    "                    embedding = decoder_embeddings,\n",
    "                    sampling_probability = 0.5,\n",
    "                    time_major = False)\n",
    "            training_decoder = tf.contrib.seq2seq.BasicDecoder(\n",
    "                    cell = attn_cell,\n",
    "                    helper = training_helper,\n",
    "                    initial_state = encoder_state,\n",
    "                    output_layer = None)\n",
    "            training_decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "                    decoder = training_decoder,\n",
    "                    impute_finished = True,\n",
    "                    maximum_iterations = tf.reduce_max(self.Y_seq_len))\n",
    "        \n",
    "        with tf.variable_scope(\"attn_cell\", reuse=True):\n",
    "            attn_meca = AttentionMechanism(img, attention_size, tiles=beam_width)\n",
    "            recu_cell = tf.nn.rnn_cell.LSTMCell(size_layer, reuse = True)\n",
    "            attn_cell = AttentionCell(recu_cell, attn_meca, 1.0,\n",
    "                        attention_size, attention_size, size_layer, len(encode_maps))\n",
    "            \n",
    "            encoder_state = attn_cell.initial_state()\n",
    "            \n",
    "            predicting_decoder = tf.contrib.seq2seq.BeamSearchDecoder(\n",
    "                cell = attn_cell,\n",
    "                embedding = decoder_embeddings,\n",
    "                start_tokens = tf.tile(tf.constant([GO], dtype=tf.int32), [batch_size]),\n",
    "                end_token = EOS,\n",
    "                initial_state = tf.contrib.seq2seq.tile_batch(encoder_state, beam_width),\n",
    "                beam_width = beam_width,\n",
    "                output_layer = None,\n",
    "                length_penalty_weight = 0.0)\n",
    "            predicting_decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "                decoder = predicting_decoder,\n",
    "                impute_finished = False,\n",
    "                maximum_iterations = x_len)\n",
    "            \n",
    "        self.training_logits = training_decoder_output.rnn_output\n",
    "        self.predicting_ids = predicting_decoder_output.predicted_ids\n",
    "        \n",
    "        masks = tf.sequence_mask(self.Y_seq_len, tf.reduce_max(self.Y_seq_len), dtype=tf.float32)\n",
    "        self.cost = tf.contrib.seq2seq.sequence_loss(logits = self.training_logits,\n",
    "                                                     targets = self.Y,\n",
    "                                                     weights = masks)\n",
    "        self.optimizer = tf.train.AdamOptimizer(learning_rate).minimize(self.cost)\n",
    "        y_t = tf.argmax(self.training_logits,axis=2)\n",
    "        y_t = tf.cast(y_t, tf.int32)\n",
    "        self.prediction = tf.boolean_mask(y_t, masks)\n",
    "        mask_label = tf.boolean_mask(self.Y, masks)\n",
    "        correct_pred = tf.equal(self.prediction, mask_label)\n",
    "        correct_index = tf.cast(correct_pred, tf.float32)\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W0829 22:55:10.605042 139914025953088 deprecation.py:506] From /home/husein/.local/lib/python3.6/site-packages/tensorflow/python/util/deprecation.py:507: calling count_nonzero (from tensorflow.python.ops.math_ops) with axis is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "reduction_indices is deprecated, use axis instead\n",
      "W0829 22:55:10.637653 139914025953088 deprecation.py:323] From <ipython-input-16-e4ec94e9331f>:19: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.keras.layers.Conv2D` instead.\n",
      "W0829 22:55:10.642420 139914025953088 deprecation.py:506] From /home/husein/.local/lib/python3.6/site-packages/tensorflow/python/ops/init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "W0829 22:55:10.882626 139914025953088 deprecation.py:323] From <ipython-input-16-e4ec94e9331f>:20: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.MaxPooling2D instead.\n",
      "W0829 22:55:11.109754 139914025953088 deprecation.py:323] From <ipython-input-14-b2dd412390f9>:50: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.cast` instead.\n",
      "W0829 22:55:11.177250 139914025953088 deprecation.py:323] From <ipython-input-12-f29c2694405e>:40: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.dense instead.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"add_1:0\", shape=(?, 6, 28, 512), dtype=float32)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0829 22:55:11.449484 139914025953088 deprecation.py:323] From <ipython-input-16-e4ec94e9331f>:42: LSTMCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.LSTMCell, and will be replaced by that in Tensorflow 2.0.\n",
      "W0829 22:55:11.623882 139914025953088 deprecation.py:506] From /home/husein/.local/lib/python3.6/site-packages/tensorflow/python/ops/rnn_cell_impl.py:961: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "W0829 22:55:12.201008 139914025953088 deprecation.py:506] From <ipython-input-13-d84cd8088212>:75: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "W0829 22:55:12.309709 139914025953088 deprecation.py:323] From /home/husein/.local/lib/python3.6/site-packages/tensorflow/contrib/seq2seq/python/ops/helper.py:107: multinomial (from tensorflow.python.ops.random_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.random.categorical` instead.\n",
      "W0829 22:55:12.324810 139914025953088 deprecation.py:323] From /home/husein/.local/lib/python3.6/site-packages/tensorflow/contrib/seq2seq/python/ops/helper.py:379: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "W0829 22:55:12.765539 139914025953088 deprecation.py:323] From /home/husein/.local/lib/python3.6/site-packages/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py:985: to_int64 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.cast` instead.\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/ops/gradients_util.py:93: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Model()\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pad_sentence_batch(sentence_batch, pad_int):\n",
    "    padded_seqs = []\n",
    "    seq_lens = []\n",
    "    max_sentence_len = max([len(sentence) for sentence in sentence_batch])\n",
    "    for sentence in sentence_batch:\n",
    "        padded_seqs.append(sentence + [pad_int] * (max_sentence_len - len(sentence)))\n",
    "        seq_lens.append(len(sentence))\n",
    "    return padded_seqs, seq_lens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2.9175596, 0.10526316)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_x = train_X[:5]\n",
    "batch_x = np.array(batch_x).reshape((len(batch_x), image_height, image_width,image_channel))\n",
    "y = train_Y[:5]\n",
    "batch_y, _ = pad_sentence_batch(y, 0)\n",
    "loss, logits, acc = sess.run([model.cost, model.training_logits, model.accuracy], feed_dict = {model.X: batch_x,\n",
    "                                                          model.Y: batch_y})\n",
    "loss, acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 625/625 [01:30<00:00,  7.02it/s, accuracy=0.906, cost=0.305]\n",
      "minibatch loop: 100%|██████████| 157/157 [00:09<00:00, 12.39it/s, accuracy=0.948, cost=0.205]\n",
      "minibatch loop:   0%|          | 1/625 [00:00<01:29,  6.94it/s, accuracy=0.932, cost=0.252]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 1, training avg loss 1.519639, training avg acc 0.485971\n",
      "epoch 1, testing avg loss 0.280290, testing avg acc 0.919022\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 625/625 [01:27<00:00,  7.17it/s, accuracy=0.991, cost=0.0442]\n",
      "minibatch loop: 100%|██████████| 157/157 [00:08<00:00, 17.64it/s, accuracy=0.991, cost=0.0261]\n",
      "minibatch loop:   0%|          | 1/625 [00:00<01:28,  7.07it/s, accuracy=0.994, cost=0.0253]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 2, training avg loss 0.098987, training avg acc 0.973223\n",
      "epoch 2, testing avg loss 0.036170, testing avg acc 0.991269\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 625/625 [01:27<00:00,  7.08it/s, accuracy=0.994, cost=0.0228] \n",
      "minibatch loop: 100%|██████████| 157/157 [00:08<00:00, 17.64it/s, accuracy=1, cost=0.00788]    \n",
      "minibatch loop:   0%|          | 1/625 [00:00<01:26,  7.23it/s, accuracy=0.999, cost=0.00922]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 3, training avg loss 0.026419, training avg acc 0.993798\n",
      "epoch 3, testing avg loss 0.015001, testing avg acc 0.996805\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 625/625 [01:28<00:00,  7.09it/s, accuracy=0.997, cost=0.0117] \n",
      "minibatch loop: 100%|██████████| 157/157 [00:08<00:00, 17.56it/s, accuracy=1, cost=0.00262]    \n",
      "minibatch loop:   0%|          | 1/625 [00:00<01:28,  7.08it/s, accuracy=0.999, cost=0.00642]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 4, training avg loss 0.013676, training avg acc 0.996753\n",
      "epoch 4, testing avg loss 0.009664, testing avg acc 0.997876\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 625/625 [01:27<00:00,  6.94it/s, accuracy=1, cost=0.00306]    \n",
      "minibatch loop: 100%|██████████| 157/157 [00:08<00:00, 17.63it/s, accuracy=1, cost=0.00112]    \n",
      "minibatch loop:   0%|          | 1/625 [00:00<01:28,  7.06it/s, accuracy=0.995, cost=0.042]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 5, training avg loss 0.012101, training avg acc 0.997094\n",
      "epoch 5, testing avg loss 0.009274, testing avg acc 0.998141\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 625/625 [01:27<00:00,  7.12it/s, accuracy=0.999, cost=0.00509]\n",
      "minibatch loop: 100%|██████████| 157/157 [00:08<00:00, 17.68it/s, accuracy=1, cost=0.00279]    \n",
      "minibatch loop:   0%|          | 1/625 [00:00<01:27,  7.16it/s, accuracy=1, cost=0.00305]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 6, training avg loss 0.009302, training avg acc 0.997793\n",
      "epoch 6, testing avg loss 0.010704, testing avg acc 0.997450\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 625/625 [01:27<00:00,  7.08it/s, accuracy=0.999, cost=0.00318]\n",
      "minibatch loop: 100%|██████████| 157/157 [00:08<00:00, 17.55it/s, accuracy=1, cost=0.000676]   \n",
      "minibatch loop:   0%|          | 1/625 [00:00<01:30,  6.93it/s, accuracy=0.998, cost=0.00467]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 7, training avg loss 0.005349, training avg acc 0.998737\n",
      "epoch 7, testing avg loss 0.006314, testing avg acc 0.998461\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 625/625 [01:27<00:00,  7.15it/s, accuracy=0.999, cost=0.00234]\n",
      "minibatch loop: 100%|██████████| 157/157 [00:08<00:00, 17.59it/s, accuracy=1, cost=0.000823]   \n",
      "minibatch loop:   0%|          | 1/625 [00:00<01:26,  7.18it/s, accuracy=1, cost=0.00265]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 8, training avg loss 0.007786, training avg acc 0.998095\n",
      "epoch 8, testing avg loss 0.007873, testing avg acc 0.998013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 625/625 [01:27<00:00,  7.15it/s, accuracy=0.999, cost=0.00574]\n",
      "minibatch loop: 100%|██████████| 157/157 [00:08<00:00, 17.64it/s, accuracy=1, cost=0.000296]   \n",
      "minibatch loop:   0%|          | 1/625 [00:00<01:34,  6.59it/s, accuracy=1, cost=0.000733]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 9, training avg loss 0.003831, training avg acc 0.999121\n",
      "epoch 9, testing avg loss 0.003734, testing avg acc 0.999153\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 625/625 [01:27<00:00,  6.96it/s, accuracy=1, cost=0.00114]    \n",
      "minibatch loop: 100%|██████████| 157/157 [00:08<00:00, 17.64it/s, accuracy=1, cost=0.000188]   \n",
      "minibatch loop:   0%|          | 1/625 [00:00<01:24,  7.35it/s, accuracy=1, cost=0.00105]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 10, training avg loss 0.004331, training avg acc 0.998917\n",
      "epoch 10, testing avg loss 0.003307, testing avg acc 0.999179\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 625/625 [01:27<00:00,  7.04it/s, accuracy=0.999, cost=0.00771]\n",
      "minibatch loop: 100%|██████████| 157/157 [00:08<00:00, 17.63it/s, accuracy=1, cost=0.000244]   \n",
      "minibatch loop:   0%|          | 1/625 [00:00<01:29,  6.94it/s, accuracy=1, cost=0.000596]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 11, training avg loss 0.002805, training avg acc 0.999352\n",
      "epoch 11, testing avg loss 0.003485, testing avg acc 0.999247\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 625/625 [01:27<00:00,  7.11it/s, accuracy=0.999, cost=0.00233]\n",
      "minibatch loop: 100%|██████████| 157/157 [00:08<00:00, 17.59it/s, accuracy=1, cost=0.000613]   \n",
      "minibatch loop:   0%|          | 1/625 [00:00<01:27,  7.16it/s, accuracy=0.999, cost=0.00137]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 12, training avg loss 0.003280, training avg acc 0.999200\n",
      "epoch 12, testing avg loss 0.003192, testing avg acc 0.999281\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 625/625 [01:27<00:00,  7.15it/s, accuracy=1, cost=0.00131]    \n",
      "minibatch loop: 100%|██████████| 157/157 [00:08<00:00, 17.63it/s, accuracy=1, cost=0.000401]   \n",
      "minibatch loop:   0%|          | 1/625 [00:00<01:27,  7.14it/s, accuracy=0.999, cost=0.00215]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 13, training avg loss 0.002531, training avg acc 0.999378\n",
      "epoch 13, testing avg loss 0.003290, testing avg acc 0.999273\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 625/625 [01:27<00:00,  7.08it/s, accuracy=1, cost=0.000158]   \n",
      "minibatch loop: 100%|██████████| 157/157 [00:08<00:00, 17.56it/s, accuracy=1, cost=5.84e-5]    \n",
      "minibatch loop:   0%|          | 1/625 [00:00<01:27,  7.11it/s, accuracy=1, cost=7.04e-5]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 14, training avg loss 0.001583, training avg acc 0.999608\n",
      "epoch 14, testing avg loss 0.001468, testing avg acc 0.999667\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 625/625 [01:27<00:00,  7.10it/s, accuracy=1, cost=8.3e-5]      \n",
      "minibatch loop: 100%|██████████| 157/157 [00:09<00:00, 17.41it/s, accuracy=1, cost=5.81e-5]    \n",
      "minibatch loop:   0%|          | 1/625 [00:00<01:25,  7.30it/s, accuracy=1, cost=3.57e-5]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 15, training avg loss 0.001462, training avg acc 0.999669\n",
      "epoch 15, testing avg loss 0.000975, testing avg acc 0.999729\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 625/625 [01:27<00:00,  7.18it/s, accuracy=1, cost=9.13e-5]     \n",
      "minibatch loop: 100%|██████████| 157/157 [00:08<00:00, 17.60it/s, accuracy=1, cost=2.84e-5]    \n",
      "minibatch loop:   0%|          | 1/625 [00:00<01:27,  7.15it/s, accuracy=1, cost=3.45e-5]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 16, training avg loss 0.000887, training avg acc 0.999786\n",
      "epoch 16, testing avg loss 0.000834, testing avg acc 0.999796\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 625/625 [01:27<00:00,  7.01it/s, accuracy=1, cost=9.9e-5]      \n",
      "minibatch loop: 100%|██████████| 157/157 [00:08<00:00, 17.57it/s, accuracy=1, cost=4.35e-5]    \n",
      "minibatch loop:   0%|          | 1/625 [00:00<01:27,  7.10it/s, accuracy=1, cost=5.74e-5]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 17, training avg loss 0.003594, training avg acc 0.999160\n",
      "epoch 17, testing avg loss 0.001189, testing avg acc 0.999729\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 625/625 [01:28<00:00,  7.07it/s, accuracy=1, cost=0.000129]    \n",
      "minibatch loop: 100%|██████████| 157/157 [00:08<00:00, 17.59it/s, accuracy=1, cost=3.95e-5]     \n",
      "minibatch loop:   0%|          | 1/625 [00:00<01:28,  7.06it/s, accuracy=1, cost=3.42e-5]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 18, training avg loss 0.000426, training avg acc 0.999906\n",
      "epoch 18, testing avg loss 0.000875, testing avg acc 0.999742\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 625/625 [01:27<00:00,  7.17it/s, accuracy=0.999, cost=0.00295] \n",
      "minibatch loop: 100%|██████████| 157/157 [00:08<00:00, 17.69it/s, accuracy=1, cost=0.00165]    \n",
      "minibatch loop:   0%|          | 1/625 [00:00<01:27,  7.17it/s, accuracy=0.999, cost=0.00196]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 19, training avg loss 0.001924, training avg acc 0.999559\n",
      "epoch 19, testing avg loss 0.004051, testing avg acc 0.999055\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 625/625 [01:27<00:00,  7.15it/s, accuracy=1, cost=5.21e-5]    \n",
      "minibatch loop: 100%|██████████| 157/157 [00:08<00:00, 17.54it/s, accuracy=1, cost=8.75e-5]    "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 20, training avg loss 0.001526, training avg acc 0.999603\n",
      "epoch 20, testing avg loss 0.000684, testing avg acc 0.999837\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "for e in range(epoch):\n",
    "    pbar = tqdm(\n",
    "        range(0, len(train_X), batch_size), desc = 'minibatch loop')\n",
    "    train_loss, train_acc, test_loss, test_acc = [], [], [], []\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(train_X))\n",
    "        batch_x = train_X[i : index]\n",
    "        batch_x = np.array(batch_x).reshape((len(batch_x), image_height, image_width,image_channel))\n",
    "        y = train_Y[i : index]\n",
    "        batch_y, _ = pad_sentence_batch(y, 0)\n",
    "        feed = {model.X: batch_x,\n",
    "                model.Y: batch_y}\n",
    "        accuracy, loss, _ = sess.run([model.accuracy,model.cost,model.optimizer],\n",
    "                                    feed_dict = feed)\n",
    "        train_loss.append(loss)\n",
    "        train_acc.append(accuracy)\n",
    "        pbar.set_postfix(cost = loss, accuracy = accuracy)\n",
    "    \n",
    "    \n",
    "    pbar = tqdm(\n",
    "        range(0, len(test_X), batch_size), desc = 'minibatch loop')\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(test_X))\n",
    "        batch_x = test_X[i : index]\n",
    "        batch_x = np.array(batch_x).reshape((len(batch_x), image_height, image_width,image_channel))\n",
    "        y = test_Y[i : index]\n",
    "        batch_y, _ = pad_sentence_batch(y, 0)\n",
    "        feed = {model.X: batch_x,\n",
    "                model.Y: batch_y,}\n",
    "        accuracy, loss = sess.run([model.accuracy,model.cost],\n",
    "                                    feed_dict = feed)\n",
    "\n",
    "        test_loss.append(loss)\n",
    "        test_acc.append(accuracy)\n",
    "        pbar.set_postfix(cost = loss, accuracy = accuracy)\n",
    "    \n",
    "    print('epoch %d, training avg loss %f, training avg acc %f'%(e+1,\n",
    "                                                                 np.mean(train_loss),np.mean(train_acc)))\n",
    "    print('epoch %d, testing avg loss %f, testing avg acc %f'%(e+1,\n",
    "                                                              np.mean(test_loss),np.mean(test_acc)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(9, 15)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "decoded = sess.run(model.predicting_ids, feed_dict = {model.X: batch_x[:1],\n",
    "                                          model.Y: batch_y[:1]})[0]\n",
    "decoded.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9+(0+1)\n",
      "9+(0+1)\n",
      "9+(0+1)\n",
      "9+(0+1)\n",
      "9+(0+1)\n",
      "9+(0+1)\n",
      "9+(0+1)\n",
      "9+(0+1)\n",
      "9+(0+1)\n",
      "9+(0+1)\n",
      "9+(0+1)\n",
      "9+(0+1)\n",
      "9+(0+1)\n",
      "9+(0+1)\n",
      "9+(0+1)\n"
     ]
    }
   ],
   "source": [
    "for i in range(decoded.shape[1]):\n",
    "    d = decoded[:,0]\n",
    "    print(''.join([decode_maps[i] for i in d if i not in [0,1,2]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAACDCAYAAACUaEA8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOy9d5hl6VXe+1s7nHwqh+6qznF6Qk/WJA0z0khWGAmBBTIIgwCBkA22L9gXG/A1ugb84AsIfG2RFBAYhECDQEJCiBlJM5okTc49PR2mOlXoyunkvT//sb59zqnqqu6qru7p7pn9Pk8/fersffb+dlp7fWu9611ijCFGjBgxYlx6cC70AGLEiBEjxtkhNuAxYsSIcYkiNuAxYsSIcYkiNuAxYsSIcYkiNuAxYsSIcYkiNuAxYsSIcYkiNuCvM4jIZ0Xk1+3n20Vk/4Ue05kgIg+LyLWv4f7eKyJ/9Vrt71xBRO4UkeMXYL/dIvKyiKRfw33+joj8q9dqf5cqYgP+OoYx5kFjzO4zrSciPy4iD61m2yLSLyJfEpEJETkuIh89mzGKyHuBWWPM003f/byIDIvIjIh8RkSSZ7HdXxOR50WkJiIfa15mjPl74AoR2Xs2Y17FGLaIiBER73zu5zT7v1VEHhORWRF5TkTefJab+k/AZ40xRbvdpL0uM/Y6/cJZjC0hIveIyIA9R3cuWuW3gV8WkcRZjvkNgdiAX8S4UA/+CvHnwKtAL3A38N9E5C1LrSgi9y/xgEb4KPC/m9Z9B2ow7gI2A9uA/3eZ7X5WRH58me0eBH4R+Ooyy/8S+Mgyyy55iEgH8PfAbwFtwP8H/L2ItC+z/pIVffbl+SH0ekf4GLATvT5vAX5RRN65zO8HRGTLMsN8CPiXwPDiBcaYIeBl4HuX+W0MYgP+msPe0L8kIi+JyKSI/ImIpOyyO603+x9FZBj4E/v9e0TkGRGZEpFHmj1HEblWRJ6yXtZfAammZQum3CKyUUS+KCKjIjIuIv9LRPYAfwjcIiJzIjK1gmPIAXcCv2GMqRpjngXuAX5yleciAbwVeKDp6w8BnzbGvGiMmQR+Dfjx1WwXwBjzp8aYrwGzy6xyP/riWelYv2C9zWkR+baIXNG0LG2n/Efs8odsuOHbdpUpe25vEZGPicifN/12gZcuIj8hIvvs9TwsIj+z2mO3uBUYNsZ8wRgTGGP+HBgF/vkqt3MTMGWMaQ7dfAj4NWPMpDFmH/BJVnmNjDEVY8zvGWMeAoJlVrufVVyjNyJiA35h8CPAO4DtwC7gPzctWwd0oN7NR2xs+DPAzwCdwB8BX7bT2ATwd6gH2wF8AXj/UjsUERf4CnAE2AL0A5+3D+BHgUeNMTljTJtd/4Mi8twy45dF/0efr1zpCbDYCYSLjMMVwLNNfz8L9IpI5yq3fSbsA7aISMsK1/8aOt4e4CngL5qW/TZwPWo0O1DPPwS+xy5vs+f20RXs5yTwHqAF+Angd0XkuqVWFJHfF5HfP822ZIm/V3uNrgLqeRTrwa/n1Gt0Bece+4Crz8N2XzeIDfiFwf8yxhwzxkwAvwH8cNOyEPhVY0zZxhw/AvyRMea71pP6U6AM3Gz/+cDvWU/4HuDxZfb5JqAP+L+NMfPGmJL1fpaEMeZzxpglY8TGmFngYeD/EZGUNTDvBzKrOAegU/vFHnIOmG76O/qcX+W2z4Rov20rWdkY8xljzKwxpoyGEK4WkVYRcdCZx78zxpyw1+gRu96qYYz5qjHmkFE8APwTcPsy6/5rY8y/XmZTjwJ9IvLDIuKLyIdQh2Gt1yhn/198jc719cHud0XX542K2IBfGBxr+nwENawRRo0xpaa/NwP/3oZPpmyIY6P9TR9wwixUJDuyzD43AkeMMbW1Dx/QWcRW9Fj+AI2RNodrmsf7ZuArTd/9J7vaJKc++HOo9xkh+jxrt/tc03Y/CPx+03ZP540uRrTflYSMXBH5TRE5JCIzwIBd1GX/pYBDq9j36fb1LhH5jmhyeAp4t93HqmCMGQfeB/wCMAK8E7gPe41E5M2LrtGCayaNhOfiazRn/198jaLrs2nRdjcBzzV998FVHEaeFVyfNzIu5iTZ6xkbmz5vAgab/l6cTDqGxpp/Y/FGROQOoF9EpMmIb2JpY3IM2CQi3hJGfNWSlMaYI+hUPxrL54DHmpa3NS27H/iYMeb+RZs5qIul3xhzwn73Ijpt/mv799XAiDVINM8KROSzwP3GmM+udvzAHmDAGDOzgnU/iBrDt6HGuxU1bAKMASXUu3120e+WOq/zLPSC10UfbMLwb4AfA75kjKmKyN9xaihkRbAe/I122x5wGPgdu+whmrxbewst5e0+B/x80zYnRWQIvS732q+vRq8bxpiji7Y7ANxpjBk4i0PYw6nnNEYTYg/8wuBnRWSDKFPgV4DTcZI/CXxURG4SRVZE7haRPDpNrgH/1k6T/zkaKlkKjwFDwG/abaRE5Da7bATYIKugbInIHhHJi9LB/iXwz4CPr/T3oIks1Cu8o+nrPwM+LCKXi0gbmh/47Gq2a8fniyaHHcCzx+s2rXIHGteO1v+YfdEshTwathpHje9/azqGEM1RfFxE+qy3fos1xqNoSGxb07aeAb7HeqqtwC81LUsA0e9qIvIu9LyeFUQT3L6N8/82cMwY8/VVbuYxoE1E+pu++zPgP4tIu4hcBvw0Z3eNkvYaASTsNWp+WS24RjFORWzALww+h8Y2D6Pe8q8vt6Ix5gn0AflfqNd3EJvxtwbwn9u/J4B/AXxxme0EwHuBHcBRdCr9L+zib6Ie1LCIjAGIyI+IyIunOYZ32PFPoknQdxpjRk971Evjj4AfbRrnP6KUt2/ZcR4BfvUstvtJoIjmF37Ffv7RpuU/bPcdYSMa118Kf2bHcQJ4CfjOouX/AXgezT9MAP8dcIwxBTTH8bANH9xsjLkXfWE/BzyJJpaBem7h36Kzj0nU8//ycgcoIn8oIn+43HI0mTqGzr7WA99/mnWXhL3HPovS/SL8KnrfHkEZRL9lr9tqsR+9Lv3A1+3nzQAish64HE3Sx1gGEjd0eG1hp5Q/ZYy570KP5WKBiDwM/FxzMc953t97gR81xnyg6btngLuiUE2MBkSkG3gQuDYq5nkN9vk7wCFjzGryGm84xAb8NUZswGPEiHGuEIdQYsSIEeMSxZoMuIi8U0T2i8jBJmpYjNPAGLMl9r5jxIhxLnDWIRSb0X8FeDuaEHsc+GFjzEvnbngxYsSIEWM5rMUDfxNw0Bhz2GaqP49yZWPEiBEjxmuAtRTy9LOwovA4KnyzLLo6XLNlo7/s8hfnOwDwD5UWfF/tyXLFugZD7cWRbv1gWPB9jBgxYrwe8eRz5TFjTPfi7897JaaIfAQr27mp3+Oxr29ccr2jtTkeKW4A4PPDb2L/N7YDUN5eYnPfOKOzKsGQvyfPjRtVvKzc5jKxR/ilH/wbAN6XHaDdXa3UwxsDc2GJoKkw0EXIOanT/CJGjBgXC9z1B5eUyFhLCOUEC0vCN9jvFsAY88fGmBuMMTd0d7qLF8eIESNGjLPEWjzwx4GdIrIVNdw/hFaOnRU2eTnyGbX/Ozf9vQppAgPVLj5++G1497cC4BcaMh5TOxy+9+5H2eKPne1uX7cohBWmwwoAU6HDYJDnu/M7APCdGrdnXmGDp7pE3W6SpCwf2ooRI8bFibM24MaYmoj8HFoC6wKfMcacrvT6jIjCH9c3Oerd7iD/Zuu3+Pz7VeJj/ze241ytSpY/vftR3pZ7iT2+Gh9f4vBJhIKp8t2y6iR9afxaBmY7mK+o1Em56vHX3vX8+DaVp745fYhtXgEgDkHFiHEJYU0xcGPMPwD/cI7GEiNGjBgxVoGLXk42Cq3s3PT3+sVPNJZ1uxU2ebmlf/gGRdVognc0EA6UewF4YWw9Y0OtOHM6tZEA5jz47aPvAuDGvYf4yXXa2+HKxDjr3QyuxEW6lzKmQ5UsGQlCRoM0U6HOrPJOiW5HZ1vdrqHdSb+hr/VkoOficG2hKbxUbMtFb8BBp/XXL5n/jBtWNyMwIUOBPrjfLW3j0QlVMR0fy5M+6mOjJIhNI9Qy+uA+PbeLX75Mewb8lz1f5brkML2uNoKPY+MXL+ZCpduOBDVmQx/H9iXOS43pUK/bV2av5QuHr2VuThlHuVyJOzYcBOBfdT1Aq2N4o1ILlPmmPIyI+RaFZz98Snj24jxLl4QBj3FmVE3AZFjimXIPAJ8aeDNDL+vnzLCDNw9uRR9wCfU3ntWV80rCLMrB/zV5N7962Ve4PaWJ4aR7qgEPjG6gbGqM2UTpYE29vJI5df2UVGlzCnS7usP1boKME798zwbRDGssKHKgph7ilyav48nxTVRDfSH/u23fICG63nMz/cwdbiU9osvm1id5LqPS3ofbOtjszVy0xul8oGoC9lWrANw3t5f/+fhbANj8BYdeqswNaqOhTxTu4G23X/xF5W/cuVOMGDFiXOKIPfDXCebCMs+U2/jD43cCcOLVLjLW60pMG5wlOmHaGTdu0ZCY1EYoE6MtHNy6jmuSJwFoX/SbqgkYs2Gaw7UMX5u5HoBvDO3i5FgLYVm9OTcdENjPBIKTrvG2XS8DsDszQp8/CcA6b5qN3gzrXfXIY8/89JgLtVfys5VOfvfI2wF4ZX8f3qxL0KuzoVf61rM3fRSAnuQcYTok9PVe8KcdToxpx7OBDd0M+WNssE1w3mjhsk88fQedD0f3mz4g47fpOfw31z5wgUa1OsQG/BJHwYYwjtRcHp7fxb79Ws2aPu6RsL3E7Wz6FBjbvMo4grG2VtwQX4Jlp2bTYalOT7xn9AYeOahxdmc0QWLWwUZJkIDGS0Mg9H2+feBaAL6VNtTS9u3RXuGde17iiqzWALS5BXwbpO9xZ+nzZul1dTStTnqlp+V1gyhcNRkWGQkcni9r+OPrk1dyYJ9+zpzQi1dode1vHHpcvfjf2/4U89cmeGjmSgASU0JwVM/jH2XeTPflM+Qdbcnas0S47PWMn732AT6duQWA+Wdb2X3XIT66Ttu63po+dkkkMeMQSowYMWJcoog98EsYc2GJw9bLvWf6Rv765WtJnlQvzJ8DJ0paLqEYbBz9BxCkoNKunt763il6/SkSsnQj9OnQ8HRhMwCPHdmMN6hsFX9acMvg6oQAp7pwpxJCcrLxd2Cn9LVsim8duo5/yl8DQJg2GFd/67VUuHv3C3ywQwuOLvdLbyj9lqoJmLZMk5eqWX5/6K08/qqeexlKkZrSa+TPQ5BE2ycDVePWE8bdbpEt6XHub9VpWHLCIzWmvyscauXvuq8j26thmZuS43S52dfq8C4IfHHZm9BnpK3lOe64TsN6XKf/bfP0gWp3L37vGy5xAx4xL8YDG78NU8yGOj0MELrdWTodvTm7XPd1MwWPptUFE/D1ub0AfPXIFZiBLAn7UEeMk2Y0QiZgXKFmn9VyuyHsUsvbm5ml053Dp2HAo/0NBQWer6zj2SkN09TGUqSnrRGZs/u0u13qpdEM1xp4dwoS08Cw2LEJoa+fKy0eXypfzfAuZQb89LoH2O0rzetcl/8386YHa3kATgZ5SqFPp5UcWOfO0GvfUOeTKx8xTSbDEi9VdCz//4m7eGZgI/5RfWGmT0r9JSmhnjdvVg3TiVIbh6qavdCQVABOdGEaL9nEhMOjL+ygK6nHt7P7frreOIQUNnk5Np1iAS+tHMwlZ8ADEzJpH7bBwOX58ib+YlBVbMcKWUpVPaRq1SOTKvOBLdon9wMtT9P6OgkYhdZKDtY8Xi2qwuTMTJrUtOBaJd6IKtiMyOMOE0ItA7WMbqfaUaOvdwqAG9qO0ufOkpTGrRHtbyRI8IfH7uCVl2ycfdDF12dfjYk5s+FeCmJoGP4QsE1G/FmBQ0kem98JwLEdbfzyDi38vTU5sSTF8WxQNQEjgZ6wT0/cxjdP7AJg7GQLlBwkq17ZlVsG+eWNXwWg2w1wz1MEctJ63Q8W1/O3Y+oavji0Hm8wSWpcX25OxeA05zYMOMqO44GXd7HzOk1Cf3/L0+xIDZPrngegPNGKW9ZtJGahMucyWFSdoYkgBW+sMPglj9eJSYsRI0aMNx4uGQ+8bNS9GAnKPGVZEJ8buYnHX94KNX0PSU1wrHchNWEqmeVv3asBqG5weUvuJda5Wo54votJAhNSNgu5e1HBxFoKJ6KwEcBArZenRtUbluEUXgmcYGGxToTQVc8boNIKpd4Ap13n0v2d09y57gAA72t5ho2eg2tj4NNhkeP2MB6Yv4LDI12kRnT8yaml6YkRIo/fLAqnS3h6Tz3yLKVocKpCRJEZTHZyf88eANranuRKKZ6TsFhII2yyf6aXsSH1SNNHfbx5qGX1MXmBPv4mfwMApdZn2ebP0OWce/rjlL1235nbziOvqC5+8nCSxCy4JT1xzhLMIlt8ya27DnFdZgCArBNyffIEv3L51wD4Dd5FKdDjS0wLTkWYq2pYZiZMUQjnYyrnJYRLwoAHJqRkjeGBaisfP6z81xP7ekmPOnVjJaZBmZNADdb0hFYj/mnuLXym5zbeeYUKJt7Zuo9dvk4z+7wanedYEyLEMGeqHKvpU9XhVOi1XOe1GvDhQH//pfFrGRlSTm96SnBLZtnQiXE1bAJQaQtJ9hb4wK6nAPie3MvstLHlVscl56TqL8wjNeHLM5pg/POXb8QcyeDpbHxZeiKo0Y5i2UESgkRjim+8hUbdqVIv83ebkp9i9IWUmLUrD/jcU70RgKFrWvmp3m9zpeVKrlXTo2JfEhOlDM68Wx9X89gSAynuKej+H926lR/b/B3em30FONcGXLc1VGqFWX1EEzPgFZe+vhh9xwU5XbglM85OfxzQOG9gQvy0Ns/66O4H+bR7KwDFpzpxi8KRca3CfaxrOzv9J9hkw2dvZI2USwXxFYoRI0aMSxQXvQc+HRYZCUIOVDsB+PLEtRwfUo8hNeaQnAIJFnptACyaooe+UBtN8s0jmhT6WvdeerdMAPD+TU+zMznCRk//7nUrdJzl1LhZ3ezF8maemNsKwLb0KNektDpuhz9z1iyGgqlyrKbnYqjQghRt5WOVU4458nJDV6iloZrTFYK2Gru6xnlLbh8A1ybnaXUW0qYmAmXvvFTeyHcm9BhqR3JkRgWv0ERPjDxCG+kIXbtTh7rHX81CNR8SpnVlv71ET5tmP9N+lUP7+ur0x8Ss1MMEEuo/r2jDBlVwAr1lH3Z3Ugld/vMGTSquRZQpMKau4VINnQWzOAmpFyclQ8DOnoZbWzjU28NUWoWh1p/lvheOI2Q8LDJQ7QPg6Gw7iUmnPhYj0EQOqt/rtbRQzQM22bo1OUqmaT1XHPqtzvtVqWNsaNXZ1r5MB8lxoXRU6Uj/1LKHN2dfoc/Ta3++krQxzh0uSgPezDR5qZrl8+M38/WXNfZpJpIkJ2xZ8JyyH1bCfHArBrcCvq1OTEx7TI9oeOUPX3k73voCV/drNeB7up7lnVltQZc03hkNbUT7GgmKvFBR4/qZ4Tfz+HPbcUr6W5MwdG5RIvS/33kv16WOs8GyKFbzkqgaw3BVwybHJttw56L4/xIrRw+xo/HRaosa0P7+CX6s71F2+DO6f1kYRy6EFQYDHdO3pi/jxai6c0q53ktO41EDE1iadrnT1F8YTleZHetGeWvPfkArBTOuGol13jTfbtnNfQd3A1B8NUP6pKVClhcyWyQAO2T8cY+Xx3p4pENjxH7mAH2e/i4nybOe/hsjixgxIGH0AhG8gu6jXHYpBj7Vc2jkQgyDNY8/G9QQx/EDPWTnbE4nAONJRNCps35AQ1SlzRU+fN3DANycfpXWRfdUdD6yUqEtoU5GkA6RwMWf1WWjMzmOVju4PKH3fqfjxmGUixwXpQEPMYxabvefnLydB17eRXLAyptONBWohIBA0BRrNV59IzhVawRoSoxFSnwFg1OLDIVDuZzlJV+Tozuyo9xoveX2FXh2kQE/UsvwWwPvAGDghT5yxx38Oev1Jh1mJ9S4/8rk9/HLN3yNd2fVe1uNAS8ZmAjUYyrMpEgVIt73QsNqpIk26FnD2qJWfmfbKFcnT9DrquFeHJOvEvBkSQ3js2P9eFM2Dju7xAszSlQ6EKSEqlK2qXTX2LNTX4h39z7PjenDtDmaNG1zGrG7gjFs8ceYq+k5eDjYTlH0LZCYEvz5hQVJkTFNTgjF/W18vHQXAI9v2cq7O54D4LrkIBu89IpzDa4IKdEAfWgEqUWJcBDTiDtL1eAW7fme8BkutVAy5444XTUBJ4JWxovqLTslqc8GahmbE7B5BJpe2EESkrkyO5IjAGz13GXvqZQE9CbVi5H2CrWxdP1FUJpO8q2pPexK6HYyiQo5eeMUTl2KiF+vMWLEiHGJ4iL1wEMGA6V1DRVaYNarhz6aCxiMNOK7AKUeQ9CjXl4qV2Z+JItrp4deUaf/zU0Noko2ryCEnjA/rp7Py129TLSq51E1tdN6cs10wYHqeibmdRvevODPN8bqFEydlVGe9Xm13E0hc3Atp0mPox6vXaLy0mnMTGq5kCu3qkf8A12P0+0anOaAqkVU1HKiopV8M4UUXjEq4Tx1/1GcPUiK0hP71EV829UvcWeblilfkzxuvcJTy7QDE5KRIr+w7l4Abm09xD9svAqA51/eSGrIryslesXG+dRzK1QKGrv/5vQeRi/Tz8H6R+lwh2mVlVEMA2Oo2nlWe6rIWDqatakH3JxXiTxgqQqFWmJJ/fOzhS8u/e40nWm9SYfba8wlI5UxVRKMzoVbohEis/9HTJqQZWJcQIcTsDOtHnZ/9xTHaw7uiHrr3oTPQ4e3c0vrIQA2e6+Qi128ixpnNOAishH4M6AXfYT/2BjzP0SkA/grYAswAHzAGDO53HZWg+ak0kQxgzvvYEOmiGkKDbhidTz0CQvXl9jVr9TAmzoH2HXVMMcrmvB8cHwHL77aR+pVDcUkpsGzVYtu2ZCcEoKkno6XWtfxSIdW/3U4z7PZWz7MUSNgItQHZqKWo1DS9ZyKsNjiRSXM3ozL0WI7h3Ltdh/TC5oJT4dFxoKoNVqak/Zl1uYWmA27GKvq3yILjcuC0EZzCMWHsKvClqxSyzZ6U6fVvJgNfV6c0bRccTxNJjr34SmHVFcxDFJQ7g7YvUOV7d7e/gI3JvWF0esmln0JuuLQ5WZptZa5193P5Sn93RdzN/DlF/cir6Tq5y+K9UsATpl6JWgw6XF8WvnNRzq7mEieIGdjHyuJ4wb2ZA3N5klMRYnD5fMrYjTcci7hIPR5NX5+4z8B8I8te3lwWENZPdk5nj+4AW+ucR9Ws7r/as7QlS+wPaH3/ukcjlYnwdszSn3ctH2cP8vdyqPlHQAkRzyqwymemdsEwBXJ4+QdfUjeSBo0lxJW8n6tAf/eGHM5cDPwsyJyOfCfgG8YY3YC37B/x4gRI0aM1whn9MCNMUPAkP08KyL7gH7gfcCddrU/Be4H/uO5GljVZiOLFR+vJAsSWVHSspqHUpch6NR57e07DvHB7u8CcHVinLzjMZvWjPp6f5JPlW/n+KwyT5yqq1V+aLGI1AwJK8xUPJrlc0kt2Nh5+Qjd7ggZK3LT3JKpahzGgywvlJRB8YVj11Id02l7unLqMUXeY2pCeOjxPaRv1u10dj9AW6iu5HCQ4ZnSLu4dvRyA54/11b3edLbChrYp1mdm6tuMvOxToiGmkdCtZQ3JTJUtKfXAu09TPhkSMhWmOVlQL9+dcxuzn3ChOxq6mrgEKHcY/O4iV7WpB35FYrheuLSSJG3kNfa4GVzRSqGrMsf5dtt2ZvNW8XBO6jopTqDXzVRseGVemBzXEMqzXRt5a3YfG+yJW0masRDqPioVDydSVKywrIuzHBNnLXDFocfNcoMVl9rd9SA/2aHMkq/MXcXLg70YT8+laaJptl05zs9sfZDNNj7okTl14xYZJ1Ev1PFlnN25Eb6b3aILjU9yQvjHF6/Q7V5doK/jEYA4lHKRYlUxcBHZAlwLfBfotcYdYBgNsZwzlGxdcBA40Fx6HYJ91qi0G5wNBb5/t7IP3tn6HFcn1Lh1WTlI37ILbk4fYXZjmv85faf+tpDDtQ+/zKtBqFfcTTlMT2qI4dnCJrb5Y5SMPlQHqy18dVrLqe/5zo04ZYf+yzWmeHK8BSk3WdJF0++oylBmIDXi8tRJbaj69dRVuNYi/On+mygfzZG0/N90lbpBKfameHWLi9Oj2zHhaabwogYWVLTqnVtf5taMlssvppgtRmCceoNcIwurWxeHaQJ7LYL+Em/beoDva3sSgF43xJezq07MWGN+feoIV3YP8eCYvkxq04n6S9eEyg6J8hj+nFAZ13vm1ZkOxnuyhNgLeh7a9przaNAieQCfCtOhnvxy6FMteSTL0f6Fal6PvTs7x7bESTassgFBVhyuTB8nk9VK24qbwp0TZFLP4+FCF6NteoG3rvmoYpwPrNiAi0gO+Bvg/zLGzEiTXrQxxogsHS0UkY8AHwHY1L/y94XT5OIs3nKk+VDtrfAje57iA22PA7DZM+QWJa4iydE+N+DG9GHeuV3fM18u7aVmO3XXE5vWw/SKUk/sfOXYlRTCBF02i3rfyB6mPquGd/N4jUoeJkY0XnzTe/fxTEq7pFQqedyK1GmEzR6bhKrhPPukdoL/5Mnb6e7V4orqwTy54UaxjHGoe9f+jFCcTDGUUq6eM+M1aGX2HDUX70ScbNNSZWNqgl5bkZJxTv+gJyTAd6Ps6/LrGWkkkHP5EnuyQ+z2dR/Npe2TQYHDtYXXvtsmBBZ3PXHFqVPXtvpFfqDrCUY26/EeHt1YT6hKCARNmiBFU+dozxRTjAYtjAaaknEpM2Ub/o4EOcaDHHlHx9njztHqVFnn6flvyxWZyOuYqvN6Hc5GYfFcYDqs8J2ixqc/88ytpAaS9aKitSC6LhnHp9OdY2ObKlG+3JonMePW79VK4FI9hzTJGOceK/IjRMRHjfdfGGO+aL8eEZH1dvl64ORSvzXG/LEx5gZjzA3dnfHNECNGjBjnCithoQjwaWCfMebjTYu+DHwI+E37/5fWOpiov+OrtYBXyxqrLs0nyDTHkx0VRgJIZivsSg2x2VMX6XTKdFAF0AMAACAASURBVEnx6XOLdQpVvqVIIbcwthp5Hm6JemOEseNt7Muv4/i0xrnD+zppmW3EkCd3O7z7fd8B4LrcEW5o1Zj7X6ZuYP7RrobwU9NYxCgNTmyxUmImQXGf6nqnYIHCnxM0vGp/DpwBj/JxZdYkaTBbToEDYULPSyJTJeNU8FdBmvCtaxt1x2lGM7slsJS7re0TXJM6SsbOeFxxOFrTsNMjxY18fvhN7P+GMiqcq6f58G7tsvO23Evs8f0lmROtTpp+b4peG/M/kAsJEpHQlOiMqWl4bkkPcG4iwzcn97DFHwVgPkxyr+0J+Y9H9jA3lcbx9WLvWD/KRzZ+mzZXL9QPbX6C/x28CYDibBdu6fSiXecTBQOvlvW+YNbHnwPPygwEvqw5Du/hstmbYWNWZyr7cusJki6uncmcLOQZrmnV73Q48rppiPJ6wkpiGrcBPwo8LyLP2O9+GTXcfy0iHwaOAB9Y62AKVgHvxUoff7LvZh3gYHKBkWrWgxDR6f5K4CDkHZcrLLVtd9dJHu/UqXI44dcTdaChlCjWKoFQrnnctUGpVwfe37PAEP307kd5W+4lAPb4Po+WNVF4X/YypnOdmMmIKL04DtTQ+DDlhTHVxXKr0WevpNWjYRP1uM5LXkzvcxoUP9cNSUl1VZHgolVRlOqpVr+5FVutWy/Oltw43e58Pe79XKXEfbZb0P98/C1s/oJDLzrYucEWPlG4A4C33f7SaceRd6psSquB8TpKVKdslWJN6lolYDVLLC3UmfZ4cWIdD2X1pTsbpHhoRJsvFw+3kJ5y6ufm0PAGfnHw/dy95wUAasalVLmwXQ0iPZ0D1U4GClq96845SK2RwK/lVF8GoD8zTbdTgNMkL5eCKw4drsu72p8HYH9/L0OD6/HmdR8njnTyBSufu7P/H9jjB2tS0oxx7rESFspDnMpxiHDXuR1OjBgxYsRYKS6qSsyqpYjNBmkVFcLqUdQWNgeIvKcwFI5VO5gIVOs4I8t7CK44pPDqmhdAvU9gkAS3KLgRTa7JmzWOYUN2ire3qIf2Q+3z8BON5d1uZUEibp2dim/MTvJydiO1nB3PnCzQuj4bRPom0bmQYOFYF6zbFGpqz5Rsb8SVxVAqxqVqNcclkFOaB0QVnqEHXlIXZpwKAVLXEQf4xNPqZXc+nKBZvGP8tgr/5toHVjSWDgf2ZlSX5pn1G3hxToW13JJvQyaNExDNRrx5YWI6y2yfJkMfH9/M8JAWTSVsRW5UrOiUHYL5NP94/AZ7vI1QjF84dWYTJbr9WWF0Psfhiob6tnmHWW+LsdYqADVsz/cXxm7kkccvAyA7LjjVBjW01GG49QYVB3t/5xOsO0vHuNVJs87VBG5Xeo5jaUNyQo8/OeLxbIcm5R9p304q+8pZCbDFOH+4qAx4hJRTpaNFDeFoKruQsiUNWl1pMsW3RndztRWealtU0bgYDg4ddp7dnijgpvRJCVKmbpSa9wOAH5L1yvR7Goe9IrE4DrjwRu622/n+jqeYvyHJdwuqopgtCkShIEHTx02qd9LEIlkp60GMii3pDxvfgY1P5/T4trROsM6bIrWC6W9UBVu2rBGn2ijTX7K7vX3RjpRbGK7l2ebN1pf9rDXSn87cwvyzrey+S0u0P7ruMW61DQYWs1AWI+8kuCap3PLRnhc4eFKZO0HKw7iNayYGnJrlfZeEYtGnGKixubXrMDMVzXdMHutVQ9907t0K9XyEUwOpNcnZNh+zabBe/DmYeqGTP3JvB+Cy3UN0W+bOWmVYR0O9h4eKLZqfAfwZ7X5USVt5hHzIloyG63b642vqou7bMGTCCRbQGvw5oTig2/0D/3vovnyWjrQyhzOXWPPf1ysuKgMeeYhZp0zZNidGVEPERDFq04h1etMux6dbGVyv3lUpcfpKfl9cel29Q+9uf5Yj6zUZeOjEptP+LjQOwbJRpIWIXiA7/XG2ZMZ5xMYpQ9+lTks26IOyxCZPZ7yDpFIDm5OIft3wLPyhcbEC1tCfmqLNKeOsMAoe4lCu2WRhjVO8/Migu2UhmPHtb4QWp1Snbe5NuLS1KD//jutehusav9/m1VZscDT5rAbmqtQxtnWr0Xp5OENywoUmWl1zPByg01cnoC8xSXdGP4+lDNCUANQ/G4nKJXIPjS8aH715gz8nTM3p9R6odrHXqvit8FZZNYKUUI1OW77KjpRNyC92PlaJvJ269KenMG1VgpP6svMKtrE0MDeX4nClm+vsy/Ri7V5/tDbHaLDw5bLN09nf6Zy7SxVxfVWMGDFiXKK4qDzwCL7U6MkpBW0y20roOw3mSRPbIDEtzJ7M8eImjdPtTAyTsQUay1Geou973FlSro3LmkVCTc3Vh0WXmVqS6vksvTsD6hWVWShsrJHpUW+ysr+lHq9dTCc0AuKpm5nzyuSdYMWVkaXQ1wpYNLSwuIQ+8kydKvWGFcXAx5EQXxoMjig8sumUu2x10+/Iq9/mzfHPepS18kpnL8HQ0tdYQqC2tFcqRkNw0YxltQ2X63D0nM9NaZz9G1OXMx7o8WadMr4EZB3btMKdrhdRdbvessJQgQkZCgocsOJSI3O5OqUPDLUMlLboNj98zSPcmFLKak7WxprpsLf2NdmjPN63mcEhLUzzilLPK4SVi9TlZqG8xX1ze/n0/lsIn1Vhs913HeKH1j0GwK3pY2cM2V1quKgMeMQh3uhNsS2vU+WDLT3UMm49FuiWTb3zujcvpI773ONeD0D1Gpes1Y7Y4et0PpJMjRJLEdd8PGxhumJV7sr6QNfjx67US8SNZ2jzi/VmBCs1PhnR1lbkrW5K3q2X7rvFM1uIOl0so5QxgFJ/ld7+SXqy+nJ7YTBHOLH0g2Uc8BLNLbZkSfnYpeBI2EgiL2XQmmP39kV3Pqv2osR03nHrFZOeX+OU3TXnAQLhSElDZGXjcXJeT6I3L6oyGIVb1jAutwTZQ3rPPji0l2+llTaJYwhTBmnRe6ajfZ4f3vIEAD/Q8tyyuiI1Ao7X0nxq4DYAZl7sJBPVEYSabE/ldJs7kiNs9fQErDWhmLe/35kY5sr2IY5mtVrZOG4jgV1xKATJ0wjVvvaI6JbPVnJ1eYuv/N0t9D5RJbT3/sDUdj7/fl3/1q3Hzts4VlppfK4Rh1BixIgR4xLFReWBR55Wn1vlRzpVBe3kphxPFrfgltVLSE5JoxFDySBGCBPqBd3btpuM9ZQ/0PY4vW6xnhitGsNE6DJQ04TnX568mSMnlNGQKTRU7kDpcZU2/fvOa/fxnrZn6HBW965rdRLcnH6Vn7pGZwSfrN2OV9BjaC4aWg4RXazcadh2k7Jsvm/dM+TdEt+ctP1Bl6iSrMMB1y5PSZWM+Cuit7kibPImaM3qlH8sm6/rTke6INLkgUdt6WqvgWaGf4YkbLPgmdSE0ZJ6P98Z3MLsEZ1SZ8ucsySjVzSNcN5U80xFCH2Hak5DPGPbPfZ1a1hiOLuffjdc8loExjBQ7aonRn3bFCQ6puUoo2s+DnteU1Ih65UxfpMOT9M5vVgxUO3iy1/Vwr+OgzrQcl6Pqfd9R+shlHONxZXGAPu/sX3JSmM4vU772eKiNOBdbhZf1ID81PoHAXiqqNWPbsnFb+pv6VQNqVF9Iiu1Vv6qqOGUucuS9CenSNog3lCljeen+jg4rKXJtdEUyXHb0b2o26oLQSWg1mopeOlxtvuTK2ZNRNO6wUA4VO1ksNxmN7pyBbtaSii328/9ZX5wvSr8vTt7kIKBg2kr/LgMk2Ut8HDZ4BX4ma163j8R3Mn8rL7oTikrN9Qf8FroEFzgCV3NStvWsmByNbpT+oAdcdsXGPdzaQgXvDSaGSw1Y5t6gDfpcXBGz+GBtnX0uUcWcMabJSQOl3so2aYNucUverPAzzhniF4mHU6NPelB3FYdTzXv1o8BuGiFrbb4Y3zv3SpncU/njYDwAzfr33e3PsvVCb0P1kK1bEYUc19caQzQS3VVlcZrxUVlwJsRJRu3+RNszY7zVKtS/WrZho63lpY3ElISCm5Rf/dPh2+g0hli7FPllBxNylgDlC4KnqWguWX1LOsJrSajGK7SQkbdef5y6hb+6qXr4aiOJzPd2N+ZuN5BemGy6ub0q/acJCgETdlKMY0CJ2eh4qFTgUJB3/zHqh0cqZ1gs73ap4uZuuKwwcuxzXZ36c3N8kpey7nNpJwqCWBPj+eEuIRcyKhcELXW21Lmx697lCvTxwHoTszx5UC1UKpT7bhlqfPnz5Wu9ykFP6ahaePNCQMH9aX7F95NXLZliF63oVU+Z4ufHi9t5VPP3EbKNvBeoDx4Hl7Wi9HhJLgxdYSfvFK9x0+W30zixMXL945ogbelqnS4OuYfepf2A/DthVWdnfNDHzwXhWprRRwDjxEjRoxLFBetBx6hw4Hb8gc4sk0ZBY+XtxEmbEeRGcGfZ0FMPIovJ6YhM+RgovLxSARrgS73Ii8s6t3rANZDcs5yvj0XJKnN+6QKUfxYPX1YeaXlUnBoVM7RVFZvZKGD5pbAO6Ge3GdzN7PtqlG63TVW0S3lATaFUKrGI7ANntdaTn42iJhDqVyFHakRbk1p0UnVuDycVTGrY9k2ElOyYLZ1XvS+TeO+TE4JQUbDD6PzOU7U2tjjqwa3Ly6z9j48WOrFzHl1BUuv1Jhh1TJCpS2kr1XDAVv8MdwVSiOsFBknwVavwrakzr5SuQqB7arkzbqcKLVxqLp0H9cLCV9c9iZe+/DO4kpjUNriaiqN14qL3oC3OiluSg7Ttv5bANyXO8k/HNV2Y1NH2kgNuySnI92UBj2sod+xsqcz6nAPmsTEcqgzbvmM05TA6LrjYZGBml7Idq9A38ZxJo6va4xnhYbCLUHqiFqjz7i3sv5Gfdg7strF3q/PzcNlW6q5lUaDg+J8kmPVDgrWoK1oDPZN50m4IEzTHP4RY+ox8WLNZyZMUTY61sxZduNZKRzH1MejY2lILBQKPscrHYxa5cltiZNc1a7HPtDWQ2XWJ2EaYbjzkRwUQ0NvpWLqfP1ixV/QyX4yKHCgqiGqgUIn3ox7qvomUM1A+2UT/ORmTYpv8IqnbZ12rlDXlykI9z+9h+wNVn2y55u0X5wh8fOO6IVxLiqN14o4hBIjRowYlygueg/cF5ceN0NKdOrotDxPeYMO+z6zm2nTCpa94s8Krm1+bFhUXbkIS+lnR01iqy2GXJtmkLYlRs+oNRHanQzWPP5kRMWNHjm4De9EkkQp2sHKp+pe0RBaDYryrF8X9S9kDpIU6lRJLxkQOXMqxrX8DqrGPSX/eDq4dlueEzToiktM2SOWwsR8hiOVbgopbaKQNN45D6NUCeqt9hzHEDSFkDANeqZ/PMmnk7ew+doxAK5LHedDneq59t88xSezt1Eet4nCgoNbVjVKWBjqWqzCuFosCM3Zz4tZJKOh4W8n1H175IndZCcaNFlMQ/mxljP13pfAqvtfni2k1nReCg7DJe1POhGk4MLKpl9wnKtK47XgojfgETKO3i2X+/N022rLO1v2cf/GPdzzhFZhhUM+/nzUiGFh84eoXD5qokBTg16VaZV6xaO7ZY6f3a3xrauSQ/UK0aVQNlUGa2o5nizt4JlhLev3jyVJnZSzMwZN9LzFcKEuiZtIVqnVObtyWnaLLwHuKkKmkaH0nHBBnL0+Pvt/FM0pFJIcr7RTsBaqfeW7OiOiEFXZhEzYi2SMSv02IzrX3rxQKCQ4VtW8yU2pY+yxl7Ct5WluuukQA1Wl9R0pd3FgvodHD20FwH81VaelOqt54y2BBectkgBedA2GgyzH5tvtuB28uYUdmU7J01wgRDLOnmNfoBJyPppFx1gdLgkD7opTl+hMun5dCW2zN8NO/yGuv2MAgN87dBcjJ6zpEIOTDDBT+jZMTLj4cw2PZnE7LpxGh5MNbbN1T2ezl6hrcSyFQljlJWsMPjVwG4UhNTApS1k82wRZFHt0Z516V5YDuU76vGl8sYlCNySSqDgTRbdqXFYjR56wLmPWrRCmQruPhR61mAYvvFZ2OVnOM2i7HOelRIvV/FirJx7NcErGMFLVHEOpkCBVaui0iIHQa+QwxAvryV5XGtTJ7U6CTV6VG5JaHDWROcwzmR5OzOt2j0ytw5uP2ratzQtv9sCjl5/jhITG4XigF/hAeRMjc3rP+LMLdWeCRMOpqOZC1qdnbOcdWG33ndWgMcsJCZtzDCVhqqzXdzTIMxdqvmM5bZcY5x9xDDxGjBgxLlFcEh74csg4CbaLR8ZSdnbu+RyHdmi8+MXiBr59cgcD81pAIYsVB5u9UYFaGoKMeh4dqXl6XBtzX8EpCuxcOQgdnEgdsLSQFbMAxjZKWMIjjqSMo+Kk1ITwyBPa2zF9U5X3djzNiGW6VKsebtlO96u2mMd6jEYW8goLQZJKNE6zdCl3M5J2Q21+Acmoxx8kPdzSws5CddXGWY8Xx9dxX+YKAFpbniJpZwprZaREXX4Ga2kGy3rs4byPU1mmyYQLfrJWDzUtPlIHwbHfZkTY7o/zznVaMffJkx2Ew2tr3ru4ICxICGFSB5pJVMm7RQaqWqH7qYHbmHlRZ1iZwsJQSehDqVN/d+uNL/ODXY+fdeed1SA6b0m/xnyiIdHplYTDIzrbfLh9FzcmtXgmvYL7Kcb5wYoNuIi4wBPACWPMe0RkK/B5oBN4EvhRY8xyPdLPG6LKQYC8U6RktKnDk+FWjgx14k/qHe/NqQh/8wMfKf4FaSh3GDZu0wTc+3ufos+KwDtnyNRkHJ8rbLjlhzY/yScmtDrLjKZPnX5HtLIa+EVD9oTNcIaG4jqdhoa+EPhgKpFhNlRa9OF4dqwPgH2T+lLK/UMOvxjWjyX0GqEUMaaegDIll/FqlolQ91E0FXJy+mlvylqSnsQsaVvTHSZSSNjg2hsHfHtC/RmHyekshzr1BTqYzbPO1e48a+3eUjJ6Igeq63nwqEoq+BOWbhcZvKZKReMYXLcphLLENpNiawnExaXMhsQEoHkF4zUM+IIKV9PYRzUr1NLUG0xLUxLVLSqNMzLkQRKqXWoUd7SN4RMwaDV5puYy9bxNXfck2p0HQU53viUzvubOOyuFaw846deYjSyEAacMtSb93Xkb7umMbfcFw2pO/b8D9jX9/d+B3zXG7AAmgQ+fy4HFiBEjRozTY0UeuIhsAO4GfgP4BRER4K3AB+0qfwp8DPiD8zDGM6IuBlR1+OKkMlL+bt/VuINJElOWBlVcmJAyTsN7qrQa/F0z3LVOm8RekzxOp9ViOXOowWe9pdltSIyTyuhYgkR6YdFLDTzLkkhOB2RfOgkVm6l0HLwJ9YhLG1vxZyoYT/c7vyFN7aR+Hhvo4IHnu+j/lv4uPzINNfWWqt0ZZjYlG8lM0zhep+Dy5NhGrstpA4Dd/qucyY/L2mRvq1sg4emGSgJeCXIn9BgT40VmtyutLPRd5pJpnspo0+G3tLVzuT99hr2sDFGl4rFqByWr6JiabzB8ItRZHz6kElVSTpWl0HxNXXQWtd3XWVRrtshYRo8pSAteYel91DJgrpvhzRtVp8aRkHv3qUok077OfiJxtHRA3wb18HdmT1LFrVNDS3OJU0SrIlGuao66nrxqui97is4popCgMbIg1OgE2kgcoBAmmLWSmSEm5qNcIKw0hPJ7wC8Ceft3JzBljIkIT8eB/rUOpmqnytNhqT4zjh7e6OYtGep0OKf+O/3/3vmr+daJnQC4x1NK47NRiogSFs0AjajqIEA1b1iXn2d3SkvNN3hnz5xwXRvSWOKOTsza6s5XpyEMMdaASz5L0KovjNTzGs+XtBr0fLlGJdei47rXkBwvLSATz+/QZcV2V18WzVKvNqDlzQszxRSDFZ22T6cPn7GnYcZy69f503RklfkwVO2g9VCJxGHb+9FxaKlGQfA2qnmX2Rk9jpFaK6VzXOFYDn1MwTZbXqJX54J+lqug/0QKjAA/s/VBPhHcCcD8TJcqVTbvIgqh5Aw7Oif4vk5VirzMH+OHO1VIabjWykyYrlezZp0yfX6jX+sThW3876dV/jQ1kFwoWgUENrpV2lzhw9eorPLN6VdpfY06wYf2yZorJfFKDVquBMCo8uef7e1ntj1h1w+IKYUXBmc04CLyHuCkMeZJEblztTsQkY8AHwHY1L/87gphhRGrtHeg2s635y4DoNUrUA59jhQ10bM5Pc6hgnovk+UMKa/KxrQ+HM9MbmDypL5jslOCVzQLOLVGGlSzIKWGG8C0V3nbupe5PKkGPCWry+1GmhSbvAk6Mvo0Hsu3Us07+HO6jzAF01t0u9NbukhNhAQJ/V1iLiR/UJOmkkhAGBK2qY88uy1XN0yzGzxmNzTGNncrBBO6jfQJITm18Hijz25RmJ9OM17NAtSTmadD2iYet3jj9GeVLjY53o9brDbIzJ5LYYu+QGopfVmaeR3fWDXPhH1D9pnqaamYp8N0WORITfdxrNSOU9Cxu1ECs+lQ6q3eKjBXSFEKV7bP5jzKtsRJum3Ho+lc50IFxiZL7s0LI3M5hm0y8urEGLenopM/TtnUqKIvt7IJ68M8WE1xuNgNczYGP9tUm2ALfqJEdjJXZkdSX5ZbPXfNnXfWAjEGpyo4Nmk+X0kwFSqVMTAz510pMcbSWImlug34XhF5N5ACWoD/AbSJiGe98A3AiaV+bIz5Y+CPAW64OnU+ZINixIgR4w2JMxpwY8wvAb8EYD3w/2CM+RER+QLwAygT5UPAl1a786jCbigocLDawhcntRfgfQO7cWyVXT5doha4TE6r9xiOJ5CousCAUxWeimK9VchEJdHzC5sPRN53RVlolDsM9Ku3fMOmY1yZPs46V3+QPANDYzGijiYbvAI/vVkbIfxBeAejhV6M2+htWd6q8Zy37H6FrZkxXprVLi2PvrKNju+ol9n9lEOlNUGpS73HSl4odtu446Ya23YM833rnwVgOkjztUEV9hoKu/GKbj28ov0qreZ0SWDWY7SiXmbBnPm9HYWQMk6N9akZAB7ebUiPZ0nn1BMsdvvUUpaamFQP2J/Sc/HNE7u0JyjQ5+5nvXd2HvhEEPBPM6rlfe/Le0hO22bLtYXrmaZydQmEIHCYtaybpSPhS8MlrFcbLuVVRowUf157Vn7K03v2st2DdNv7pxBWORY4DFiRquFaG4dKPQDM15JMVZtoik37MAK1vNRnhl35Aht97Q17rpUHTwfHnshMssJUMgo9ijY9sZGShBvgExVKxe73hcJaeOD/Efi8iPw68DTw6dVuIKqwGwkS/OaRd7F/v4bRE2MuWBrdTK0VJ4CMNcZuuTHFNI4mVqJYrzarXTrpFPqiXb17LC3rqkFu6dIEVN4tsc0fq8d9V4vI2G3wcuxMDAOwrXWME11dhL5tPLt7ih/Z9jQA78k/yw7fcKRF//64+3bun7QNB3J5EtOm/qCUOqQ+5s3bTvLB/sd4h1UlPF5L81xOz9lgpoNqrtHsQsKGUqBqfTjMVNRwlFZgwCO0ObA3o3H5v994JaPX5UkPq2FMzC5sKOFUVOIXYOx4G0/3aBOOWzOHWL/iPTZQCCscqbVwcF5DZkz5dalVp2YWJImN08hphJ4hDKWu53I+WG5OTe+9clXP5VSY4UhNje3hagefG72JB/ft0nWnPMJWfePkOgr4boA32/QiapIxrmah7Urdzs9sfZDNNjb/WigPngnGAaxj5btBnW4Y48JhVQbcGHM/cL/9fBh407kfUowYMWLEWAkuaCVmaKdqs2GKQjWBaxNUqQnBn9U3fWLOkBqrkhjX8MP8lhyVnJ22p6zuRTTjbXIIIq1oY5OW1aw2CO7YqXSuu3r284788wDs8KPQydqTRJHX50sIyQBbNEl/6zS76iyXGr4k6LbCz/+i67twq673wENXgnHqnmWl3XDTjUpv/EDP41yXHKbN0ctWdYvsbdHUw77OXirDfj1kQ23heZFw9e3hAHyEdZ4mMfeuH+S749vxZzQc4lYaiVJtIWbw53QfyZMuL06o3/18az8dzsI+kKdDFFobCSo8Mn8V+0a1cCkx5dSLZWRRD0po0EKDjKG1pUC3p6GfxBmm+IEJGbK9TA9VtjBa0HCdW5IFTTOaC3mCJFTWVXlzj57/vFNisKYJ9N88/C6OvbCO/AmbcC1DkLCMDT9BBUhFDNKm+I4Kqpl6EnVb4uRrpjrYjCjlWqz4jZ6Yos9QkNNnpTs9R7er06EzFbvFOH+4oAY8Kmfu82a5qXuAv11vy6RHMviWf9uyfxb35GSd+ZAvVZjfrnS4Qo8LboPaVctKYxrt6pQ6tKXA1ZzB7S6xt0uF/W/KHmS3r4Yi56ytdLoZHTaesys7wnPdffXvb+08zBU2vNLqpHAQelw1FNv9SbZnNF58f2+ZYpiqS7j6m+a5u0uF429NjdDTVInnUOGW7AEAnujYzAuplkYHokWQKsyUNfQxXGtjLBiifQVc94zj02crKrsS87i5GkFabxszK5gmtbyoyTRAYloYeUYN7+9U3kZ2z1e5LqnH3+sml2WlNKs7fre0kb89ejXFg3pfZKalkddYIh1eN+AdVW5dP8Bl9nznVsGA+cfxqxi2gmhJxxC6Uo+3N/dNDRLgpgLyvjoWLiEvlTWcNTTZQnLCqTshzVWalwIi9k4QOAsEuWoZQ99mDe+8r/sZ+mzM33kN5VNjLMQFNeBRF/o+1+UH2x9jfqdyTL82vZf8UTUqtbYkeJ315NzknnxdtztICEGykVipZQ3lDerS7Nk6yPBsvl6Esqv9JO/ueL7+UPd5tVUnK1eCDlcH8/0tz3DH5S83fV+i16r5+Yti7XlH2JHScXV3zuJ2T3N5u9LH3tn+PFdZeuNiQ5RxEnRadbr2ZIEwaRovsKrg1BOaBq8oHDmiseS/yV7P5f1fpdXGM08X+fdwWefqOf1A53c5uqGd5wsbdaHxSY/YpHHFaD4ikk01pq78one4pQAAD6tJREFUOHOgnf/K3fyXPV8F4M2pEZJu41gKYYUJW4x1uJbj0XlNzH7mpVsIjmZJjVt9mXKDz78UzTs69kxLiVtaDtblEJKy8hf03V3PcXSDGvDRoV57MNT/j/brViAou0zbhORwrY0DRV0/CBwct+G5y2JJhcWoc/eX0c55jRCYkIKp1jsG1WoO9vEh9KDaFrK1RWew2/2TK3IAYpxfxGc+RowYMS5RXBRqhDknxW6/yEe77wfgtrce4Kt79wLwnSd241QT5HdqHHZqpNZoOJwIQCC0MVlSIds2aUn0h/sfYrs/qrFoICMB3a7XpF2cPC/H0mq9klaHRR1Lssv+Jic+19v+jb+5+4scrXaQtaGYG1OD9J8mdhwdX1diDmmvEIxYr6gEREWSoRaMOLPqaw8VWjhWa2WbH2lLL++Du+LUdb0v9+f5uf5v8BlPuw49VthF9oQtuR8o41RCCuv0vFYDp4kR5DDttnNPl8oclDqfrc+EMk6NiSDFN+euB+DekcsYsDMF/6RPelKwRAzLMlo4vjoLxhMCW2aQT1XodmfqlYtn8hCbC3kuSwyxKa+FYYMtPSRmBSc4VUPeqYA/7PNsl4bJ3tH+Am9tVUXDfevXsW+2H+yMyZujfgzuaUTZxWiB0NFJnQF8u/MysqLhs163QvdpQk/nAiGG6dCwr6jHVC36eHa4oQfGM6TtbCzjVHHPwww2xupwURhwUMO3106B9/gnuSr5NQCqfV9nNkwxHljFwSuKVK3BmQ+TDFbb+eaYVm1e23aMt+RUb2uHP8N6N3NJTO8yToIttvqz1Zkl75Qay86QgOuwgiebkhO05IvMdNhWYWWnniBzAptgtNS1sbkss2GaQqhaJZ7jnvY8Rcu63CwtTokwaghcFLqe0YSbNzIFIuQKmsibvixPGDQ4+ZlBh8e/qVohj27eysZeNZLrsjMcm21jaEQrGmU8QcKq8yVmRDVsast3pYkMeKUFqr360ruic5h13uxZJdd8CelJacxfOsrUJtL18yhBYwxOTatea6EOwJGQnZaz/Wtb/o4n123hyyNXA7D/O1vIDNkw0GlI6doxCiq2w/mnT34PD+7cAcDPb76Xm5KTC0JP5xoFU+FYrYX9M1aCec6rH28tA25rhQ0pvW5tTm25zcR4DXHxW7cYMWLEiLEkLhoPvBm+uOxNNE/rQwIzdcp6ZTPDnBnkgy0v1r+LinFyzmtPv1oL6pWP4tf7Nyr803rHUZjg9swrTG9N8+kJrQwMJpN4hYgyoUU9kbDX/GyKF4ob6snRjBPUW9adCXmnwknrZbcchDBpGSmpJGE+pdotaII5Ymy4VYMzBZ6lGAYTaUaTGuo56fUhNchGTI+mphtuRZtiLNcfNPRUlxug3Bly/Q5VW/xQ98Ns9swpyeKVoNupcVXmOABP9Wzk2JyPsbTNxIzUi8ZUSyekK6X0kpRU2WC944yfYKN3gISlr/zXvvVUiil7fIK3WOVrgcaKqRdjGddloF37eh5a38PlifFz2mt0Mcom5EStnUMTWkEaFRsBVFtD7tqxn7fkNUzU4VyUpuMNh0vmKixlxDKSWHOzgIsNqxUsitbvdefYmhylo0un/zOjiTonW2xvTi9SZhxNcO/QZVybGQCgwxld8dQ87wR8dLM2fP7zf3kzx/9GmwFLmCb0G42FI1phBDGN+K9bBWYXbtcsEylaTlQwdKGWFkrdusLWa07wvd3PALDbn6H1LF/grU6Ct9tK18t2DvLUhq189vBNAEycaEMsL9qkQtIdRfZYttAWfwK/KT6dE5+bUgMAfOxNX+b32u4ClJGTmHTwrAKhW1wiLl5npQi1wCoDBqlV9TRdDSI55mM1nxcL/fyf9s41Nq7iCsDfubt317vx5mE7fmA7DxInIe+YiIRCEx5pS/iTVvyBtiqVkFAlqFqJ/qBFlfqrv0orUQFVUSmUVqVC0JIKRIGoApUQIAkhIUEhTuISbOfhvMzaXnt9d/pjZtcbxy5xZO96veeTrnZ37t17Z8/OPXfmzJlzkqes7GK9kjNRBYmA+RXnaHaZqkqtgzRdKRkFrvx/El6YZv8sjTPt4pXztQlSA1a5x08JMmjwnC9ytNuj47/VfNiwALDKJ+EMvV82STYvXEkibidcm+e9wpEH6gH43bFNnD5aTbTb9nojF/OU+RUkd77S6K8Z16kOKoRUraFiibXj39Wwl5ti7QDUhK7erz/uRZjnHoqNoQzzwwfZsNIq9M5lc7gQ2AllXwLmhnpodIuF6kKZS/yh416EJb69vRJeO8tW/RmAQ0saeevCUt7abV0lK7pCRFz4gXDKZozKukMGFYZo1GXyiZ4i4U1OzJGsC+ebyXW8dGRNLp6NF0B6hvPrnzXArHAfvoY9mVKoDVxRFKVE0R74NCEqPi3hJD9pfg2AX3EHB1I2mJTfGyZycdibI5wSwhfCvNZpvUKaIueoDx0FoPYKTClznFvjRi/DUt8GBGtZdpIX6m7glcM2KNfQidhwNqQ+2xu/LEcojJrgOWtOyfiA5CXhCEEQtR/6aw2Z5hTr6q29ujXWTl1oOM/lRJB1L8yFYI8mgeSIo8bu7ecHOcueY6X/ORsq2rl1i/WW+uX+rQwcsnMK5oIQShmGnHde0JTiB0tt4uBV0a5xrSgdD92BPW9bXy0D52JUuhjzgQ+D1fZPW1xznmXRThKeJm6YSqgCnyb4EqIhXEncs8bVBxt38AS3AfBxciGhAY+QmzzzBiF6XjjZZafEPprbzMaYVcS147g/Q+JR48IBxGWQquq3aW1tB+C5uhtp+9TGQol9HsYLhCBvlWYu2cQg+L0Z4l3DrpO9jVaDDcU8hiog4xJfBBFI26i7pOvTXD/vBN+daxXcgvDgpKysnWiyppWE1w5AetUbPBHbBED/ezVIRkjPtP9T1exeFrvwA00hf9ISOgy6gfjFdAVev0fglkgMzs5Qt8CuvHxo/uss9y8SH8eqVmXyUROKoihKiaI98GlGdiXo8sh5bqmxUQyPLa4ilZpNxZlsTBFDuE+IdNqh88F5DRydZV3HmsJnc+cYD3EvwrW+EJF2AE7VzaI/bc/fOWM2/b1hQr0uFkyPR0W3+97pIRIHhhM8m1iUGW7x0unWOKm5hnTCurNk4gFe3HbdF9SfY8Oc4yyL2IUl1V5pLNqCS1d+fiV2jAPX2ETQ21sqSfX4yGw7qbhoTjfzwvb3+eNM8zceZjvfyIUzzrKvvolUtR2GbV5yhHtqdgG2PdWWyMK4ckKMmSTfpFFYv6bCvP+v5oJd78s470KIHhu69OaYGxpkXhHCeE4kAyZNW9oqu5d71vLUrk3Ej1uFGrlog0INzrSKMrk4zYaV1gb+s8ZXuc73r8qOHJgMSWNdXbqDgDOB8/UOElwI4nSlrcnmb8db6d9jHxi1e4aIfJHOZdNJNkU5u9plMWoYZP3idjZXfQpAdShJ3LnSVIeS1If6cnbv4RAJpcX5oI8TzrbUOTSL00MJqsPWzt4YupgLyFXtxSZNeWbvg5MBnMsMy7HKS1HvmkF23kMpDqGGtj3GmPUjy8u2B/7ZUJKd/fZh8vzJGzi8YxHeGuuSdt/Sd9lSaRcsXK0yKzZR8alz8caXxzpYuriTI73294b7PUK95BIux0747J1p9+2cs4jEjMNcE47mznOlhMRjlgzHglmU+2ofgUnSFbQDcOPKI7y7sAWAf964io536hm8ziqR2xd/xLrKz2y9KzpoDCXHUNIeUNoPWbCKcY5rXqsjA0B+3NkokxWzZ2Qd7OvIPaq0pzo6HlIURSlRyqoHnjYBn6StrfXN5Gp++8GtAMx/waOONMlO6+LweN9mtnz1UNHqOVHEXe+5xT/DilldtNXaxLoDPTEkkNxCGz8JwTHb23rUfI3UGp9b4tZ+vtDvvyqb+Ejy7b5NYVjl28TMWyoPwtLh4y43X5V+L1tRJouyUuD5PP7hZqrfybplWTvj2ZusyeGH694qUq0mlqzppy6U5ttVu8AGbWS7rCadiuei7IX7TW6Cc2AozmPp2+hYZe3V36/aiR+2cplIN7bssP36y4bt0ys0gqJMJmpCURRFKVHKqgeeH+Xw6O1/ZEX8OwDU/jzNff94jbsqe/KOLk2vhnyyPfCa0AxqQnB9w14AeoZi7N65Jrf6UTLg97pAUwOARPlP7bUAbKw8yopIz2XnVhSl+BTUjVBEzgC9QHfBLloa1KAyGQ2Vy+ioXEZnOstlvjFm7sjCgipwABHZPZo/YzmjMhkdlcvoqFxGpxzlojZwRVGUEkUVuKIoSolSDAX++yJcc6qjMhkdlcvoqFxGp+zkUnAbuKIoijIxqAlFURSlRCmYAheRO0TksIi0icjDhbruVERE2kXkgIjsE5HdrqxKRN4QkSPudTITkE8JRORpETktIh/nlY0qB7E85trPfhFpLV7NJ48xZPILEelw7WWfiNyZt++nTiaHReQbxan15CMizSLybxE5JCIHReRHrrys20tBFLiIhIDHga3AcuAeEVleiGtPYW41xqzNc3t6GNhhjGkBdrjP051ngDtGlI0lh61Ai9vuB54sUB0LzTNcLhOA37j2stYY8yqAu4fuBla47zzh7rXpyBDwkDFmObAReMD9/rJuL4Xqgd8AtBljjhljBoHngW0FunapsA141r1/FvhmEetSEIwxbwPnRhSPJYdtwJ+MZRcwW0QaClPTwjGGTMZiG/C8MWbAGHMcaMPea9MOY0yXMWave/8F8AnQSJm3l0Ip8EbgRN7nz11ZuWKA10Vkj4jc78rqjDFd7v1JoK44VSs6Y8mh3NvQg84U8HSeea0sZSIiC4B1wHuUeXvRSczicLMxphU7zHtARDbl7zTWNajs3YNUDjmeBBYBa4Eu4NHiVqd4iEgl8CLwY2PMJUF6yrG9FEqBdwD5udSaXFlZYozpcK+ngb9jh72nskM893q6eDUsKmPJoWzbkDHmlDEmMMZkgKcYNpOUlUxExMcq778YY15yxWXdXgqlwD8AWkRkoYhEsBMv2wt07SmFiMwQkUT2PfB14GOsPO51h90LvFycGhadseSwHfie8y7YCFzMGzpPa0bYbr+FbS9gZXK3iERFZCF2wu79QtevEIiIAH8APjHG/DpvV3m3F2NMQTbgTuBT4CjwSKGuO9U24FrgI7cdzMoCqMbOoh8B3gSqil3XAsjir1iTQBpro7xvLDkAgvVkOgocANYXu/4FlMlz7jfvxyqmhrzjH3EyOQxsLXb9J1EuN2PNI/uBfW67s9zbi67EVBRFKVF0ElNRFKVEUQWuKIpSoqgCVxRFKVFUgSuKopQoqsAVRVFKFFXgiqIoJYoqcEVRlBJFFbiiKEqJ8j+A8VZ6MGEB0wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(batch_x[0][:,:,0])\n",
    "decoded = ''.join([decode_maps[i] for i in decoded[:,0] if i not in [0,1,2]])\n",
    "actual = ''.join([decode_maps[i] for i in batch_y[0] if i not in [0,1,2]])\n",
    "plt.title('predict: %s, actual: %s'%(decoded, actual))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}