Setting up.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/shivanikohli/Documents/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:34: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import keras as K\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from numpy import array\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.utils import to_categorical\n",
    "from keras.utils.vis_utils import plot_model\n",
    "from keras.models import Sequential\n",
    "from keras.layers import LSTM\n",
    "from keras.layers import Dense\n",
    "from keras.layers import Embedding\n",
    "from keras.layers import RepeatVector\n",
    "from keras.layers import TimeDistributed\n",
    "from keras.callbacks import ModelCheckpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"Data/Gita-data.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>SA</th>\n",
       "      <th>EN</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>c:1v1</td>\n",
       "      <td>धृतराष्ट्र उवाच |धर्मक्षेत्रे कुरुक्षेत्रे समव...</td>\n",
       "      <td>Dhritarashtra said: O Sanjay, after gathering ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>c:1v2</td>\n",
       "      <td>सञ्जय उवाच ।दृष्ट्वा तु पाण्डवानीकं व्यूढं दुर...</td>\n",
       "      <td>Sanjay said: On observing the Pandava army sta...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>c:1v3</td>\n",
       "      <td>पश्यैतां पाण्डुपुत्राणामाचार्य महतीं चमूम् ।व्...</td>\n",
       "      <td>Duryodhan said: Respected teacher! Behold the ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>c:1v4</td>\n",
       "      <td>अत्र शूरा महेष्वासा भीमार्जुनसमा युधि</td>\n",
       "      <td>Behold in their ranks are many powerful warrio...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>c:1v5</td>\n",
       "      <td>अत्र शूरा महेष्वासा भीमार्जुनसमा युधि</td>\n",
       "      <td>Behold in their ranks are many powerful warrio...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      id                                                 SA  \\\n",
       "0  c:1v1  धृतराष्ट्र उवाच |धर्मक्षेत्रे कुरुक्षेत्रे समव...   \n",
       "1  c:1v2  सञ्जय उवाच ।दृष्ट्वा तु पाण्डवानीकं व्यूढं दुर...   \n",
       "2  c:1v3  पश्यैतां पाण्डुपुत्राणामाचार्य महतीं चमूम् ।व्...   \n",
       "3  c:1v4             अत्र शूरा महेष्वासा भीमार्जुनसमा युधि    \n",
       "4  c:1v5             अत्र शूरा महेष्वासा भीमार्जुनसमा युधि    \n",
       "\n",
       "                                                  EN  \n",
       "0  Dhritarashtra said: O Sanjay, after gathering ...  \n",
       "1  Sanjay said: On observing the Pandava army sta...  \n",
       "2  Duryodhan said: Respected teacher! Behold the ...  \n",
       "3  Behold in their ranks are many powerful warrio...  \n",
       "4  Behold in their ranks are many powerful warrio...  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"Data/bible.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>data_id</th>\n",
       "      <th>book_id</th>\n",
       "      <th>chapter</th>\n",
       "      <th>verse</th>\n",
       "      <th>sanskrit</th>\n",
       "      <th>english</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>40001001</td>\n",
       "      <td>40</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>\\u0907\\u092c\\u094d\\u0930\\u093e\\u0939\\u0940\\u09...</td>\n",
       "      <td>The book of the generation of Jesus Christ, th...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>40001002</td>\n",
       "      <td>40</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>\\u0907\\u092c\\u094d\\u0930\\u093e\\u0939\\u0940\\u09...</td>\n",
       "      <td>Abraham begat Isaac; and Isaac begat Jacob; an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>40001003</td>\n",
       "      <td>40</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>\\u0924\\u0938\\u094d\\u092e\\u093e\\u0926\\u094d \\u0...</td>\n",
       "      <td>And Judas begat Phares and Zara of Thamar; and...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>40001004</td>\n",
       "      <td>40</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>\\u0924\\u0938\\u094d\\u092f \\u092a\\u0941\\u0924\\u0...</td>\n",
       "      <td>And Aram begat Aminadab; and Aminadab begat Na...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>40001005</td>\n",
       "      <td>40</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>\\u0924\\u0938\\u094d\\u092e\\u093e\\u0926\\u094d \\u0...</td>\n",
       "      <td>And Salmon begat Booz of Rachab; and Booz bega...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    data_id  book_id  chapter  verse  \\\n",
       "0  40001001       40        1      1   \n",
       "1  40001002       40        1      2   \n",
       "2  40001003       40        1      3   \n",
       "3  40001004       40        1      4   \n",
       "4  40001005       40        1      5   \n",
       "\n",
       "                                            sanskrit  \\\n",
       "0  \\u0907\\u092c\\u094d\\u0930\\u093e\\u0939\\u0940\\u09...   \n",
       "1  \\u0907\\u092c\\u094d\\u0930\\u093e\\u0939\\u0940\\u09...   \n",
       "2  \\u0924\\u0938\\u094d\\u092e\\u093e\\u0926\\u094d \\u0...   \n",
       "3  \\u0924\\u0938\\u094d\\u092f \\u092a\\u0941\\u0924\\u0...   \n",
       "4  \\u0924\\u0938\\u094d\\u092e\\u093e\\u0926\\u094d \\u0...   \n",
       "\n",
       "                                             english  \n",
       "0  The book of the generation of Jesus Christ, th...  \n",
       "1  Abraham begat Isaac; and Isaac begat Jacob; an...  \n",
       "2  And Judas begat Phares and Zara of Thamar; and...  \n",
       "3  And Aram begat Aminadab; and Aminadab begat Na...  \n",
       "4  And Salmon begat Booz of Rachab; and Booz bega...  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = df[:int(0.7*len(df))]\n",
    "test = df[int(0.7*len(df)):]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainArray = np.array(train)\n",
    "testArray = np.array(test)\n",
    "x_train = trainArray[:,2-1]\n",
    "y_train = trainArray[:,-1]\n",
    "x_test = testArray[:,2-1]\n",
    "y_test = testArray[:,-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_tokenizer(lines):\n",
    "    tokenizer = Tokenizer()\n",
    "    tokenizer.fit_on_texts(lines)\n",
    "    return tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def max_length(lines):\n",
    "    return max(len(line.split()) for line in lines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "eng_tokenizer = create_tokenizer(df.iloc[:,5])\n",
    "eng_vocab_size = len(eng_tokenizer.word_index)+1\n",
    "eng_length = max_length(df.iloc[:,5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "san_tokenizer = create_tokenizer(df.iloc[:,4])\n",
    "san_vocab_size = len(san_tokenizer.word_index)+1\n",
    "san_length = max_length(df.iloc[:,4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def encode_sequences(tokenizer, length, lines):\n",
    "    X = K.preprocessing.text.Tokenizer(lines)\n",
    "    X = pad_sequences(X, maxlen = length, padding = 'post')\n",
    "    return X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def encode_output(sequences, vocab_size):\n",
    "    ylist = list()\n",
    "    for sequence in sequences:\n",
    "        encoded = to_categorical(sequence, num_classes=vocab_size)\n",
    "        ylist.append(encoded)\n",
    "    y = array(ylist)\n",
    "    y = y.reshape(sequences.shape[0], sequences.shape[1], vocab_size)\n",
    "    return y\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "`sequences` must be iterable.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-15-467a1e34c49b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtrainX\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mencode_sequences\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msan_tokenizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msan_length\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrainArray\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mtrainY\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mencode_sequences\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0meng_tokenizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0meng_length\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrainArray\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mtrainY\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mencode_output\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrainY\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0meng_vocab_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;31m# prepare validation data\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mtestX\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mencode_sequences\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msan_tokenizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msan_length\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtestArray\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-13-4a7915eb2059>\u001b[0m in \u001b[0;36mencode_sequences\u001b[0;34m(tokenizer, length, lines)\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mencode_sequences\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtokenizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlength\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlines\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m     \u001b[0mX\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mK\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpreprocessing\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTokenizer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlines\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m     \u001b[0mX\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpad_sequences\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmaxlen\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlength\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpadding\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'post'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Documents/anaconda3/lib/python3.6/site-packages/keras/preprocessing/sequence.py\u001b[0m in \u001b[0;36mpad_sequences\u001b[0;34m(sequences, maxlen, dtype, padding, truncating, value)\u001b[0m\n\u001b[1;32m     39\u001b[0m     \"\"\"\n\u001b[1;32m     40\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msequences\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'__len__'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 41\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'`sequences` must be iterable.'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     42\u001b[0m     \u001b[0mlengths\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     43\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0msequences\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: `sequences` must be iterable."
     ]
    }
   ],
   "source": [
    "trainX = encode_sequences(san_tokenizer, san_length, trainArray[:,2-1])\n",
    "trainY = encode_sequences(eng_tokenizer, eng_length, trainArray[:,-1])\n",
    "trainY = encode_output(trainY, eng_vocab_size)\n",
    "# prepare validation data\n",
    "testX = encode_sequences(san_tokenizer, san_length, testArray[:,2-1])\n",
    "testY = encode_sequences(eng_tokenizer, eng_length,  testArray[:,-1])\n",
    "testY = encode_output(testY, eng_vocab_size)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}