Chapter 28 Project - text generation with alice in wonderland.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Small LSTM RNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.layers import Dropout\n",
    "from keras.layers import LSTM\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from keras.utils import np_utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = \"wonderland.txt\"\n",
    "raw_text = open(filename).read()\n",
    "raw_text = raw_text.lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "chars = sorted(list(set(raw_text)))\n",
    "char_to_int = dict((c,i) for i,c in enumerate(chars))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['\\n',\n",
       " ' ',\n",
       " '!',\n",
       " '\"',\n",
       " \"'\",\n",
       " '(',\n",
       " ')',\n",
       " '*',\n",
       " ',',\n",
       " '-',\n",
       " '.',\n",
       " '0',\n",
       " '3',\n",
       " ':',\n",
       " ';',\n",
       " '?',\n",
       " '[',\n",
       " ']',\n",
       " '_',\n",
       " 'a',\n",
       " 'b',\n",
       " 'c',\n",
       " 'd',\n",
       " 'e',\n",
       " 'f',\n",
       " 'g',\n",
       " 'h',\n",
       " 'i',\n",
       " 'j',\n",
       " 'k',\n",
       " 'l',\n",
       " 'm',\n",
       " 'n',\n",
       " 'o',\n",
       " 'p',\n",
       " 'q',\n",
       " 'r',\n",
       " 's',\n",
       " 't',\n",
       " 'u',\n",
       " 'v',\n",
       " 'w',\n",
       " 'x',\n",
       " 'y',\n",
       " 'z']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chars"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total Characters:  144409\n",
      "Total Vocab:  45\n"
     ]
    }
   ],
   "source": [
    "n_chars = len(raw_text)\n",
    "n_vocab = len(chars)\n",
    "\n",
    "print(\"Total Characters: \", n_chars)\n",
    "print(\"Total Vocab: \",n_vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "seq_length = 100\n",
    "dataX = []\n",
    "dataY = []\n",
    "for i in range(0,n_chars-seq_length,1):\n",
    "    seq_in = raw_text[i:i+seq_length]\n",
    "    seq_out = raw_text[i+seq_length]\n",
    "    dataX.append([char_to_int[char] for char in seq_in])\n",
    "    dataY.append(char_to_int[seq_out])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_patterns = len(dataX)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total Patterns:  144309\n"
     ]
    }
   ],
   "source": [
    "print(\"Total Patterns: \", n_patterns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = numpy.reshape(dataX, (n_patterns,seq_length,1))\n",
    "X = X/float(n_vocab)\n",
    "y = np_utils.to_categorical(dataY)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(LSTM(256,input_shape = (X.shape[1],X.shape[2])))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(y.shape[1],activation=\"softmax\"))\n",
    "model.compile(loss=\"categorical_crossentropy\",optimizer=\"adam\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "filepath = \"weights-improvement-{epoch:02d}-{loss:.4f}.hdf5\"\n",
    "checkpoint = ModelCheckpoint(filepath,monitor = \"loss\",verbose=1,save_best_only = True,mode = \"min\")\n",
    "callbacks_list = [checkpoint]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "144309/144309 [==============================] - 513s 4ms/step - loss: 2.5561\n",
      "\n",
      "Epoch 00001: loss improved from 2.66031 to 2.55607, saving model to weights-improvement-01-2.5561.hdf5\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.callbacks.History at 0x7f56c1e576d8>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X,y,epochs=1,batch_size=128,callbacks=callbacks_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = \"weights-improvement-03-2.6603.hdf5\"\n",
    "model.load_weights(filename)\n",
    "model.compile(loss = \"categorical_crossentropy\",optimizer = \"adam\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "int_to_char = dict((i,c) for i,c in enumerate(chars))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Seed:\n",
      "\" been to the seaside once in\n",
      "her life, and had come to the general conclusion, that wherever you go\n",
      "t \"\n"
     ]
    }
   ],
   "source": [
    "start = numpy.random.randint(0,len(dataX)-1)\n",
    "pattern = dataX[start]\n",
    "print(\"Seed:\")\n",
    "print(\"\\\"\",\"\".join([int_to_char[value] for value in pattern]),\"\\\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "he toet to the toet to the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the toet the t\n",
      "Done\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "for i in range(1000):\n",
    "    x = numpy.reshape(pattern,(1,len(pattern),1))\n",
    "    x = x/float(n_vocab)\n",
    "    prediction = model.predict(x,verbose=0)\n",
    "    index = numpy.argmax(prediction)\n",
    "    result = int_to_char[index]\n",
    "    seq_in = [int_to_char[value] for value in pattern]\n",
    "    sys.stdout.write(result)\n",
    "    pattern.append(index)\n",
    "    pattern = pattern[1:len(pattern)]\n",
    "print(\"\\nDone\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Larger LSTM NN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(LSTM(256,input_shape = (X.shape[1],X.shape[2]),return_sequences=True))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(LSTM(256))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(y.shape[1],activation=\"softmax\"))\n",
    "model.compile(loss=\"categorical_crossentropy\",optimizer=\"adam\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "filepath = \"weights-improvement-{epoch:02d}-{loss:.4f}-bigger.hdf5\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoint = ModelCheckpoint(filepath,monitor = \"loss\",verbose=1,save_best_only=True,mode = \"min\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "callbacks_list =[checkpoint]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "144309/144309 [==============================] - 1768s 12ms/step - loss: 2.7873\n",
      "\n",
      "Epoch 00001: loss improved from inf to 2.78733, saving model to weights-improvement-01-2.7873-bigger.hdf5\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.callbacks.History at 0x7f56c87ad278>"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X,y,epochs=1,batch_size=64,callbacks=callbacks_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = \"weights-improvement-01-2.7873-bigger.hdf5\"\n",
    "model.load_weights(filename)\n",
    "model.compile(loss = \"categorical_crossentropy\",optimizer=\"adam\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Seed:\n",
      "\" tes. alice thought to\n",
      "herself, 'i don't see how he can even finish, if he doesn't begin.' but\n",
      "she wa \"\n"
     ]
    }
   ],
   "source": [
    "start = numpy.random.randint(0,len(dataX)-1)\n",
    "pattern = dataX[start]\n",
    "print(\"Seed:\")\n",
    "print(\"\\\"\",''.join([int_to_char[value] for value in pattern]),\"\\\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "r a sooe the mooee the tooe the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the mooee the toee the\n",
      "Done.\n"
     ]
    }
   ],
   "source": [
    "for i in range(1000):\n",
    "    x = numpy.reshape(pattern,(1,len(pattern),1))\n",
    "    x = x/float(n_vocab)\n",
    "    prediction = model.predict(x,verbose=0)\n",
    "    index = numpy.argmax(prediction)\n",
    "    result = int_to_char[index]\n",
    "    seq_in = [int_to_char[value] for value in pattern]\n",
    "    sys.stdout.write(result)\n",
    "    pattern.append(index)\n",
    "    pattern = pattern[1:len(pattern)]\n",
    "print(\"\\nDone.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}