Chapter 26 Sequence classification of movie reviews.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Simple LSTM for sequence classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy\n",
    "from keras.datasets import imdb\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.layers import LSTM\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.preprocessing import sequence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "numpy.random.seed(7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/arvind/.local/lib/python3.6/site-packages/keras/datasets/imdb.py:49: UserWarning: The `nb_words` argument in `load_data` has been renamed `num_words`.\n",
      "  warnings.warn('The `nb_words` argument in `load_data` '\n"
     ]
    }
   ],
   "source": [
    "top_words = 5000\n",
    "(X_train,y_train), (X_test,y_test) = imdb.load_data(nb_words = top_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_review_length = 500\n",
    "X_train = sequence.pad_sequences(X_train,maxlen=max_review_length)\n",
    "X_test = sequence.pad_sequences(X_test,maxlen=max_review_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_vecor_length = 32"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_3\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_3 (Embedding)      (None, 500, 32)           160000    \n",
      "_________________________________________________________________\n",
      "lstm_3 (LSTM)                (None, 100)               53200     \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 1)                 101       \n",
      "=================================================================\n",
      "Total params: 213,301\n",
      "Trainable params: 213,301\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0427 15:29:52.422997 139827340994368 deprecation.py:323] From /home/arvind/.local/lib/python3.6/site-packages/keras/optimizers.py:550: BaseResourceVariable.constraint (from tensorflow.python.ops.resource_variable_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Apply a constraint manually following the optimizer update step.\n",
      "W0427 15:29:52.755425 139827340994368 module_wrapper.py:136] From /home/arvind/.local/lib/python3.6/site-packages/tensorflow_core/python/util/module_wrapper.py:163: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 25000 samples, validate on 25000 samples\n",
      "Epoch 1/3\n",
      "25000/25000 [==============================] - 224s 9ms/step - loss: 0.5025 - accuracy: 0.7446 - val_loss: 0.3602 - val_accuracy: 0.8495\n",
      "Epoch 2/3\n",
      "25000/25000 [==============================] - 243s 10ms/step - loss: 0.3215 - accuracy: 0.8703 - val_loss: 0.3377 - val_accuracy: 0.8660\n",
      "Epoch 3/3\n",
      "25000/25000 [==============================] - 232s 9ms/step - loss: 0.2703 - accuracy: 0.8932 - val_loss: 0.3378 - val_accuracy: 0.8673\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.callbacks.History at 0x7f2bafd01f28>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "model = Sequential()\n",
    "model.add(Embedding(top_words,embedding_vecor_length,input_length = max_review_length))\n",
    "model.add(LSTM(100))\n",
    "model.add(Dense(1,activation = \"sigmoid\"))\n",
    "model.compile(loss = \"binary_crossentropy\",optimizer = \"adam\",metrics = [\"accuracy\"])\n",
    "print(model.summary())\n",
    "model.fit(X_train,y_train,validation_data = (X_test,y_test),epochs=3,batch_size=64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 86.73%\n"
     ]
    }
   ],
   "source": [
    "scores = model.evaluate(X_test,y_test,verbose=0)\n",
    "print(\"Accuracy: %.2f%%\" %(scores[1]*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model with dropout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import Dropout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/arvind/.local/lib/python3.6/site-packages/ipykernel_launcher.py:2: UserWarning: The `dropout` argument is no longer support in `Embedding`. You can apply a `keras.layers.SpatialDropout1D` layer right after the `Embedding` layer to get the same behavior.\n",
      "  \n",
      "W0427 21:04:17.649214 140132436387648 deprecation.py:506] From /home/arvind/.local/lib/python3.6/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1633: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n",
      "W0427 21:04:18.111909 140132436387648 deprecation.py:323] From /home/arvind/.local/lib/python3.6/site-packages/tensorflow_core/python/ops/nn_impl.py:183: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_2\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (None, 500, 32)           160000    \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 500, 32)           0         \n",
      "_________________________________________________________________\n",
      "lstm_1 (LSTM)                (None, 100)               53200     \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 100)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 1)                 101       \n",
      "=================================================================\n",
      "Total params: 213,301\n",
      "Trainable params: 213,301\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(Embedding(top_words, embedding_vecor_length,input_length = max_review_length,dropout = 0.2))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(LSTM(100))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(1,activation=\"sigmoid\"))\n",
    "model.compile(loss = \"binary_crossentropy\",optimizer = \"adam\",metrics=[\"accuracy\"])\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0427 21:04:48.839244 140132436387648 deprecation.py:323] From /home/arvind/.local/lib/python3.6/site-packages/keras/optimizers.py:550: BaseResourceVariable.constraint (from tensorflow.python.ops.resource_variable_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Apply a constraint manually following the optimizer update step.\n",
      "W0427 21:04:49.529098 140132436387648 module_wrapper.py:136] From /home/arvind/.local/lib/python3.6/site-packages/tensorflow_core/python/util/module_wrapper.py:163: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/3\n",
      "25000/25000 [==============================] - 176s 7ms/step - loss: 0.5164 - accuracy: 0.7501\n",
      "Epoch 2/3\n",
      "25000/25000 [==============================] - 175s 7ms/step - loss: 0.2949 - accuracy: 0.8808\n",
      "Epoch 3/3\n",
      "25000/25000 [==============================] - 176s 7ms/step - loss: 0.2541 - accuracy: 0.9004\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.callbacks.History at 0x7f7304062400>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train,y_train,epochs=3,batch_size=64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "scores = model.evaluate(X_test,y_test,verbose=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 87.36%\n"
     ]
    }
   ],
   "source": [
    "print(\"Accuracy: %.2f%%\" %(scores[1]*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Precise LSTM dropout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/arvind/.local/lib/python3.6/site-packages/ipykernel_launcher.py:2: UserWarning: The `dropout` argument is no longer support in `Embedding`. You can apply a `keras.layers.SpatialDropout1D` layer right after the `Embedding` layer to get the same behavior.\n",
      "  \n",
      "/home/arvind/.local/lib/python3.6/site-packages/ipykernel_launcher.py:3: UserWarning: Update your `LSTM` call to the Keras 2 API: `LSTM(100, dropout=0.2, recurrent_dropout=0.2)`\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_4\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_3 (Embedding)      (None, 500, 32)           160000    \n",
      "_________________________________________________________________\n",
      "lstm_3 (LSTM)                (None, 100)               53200     \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 1)                 101       \n",
      "=================================================================\n",
      "Total params: 213,301\n",
      "Trainable params: 213,301\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(Embedding(top_words, embedding_vecor_length,input_length = max_review_length,dropout = 0.2))\n",
    "model.add(LSTM(100,dropout_W=0.2,dropout_U=0.2))\n",
    "model.add(Dense(1,activation=\"sigmoid\"))\n",
    "model.compile(loss = \"binary_crossentropy\",optimizer = \"adam\",metrics=[\"accuracy\"])\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/arvind/.local/lib/python3.6/site-packages/ipykernel_launcher.py:1: UserWarning: The `nb_epoch` argument in `fit` has been renamed `epochs`.\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/3\n",
      "25000/25000 [==============================] - 179s 7ms/step - loss: 0.4933 - accuracy: 0.7615\n",
      "Epoch 2/3\n",
      "25000/25000 [==============================] - 180s 7ms/step - loss: 0.3768 - accuracy: 0.8391\n",
      "Epoch 3/3\n",
      "25000/25000 [==============================] - 179s 7ms/step - loss: 0.3370 - accuracy: 0.8611\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.callbacks.History at 0x7f728287f3c8>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train, y_train, nb_epoch=3, batch_size=64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 84.22%\n"
     ]
    }
   ],
   "source": [
    "scores = model.evaluate(X_test,y_test,verbose=0)\n",
    "print(\"Accuracy: %.2f%%\" %(scores[1]*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# LSTM CNN for sequence classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers.convolutional import Convolution1D\n",
    "from keras.layers.convolutional import MaxPooling1D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/arvind/.local/lib/python3.6/site-packages/ipykernel_launcher.py:3: UserWarning: Update your `Conv1D` call to the Keras 2 API: `Conv1D(padding=\"same\", activation=\"relu\", filters=32, kernel_size=3)`\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n",
      "/home/arvind/.local/lib/python3.6/site-packages/ipykernel_launcher.py:4: UserWarning: Update your `MaxPooling1D` call to the Keras 2 API: `MaxPooling1D(pool_size=2)`\n",
      "  after removing the cwd from sys.path.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_6\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_5 (Embedding)      (None, 500, 32)           160000    \n",
      "_________________________________________________________________\n",
      "conv1d_2 (Conv1D)            (None, 500, 32)           3104      \n",
      "_________________________________________________________________\n",
      "max_pooling1d_2 (MaxPooling1 (None, 250, 32)           0         \n",
      "_________________________________________________________________\n",
      "lstm_5 (LSTM)                (None, 100)               53200     \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 1)                 101       \n",
      "=================================================================\n",
      "Total params: 216,405\n",
      "Trainable params: 216,405\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(Embedding(top_words,embedding_vecor_length,input_length=max_review_length))\n",
    "model.add(Convolution1D(nb_filter=32,filter_length=3,padding=\"same\",activation=\"relu\"))\n",
    "model.add(MaxPooling1D(pool_length=2))\n",
    "model.add(LSTM(100))\n",
    "model.add(Dense(1,activation=\"sigmoid\"))\n",
    "model.compile(loss=\"binary_crossentropy\",optimizer = \"adam\",metrics=[\"accuracy\"])\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/arvind/.local/lib/python3.6/site-packages/ipykernel_launcher.py:1: UserWarning: The `nb_epoch` argument in `fit` has been renamed `epochs`.\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/3\n",
      "25000/25000 [==============================] - 96s 4ms/step - loss: 0.4408 - accuracy: 0.7787\n",
      "Epoch 2/3\n",
      "25000/25000 [==============================] - 95s 4ms/step - loss: 0.2502 - accuracy: 0.9018\n",
      "Epoch 3/3\n",
      "25000/25000 [==============================] - 95s 4ms/step - loss: 0.2203 - accuracy: 0.9158\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.callbacks.History at 0x7f7281337eb8>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train,y_train,nb_epoch=3,batch_size=64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "scores = model.evaluate(X_test,y_test,verbose=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 88.24%\n"
     ]
    }
   ],
   "source": [
    "print(\"Accuracy: %.2f%%\"%(scores[1]*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}