Sarcasm_classification_Reddit_hyperband.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "k9CqgGeuPtsu"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "HUasb8WjQHgW"
   },
   "outputs": [],
   "source": [
    "# https://www.kaggle.com/code/prashantkikani/are-you-being-sarcastic-sarcasm-detection-nlp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-03T06:44:45.437457Z",
     "start_time": "2022-11-03T06:44:35.221735Z"
    },
    "id": "ReWTpeEYPvWT"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Furqan Ali\\AppData\\Local\\Temp\\ipykernel_51164\\702577804.py:6: FutureWarning: Passing a negative integer is deprecated in version 1.0 and will not be supported in future version. Instead, use None to not limit the column width.\n",
      "  pd.set_option('max_colwidth', -1)\n",
      "C:\\Users\\Furqan Ali\\AppData\\Local\\Temp\\ipykernel_51164\\702577804.py:12: DeprecationWarning: `import kerastuner` is deprecated, please use `import keras_tuner`.\n",
      "  import kerastuner as kt\n"
     ]
    }
   ],
   "source": [
    "# pandas to open data files & processing it.\n",
    "import pandas as pd\n",
    "# to see all columns\n",
    "pd.set_option('display.max_columns', None)\n",
    "# To see whole text\n",
    "pd.set_option('max_colwidth', -1)\n",
    "\n",
    "# numpy for numeric data processing\n",
    "import numpy as np\n",
    "import seaborn as sbs\n",
    "import tensorflow as tf\n",
    "import kerastuner as kt\n",
    "\n",
    "# keras for deep learning model creation\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras_preprocessing.sequence import pad_sequences\n",
    "from keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, Flatten, Bidirectional, GlobalMaxPool1D\n",
    "from keras.models import Model\n",
    "from keras.utils import plot_model\n",
    "\n",
    "# to fix random seeds\n",
    "import random\n",
    "import torch\n",
    "import os\n",
    "\n",
    "# Regular Expression for text cleaning\n",
    "import re\n",
    "\n",
    "# to track the progress - progress bar\n",
    "from tqdm.notebook import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-03T06:44:49.267268Z",
     "start_time": "2022-11-03T06:44:45.437457Z"
    },
    "id": "-00gIOSgPvYl"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1010826, 10)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>comment</th>\n",
       "      <th>author</th>\n",
       "      <th>subreddit</th>\n",
       "      <th>score</th>\n",
       "      <th>ups</th>\n",
       "      <th>downs</th>\n",
       "      <th>date</th>\n",
       "      <th>created_utc</th>\n",
       "      <th>parent_comment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>NC and NH.</td>\n",
       "      <td>Trumpbart</td>\n",
       "      <td>politics</td>\n",
       "      <td>2</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>2016-10</td>\n",
       "      <td>2016-10-16 23:55:23</td>\n",
       "      <td>Yeah, I get that argument. At this point, I'd prefer is she lived in NC as well.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>You do know west teams play against west teams more than east teams right?</td>\n",
       "      <td>Shbshb906</td>\n",
       "      <td>nba</td>\n",
       "      <td>-4</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>2016-11</td>\n",
       "      <td>2016-11-01 00:24:10</td>\n",
       "      <td>The blazers and Mavericks (The wests 5 and 6 seed) did not even carry a good enough record to make the playoffs in the east last year.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>They were underdogs earlier today, but since Gronk's announcement this afternoon, the Vegas line has moved to patriots -1</td>\n",
       "      <td>Creepeth</td>\n",
       "      <td>nfl</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2016-09</td>\n",
       "      <td>2016-09-22 21:45:37</td>\n",
       "      <td>They're favored to win.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>This meme isn't funny none of the \"new york nigga\" ones are.</td>\n",
       "      <td>icebrotha</td>\n",
       "      <td>BlackPeopleTwitter</td>\n",
       "      <td>-8</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>2016-10</td>\n",
       "      <td>2016-10-18 21:03:47</td>\n",
       "      <td>deadass don't kill my buzz</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>I could use one of those tools.</td>\n",
       "      <td>cush2push</td>\n",
       "      <td>MaddenUltimateTeam</td>\n",
       "      <td>6</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>2016-12</td>\n",
       "      <td>2016-12-30 17:00:13</td>\n",
       "      <td>Yep can confirm I saw the tool they use for that. It was made by our boy EASports_MUT</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   label  \\\n",
       "0  0       \n",
       "1  0       \n",
       "2  0       \n",
       "3  0       \n",
       "4  0       \n",
       "\n",
       "                                                                                                                     comment  \\\n",
       "0  NC and NH.                                                                                                                  \n",
       "1  You do know west teams play against west teams more than east teams right?                                                  \n",
       "2  They were underdogs earlier today, but since Gronk's announcement this afternoon, the Vegas line has moved to patriots -1   \n",
       "3  This meme isn't funny none of the \"new york nigga\" ones are.                                                                \n",
       "4  I could use one of those tools.                                                                                             \n",
       "\n",
       "      author           subreddit  score  ups  downs     date  \\\n",
       "0  Trumpbart  politics            2     -1   -1      2016-10   \n",
       "1  Shbshb906  nba                -4     -1   -1      2016-11   \n",
       "2  Creepeth   nfl                 3      3    0      2016-09   \n",
       "3  icebrotha  BlackPeopleTwitter -8     -1   -1      2016-10   \n",
       "4  cush2push  MaddenUltimateTeam  6     -1   -1      2016-12   \n",
       "\n",
       "           created_utc  \\\n",
       "0  2016-10-16 23:55:23   \n",
       "1  2016-11-01 00:24:10   \n",
       "2  2016-09-22 21:45:37   \n",
       "3  2016-10-18 21:03:47   \n",
       "4  2016-12-30 17:00:13   \n",
       "\n",
       "                                                                                                                           parent_comment  \n",
       "0  Yeah, I get that argument. At this point, I'd prefer is she lived in NC as well.                                                        \n",
       "1  The blazers and Mavericks (The wests 5 and 6 seed) did not even carry a good enough record to make the playoffs in the east last year.  \n",
       "2  They're favored to win.                                                                                                                 \n",
       "3  deadass don't kill my buzz                                                                                                              \n",
       "4  Yep can confirm I saw the tool they use for that. It was made by our boy EASports_MUT                                                   "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sarcasm_data = pd.read_csv(\"reddit_dataset/train-balanced-sarcasm.csv\")\n",
    "print(sarcasm_data.shape)\n",
    "sarcasm_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-03T06:44:49.458755Z",
     "start_time": "2022-11-03T06:44:49.268269Z"
    },
    "id": "s4gA_VpUPvai"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>comment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>NC and NH.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>You do know west teams play against west teams more than east teams right?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>They were underdogs earlier today, but since Gronk's announcement this afternoon, the Vegas line has moved to patriots -1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>This meme isn't funny none of the \"new york nigga\" ones are.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>I could use one of those tools.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   label  \\\n",
       "0  0       \n",
       "1  0       \n",
       "2  0       \n",
       "3  0       \n",
       "4  0       \n",
       "\n",
       "                                                                                                                     comment  \n",
       "0  NC and NH.                                                                                                                 \n",
       "1  You do know west teams play against west teams more than east teams right?                                                 \n",
       "2  They were underdogs earlier today, but since Gronk's announcement this afternoon, the Vegas line has moved to patriots -1  \n",
       "3  This meme isn't funny none of the \"new york nigga\" ones are.                                                               \n",
       "4  I could use one of those tools.                                                                                            "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# We just need comment & label column.\n",
    "# So, let's remove others.\n",
    "\n",
    "sarcasm_data.drop(['author', 'subreddit', 'score', 'ups', 'downs', 'date', 'created_utc', 'parent_comment'], axis=1, inplace=True)\n",
    "# remove empty rows\n",
    "sarcasm_data.dropna(inplace=True)\n",
    "sarcasm_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-03T06:44:49.474891Z",
     "start_time": "2022-11-03T06:44:49.459760Z"
    },
    "id": "OpOO793GPvcz"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    505405\n",
       "1    505368\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sarcasm_data['label'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-03T06:44:49.488987Z",
     "start_time": "2022-11-03T06:44:49.475893Z"
    },
    "id": "tUwVSGa8Pve4"
   },
   "outputs": [],
   "source": [
    "# So, there are 505368 sentences with sarcastic news headlines !!\n",
    "\n",
    "# Let's do some pre-processing on our text data.\n",
    "# These are the common practices which can improve performance in almost any NLP task.\n",
    "\n",
    "\n",
    "# One common thing we can do is to remove contractions.\n",
    "\n",
    "# Like, \"ain't\" to \"is not\", \"can't\" to \"can not\" etc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-03T06:44:49.504991Z",
     "start_time": "2022-11-03T06:44:49.490762Z"
    },
    "id": "ZUQn3Os8PvhR"
   },
   "outputs": [],
   "source": [
    "mispell_dict = {\"ain't\": \"is not\", \"cannot\": \"can not\", \"aren't\": \"are not\", \"can't\": \"can not\", \"'cause\": \"because\", \"could've\": \"could have\", \"couldn't\": \"could not\", \"didn't\": \"did not\",\n",
    "                \"doesn't\": \"does not\",\n",
    "                \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\", \"he'd\": \"he would\", \"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\",\n",
    "                \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\", \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\", \"I'm\": \"I am\",\n",
    "                \"I've\": \"I have\", \"i'd\": \"i would\", \"i'd've\": \"i would have\", \"i'll\": \"i will\", \"i'll've\": \"i will have\", \"i'm\": \"i am\", \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\",\n",
    "                \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\", \"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\", \"mayn't\": \"may not\", \"might've\": \"might have\",\n",
    "                \"mightn't\": \"might not\", \"mightn't've\": \"might not have\", \"must've\": \"must have\", \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\",\n",
    "                \"needn't've\": \"need not have\", \"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \"sha'n't\": \"shall not\",\n",
    "                \"shan't've\": \"shall not have\", \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\",\n",
    "                \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\", \"so's\": \"so as\", \"this's\": \"this is\", \"that'd\": \"that would\",\n",
    "                \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\", \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\", \"they'd\": \"they would\",\n",
    "                \"they'd've\": \"they would have\", \"they'll\": \"they will\", \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\", \"wasn't\": \"was not\",\n",
    "                \"we'd\": \"we would\", \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\", \"we've\": \"we have\", \"weren't\": \"were not\",\n",
    "                \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\", \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\",\n",
    "                \"where'd\": \"where did\", \"where's\": \"where is\", \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\",\n",
    "                \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\", \"wont\": \"will not\", \"won't've\": \"will not have\", \"would've\": \"would have\",\n",
    "                \"wouldn't\": \"would not\",\n",
    "                \"wouldn't've\": \"would not have\", \"y'all\": \"you all\", \"y'all'd\": \"you all would\", \"y'all'd've\": \"you all would have\", \"y'all're\": \"you all are\", \"y'all've\": \"you all have\",\n",
    "                \"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\", \"you're\": \"you are\", \"you've\": \"you have\", 'colour': 'color',\n",
    "                'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor',\n",
    "                'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What',\n",
    "                'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I',\n",
    "                'theBest': 'the best', 'howdoes': 'how does', 'Etherium': 'Ethereum',\n",
    "                'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', \"whst\": 'what',\n",
    "                'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}\n",
    "\n",
    "mispell_dict = {k.lower(): v.lower() for k, v in mispell_dict.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-03T06:44:55.687031Z",
     "start_time": "2022-11-03T06:44:49.505991Z"
    },
    "id": "6vAGNbIyPvjV"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>comment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>nc and nh.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>you do know west teams play against west teams more than east teams right ?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>they were underdogs earlier today , but since gronk ' s announcement this afternoon , the vegas line has moved to patriots -1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>this meme is not funny none of the \" new york nigga \" ones are.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>i could use one of those tools.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   label  \\\n",
       "0  0       \n",
       "1  0       \n",
       "2  0       \n",
       "3  0       \n",
       "4  0       \n",
       "\n",
       "                                                                                                                         comment  \n",
       "0  nc and nh.                                                                                                                     \n",
       "1  you do know west teams play against west teams more than east teams right ?                                                    \n",
       "2  they were underdogs earlier today , but since gronk ' s announcement this afternoon , the vegas line has moved to patriots -1  \n",
       "3  this meme is not funny none of the \" new york nigga \" ones are.                                                                \n",
       "4  i could use one of those tools.                                                                                                "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let's make our preprocessing function.\n",
    "\n",
    "def preprocessing_text(s):\n",
    "    # making our string lowercase & removing extra spaces\n",
    "    s = str(s).lower().strip()\n",
    "    \n",
    "    # remove contractions.\n",
    "    s = \" \".join([mispell_dict[word] if word in mispell_dict.keys() else word for word in s.split()])\n",
    "    \n",
    "    # removing \\n\n",
    "    s = re.sub('\\n', '', s)\n",
    "    \n",
    "    # put spaces before & after punctuations to make words seprate. Like \"king?\" to \"king\", \"?\".\n",
    "    s = re.sub(r\"([?!,+=—&%\\'\\\";:¿।।।|\\(\\){}\\[\\]//])\", r\" \\1 \", s)\n",
    "    \n",
    "    # Remove more than 2 continues spaces with 1 space.\n",
    "    s = re.sub('[ ]{2,}', ' ', s).strip()\n",
    "    \n",
    "    return s\n",
    "\n",
    "\n",
    "# apply preprocessing_text function\n",
    "sarcasm_data['comment'] = sarcasm_data['comment'].apply(preprocessing_text)\n",
    "sarcasm_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-03T06:45:15.109282Z",
     "start_time": "2022-11-03T06:44:55.688090Z"
    },
    "id": "CvDlpM5-Pvla"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 0 ns\n",
      "Wall time: 0 ns\n"
     ]
    }
   ],
   "source": [
    "# total unique words we are going to use.\n",
    "TOTAL_WORDS = 60000\n",
    "\n",
    "# max number of words one sentence can have\n",
    "MAX_LEN = 50\n",
    "\n",
    "# width of of 1D embedding vector\n",
    "EMBEDDING_SIZE = 300\n",
    "\n",
    "\n",
    "%time\n",
    "tokenizer = Tokenizer(num_words=TOTAL_WORDS)\n",
    "tokenizer.fit_on_texts(list(sarcasm_data['comment']))\n",
    "\n",
    "train_data = tokenizer.texts_to_sequences(sarcasm_data['comment'])\n",
    "train_data = pad_sequences(train_data, maxlen = MAX_LEN)\n",
    "target = sarcasm_data['label']\n",
    "\n",
    "# Let's open embedding file now & store in a matrix."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-03T06:47:01.273053Z",
     "start_time": "2022-11-03T06:45:15.109282Z"
    },
    "id": "x6ceg2bVPvne"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 0 ns\n",
      "Wall time: 0 ns\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "04ed5e43dbf14a73a2b9ad4ca329d8ad",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%time\n",
    "EMBEDDING_FILE = 'embeddings/crawl-300d-2M.vec'\n",
    "\n",
    "def get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32')\n",
    "\n",
    "embeddings_index = dict(get_coefs(*o.rstrip().rsplit(' ')) for o in tqdm(open(EMBEDDING_FILE, encoding=\"utf8\")))\n",
    "\n",
    "word_index = tokenizer.word_index\n",
    "nb_words = min(TOTAL_WORDS, len(word_index))\n",
    "embedding_matrix = np.zeros((nb_words, EMBEDDING_SIZE))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-03T06:47:01.476992Z",
     "start_time": "2022-11-03T06:47:01.273053Z"
    },
    "id": "R03SFNhSPvpp"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d7b9d439fcbf4e67af0e241189cf100e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/166508 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for word, i in tqdm(word_index.items()):\n",
    "    if i >= TOTAL_WORDS: continue\n",
    "    embedding_vector = embeddings_index.get(word)\n",
    "    if embedding_vector is not None: embedding_matrix[i] = embedding_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-03T06:47:01.508246Z",
     "start_time": "2022-11-03T06:47:01.476992Z"
    },
    "id": "brAe42yqPvr0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(60000, 300)\n"
     ]
    }
   ],
   "source": [
    "# Let's build our NLP deep learning model now..\n",
    "\n",
    "print(embedding_matrix.shape)\n",
    "\n",
    "def seed_everything(seed):\n",
    "    random.seed(seed)\n",
    "    np.random.seed(seed)\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed(seed)\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "    tf.random.set_seed(seed)\n",
    "\n",
    "# We fix all the random seed so that, we can reproduce the results.\n",
    "seed_everything(2020)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-03T06:47:01.523870Z",
     "start_time": "2022-11-03T06:47:01.508246Z"
    }
   },
   "outputs": [],
   "source": [
    "######################### TUNER work here  ########################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-03T06:47:01.539493Z",
     "start_time": "2022-11-03T06:47:01.523870Z"
    }
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.losses import MeanSquaredLogarithmicError\n",
    "msle = MeanSquaredLogarithmicError()\n",
    "\n",
    "\n",
    "# hp_units1 = hp.Int('units1', min_value=32, max_value=512, step=32)\n",
    "# hp_units2 = hp.Int('units2', min_value=32, max_value=512, step=32)\n",
    "# hp_units3 = hp.Int('units3', min_value=32, max_value=512, step=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-03T06:47:01.555515Z",
     "start_time": "2022-11-03T06:47:01.539493Z"
    }
   },
   "outputs": [],
   "source": [
    "###################################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-03T06:47:01.571149Z",
     "start_time": "2022-11-03T06:47:01.555515Z"
    },
    "id": "4NSgBjihPvt4"
   },
   "outputs": [],
   "source": [
    "# input_layer = Input(shape=(MAX_LEN,))\n",
    "\n",
    "# embedding_layer = Embedding(TOTAL_WORDS, EMBEDDING_SIZE, weights = [embedding_matrix])(input_layer)\n",
    "\n",
    "# LSTM_layer = Bidirectional(LSTM(128, return_sequences = True))(embedding_layer)\n",
    "# maxpool_layer = GlobalMaxPool1D()(LSTM_layer)\n",
    "\n",
    "# dense_layer_1 = Dense(64, activation=\"relu\")(maxpool_layer)\n",
    "# dropout_1 = Dropout(0.5)(dense_layer_1)\n",
    "\n",
    "# dense_layer_2 = Dense(32, activation=\"relu\")(dropout_1)\n",
    "# dropout_2 = Dropout(0.5)(dense_layer_2)\n",
    "\n",
    "# output_layer = Dense(1, activation=\"sigmoid\")(dropout_2)\n",
    "\n",
    "# model = Model(input_layer, output_layer)\n",
    "\n",
    "# model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "# model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-03T06:47:01.586775Z",
     "start_time": "2022-11-03T06:47:01.571149Z"
    }
   },
   "outputs": [],
   "source": [
    "def build_model(hp):\n",
    "    \n",
    "    input_layer = Input(shape=(MAX_LEN,))\n",
    "\n",
    "    embedding_layer = Embedding(TOTAL_WORDS, EMBEDDING_SIZE, weights = [embedding_matrix])(input_layer)\n",
    "\n",
    "    hp_units1 = hp.Int('units1', min_value=32, max_value=512, step=32)\n",
    "    hp_units2 = hp.Int('units2', min_value=32, max_value=512, step=32)\n",
    "    hp_units3 = hp.Int('units3', min_value=32, max_value=512, step=32)\n",
    "    dropout1 = hp.Float(\n",
    "                    'dropout_1',\n",
    "                    min_value=0.0,\n",
    "                    max_value=0.7,\n",
    "                    default=0.25,\n",
    "                    step=0.08\n",
    "                )\n",
    "    dropout2 = hp.Float(\n",
    "                    'dropout_2',\n",
    "                    min_value=0.0,\n",
    "                    max_value=0.5,\n",
    "                    default=0.25,\n",
    "                    step=0.08\n",
    "                )\n",
    "    dropout3 = hp.Float(\n",
    "                    'dropout_3',\n",
    "                    min_value=0.0,\n",
    "                    max_value=0.5,\n",
    "                    default=0.25,\n",
    "                    step=0.08\n",
    "                )\n",
    "    \n",
    "    af1 = hp.Choice(\n",
    "                    'activation',\n",
    "                    values=['relu', 'tanh', 'sigmoid', 'elu', 'LeakyReLU'],\n",
    "                    default='relu'\n",
    "                )\n",
    "    af2 = hp.Choice(\n",
    "                    'activation_2',\n",
    "                    values=['relu', 'tanh', 'sigmoid', 'elu', 'LeakyReLU'],\n",
    "                    default='relu'\n",
    "                )\n",
    "    af3 = hp.Choice(\n",
    "                    'activation_3',\n",
    "                    values=['relu', 'tanh', 'sigmoid', 'elu', 'LeakyReLU'],\n",
    "                    default='relu'\n",
    "                )\n",
    "\n",
    "    LSTM_layer = Bidirectional(LSTM(hp_units1, return_sequences = True))(embedding_layer)\n",
    "    maxpool_layer = GlobalMaxPool1D()(LSTM_layer)\n",
    "\n",
    "    dense_layer_1 = Dense(hp_units2, activation=af1)(maxpool_layer)\n",
    "    dropout_1 = Dropout(dropout1)(dense_layer_1)\n",
    "\n",
    "    dense_layer_2 = Dense(hp_units3, activation=af2)(dropout_1)\n",
    "    dropout_2 = Dropout(dropout2)(dense_layer_2)\n",
    "\n",
    "    output_layer = Dense(1, activation=\"sigmoid\")(dropout_2)\n",
    "\n",
    "    # Tune the learning rate for the optimizer\n",
    "    # Choose an optimal value from 0.01, 0.001, or 0.0001\n",
    "    hp_learning_rate = hp.Choice('learning_rate', values=[1e-1, 1e-2, 1e-3, 1e-4, 1e-5, 1e-6])\n",
    "\n",
    "\n",
    "    model = Model(input_layer, output_layer)\n",
    "\n",
    "    model.compile(loss='binary_crossentropy', optimizer=tf.keras.optimizers.Adam(learning_rate=hp_learning_rate),\n",
    "                  metrics=['accuracy'])\n",
    "    model.summary()\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-04T01:37:15.151630Z",
     "start_time": "2022-11-03T06:47:01.586775Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trial 25 Complete [01h 05m 03s]\n",
      "val_accuracy: 0.7413371205329895\n",
      "\n",
      "Best val_accuracy So Far: 0.742168128490448\n",
      "Total elapsed time: 17h 36m 20s\n",
      "\n",
      "Search: Running Trial #26\n",
      "\n",
      "Value             |Best Value So Far |Hyperparameter\n",
      "448               |352               |units1\n",
      "192               |480               |units2\n",
      "160               |224               |units3\n",
      "1e-06             |0.1               |learning_rate\n",
      "10                |4                 |tuner/epochs\n",
      "4                 |2                 |tuner/initial_epoch\n",
      "1                 |2                 |tuner/bracket\n",
      "1                 |1                 |tuner/round\n",
      "0023              |0004              |tuner/trial_id\n",
      "\n",
      "Model: \"model\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " input_1 (InputLayer)        [(None, 50)]              0         \n",
      "                                                                 \n",
      " embedding (Embedding)       (None, 50, 300)           18000000  \n",
      "                                                                 \n",
      " bidirectional (Bidirectiona  (None, 50, 896)          2684416   \n",
      " l)                                                              \n",
      "                                                                 \n",
      " global_max_pooling1d (Globa  (None, 896)              0         \n",
      " lMaxPooling1D)                                                  \n",
      "                                                                 \n",
      " dense (Dense)               (None, 192)               172224    \n",
      "                                                                 \n",
      " dropout (Dropout)           (None, 192)               0         \n",
      "                                                                 \n",
      " dense_1 (Dense)             (None, 160)               30880     \n",
      "                                                                 \n",
      " dropout_1 (Dropout)         (None, 160)               0         \n",
      "                                                                 \n",
      " dense_2 (Dense)             (None, 1)                 161       \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 20,887,681\n",
      "Trainable params: 20,887,681\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Epoch 5/10\n",
      "    6/25270 [..............................] - ETA: 21:26 - loss: 0.7022 - accuracy: 0.5052WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.0244s vs `on_train_batch_end` time: 0.0258s). Check your callbacks.\n",
      "25270/25270 [==============================] - 721s 28ms/step - loss: 0.5536 - accuracy: 0.7173 - val_loss: 0.5208 - val_accuracy: 0.7397\n",
      "Epoch 6/10\n",
      " 5807/25270 [=====>........................] - ETA: 3:26:59 - loss: 0.4999 - accuracy: 0.7591"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Input \u001b[1;32mIn [17]\u001b[0m, in \u001b[0;36m<cell line: 10>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;66;03m# HyperBand algorithm from keras tuner\u001b[39;00m\n\u001b[0;32m      2\u001b[0m tuner \u001b[38;5;241m=\u001b[39m kt\u001b[38;5;241m.\u001b[39mHyperband(\n\u001b[0;32m      3\u001b[0m     build_model,\n\u001b[0;32m      4\u001b[0m     objective\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mval_accuracy\u001b[39m\u001b[38;5;124m'\u001b[39m,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m      7\u001b[0m     project_name\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mkeras_tuner_demo\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[0;32m      8\u001b[0m )\n\u001b[1;32m---> 10\u001b[0m \u001b[43mtuner\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msearch\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtrain_data\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtarget\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mepochs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m10\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mvalidation_split\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m0.2\u001b[39;49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32mc:\\users\\furqan ali\\desktop\\python\\env\\lib\\site-packages\\keras_tuner\\engine\\base_tuner.py:183\u001b[0m, in \u001b[0;36mBaseTuner.search\u001b[1;34m(self, *fit_args, **fit_kwargs)\u001b[0m\n\u001b[0;32m    180\u001b[0m     \u001b[38;5;28;01mcontinue\u001b[39;00m\n\u001b[0;32m    182\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mon_trial_begin(trial)\n\u001b[1;32m--> 183\u001b[0m results \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun_trial\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtrial\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mfit_args\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mfit_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    184\u001b[0m \u001b[38;5;66;03m# `results` is None indicates user updated oracle in `run_trial()`.\u001b[39;00m\n\u001b[0;32m    185\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m results \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
      "File \u001b[1;32mc:\\users\\furqan ali\\desktop\\python\\env\\lib\\site-packages\\keras_tuner\\tuners\\hyperband.py:384\u001b[0m, in \u001b[0;36mHyperband.run_trial\u001b[1;34m(self, trial, *fit_args, **fit_kwargs)\u001b[0m\n\u001b[0;32m    382\u001b[0m     fit_kwargs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mepochs\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m hp\u001b[38;5;241m.\u001b[39mvalues[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtuner/epochs\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[0;32m    383\u001b[0m     fit_kwargs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124minitial_epoch\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m hp\u001b[38;5;241m.\u001b[39mvalues[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtuner/initial_epoch\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[1;32m--> 384\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mHyperband\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun_trial\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtrial\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mfit_args\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mfit_kwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32mc:\\users\\furqan ali\\desktop\\python\\env\\lib\\site-packages\\keras_tuner\\engine\\tuner.py:295\u001b[0m, in \u001b[0;36mTuner.run_trial\u001b[1;34m(self, trial, *args, **kwargs)\u001b[0m\n\u001b[0;32m    293\u001b[0m     callbacks\u001b[38;5;241m.\u001b[39mappend(model_checkpoint)\n\u001b[0;32m    294\u001b[0m     copied_kwargs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcallbacks\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m callbacks\n\u001b[1;32m--> 295\u001b[0m     obj_value \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_build_and_fit_model\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtrial\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mcopied_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    297\u001b[0m     histories\u001b[38;5;241m.\u001b[39mappend(obj_value)\n\u001b[0;32m    298\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m histories\n",
      "File \u001b[1;32mc:\\users\\furqan ali\\desktop\\python\\env\\lib\\site-packages\\keras_tuner\\engine\\tuner.py:222\u001b[0m, in \u001b[0;36mTuner._build_and_fit_model\u001b[1;34m(self, trial, *args, **kwargs)\u001b[0m\n\u001b[0;32m    220\u001b[0m hp \u001b[38;5;241m=\u001b[39m trial\u001b[38;5;241m.\u001b[39mhyperparameters\n\u001b[0;32m    221\u001b[0m model \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_try_build(hp)\n\u001b[1;32m--> 222\u001b[0m results \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mhypermodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhp\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    223\u001b[0m tuner_utils\u001b[38;5;241m.\u001b[39mvalidate_trial_results(\n\u001b[0;32m    224\u001b[0m     results, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moracle\u001b[38;5;241m.\u001b[39mobjective, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mHyperModel.fit()\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    225\u001b[0m )\n\u001b[0;32m    226\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m results\n",
      "File \u001b[1;32mc:\\users\\furqan ali\\desktop\\python\\env\\lib\\site-packages\\keras_tuner\\engine\\hypermodel.py:140\u001b[0m, in \u001b[0;36mHyperModel.fit\u001b[1;34m(self, hp, model, *args, **kwargs)\u001b[0m\n\u001b[0;32m    116\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mfit\u001b[39m(\u001b[38;5;28mself\u001b[39m, hp, model, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[0;32m    117\u001b[0m     \u001b[38;5;124;03m\"\"\"Train the model.\u001b[39;00m\n\u001b[0;32m    118\u001b[0m \n\u001b[0;32m    119\u001b[0m \u001b[38;5;124;03m    Args:\u001b[39;00m\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    138\u001b[0m \u001b[38;5;124;03m        If return a float, it should be the `objective` value.\u001b[39;00m\n\u001b[0;32m    139\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[1;32m--> 140\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32mc:\\users\\furqan ali\\desktop\\python\\env\\lib\\site-packages\\keras\\utils\\traceback_utils.py:64\u001b[0m, in \u001b[0;36mfilter_traceback.<locals>.error_handler\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     62\u001b[0m filtered_tb \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m     63\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m---> 64\u001b[0m   \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     65\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:  \u001b[38;5;66;03m# pylint: disable=broad-except\u001b[39;00m\n\u001b[0;32m     66\u001b[0m   filtered_tb \u001b[38;5;241m=\u001b[39m _process_traceback_frames(e\u001b[38;5;241m.\u001b[39m__traceback__)\n",
      "File \u001b[1;32mc:\\users\\furqan ali\\desktop\\python\\env\\lib\\site-packages\\keras\\engine\\training.py:1414\u001b[0m, in \u001b[0;36mModel.fit\u001b[1;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[0;32m   1412\u001b[0m logs \u001b[38;5;241m=\u001b[39m tmp_logs  \u001b[38;5;66;03m# No error, now safe to assign to logs.\u001b[39;00m\n\u001b[0;32m   1413\u001b[0m end_step \u001b[38;5;241m=\u001b[39m step \u001b[38;5;241m+\u001b[39m data_handler\u001b[38;5;241m.\u001b[39mstep_increment\n\u001b[1;32m-> 1414\u001b[0m \u001b[43mcallbacks\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mon_train_batch_end\u001b[49m\u001b[43m(\u001b[49m\u001b[43mend_step\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlogs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1415\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstop_training:\n\u001b[0;32m   1416\u001b[0m   \u001b[38;5;28;01mbreak\u001b[39;00m\n",
      "File \u001b[1;32mc:\\users\\furqan ali\\desktop\\python\\env\\lib\\site-packages\\keras\\callbacks.py:438\u001b[0m, in \u001b[0;36mCallbackList.on_train_batch_end\u001b[1;34m(self, batch, logs)\u001b[0m\n\u001b[0;32m    431\u001b[0m \u001b[38;5;124;03m\"\"\"Calls the `on_train_batch_end` methods of its callbacks.\u001b[39;00m\n\u001b[0;32m    432\u001b[0m \n\u001b[0;32m    433\u001b[0m \u001b[38;5;124;03mArgs:\u001b[39;00m\n\u001b[0;32m    434\u001b[0m \u001b[38;5;124;03m    batch: Integer, index of batch within the current epoch.\u001b[39;00m\n\u001b[0;32m    435\u001b[0m \u001b[38;5;124;03m    logs: Dict. Aggregated metric results up until this batch.\u001b[39;00m\n\u001b[0;32m    436\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m    437\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_should_call_train_batch_hooks:\n\u001b[1;32m--> 438\u001b[0m   \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_batch_hook\u001b[49m\u001b[43m(\u001b[49m\u001b[43mModeKeys\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mTRAIN\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mend\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbatch\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlogs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlogs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32mc:\\users\\furqan ali\\desktop\\python\\env\\lib\\site-packages\\keras\\callbacks.py:297\u001b[0m, in \u001b[0;36mCallbackList._call_batch_hook\u001b[1;34m(self, mode, hook, batch, logs)\u001b[0m\n\u001b[0;32m    295\u001b[0m   \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_batch_begin_hook(mode, batch, logs)\n\u001b[0;32m    296\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m hook \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mend\u001b[39m\u001b[38;5;124m'\u001b[39m:\n\u001b[1;32m--> 297\u001b[0m   \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_batch_end_hook\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmode\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbatch\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlogs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    298\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m    299\u001b[0m   \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[0;32m    300\u001b[0m       \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mUnrecognized hook: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mhook\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m. Expected values are [\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mbegin\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m, \u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mend\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m]\u001b[39m\u001b[38;5;124m'\u001b[39m)\n",
      "File \u001b[1;32mc:\\users\\furqan ali\\desktop\\python\\env\\lib\\site-packages\\keras\\callbacks.py:318\u001b[0m, in \u001b[0;36mCallbackList._call_batch_end_hook\u001b[1;34m(self, mode, batch, logs)\u001b[0m\n\u001b[0;32m    315\u001b[0m   batch_time \u001b[38;5;241m=\u001b[39m time\u001b[38;5;241m.\u001b[39mtime() \u001b[38;5;241m-\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_batch_start_time\n\u001b[0;32m    316\u001b[0m   \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_batch_times\u001b[38;5;241m.\u001b[39mappend(batch_time)\n\u001b[1;32m--> 318\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_batch_hook_helper\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhook_name\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbatch\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlogs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    320\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_batch_times) \u001b[38;5;241m>\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_num_batches_for_timing_check:\n\u001b[0;32m    321\u001b[0m   end_hook_name \u001b[38;5;241m=\u001b[39m hook_name\n",
      "File \u001b[1;32mc:\\users\\furqan ali\\desktop\\python\\env\\lib\\site-packages\\keras\\callbacks.py:353\u001b[0m, in \u001b[0;36mCallbackList._call_batch_hook_helper\u001b[1;34m(self, hook_name, batch, logs)\u001b[0m\n\u001b[0;32m    350\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_check_timing:\n\u001b[0;32m    351\u001b[0m   start_time \u001b[38;5;241m=\u001b[39m time\u001b[38;5;241m.\u001b[39mtime()\n\u001b[1;32m--> 353\u001b[0m logs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_process_logs\u001b[49m\u001b[43m(\u001b[49m\u001b[43mlogs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mis_batch_hook\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[0;32m    354\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m callback \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcallbacks:\n\u001b[0;32m    355\u001b[0m   hook \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mgetattr\u001b[39m(callback, hook_name)\n",
      "File \u001b[1;32mc:\\users\\furqan ali\\desktop\\python\\env\\lib\\site-packages\\keras\\callbacks.py:272\u001b[0m, in \u001b[0;36mCallbackList._process_logs\u001b[1;34m(self, logs, is_batch_hook)\u001b[0m\n\u001b[0;32m    270\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m is_batch_hook \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_batch_hooks_support_tf_logs:\n\u001b[0;32m    271\u001b[0m   \u001b[38;5;28;01mreturn\u001b[39;00m logs\n\u001b[1;32m--> 272\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mtf_utils\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msync_to_numpy_or_python_type\u001b[49m\u001b[43m(\u001b[49m\u001b[43mlogs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32mc:\\users\\furqan ali\\desktop\\python\\env\\lib\\site-packages\\keras\\utils\\tf_utils.py:607\u001b[0m, in \u001b[0;36msync_to_numpy_or_python_type\u001b[1;34m(tensors)\u001b[0m\n\u001b[0;32m    604\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m t\n\u001b[0;32m    605\u001b[0m   \u001b[38;5;28;01mreturn\u001b[39;00m t\u001b[38;5;241m.\u001b[39mitem() \u001b[38;5;28;01mif\u001b[39;00m np\u001b[38;5;241m.\u001b[39mndim(t) \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m0\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m t\n\u001b[1;32m--> 607\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mtf\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mnest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmap_structure\u001b[49m\u001b[43m(\u001b[49m\u001b[43m_to_single_numpy_or_python_type\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtensors\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32mc:\\users\\furqan ali\\desktop\\python\\env\\lib\\site-packages\\tensorflow\\python\\util\\nest.py:916\u001b[0m, in \u001b[0;36mmap_structure\u001b[1;34m(func, *structure, **kwargs)\u001b[0m\n\u001b[0;32m    912\u001b[0m flat_structure \u001b[38;5;241m=\u001b[39m (flatten(s, expand_composites) \u001b[38;5;28;01mfor\u001b[39;00m s \u001b[38;5;129;01min\u001b[39;00m structure)\n\u001b[0;32m    913\u001b[0m entries \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mzip\u001b[39m(\u001b[38;5;241m*\u001b[39mflat_structure)\n\u001b[0;32m    915\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m pack_sequence_as(\n\u001b[1;32m--> 916\u001b[0m     structure[\u001b[38;5;241m0\u001b[39m], [func(\u001b[38;5;241m*\u001b[39mx) \u001b[38;5;28;01mfor\u001b[39;00m x \u001b[38;5;129;01min\u001b[39;00m entries],\n\u001b[0;32m    917\u001b[0m     expand_composites\u001b[38;5;241m=\u001b[39mexpand_composites)\n",
      "File \u001b[1;32mc:\\users\\furqan ali\\desktop\\python\\env\\lib\\site-packages\\tensorflow\\python\\util\\nest.py:916\u001b[0m, in \u001b[0;36m<listcomp>\u001b[1;34m(.0)\u001b[0m\n\u001b[0;32m    912\u001b[0m flat_structure \u001b[38;5;241m=\u001b[39m (flatten(s, expand_composites) \u001b[38;5;28;01mfor\u001b[39;00m s \u001b[38;5;129;01min\u001b[39;00m structure)\n\u001b[0;32m    913\u001b[0m entries \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mzip\u001b[39m(\u001b[38;5;241m*\u001b[39mflat_structure)\n\u001b[0;32m    915\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m pack_sequence_as(\n\u001b[1;32m--> 916\u001b[0m     structure[\u001b[38;5;241m0\u001b[39m], [\u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mx\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m x \u001b[38;5;129;01min\u001b[39;00m entries],\n\u001b[0;32m    917\u001b[0m     expand_composites\u001b[38;5;241m=\u001b[39mexpand_composites)\n",
      "File \u001b[1;32mc:\\users\\furqan ali\\desktop\\python\\env\\lib\\site-packages\\keras\\utils\\tf_utils.py:601\u001b[0m, in \u001b[0;36msync_to_numpy_or_python_type.<locals>._to_single_numpy_or_python_type\u001b[1;34m(t)\u001b[0m\n\u001b[0;32m    598\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_to_single_numpy_or_python_type\u001b[39m(t):\n\u001b[0;32m    599\u001b[0m   \u001b[38;5;66;03m# Don't turn ragged or sparse tensors to NumPy.\u001b[39;00m\n\u001b[0;32m    600\u001b[0m   \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(t, tf\u001b[38;5;241m.\u001b[39mTensor):\n\u001b[1;32m--> 601\u001b[0m     t \u001b[38;5;241m=\u001b[39m \u001b[43mt\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mnumpy\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    602\u001b[0m   \u001b[38;5;66;03m# Strings, ragged and sparse tensors don't have .item(). Return them as-is.\u001b[39;00m\n\u001b[0;32m    603\u001b[0m   \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(t, (np\u001b[38;5;241m.\u001b[39mndarray, np\u001b[38;5;241m.\u001b[39mgeneric)):\n",
      "File \u001b[1;32mc:\\users\\furqan ali\\desktop\\python\\env\\lib\\site-packages\\tensorflow\\python\\framework\\ops.py:1159\u001b[0m, in \u001b[0;36m_EagerTensorBase.numpy\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m   1136\u001b[0m \u001b[38;5;124;03m\"\"\"Copy of the contents of this Tensor into a NumPy array or scalar.\u001b[39;00m\n\u001b[0;32m   1137\u001b[0m \n\u001b[0;32m   1138\u001b[0m \u001b[38;5;124;03mUnlike NumPy arrays, Tensors are immutable, so this method has to copy\u001b[39;00m\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m   1156\u001b[0m \u001b[38;5;124;03m    NumPy dtype.\u001b[39;00m\n\u001b[0;32m   1157\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m   1158\u001b[0m \u001b[38;5;66;03m# TODO(slebedev): Consider avoiding a copy for non-CPU or remote tensors.\u001b[39;00m\n\u001b[1;32m-> 1159\u001b[0m maybe_arr \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_numpy\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m  \u001b[38;5;66;03m# pylint: disable=protected-access\u001b[39;00m\n\u001b[0;32m   1160\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m maybe_arr\u001b[38;5;241m.\u001b[39mcopy() \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(maybe_arr, np\u001b[38;5;241m.\u001b[39mndarray) \u001b[38;5;28;01melse\u001b[39;00m maybe_arr\n",
      "File \u001b[1;32mc:\\users\\furqan ali\\desktop\\python\\env\\lib\\site-packages\\tensorflow\\python\\framework\\ops.py:1125\u001b[0m, in \u001b[0;36m_EagerTensorBase._numpy\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m   1123\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_numpy\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m   1124\u001b[0m   \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m-> 1125\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_numpy_internal\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1126\u001b[0m   \u001b[38;5;28;01mexcept\u001b[39;00m core\u001b[38;5;241m.\u001b[39m_NotOkStatusException \u001b[38;5;28;01mas\u001b[39;00m e:  \u001b[38;5;66;03m# pylint: disable=protected-access\u001b[39;00m\n\u001b[0;32m   1127\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m core\u001b[38;5;241m.\u001b[39m_status_to_exception(e) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28mNone\u001b[39m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# HyperBand algorithm from keras tuner\n",
    "tuner = kt.Hyperband(\n",
    "    build_model,\n",
    "    objective='val_accuracy',\n",
    "    max_epochs=10,\n",
    "    directory='keras_tuner_dir_2',\n",
    "    project_name='keras_tuner_demo',\n",
    "    max_trials = 2,\n",
    "    executions_per_trial = 1,\n",
    ")\n",
    "\n",
    "tuner.search(train_data, target, epochs=10, validation_split=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "uQd60JqLPvwZ"
   },
   "outputs": [],
   "source": [
    "# Before training the model first, let's understand our model first.\n",
    "# input_layer : Input layer with which we will get text sentence.\n",
    "\n",
    "# embedding_layer : Embedding layer with which we will map each word with it's corresponding embedding vector.\n",
    "\n",
    "# LSTM_layer : LSTM layer with 128 LSTM cells.\n",
    "\n",
    "# We are using Bidirectional to run LSTM from both side of the text sentence.\n",
    "# Left to Right\n",
    "# Right to Left\n",
    "# Purpose of this is to give our model both side context.\n",
    "# It's also possible to not use this. But using this have provided good results.\n",
    "\n",
    "# maxpool_layer : Max pool layer is used to minimize the image size by pooling maximum number out of 2x2 grid.maxpool\n",
    "\n",
    "# dense_layer_1 : Feed-forward dense layer to classify the features captured by LSTM layer.\n",
    "\n",
    "# dropout_1 : Dropout is interesting trick. In Dropout, we randomly turn off some percentage of our neurons so that their's output can't go to next layer. Here we are turning off 20% of our total neurons.\n",
    "# Purpose of doing this is again to make our training robust.\n",
    "# Network should not depend some specific neurons to make predictions. And random turn will allow us to do that.\n",
    "# Picture below help us to understand it.dropout\n",
    "\n",
    "# dense_layer_2 & dropout_2 are same as above.\n",
    "\n",
    "# output_layer : To get the output prediction from the neural network."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "id": "9_0j9X4VPvyd"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You must install pydot (`pip install pydot`) and install graphviz (see instructions at https://graphviz.gitlab.io/download/) for plot_model/model_to_dot to work.\n"
     ]
    }
   ],
   "source": [
    "plot_model(model, show_shapes=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "id": "FkWtrudxPv0v"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "1580/1580 [==============================] - 79s 46ms/step - loss: 0.5654 - accuracy: 0.7103 - val_loss: 0.5220 - val_accuracy: 0.7392\n",
      "Epoch 2/2\n",
      "1580/1580 [==============================] - 73s 46ms/step - loss: 0.5164 - accuracy: 0.7474 - val_loss: 0.5161 - val_accuracy: 0.7414\n"
     ]
    }
   ],
   "source": [
    "BATCH_SIZE = 512\n",
    "EPOCHS = 2\n",
    "\n",
    "history = model.fit(\n",
    "    train_data, target,\n",
    "    batch_size=BATCH_SIZE,\n",
    "    epochs=EPOCHS,\n",
    "    # We are using randomly selected 20% sentences as validation data.\n",
    "    validation_split=0.2\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "id": "lkplWhauPv26"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>comment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>821405</th>\n",
       "      <td>1</td>\n",
       "      <td>they were rescued by an army of twitter hash tag.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>931327</th>\n",
       "      <td>1</td>\n",
       "      <td>oh ok i guess i am deserving of downvotes.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>341335</th>\n",
       "      <td>1</td>\n",
       "      <td>it is not magic they use jutsu ugh</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>513336</th>\n",
       "      <td>1</td>\n",
       "      <td>this is so funny and clever</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79534</th>\n",
       "      <td>1</td>\n",
       "      <td>he is probably pissed about no new macbook pros.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>328832</th>\n",
       "      <td>1</td>\n",
       "      <td>why does not he try to steer out of the way ?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118997</th>\n",
       "      <td>1</td>\n",
       "      <td>yeah , we can drop all that stem stuff now.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>343211</th>\n",
       "      <td>1</td>\n",
       "      <td>for children ' s day they are sending a bunch of adult children to compete against each other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>872120</th>\n",
       "      <td>1</td>\n",
       "      <td>amtrak is the greatest example of why laissez faire capitalism is horrible</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>277118</th>\n",
       "      <td>1</td>\n",
       "      <td>what do you do with all your buffs ?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>606485</th>\n",
       "      <td>1</td>\n",
       "      <td>may they live happily ever after together...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78901</th>\n",
       "      <td>1</td>\n",
       "      <td>gg overwatch</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>872915</th>\n",
       "      <td>1</td>\n",
       "      <td>dae chief keef is not *real* rap</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>559565</th>\n",
       "      <td>1</td>\n",
       "      <td>no that is only in the tekno remixez</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9641</th>\n",
       "      <td>1</td>\n",
       "      <td>you dropped your</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>158207</th>\n",
       "      <td>1</td>\n",
       "      <td>should have put antifreeze</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>566806</th>\n",
       "      <td>1</td>\n",
       "      <td>leap years.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>170990</th>\n",
       "      <td>1</td>\n",
       "      <td>con tutta la coca che hanno in corpo va bene se non camminano sulle acque</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>877074</th>\n",
       "      <td>1</td>\n",
       "      <td>do not worry , this expansion is as much about the alliance as it is the horde !</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>967012</th>\n",
       "      <td>1</td>\n",
       "      <td>nsfw</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        label  \\\n",
       "821405  1       \n",
       "931327  1       \n",
       "341335  1       \n",
       "513336  1       \n",
       "79534   1       \n",
       "328832  1       \n",
       "118997  1       \n",
       "343211  1       \n",
       "872120  1       \n",
       "277118  1       \n",
       "606485  1       \n",
       "78901   1       \n",
       "872915  1       \n",
       "559565  1       \n",
       "9641    1       \n",
       "158207  1       \n",
       "566806  1       \n",
       "170990  1       \n",
       "877074  1       \n",
       "967012  1       \n",
       "\n",
       "                                                                                              comment  \n",
       "821405  they were rescued by an army of twitter hash tag.                                              \n",
       "931327  oh ok i guess i am deserving of downvotes.                                                     \n",
       "341335  it is not magic they use jutsu ugh                                                             \n",
       "513336  this is so funny and clever                                                                    \n",
       "79534   he is probably pissed about no new macbook pros.                                               \n",
       "328832  why does not he try to steer out of the way ?                                                  \n",
       "118997  yeah , we can drop all that stem stuff now.                                                    \n",
       "343211  for children ' s day they are sending a bunch of adult children to compete against each other  \n",
       "872120  amtrak is the greatest example of why laissez faire capitalism is horrible                     \n",
       "277118  what do you do with all your buffs ?                                                           \n",
       "606485  may they live happily ever after together...                                                   \n",
       "78901   gg overwatch                                                                                   \n",
       "872915  dae chief keef is not *real* rap                                                               \n",
       "559565  no that is only in the tekno remixez                                                           \n",
       "9641    you dropped your                                                                               \n",
       "158207  should have put antifreeze                                                                     \n",
       "566806  leap years.                                                                                    \n",
       "170990  con tutta la coca che hanno in corpo va bene se non camminano sulle acque                      \n",
       "877074  do not worry , this expansion is as much about the alliance as it is the horde !               \n",
       "967012  nsfw                                                                                           "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let's test our model on some random input now.\n",
    "\n",
    "sarcasm_data[sarcasm_data['label']==1].sample(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "id": "spvwPJJPPv5T"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sun rises from the east\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[   0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "           0, 1664, 9351,   55,    1, 1061]])"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence = \"sun rises from the east\"\n",
    "sentence = preprocessing_text(sentence)\n",
    "print(sentence)\n",
    "\n",
    "sentence = tokenizer.texts_to_sequences([sentence])\n",
    "sentence = pad_sequences(sentence, maxlen = MAX_LEN)\n",
    "sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "id": "v6UBiBKpPv7R"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/1 [==============================] - 0s 434ms/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.3057979"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Make the prediction.\n",
    "prediction = model.predict(sentence)\n",
    "prediction[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "id": "D1rQFlJEPv9m"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "So, it's saying sentence have probability of 30.580 percent\n"
     ]
    }
   ],
   "source": [
    "print(\"So, it's saying sentence have probability of %.3f percent\"%(prediction[0][0]*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "id": "9b5wXamUPv_2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "is not it great that , your girlfriend dumped you ?\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[   0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "           0,    0,    0,    0,    0,    0,    0,    0,    2,    7,    8,\n",
       "         121,   10,   39, 1642, 9632,    6]])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence = \"Isn't it great that, your girlfriend dumped you?\"\n",
    "sentence = preprocessing_text(sentence)\n",
    "print(sentence)\n",
    "\n",
    "sentence = tokenizer.texts_to_sequences([sentence])\n",
    "sentence = pad_sequences(sentence, maxlen = MAX_LEN)\n",
    "sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "id": "PMF7t_33PwB2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/1 [==============================] - 0s 14ms/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.58195543"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Make the prediction.\n",
    "prediction = model.predict(sentence)\n",
    "prediction[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "id": "W4HPuljpPwFS"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "So, it's saying sentence have probability of 58.196 percent\n"
     ]
    }
   ],
   "source": [
    "print(\"So, it's saying sentence have probability of %.3f percent\"%(prediction[0][0]*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "A_PUwjHYRn7n"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "XbEBfVM3Rn9s"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "D9CqGpM4Rn_2"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "fnRwk4qQRoBs"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}