Bert_Phishing_Test.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "2bzif-_OCWTV"
   },
   "source": [
    "# BERT ile Phishing URL'leri Tespit Sistemi"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "47Wv7Ff8CdFe"
   },
   "source": [
    "## Gerekli Paketleri Yükleme"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "vXIwgShLCPtt",
    "outputId": "6606b58b-2d9f-4d4c-e270-483486124898"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m570.5/570.5 kB\u001b[0m \u001b[31m3.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m950.8/950.8 kB\u001b[0m \u001b[31m16.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.2/5.2 MB\u001b[0m \u001b[31m20.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.2/5.2 MB\u001b[0m \u001b[31m28.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h"
     ]
    }
   ],
   "source": [
    "!pip install -q keras-nlp tf-keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "6FPdJ8ndCgw4",
    "outputId": "3de95da3-faa1-428f-91ae-fae04032ece4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import warnings\n",
    "\n",
    "import re\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "import keras\n",
    "import keras_nlp\n",
    "from sklearn.metrics import classification_report, confusion_matrix\n",
    "from google.colab import drive\n",
    "\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "by8NvSEeCyro",
    "outputId": "190d95b1-feac-44f2-fe2b-dc4af63a4d78"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mounted at /gdrive\n",
      "/gdrive\n"
     ]
    }
   ],
   "source": [
    "drive.mount(\"/gdrive\")\n",
    "\n",
    "%cd /gdrive\n",
    "\n",
    "os.chdir(\"/gdrive/MyDrive/Mezuniyet Tezi\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "iQ6BGCYXDQx-"
   },
   "source": [
    "## Veri Seti ve Model Yüklenmesi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "9zWrQSSiC4TT",
    "outputId": "6625584b-a0ca-4c36-ce29-93c23359841d"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`compile()` was not called as part of model loading because the model's `compile()` method is custom. All subclassed Models that have `compile()` overridden should also override `get_compile_config()` and `compile_from_config(config)`. Alternatively, you can call `compile()` manually after loading.\n",
      "WARNING:absl:Skipping variable loading for optimizer 'Adam', because it has 1 variables whereas the saved optimizer has 403 variables. \n"
     ]
    }
   ],
   "source": [
    "model = keras.saving.load_model(\"./model/url_detection_model.keras\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "jmYqxbUXDSze"
   },
   "outputs": [],
   "source": [
    "val_df = pd.read_pickle(\"./veri-seti-pickle/pickle_val_20000.pkl\")\n",
    "\n",
    "x_test = tf.convert_to_tensor(val_df['url'])\n",
    "y_test = tf.convert_to_tensor(val_df['label'])\n",
    "\n",
    "#Tensorflow dataset'e dönüşüm yapıldı.\n",
    "val_data = tf.data.Dataset.from_tensor_slices((x_test,y_test))\n",
    "val_data = val_data.batch(32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 433
    },
    "id": "xmk0u_nuD6hp",
    "outputId": "f2f5437a-744f-4ad7-ec29-6a6745de3a90"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">Preprocessor: \"bert_preprocessor\"</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1mPreprocessor: \"bert_preprocessor\"\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
       "┃<span style=\"font-weight: bold\"> Tokenizer (type)                                   </span>┃<span style=\"font-weight: bold\">                                             Vocab # </span>┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
       "│ bert_tokenizer (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">BertTokenizer</span>)                     │                                              <span style=\"color: #00af00; text-decoration-color: #00af00\">30,522</span> │\n",
       "└────────────────────────────────────────────────────┴─────────────────────────────────────────────────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
       "┃\u001b[1m \u001b[0m\u001b[1mTokenizer (type)                                  \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m                                            Vocab #\u001b[0m\u001b[1m \u001b[0m┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
       "│ bert_tokenizer (\u001b[38;5;33mBertTokenizer\u001b[0m)                     │                                              \u001b[38;5;34m30,522\u001b[0m │\n",
       "└────────────────────────────────────────────────────┴─────────────────────────────────────────────────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">Model: \"bert_classifier\"</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1mModel: \"bert_classifier\"\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
       "┃<span style=\"font-weight: bold\"> Layer (type)                                  </span>┃<span style=\"font-weight: bold\"> Output Shape                           </span>┃<span style=\"font-weight: bold\">         Param # </span>┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
       "│ padding_mask (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">InputLayer</span>)                     │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>)                           │               <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │\n",
       "├───────────────────────────────────────────────┼────────────────────────────────────────┼─────────────────┤\n",
       "│ segment_ids (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">InputLayer</span>)                      │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>)                           │               <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │\n",
       "├───────────────────────────────────────────────┼────────────────────────────────────────┼─────────────────┤\n",
       "│ token_ids (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">InputLayer</span>)                        │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>)                           │               <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │\n",
       "├───────────────────────────────────────────────┼────────────────────────────────────────┼─────────────────┤\n",
       "│ bert_backbone (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">BertBackbone</span>)                  │ {sequence_output: (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">768</span>),   │     <span style=\"color: #00af00; text-decoration-color: #00af00\">109,482,240</span> │\n",
       "│                                               │ pooled_output: (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">768</span>)}            │                 │\n",
       "├───────────────────────────────────────────────┼────────────────────────────────────────┼─────────────────┤\n",
       "│ classifier_dropout (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dropout</span>)                  │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">768</span>)                            │               <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │\n",
       "├───────────────────────────────────────────────┼────────────────────────────────────────┼─────────────────┤\n",
       "│ logits (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)                                │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">2</span>)                              │           <span style=\"color: #00af00; text-decoration-color: #00af00\">1,538</span> │\n",
       "└───────────────────────────────────────────────┴────────────────────────────────────────┴─────────────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
       "┃\u001b[1m \u001b[0m\u001b[1mLayer (type)                                 \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mOutput Shape                          \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m        Param #\u001b[0m\u001b[1m \u001b[0m┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
       "│ padding_mask (\u001b[38;5;33mInputLayer\u001b[0m)                     │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;45mNone\u001b[0m)                           │               \u001b[38;5;34m0\u001b[0m │\n",
       "├───────────────────────────────────────────────┼────────────────────────────────────────┼─────────────────┤\n",
       "│ segment_ids (\u001b[38;5;33mInputLayer\u001b[0m)                      │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;45mNone\u001b[0m)                           │               \u001b[38;5;34m0\u001b[0m │\n",
       "├───────────────────────────────────────────────┼────────────────────────────────────────┼─────────────────┤\n",
       "│ token_ids (\u001b[38;5;33mInputLayer\u001b[0m)                        │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;45mNone\u001b[0m)                           │               \u001b[38;5;34m0\u001b[0m │\n",
       "├───────────────────────────────────────────────┼────────────────────────────────────────┼─────────────────┤\n",
       "│ bert_backbone (\u001b[38;5;33mBertBackbone\u001b[0m)                  │ {sequence_output: (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m768\u001b[0m),   │     \u001b[38;5;34m109,482,240\u001b[0m │\n",
       "│                                               │ pooled_output: (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m768\u001b[0m)}            │                 │\n",
       "├───────────────────────────────────────────────┼────────────────────────────────────────┼─────────────────┤\n",
       "│ classifier_dropout (\u001b[38;5;33mDropout\u001b[0m)                  │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m768\u001b[0m)                            │               \u001b[38;5;34m0\u001b[0m │\n",
       "├───────────────────────────────────────────────┼────────────────────────────────────────┼─────────────────┤\n",
       "│ logits (\u001b[38;5;33mDense\u001b[0m)                                │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m2\u001b[0m)                              │           \u001b[38;5;34m1,538\u001b[0m │\n",
       "└───────────────────────────────────────────────┴────────────────────────────────────────┴─────────────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Total params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">109,483,778</span> (417.65 MB)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Total params: \u001b[0m\u001b[38;5;34m109,483,778\u001b[0m (417.65 MB)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">109,483,778</span> (417.65 MB)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Trainable params: \u001b[0m\u001b[38;5;34m109,483,778\u001b[0m (417.65 MB)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Non-trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (0.00 B)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Non-trainable params: \u001b[0m\u001b[38;5;34m0\u001b[0m (0.00 B)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Y-SZtW5sDw38"
   },
   "source": [
    "# Test Kodları"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 304
    },
    "collapsed": true,
    "id": "ZAVMyxyrDzXX",
    "outputId": "ecda82d9-a9bc-432c-8b61-2ae304c4d229"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 49/625 [=>............................] - ETA: 8:41 - loss: 0.1142 - sparse_categorical_accuracy: 0.9758"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-7-113a54cbdef7>\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/keras_nlp/src/utils/pipeline_model.py\u001b[0m in \u001b[0;36mevaluate\u001b[0;34m(self, x, y, batch_size, sample_weight, **kwargs)\u001b[0m\n\u001b[1;32m    218\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpreprocess_samples\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_parallel_calls\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mAUTOTUNE\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    219\u001b[0m         ).prefetch(tf.data.AUTOTUNE)\n\u001b[0;32m--> 220\u001b[0;31m         return super().evaluate(\n\u001b[0m\u001b[1;32m    221\u001b[0m             \u001b[0mx\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    222\u001b[0m             \u001b[0my\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/keras/src/utils/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     63\u001b[0m         \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     64\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 65\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     66\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m             \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/keras/src/engine/training.py\u001b[0m in \u001b[0;36mevaluate\u001b[0;34m(self, x, y, batch_size, verbose, sample_weight, steps, callbacks, max_queue_size, workers, use_multiprocessing, return_dict, **kwargs)\u001b[0m\n\u001b[1;32m   2294\u001b[0m                         ):\n\u001b[1;32m   2295\u001b[0m                             \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_test_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2296\u001b[0;31m                             logs = test_function_runner.run_step(\n\u001b[0m\u001b[1;32m   2297\u001b[0m                                 \u001b[0mdataset_or_iterator\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2298\u001b[0m                                 \u001b[0mdata_handler\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/keras/src/engine/training.py\u001b[0m in \u001b[0;36mrun_step\u001b[0;34m(self, dataset_or_iterator, data_handler, step, unused_shards)\u001b[0m\n\u001b[1;32m   4111\u001b[0m         \u001b[0mlogs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtmp_logs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   4112\u001b[0m         \u001b[0mend_step\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstep\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep_increment\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 4113\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_callbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_test_batch_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mend_step\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   4114\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   4115\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/keras/src/callbacks.py\u001b[0m in \u001b[0;36mon_test_batch_end\u001b[0;34m(self, batch, logs)\u001b[0m\n\u001b[1;32m    495\u001b[0m         \"\"\"\n\u001b[1;32m    496\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_should_call_test_batch_hooks\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 497\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_batch_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mModeKeys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTEST\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"end\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    498\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    499\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mon_predict_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/keras/src/callbacks.py\u001b[0m in \u001b[0;36m_call_batch_hook\u001b[0;34m(self, mode, hook, batch, logs)\u001b[0m\n\u001b[1;32m    320\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_batch_begin_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    321\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"end\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 322\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_batch_end_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    323\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    324\u001b[0m             raise ValueError(\n",
      "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/keras/src/callbacks.py\u001b[0m in \u001b[0;36m_call_batch_end_hook\u001b[0;34m(self, mode, batch, logs)\u001b[0m\n\u001b[1;32m    343\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_batch_times\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_time\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    344\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 345\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_batch_hook_helper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhook_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    346\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    347\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_batch_times\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_batches_for_timing_check\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/keras/src/callbacks.py\u001b[0m in \u001b[0;36m_call_batch_hook_helper\u001b[0;34m(self, hook_name, batch, logs)\u001b[0m\n\u001b[1;32m    391\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mcallback\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    392\u001b[0m             \u001b[0mhook\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcallback\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 393\u001b[0;31m             \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    394\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    395\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_check_timing\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/keras/src/callbacks.py\u001b[0m in \u001b[0;36mon_test_batch_end\u001b[0;34m(self, batch, logs)\u001b[0m\n\u001b[1;32m   1095\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mon_test_batch_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1096\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_called_in_fit\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1097\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_batch_update_progbar\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1098\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1099\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mon_predict_batch_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/keras/src/callbacks.py\u001b[0m in \u001b[0;36m_batch_update_progbar\u001b[0;34m(self, batch, logs)\u001b[0m\n\u001b[1;32m   1167\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mverbose\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1168\u001b[0m             \u001b[0;31m# Only block async when verbose = 1.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1169\u001b[0;31m             \u001b[0mlogs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_utils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msync_to_numpy_or_python_type\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1170\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprogbar\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mseen\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlogs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfinalize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1171\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/keras/src/utils/tf_utils.py\u001b[0m in \u001b[0;36msync_to_numpy_or_python_type\u001b[0;34m(tensors)\u001b[0m\n\u001b[1;32m    692\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndim\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    693\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 694\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnest\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmap_structure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_to_single_numpy_or_python_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtensors\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    695\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    696\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tensorflow/python/util/nest.py\u001b[0m in \u001b[0;36mmap_structure\u001b[0;34m(func, *structure, **kwargs)\u001b[0m\n\u001b[1;32m    629\u001b[0m     \u001b[0mValueError\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mIf\u001b[0m \u001b[0mwrong\u001b[0m \u001b[0mkeyword\u001b[0m \u001b[0marguments\u001b[0m \u001b[0mare\u001b[0m \u001b[0mprovided\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    630\u001b[0m   \"\"\"\n\u001b[0;32m--> 631\u001b[0;31m   return nest_util.map_structure(\n\u001b[0m\u001b[1;32m    632\u001b[0m       \u001b[0mnest_util\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mModality\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCORE\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mstructure\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    633\u001b[0m   )\n",
      "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tensorflow/python/util/nest_util.py\u001b[0m in \u001b[0;36mmap_structure\u001b[0;34m(modality, func, *structure, **kwargs)\u001b[0m\n\u001b[1;32m   1064\u001b[0m   \"\"\"\n\u001b[1;32m   1065\u001b[0m   \u001b[0;32mif\u001b[0m \u001b[0mmodality\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mModality\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCORE\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1066\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0m_tf_core_map_structure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mstructure\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1067\u001b[0m   \u001b[0;32melif\u001b[0m \u001b[0mmodality\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mModality\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDATA\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1068\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0m_tf_data_map_structure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mstructure\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tensorflow/python/util/nest_util.py\u001b[0m in \u001b[0;36m_tf_core_map_structure\u001b[0;34m(func, *structure, **kwargs)\u001b[0m\n\u001b[1;32m   1104\u001b[0m   return _tf_core_pack_sequence_as(\n\u001b[1;32m   1105\u001b[0m       \u001b[0mstructure\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1106\u001b[0;31m       \u001b[0;34m[\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mentries\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1107\u001b[0m       \u001b[0mexpand_composites\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mexpand_composites\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1108\u001b[0m   )\n",
      "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tensorflow/python/util/nest_util.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m   1104\u001b[0m   return _tf_core_pack_sequence_as(\n\u001b[1;32m   1105\u001b[0m       \u001b[0mstructure\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1106\u001b[0;31m       \u001b[0;34m[\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mentries\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1107\u001b[0m       \u001b[0mexpand_composites\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mexpand_composites\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1108\u001b[0m   )\n",
      "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/keras/src/utils/tf_utils.py\u001b[0m in \u001b[0;36m_to_single_numpy_or_python_type\u001b[0;34m(t)\u001b[0m\n\u001b[1;32m    685\u001b[0m         \u001b[0;31m# Don't turn ragged or sparse tensors to NumPy.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    686\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 687\u001b[0;31m             \u001b[0mt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    688\u001b[0m         \u001b[0;31m# Strings, ragged and sparse tensors don't have .item(). Return them\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    689\u001b[0m         \u001b[0;31m# as-is.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36mnumpy\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    392\u001b[0m     \"\"\"\n\u001b[1;32m    393\u001b[0m     \u001b[0;31m# TODO(slebedev): Consider avoiding a copy for non-CPU or remote tensors.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 394\u001b[0;31m     \u001b[0mmaybe_arr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_numpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    395\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mmaybe_arr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmaybe_arr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mmaybe_arr\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    396\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36m_numpy\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    358\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_numpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    359\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 360\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_numpy_internal\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    361\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    362\u001b[0m       \u001b[0;32mraise\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_status_to_exception\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;32mNone\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model.evaluate(x_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "dsx2XJmaD3OM"
   },
   "outputs": [],
   "source": [
    "y_preds = model.predict(x_test)\n",
    "y_preds = [np.argmax(pred) for pred in y_preds]\n",
    "\n",
    "print(classification_report(y_test, y_preds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 447
    },
    "id": "Jipp249yECO9",
    "outputId": "a9d0e029-2047-416d-f5d9-1c936a6f98e0"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: >"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAhAAAAGdCAYAAABDxkoSAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAskElEQVR4nO3de1xUdf7H8fcgMiAGhsgg3qIsL6vrtZRMy2SltItprZSaleZl0RTKC1thtRqFtSqZmt2wTctqszVNjfBWimK43m+5WXgJ1DUhUUdh5vdHP+fXHKnj6TfIZK/nPs7jsZ7zne98z27uvh+fz/ecsbndbrcAAAAsCKjqBQAAgN8eAgQAALCMAAEAACwjQAAAAMsIEAAAwDICBAAAsIwAAQAALCNAAAAAywgQAADAssCqXsA5Zw5tr+olAH6nZmxCVS8B8EtnnAcqdf6zR7/22VzVI6/02Vz+xG8CBAAAfsNVXtUr8Hu0MAAAgGVUIAAAMHK7qnoFfo8AAQCAkYsAYYYAAQCAgZsKhCn2QAAAAMuoQAAAYEQLwxQBAgAAI1oYpmhhAAAAy6hAAABgxIukTBEgAAAwooVhihYGAACwjAoEAABGPIVhigABAIABL5IyRwsDAABYRgUCAAAjWhimCBAAABjRwjBFgAAAwIj3QJhiDwQAALCMCgQAAEa0MEwRIAAAMGITpSlaGAAAwDIqEAAAGNHCMEWAAADAiBaGKVoYAADAMioQAAAYuN28B8IMAQIAACP2QJiihQEAACyjAgEAgBGbKE0RIAAAMKKFYYoAAQCAET+mZYo9EAAAwDIqEAAAGNHCMEWAAADAiE2UpmhhAAAAy6hAAABgRAvDFAECAAAjWhimaGEAAADLqEAAAGBEBcIUAQIAAAN+jdMcLQwAAGAZFQgAAIxoYZgiQAAAYMRjnKYIEAAAGFGBMMUeCAAAYBkVCAAAjGhhmCJAAABgRAvDFC0MAABgGRUIAACMaGGYIkAAAGBEC8MULQwAAGAZFQgAAIyoQJgiQAAAYMQeCFO0MAAAgGVUIAAAMKKFYYoAAQCAES0MUwQIAACMqECYYg8EAACwjAoEAABGtDBMESAAADCihWGKFgYAALCMCgQAAEZUIEwRIAAAMHK7q3oFfo8WBgAAsIwKBAAARrQwTFGBAADAyOXy3WFBeXm5nnzyScXGxiokJERXXXWV/va3v8n9k5aK2+1WWlqa6tatq5CQEMXHx+urr77ymufYsWPq16+fwsLCVKtWLQ0aNEgnTpzwGrNlyxZ17txZwcHBatCggTIyMiytlQABAICfeP755zVz5kxNnz5dO3fu1PPPP6+MjAy99NJLnjEZGRnKzMzUrFmztH79eoWGhiohIUGnT5/2jOnXr5+2b9+u7OxsLVq0SKtXr9aQIUM810tKStS9e3c1atRI+fn5mjx5sp566inNnj37gtdqc7v9Y6fImUPbq3oJgN+pGZtQ1UsA/NIZ54FKnf/U24/7bK6Q/pMueOxtt90mh8Oh119/3XOuT58+CgkJ0dtvvy23262YmBg9+uijeuyxxyRJxcXFcjgcysrKUmJionbu3KnmzZtrw4YNat++vSRp6dKl6tGjhw4cOKCYmBjNnDlTjz/+uAoLCxUUFCRJGj9+vD766CPt2rXrgtZKBQIAAKMqamFcf/31ysnJ0Z49eyRJmzdv1hdffKFbb71VkrRv3z4VFhYqPj7e85nw8HB16NBBubm5kqTc3FzVqlXLEx4kKT4+XgEBAVq/fr1nTJcuXTzhQZISEhK0e/duff/99xe0VjZRAgBg5MPivNPplNPp9Dpnt9tlt9vPGzt+/HiVlJSoadOmqlatmsrLyzVp0iT169dPklRYWChJcjgcXp9zOByea4WFhYqKivK6HhgYqIiICK8xsbGx581x7trll19uel9UIAAAqETp6ekKDw/3OtLT0ysc+95772nu3LmaN2+eNm7cqDlz5uiFF17QnDlzLvKqzVGBAADAyIePcaampiolJcXrXEXVB0kaM2aMxo8fr8TERElSy5Yt9e233yo9PV0DBw5UdHS0JKmoqEh169b1fK6oqEitW7eWJEVHR+vw4cNe85aVlenYsWOez0dHR6uoqMhrzLk/nxtjhgoEAABGPtwDYbfbFRYW5nX8XIA4efKkAgK8/6+5WrVqcv1voImNjVV0dLRycnI810tKSrR+/XrFxcVJkuLi4nT8+HHl5+d7xixfvlwul0sdOnTwjFm9erXOnj3rGZOdna0mTZpcUPtCIkAAAOA3br/9dk2aNEmLFy/WN998owULFujvf/+77rrrLkmSzWbT6NGjNXHiRC1cuFBbt27V/fffr5iYGPXq1UuS1KxZM91yyy16+OGHlZeXpzVr1mjEiBFKTExUTEyMJOm+++5TUFCQBg0apO3bt2v+/PmaNm3aeZWSX0ILAwAAI3fVvInypZde0pNPPqm//OUvOnz4sGJiYjR06FClpaV5xowdO1alpaUaMmSIjh8/rhtuuEFLly5VcHCwZ8zcuXM1YsQIdevWTQEBAerTp48yMzM918PDw/Xpp58qKSlJ7dq1U2RkpNLS0rzeFWGG90AAfoz3QAAVq+z3QJycneyzuWoMmeKzufwJLQwAAGAZLQwAAIz4MS1TBAgAAIyqaA/EbwktDAAAYBkVCAAAjFx+8XyBXyNAAABgxB4IUwQIAACMCBCm2AMBAAAsowIBAICRf7xj0a8RIC5BpSdPafob85TzxXod+75ETa+O1fgRD6lF06s9Y77+9oCmzH5LX27eofLycl3ZqL6mPD1WdR11dLDwsG65d1iFc78w4TEl3HS9JGnbrq80dfbb2rHnP5LNppZNr1bK0AFq0ji2ws8C/mTsmCT16nWrmjRprFOnTmvdui/118ef1Z49X0uSGjWqr6/2rKvws/feO1T//HCx588DBtyj0aOG6OqrY1VSckL//HCRRo164qLcByoJLQxTBIhL0ITJL2vvvv16NnWUoiIjtCh7lR5+7Gl99OY0OerU1v6Dhbr/kb+q963x+ssDiapZo4b2flOgoKDqkqToOrW14p+ve835/sfZypr/kTp3aCNJOnnqlIaN+5tuuv5aPT56iMrLyzUj610NHfs3Zb83W9UD+UcL/q1zlzjNnDVH+V9uVmBgNT3zt/FavGieWrXuqpMnT2n//kNq0LCN12cGD+qnlJRhWrpshefcqFEPa/SooUpNnai8Df9WaI0aanRF/Yt9O8BFx//KX2JOO536bPU6ZU4cr/at/iBJ+ssDiVq59kvNX7hMjwy6T5mvz1XnDu2UMux+z+ca1Pu/33+vVq2aIiO8f851+RfrlXBTJ9UICZEk7Ss4qOKSExrx4L2KjoqUJA0b2Fd9BiXru6IjalivrgB/dvvt/b3+PHhwsg4d3KK2bf+oL75YL5fLpaKiI15j7rzzFn3wwSKVlp6UJNWqFa6nnxqru3o/oBUr1njGbd22s/JvAJWLxzhNsYnyElNe7lK5y6WgoCCv88H2IP176065XC6tXpevRvXrauiYZ3TjXQ/ovuHjlPPF+p+dc/vu/2jX3n3q3aOb59wVDeqpVthl+vCTz3T27Fmddjq14JPPdGWj+oqJjqq0+wMqS3h4mCTp+2PHK7zepk1LtW7dQm9mveM5F9+tswICbKoXE60tm1fo6/9s0Ly5M1W/PgH6N8/t8t1xibIcII4ePaqMjAzdddddiouLU1xcnO666y5NnjxZR44cMZ8AlSq0Roha/aGJXvnH+zp89JjKy8v1cfYqbd6xR0ePfa9jx4t18tRpvfHOAnW6ro1emTxBN3fuoOS0DG3YVPEvop4LBq1bNPX6njemPqNF2avV/pZ71aFHP32Rt0kzn3tCgdWqXazbBXzCZrPphRee0po1edq+Y3eFYx58MFE7d+7RunX5nnOxsY0UEBCgceNG6tHHnlLivUN1eUQtLfnkHVWvXv1iLR+oEpYCxIYNG3TNNdcoMzNT4eHh6tKli7p06aLw8HBlZmaqadOm+vLLL03ncTqdKikp8TqczjO/+ibgLT11lNxut7rdM1jtuvfVvA8X69abb5DNZpPrf8tyN11/ne6/53Y1bRyrwff11o1x7fT+x8vOm+u006lPcj73qj6cOz8hY4batGiquS+n662XJunq2AZKSp2k007nRblPwFcyMyfpD82bqP+ApAqvBwcHK7FvL72Z9a7X+YAAm4KCgpSckqbs7FXKy9uoAQOS1LhxrG76383G+I1yuX13XKIs7YEYOXKk7rnnHs2aNUs2m83rmtvt1rBhwzRy5Ejl5ub+4jzp6el6+umnvc49kTJcTz5a8V9eWNOgXrSypk3UyVOnVXrypOrUjtBjT7+g+nUdujz8MgVWq6arDJu8YhvW17+3nt+3zV6Vq1POM7q9+01e5z/57HMdLDqst19OV0DAjzn0+SeS1emO+7VizQbdevMNlXZ/gC9NnTpRPW6NV7f4Pjp48LsKx/Tp3VM1aoTo7bc/8Dr/XeFhSdLOnV95zh09ekxHjx5Tgwb1Km/RqHRunsIwZakCsXnzZiUnJ58XHqQfS4DJycnatGmT6TypqakqLi72OsaOeNjKUnABaoQEq07tCBX/cEJrN2xS107XqXr16vpD08b6Zv8hr7HfHjikuo7z9y58+EmOul7fXhG1wr3On3I6FWCzef2zYAsIkGSTi794+I2YOnWi7rzjFiXc0lfffLP/Z8c98ECiFi3K1tGjx7zO5+ZukCRdc82VnnOXX15LkZERKig4UDmLBvyEpQARHR2tvLy8n72el5cnh8NhOo/dbldYWJjXYbcHmX4OF2ZN3r/1Rd5GHfiuSGu/3KRByWmKbVhPvW69WZL0YN87tXTFGn2wKFsFB7/TvAWfaNXaL5XY6xaveQoOfqf8LTvUu2f8ed8R176VSn4o1aSps/X1twe0d1+Bnnx+ugKrBei6Ni0uyn0C/x+ZmZN037136f6BI/TDDyfkcNSRw1FHwcHBXuOuuuoKde7cQW+8+c55c3z11T4tXLhUf3/xaXXs2E5/aN5Eb7w+Rbt379XKlWsv1q2gMtDCMGWphfHYY49pyJAhys/PV7du3TxhoaioSDk5OXr11Vf1wgsvVMpCceF+KD2paa+9raIj/1X4ZTUV3yVOjwy6z/Nuhm6dOyoteahem/ehnnvpdV3RIEZ/f3qs2rZs5jXPgk9y5KhTW9e3b33ed1zZsL5eejZVs+a8p/5J42ULCFCzxrGamfGk6tSOuBi3Cfy/DBs6UJKU85l3W2LQ4GT94x/ve/78wMC+OnDgO2Vnr6pwngcfGq0XJj+lf300Ry6XW59/vk633d5fZWVllbd4VL5L+OkJX7G53dbe1zl//nxNmTJF+fn5Ki8vl/TjewPatWunlJQU/fnPf/5VCzlzqOInAIDfs5qxCVW9BMAvnXFWbouo9Jl+PpsrNG2uz+byJ5ZfJNW3b1/17dtXZ8+e1dGjRyVJkZGRPLIEAMDvyK9+E2X16tVVty4vSwEAXILYDG6KV1kDAGB0CW9+9BVeZQ0AACyjAgEAgBFPYZgiQAAAYEQLwxQtDAAAYBkVCAAADPgtDHMECAAAjGhhmKKFAQAALKMCAQCAERUIUwQIAACMeIzTFAECAAAjKhCm2AMBAAAsowIBAICBmwqEKQIEAABGBAhTtDAAAIBlVCAAADDiTZSmCBAAABjRwjBFCwMAAFhGBQIAACMqEKYIEAAAGLjdBAgztDAAAIBlVCAAADCihWGKAAEAgBEBwhQBAgAAA15lbY49EAAAwDIqEAAAGFGBMEWAAADAiDdZm6KFAQAALKMCAQCAAZsozREgAAAwIkCYooUBAAAsowIBAIARmyhNESAAADBgD4Q5WhgAAMAyKhAAABjRwjBFgAAAwIAWhjkCBAAARlQgTLEHAgAAWEYFAgAAAzcVCFMECAAAjAgQpmhhAAAAy6hAAABgQAvDHAECAAAjAoQpWhgAAMAyAgQAAAZul+8Oqw4ePKj+/furdu3aCgkJUcuWLfXll1/+39rcbqWlpalu3boKCQlRfHy8vvrqK685jh07pn79+iksLEy1atXSoEGDdOLECa8xW7ZsUefOnRUcHKwGDRooIyPD0joJEAAAGFRVgPj+++/VqVMnVa9eXUuWLNGOHTv04osv6vLLL/eMycjIUGZmpmbNmqX169crNDRUCQkJOn36tGdMv379tH37dmVnZ2vRokVavXq1hgwZ4rleUlKi7t27q1GjRsrPz9fkyZP11FNPafbs2Re8Vpvb7faL93WeObS9qpcA+J2asQlVvQTAL51xHqjU+Yu63uizuRwrVl3w2PHjx2vNmjX6/PPPK7zudrsVExOjRx99VI899pgkqbi4WA6HQ1lZWUpMTNTOnTvVvHlzbdiwQe3bt5ckLV26VD169NCBAwcUExOjmTNn6vHHH1dhYaGCgoI83/3RRx9p165dF7RWKhAAAFQip9OpkpISr8PpdFY4duHChWrfvr3uueceRUVFqU2bNnr11Vc91/ft26fCwkLFx8d7zoWHh6tDhw7Kzc2VJOXm5qpWrVqe8CBJ8fHxCggI0Pr16z1junTp4gkPkpSQkKDdu3fr+++/v6D7IkAAAGDktvnsSE9PV3h4uNeRnp5e4dd+/fXXmjlzpq6++motW7ZMw4cP1yOPPKI5c+ZIkgoLCyVJDofD63MOh8NzrbCwUFFRUV7XAwMDFRER4TWmojl++h1meIwTAAADX74HIjU1VSkpKV7n7HZ7hWNdLpfat2+vZ599VpLUpk0bbdu2TbNmzdLAgQN9tygfoAIBAEAlstvtCgsL8zp+LkDUrVtXzZs39zrXrFkzFRQUSJKio6MlSUVFRV5jioqKPNeio6N1+PBhr+tlZWU6duyY15iK5vjpd5ghQAAAYOB22Xx2WNGpUyft3r3b69yePXvUqFEjSVJsbKyio6OVk5PjuV5SUqL169crLi5OkhQXF6fjx48rPz/fM2b58uVyuVzq0KGDZ8zq1at19uxZz5js7Gw1adLE64mPX0KAAADAoKoe40xOTta6dev07LPPau/evZo3b55mz56tpKQkSZLNZtPo0aM1ceJELVy4UFu3btX999+vmJgY9erVS9KPFYtbbrlFDz/8sPLy8rRmzRqNGDFCiYmJiomJkSTdd999CgoK0qBBg7R9+3bNnz9f06ZNO6/V8kvYAwEAgJ+49tprtWDBAqWmpuqZZ55RbGyspk6dqn79+nnGjB07VqWlpRoyZIiOHz+uG264QUuXLlVwcLBnzNy5czVixAh169ZNAQEB6tOnjzIzMz3Xw8PD9emnnyopKUnt2rVTZGSk0tLSvN4VYYb3QAB+jPdAABWr7PdAHIy72Wdz1ctd7rO5/AkVCAAADPg1TnPsgQAAAJZRgQAAwMDq0xO/RwQIAAAM/GN3oH8jQAAAYEAFwhx7IAAAgGVUIAAAMKACYY4AAQCAAXsgzNHCAAAAllGBAADAgBaGOQIEAAAGbjcBwgwtDAAAYBkVCAAADPgtDHMECAAADFy0MEzRwgAAAJZRgQAAwIBNlOYIEAAAGPAYpzkCBAAABryJ0hx7IAAAgGVUIAAAMKCFYY4AAQCAAY9xmqOFAQAALKMCAQCAAY9xmiNAAABgwFMY5mhhAAAAy6hAAABgwCZKcwQIAAAM2ANhjhYGAACwjAoEAAAGbKI0R4AAAMCAPRDm/CZA1Liie1UvAfA7pw59XtVLAH6X2ANhjj0QAADAMr+pQAAA4C9oYZgjQAAAYMAeSnO0MAAAgGVUIAAAMKCFYY4AAQCAAU9hmKOFAQAALKMCAQCAgauqF/AbQIAAAMDALVoYZmhhAAAAy6hAAABg4OJFEKYIEAAAGLhoYZgiQAAAYMAeCHPsgQAAAJZRgQAAwIDHOM0RIAAAMKCFYY4WBgAAsIwKBAAABrQwzBEgAAAwIECYo4UBAAAsowIBAIABmyjNESAAADBwkR9M0cIAAACWUYEAAMCA38IwR4AAAMCAH+M0R4AAAMCAxzjNsQcCAABYRgUCAAADl409EGYIEAAAGLAHwhwtDAAAYBkVCAAADNhEaY4AAQCAAW+iNEcLAwAAWEYFAgAAA95EaY4KBAAABm4fHr/Wc889J5vNptGjR3vOnT59WklJSapdu7Zq1qypPn36qKioyOtzBQUF6tmzp2rUqKGoqCiNGTNGZWVlXmNWrlyptm3bym63q3HjxsrKyrK8PgIEAAB+ZsOGDXrllVf0xz/+0et8cnKyPv74Y73//vtatWqVDh06pN69e3uul5eXq2fPnjpz5ozWrl2rOXPmKCsrS2lpaZ4x+/btU8+ePdW1a1dt2rRJo0eP1uDBg7Vs2TJLa7S53W6/eNw1MKheVS8B8DunDn1e1UsA/FL1yCsrdf636vX32Vz3H3zb0vgTJ06obdu2mjFjhiZOnKjWrVtr6tSpKi4uVp06dTRv3jzdfffdkqRdu3apWbNmys3NVceOHbVkyRLddtttOnTokBwOhyRp1qxZGjdunI4cOaKgoCCNGzdOixcv1rZt2zzfmZiYqOPHj2vp0qUXvE4qEAAAGLh8eFiVlJSknj17Kj4+3ut8fn6+zp4963W+adOmatiwoXJzcyVJubm5atmypSc8SFJCQoJKSkq0fft2zxjj3AkJCZ45LhSbKAEAMPBlad7pdMrpdHqds9vtstvt54199913tXHjRm3YsOG8a4WFhQoKClKtWrW8zjscDhUWFnrG/DQ8nLt+7tovjSkpKdGpU6cUEhJyQfdFBQIAgEqUnp6u8PBwryM9Pf28cfv379eoUaM0d+5cBQcHV8FKrSFAAABg4LL57khNTVVxcbHXkZqaet535ufn6/Dhw2rbtq0CAwMVGBioVatWKTMzU4GBgXI4HDpz5oyOHz/u9bmioiJFR0dLkqKjo897KuPcn83GhIWFXXD1QSJAAABwHl/ugbDb7QoLC/M6KmpfdOvWTVu3btWmTZs8R/v27dWvXz/Pv69evbpycnI8n9m9e7cKCgoUFxcnSYqLi9PWrVt1+PBhz5js7GyFhYWpefPmnjE/nePcmHNzXCj2QAAA4Acuu+wytWjRwutcaGioateu7Tk/aNAgpaSkKCIiQmFhYRo5cqTi4uLUsWNHSVL37t3VvHlzDRgwQBkZGSosLNQTTzyhpKQkT2gZNmyYpk+frrFjx+qhhx7S8uXL9d5772nx4sWW1kuAAADAwF9/TGvKlCkKCAhQnz595HQ6lZCQoBkzZniuV6tWTYsWLdLw4cMVFxen0NBQDRw4UM8884xnTGxsrBYvXqzk5GRNmzZN9evX12uvvaaEhARLa+E9EIAf4z0QQMUq+z0Qsxr47j0Qw/Zbew/EbwV7IAAAgGW0MAAAMPDXFoY/IUAAAGBAgDBHCwMAAFhGBQIAAAO/eLrAzxEgAAAwcNmqegX+jwABAIABeyDMsQcCAABYRgUCAAADKhDmCBAAABiwidIcLQwAAGAZFQgAAAx4CsMcAQIAAAP2QJijhQEAACyjAgEAgAGbKM0RIAAAMHARIUzRwgAAAJZRgQAAwIBNlOYIEAAAGNDAMEeAAADAgAqEOfZAAAAAy6hAAABgwJsozREgAAAw4DFOc7QwAACAZVQgAAAwoP5gjgABAIABT2GYo4UBAAAsowIBAIABmyjNESAAADAgPpijhQEAACyjAgEAgAGbKM0RIAAAMGAPhDkCBAAABsQHc+yBAAAAllGBAADAgD0Q5ggQAAAYuGlimKKFAQAALKMCAQCAAS0McwQIAAAMeIzTHC0MAABgGRUIAAAMqD+YowLxO9D5hg76aEGWCr7JV9mZg7rjjgSv61FRkXr9tSkq+CZfJcf3avHHb6tx41ivMXa7XZnTJqnou206fmyP3ps/W1FRkRfzNoD/t9LSk3pu6iz9qfdAtet6p/oNTdHWnbs910+ePKVJL85Qt1791a7rnbqj3xDNX7DYa44HRoxVi063eh1PZ7x03nd9tDhbd90/XG273qEuPRM18cWXK/3+4DsuuX12XKqoQPwOhIbW0JYtO/Rm1rv65/uvn3f9ww/e0NmzZ9W7z0Mq+eGERo8aomVL3lXLVjfp5MlTkqQXX3hKPW7tpsR7h6q4uESZ0ybpg/deU5ebel3kuwF+vbTnpmnv198oPe0xRUXW1sfLluvhUX/Vv+a+IkedSGW8NFvr8zcrPW2s6tV1aG1evia++LKiImura+eOnnnuvuMWjRg8wPPn4GC71/fMefdDzXnnQz2aNEgtmzfRqdNOHfqu6KLdJ3AxECB+B5YuW6Gly1ZUeO3qq69Ux47t9MfWXbVjxx5JUtKI8Tq4f5MS+/bSG2++o7Cwy/TQg4nqf/8IrVi5RpI06OFkbd+6Wh2ua6v1eRsv2r0Av9Zpp1OfrfpCmc9NUPvWLSVJSYP6a9Wa9Zq/YLEeGTJQm7bu1J23xuu6tn+UJN1zZw+9/68l2rpzt1eACLbbFVk7osLvKS75QS/NfkvTMyaoY/s2nvNNDFU9+DeewjBHC+N3zm4PkiSdPu30nHO73XI6z6hTp+skSe3a/lFBQUHKyfncM2b37v/o228PqGPHdhd3wcCvVF5WrvJyl+xB1b3O2+1B2rhluySpdctmWvHFOhUdOSq32628/M36puCgrr+urddnFmev0A09+qpX/2GaMvNNnTp92nMtd8O/5XK7VHTkv7r9viHq1qu/Hn3yWX1XdKTybxI+4/bhvy5VVCB+53bt2qtvvz2gSRNTNfwv41RaelKjRz2sBg1iVDc6SpLkiK4jp9Op4uISr88ePnxE0dF1qmLZgGWhoTXUqkUzzcp6R1c2aqjaEbX0yWertHnbLjWsV1eS9Nfk4Xrq+Ux16zVAgdWqyRZg01PjRnkqFpLU8083KSbaoTqREdqzd5+mzHxD3xQc0LT0JyVJBw4VyuVy67W35mv86GGqGVpDL736loaM/qs+fGuGqlevXuH64F+oQJjzeYDYv3+/JkyYoDfeeONnxzidTjmdTq9zbrdbNpvN18uBibKyMt3z58GaPftFHT28Q2VlZcrJ+VxLluTw3wcuOelPPqa09Cm6uVd/VasWoGbXNNat8Tdqx+69kqS5HyzUlu27NP35Caob7VD+pq2a9OIMRUXWVty1P7Yj7rmzh2e+a66KVZ3ICA16JFUFBw6pYf0YuVwulZWVafzoYerU4ccKXcZT43TTHf2Ut3GL5xzwW+fzFsaxY8c0Z86cXxyTnp6u8PBwr8Pt+sHXS8EF2vjvrWp/bXdFRDZV/YZt1PP2/qpd+3J9va9AklRUeER2u13h4WFen4uKqqPCQsqy+O1oWD9GWS9PVt5nC/TZh//Qu69NU1lZuerHROu006lpr8zRmEeG6KYbOqpJ41jdd/cduqVbF2W988+fnbNl86aSpP0Hv5Mk1Yn8cW/EVbENPWMiLq+lWuFh+q7ocCXeHXyJFoY5yxWIhQsX/uL1r7/+2nSO1NRUpaSkeJ27vHZTq0uBj5WU/BjiGjeOVbt2rTThqcmSpPyNW3TmzBndfPMNWrDgE0nSNddcpUaN6mvduvwqWy/wa9UICVaNkGAVl/ygtXn5SvnLQyorK1NZWZkCDJW3atUC5HL9fEF711f/kSTPpso2LZtLkr4pOKDoqB9bfMUlP+h4cYnqOqIq43ZQCWhhmLMcIHr16iWbzSa3++dTlVnp2263y273fuyJcnnlCQ2t4fVeh9grGqpVqz/o2LHvtX//IfXpc5uOHvmvCvYfVIsWTTXlxWf0r4VLlf3Zakk/Bos33nxXL2RM0PfHjquk5AdNmzpRublf8gQGflPWrM+X2+3WFQ3rq+DAIb348uuKbVhfvXp2V/XAQLVv01Ivvvy67Ha7YqKj9OW/t2rhkhyNeeRhSVLBgUP6JHulOsddq1rhYdqzd5+ez3xF7Vu38DxlcUXD+rq5c5yem/qKJox7RDVDa2jqrDcV27C+rmvXqipvH/Apm/uXkkAF6tWrpxkzZujOO++s8PqmTZvUrl07lZeXW1pIYFA9S+Nx4W7sEqeczz447/yct97ToMHJGpH0kB5NGS6HI1LffXdYb8/9QBMnTdXZs2c9Y+12uyZnpCmx752y2+36NHulRoz8q4rYWV6pTh363HwQLtjSnNWaOutNFR05qvCwy/SnG2/QI0MH6rKaoZKko/89pqmzsrQ2b6OKS35QTHSU7r7zVt3f9y7ZbDZ9V3REqc9k6Kuvv9Wp06cVHVVH3bpcr6EPJKpmaKjne06Ulur5zNnKWbVWNptN7Vu31PjRw1TXwaZjX6keeWWlzj+gUW+fzfWPbz/02Vz+xHKAuOOOO9S6dWs988wzFV7fvHmz2rRp84slv4oQIIDzESCAilV2gOjvwwDx9iUaICy3MMaMGaPS0tKfvd64cWOtWFHxS4sAAMClwXKA6Ny58y9eDw0N1Y033virFwQAQFW7lH/Dwld4kRQAAAaX8uOXvsKrrAEAgGVUIAAAMOA9EOYIEAAAGLAHwhwBAgAAA/ZAmGMPBAAAsIwKBAAABuyBMEeAAADAwOJLmn+XaGEAAADLqEAAAGDAUxjmCBAAABiwB8IcLQwAAGAZAQIAAAO3D/9lRXp6uq699lpddtllioqKUq9evbR7926vMadPn1ZSUpJq166tmjVrqk+fPioqKvIaU1BQoJ49e6pGjRqKiorSmDFjVFZW5jVm5cqVatu2rex2uxo3bqysrCxLayVAAABg4JLbZ4cVq1atUlJSktatW6fs7GydPXtW3bt3V2lpqWdMcnKyPv74Y73//vtatWqVDh06pN69e3uul5eXq2fPnjpz5ozWrl2rOXPmKCsrS2lpaZ4x+/btU8+ePdW1a1dt2rRJo0eP1uDBg7Vs2bILXqvN7SfPqgQG1avqJQB+59Shz6t6CYBfqh55ZaXO36NhD5/N9UnBJ7/6s0eOHFFUVJRWrVqlLl26qLi4WHXq1NG8efN09913S5J27dqlZs2aKTc3Vx07dtSSJUt022236dChQ3I4HJKkWbNmady4cTpy5IiCgoI0btw4LV68WNu2bfN8V2Jioo4fP66lS5de0NqoQAAAYOB2u312OJ1OlZSUeB1Op/OC1lFcXCxJioiIkCTl5+fr7Nmzio+P94xp2rSpGjZsqNzcXElSbm6uWrZs6QkPkpSQkKCSkhJt377dM+anc5wbc26OC0GAAADAwOXDIz09XeHh4V5Henq6+RpcLo0ePVqdOnVSixYtJEmFhYUKCgpSrVq1vMY6HA4VFhZ6xvw0PJy7fu7aL40pKSnRqVOnzP8DEo9xAgBwHl/+mFZqaqpSUlK8ztntdtPPJSUladu2bfriiy98thZfIkAAAFCJ7Hb7BQWGnxoxYoQWLVqk1atXq379+p7z0dHROnPmjI4fP+5VhSgqKlJ0dLRnTF5entd8557S+OkY45MbRUVFCgsLU0hIyAWtkRYGAAAGVfUUhtvt1ogRI7RgwQItX75csbGxXtfbtWun6tWrKycnx3Nu9+7dKigoUFxcnCQpLi5OW7du1eHDhz1jsrOzFRYWpubNm3vG/HSOc2POzXEhqEAAAGBQVQ8oJiUlad68efrXv/6lyy67zLNnITw8XCEhIQoPD9egQYOUkpKiiIgIhYWFaeTIkYqLi1PHjh0lSd27d1fz5s01YMAAZWRkqLCwUE888YSSkpI8lZBhw4Zp+vTpGjt2rB566CEtX75c7733nhYvXnzBa+UxTsCP8RgnULHKfoyzW/3uPpsr58CnFzzWZrNVeP7NN9/UAw88IOnHF0k9+uijeuedd+R0OpWQkKAZM2Z42hOS9O2332r48OFauXKlQkNDNXDgQD333HMKDPy/usHKlSuVnJysHTt2qH79+nryySc933FBayVAAP6LAAFUrLIDRNf6f/LZXCsOZPtsLn9CCwMAAANfPoVxqWITJQAAsIwKBAAABi7/6O77NQIEAAAGxAdztDAAAIBlVCAAADCw+gKo3yMCBAAABgQIcwQIAAAM/OQVSX6NPRAAAMAyKhAAABjQwjBHgAAAwIA3UZqjhQEAACyjAgEAgAGbKM0RIAAAMGAPhDlaGAAAwDIqEAAAGNDCMEeAAADAgBaGOVoYAADAMioQAAAY8B4IcwQIAAAMXOyBMEWAAADAgAqEOfZAAAAAy6hAAABgQAvDHAECAAADWhjmaGEAAADLqEAAAGBAC8McAQIAAANaGOZoYQAAAMuoQAAAYEALwxwBAgAAA1oY5mhhAAAAy6hAAABg4Ha7qnoJfo8AAQCAgYsWhikCBAAABm42UZpiDwQAALCMCgQAAAa0MMwRIAAAMKCFYY4WBgAAsIwKBAAABryJ0hwBAgAAA95EaY4WBgAAsIwKBAAABmyiNEeAAADAgMc4zdHCAAAAllGBAADAgBaGOQIEAAAGPMZpjgABAIABFQhz7IEAAACWUYEAAMCApzDMESAAADCghWGOFgYAALCMCgQAAAY8hWGOAAEAgAE/pmWOFgYAALCMCgQAAAa0MMwRIAAAMOApDHO0MAAAgGVUIAAAMGATpTkCBAAABrQwzBEgAAAwIECYYw8EAACwjAoEAAAG1B/M2dzUafATTqdT6enpSk1Nld1ur+rlAH6BvxfA+QgQ8FJSUqLw8HAVFxcrLCysqpcD+AX+XgDnYw8EAACwjAABAAAsI0AAAADLCBDwYrfbNWHCBDaKAT/B3wvgfGyiBAAAllGBAAAAlhEgAACAZQQIAABgGQECAABYRoCAx8svv6wrrrhCwcHB6tChg/Ly8qp6SUCVWr16tW6//XbFxMTIZrPpo48+quolAX6DAAFJ0vz585WSkqIJEyZo48aNatWqlRISEnT48OGqXhpQZUpLS9WqVSu9/PLLVb0UwO/wGCckSR06dNC1116r6dOnS5JcLpcaNGigkSNHavz48VW8OqDq2Ww2LViwQL169arqpQB+gQoEdObMGeXn5ys+Pt5zLiAgQPHx8crNza3ClQEA/BUBAjp69KjKy8vlcDi8zjscDhUWFlbRqgAA/owAAQAALCNAQJGRkapWrZqKioq8zhcVFSk6OrqKVgUA8GcECCgoKEjt2rVTTk6O55zL5VJOTo7i4uKqcGUAAH8VWNULgH9ISUnRwIED1b59e1133XWaOnWqSktL9eCDD1b10oAqc+LECe3du9fz53379mnTpk2KiIhQw4YNq3BlQNXjMU54TJ8+XZMnT1ZhYaFat26tzMxMdejQoaqXBVSZlStXqmvXruedHzhwoLKysi7+ggA/QoAAAACWsQcCAABYRoAAAACWESAAAIBlBAgAAGAZAQIAAFhGgAAAAJYRIAAAgGUECAAAYBkBAgAAWEaAAAAAlhEgAACAZQQIAABg2f8APFl7N+6zOUEAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "cm = confusion_matrix(y_test, y_preds)\n",
    "sns.heatmap(cm, annot=True, fmt=\"d\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "WcKecLIEEIFy"
   },
   "source": [
    "## Elle Deneme İçin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "rnfhgag2EKb7"
   },
   "outputs": [],
   "source": [
    "def preprocessInput(text):\n",
    "  regex = \"^(?:\\w*\\:\\/\\/)?(.*)\"\n",
    "  total_text = re.findall(regex, text)\n",
    "  lastStr = ''.join(total_text)\n",
    "  lastStr = lastStr.replace('www.','')\n",
    "  lastStr = lastStr.replace('.',' ')\n",
    "  lastStr = lastStr.replace('/',' ')\n",
    "  print(\"Preprocess Sonucu: \"+lastStr)\n",
    "  return lastStr\n",
    "\n",
    "def predictLabel(text):\n",
    "  url = preprocessInput(text)\n",
    "  pred = model.predict([url])\n",
    "  pred = np.argmax(pred,axis=1)\n",
    "\n",
    "  if pred == 1:\n",
    "    print(\"Phishing URL\")\n",
    "  else:\n",
    "    print(\"Normal URL\")\n",
    "  return pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "FH_6v2ueEPQQ"
   },
   "outputs": [],
   "source": [
    "urlInput = str(input(\"Lütfen URL verin: \"))\n",
    "binary_pred = predictLabel(urlInput)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "gpuType": "T4",
   "machine_shape": "hm",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  },
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}