pyspark_zero_to_hero.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "564e3e8b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f36093ea",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-01-31T21:41:58.771865Z",
     "start_time": "2023-01-31T21:41:58.765863Z"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "#os.environ['SPARK_HOME'] = 'C:\\spark\\spark-3.3.0-bin-hadoop3'\n",
    "os.environ[\"JAVA_HOME\"] = 'C:\\Program Files\\Java\\jdk-19'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "66e90f9f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-01-31T20:15:33.840330Z",
     "start_time": "2023-01-31T20:13:18.507334Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting pyspark\n",
      "  Downloading pyspark-3.3.1.tar.gz (281.4 MB)\n",
      "Collecting py4j==0.10.9.5\n",
      "  Downloading py4j-0.10.9.5-py2.py3-none-any.whl (199 kB)\n",
      "Building wheels for collected packages: pyspark\n",
      "  Building wheel for pyspark (setup.py): started\n",
      "  Building wheel for pyspark (setup.py): finished with status 'done'\n",
      "  Created wheel for pyspark: filename=pyspark-3.3.1-py2.py3-none-any.whl size=281845513 sha256=6c0d2b530d16089c9dfeee32b5f0cc5a2df87582df300e21ee413b408fac1bdb\n",
      "  Stored in directory: c:\\users\\furqan ali\\appdata\\local\\pip\\cache\\wheels\\43\\dc\\11\\ec201cd671da62fa9c5cc77078235e40722170ceba231d7598\n",
      "Successfully built pyspark\n",
      "Installing collected packages: py4j, pyspark\n",
      "Successfully installed py4j-0.10.9.5 pyspark-3.3.1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: You are using pip version 20.2.1; however, version 23.0 is available.\n",
      "You should consider upgrading via the 'c:\\users\\furqan ali\\desktop\\python\\env\\scripts\\python.exe -m pip install --upgrade pip' command.\n"
     ]
    }
   ],
   "source": [
    "!pip install pyspark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7cf55482",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-01-31T21:42:02.225447Z",
     "start_time": "2023-01-31T21:42:02.065476Z"
    }
   },
   "outputs": [],
   "source": [
    "import pyspark\n",
    "from pyspark.sql import SparkSession "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9aa92705",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-01-31T21:42:06.594566Z",
     "start_time": "2023-01-31T21:42:02.637269Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "            <div>\n",
       "                <p><b>SparkSession - in-memory</b></p>\n",
       "                \n",
       "        <div>\n",
       "            <p><b>SparkContext</b></p>\n",
       "\n",
       "            <p><a href=\"http://DESKTOP-Q24DOVF:4040\">Spark UI</a></p>\n",
       "\n",
       "            <dl>\n",
       "              <dt>Version</dt>\n",
       "                <dd><code>v3.3.1</code></dd>\n",
       "              <dt>Master</dt>\n",
       "                <dd><code>local[*]</code></dd>\n",
       "              <dt>AppName</dt>\n",
       "                <dd><code>Practice</code></dd>\n",
       "            </dl>\n",
       "        </div>\n",
       "        \n",
       "            </div>\n",
       "        "
      ],
      "text/plain": [
       "<pyspark.sql.session.SparkSession at 0x232e315aa30>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark = SparkSession.builder.appName(\"Practice\").getOrCreate()\n",
    "spark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1f211de5",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-01-31T21:44:08.361771Z",
     "start_time": "2023-01-31T21:44:08.160465Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[_c0: string, _c1: string, _c2: string, _c3: string]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_pyspark = spark.read.csv('./cyberbullying_tweets_test.csv')\n",
    "df_pyspark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "00fe4738",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-01-31T21:44:24.086494Z",
     "start_time": "2023-01-31T21:44:23.837489Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+------------------+----------+-----------+\n",
      "|                 _c0|               _c1|       _c2|        _c3|\n",
      "+--------------------+------------------+----------+-----------+\n",
      "|          tweet_text|cyberbullying_type|gpt_labels|Keep/ignore|\n",
      "|In other words #k...| not_cyberbullying|         0|       keep|\n",
      "|Why is #aussietv ...| not_cyberbullying|         0|       keep|\n",
      "|@XochitlSuckkks a...| not_cyberbullying|         0|       keep|\n",
      "|@Jason_Gio meh. :...| not_cyberbullying|         0|       keep|\n",
      "|@RudhoeEnglish Th...| not_cyberbullying|         0|       keep|\n",
      "|@Raja5aab @Quicki...| not_cyberbullying|         0|       keep|\n",
      "|Itu sekolah ya bu...| not_cyberbullying|         0|       keep|\n",
      "|Karma. I hope it ...| not_cyberbullying|         0|       keep|\n",
      "|@stockputout ever...| not_cyberbullying|         0|       keep|\n",
      "|Rebecca Black Dro...| not_cyberbullying|         0|       keep|\n",
      "|@Jord_Is_Dead htt...| not_cyberbullying|         0|       keep|\n",
      "|The Bully flushes...| not_cyberbullying|         0|       keep|\n",
      "|         Ughhhh #MKR| not_cyberbullying|         0|       keep|\n",
      "|RT @Kurdsnews: Tu...| not_cyberbullying|         0|       null|\n",
      "|\"Love that the be...| not_cyberbullying|         0|       null|\n",
      "|@yasmimcaci @Bfer...| not_cyberbullying|         0|       null|\n",
      "|@sarinhacoral @Vi...| not_cyberbullying|         0|       null|\n",
      "|@0xabad1dea @kels...| not_cyberbullying|         0|       null|\n",
      "|Best pick up line...| not_cyberbullying|         0|       null|\n",
      "+--------------------+------------------+----------+-----------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_pyspark = spark.read.csv('./cyberbullying_tweets_test.csv').show()\n",
    "df_pyspark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "28f84cf2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-01-31T21:47:52.675809Z",
     "start_time": "2023-01-31T21:47:52.556476Z"
    }
   },
   "outputs": [],
   "source": [
    "df_pyspark = spark.read.option('header', 'true').csv('./cyberbullying_tweets_test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "87dfc858",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-01-31T21:47:56.978265Z",
     "start_time": "2023-01-31T21:47:56.965262Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pyspark.sql.dataframe.DataFrame"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(df_pyspark)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "0207bc16",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-01-31T21:48:34.073742Z",
     "start_time": "2023-01-31T21:48:33.999732Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Row(tweet_text='In other words #katandandre, your food was crapilicious! #mkr', cyberbullying_type='not_cyberbullying', gpt_labels='0', Keep/ignore='keep')"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_pyspark.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "be70415e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-01-31T21:50:14.252880Z",
     "start_time": "2023-01-31T21:50:14.180043Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+------------------+----------+-----------+\n",
      "|          tweet_text|cyberbullying_type|gpt_labels|Keep/ignore|\n",
      "+--------------------+------------------+----------+-----------+\n",
      "|In other words #k...| not_cyberbullying|         0|       keep|\n",
      "|Why is #aussietv ...| not_cyberbullying|         0|       keep|\n",
      "|@XochitlSuckkks a...| not_cyberbullying|         0|       keep|\n",
      "|@Jason_Gio meh. :...| not_cyberbullying|         0|       keep|\n",
      "|@RudhoeEnglish Th...| not_cyberbullying|         0|       keep|\n",
      "|@Raja5aab @Quicki...| not_cyberbullying|         0|       keep|\n",
      "|Itu sekolah ya bu...| not_cyberbullying|         0|       keep|\n",
      "|Karma. I hope it ...| not_cyberbullying|         0|       keep|\n",
      "|@stockputout ever...| not_cyberbullying|         0|       keep|\n",
      "|Rebecca Black Dro...| not_cyberbullying|         0|       keep|\n",
      "|@Jord_Is_Dead htt...| not_cyberbullying|         0|       keep|\n",
      "|The Bully flushes...| not_cyberbullying|         0|       keep|\n",
      "|         Ughhhh #MKR| not_cyberbullying|         0|       keep|\n",
      "|RT @Kurdsnews: Tu...| not_cyberbullying|         0|       null|\n",
      "|\"Love that the be...| not_cyberbullying|         0|       null|\n",
      "|@yasmimcaci @Bfer...| not_cyberbullying|         0|       null|\n",
      "|@sarinhacoral @Vi...| not_cyberbullying|         0|       null|\n",
      "|@0xabad1dea @kels...| not_cyberbullying|         0|       null|\n",
      "|Best pick up line...| not_cyberbullying|         0|       null|\n",
      "|Now I gotta walk ...| not_cyberbullying|         0|       keep|\n",
      "+--------------------+------------------+----------+-----------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_pyspark.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "8fcecf37",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-01-31T21:50:27.325568Z",
     "start_time": "2023-01-31T21:50:27.306567Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- tweet_text: string (nullable = true)\n",
      " |-- cyberbullying_type: string (nullable = true)\n",
      " |-- gpt_labels: string (nullable = true)\n",
      " |-- Keep/ignore: string (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_pyspark.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "06decd93",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-01-31T22:54:58.314549Z",
     "start_time": "2023-01-31T22:54:58.305021Z"
    }
   },
   "outputs": [],
   "source": [
    "# Now we will cover PySpark dataframe\n",
    "# Reading the dataset\n",
    "# Checking the datatypes of the column\n",
    "# Selecting column and indexing \n",
    "# Check describe option similar to pandas \n",
    "# Adding columns\n",
    "# Dropping Columns\n",
    "# Renaming the Columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7f9fa76d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql import SparkSession"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "292133be",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-01-31T22:00:46.594686Z",
     "start_time": "2023-01-31T22:00:46.580982Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "            <div>\n",
       "                <p><b>SparkSession - in-memory</b></p>\n",
       "                \n",
       "        <div>\n",
       "            <p><b>SparkContext</b></p>\n",
       "\n",
       "            <p><a href=\"http://DESKTOP-Q24DOVF:4040\">Spark UI</a></p>\n",
       "\n",
       "            <dl>\n",
       "              <dt>Version</dt>\n",
       "                <dd><code>v3.3.1</code></dd>\n",
       "              <dt>Master</dt>\n",
       "                <dd><code>local[*]</code></dd>\n",
       "              <dt>AppName</dt>\n",
       "                <dd><code>Practice</code></dd>\n",
       "            </dl>\n",
       "        </div>\n",
       "        \n",
       "            </div>\n",
       "        "
      ],
      "text/plain": [
       "<pyspark.sql.session.SparkSession at 0x232e315aa30>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark = SparkSession.builder.appName(\"Dataframe\").getOrCreate()\n",
    "spark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "10303374",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-01-31T22:01:14.394956Z",
     "start_time": "2023-01-31T22:01:14.389956Z"
    }
   },
   "outputs": [],
   "source": [
    "# Check the schema"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "88f5e353",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-01-31T22:02:28.615115Z",
     "start_time": "2023-01-31T22:02:28.490558Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- tweet_text: string (nullable = true)\n",
      " |-- cyberbullying_type: string (nullable = true)\n",
      " |-- gpt_labels: string (nullable = true)\n",
      " |-- Keep/ignore: string (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_pyspark = spark.read.option('header', 'true').csv('./cyberbullying_tweets_test.csv')\n",
    "df_pyspark.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "36706475",
   "metadata": {},
   "outputs": [],
   "source": [
    "# By default it is taking every column as string so to change that behavior we need to do this."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "7816d9a1",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-01-31T22:08:32.750311Z",
     "start_time": "2023-01-31T22:08:32.570266Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- tweet_text: string (nullable = true)\n",
      " |-- cyberbullying_type: string (nullable = true)\n",
      " |-- gpt_labels: string (nullable = true)\n",
      " |-- Keep/ignore: string (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_pyspark = spark.read.option('header', 'true').csv('./cyberbullying_tweets_test.csv', inferSchema= \"true\")\n",
    "df_pyspark.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "747c7d8e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-01-31T22:29:45.087688Z",
     "start_time": "2023-01-31T22:29:44.833514Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+------------------+----------+-----------+\n",
      "|          tweet_text|cyberbullying_type|gpt_labels|Keep/ignore|\n",
      "+--------------------+------------------+----------+-----------+\n",
      "|In other words #k...| not_cyberbullying|         0|       keep|\n",
      "|Why is #aussietv ...| not_cyberbullying|         0|       keep|\n",
      "|@XochitlSuckkks a...| not_cyberbullying|         0|       keep|\n",
      "|@Jason_Gio meh. :...| not_cyberbullying|         0|       keep|\n",
      "|@RudhoeEnglish Th...| not_cyberbullying|         0|       keep|\n",
      "|@Raja5aab @Quicki...| not_cyberbullying|         0|       keep|\n",
      "|Itu sekolah ya bu...| not_cyberbullying|         0|       keep|\n",
      "|Karma. I hope it ...| not_cyberbullying|         0|       keep|\n",
      "|@stockputout ever...| not_cyberbullying|         0|       keep|\n",
      "|Rebecca Black Dro...| not_cyberbullying|         0|       keep|\n",
      "|@Jord_Is_Dead htt...| not_cyberbullying|         0|       keep|\n",
      "|The Bully flushes...| not_cyberbullying|         0|       keep|\n",
      "|         Ughhhh #MKR| not_cyberbullying|         0|       keep|\n",
      "|RT @Kurdsnews: Tu...| not_cyberbullying|         0|       null|\n",
      "|\"Love that the be...| not_cyberbullying|         0|       null|\n",
      "|@yasmimcaci @Bfer...| not_cyberbullying|         0|       null|\n",
      "|@sarinhacoral @Vi...| not_cyberbullying|         0|       null|\n",
      "|@0xabad1dea @kels...| not_cyberbullying|         0|       null|\n",
      "|Best pick up line...| not_cyberbullying|         0|       null|\n",
      "|Now I gotta walk ...| not_cyberbullying|         0|       keep|\n",
      "+--------------------+------------------+----------+-----------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_pyspark = spark.read.csv(\"./cyberbullying_tweets_test.csv\", header= True, inferSchema=True)\n",
    "df_pyspark.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "0b42a0a5",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-01-31T22:30:21.782051Z",
     "start_time": "2023-01-31T22:30:21.767044Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- tweet_text: string (nullable = true)\n",
      " |-- cyberbullying_type: string (nullable = true)\n",
      " |-- gpt_labels: string (nullable = true)\n",
      " |-- Keep/ignore: string (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_pyspark.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "9328ad3d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-01-31T22:31:34.848986Z",
     "start_time": "2023-01-31T22:31:34.844462Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pyspark.sql.dataframe.DataFrame"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(df_pyspark)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "e3d76a59",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-01-31T22:31:58.321956Z",
     "start_time": "2023-01-31T22:31:58.268977Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Row(tweet_text='In other words #katandandre, your food was crapilicious! #mkr', cyberbullying_type='not_cyberbullying', gpt_labels='0', Keep/ignore='keep'),\n",
       " Row(tweet_text='Why is #aussietv so white? #MKR #theblock #ImACelebrityAU #today #sunrise #studio10 #Neighbours #WonderlandTen #etc', cyberbullying_type='not_cyberbullying', gpt_labels='0', Keep/ignore='keep'),\n",
       " Row(tweet_text='@XochitlSuckkks a classy whore? Or more red velvet cupcakes?', cyberbullying_type='not_cyberbullying', gpt_labels='0', Keep/ignore='keep')]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_pyspark.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "09e77637",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-01-31T22:33:20.508473Z",
     "start_time": "2023-01-31T22:33:20.461196Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+\n",
      "|          tweet_text|\n",
      "+--------------------+\n",
      "|In other words #k...|\n",
      "|Why is #aussietv ...|\n",
      "|@XochitlSuckkks a...|\n",
      "|@Jason_Gio meh. :...|\n",
      "|@RudhoeEnglish Th...|\n",
      "+--------------------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_pyspark.select(\"tweet_text\").show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "d4ee0b13",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-01-31T22:33:16.285465Z",
     "start_time": "2023-01-31T22:33:16.228537Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+----------+\n",
      "|          tweet_text|gpt_labels|\n",
      "+--------------------+----------+\n",
      "|In other words #k...|         0|\n",
      "|Why is #aussietv ...|         0|\n",
      "|@XochitlSuckkks a...|         0|\n",
      "|@Jason_Gio meh. :...|         0|\n",
      "|@RudhoeEnglish Th...|         0|\n",
      "+--------------------+----------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_pyspark.select([\"tweet_text\", \"gpt_labels\"]).show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "9558f4a5",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-01-31T22:34:15.242503Z",
     "start_time": "2023-01-31T22:34:15.236992Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('tweet_text', 'string'),\n",
       " ('cyberbullying_type', 'string'),\n",
       " ('gpt_labels', 'string'),\n",
       " ('Keep/ignore', 'string')]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_pyspark.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "c281efe1",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-01-31T22:35:21.868463Z",
     "start_time": "2023-01-31T22:35:21.234282Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+--------------------+--------------------+--------------------+------------------+\n",
      "|summary|          tweet_text|  cyberbullying_type|          gpt_labels|       Keep/ignore|\n",
      "+-------+--------------------+--------------------+--------------------+------------------+\n",
      "|  count|               47985|               47750|                1841|               503|\n",
      "|   mean|                null|                null|   1.996996996996997|2.5384615384615383|\n",
      "| stddev|                null|                null|  1.7531900008094783|1.5607361839521232|\n",
      "|    min| &amp; you might ...| \"\" you never shu...| \"\"Are you making...|           \"\"Kiley|\n",
      "|    max|😘 http://t.co/2Q...|’ he always calle...|” and still can’t...|                …\"|\n",
      "+-------+--------------------+--------------------+--------------------+------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_pyspark.describe().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "618cfd32",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-01-31T22:45:40.092403Z",
     "start_time": "2023-01-31T22:45:40.043012Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+------------------+----------+-----------+----------+\n",
      "|          tweet_text|cyberbullying_type|gpt_labels|Keep/ignore|prediction|\n",
      "+--------------------+------------------+----------+-----------+----------+\n",
      "|In other words #k...| not_cyberbullying|         0|       keep|       2.0|\n",
      "|Why is #aussietv ...| not_cyberbullying|         0|       keep|       2.0|\n",
      "|@XochitlSuckkks a...| not_cyberbullying|         0|       keep|       2.0|\n",
      "|@Jason_Gio meh. :...| not_cyberbullying|         0|       keep|       2.0|\n",
      "|@RudhoeEnglish Th...| not_cyberbullying|         0|       keep|       2.0|\n",
      "+--------------------+------------------+----------+-----------+----------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "### Adding columns in dataframe \n",
    "df_pyspark.withColumn('prediction', df_pyspark['gpt_labels']+2).show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "4841e97f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-01-31T22:51:50.505343Z",
     "start_time": "2023-01-31T22:51:50.485505Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[tweet_text: string, cyberbullying_type: string, gpt_labels: string, Keep/ignore: string]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#### Droping the columns\n",
    "df_pyspark.drop(\"prediction\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "030c7a40",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-01-31T22:54:16.516702Z",
     "start_time": "2023-01-31T22:54:16.458458Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+------------------+----------+-----------+\n",
      "|               tweet|cyberbullying_type|gpt_labels|Keep/ignore|\n",
      "+--------------------+------------------+----------+-----------+\n",
      "|In other words #k...| not_cyberbullying|         0|       keep|\n",
      "|Why is #aussietv ...| not_cyberbullying|         0|       keep|\n",
      "|@XochitlSuckkks a...| not_cyberbullying|         0|       keep|\n",
      "|@Jason_Gio meh. :...| not_cyberbullying|         0|       keep|\n",
      "|@RudhoeEnglish Th...| not_cyberbullying|         0|       keep|\n",
      "|@Raja5aab @Quicki...| not_cyberbullying|         0|       keep|\n",
      "|Itu sekolah ya bu...| not_cyberbullying|         0|       keep|\n",
      "|Karma. I hope it ...| not_cyberbullying|         0|       keep|\n",
      "|@stockputout ever...| not_cyberbullying|         0|       keep|\n",
      "|Rebecca Black Dro...| not_cyberbullying|         0|       keep|\n",
      "|@Jord_Is_Dead htt...| not_cyberbullying|         0|       keep|\n",
      "|The Bully flushes...| not_cyberbullying|         0|       keep|\n",
      "|         Ughhhh #MKR| not_cyberbullying|         0|       keep|\n",
      "|RT @Kurdsnews: Tu...| not_cyberbullying|         0|       null|\n",
      "|\"Love that the be...| not_cyberbullying|         0|       null|\n",
      "|@yasmimcaci @Bfer...| not_cyberbullying|         0|       null|\n",
      "|@sarinhacoral @Vi...| not_cyberbullying|         0|       null|\n",
      "|@0xabad1dea @kels...| not_cyberbullying|         0|       null|\n",
      "|Best pick up line...| not_cyberbullying|         0|       null|\n",
      "|Now I gotta walk ...| not_cyberbullying|         0|       keep|\n",
      "+--------------------+------------------+----------+-----------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Rename the columns\n",
    "df_pyspark.withColumnRenamed(\"tweet_text\", \"tweet\").show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "72bdba3f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-01-31T22:57:10.451506Z",
     "start_time": "2023-01-31T22:57:10.438475Z"
    }
   },
   "outputs": [],
   "source": [
    "# Now we will handle\n",
    "\n",
    "# Missing values by dropping Columns\n",
    "# Dropping Rows\n",
    "# Various Parameters in Dropping Functionalities\n",
    "# Handling Missing values By Mean, Median and Mode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "785b8340",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-01-31T23:03:02.931858Z",
     "start_time": "2023-01-31T23:03:02.765034Z"
    }
   },
   "outputs": [],
   "source": [
    "from pyspark.sql import SparkSession\n",
    "\n",
    "spark = SparkSession.builder.appName('Practice').getOrCreate()\n",
    "df_pyspark = spark.read.csv(\"./cyberbullying_tweets_test.csv\", header = True, inferSchema=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "f461d8a9",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-01-31T23:04:32.524599Z",
     "start_time": "2023-01-31T23:04:32.472993Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+------------------+----------+-----------+\n",
      "|          tweet_text|cyberbullying_type|gpt_labels|Keep/ignore|\n",
      "+--------------------+------------------+----------+-----------+\n",
      "|In other words #k...| not_cyberbullying|         0|       keep|\n",
      "|Why is #aussietv ...| not_cyberbullying|         0|       keep|\n",
      "|@XochitlSuckkks a...| not_cyberbullying|         0|       keep|\n",
      "|@Jason_Gio meh. :...| not_cyberbullying|         0|       keep|\n",
      "|@RudhoeEnglish Th...| not_cyberbullying|         0|       keep|\n",
      "|@Raja5aab @Quicki...| not_cyberbullying|         0|       keep|\n",
      "|Itu sekolah ya bu...| not_cyberbullying|         0|       keep|\n",
      "|Karma. I hope it ...| not_cyberbullying|         0|       keep|\n",
      "|@stockputout ever...| not_cyberbullying|         0|       keep|\n",
      "|Rebecca Black Dro...| not_cyberbullying|         0|       keep|\n",
      "|@Jord_Is_Dead htt...| not_cyberbullying|         0|       keep|\n",
      "|The Bully flushes...| not_cyberbullying|         0|       keep|\n",
      "|         Ughhhh #MKR| not_cyberbullying|         0|       keep|\n",
      "|RT @Kurdsnews: Tu...| not_cyberbullying|         0|       null|\n",
      "|\"Love that the be...| not_cyberbullying|         0|       null|\n",
      "|@yasmimcaci @Bfer...| not_cyberbullying|         0|       null|\n",
      "|@sarinhacoral @Vi...| not_cyberbullying|         0|       null|\n",
      "|@0xabad1dea @kels...| not_cyberbullying|         0|       null|\n",
      "|Best pick up line...| not_cyberbullying|         0|       null|\n",
      "|Now I gotta walk ...| not_cyberbullying|         0|       keep|\n",
      "+--------------------+------------------+----------+-----------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_pyspark.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "fd090f93",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-01-31T23:06:21.550416Z",
     "start_time": "2023-01-31T23:06:21.480000Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+------------------+----------+\n",
      "|          tweet_text|cyberbullying_type|gpt_labels|\n",
      "+--------------------+------------------+----------+\n",
      "|In other words #k...| not_cyberbullying|         0|\n",
      "|Why is #aussietv ...| not_cyberbullying|         0|\n",
      "|@XochitlSuckkks a...| not_cyberbullying|         0|\n",
      "|@Jason_Gio meh. :...| not_cyberbullying|         0|\n",
      "|@RudhoeEnglish Th...| not_cyberbullying|         0|\n",
      "|@Raja5aab @Quicki...| not_cyberbullying|         0|\n",
      "|Itu sekolah ya bu...| not_cyberbullying|         0|\n",
      "|Karma. I hope it ...| not_cyberbullying|         0|\n",
      "|@stockputout ever...| not_cyberbullying|         0|\n",
      "|Rebecca Black Dro...| not_cyberbullying|         0|\n",
      "|@Jord_Is_Dead htt...| not_cyberbullying|         0|\n",
      "|The Bully flushes...| not_cyberbullying|         0|\n",
      "|         Ughhhh #MKR| not_cyberbullying|         0|\n",
      "|RT @Kurdsnews: Tu...| not_cyberbullying|         0|\n",
      "|\"Love that the be...| not_cyberbullying|         0|\n",
      "|@yasmimcaci @Bfer...| not_cyberbullying|         0|\n",
      "|@sarinhacoral @Vi...| not_cyberbullying|         0|\n",
      "|@0xabad1dea @kels...| not_cyberbullying|         0|\n",
      "|Best pick up line...| not_cyberbullying|         0|\n",
      "|Now I gotta walk ...| not_cyberbullying|         0|\n",
      "+--------------------+------------------+----------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "## Drop the columns \n",
    "df_pyspark.drop(\"Keep/ignore\").show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "fec8e18d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-01-31T23:09:04.710713Z",
     "start_time": "2023-01-31T23:09:04.658713Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+------------------+----------+-----------+\n",
      "|          tweet_text|cyberbullying_type|gpt_labels|Keep/ignore|\n",
      "+--------------------+------------------+----------+-----------+\n",
      "|In other words #k...| not_cyberbullying|         0|       keep|\n",
      "|Why is #aussietv ...| not_cyberbullying|         0|       keep|\n",
      "|@XochitlSuckkks a...| not_cyberbullying|         0|       keep|\n",
      "|@Jason_Gio meh. :...| not_cyberbullying|         0|       keep|\n",
      "|@RudhoeEnglish Th...| not_cyberbullying|         0|       keep|\n",
      "|@Raja5aab @Quicki...| not_cyberbullying|         0|       keep|\n",
      "|Itu sekolah ya bu...| not_cyberbullying|         0|       keep|\n",
      "|Karma. I hope it ...| not_cyberbullying|         0|       keep|\n",
      "|@stockputout ever...| not_cyberbullying|         0|       keep|\n",
      "|Rebecca Black Dro...| not_cyberbullying|         0|       keep|\n",
      "|@Jord_Is_Dead htt...| not_cyberbullying|         0|       keep|\n",
      "|The Bully flushes...| not_cyberbullying|         0|       keep|\n",
      "|         Ughhhh #MKR| not_cyberbullying|         0|       keep|\n",
      "|Now I gotta walk ...| not_cyberbullying|         0|       keep|\n",
      "|@halalcunty @bieb...| not_cyberbullying|         0|       keep|\n",
      "|Kids Love😘❤ @ Mo...| not_cyberbullying|         0|       keep|\n",
      "|I still have Jack...| not_cyberbullying|         0|       keep|\n",
      "|@gcarothers eek. ...| not_cyberbullying|         0|     ignore|\n",
      "|@MaxBlumenthal @c...| not_cyberbullying|         0|     ignore|\n",
      "|You know there ar...| not_cyberbullying|         0|     ignore|\n",
      "+--------------------+------------------+----------+-----------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "## Drop the nan values from overall dataframe\n",
    "df_pyspark.na.drop().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d8198bf1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Any How\n",
    "# If I choose all it will drop only those rows that have all the values null in the whole row , if any single value contain non\n",
    "# null value it will not drop that then.\n",
    "df_pyspark.na.drop(how=\"all\").show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "c0f417b1",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-01-31T23:14:22.732278Z",
     "start_time": "2023-01-31T23:14:22.642458Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+------------------+----------+-----------+\n",
      "|          tweet_text|cyberbullying_type|gpt_labels|Keep/ignore|\n",
      "+--------------------+------------------+----------+-----------+\n",
      "|In other words #k...| not_cyberbullying|         0|       keep|\n",
      "|Why is #aussietv ...| not_cyberbullying|         0|       keep|\n",
      "|@XochitlSuckkks a...| not_cyberbullying|         0|       keep|\n",
      "|@Jason_Gio meh. :...| not_cyberbullying|         0|       keep|\n",
      "|@RudhoeEnglish Th...| not_cyberbullying|         0|       keep|\n",
      "|@Raja5aab @Quicki...| not_cyberbullying|         0|       keep|\n",
      "|Itu sekolah ya bu...| not_cyberbullying|         0|       keep|\n",
      "|Karma. I hope it ...| not_cyberbullying|         0|       keep|\n",
      "|@stockputout ever...| not_cyberbullying|         0|       keep|\n",
      "|Rebecca Black Dro...| not_cyberbullying|         0|       keep|\n",
      "|@Jord_Is_Dead htt...| not_cyberbullying|         0|       keep|\n",
      "|The Bully flushes...| not_cyberbullying|         0|       keep|\n",
      "|         Ughhhh #MKR| not_cyberbullying|         0|       keep|\n",
      "|RT @Kurdsnews: Tu...| not_cyberbullying|         0|       null|\n",
      "|\"Love that the be...| not_cyberbullying|         0|       null|\n",
      "|@yasmimcaci @Bfer...| not_cyberbullying|         0|       null|\n",
      "|@sarinhacoral @Vi...| not_cyberbullying|         0|       null|\n",
      "|@0xabad1dea @kels...| not_cyberbullying|         0|       null|\n",
      "|Best pick up line...| not_cyberbullying|         0|       null|\n",
      "|Now I gotta walk ...| not_cyberbullying|         0|       keep|\n",
      "+--------------------+------------------+----------+-----------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "## Threshold \n",
    "# if the thresh = 2 or any number it will check if the whole row contains atleast 2 non null values, if they contains then it \n",
    "# won't delete the whole row.\n",
    "df_pyspark.na.drop(how=\"all\", thresh= 2).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7711da5c",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Threshold \n",
    "# if the thresh = 2 or any number it will check if the whole row contains atleast 1 non null values, if they contains then it \n",
    "# won't delete the whole row.\n",
    "df_pyspark.na.drop(how=\"all\", thresh= 1).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "d29abea1",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-01-31T23:23:25.396634Z",
     "start_time": "2023-01-31T23:23:25.347634Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+------------------+----------+-----------+\n",
      "|          tweet_text|cyberbullying_type|gpt_labels|Keep/ignore|\n",
      "+--------------------+------------------+----------+-----------+\n",
      "|In other words #k...| not_cyberbullying|         0|       keep|\n",
      "|Why is #aussietv ...| not_cyberbullying|         0|       keep|\n",
      "|@XochitlSuckkks a...| not_cyberbullying|         0|       keep|\n",
      "|@Jason_Gio meh. :...| not_cyberbullying|         0|       keep|\n",
      "|@RudhoeEnglish Th...| not_cyberbullying|         0|       keep|\n",
      "|@Raja5aab @Quicki...| not_cyberbullying|         0|       keep|\n",
      "|Itu sekolah ya bu...| not_cyberbullying|         0|       keep|\n",
      "|Karma. I hope it ...| not_cyberbullying|         0|       keep|\n",
      "|@stockputout ever...| not_cyberbullying|         0|       keep|\n",
      "|Rebecca Black Dro...| not_cyberbullying|         0|       keep|\n",
      "|@Jord_Is_Dead htt...| not_cyberbullying|         0|       keep|\n",
      "|The Bully flushes...| not_cyberbullying|         0|       keep|\n",
      "|         Ughhhh #MKR| not_cyberbullying|         0|       keep|\n",
      "|RT @Kurdsnews: Tu...| not_cyberbullying|         0|       null|\n",
      "|\"Love that the be...| not_cyberbullying|         0|       null|\n",
      "|@yasmimcaci @Bfer...| not_cyberbullying|         0|       null|\n",
      "|@sarinhacoral @Vi...| not_cyberbullying|         0|       null|\n",
      "|@0xabad1dea @kels...| not_cyberbullying|         0|       null|\n",
      "|Best pick up line...| not_cyberbullying|         0|       null|\n",
      "|Now I gotta walk ...| not_cyberbullying|         0|       keep|\n",
      "+--------------------+------------------+----------+-----------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Subset\n",
    "\n",
    "df_pyspark.na.drop(how=\"any\", subset=[\"gpt_labels\"]).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2605ab7e",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Filling the missing values\n",
    "\n",
    "df_pyspark.na.fill(\"Missing Values\").show()\n",
    "# Wherever it will see NaN values it will replae it with Missing values string. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "6e26c412",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-01-31T23:28:10.902940Z",
     "start_time": "2023-01-31T23:28:10.840942Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+------------------+----------+--------------+\n",
      "|          tweet_text|cyberbullying_type|gpt_labels|   Keep/ignore|\n",
      "+--------------------+------------------+----------+--------------+\n",
      "|In other words #k...| not_cyberbullying|         0|          keep|\n",
      "|Why is #aussietv ...| not_cyberbullying|         0|          keep|\n",
      "|@XochitlSuckkks a...| not_cyberbullying|         0|          keep|\n",
      "|@Jason_Gio meh. :...| not_cyberbullying|         0|          keep|\n",
      "|@RudhoeEnglish Th...| not_cyberbullying|         0|          keep|\n",
      "|@Raja5aab @Quicki...| not_cyberbullying|         0|          keep|\n",
      "|Itu sekolah ya bu...| not_cyberbullying|         0|          keep|\n",
      "|Karma. I hope it ...| not_cyberbullying|         0|          keep|\n",
      "|@stockputout ever...| not_cyberbullying|         0|          keep|\n",
      "|Rebecca Black Dro...| not_cyberbullying|         0|          keep|\n",
      "|@Jord_Is_Dead htt...| not_cyberbullying|         0|          keep|\n",
      "|The Bully flushes...| not_cyberbullying|         0|          keep|\n",
      "|         Ughhhh #MKR| not_cyberbullying|         0|          keep|\n",
      "|RT @Kurdsnews: Tu...| not_cyberbullying|         0|Missing values|\n",
      "|\"Love that the be...| not_cyberbullying|         0|Missing values|\n",
      "|@yasmimcaci @Bfer...| not_cyberbullying|         0|Missing values|\n",
      "|@sarinhacoral @Vi...| not_cyberbullying|         0|Missing values|\n",
      "|@0xabad1dea @kels...| not_cyberbullying|         0|Missing values|\n",
      "|Best pick up line...| not_cyberbullying|         0|Missing values|\n",
      "|Now I gotta walk ...| not_cyberbullying|         0|          keep|\n",
      "+--------------------+------------------+----------+--------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_pyspark.na.fill(\"Missing values\", [\"gpt_labels\", \"keep/ignore\"]).show()\n",
    "# when i specify the column name the filling of missing values will happen only in those specified column. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "21ab9df9",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-01-31T23:48:08.415973Z",
     "start_time": "2023-01-31T23:48:08.328461Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+------------------+----------+-----------+\n",
      "|          tweet_text|cyberbullying_type|gpt_labels|Keep/ignore|\n",
      "+--------------------+------------------+----------+-----------+\n",
      "|In other words #k...| not_cyberbullying|         0|       keep|\n",
      "|Why is #aussietv ...| not_cyberbullying|         0|       keep|\n",
      "|@XochitlSuckkks a...| not_cyberbullying|         0|       keep|\n",
      "|@Jason_Gio meh. :...| not_cyberbullying|         0|       keep|\n",
      "|@RudhoeEnglish Th...| not_cyberbullying|         0|       keep|\n",
      "|@Raja5aab @Quicki...| not_cyberbullying|         0|       keep|\n",
      "|Itu sekolah ya bu...| not_cyberbullying|         0|       keep|\n",
      "|Karma. I hope it ...| not_cyberbullying|         0|       keep|\n",
      "|@stockputout ever...| not_cyberbullying|         0|       keep|\n",
      "|Rebecca Black Dro...| not_cyberbullying|         0|       keep|\n",
      "|@Jord_Is_Dead htt...| not_cyberbullying|         0|       keep|\n",
      "|The Bully flushes...| not_cyberbullying|         0|       keep|\n",
      "|         Ughhhh #MKR| not_cyberbullying|         0|       keep|\n",
      "|RT @Kurdsnews: Tu...| not_cyberbullying|         0|       null|\n",
      "|\"Love that the be...| not_cyberbullying|         0|       null|\n",
      "|@yasmimcaci @Bfer...| not_cyberbullying|         0|       null|\n",
      "|@sarinhacoral @Vi...| not_cyberbullying|         0|       null|\n",
      "|@0xabad1dea @kels...| not_cyberbullying|         0|       null|\n",
      "|Best pick up line...| not_cyberbullying|         0|       null|\n",
      "|Now I gotta walk ...| not_cyberbullying|         0|       keep|\n",
      "+--------------------+------------------+----------+-----------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "dd_pyspark = df_pyspark.na.drop(how=\"any\", subset=[\"gpt_labels\"])\n",
    "dd_pyspark.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "e06f682a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-01T00:16:39.495037Z",
     "start_time": "2023-02-01T00:16:39.485038Z"
    }
   },
   "outputs": [],
   "source": [
    "from pyspark.sql.types import *\n",
    "\n",
    "schema = StructType() \\\n",
    "      .add(\"gpt_labels\", IntegerType(), True) \\\n",
    "      .add(\"tweet_text\", StringType(), True) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "5ad9f7c0",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-01T00:25:59.610428Z",
     "start_time": "2023-02-01T00:25:59.477114Z"
    }
   },
   "outputs": [],
   "source": [
    "from pyspark.sql import SparkSession\n",
    "from pyspark.sql.functions import col, cast\n",
    "\n",
    "spark = SparkSession.builder.appName('Practice').getOrCreate()\n",
    "df_pyspark = spark.read.options(inferSchema = \"true\").csv( \"./cyberbullying_tweets_test.csv\", header = True, inferSchema=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "40a504f2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-01T00:25:59.783226Z",
     "start_time": "2023-02-01T00:25:59.769227Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- tweet_text: string (nullable = true)\n",
      " |-- cyberbullying_type: string (nullable = true)\n",
      " |-- gpt_labels: string (nullable = true)\n",
      " |-- Keep/ignore: string (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_pyspark.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "d55b3440",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-01T03:24:11.740464Z",
     "start_time": "2023-02-01T03:24:11.686943Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+------------------+----------+-----------+\n",
      "|          tweet_text|cyberbullying_type|gpt_labels|Keep/ignore|\n",
      "+--------------------+------------------+----------+-----------+\n",
      "|In other words #k...| not_cyberbullying|         0|       keep|\n",
      "|Why is #aussietv ...| not_cyberbullying|         0|       keep|\n",
      "|@XochitlSuckkks a...| not_cyberbullying|         0|       keep|\n",
      "|@Jason_Gio meh. :...| not_cyberbullying|         0|       keep|\n",
      "|@RudhoeEnglish Th...| not_cyberbullying|         0|       keep|\n",
      "|@Raja5aab @Quicki...| not_cyberbullying|         0|       keep|\n",
      "|Itu sekolah ya bu...| not_cyberbullying|         0|       keep|\n",
      "|Karma. I hope it ...| not_cyberbullying|         0|       keep|\n",
      "|@stockputout ever...| not_cyberbullying|         0|       keep|\n",
      "|Rebecca Black Dro...| not_cyberbullying|         0|       keep|\n",
      "+--------------------+------------------+----------+-----------+\n",
      "only showing top 10 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_pyspark.show(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "94bedd39",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Changing datatype of columns in pyspark dataframe. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "821ad467",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-01T03:25:20.655108Z",
     "start_time": "2023-02-01T03:25:20.634107Z"
    }
   },
   "outputs": [],
   "source": [
    "# Cast Course_Fees from integer type to float type\n",
    "df2_pyspark2 = df_pyspark.withColumn(\"gpt_labels\", \n",
    "                                  df_pyspark[\"gpt_labels\"]\n",
    "                                  .cast('int'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "ccde26d8",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-01T03:25:21.751182Z",
     "start_time": "2023-02-01T03:25:21.732174Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- tweet_text: string (nullable = true)\n",
      " |-- cyberbullying_type: string (nullable = true)\n",
      " |-- gpt_labels: integer (nullable = true)\n",
      " |-- Keep/ignore: string (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df2_pyspark2.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "de4b1f13",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-01T03:42:30.974474Z",
     "start_time": "2023-02-01T03:42:30.965952Z"
    }
   },
   "outputs": [],
   "source": [
    "# Use of imputer fucntion in the pyspark\n",
    "\n",
    "from pyspark.ml.feature import Imputer\n",
    "\n",
    "imputer = Imputer(\n",
    "\t\tinputCols = [\"gpt_labels\"],\n",
    "    \toutputCols=['{}_imputed'.format(c) for c in [\"gpt_labels\"]],\n",
    "\t\t).setStrategy('mean')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "id": "76703b39",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-01T03:42:55.004044Z",
     "start_time": "2023-02-01T03:42:54.738001Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+------------------+----------+-----------+------------------+\n",
      "|          tweet_text|cyberbullying_type|gpt_labels|Keep/ignore|gpt_labels_imputed|\n",
      "+--------------------+------------------+----------+-----------+------------------+\n",
      "|In other words #k...| not_cyberbullying|         0|       keep|                 0|\n",
      "|Why is #aussietv ...| not_cyberbullying|         0|       keep|                 0|\n",
      "|@XochitlSuckkks a...| not_cyberbullying|         0|       keep|                 0|\n",
      "|@Jason_Gio meh. :...| not_cyberbullying|         0|       keep|                 0|\n",
      "|@RudhoeEnglish Th...| not_cyberbullying|         0|       keep|                 0|\n",
      "|@Raja5aab @Quicki...| not_cyberbullying|         0|       keep|                 0|\n",
      "|Itu sekolah ya bu...| not_cyberbullying|         0|       keep|                 0|\n",
      "|Karma. I hope it ...| not_cyberbullying|         0|       keep|                 0|\n",
      "|@stockputout ever...| not_cyberbullying|         0|       keep|                 0|\n",
      "|Rebecca Black Dro...| not_cyberbullying|         0|       keep|                 0|\n",
      "|@Jord_Is_Dead htt...| not_cyberbullying|         0|       keep|                 0|\n",
      "|The Bully flushes...| not_cyberbullying|         0|       keep|                 0|\n",
      "|         Ughhhh #MKR| not_cyberbullying|         0|       keep|                 0|\n",
      "|RT @Kurdsnews: Tu...| not_cyberbullying|         0|       null|                 0|\n",
      "|\"Love that the be...| not_cyberbullying|         0|       null|                 0|\n",
      "|@yasmimcaci @Bfer...| not_cyberbullying|         0|       null|                 0|\n",
      "|@sarinhacoral @Vi...| not_cyberbullying|         0|       null|                 0|\n",
      "|@0xabad1dea @kels...| not_cyberbullying|         0|       null|                 0|\n",
      "|Best pick up line...| not_cyberbullying|         0|       null|                 0|\n",
      "|Now I gotta walk ...| not_cyberbullying|         0|       keep|                 0|\n",
      "+--------------------+------------------+----------+-----------+------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "imputer.fit(df2_pyspark2).transform(df2_pyspark2).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "e583e4f2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-01T03:44:23.213162Z",
     "start_time": "2023-02-01T03:44:23.197166Z"
    }
   },
   "outputs": [],
   "source": [
    "# Now\n",
    "# Filter operations\n",
    "# &, |, ==\n",
    "# ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "de11ca7c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-01T03:46:20.057681Z",
     "start_time": "2023-02-01T03:46:20.040135Z"
    }
   },
   "outputs": [],
   "source": [
    "from pyspark.sql import SparkSession \n",
    "\n",
    "spark = SparkSession.builder.appName(\"dataframe\").getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "id": "90686c3f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-01T03:47:30.174789Z",
     "start_time": "2023-02-01T03:47:30.001550Z"
    }
   },
   "outputs": [],
   "source": [
    "df_pyspark = spark.read.csv(\"./cyberbullying_tweets_test.csv\", header = True, inferSchema= True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "id": "50d1843b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-01T03:47:31.242629Z",
     "start_time": "2023-02-01T03:47:31.227065Z"
    }
   },
   "outputs": [],
   "source": [
    "# Cast Course_Fees from integer type to float type\n",
    "df2_pyspark2 = df_pyspark.withColumn(\"gpt_labels\", \n",
    "                                  df_pyspark[\"gpt_labels\"]\n",
    "                                  .cast('int'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "id": "37db6cf0",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-01T03:47:39.449476Z",
     "start_time": "2023-02-01T03:47:39.398959Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+------------------+----------+-----------+\n",
      "|          tweet_text|cyberbullying_type|gpt_labels|Keep/ignore|\n",
      "+--------------------+------------------+----------+-----------+\n",
      "|In other words #k...| not_cyberbullying|         0|       keep|\n",
      "|Why is #aussietv ...| not_cyberbullying|         0|       keep|\n",
      "|@XochitlSuckkks a...| not_cyberbullying|         0|       keep|\n",
      "|@Jason_Gio meh. :...| not_cyberbullying|         0|       keep|\n",
      "|@RudhoeEnglish Th...| not_cyberbullying|         0|       keep|\n",
      "|@Raja5aab @Quicki...| not_cyberbullying|         0|       keep|\n",
      "|Itu sekolah ya bu...| not_cyberbullying|         0|       keep|\n",
      "|Karma. I hope it ...| not_cyberbullying|         0|       keep|\n",
      "|@stockputout ever...| not_cyberbullying|         0|       keep|\n",
      "|Rebecca Black Dro...| not_cyberbullying|         0|       keep|\n",
      "|@Jord_Is_Dead htt...| not_cyberbullying|         0|       keep|\n",
      "|The Bully flushes...| not_cyberbullying|         0|       keep|\n",
      "|         Ughhhh #MKR| not_cyberbullying|         0|       keep|\n",
      "|RT @Kurdsnews: Tu...| not_cyberbullying|         0|       null|\n",
      "|\"Love that the be...| not_cyberbullying|         0|       null|\n",
      "|@yasmimcaci @Bfer...| not_cyberbullying|         0|       null|\n",
      "|@sarinhacoral @Vi...| not_cyberbullying|         0|       null|\n",
      "|@0xabad1dea @kels...| not_cyberbullying|         0|       null|\n",
      "|Best pick up line...| not_cyberbullying|         0|       null|\n",
      "|Now I gotta walk ...| not_cyberbullying|         0|       keep|\n",
      "+--------------------+------------------+----------+-----------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df2_pyspark2.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2e45c7dd",
   "metadata": {},
   "source": [
    "# Filter Operations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "id": "b6f4490b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-01T06:00:44.242987Z",
     "start_time": "2023-02-01T06:00:44.018315Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+------------------+----------+-----------+\n",
      "|          tweet_text|cyberbullying_type|gpt_labels|Keep/ignore|\n",
      "+--------------------+------------------+----------+-----------+\n",
      "|You never saw any...|            gender|         3|       null|\n",
      "|RT @Raul_Novoa16:...|            gender|         3|       null|\n",
      "|Rape is rape. And...|            gender|         3|       null|\n",
      "|@coiny Also, it's...|            gender|         3|       null|\n",
      "|Idgaf if you are ...|            gender|         3|       null|\n",
      "|\"#GermanProfessor...|            gender|         3|       null|\n",
      "|RT @mcclure111: #...|            gender|         3|       null|\n",
      "|SO HILARIOUS U WR...|            gender|         4|       null|\n",
      "|“We voted for a g...|            gender|         3|       null|\n",
      "|I always get fuck...|            gender|         3|       null|\n",
      "|\"@DrawPlayDave \"\"...|            gender|         3|       null|\n",
      "|almost 2017 & yal...|            gender|         4|       null|\n",
      "|\"Morrison believe...|            gender|         4|       null|\n",
      "|were there no wom...|            gender|         3|       null|\n",
      "|RT @jcsliva1999 @...|            gender|         3|       null|\n",
      "|\"RT @souperfan201...|            gender|         4|       null|\n",
      "|from what people ...|            gender|         4|       null|\n",
      "|RAPE AND PEDOPHIL...|            gender|         5|       null|\n",
      "|Thank you, uncle,...|            gender|         4|       null|\n",
      "|Hate when niggas ...|            gender|         3|       null|\n",
      "+--------------------+------------------+----------+-----------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# I want to find gpt_labels that are have values above than 2... \n",
    "\n",
    "df2_pyspark2.filter(\"gpt_labels >= 3\").show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "id": "79a8ffd3",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-01T06:02:45.969690Z",
     "start_time": "2023-02-01T06:02:45.890274Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+------------------+----------+\n",
      "|          tweet_text|cyberbullying_type|gpt_labels|\n",
      "+--------------------+------------------+----------+\n",
      "|You never saw any...|            gender|         3|\n",
      "|RT @Raul_Novoa16:...|            gender|         3|\n",
      "|Rape is rape. And...|            gender|         3|\n",
      "|@coiny Also, it's...|            gender|         3|\n",
      "|Idgaf if you are ...|            gender|         3|\n",
      "|\"#GermanProfessor...|            gender|         3|\n",
      "|RT @mcclure111: #...|            gender|         3|\n",
      "|SO HILARIOUS U WR...|            gender|         4|\n",
      "|“We voted for a g...|            gender|         3|\n",
      "|I always get fuck...|            gender|         3|\n",
      "|\"@DrawPlayDave \"\"...|            gender|         3|\n",
      "|almost 2017 & yal...|            gender|         4|\n",
      "|\"Morrison believe...|            gender|         4|\n",
      "|were there no wom...|            gender|         3|\n",
      "|RT @jcsliva1999 @...|            gender|         3|\n",
      "|\"RT @souperfan201...|            gender|         4|\n",
      "|from what people ...|            gender|         4|\n",
      "|RAPE AND PEDOPHIL...|            gender|         5|\n",
      "|Thank you, uncle,...|            gender|         4|\n",
      "|Hate when niggas ...|            gender|         3|\n",
      "+--------------------+------------------+----------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# I want to find gpt_labels that are have values above than 2... and I want to show some specifics columns only then showing \n",
    "# the whole dataframe\n",
    "\n",
    "df2_pyspark2.filter(\"gpt_labels >= 3\").select([\"tweet_text\", \"cyberbullying_type\", \"gpt_labels\"]).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "id": "6f6cd807",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-01T06:06:08.433867Z",
     "start_time": "2023-02-01T06:06:08.345651Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+------------------+----------+-----------+\n",
      "|          tweet_text|cyberbullying_type|gpt_labels|Keep/ignore|\n",
      "+--------------------+------------------+----------+-----------+\n",
      "|You never saw any...|            gender|         3|       null|\n",
      "|RT @Raul_Novoa16:...|            gender|         3|       null|\n",
      "|Rape is rape. And...|            gender|         3|       null|\n",
      "|@coiny Also, it's...|            gender|         3|       null|\n",
      "|Idgaf if you are ...|            gender|         3|       null|\n",
      "|\"#GermanProfessor...|            gender|         3|       null|\n",
      "|RT @mcclure111: #...|            gender|         3|       null|\n",
      "|SO HILARIOUS U WR...|            gender|         4|       null|\n",
      "|“We voted for a g...|            gender|         3|       null|\n",
      "|I always get fuck...|            gender|         3|       null|\n",
      "|\"@DrawPlayDave \"\"...|            gender|         3|       null|\n",
      "|almost 2017 & yal...|            gender|         4|       null|\n",
      "|\"Morrison believe...|            gender|         4|       null|\n",
      "|were there no wom...|            gender|         3|       null|\n",
      "|RT @jcsliva1999 @...|            gender|         3|       null|\n",
      "|\"RT @souperfan201...|            gender|         4|       null|\n",
      "|from what people ...|            gender|         4|       null|\n",
      "|RAPE AND PEDOPHIL...|            gender|         5|       null|\n",
      "|Thank you, uncle,...|            gender|         4|       null|\n",
      "|Hate when niggas ...|            gender|         3|       null|\n",
      "+--------------------+------------------+----------+-----------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Now I want to write multiples condition \n",
    "\n",
    "df2_pyspark2.filter((df2_pyspark2[\"gpt_labels\"] <= 5) & (df2_pyspark2[\"gpt_labels\"] > 2)).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b637e8aa",
   "metadata": {},
   "outputs": [],
   "source": [
    " # Now I want to write multiples condition  i.e. end or\n",
    "\n",
    "df2_pyspark2.filter((df2_pyspark2[\"gpt_labels\"] <= 5) | (df2_pyspark2[\"gpt_labels\"] > 2)).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "id": "d7fd518f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-01T06:08:40.387492Z",
     "start_time": "2023-02-01T06:08:40.296970Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+------------------+----------+-----------+\n",
      "|          tweet_text|cyberbullying_type|gpt_labels|Keep/ignore|\n",
      "+--------------------+------------------+----------+-----------+\n",
      "|You never saw any...|            gender|         3|       null|\n",
      "|RT @Raul_Novoa16:...|            gender|         3|       null|\n",
      "|Rape is rape. And...|            gender|         3|       null|\n",
      "|@coiny Also, it's...|            gender|         3|       null|\n",
      "|Idgaf if you are ...|            gender|         3|       null|\n",
      "|\"#GermanProfessor...|            gender|         3|       null|\n",
      "|RT @mcclure111: #...|            gender|         3|       null|\n",
      "|SO HILARIOUS U WR...|            gender|         4|       null|\n",
      "|“We voted for a g...|            gender|         3|       null|\n",
      "|I always get fuck...|            gender|         3|       null|\n",
      "|\"@DrawPlayDave \"\"...|            gender|         3|       null|\n",
      "|almost 2017 & yal...|            gender|         4|       null|\n",
      "|\"Morrison believe...|            gender|         4|       null|\n",
      "|were there no wom...|            gender|         3|       null|\n",
      "|RT @jcsliva1999 @...|            gender|         3|       null|\n",
      "|\"RT @souperfan201...|            gender|         4|       null|\n",
      "|from what people ...|            gender|         4|       null|\n",
      "|RAPE AND PEDOPHIL...|            gender|         5|       null|\n",
      "|Thank you, uncle,...|            gender|         4|       null|\n",
      "|Hate when niggas ...|            gender|         3|       null|\n",
      "+--------------------+------------------+----------+-----------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    " # Now I want to test not condition i.e. ~\n",
    "\n",
    "df2_pyspark2.filter(~(df2_pyspark2[\"gpt_labels\"] <= 2)).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "id": "2fd373b4",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-01T07:09:33.434958Z",
     "start_time": "2023-02-01T07:09:33.414957Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "            <div>\n",
       "                <p><b>SparkSession - in-memory</b></p>\n",
       "                \n",
       "        <div>\n",
       "            <p><b>SparkContext</b></p>\n",
       "\n",
       "            <p><a href=\"http://DESKTOP-Q24DOVF:4040\">Spark UI</a></p>\n",
       "\n",
       "            <dl>\n",
       "              <dt>Version</dt>\n",
       "                <dd><code>v3.3.1</code></dd>\n",
       "              <dt>Master</dt>\n",
       "                <dd><code>local[*]</code></dd>\n",
       "              <dt>AppName</dt>\n",
       "                <dd><code>Practice</code></dd>\n",
       "            </dl>\n",
       "        </div>\n",
       "        \n",
       "            </div>\n",
       "        "
      ],
      "text/plain": [
       "<pyspark.sql.session.SparkSession at 0x232e315aa30>"
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Pyspark Groupby and Aggregate function. \n",
    "\n",
    "from pyspark.sql import SparkSession\n",
    "\n",
    "spark = SparkSession.builder.appName(\"aggregate\").getOrCreate()\n",
    "spark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "id": "36c384e6",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-01T07:09:33.948949Z",
     "start_time": "2023-02-01T07:09:33.819961Z"
    }
   },
   "outputs": [],
   "source": [
    "# loading the dataset\n",
    "df_pyspark = spark.read.csv(\"./cyberbullying_tweets_test.csv\", header = True, inferSchema=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "id": "b372051d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-01T07:09:39.320785Z",
     "start_time": "2023-02-01T07:09:39.308787Z"
    }
   },
   "outputs": [],
   "source": [
    "# Cast gpt_labels from integer type to float type\n",
    "df2_pyspark2 = df_pyspark.withColumn(\"gpt_labels\", \n",
    "                                  df_pyspark[\"gpt_labels\"]\n",
    "                                  .cast('int'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "id": "14fef90b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-01T07:09:39.807476Z",
     "start_time": "2023-02-01T07:09:39.767955Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+------------------+----------+-----------+\n",
      "|          tweet_text|cyberbullying_type|gpt_labels|Keep/ignore|\n",
      "+--------------------+------------------+----------+-----------+\n",
      "|In other words #k...| not_cyberbullying|         0|       keep|\n",
      "|Why is #aussietv ...| not_cyberbullying|         0|       keep|\n",
      "|@XochitlSuckkks a...| not_cyberbullying|         0|       keep|\n",
      "|@Jason_Gio meh. :...| not_cyberbullying|         0|       keep|\n",
      "|@RudhoeEnglish Th...| not_cyberbullying|         0|       keep|\n",
      "+--------------------+------------------+----------+-----------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df2_pyspark2.show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "id": "23fa3c8f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-01T07:09:43.574982Z",
     "start_time": "2023-02-01T07:09:43.568986Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- tweet_text: string (nullable = true)\n",
      " |-- cyberbullying_type: string (nullable = true)\n",
      " |-- gpt_labels: integer (nullable = true)\n",
      " |-- Keep/ignore: string (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df2_pyspark2.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "id": "5aa9c250",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-01T07:11:19.569579Z",
     "start_time": "2023-02-01T07:11:19.153948Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+---------------+\n",
      "|          tweet_text|sum(gpt_labels)|\n",
      "+--------------------+---------------+\n",
      "|Stick to your day...|           null|\n",
      "|RT @PSogeco: Litt...|           null|\n",
      "|RT @Oh_tobs: If t...|           null|\n",
      "|@ShavaneBandoo mi...|           null|\n",
      "|It's so obvious t...|           null|\n",
      "+--------------------+---------------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "## Groupby\n",
    "\n",
    "# It is also kinda filter.... Group to find the maximum gpt_labels with same tweet... Unfortunately there isn't any\n",
    "df2_pyspark2.groupby(\"tweet_text\").sum().show(5)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "id": "115fe492",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-01T07:11:07.578541Z",
     "start_time": "2023-02-01T07:11:07.237950Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+---------------+\n",
      "|  cyberbullying_type|min(gpt_labels)|\n",
      "+--------------------+---------------+\n",
      "| gay is used as a...|           null|\n",
      "| parts of the kor...|           null|\n",
      "| that's a winning...|           null|\n",
      "| I guess not. Bec...|           null|\n",
      "| truly incredible...|           null|\n",
      "+--------------------+---------------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df2_pyspark2.groupBy(\"cyberbullying_type\").min().show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "id": "0bd48bd9",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-01T07:12:01.253003Z",
     "start_time": "2023-02-01T07:12:00.859231Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+---------------+\n",
      "|  cyberbullying_type|avg(gpt_labels)|\n",
      "+--------------------+---------------+\n",
      "| gay is used as a...|           null|\n",
      "| parts of the kor...|           null|\n",
      "| that's a winning...|           null|\n",
      "| I guess not. Bec...|           null|\n",
      "| truly incredible...|           null|\n",
      "+--------------------+---------------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df2_pyspark2.groupBy(\"cyberbullying_type\").mean().show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "id": "a1adfd1e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-01T07:12:18.265098Z",
     "start_time": "2023-02-01T07:12:17.866000Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+-----+\n",
      "|  cyberbullying_type|count|\n",
      "+--------------------+-----+\n",
      "| gay is used as a...|    1|\n",
      "| parts of the kor...|    1|\n",
      "| that's a winning...|    1|\n",
      "| I guess not. Bec...|    1|\n",
      "| truly incredible...|    1|\n",
      "+--------------------+-----+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df2_pyspark2.groupBy(\"cyberbullying_type\").count().show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "id": "02588dae",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-01T07:14:48.053922Z",
     "start_time": "2023-02-01T07:14:47.815880Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+---------------+\n",
      "|         Keep/ignore|max(gpt_labels)|\n",
      "+--------------------+---------------+\n",
      "| kazakhstan etc. ...|           null|\n",
      "|             consent|           null|\n",
      "|             wake up|           null|\n",
      "|Nonsense............|           null|\n",
      "| or weight mortif...|           null|\n",
      "+--------------------+---------------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df2_pyspark2.groupBy(\"Keep/ignore\").max().show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "id": "021439b8",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-01T07:13:13.888911Z",
     "start_time": "2023-02-01T07:13:13.766393Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------------+\n",
      "|sum(gpt_labels)|\n",
      "+---------------+\n",
      "|           1330|\n",
      "+---------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# It will sum up all the all the gpt_labels\n",
    "df2_pyspark2.agg({\"gpt_labels\": \"sum\"}).show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fa4dde38",
   "metadata": {},
   "source": [
    "# Examples of PySpark ML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "id": "2c49aead",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-01T07:23:28.207287Z",
     "start_time": "2023-02-01T07:23:28.191289Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "            <div>\n",
       "                <p><b>SparkSession - in-memory</b></p>\n",
       "                \n",
       "        <div>\n",
       "            <p><b>SparkContext</b></p>\n",
       "\n",
       "            <p><a href=\"http://DESKTOP-Q24DOVF:4040\">Spark UI</a></p>\n",
       "\n",
       "            <dl>\n",
       "              <dt>Version</dt>\n",
       "                <dd><code>v3.3.1</code></dd>\n",
       "              <dt>Master</dt>\n",
       "                <dd><code>local[*]</code></dd>\n",
       "              <dt>AppName</dt>\n",
       "                <dd><code>Practice</code></dd>\n",
       "            </dl>\n",
       "        </div>\n",
       "        \n",
       "            </div>\n",
       "        "
      ],
      "text/plain": [
       "<pyspark.sql.session.SparkSession at 0x232e315aa30>"
      ]
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Pyspark Groupby and Aggregate function. \n",
    "\n",
    "from pyspark.sql import SparkSession\n",
    "\n",
    "spark = SparkSession.builder.appName(\"Missing\").getOrCreate()\n",
    "spark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "id": "edaceabd",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-01T07:24:22.438472Z",
     "start_time": "2023-02-01T07:24:22.323956Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+---+----------+------+\n",
      "|     Name|age|Experience|Salary|\n",
      "+---------+---+----------+------+\n",
      "|    Krish| 31|        10| 30000|\n",
      "|Sudhanshu| 30|         8| 25000|\n",
      "|    Sunny| 29|         4| 20000|\n",
      "|     Paul| 24|         3| 20000|\n",
      "|   Harsha| 21|         1| 15000|\n",
      "+---------+---+----------+------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "## Read the dataset\n",
    "\n",
    "training = spark.read.csv(\"test1.csv\", header=True, inferSchema=True)\n",
    "training.show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "id": "4d4c99a2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-01T07:29:39.391608Z",
     "start_time": "2023-02-01T07:29:39.368608Z"
    }
   },
   "outputs": [],
   "source": [
    "# In pyspark we dont do like scikit learn, we group together all our dependent features together in one column and then \n",
    "# we perform further processing on top of that. \n",
    "# like [\"age, experience\"] these are 2 independent features. \n",
    "# to group features together in pyspark we use technique called vector assembler. \n",
    "\n",
    "from pyspark.ml.feature import VectorAssembler\n",
    "\n",
    "featureassembler = VectorAssembler(inputCols= [\"age\", \"Experience\"], outputCol=\"Independent Features\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "id": "73761a87",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-01T07:30:10.579488Z",
     "start_time": "2023-02-01T07:30:10.501488Z"
    }
   },
   "outputs": [],
   "source": [
    "output = featureassembler.transform(training)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "id": "6f6545b2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-01T07:31:55.277518Z",
     "start_time": "2023-02-01T07:31:55.229988Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+---+----------+------+--------------------+\n",
      "|     Name|age|Experience|Salary|Independent Features|\n",
      "+---------+---+----------+------+--------------------+\n",
      "|    Krish| 31|        10| 30000|         [31.0,10.0]|\n",
      "|Sudhanshu| 30|         8| 25000|          [30.0,8.0]|\n",
      "|    Sunny| 29|         4| 20000|          [29.0,4.0]|\n",
      "|     Paul| 24|         3| 20000|          [24.0,3.0]|\n",
      "|   Harsha| 21|         1| 15000|          [21.0,1.0]|\n",
      "|  Shubham| 23|         2| 18000|          [23.0,2.0]|\n",
      "+---------+---+----------+------+--------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "output.show(6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "id": "114079e8",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-01T07:31:15.357237Z",
     "start_time": "2023-02-01T07:31:15.353087Z"
    }
   },
   "outputs": [],
   "source": [
    "# in short i have combined age and experience column to get one indepent feature .\n",
    "# Now this new column will be my input feature and salary will be my dependent feature which we will be predicting through\n",
    "# ml algo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "id": "04085773",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-01T07:33:30.250426Z",
     "start_time": "2023-02-01T07:33:30.185258Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+------+\n",
      "|Independent Features|Salary|\n",
      "+--------------------+------+\n",
      "|         [31.0,10.0]| 30000|\n",
      "|          [30.0,8.0]| 25000|\n",
      "|          [29.0,4.0]| 20000|\n",
      "|          [24.0,3.0]| 20000|\n",
      "|          [21.0,1.0]| 15000|\n",
      "+--------------------+------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "finalized_data = output.select(\"Independent Features\", \"Salary\")\n",
    "finalized_data.show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "id": "6e0c28f1",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-01T07:37:10.608927Z",
     "start_time": "2023-02-01T07:37:10.165553Z"
    }
   },
   "outputs": [],
   "source": [
    "from pyspark.ml.regression import LinearRegression\n",
    "\n",
    "# spliting the data\n",
    "train_data, test_data = finalized_data.randomSplit([0.75, 0.25])\n",
    "# fitting the model... 2 inputs are given -> input features and predicted variable.  \n",
    "regressor = LinearRegression(featuresCol = \"Independent Features\", labelCol= \"Salary\")\n",
    "regressor=regressor.fit(train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "id": "b92039f3",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-01T07:37:24.525487Z",
     "start_time": "2023-02-01T07:37:24.494952Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DenseVector([28.4757, 1271.3568])"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Coefficients\n",
    "\n",
    "regressor.coefficients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "id": "60ab483a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-01T07:38:03.642946Z",
     "start_time": "2023-02-01T07:38:03.637942Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14299.832495812996"
      ]
     },
     "execution_count": 182,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Intercepts\n",
    "\n",
    "regressor.intercept"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "id": "ff3f28f3",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-01T07:38:35.620477Z",
     "start_time": "2023-02-01T07:38:35.553474Z"
    }
   },
   "outputs": [],
   "source": [
    "## Prediction\n",
    "\n",
    "pred_results = regressor.evaluate(test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "id": "68273474",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-01T07:38:49.058468Z",
     "start_time": "2023-02-01T07:38:48.987944Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+------+------------------+\n",
      "|Independent Features|Salary|        prediction|\n",
      "+--------------------+------+------------------+\n",
      "|         [31.0,10.0]| 30000|27896.147403685147|\n",
      "+--------------------+------+------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "pred_results.predictions.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "id": "efaa8093",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-01T07:40:40.991492Z",
     "start_time": "2023-02-01T07:40:40.979982Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2103.852596314853, 4426195.747020748)"
      ]
     },
     "execution_count": 189,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_results.meanAbsoluteError, pred_results.meanSquaredError"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "75d86d09",
   "metadata": {},
   "source": [
    "# From here starts the tutorial of databrick"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "be956482",
   "metadata": {},
   "source": [
    "# Databricks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b4ddb76a",
   "metadata": {},
   "outputs": [],
   "source": [
    "file = \"/FileStore/tables/tips.csv\"\n",
    "\n",
    "df = spark.read.csv(file, header=True, inferSchema=True)\n",
    "df.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b8fc6331",
   "metadata": {},
   "outputs": [],
   "source": [
    "#### Droping the columns\n",
    "df = df.drop(\"_c7\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3e2392e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fa5847c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9326b783",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aa37a201",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Handling Categorical features\n",
    "\n",
    "from pyspark.ml.feature import StringIndexer\n",
    "\n",
    "indexer = StringIndexer(inputCol = \"sex\", outputCol = \"sex_indexed\")\n",
    "df_r = indexer.fit(df).transform(df)\n",
    "df_r.show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fb97f958",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Above line of code was for the conversion of one column\n",
    "# Now we will convert multiple columns\n",
    "\n",
    "indexer = StringIndexer(inputCols=[\"smoker\", \"day\", \"time\"], outputCols=[\"smoker_indexed\", \"day_indexed\", \"time_indexed\"])\n",
    "df_r = indexer.fit(df_r).transform(df_r)\n",
    "df_r.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "90906202",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.feature import VectorAssembler\n",
    "\n",
    "featureAssembler = VectorAssembler(inputCols=[\"tip\", \"size\", \"sex_indexed\", \"smoker_indexed\", \"day_indexed\", \"time_indexed\"],\n",
    "               outputCol = \"Independent_features\"\n",
    "               )\n",
    "output = featureAssembler.transform(df_r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ad9d30fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "output.show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8774e87a",
   "metadata": {},
   "outputs": [],
   "source": [
    "output.select(\"Independent_features\").show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "07dee9f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "finalized_data = output.select(\"Independent_features\", \"total_bill\")\n",
    "finalized_data.show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9985f188",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ok the preprocessing steps are done. now we just need to import the model and fit the linear regression model on top of \n",
    "# the preprocessed data. \n",
    "\n",
    "from pyspark.ml.regression import LinearRegression\n",
    "\n",
    "train_data, test_data = finalized_data.randomSplit([0.75, 0.25])\n",
    "regressor = LinearRegression(featuresCol=\"Independent_features\", labelCol=\"total_bill\")\n",
    "regressor = regressor.fit(train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "559e2a07",
   "metadata": {},
   "outputs": [],
   "source": [
    "regressor.coefficients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b3c82612",
   "metadata": {},
   "outputs": [],
   "source": [
    "regressor.intercept"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "32cef943",
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_results = regressor.evaluate(test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "045c3722",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Final Comparison\n",
    "pred_results.predictions.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dfcd9a6c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Performance Metrics\n",
    "pred_results.r2, pred_results.meanAbsoluteError, pred_results.meanSquaredError"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "42b57aaf",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f2f85a7c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8fc41d09",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "63a86f43",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}