British airways data science project .ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4f1fba13",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ae4fc863",
   "metadata": {},
   "outputs": [],
   "source": [
    "base_url = \"https://www.airlinequality.com/airline-reviews/british-airways\"\n",
    "pages = 10\n",
    "page_size = 100\n",
    "\n",
    "reviews= []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1a791c4b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scraping page 1\n",
      "   ---> 100 total reviews\n",
      "Scraping page 2\n",
      "   ---> 200 total reviews\n",
      "Scraping page 3\n",
      "   ---> 300 total reviews\n",
      "Scraping page 4\n",
      "   ---> 400 total reviews\n",
      "Scraping page 5\n",
      "   ---> 500 total reviews\n",
      "Scraping page 6\n",
      "   ---> 600 total reviews\n",
      "Scraping page 7\n",
      "   ---> 700 total reviews\n",
      "Scraping page 8\n",
      "   ---> 800 total reviews\n",
      "Scraping page 9\n",
      "   ---> 900 total reviews\n",
      "Scraping page 10\n",
      "   ---> 1000 total reviews\n"
     ]
    }
   ],
   "source": [
    "base_url = \"https://www.airlinequality.com/airline-reviews/british-airways\"\n",
    "pages = 10\n",
    "page_size = 100\n",
    "\n",
    "reviews = []\n",
    "\n",
    "# for i in range(1, pages + 1):\n",
    "for i in range(1, pages + 1):\n",
    "\n",
    "    print(f\"Scraping page {i}\")\n",
    "\n",
    "    # Create URL to collect links from paginated data\n",
    "    url = f\"{base_url}/page/{i}/?sortby=post_date%3ADesc&pagesize={page_size}\"\n",
    "\n",
    "    # Collect HTML data from this page\n",
    "    response = requests.get(url)\n",
    "\n",
    "    # Parse content\n",
    "    content = response.content\n",
    "    parsed_content = BeautifulSoup(content, 'html.parser')\n",
    "    for para in parsed_content.find_all(\"div\", {\"class\": \"text_content\"}):\n",
    "        reviews.append(para.get_text())\n",
    "    \n",
    "    print(f\"   ---> {len(reviews)} total reviews\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "26946bfc",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'df' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[4], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mdf\u001b[49m\u001b[38;5;241m.\u001b[39mto_csv(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdata/BA_reviews.csv\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
      "\u001b[0;31mNameError\u001b[0m: name 'df' is not defined"
     ]
    }
   ],
   "source": [
    "df.to_csv(\"data/BA_reviews.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "dde407a7",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'pd' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[1], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m df \u001b[38;5;241m=\u001b[39m \u001b[43mpd\u001b[49m\u001b[38;5;241m.\u001b[39mDataFrame()\n\u001b[1;32m      2\u001b[0m df[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mreviews\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m reviews\n\u001b[1;32m      3\u001b[0m df\u001b[38;5;241m.\u001b[39mhead()\n",
      "\u001b[0;31mNameError\u001b[0m: name 'pd' is not defined"
     ]
    }
   ],
   "source": [
    "df = pd.DataFrame()\n",
    "df[\"reviews\"] = reviews\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "093a65a1",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'df' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[2], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mdf\u001b[49m\u001b[38;5;241m.\u001b[39mto_csv(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdata/BA_reviews.csv\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
      "\u001b[0;31mNameError\u001b[0m: name 'df' is not defined"
     ]
    }
   ],
   "source": [
    "df.to_csv(\"data/BA_reviews.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "dd708217",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'df' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[3], line 4\u001b[0m\n\u001b[1;32m      2\u001b[0m Df \u001b[38;5;241m=\u001b[39m pd\u001b[38;5;241m.\u001b[39mread_csv(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m/Users/ayodejioyesanya/Desktop/BA_reviews_20231121_123616.csv\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m      3\u001b[0m Df\n\u001b[0;32m----> 4\u001b[0m docs \u001b[38;5;241m=\u001b[39m \u001b[43mdf\u001b[49m\u001b[38;5;241m.\u001b[39mreviews\u001b[38;5;241m.\u001b[39mtolist()\n\u001b[1;32m      5\u001b[0m docs [\u001b[38;5;241m0\u001b[39m]\n",
      "\u001b[0;31mNameError\u001b[0m: name 'df' is not defined"
     ]
    }
   ],
   "source": [
    "import pandas as pd \n",
    "Df = pd.read_csv(\"/Users/ayodejioyesanya/Desktop/BA_reviews_20231121_123616.csv\")\n",
    "Df\n",
    "docs = df.reviews.tolist()\n",
    "docs [0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1381fb0d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'✅ Trip Verified |  4 Hours before takeoff we received a Mail stating a cryptic message that there are disruptions to be expected as there is a limit on how many planes can leave at the same time. So did the capacity of the Heathrow Airport really hit British Airways by surprise, 4h before departure? Anyhow - we took the one hour delay so what - but then we have been forced to check in our Hand luggage. I travel only with hand luggage to avoid waiting for the ultra slow processing of the checked in luggage. Overall 2h later at home than planed, with really no reason, just due to incompetent people. Service level far worse then Ryanair and triple the price. Really never again. Thanks for nothing.'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd \n",
    "Df = pd.read_csv(\"/Users/ayodejioyesanya/Desktop/BA_reviews_20231121_123616.csv\")\n",
    "Df\n",
    "docs = Df.reviews.tolist()\n",
    "docs [0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "45f21b30",
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'data/BA_reviews.csv'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[5], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m df \u001b[38;5;241m=\u001b[39m \u001b[43mpd\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread_csv\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mdata/BA_reviews.csv\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m      2\u001b[0m \u001b[38;5;28mprint\u001b[39m(df\u001b[38;5;241m.\u001b[39mhead())\n\u001b[1;32m      3\u001b[0m df[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mreviews\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m df[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mreviews\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;241m.\u001b[39mstr\u001b[38;5;241m.\u001b[39mreplace(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m✅ Trip Verified\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m'\u001b[39m)\n",
      "File \u001b[0;32m~/anaconda3/lib/python3.10/site-packages/pandas/io/parsers/readers.py:948\u001b[0m, in \u001b[0;36mread_csv\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, date_format, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, encoding_errors, dialect, on_bad_lines, delim_whitespace, low_memory, memory_map, float_precision, storage_options, dtype_backend)\u001b[0m\n\u001b[1;32m    935\u001b[0m kwds_defaults \u001b[38;5;241m=\u001b[39m _refine_defaults_read(\n\u001b[1;32m    936\u001b[0m     dialect,\n\u001b[1;32m    937\u001b[0m     delimiter,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    944\u001b[0m     dtype_backend\u001b[38;5;241m=\u001b[39mdtype_backend,\n\u001b[1;32m    945\u001b[0m )\n\u001b[1;32m    946\u001b[0m kwds\u001b[38;5;241m.\u001b[39mupdate(kwds_defaults)\n\u001b[0;32m--> 948\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_read\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfilepath_or_buffer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkwds\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/anaconda3/lib/python3.10/site-packages/pandas/io/parsers/readers.py:611\u001b[0m, in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m    608\u001b[0m _validate_names(kwds\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnames\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m))\n\u001b[1;32m    610\u001b[0m \u001b[38;5;66;03m# Create the parser.\u001b[39;00m\n\u001b[0;32m--> 611\u001b[0m parser \u001b[38;5;241m=\u001b[39m \u001b[43mTextFileReader\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfilepath_or_buffer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwds\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    613\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m chunksize \u001b[38;5;129;01mor\u001b[39;00m iterator:\n\u001b[1;32m    614\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m parser\n",
      "File \u001b[0;32m~/anaconda3/lib/python3.10/site-packages/pandas/io/parsers/readers.py:1448\u001b[0m, in \u001b[0;36mTextFileReader.__init__\u001b[0;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[1;32m   1445\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moptions[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhas_index_names\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m kwds[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhas_index_names\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[1;32m   1447\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mhandles: IOHandles \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m-> 1448\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_engine \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_make_engine\u001b[49m\u001b[43m(\u001b[49m\u001b[43mf\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mengine\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/anaconda3/lib/python3.10/site-packages/pandas/io/parsers/readers.py:1705\u001b[0m, in \u001b[0;36mTextFileReader._make_engine\u001b[0;34m(self, f, engine)\u001b[0m\n\u001b[1;32m   1703\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mb\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m mode:\n\u001b[1;32m   1704\u001b[0m         mode \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mb\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m-> 1705\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mhandles \u001b[38;5;241m=\u001b[39m \u001b[43mget_handle\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1706\u001b[0m \u001b[43m    \u001b[49m\u001b[43mf\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1707\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmode\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1708\u001b[0m \u001b[43m    \u001b[49m\u001b[43mencoding\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43moptions\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mencoding\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1709\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcompression\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43moptions\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcompression\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1710\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmemory_map\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43moptions\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mmemory_map\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1711\u001b[0m \u001b[43m    \u001b[49m\u001b[43mis_text\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mis_text\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1712\u001b[0m \u001b[43m    \u001b[49m\u001b[43merrors\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43moptions\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mencoding_errors\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mstrict\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1713\u001b[0m \u001b[43m    \u001b[49m\u001b[43mstorage_options\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43moptions\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mstorage_options\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1714\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1715\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mhandles \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m   1716\u001b[0m f \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mhandles\u001b[38;5;241m.\u001b[39mhandle\n",
      "File \u001b[0;32m~/anaconda3/lib/python3.10/site-packages/pandas/io/common.py:863\u001b[0m, in \u001b[0;36mget_handle\u001b[0;34m(path_or_buf, mode, encoding, compression, memory_map, is_text, errors, storage_options)\u001b[0m\n\u001b[1;32m    858\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(handle, \u001b[38;5;28mstr\u001b[39m):\n\u001b[1;32m    859\u001b[0m     \u001b[38;5;66;03m# Check whether the filename is to be opened in binary mode.\u001b[39;00m\n\u001b[1;32m    860\u001b[0m     \u001b[38;5;66;03m# Binary mode does not support 'encoding' and 'newline'.\u001b[39;00m\n\u001b[1;32m    861\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m ioargs\u001b[38;5;241m.\u001b[39mencoding \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mb\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m ioargs\u001b[38;5;241m.\u001b[39mmode:\n\u001b[1;32m    862\u001b[0m         \u001b[38;5;66;03m# Encoding\u001b[39;00m\n\u001b[0;32m--> 863\u001b[0m         handle \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mopen\u001b[39;49m\u001b[43m(\u001b[49m\n\u001b[1;32m    864\u001b[0m \u001b[43m            \u001b[49m\u001b[43mhandle\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    865\u001b[0m \u001b[43m            \u001b[49m\u001b[43mioargs\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmode\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    866\u001b[0m \u001b[43m            \u001b[49m\u001b[43mencoding\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mioargs\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mencoding\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    867\u001b[0m \u001b[43m            \u001b[49m\u001b[43merrors\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43merrors\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    868\u001b[0m \u001b[43m            \u001b[49m\u001b[43mnewline\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m    869\u001b[0m \u001b[43m        \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    870\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    871\u001b[0m         \u001b[38;5;66;03m# Binary mode\u001b[39;00m\n\u001b[1;32m    872\u001b[0m         handle \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mopen\u001b[39m(handle, ioargs\u001b[38;5;241m.\u001b[39mmode)\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'data/BA_reviews.csv'"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv(\"data/BA_reviews.csv\")\n",
    "print(df.head())\n",
    "df['reviews'] = df['reviews'].str.replace('✅ Trip Verified', '')\n",
    "print(df.head())\n",
    "df.to_csv(\"data/cleaned_BA_reviews.csv\", index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "fd97e2a5",
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "unterminated string literal (detected at line 1) (2868649243.py, line 1)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;36m  Cell \u001b[0;32mIn[6], line 1\u001b[0;36m\u001b[0m\n\u001b[0;31m    df = pd.read_csv(\"\"/Users/ayodejioyesanya/Desktop/BA_reviews_20231121_123616.csv\")\u001b[0m\n\u001b[0m                                                                                    ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m unterminated string literal (detected at line 1)\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv(\"\"/Users/ayodejioyesanya/Desktop/BA_reviews_20231121_123616.csv\")\n",
    "print(df.head())\n",
    "df['reviews'] = df['reviews'].str.replace('✅ Trip Verified', '')\n",
    "print(df.head())\n",
    "df.to_csv(\"data/cleaned_BA_reviews_20231121_123616.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "9f953c14",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Unnamed: 0                                            reviews\n",
      "0           0  ✅ Trip Verified |  4 Hours before takeoff we r...\n",
      "1           1  ✅ Trip Verified |  I recently had a delay on B...\n",
      "2           2  Not Verified |  Boarded on time, but it took a...\n",
      "3           3  ✅ Trip Verified |  5 days before the flight, w...\n",
      "4           4  Not Verified |  \\r\\nWe traveled to Lisbon for ...\n",
      "   Unnamed: 0                                            reviews\n",
      "0           0   |  4 Hours before takeoff we received a Mail ...\n",
      "1           1   |  I recently had a delay on British Airways ...\n",
      "2           2  Not Verified |  Boarded on time, but it took a...\n",
      "3           3   |  5 days before the flight, we were advised ...\n",
      "4           4  Not Verified |  \\r\\nWe traveled to Lisbon for ...\n"
     ]
    },
    {
     "ename": "OSError",
     "evalue": "Cannot save file into a non-existent directory: 'data'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mOSError\u001b[0m                                   Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[7], line 5\u001b[0m\n\u001b[1;32m      3\u001b[0m df[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mreviews\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m df[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mreviews\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;241m.\u001b[39mstr\u001b[38;5;241m.\u001b[39mreplace(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m✅ Trip Verified\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m      4\u001b[0m \u001b[38;5;28mprint\u001b[39m(df\u001b[38;5;241m.\u001b[39mhead())\n\u001b[0;32m----> 5\u001b[0m \u001b[43mdf\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mto_csv\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mdata/cleaned_BA_reviews_20231121_123616.csv\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mindex\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/anaconda3/lib/python3.10/site-packages/pandas/core/generic.py:3902\u001b[0m, in \u001b[0;36mNDFrame.to_csv\u001b[0;34m(self, path_or_buf, sep, na_rep, float_format, columns, header, index, index_label, mode, encoding, compression, quoting, quotechar, lineterminator, chunksize, date_format, doublequote, escapechar, decimal, errors, storage_options)\u001b[0m\n\u001b[1;32m   3891\u001b[0m df \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(\u001b[38;5;28mself\u001b[39m, ABCDataFrame) \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mto_frame()\n\u001b[1;32m   3893\u001b[0m formatter \u001b[38;5;241m=\u001b[39m DataFrameFormatter(\n\u001b[1;32m   3894\u001b[0m     frame\u001b[38;5;241m=\u001b[39mdf,\n\u001b[1;32m   3895\u001b[0m     header\u001b[38;5;241m=\u001b[39mheader,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   3899\u001b[0m     decimal\u001b[38;5;241m=\u001b[39mdecimal,\n\u001b[1;32m   3900\u001b[0m )\n\u001b[0;32m-> 3902\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mDataFrameRenderer\u001b[49m\u001b[43m(\u001b[49m\u001b[43mformatter\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mto_csv\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   3903\u001b[0m \u001b[43m    \u001b[49m\u001b[43mpath_or_buf\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3904\u001b[0m \u001b[43m    \u001b[49m\u001b[43mlineterminator\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlineterminator\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3905\u001b[0m \u001b[43m    \u001b[49m\u001b[43msep\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msep\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3906\u001b[0m \u001b[43m    \u001b[49m\u001b[43mencoding\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mencoding\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3907\u001b[0m \u001b[43m    \u001b[49m\u001b[43merrors\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43merrors\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3908\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcompression\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcompression\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3909\u001b[0m \u001b[43m    \u001b[49m\u001b[43mquoting\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mquoting\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3910\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcolumns\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcolumns\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3911\u001b[0m \u001b[43m    \u001b[49m\u001b[43mindex_label\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mindex_label\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3912\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmode\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmode\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3913\u001b[0m \u001b[43m    \u001b[49m\u001b[43mchunksize\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunksize\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3914\u001b[0m \u001b[43m    \u001b[49m\u001b[43mquotechar\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mquotechar\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3915\u001b[0m \u001b[43m    \u001b[49m\u001b[43mdate_format\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdate_format\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3916\u001b[0m \u001b[43m    \u001b[49m\u001b[43mdoublequote\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdoublequote\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3917\u001b[0m \u001b[43m    \u001b[49m\u001b[43mescapechar\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mescapechar\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3918\u001b[0m \u001b[43m    \u001b[49m\u001b[43mstorage_options\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstorage_options\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3919\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/anaconda3/lib/python3.10/site-packages/pandas/io/formats/format.py:1152\u001b[0m, in \u001b[0;36mDataFrameRenderer.to_csv\u001b[0;34m(self, path_or_buf, encoding, sep, columns, index_label, mode, compression, quoting, quotechar, lineterminator, chunksize, date_format, doublequote, escapechar, errors, storage_options)\u001b[0m\n\u001b[1;32m   1131\u001b[0m     created_buffer \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m   1133\u001b[0m csv_formatter \u001b[38;5;241m=\u001b[39m CSVFormatter(\n\u001b[1;32m   1134\u001b[0m     path_or_buf\u001b[38;5;241m=\u001b[39mpath_or_buf,\n\u001b[1;32m   1135\u001b[0m     lineterminator\u001b[38;5;241m=\u001b[39mlineterminator,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1150\u001b[0m     formatter\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfmt,\n\u001b[1;32m   1151\u001b[0m )\n\u001b[0;32m-> 1152\u001b[0m \u001b[43mcsv_formatter\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msave\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1154\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m created_buffer:\n\u001b[1;32m   1155\u001b[0m     \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(path_or_buf, StringIO)\n",
      "File \u001b[0;32m~/anaconda3/lib/python3.10/site-packages/pandas/io/formats/csvs.py:247\u001b[0m, in \u001b[0;36mCSVFormatter.save\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    243\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m    244\u001b[0m \u001b[38;5;124;03mCreate the writer & save.\u001b[39;00m\n\u001b[1;32m    245\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m    246\u001b[0m \u001b[38;5;66;03m# apply compression and byte/text conversion\u001b[39;00m\n\u001b[0;32m--> 247\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[43mget_handle\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    248\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfilepath_or_buffer\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    249\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmode\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    250\u001b[0m \u001b[43m    \u001b[49m\u001b[43mencoding\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mencoding\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    251\u001b[0m \u001b[43m    \u001b[49m\u001b[43merrors\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43merrors\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    252\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcompression\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcompression\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    253\u001b[0m \u001b[43m    \u001b[49m\u001b[43mstorage_options\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstorage_options\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    254\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mas\u001b[39;00m handles:\n\u001b[1;32m    255\u001b[0m     \u001b[38;5;66;03m# Note: self.encoding is irrelevant here\u001b[39;00m\n\u001b[1;32m    256\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mwriter \u001b[38;5;241m=\u001b[39m csvlib\u001b[38;5;241m.\u001b[39mwriter(\n\u001b[1;32m    257\u001b[0m         handles\u001b[38;5;241m.\u001b[39mhandle,\n\u001b[1;32m    258\u001b[0m         lineterminator\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlineterminator,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    263\u001b[0m         quotechar\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mquotechar,\n\u001b[1;32m    264\u001b[0m     )\n\u001b[1;32m    266\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_save()\n",
      "File \u001b[0;32m~/anaconda3/lib/python3.10/site-packages/pandas/io/common.py:739\u001b[0m, in \u001b[0;36mget_handle\u001b[0;34m(path_or_buf, mode, encoding, compression, memory_map, is_text, errors, storage_options)\u001b[0m\n\u001b[1;32m    737\u001b[0m \u001b[38;5;66;03m# Only for write methods\u001b[39;00m\n\u001b[1;32m    738\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mr\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m mode \u001b[38;5;129;01mand\u001b[39;00m is_path:\n\u001b[0;32m--> 739\u001b[0m     \u001b[43mcheck_parent_directory\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mstr\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mhandle\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    741\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m compression:\n\u001b[1;32m    742\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m compression \u001b[38;5;241m!=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mzstd\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m    743\u001b[0m         \u001b[38;5;66;03m# compression libraries do not like an explicit text-mode\u001b[39;00m\n",
      "File \u001b[0;32m~/anaconda3/lib/python3.10/site-packages/pandas/io/common.py:604\u001b[0m, in \u001b[0;36mcheck_parent_directory\u001b[0;34m(path)\u001b[0m\n\u001b[1;32m    602\u001b[0m parent \u001b[38;5;241m=\u001b[39m Path(path)\u001b[38;5;241m.\u001b[39mparent\n\u001b[1;32m    603\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m parent\u001b[38;5;241m.\u001b[39mis_dir():\n\u001b[0;32m--> 604\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mOSError\u001b[39;00m(\u001b[38;5;124mrf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCannot save file into a non-existent directory: \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mparent\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
      "\u001b[0;31mOSError\u001b[0m: Cannot save file into a non-existent directory: 'data'"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv(\"/Users/ayodejioyesanya/Desktop/BA_reviews_20231121_123616.csv\")\n",
    "print(df.head())\n",
    "df['reviews'] = df['reviews'].str.replace('✅ Trip Verified', '')\n",
    "print(df.head())\n",
    "df.to_csv(\"data/cleaned_BA_reviews_20231121_123616.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "dde89cc8",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(\"/Users/ayodejioyesanya/Desktop/cleaned_BA_reviews_20231121_123616.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d89f4692",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd \n",
    "Df = pd.read_csv(\"/Users/ayodejioyesanya/Desktop/BA_reviews_20231121_123616.csv\")\n",
    "Df\n",
    "docs = Df.reviews.tolist()\n",
    "docs [0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d3e2f4ce",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' |  4 Hours before takeoff we received a Mail stating a cryptic message that there are disruptions to be expected as there is a limit on how many planes can leave at the same time. So did the capacity of the Heathrow Airport really hit British Airways by surprise, 4h before departure? Anyhow - we took the one hour delay so what - but then we have been forced to check in our Hand luggage. I travel only with hand luggage to avoid waiting for the ultra slow processing of the checked in luggage. Overall 2h later at home than planed, with really no reason, just due to incompetent people. Service level far worse then Ryanair and triple the price. Really never again. Thanks for nothing.'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd \n",
    "Df = pd.read_csv(\"/Users/ayodejioyesanya/Desktop/cleaned_BA_reviews_20231121_123616.csv\")\n",
    "Df\n",
    "docs = Df.reviews.tolist()\n",
    "docs [0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0cd2837a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.decomposition import LatentDirichletAllocation\n",
    "from sklearn.manifold import TSNE\n",
    "from umap import UMAP\n",
    "from wordcloud import WordCloud\n",
    "from nltk.sentiment import SentimentIntensityAnalyzer\n",
    "# Assuming you have a function 'use_embed' for USE embeddings\n",
    "# import tensorflow_hub as hub\n",
    "\n",
    "# Function to perform clustering\n",
    "def perform_clustering(embeddings, n_clusters=3):\n",
    "    kmeans = KMeans(n_clusters=n_clusters, n_init=10, random_state=42)\n",
    "    cluster_labels = kmeans.fit_predict(embeddings)\n",
    "    return cluster_labels\n",
    "\n",
    "# Function to extract keywords using TF-IDF\n",
    "def extract_keywords(reviews, n_keywords=10):\n",
    "    vectorizer = TfidfVectorizer(max_features=n_keywords)\n",
    "    tfidf_matrix = vectorizer.fit_transform(reviews)\n",
    "    feature_names = vectorizer.get_feature_names_out()\n",
    "    return pd.DataFrame(tfidf_matrix.toarray(), columns=feature_names)\n",
    "\n",
    "# Function to perform topic modeling using LDA\n",
    "def perform_topic_modeling(reviews, n_topics=5):\n",
    "    vectorizer = TfidfVectorizer(max_features=5000, stop_words='english')\n",
    "    tfidf_matrix = vectorizer.fit_transform(reviews)\n",
    "\n",
    "    lda = LatentDirichletAllocation(n_components=n_topics, random_state=42)\n",
    "    lda.fit(tfidf_matrix)\n",
    "\n",
    "    return lda, vectorizer\n",
    "\n",
    "# Function to visualize word cloud\n",
    "def visualize_word_cloud(text, title):\n",
    "    wordcloud = WordCloud(width=800, height=400, background_color='white').generate(text)\n",
    "\n",
    "    plt.figure(figsize=(10, 5))\n",
    "    plt.imshow(wordcloud, interpolation='bilinear')\n",
    "    plt.axis('off')\n",
    "    plt.title(title)\n",
    "    plt.show()\n",
    "\n",
    "# Function to perform sentiment analysis using VADER\n",
    "def perform_sentiment_analysis(reviews):\n",
    "    analyzer = SentimentIntensityAnalyzer()\n",
    "    sentiments = [analyzer.polarity_scores(review)['compound'] for review in reviews]\n",
    "    return sentiments\n",
    "\n",
    "# Function to visualize t-SNE\n",
    "def visualize_tsne(embeddings, labels):\n",
    "    tsne = TSNE(n_components=2, perplexity=5, random_state=42, n_jobs=1)  # Adjusted perplexity value and set n_jobs explicitly\n",
    "    tsne_result = tsne.fit_transform(embeddings)\n",
    "\n",
    "    plt.figure(figsize=(8, 6))\n",
    "    sns.scatterplot(x=tsne_result[:, 0], y=tsne_result[:, 1], hue=labels, palette=\"viridis\")\n",
    "    plt.title('t-SNE Visualization')\n",
    "    plt.show()\n",
    "\n",
    "# Function to perform UMAP dimensionality reduction\n",
    "def visualize_umap(embeddings, labels):\n",
    "    umap_result = UMAP(n_neighbors=5, min_dist=0.3, random_state=42, n_jobs=1).fit_transform(embeddings)  # Set n_jobs explicitly\n",
    "\n",
    "    plt.figure(figsize=(8, 6))\n",
    "    sns.scatterplot(x=umap_result[:, 0], y=umap_result[:, 1], hue=labels, palette=\"viridis\")\n",
    "    plt.title('UMAP Visualization')\n",
    "    plt.show()\n",
    "\n",
    "# Example British Airways reviews\n",
    "reviews = [\n",
    "    \"London to Gothenburg. BA are getting a lot of bad press - deservedly so at present with strikes, IT glitches, baggage problems...\",\n",
    "    \"London Heathrow to Inverness. Having previously written a review about the shockingly appalling experience with BA so far this summer...\",\n",
    "    \"Heathrow to Glasgow. Again flight is delayed. It’s easier to use the train than fly with BA the staff don’t even get embarrassed anymore...\",\n",
    "    \"I was flying BA to Delhi in economy because my original flight with Swissair was cancelled. The 777 aircraft is looking old, the economy class seat is small and fairly cramped...\",\n",
    "    \"BA 2616 and 2617 return trip from Gatwick to Cagliari. Both flights on a Saturday and were full. Aircraft was an A320-200 on both legs...\",\n",
    "    \"The flight was delayed but the crew was helpful.\"\n",
    "]\n",
    "\n",
    "# Assuming 'review_embeddings' is the result of USE embeddings\n",
    "# Use your actual embedding function or method\n",
    "# import tensorflow as tf\n",
    "# use_embed = hub.load(\"https://tfhub.dev/google/universal-sentence-encoder/4\")\n",
    "# review_embeddings = use_embed(reviews).numpy()\n",
    "\n",
    "# Visualize t-SNE with modified perplexity\n",
    "visualize_tsne(review_embeddings, labels=reviews)\n",
    "\n",
    "# Perform clustering\n",
    "cluster_labels = perform_clustering(review_embeddings, n_clusters=3)\n",
    "\n",
    "# Visualize UMAP\n",
    "visualize_umap(review_embeddings, labels=cluster_labels)\n",
    "\n",
    "# Extract keywords using TF-IDF\n",
    "keywords_df = extract_keywords(reviews, n_keywords=10)\n",
    "print(\"Top Keywords:\")\n",
    "print(keywords_df)\n",
    "\n",
    "# Perform topic modeling using LDA\n",
    "lda_model, vectorizer = perform_topic_modeling(reviews)\n",
    "print(\"Top Words in Topics:\")\n",
    "for i, topic in enumerate(lda_model.components_):\n",
    "    top_words_idx = topic.argsort()[-5:][::-1]\n",
    "    top_words = [vectorizer.get_feature_names_out()[idx] for idx in top_words_idx]\n",
    "\n",
    "    print(f\"Topic {i}: {', '.join(top_words)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d887c7e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to perform clustering\n",
    "def perform_clustering(embeddings, n_clusters=3):\n",
    "    kmeans = KMeans(n_clusters=n_clusters, n_init=10, random_state=42)\n",
    "    cluster_labels = kmeans.fit_predict(embeddings)\n",
    "    return cluster_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "198457c7",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}