Web_Scrapping.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyPahgmb+Jq9jFQEu03S6n6S",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/AusafMo/Flask-Web-App-Tutorial/blob/main/Web_Scrapping.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Script"
      ],
      "metadata": {
        "id": "iOWc7hAbP6ZP"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from bs4 import BeautifulSoup\n",
        "import requests\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import re"
      ],
      "metadata": {
        "id": "Opb1RxMBP3py"
      },
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_name(soup):\n",
        "    try:\n",
        "        name = soup.find(\"span\", attrs = { \"id\":\"productTitle\"}).text.strip()\n",
        "    except AttributeError:\n",
        "        name = \"\"\n",
        "    return name\n",
        "\n",
        "\n",
        "def get_price(soup):\n",
        "  try:\n",
        "      price = soup.find(\"span\", attrs = { \"class\":\"a-price-whole\" }).text.strip()\n",
        "  except AttributeError:\n",
        "      price = \"\"\n",
        "  return price\n",
        "\n",
        "\n",
        "def get_rating(soup):\n",
        "  try:\n",
        "      rating = soup.find(\"span\", attrs = { \"class\":\"a-icon-alt\" }).text\n",
        "  except AttributeError:\n",
        "      rating = \"\"\n",
        "  return rating\n",
        "\n",
        "\n",
        "def get_reviews(soup):\n",
        "  try:\n",
        "      reviews = soup.find(\"span\", attrs = { \"id\":\"acrCustomerReviewText\" }).text.strip()\n",
        "  except AttributeError:\n",
        "      reviews = \"\"\n",
        "  return reviews\n",
        "\n",
        "\n",
        "def get_description(soup):\n",
        "  try:\n",
        "      description = soup.find(\"ul\", attrs = { \"class\":\"a-unordered-list a-vertical a-spacing-mini\" }).text.strip().split(',')\n",
        "  except AttributeError:\n",
        "    description = \"\"\n",
        "  return description\n",
        "\n",
        "\n",
        "def get_asin(soup):\n",
        "    asin = re.search(r'/[dg]p/([^/]+)', soup, flags=re.IGNORECASE)\n",
        "    if asin:\n",
        "      return asin.group(1)\n",
        "    return \"\"\n",
        "\n",
        "\n",
        "def get_manufacturer(soup):\n",
        "  try:\n",
        "    manu = soup.find(\"table\", attrs = {\"id\":\"productDetails_techSpec_section_1\"}).find('th', string=' Manufacturer ').find_next_sibling('td').text\n",
        "    manu = str(BeautifulSoup(manu, 'html.parser'))\n",
        "    manu = manu.strip()\n",
        "    manu = manu.replace(\"\\u200e\", \"\" )\n",
        "  except AttributeError:\n",
        "      manu = \"\"\n",
        "  return manu\n",
        "\n",
        "\n",
        "\n",
        "if __name__ == '__main__':\n",
        "\n",
        "  d = {\"PRODUCT URL\" : [], \"PRODUCT NAME\" : [], \"PRODUCT PRICE\" : [], \"RATING\" : [], \"NUMBER OF REVIEWS\" : [], \"DESCRIPTION\" : [], \"ASIN\" : [], \"MANUFACTURER\" : [] }\n",
        "  HEADERS = ({'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36', 'Acccept-Language': 'en-US, en; q = 0.5'})\n",
        "  all_links = []\n",
        "  \n",
        "  for x in range(1, 21):\n",
        "\n",
        "      URL = \"https://www.amazon.in/s?k=bags&page={}&crid=2M096C61O4MLT&qid=1653308124&sprefix=ba%2Caps%2C283&ref=sr_pg_1\".format(x)\n",
        "\n",
        "      webpage = requests.get(URL, headers = HEADERS)\n",
        "\n",
        "      soup = BeautifulSoup(webpage.content, \"html.parser\")\n",
        "\n",
        "      links = soup.find_all(\"a\", attrs = {'class' : 'a-link-normal s-underline-text s-underline-link-text s-link-style a-text-normal'})\n",
        "      \n",
        "      for link in links:\n",
        "            all_links.append(link.get('href'))\n",
        "\n",
        "  for link in all_links:\n",
        "      new_webpage = requests.get(\"https://amazon.in\" + link, headers = HEADERS)\n",
        "      new_soup = BeautifulSoup(new_webpage.content, 'html.parser')\n",
        "\n",
        "      d['PRODUCT URL'].append(\"https://amazon.in\" + link)\n",
        "      d['PRODUCT NAME'].append(get_name(new_soup))\n",
        "      d['PRODUCT PRICE'].append(get_price(new_soup))\n",
        "      d['RATING'].append(get_rating(new_soup))\n",
        "      d['NUMBER OF REVIEWS'].append(get_reviews(new_soup))\n",
        "      d['DESCRIPTION'].append(get_description(new_soup))\n",
        "      d['ASIN'].append(get_asin(\"https://amazon.in\" + link))\n",
        "      d['MANUFACTURER'].append(get_manufacturer(new_soup))\n",
        "\n",
        "  product_list = pd.DataFrame.from_dict(d)\n",
        "  product_list['PRODUCT NAME'].replace('', np.nan, inplace=True)\n",
        "  product_list = product_list.dropna(subset=['PRODUCT NAME'])\n",
        "  product_list.to_csv(\"amazon_data.csv\", header=True, index=False)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5fF6WPpdkQe5",
        "outputId": "360120dd-3925-405d-fcb7-6247053cf204"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-5-1056510cec58>:51: MarkupResemblesLocatorWarning: The input looks more like a filename than markup. You may want to open this file and pass the filehandle into Beautiful Soup.\n",
            "  manu = str(BeautifulSoup(manu, 'html.parser'))\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "product_list"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 710
        },
        "id": "sB67raCH-BTU",
        "outputId": "97f48c23-f2a3-4d05-fb6e-13398ee12223"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                           PRODUCT URL  \\\n",
              "0    https://amazon.in/American-Tourister-AMT-SCH-0...   \n",
              "1    https://amazon.in/Wesley-Milestone-Waterproof-...   \n",
              "2    https://amazon.in/Lavie-Sport-Duffle-Luggage-T...   \n",
              "3    https://amazon.in/ADISA-Laptop-Backpack-Office...   \n",
              "4    https://amazon.in/Skybags-Brat-Black-Casual-Ba...   \n",
              "..                                                 ...   \n",
              "272  https://amazon.in/Gear-Polyester-Laptop-Backpa...   \n",
              "274  https://amazon.in/Casual-Travel-Backpack-Schoo...   \n",
              "283  https://amazon.in/F-Gear-President-Brown-Lapto...   \n",
              "284  https://amazon.in/F-Gear-Talent-Laptop-Backpac...   \n",
              "286  https://amazon.in/xcluma-Diaper-Changing-Mothe...   \n",
              "\n",
              "                                          PRODUCT NAME PRODUCT PRICE  \\\n",
              "0    American Tourister 32 Ltrs Black Casual Backpa...        1,199.   \n",
              "1    Wesley Milestone 2.0 Casual Waterproof Laptop ...          565.   \n",
              "2    Lavie Sport Lino Large Size 63 cms Wheel Duffl...          949.   \n",
              "3    ADISA 15.6 inch Laptop Backpack Office Bag Col...          499.   \n",
              "4            Skybags Brat Black 46 Cms Casual Backpack          659.   \n",
              "..                                                 ...           ...   \n",
              "272        Gear ECO 1 Black Orange Red Laptop Backpack          854.   \n",
              "274     Airish 45 ltrs (46 Cms)Backpack(ARU-202_Green)           664   \n",
              "283  F Gear President Brown 30 liter Laptop Backpac...        1,011.   \n",
              "284  F Gear Talent Laptop Backpack With Rain Cover ...           931   \n",
              "286  xcluma Baby Diaper Nappy Changing Baby Diaper ...          999.   \n",
              "\n",
              "                 RATING NUMBER OF REVIEWS  \\\n",
              "0    4.1 out of 5 stars    52,287 ratings   \n",
              "1    4.3 out of 5 stars     9,793 ratings   \n",
              "2    3.9 out of 5 stars     6,296 ratings   \n",
              "3    3.8 out of 5 stars       564 ratings   \n",
              "4    4.1 out of 5 stars     3,770 ratings   \n",
              "..                  ...               ...   \n",
              "272  4.3 out of 5 stars     1,357 ratings   \n",
              "274  3.6 out of 5 stars       154 ratings   \n",
              "283  4.3 out of 5 stars       510 ratings   \n",
              "284  4.4 out of 5 stars     1,103 ratings   \n",
              "286  4.3 out of 5 stars     1,537 ratings   \n",
              "\n",
              "                                           DESCRIPTION        ASIN  \\\n",
              "0    [Laptop Compatibility: No,  Strap Type: Adjust...  B07CJCGM1M   \n",
              "1    [30L Capacity: The Backpack has a padded lapto...  B084JGJ8PF   \n",
              "2    [The Lino wheel Duffle Bag from Lavie Sport is...  B097RJ22Q3   \n",
              "3    [Material: Water Resistant Light-Weight Polyes...  B09TPX22NF   \n",
              "4    [Combination of functional & safety features i...  B08Z1HHHTD   \n",
              "..                                                 ...         ...   \n",
              "272  [Outer Material: Polyester,  Color: Jet Black ...  B012NW1H1I   \n",
              "274  [Fit Type: Regular    SPACIOUS AND LIGHTWEIGHT...  B09PVCF9F8   \n",
              "283  [Professional,  Good Looking and Huge.    Larg...  B06XW7FSHF   \n",
              "284  [Water Proof and Lightweight.    Dimension HxL...  B078KS2FX1   \n",
              "286  [1 main bag,  1 messenger bag,  1 bottle bag (...  B07PFL5Z3W   \n",
              "\n",
              "                                          MANUFACTURER  \n",
              "0                                            Samsonite  \n",
              "1    Wesley, Longani Trading Company, F-82, Sector-...  \n",
              "2                                                       \n",
              "3                                                       \n",
              "4                                                       \n",
              "..                                                 ...  \n",
              "272                                                     \n",
              "274                                                     \n",
              "283                                                     \n",
              "284                                                     \n",
              "286                                                     \n",
              "\n",
              "[259 rows x 8 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-15deae38-3a8f-4ed9-87c9-c9572f3272f7\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>PRODUCT URL</th>\n",
              "      <th>PRODUCT NAME</th>\n",
              "      <th>PRODUCT PRICE</th>\n",
              "      <th>RATING</th>\n",
              "      <th>NUMBER OF REVIEWS</th>\n",
              "      <th>DESCRIPTION</th>\n",
              "      <th>ASIN</th>\n",
              "      <th>MANUFACTURER</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>https://amazon.in/American-Tourister-AMT-SCH-0...</td>\n",
              "      <td>American Tourister 32 Ltrs Black Casual Backpa...</td>\n",
              "      <td>1,199.</td>\n",
              "      <td>4.1 out of 5 stars</td>\n",
              "      <td>52,287 ratings</td>\n",
              "      <td>[Laptop Compatibility: No,  Strap Type: Adjust...</td>\n",
              "      <td>B07CJCGM1M</td>\n",
              "      <td>Samsonite</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>https://amazon.in/Wesley-Milestone-Waterproof-...</td>\n",
              "      <td>Wesley Milestone 2.0 Casual Waterproof Laptop ...</td>\n",
              "      <td>565.</td>\n",
              "      <td>4.3 out of 5 stars</td>\n",
              "      <td>9,793 ratings</td>\n",
              "      <td>[30L Capacity: The Backpack has a padded lapto...</td>\n",
              "      <td>B084JGJ8PF</td>\n",
              "      <td>Wesley, Longani Trading Company, F-82, Sector-...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>https://amazon.in/Lavie-Sport-Duffle-Luggage-T...</td>\n",
              "      <td>Lavie Sport Lino Large Size 63 cms Wheel Duffl...</td>\n",
              "      <td>949.</td>\n",
              "      <td>3.9 out of 5 stars</td>\n",
              "      <td>6,296 ratings</td>\n",
              "      <td>[The Lino wheel Duffle Bag from Lavie Sport is...</td>\n",
              "      <td>B097RJ22Q3</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>https://amazon.in/ADISA-Laptop-Backpack-Office...</td>\n",
              "      <td>ADISA 15.6 inch Laptop Backpack Office Bag Col...</td>\n",
              "      <td>499.</td>\n",
              "      <td>3.8 out of 5 stars</td>\n",
              "      <td>564 ratings</td>\n",
              "      <td>[Material: Water Resistant Light-Weight Polyes...</td>\n",
              "      <td>B09TPX22NF</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>https://amazon.in/Skybags-Brat-Black-Casual-Ba...</td>\n",
              "      <td>Skybags Brat Black 46 Cms Casual Backpack</td>\n",
              "      <td>659.</td>\n",
              "      <td>4.1 out of 5 stars</td>\n",
              "      <td>3,770 ratings</td>\n",
              "      <td>[Combination of functional &amp; safety features i...</td>\n",
              "      <td>B08Z1HHHTD</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>272</th>\n",
              "      <td>https://amazon.in/Gear-Polyester-Laptop-Backpa...</td>\n",
              "      <td>Gear ECO 1 Black Orange Red Laptop Backpack</td>\n",
              "      <td>854.</td>\n",
              "      <td>4.3 out of 5 stars</td>\n",
              "      <td>1,357 ratings</td>\n",
              "      <td>[Outer Material: Polyester,  Color: Jet Black ...</td>\n",
              "      <td>B012NW1H1I</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>274</th>\n",
              "      <td>https://amazon.in/Casual-Travel-Backpack-Schoo...</td>\n",
              "      <td>Airish 45 ltrs (46 Cms)Backpack(ARU-202_Green)</td>\n",
              "      <td>664</td>\n",
              "      <td>3.6 out of 5 stars</td>\n",
              "      <td>154 ratings</td>\n",
              "      <td>[Fit Type: Regular    SPACIOUS AND LIGHTWEIGHT...</td>\n",
              "      <td>B09PVCF9F8</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>283</th>\n",
              "      <td>https://amazon.in/F-Gear-President-Brown-Lapto...</td>\n",
              "      <td>F Gear President Brown 30 liter Laptop Backpac...</td>\n",
              "      <td>1,011.</td>\n",
              "      <td>4.3 out of 5 stars</td>\n",
              "      <td>510 ratings</td>\n",
              "      <td>[Professional,  Good Looking and Huge.    Larg...</td>\n",
              "      <td>B06XW7FSHF</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>284</th>\n",
              "      <td>https://amazon.in/F-Gear-Talent-Laptop-Backpac...</td>\n",
              "      <td>F Gear Talent Laptop Backpack With Rain Cover ...</td>\n",
              "      <td>931</td>\n",
              "      <td>4.4 out of 5 stars</td>\n",
              "      <td>1,103 ratings</td>\n",
              "      <td>[Water Proof and Lightweight.    Dimension HxL...</td>\n",
              "      <td>B078KS2FX1</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>286</th>\n",
              "      <td>https://amazon.in/xcluma-Diaper-Changing-Mothe...</td>\n",
              "      <td>xcluma Baby Diaper Nappy Changing Baby Diaper ...</td>\n",
              "      <td>999.</td>\n",
              "      <td>4.3 out of 5 stars</td>\n",
              "      <td>1,537 ratings</td>\n",
              "      <td>[1 main bag,  1 messenger bag,  1 bottle bag (...</td>\n",
              "      <td>B07PFL5Z3W</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>259 rows × 8 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-15deae38-3a8f-4ed9-87c9-c9572f3272f7')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-15deae38-3a8f-4ed9-87c9-c9572f3272f7 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-15deae38-3a8f-4ed9-87c9-c9572f3272f7');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "The Manufacurer column is largely NULL, this is because of the variation of the product page format. Hardcoding various formats is a possible solution however it will take time ."
      ],
      "metadata": {
        "id": "nOzhNoGsZIbk"
      }
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "XFPiEUfHZlgh"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}