CREATE_DATAWEBSCRAPE.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<h1 class=\"bold-title-l title\">\n",
       "         List of companies in India\n",
       "        </h1>]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final=pd.DataFrame()\n",
    "for j in range(1,1001):\n",
    "    headers={'User-Agent':'Mozilla/5.0 (Windows NT 6.3; Win 64 ; x64) Apple WeKit /537.36(KHTML , like Gecko) Chrome/80.0.3987.162 Safari/537.36'}\n",
    "    url=requests.get('https://www.ambitionbox.com/list-of-companies?page={}'.format(j),headers=headers).text\n",
    "    doc=BeautifulSoup(url,\"lxml\")\n",
    "    struc=BeautifulSoup( doc.prettify(),\"lxml\")\n",
    "    \n",
    "# title=struc.find(id='productTitle').get_text()\n",
    "# price=struc.find(id='priceblock_dealprice').get_text()\n",
    "# print(title)\n",
    "# print(price)\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DKM Online\n",
      "KP GROUP\n",
      "Rezayat Group\n",
      "Fourth Dimension...\n",
      "DELTA\n",
      "ERD Technologies\n",
      "OSP Labs\n",
      "Renu Electronics\n",
      "Autoprint Machin...\n",
      "Shiv Nadar Schoo...\n",
      "World Health Par...\n",
      "Ravenbhel Health...\n",
      "Jaika Motors\n",
      "Shiksha.com\n",
      "Triumph Internat...\n",
      "Great Eastern Sh...\n",
      "Menon & Menon\n",
      "iQuanti\n",
      "Harbinger Group\n",
      "eGain Communicat...\n",
      "DP Chocolates\n",
      "Indus Hospital\n",
      "VPS Rockland\n",
      "Vijayasri Organi...\n",
      "Rural Developmen...\n",
      "TechnoServe\n",
      "Itelligence\n",
      "BI WORLDWIDE\n",
      "Brilliant Bio Ph...\n",
      "Geological Surve...\n"
     ]
    }
   ],
   "source": [
    "for i in struc.find_all('h2'):\n",
    "    print(i.text.strip())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4.2\n",
      "3.9\n",
      "4.3\n",
      "3.8\n",
      "4.4\n",
      "3.6\n",
      "4.3\n",
      "3.7\n",
      "4.2\n",
      "4.0\n",
      "4.4\n",
      "3.6\n",
      "4.3\n",
      "4.0\n",
      "4.1\n",
      "4.2\n",
      "3.6\n",
      "4.1\n",
      "4.3\n",
      "3.8\n",
      "4.0\n",
      "4.0\n",
      "3.8\n",
      "3.8\n",
      "4.2\n",
      "4.0\n",
      "4.0\n",
      "4.2\n",
      "4.0\n",
      "3.0\n"
     ]
    }
   ],
   "source": [
    "for i in struc.find_all('p',class_='rating'):\n",
    "    print(i.text.strip())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\dubey\\AppData\\Local\\Temp\\ipykernel_36160\\233627761.py:53: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  final=final.append(df,ignore_index=True)\n"
     ]
    }
   ],
   "source": [
    "final=pd.DataFrame()\n",
    "for j in range(1,1001):\n",
    "    headers={'User-Agent':'Mozilla/5.0 (Windows NT 6.3; Win 64 ; x64) Apple WeKit /537.36(KHTML , like Gecko) Chrome/80.0.3987.162 Safari/537.36'}\n",
    "    url=requests.get('https://www.ambitionbox.com/list-of-companies?page={}'.format(j),headers=headers).text\n",
    "    doc=BeautifulSoup(url,\"lxml\")\n",
    "    struc=BeautifulSoup( doc.prettify(),\"lxml\")\n",
    "    company=struc.find_all('div',class_='company-content-wrapper')\n",
    "    name=[]\n",
    "    ratings=[]\n",
    "    reviews=[]\n",
    "    ownership=[]\n",
    "    hq=[]\n",
    "    years=[]\n",
    "    emp_count=[]\n",
    "    \n",
    "    for i in company:\n",
    "        try:\n",
    "            name.append(i.find('h2').text.strip())\n",
    "        except:\n",
    "            name.append(np.nan)\n",
    "\n",
    "        try:\n",
    "            ratings.append(i.find('p',class_='rating').text.strip())\n",
    "        except:\n",
    "            ratings.append(np.nan)\n",
    "\n",
    "        try:\n",
    "            reviews.append(i.find('a' , class_='review-count').text.strip())\n",
    "        except:\n",
    "            reviews.append(np.nan)\n",
    "\n",
    "        try:\n",
    "\n",
    "            ownership.append(i.find_all('p',class_='infoEntity')[0].text.strip())\n",
    "        except:\n",
    "            ownership.append(np.nan)\n",
    "        try:\n",
    "            hq.append(i.find_all('p',class_='infoEntity')[1].text.strip())\n",
    "        except:\n",
    "            hq.append(np.nan)\n",
    "        try:\n",
    "            years.append(i.find_all('p',class_='infoEntity')[2].text.strip())\n",
    "        except:\n",
    "            years.append(np.nan)\n",
    "        try:\n",
    "            emp_count.append(i.find_all('p',class_='infoEntity')[3].text.strip())\n",
    "        except:\n",
    "            emp_count.append(np.nan)\n",
    "    \n",
    "    \n",
    "    \n",
    "    df=pd.DataFrame({'name':name,'ratings':ratings,'ownership':ownership,'hq':hq,'years':years,'emp_count':emp_count})\n",
    "    final=final.append(df,ignore_index=True)\n",
    "\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>ratings</th>\n",
       "      <th>ownership</th>\n",
       "      <th>hq</th>\n",
       "      <th>years</th>\n",
       "      <th>emp_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TCS</td>\n",
       "      <td>3.9</td>\n",
       "      <td>Public</td>\n",
       "      <td>Mumbai,Maharashtra + 268 more</td>\n",
       "      <td>54 years old</td>\n",
       "      <td>1 Lakh+ Employees (India)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Accenture</td>\n",
       "      <td>4.1</td>\n",
       "      <td>Public</td>\n",
       "      <td>Dublin + 131 more</td>\n",
       "      <td>33 years old</td>\n",
       "      <td>1 Lakh+ Employees (India)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Cognizant</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Private</td>\n",
       "      <td>Teaneck. New Jersey. + 101 more</td>\n",
       "      <td>28 years old</td>\n",
       "      <td>1 Lakh+ Employees (India)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ICICI Bank</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Public</td>\n",
       "      <td>Mumbai,Maharashtra + 1051 more</td>\n",
       "      <td>28 years old</td>\n",
       "      <td>1 Lakh+ Employees (India)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>HDFC Bank</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Public</td>\n",
       "      <td>Mumbai,Maharashtra + 1233 more</td>\n",
       "      <td>28 years old</td>\n",
       "      <td>50k-1 Lakh Employees (India)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         name ratings ownership                               hq  \\\n",
       "0         TCS     3.9    Public    Mumbai,Maharashtra + 268 more   \n",
       "1   Accenture     4.1    Public                Dublin + 131 more   \n",
       "2   Cognizant     4.0   Private  Teaneck. New Jersey. + 101 more   \n",
       "3  ICICI Bank     4.0    Public   Mumbai,Maharashtra + 1051 more   \n",
       "4   HDFC Bank     4.0    Public   Mumbai,Maharashtra + 1233 more   \n",
       "\n",
       "          years                     emp_count  \n",
       "0  54 years old     1 Lakh+ Employees (India)  \n",
       "1  33 years old     1 Lakh+ Employees (India)  \n",
       "2  28 years old     1 Lakh+ Employees (India)  \n",
       "3  28 years old     1 Lakh+ Employees (India)  \n",
       "4  28 years old  50k-1 Lakh Employees (India)  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<function str.strip(chars=None, /)>,\n",
       " <function str.strip(chars=None, /)>,\n",
       " <function str.strip(chars=None, /)>,\n",
       " <function str.strip(chars=None, /)>,\n",
       " <function str.strip(chars=None, /)>,\n",
       " <function str.strip(chars=None, /)>,\n",
       " <function str.strip(chars=None, /)>,\n",
       " <function str.strip(chars=None, /)>,\n",
       " <function str.strip(chars=None, /)>,\n",
       " <function str.strip(chars=None, /)>,\n",
       " <function str.strip(chars=None, /)>,\n",
       " <function str.strip(chars=None, /)>,\n",
       " <function str.strip(chars=None, /)>,\n",
       " <function str.strip(chars=None, /)>,\n",
       " <function str.strip(chars=None, /)>,\n",
       " <function str.strip(chars=None, /)>,\n",
       " <function str.strip(chars=None, /)>,\n",
       " <function str.strip(chars=None, /)>,\n",
       " <function str.strip(chars=None, /)>,\n",
       " <function str.strip(chars=None, /)>,\n",
       " <function str.strip(chars=None, /)>,\n",
       " <function str.strip(chars=None, /)>,\n",
       " <function str.strip(chars=None, /)>,\n",
       " <function str.strip(chars=None, /)>,\n",
       " <function str.strip(chars=None, /)>,\n",
       " <function str.strip(chars=None, /)>,\n",
       " <function str.strip(chars=None, /)>,\n",
       " <function str.strip(chars=None, /)>,\n",
       " <function str.strip(chars=None, /)>,\n",
       " <function str.strip(chars=None, /)>]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "f9a312619f050551a366f6112b405fdb2489e028f80049bc7feab12ad677bacc"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}