scrape_specific_page.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium.webdriver import Firefox\n",
    "from selenium.webdriver.firefox.options import Options\n",
    "from selenium.webdriver.firefox.service import Service\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from selenium.common.exceptions import TimeoutException"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "#driver_path = '/usr/bin/geckodriver'\n",
    "driver_path = 'driver/geckodriver'\n",
    "service = Service(driver_path)\n",
    "ff_options = Options()\n",
    "# ff_options.add_argument('--headless')\n",
    "#ff_options.add_argument('--window-size=1920,1080')\n",
    "driver = Firefox(service=service, options=ff_options)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_url = \"https://rarediseases.info.nih.gov/\"\n",
    "disease_urls = [\n",
    "    \"diseases/4017/occipital-horn-syndrome\",\n",
    "    \"diseases/13392/16p1311-microduplication-syndrome\"  # Symptoms and causes empty\n",
    "]\n",
    "\n",
    "request_url = base_url + disease_urls[0]\n",
    "\n",
    "driver.get(request_url)\n",
    "\n",
    "timeout = 30\n",
    "try:\n",
    "    WebDriverWait(driver, timeout).until(EC.visibility_of_element_located((By.ID, \"symptoms\")))\n",
    "except TimeoutException:\n",
    "    driver.quit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Specific learning disability',\n",
       " 'Large fontanelles',\n",
       " 'Joint hyperflexibility',\n",
       " 'Intellectual disability',\n",
       " 'Hyperextensible skin',\n",
       " 'Global developmental delay',\n",
       " 'Exostoses',\n",
       " 'Delayed cranial suture closure',\n",
       " 'Cerebral calcification',\n",
       " 'Abnormality of the face',\n",
       " 'Abnormal skull morphology',\n",
       " 'Venous insufficiency',\n",
       " 'Vascular dilatation',\n",
       " 'Synostosis of joints',\n",
       " 'Short palm',\n",
       " 'Scarring',\n",
       " 'Rickets',\n",
       " 'Poor suck',\n",
       " 'Platyspondyly',\n",
       " 'Pectus excavatum',\n",
       " 'Pectus carinatum',\n",
       " 'Osteoporosis',\n",
       " 'Osteopenia',\n",
       " 'Osteomalacia',\n",
       " 'Long philtrum',\n",
       " 'Keloids; Keloid scar',\n",
       " 'Jaundice',\n",
       " 'Hypotonia',\n",
       " 'High, narrow palate',\n",
       " 'Hiatus hernia',\n",
       " 'Hepatitis',\n",
       " 'Gastroparesis',\n",
       " 'Gastroesophageal reflux',\n",
       " 'Esophagitis',\n",
       " 'Dysphagia',\n",
       " 'Cholestasis',\n",
       " 'Bruising susceptibility',\n",
       " 'Brachydactyly',\n",
       " 'Atypical scarring of skin',\n",
       " 'Abnormality of the wrist',\n",
       " 'Abnormality of the sense of smell',\n",
       " 'Abnormal esophagus physiology',\n",
       " 'Thick hair',\n",
       " 'Scoliosis',\n",
       " 'Recurrent urinary tract infections',\n",
       " 'Pes planus',\n",
       " 'Osteolysis',\n",
       " 'Narrow chest',\n",
       " 'Large iliac wings',\n",
       " 'Kyphosis',\n",
       " 'Inguinal hernia',\n",
       " 'Humerus varus',\n",
       " 'Hip dysplasia',\n",
       " 'Hip dislocation',\n",
       " 'High forehead',\n",
       " 'Genu valgum',\n",
       " 'Femoral hernia',\n",
       " 'Downslanted palpebral fissures',\n",
       " 'Down-sloping shoulders',\n",
       " 'Coxa vara',\n",
       " 'Coxa valga',\n",
       " 'Coarse hair',\n",
       " 'Bladder diverticulum',\n",
       " 'Avascular necrosis of the capital femoral epiphysis',\n",
       " 'Aplastic clavicle',\n",
       " 'Aplasia/hypoplasia of the humerus',\n",
       " 'Absent tibia',\n",
       " 'Abnormality of the pubic bone',\n",
       " 'Abnormality of fibula morphology']"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# For Scraping symptoms\n",
    "\n",
    "symptoms_div = driver.find_element(By.ID, 'symptoms')\n",
    "\n",
    "symptoms = []\n",
    "\n",
    "if 'This section is currently in development.' not in symptoms_div.text:\n",
    "    symptoms_table = symptoms_div.find_element(By.XPATH, './/div/div/div[4]/div[2]/div/div[2]')\n",
    "\n",
    "    for symptom_div in symptoms_table.find_elements(By.XPATH, '*'):\n",
    "        symptom_name = symptom_div.find_element(By.XPATH, './/div').text\n",
    "        symptoms.append(symptom_name)\n",
    "\n",
    "symptoms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Genetic Disease', 'Inheritance']"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# For scraping causes\n",
    "causes_div = driver.find_element(By.ID, 'causes')\n",
    "\n",
    "causes = []\n",
    "\n",
    "if 'This section is currently in development.' not in causes_div.text:\n",
    "    causes_table = causes_div.find_element(By.XPATH, './/div/div')\n",
    "\n",
    "    for causes_div in causes_table.find_elements(By.XPATH, '*'):\n",
    "        try:\n",
    "            causes_div.find_element(By.XPATH, './/app-causes-tabs')\n",
    "        except:\n",
    "            continue\n",
    "        causes_name = causes_div.find_element(By.XPATH, './/app-causes-tabs/div/div/h5').text\n",
    "        causes.append(causes_name)\n",
    "\n",
    "causes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.quit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  },
  "vscode": {
   "interpreter": {
    "hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}