Sanity_Check_on_Undersample.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import subprocess\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This snippet assumes you run setup first\n",
    "\n",
    "# This code copies file in your Google Bucket and loads it into a dataframe\n",
    "\n",
    "# Replace 'test.csv' with THE NAME of the file you're going to download from the bucket (don't delete the quotation marks)\n",
    "name_of_file_in_bucket = 'Feature_Matrix.csv'\n",
    "\n",
    "########################################################################\n",
    "##\n",
    "################# DON'T CHANGE FROM HERE ###############################\n",
    "##\n",
    "########################################################################\n",
    "\n",
    "# get the bucket name\n",
    "my_bucket = os.getenv('WORKSPACE_BUCKET')\n",
    "\n",
    "# copy csv file from the bucket to the current working space\n",
    "os.system(f\"gsutil cp '{my_bucket}/data/{name_of_file_in_bucket}' .\")\n",
    "\n",
    "print(f'[INFO] {name_of_file_in_bucket} is successfully downloaded into your working space')\n",
    "# save dataframe in a csv file in the same workspace as the notebook\n",
    "my_dataframe = pd.read_csv(name_of_file_in_bucket)\n",
    "my_dataframe.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove useless columns from data frame\n",
    "\n",
    "my_dataframe = my_dataframe.drop(columns = [\"Unnamed: 0\", \"condition_occurrence_person_id\", \"drug_exposure_person_id\", \"procedure_person_id\"])\n",
    "my_dataframe\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_dataframe\n",
    "my_dataframe.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "data_frame_dropped_na_by_subset = my_dataframe.dropna(how = \"all\", subset = ['has_Opioid_abuse', 'is_exposed_to_Opioids', 'has_Anxiety',\n",
    "       'has_Bipolar_disorder', 'has_Depression', 'has_Hypertension',\n",
    "       'has_Opioid_dependence', 'has_Pain', 'has_Rhinitis',\n",
    "       'has_Non_Opioid_Substance_Abuse', 'is_exposed_to_ibuprofen',\n",
    "       'is_exposed_to_buprenorphine', 'is_exposed_to_fentanyl',\n",
    "       'is_exposed_to_morphine', 'is_exposed_to_oxycodone',\n",
    "       'is_exposed_to_hydromorphone', 'is_exposed_to_aspirin',\n",
    "       'is_exposed_to_codeine', 'is_exposed_to_tramadol',\n",
    "       'is_exposed_to_nalbuphine', 'is_exposed_to_meperidine',\n",
    "       'is_exposed_to_naltrexone', 'is_exposed_to_acetaminophen',\n",
    "       'had_Mammography', 'had_Knee_procedure', 'had_Tooth_procedure',\n",
    "       'had_Hip_procedure', 'had_Vascular_procedure', 'had_Brain_procedure',\n",
    "       'had_Heart_procedure', 'had_procedural_ED_visit',\n",
    "       'had_Head_Or_Neck_procedure'])\n",
    "\n",
    "data_frame_dropped_na_by_subset\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_frame_dropped_na_by_threshold = my_dataframe.dropna(thresh = 4) # Remove rows with NA values for all feature columns\n",
    "data_frame_dropped_na_by_threshold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Testing if all persons in undersample have been exposed to opioids\n",
    "data_frame_with_person_ids_and_is_exposed_to_opioids = data_frame_dropped_na_by_threshold[[\"person_id\", \"is_exposed_to_Opioids\"]]\n",
    "data_frame_with_person_ids_and_is_exposed_to_opioids = data_frame_with_person_ids_and_is_exposed_to_opioids.fillna(0)\n",
    "data_frame_with_person_ids_and_is_exposed_to_opioids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# All persons in undersample have been exposed to opioids\n",
    "data_frame_with_person_ids_and_is_exposed_to_opioids_grouped_by_person_id = data_frame_with_person_ids_and_is_exposed_to_opioids.groupby(by = \"person_id\").max()\n",
    "print(sum(data_frame_with_person_ids_and_is_exposed_to_opioids_grouped_by_person_id[\"is_exposed_to_Opioids\"] == 1.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_frame_with_person_ids_and_has_opioid_abuse = data_frame_dropped_na_by_threshold[[\"person_id\", \"has_Opioid_abuse\"]]\n",
    "data_frame_with_person_ids_and_has_opioid_abuse = data_frame_with_person_ids_and_has_opioid_abuse.fillna(0)\n",
    "data_frame_with_person_ids_and_has_opioid_abuse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# All persons in undersample have been exposed to opioids\n",
    "data_frame_with_person_ids_and_has_opioid_abuse_grouped_by_person_id = data_frame_with_person_ids_and_has_opioid_abuse.groupby(by = \"person_id\").max()\n",
    "print(sum(data_frame_with_person_ids_and_has_opioid_abuse_grouped_by_person_id[\"has_Opioid_abuse\"] == 1.0))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}