create_TSVs.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "42940e46-a178-48fe-bda6-fb175b6c9636",
   "metadata": {
    "tags": []
   },
   "source": [
    "## This notebook constructs the KG/TSVs for Clinical Trials Using Gwenlyn's analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "56e79379-d0bf-4694-8bdb-32cdcbacece3",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>.container { width:100% !important; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.output_result { max-width:100% !important; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# display cells to maximum width \n",
    "from IPython.display import display, HTML\n",
    "display(HTML(\"<style>.container { width:100% !important; }</style>\"))\n",
    "display(HTML(\"<style>.output_result { max-width:100% !important; }</style>\"))\n",
    "\n",
    "# lets you preint multiple outputs per cell, not just last\n",
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = \"all\"\n",
    "\n",
    "import pandas as pd\n",
    "import pathlib\n",
    "import re\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a23e0697-e18b-441a-901a-abac3e95f679",
   "metadata": {
    "tags": []
   },
   "source": [
    "#### Get Gwen's TSV, clean Conditions and Interventions/Treatments columns, get each per NCT ID per each row"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "8daf224c-6c21-4339-bee5-191af38d438c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nct_id</th>\n",
       "      <th>primary_purpose</th>\n",
       "      <th>intervention_model</th>\n",
       "      <th>start_month_year</th>\n",
       "      <th>n_conditions</th>\n",
       "      <th>n_interventions</th>\n",
       "      <th>is_fda_regulated_drug</th>\n",
       "      <th>conditions</th>\n",
       "      <th>exp_only</th>\n",
       "      <th>ctr_only</th>\n",
       "      <th>both</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NCT01049399</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2009-12</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>?</td>\n",
       "      <td>Progressive Supranuclear Palsy|1.000</td>\n",
       "      <td>tideglusib</td>\n",
       "      <td>placebo</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NCT00807209</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2008-12</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>?</td>\n",
       "      <td>Postoperative Pain|-0.092</td>\n",
       "      <td>High Dose SKY0402|Low Dose SKY0402</td>\n",
       "      <td>Placebo</td>\n",
       "      <td>Bupivacaine via epidural|Fentanyl via PCA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NCT05287724</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Crossover Assignment</td>\n",
       "      <td>2022-6-19</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>t</td>\n",
       "      <td>Pruritus|1.000|Skin Disorder|-0.091</td>\n",
       "      <td>N-acetyl cysteine|Placebo</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NCT01432236</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Crossover Assignment</td>\n",
       "      <td>2011-10</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>?</td>\n",
       "      <td>Fibromyalgia|1.000</td>\n",
       "      <td>Pregabalin</td>\n",
       "      <td>placebo</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NCT04626921</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Single Group Assignment</td>\n",
       "      <td>2020-10-22</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>t</td>\n",
       "      <td>Relapsing Multiple Sclerosis|1.000</td>\n",
       "      <td>CNM-Au8</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>137111</th>\n",
       "      <td>NCT06310811</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Single Group Assignment</td>\n",
       "      <td>2024-3-7</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>f</td>\n",
       "      <td>Safety|1.000|Effective|-0.048</td>\n",
       "      <td>RD06-04 Cells injection</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>137112</th>\n",
       "      <td>NCT02375672</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Single Group Assignment</td>\n",
       "      <td>2015-5-28</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>?</td>\n",
       "      <td>Colorectal Cancer|1.000</td>\n",
       "      <td>Pembrolizumab|mFOLFOX6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>137113</th>\n",
       "      <td>NCT04252118</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2020-1-27</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>f</td>\n",
       "      <td>COVID-19|1.000</td>\n",
       "      <td>MSCs</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>137114</th>\n",
       "      <td>NCT04264533</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2020-2-14</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>f</td>\n",
       "      <td>Vitamin C|1.000|Pneumonia, Viral|0.462|Pneumon...</td>\n",
       "      <td>VC</td>\n",
       "      <td>Sterile Water for Injection</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>137115</th>\n",
       "      <td>NCT04520607</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2020-8-25</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>t</td>\n",
       "      <td>Osteoarthritis, Knee|0.728</td>\n",
       "      <td>Lorecivivint|Placebo</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>137116 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             nct_id primary_purpose       intervention_model start_month_year  \\\n",
       "0       NCT01049399       Treatment      Parallel Assignment          2009-12   \n",
       "1       NCT00807209       Treatment      Parallel Assignment          2008-12   \n",
       "2       NCT05287724       Treatment     Crossover Assignment        2022-6-19   \n",
       "3       NCT01432236       Treatment     Crossover Assignment          2011-10   \n",
       "4       NCT04626921       Treatment  Single Group Assignment       2020-10-22   \n",
       "...             ...             ...                      ...              ...   \n",
       "137111  NCT06310811       Treatment  Single Group Assignment         2024-3-7   \n",
       "137112  NCT02375672       Treatment  Single Group Assignment        2015-5-28   \n",
       "137113  NCT04252118       Treatment      Parallel Assignment        2020-1-27   \n",
       "137114  NCT04264533       Treatment      Parallel Assignment        2020-2-14   \n",
       "137115  NCT04520607       Treatment      Parallel Assignment        2020-8-25   \n",
       "\n",
       "        n_conditions  n_interventions is_fda_regulated_drug  \\\n",
       "0                  1                3                     ?   \n",
       "1                  1                5                     ?   \n",
       "2                  2                2                     t   \n",
       "3                  1                2                     ?   \n",
       "4                  1                1                     t   \n",
       "...              ...              ...                   ...   \n",
       "137111             2                1                     f   \n",
       "137112             1                2                     ?   \n",
       "137113             1                1                     f   \n",
       "137114             3                2                     f   \n",
       "137115             1                2                     t   \n",
       "\n",
       "                                               conditions  \\\n",
       "0                    Progressive Supranuclear Palsy|1.000   \n",
       "1                               Postoperative Pain|-0.092   \n",
       "2                     Pruritus|1.000|Skin Disorder|-0.091   \n",
       "3                                      Fibromyalgia|1.000   \n",
       "4                      Relapsing Multiple Sclerosis|1.000   \n",
       "...                                                   ...   \n",
       "137111                      Safety|1.000|Effective|-0.048   \n",
       "137112                            Colorectal Cancer|1.000   \n",
       "137113                                     COVID-19|1.000   \n",
       "137114  Vitamin C|1.000|Pneumonia, Viral|0.462|Pneumon...   \n",
       "137115                         Osteoarthritis, Knee|0.728   \n",
       "\n",
       "                                  exp_only                     ctr_only  \\\n",
       "0                               tideglusib                      placebo   \n",
       "1       High Dose SKY0402|Low Dose SKY0402                      Placebo   \n",
       "2                N-acetyl cysteine|Placebo                          NaN   \n",
       "3                               Pregabalin                      placebo   \n",
       "4                                  CNM-Au8                          NaN   \n",
       "...                                    ...                          ...   \n",
       "137111             RD06-04 Cells injection                          NaN   \n",
       "137112              Pembrolizumab|mFOLFOX6                          NaN   \n",
       "137113                                MSCs                          NaN   \n",
       "137114                                  VC  Sterile Water for Injection   \n",
       "137115                Lorecivivint|Placebo                          NaN   \n",
       "\n",
       "                                             both  \n",
       "0                                             NaN  \n",
       "1       Bupivacaine via epidural|Fentanyl via PCA  \n",
       "2                                             NaN  \n",
       "3                                             NaN  \n",
       "4                                             NaN  \n",
       "...                                           ...  \n",
       "137111                                        NaN  \n",
       "137112                                        NaN  \n",
       "137113                                        NaN  \n",
       "137114                                        NaN  \n",
       "137115                                        NaN  \n",
       "\n",
       "[137116 rows x 11 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trials_list = pd.read_csv(\"trials_list.txt.gz\", sep='\\t', index_col=False, header=0)\n",
    "trials_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "623852bf-4b74-42aa-8b28-b74189243dbd",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# trials_list_check = trials_list[trials_list['ctr_only'].isnull()] # sometimes exp_only or ctr_only have nulls\n",
    "# trials_list_check = trials_list[trials_list['exp_only'].isnull()]\n",
    "\n",
    "# trials_list_check"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "81b32204-e7e4-4073-9ed5-fe3f85206c1e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>348</th>\n",
       "      <th>349</th>\n",
       "      <th>350</th>\n",
       "      <th>351</th>\n",
       "      <th>352</th>\n",
       "      <th>353</th>\n",
       "      <th>354</th>\n",
       "      <th>355</th>\n",
       "      <th>356</th>\n",
       "      <th>357</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Progressive Supranuclear Palsy</td>\n",
       "      <td>1.000</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Postoperative Pain</td>\n",
       "      <td>-0.092</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Pruritus</td>\n",
       "      <td>1.000</td>\n",
       "      <td>Skin Disorder</td>\n",
       "      <td>-0.091</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Fibromyalgia</td>\n",
       "      <td>1.000</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Relapsing Multiple Sclerosis</td>\n",
       "      <td>1.000</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>137111</th>\n",
       "      <td>Safety</td>\n",
       "      <td>1.000</td>\n",
       "      <td>Effective</td>\n",
       "      <td>-0.048</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>137112</th>\n",
       "      <td>Colorectal Cancer</td>\n",
       "      <td>1.000</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>137113</th>\n",
       "      <td>COVID-19</td>\n",
       "      <td>1.000</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>137114</th>\n",
       "      <td>Vitamin C</td>\n",
       "      <td>1.000</td>\n",
       "      <td>Pneumonia, Viral</td>\n",
       "      <td>0.462</td>\n",
       "      <td>Pneumonia, Ventilator-Associated</td>\n",
       "      <td>0.071</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>137115</th>\n",
       "      <td>Osteoarthritis, Knee</td>\n",
       "      <td>0.728</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>137116 rows × 358 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   0       1                 2       3    \\\n",
       "0       Progressive Supranuclear Palsy   1.000              None    None   \n",
       "1                   Postoperative Pain  -0.092              None    None   \n",
       "2                             Pruritus   1.000     Skin Disorder  -0.091   \n",
       "3                         Fibromyalgia   1.000              None    None   \n",
       "4         Relapsing Multiple Sclerosis   1.000              None    None   \n",
       "...                                ...     ...               ...     ...   \n",
       "137111                          Safety   1.000         Effective  -0.048   \n",
       "137112               Colorectal Cancer   1.000              None    None   \n",
       "137113                        COVID-19   1.000              None    None   \n",
       "137114                       Vitamin C   1.000  Pneumonia, Viral   0.462   \n",
       "137115            Osteoarthritis, Knee   0.728              None    None   \n",
       "\n",
       "                                     4      5     6     7     8     9    ...  \\\n",
       "0                                   None   None  None  None  None  None  ...   \n",
       "1                                   None   None  None  None  None  None  ...   \n",
       "2                                   None   None  None  None  None  None  ...   \n",
       "3                                   None   None  None  None  None  None  ...   \n",
       "4                                   None   None  None  None  None  None  ...   \n",
       "...                                  ...    ...   ...   ...   ...   ...  ...   \n",
       "137111                              None   None  None  None  None  None  ...   \n",
       "137112                              None   None  None  None  None  None  ...   \n",
       "137113                              None   None  None  None  None  None  ...   \n",
       "137114  Pneumonia, Ventilator-Associated  0.071  None  None  None  None  ...   \n",
       "137115                              None   None  None  None  None  None  ...   \n",
       "\n",
       "         348   349   350   351   352   353   354   355   356   357  \n",
       "0       None  None  None  None  None  None  None  None  None  None  \n",
       "1       None  None  None  None  None  None  None  None  None  None  \n",
       "2       None  None  None  None  None  None  None  None  None  None  \n",
       "3       None  None  None  None  None  None  None  None  None  None  \n",
       "4       None  None  None  None  None  None  None  None  None  None  \n",
       "...      ...   ...   ...   ...   ...   ...   ...   ...   ...   ...  \n",
       "137111  None  None  None  None  None  None  None  None  None  None  \n",
       "137112  None  None  None  None  None  None  None  None  None  None  \n",
       "137113  None  None  None  None  None  None  None  None  None  None  \n",
       "137114  None  None  None  None  None  None  None  None  None  None  \n",
       "137115  None  None  None  None  None  None  None  None  None  None  \n",
       "\n",
       "[137116 rows x 358 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores_list = []\n",
    "regexp = re.compile(r\"^[-+]?[0-9]*\\.?[0-9]+$\")\n",
    "conditions = trials_list[\"conditions\"].str.split(\"|\", expand=True)\n",
    "conditions\n",
    "high_score_conditions = []\n",
    "low_score_conditions = []\n",
    "columns = conditions.columns.tolist()\n",
    "for index, row in conditions.iterrows():\n",
    "    for col in columns:\n",
    "        value = row[col]\n",
    "        # print(type(value))\n",
    "        if not value is None:\n",
    "            # print(value)\n",
    "            if regexp.search(value):\n",
    "                condition = conditions.loc[index,col-1]\n",
    "                # print(condition)\n",
    "                score = float(value)\n",
    "                scores_list.append(score)\n",
    "                if score > 0.9:\n",
    "                    high_score_conditions.extend([[index, condition]]) \n",
    "                else:\n",
    "                    low_score_conditions.extend([[index, condition]]) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "934ca06c-24bd-4f28-b6d1-5d413de02387",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[137101, 'Provoked, Localized Vulvodynia'],\n",
       " [137105, 'Acute Myeloid Leukemia'],\n",
       " [137106, 'Discoid Lupus Erythematosus'],\n",
       " [137108, 'Postoperative Pain'],\n",
       " [137108, 'Oocyte Retrieval'],\n",
       " [137110, 'Allergic Conjunctivitis'],\n",
       " [137111, 'Safety'],\n",
       " [137112, 'Colorectal Cancer'],\n",
       " [137113, 'COVID-19'],\n",
       " [137114, 'Vitamin C']]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "high_score_conditions[-10:]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3928930d-bfaa-497f-b80e-4e76b5634e2d",
   "metadata": {},
   "source": [
    "#### HISTOGRAM of scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b358c809-ece4-4030-8813-a0138c7c2579",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# import matplotlib.pyplot as plt\n",
    "\n",
    "# from matplotlib.ticker import PercentFormatter\n",
    "\n",
    "# # data = [1000, 1000, 5000, 3000, 4000, 16000, 2000]\n",
    "\n",
    "# plt.hist(scores_list, weights=np.ones(len(scores_list)) / len(scores_list), bins=20, edgecolor='black')\n",
    "\n",
    "# plt.gca().yaxis.set_major_formatter(PercentFormatter(1))\n",
    "# plt.xlim(0.5, 1)\n",
    "# plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a9f73837-c87c-4e53-89d7-9d50bcd2b50a",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nct_id</th>\n",
       "      <th>primary_purpose</th>\n",
       "      <th>intervention_model</th>\n",
       "      <th>start_month_year</th>\n",
       "      <th>n_conditions</th>\n",
       "      <th>n_interventions</th>\n",
       "      <th>is_fda_regulated_drug</th>\n",
       "      <th>conditions</th>\n",
       "      <th>exp_only</th>\n",
       "      <th>ctr_only</th>\n",
       "      <th>both</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NCT01049399</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2009-12</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>?</td>\n",
       "      <td>Progressive Supranuclear Palsy|1.000</td>\n",
       "      <td>tideglusib</td>\n",
       "      <td>placebo</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NCT00807209</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2008-12</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>?</td>\n",
       "      <td>Postoperative Pain|-0.092</td>\n",
       "      <td>High Dose SKY0402|Low Dose SKY0402</td>\n",
       "      <td>Placebo</td>\n",
       "      <td>Bupivacaine via epidural|Fentanyl via PCA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NCT05287724</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Crossover Assignment</td>\n",
       "      <td>2022-6-19</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>t</td>\n",
       "      <td>Pruritus|1.000|Skin Disorder|-0.091</td>\n",
       "      <td>N-acetyl cysteine|Placebo</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NCT01432236</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Crossover Assignment</td>\n",
       "      <td>2011-10</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>?</td>\n",
       "      <td>Fibromyalgia|1.000</td>\n",
       "      <td>Pregabalin</td>\n",
       "      <td>placebo</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NCT04626921</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Single Group Assignment</td>\n",
       "      <td>2020-10-22</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>t</td>\n",
       "      <td>Relapsing Multiple Sclerosis|1.000</td>\n",
       "      <td>CNM-Au8</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>137111</th>\n",
       "      <td>NCT06310811</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Single Group Assignment</td>\n",
       "      <td>2024-3-7</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>f</td>\n",
       "      <td>Safety|1.000|Effective|-0.048</td>\n",
       "      <td>RD06-04 Cells injection</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>137112</th>\n",
       "      <td>NCT02375672</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Single Group Assignment</td>\n",
       "      <td>2015-5-28</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>?</td>\n",
       "      <td>Colorectal Cancer|1.000</td>\n",
       "      <td>Pembrolizumab|mFOLFOX6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>137113</th>\n",
       "      <td>NCT04252118</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2020-1-27</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>f</td>\n",
       "      <td>COVID-19|1.000</td>\n",
       "      <td>MSCs</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>137114</th>\n",
       "      <td>NCT04264533</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2020-2-14</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>f</td>\n",
       "      <td>Vitamin C|1.000|Pneumonia, Viral|0.462|Pneumon...</td>\n",
       "      <td>VC</td>\n",
       "      <td>Sterile Water for Injection</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>137115</th>\n",
       "      <td>NCT04520607</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2020-8-25</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>t</td>\n",
       "      <td>Osteoarthritis, Knee|0.728</td>\n",
       "      <td>Lorecivivint|Placebo</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>137116 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             nct_id primary_purpose       intervention_model start_month_year  \\\n",
       "0       NCT01049399       Treatment      Parallel Assignment          2009-12   \n",
       "1       NCT00807209       Treatment      Parallel Assignment          2008-12   \n",
       "2       NCT05287724       Treatment     Crossover Assignment        2022-6-19   \n",
       "3       NCT01432236       Treatment     Crossover Assignment          2011-10   \n",
       "4       NCT04626921       Treatment  Single Group Assignment       2020-10-22   \n",
       "...             ...             ...                      ...              ...   \n",
       "137111  NCT06310811       Treatment  Single Group Assignment         2024-3-7   \n",
       "137112  NCT02375672       Treatment  Single Group Assignment        2015-5-28   \n",
       "137113  NCT04252118       Treatment      Parallel Assignment        2020-1-27   \n",
       "137114  NCT04264533       Treatment      Parallel Assignment        2020-2-14   \n",
       "137115  NCT04520607       Treatment      Parallel Assignment        2020-8-25   \n",
       "\n",
       "        n_conditions  n_interventions is_fda_regulated_drug  \\\n",
       "0                  1                3                     ?   \n",
       "1                  1                5                     ?   \n",
       "2                  2                2                     t   \n",
       "3                  1                2                     ?   \n",
       "4                  1                1                     t   \n",
       "...              ...              ...                   ...   \n",
       "137111             2                1                     f   \n",
       "137112             1                2                     ?   \n",
       "137113             1                1                     f   \n",
       "137114             3                2                     f   \n",
       "137115             1                2                     t   \n",
       "\n",
       "                                               conditions  \\\n",
       "0                    Progressive Supranuclear Palsy|1.000   \n",
       "1                               Postoperative Pain|-0.092   \n",
       "2                     Pruritus|1.000|Skin Disorder|-0.091   \n",
       "3                                      Fibromyalgia|1.000   \n",
       "4                      Relapsing Multiple Sclerosis|1.000   \n",
       "...                                                   ...   \n",
       "137111                      Safety|1.000|Effective|-0.048   \n",
       "137112                            Colorectal Cancer|1.000   \n",
       "137113                                     COVID-19|1.000   \n",
       "137114  Vitamin C|1.000|Pneumonia, Viral|0.462|Pneumon...   \n",
       "137115                         Osteoarthritis, Knee|0.728   \n",
       "\n",
       "                                  exp_only                     ctr_only  \\\n",
       "0                               tideglusib                      placebo   \n",
       "1       High Dose SKY0402|Low Dose SKY0402                      Placebo   \n",
       "2                N-acetyl cysteine|Placebo                          NaN   \n",
       "3                               Pregabalin                      placebo   \n",
       "4                                  CNM-Au8                          NaN   \n",
       "...                                    ...                          ...   \n",
       "137111             RD06-04 Cells injection                          NaN   \n",
       "137112              Pembrolizumab|mFOLFOX6                          NaN   \n",
       "137113                                MSCs                          NaN   \n",
       "137114                                  VC  Sterile Water for Injection   \n",
       "137115                Lorecivivint|Placebo                          NaN   \n",
       "\n",
       "                                             both  \n",
       "0                                             NaN  \n",
       "1       Bupivacaine via epidural|Fentanyl via PCA  \n",
       "2                                             NaN  \n",
       "3                                             NaN  \n",
       "4                                             NaN  \n",
       "...                                           ...  \n",
       "137111                                        NaN  \n",
       "137112                                        NaN  \n",
       "137113                                        NaN  \n",
       "137114                                        NaN  \n",
       "137115                                        NaN  \n",
       "\n",
       "[137116 rows x 11 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trials_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "aa23b4bf-b4a9-499b-81a6-d8ed6e140969",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nct_id</th>\n",
       "      <th>primary_purpose</th>\n",
       "      <th>intervention_model</th>\n",
       "      <th>start_month_year</th>\n",
       "      <th>n_conditions</th>\n",
       "      <th>n_interventions</th>\n",
       "      <th>is_fda_regulated_drug</th>\n",
       "      <th>conditions</th>\n",
       "      <th>exp_only</th>\n",
       "      <th>ctr_only</th>\n",
       "      <th>both</th>\n",
       "      <th>condition</th>\n",
       "      <th>predicate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NCT01049399</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2009-12</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>?</td>\n",
       "      <td>Progressive Supranuclear Palsy|1.000</td>\n",
       "      <td>tideglusib</td>\n",
       "      <td>placebo</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Progressive Supranuclear Palsy</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NCT05287724</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Crossover Assignment</td>\n",
       "      <td>2022-6-19</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>t</td>\n",
       "      <td>Pruritus|1.000|Skin Disorder|-0.091</td>\n",
       "      <td>N-acetyl cysteine|Placebo</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Pruritus</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NCT01432236</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Crossover Assignment</td>\n",
       "      <td>2011-10</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>?</td>\n",
       "      <td>Fibromyalgia|1.000</td>\n",
       "      <td>Pregabalin</td>\n",
       "      <td>placebo</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Fibromyalgia</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NCT04626921</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Single Group Assignment</td>\n",
       "      <td>2020-10-22</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>t</td>\n",
       "      <td>Relapsing Multiple Sclerosis|1.000</td>\n",
       "      <td>CNM-Au8</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Relapsing Multiple Sclerosis</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NCT04681417</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2021-3-25</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>f</td>\n",
       "      <td>Retinoblastoma|1.000</td>\n",
       "      <td>Melphalan or Melphalan + Topotecan</td>\n",
       "      <td>Carboplatin administered on Day 1|etoposide, carboplatin and vincristine or local ophthalmological treatment without IV chemotherapy</td>\n",
       "      <td>Cryotherapy (local treatment)|Intravitreal Melphalan chemotherapy injections (local treatment)|Iodine-125 plaques (local treatment)|Thermotherapy (local treatment)</td>\n",
       "      <td>Retinoblastoma</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>NCT04036227</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2019-7-3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>f</td>\n",
       "      <td>Healthy|1.000</td>\n",
       "      <td>GS-248</td>\n",
       "      <td>Placebo</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Healthy</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>NCT05624918</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Single Group Assignment</td>\n",
       "      <td>2024-4</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>f</td>\n",
       "      <td>Pancreatic Adenocarcinoma|1.000|Resectable Pancreatic Cancer|0.566</td>\n",
       "      <td>Gemcitabine|Nab paclitaxel|NovoTTF-200T(P)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Pancreatic Adenocarcinoma</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>NCT02886065</td>\n",
       "      <td>Prevention</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2017-3-7</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>t</td>\n",
       "      <td>Smoldering Multiple Myeloma|1.000</td>\n",
       "      <td>Citarinostat|Hiltonol|Lenalidomide|PVX-410</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Smoldering Multiple Myeloma</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>NCT03683576</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2018-10-22</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>t</td>\n",
       "      <td>Asthma|1.000</td>\n",
       "      <td>GB001</td>\n",
       "      <td>Placebo</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Asthma</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>NCT00738673</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Single Group Assignment</td>\n",
       "      <td>2008-7</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>?</td>\n",
       "      <td>Prostate Cancer|1.000</td>\n",
       "      <td>degarelix</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Prostate Cancer</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        nct_id primary_purpose       intervention_model start_month_year  \\\n",
       "0  NCT01049399       Treatment      Parallel Assignment          2009-12   \n",
       "1  NCT05287724       Treatment     Crossover Assignment        2022-6-19   \n",
       "2  NCT01432236       Treatment     Crossover Assignment          2011-10   \n",
       "3  NCT04626921       Treatment  Single Group Assignment       2020-10-22   \n",
       "4  NCT04681417       Treatment      Parallel Assignment        2021-3-25   \n",
       "5  NCT04036227       Treatment      Parallel Assignment         2019-7-3   \n",
       "6  NCT05624918       Treatment  Single Group Assignment           2024-4   \n",
       "7  NCT02886065      Prevention      Parallel Assignment         2017-3-7   \n",
       "8  NCT03683576       Treatment      Parallel Assignment       2018-10-22   \n",
       "9  NCT00738673       Treatment  Single Group Assignment           2008-7   \n",
       "\n",
       "   n_conditions  n_interventions is_fda_regulated_drug  \\\n",
       "0             1                3                     ?   \n",
       "1             2                2                     t   \n",
       "2             1                2                     ?   \n",
       "3             1                1                     t   \n",
       "4             1                7                     f   \n",
       "5             1                2                     f   \n",
       "6             2                3                     f   \n",
       "7             1                4                     t   \n",
       "8             1                2                     t   \n",
       "9             1                1                     ?   \n",
       "\n",
       "                                                           conditions  \\\n",
       "0                                Progressive Supranuclear Palsy|1.000   \n",
       "1                                 Pruritus|1.000|Skin Disorder|-0.091   \n",
       "2                                                  Fibromyalgia|1.000   \n",
       "3                                  Relapsing Multiple Sclerosis|1.000   \n",
       "4                                                Retinoblastoma|1.000   \n",
       "5                                                       Healthy|1.000   \n",
       "6  Pancreatic Adenocarcinoma|1.000|Resectable Pancreatic Cancer|0.566   \n",
       "7                                   Smoldering Multiple Myeloma|1.000   \n",
       "8                                                        Asthma|1.000   \n",
       "9                                               Prostate Cancer|1.000   \n",
       "\n",
       "                                     exp_only  \\\n",
       "0                                  tideglusib   \n",
       "1                   N-acetyl cysteine|Placebo   \n",
       "2                                  Pregabalin   \n",
       "3                                     CNM-Au8   \n",
       "4          Melphalan or Melphalan + Topotecan   \n",
       "5                                      GS-248   \n",
       "6  Gemcitabine|Nab paclitaxel|NovoTTF-200T(P)   \n",
       "7  Citarinostat|Hiltonol|Lenalidomide|PVX-410   \n",
       "8                                       GB001   \n",
       "9                                   degarelix   \n",
       "\n",
       "                                                                                                                               ctr_only  \\\n",
       "0                                                                                                                               placebo   \n",
       "1                                                                                                                                   NaN   \n",
       "2                                                                                                                               placebo   \n",
       "3                                                                                                                                   NaN   \n",
       "4  Carboplatin administered on Day 1|etoposide, carboplatin and vincristine or local ophthalmological treatment without IV chemotherapy   \n",
       "5                                                                                                                               Placebo   \n",
       "6                                                                                                                                   NaN   \n",
       "7                                                                                                                                   NaN   \n",
       "8                                                                                                                               Placebo   \n",
       "9                                                                                                                                   NaN   \n",
       "\n",
       "                                                                                                                                                                  both  \\\n",
       "0                                                                                                                                                                  NaN   \n",
       "1                                                                                                                                                                  NaN   \n",
       "2                                                                                                                                                                  NaN   \n",
       "3                                                                                                                                                                  NaN   \n",
       "4  Cryotherapy (local treatment)|Intravitreal Melphalan chemotherapy injections (local treatment)|Iodine-125 plaques (local treatment)|Thermotherapy (local treatment)   \n",
       "5                                                                                                                                                                  NaN   \n",
       "6                                                                                                                                                                  NaN   \n",
       "7                                                                                                                                                                  NaN   \n",
       "8                                                                                                                                                                  NaN   \n",
       "9                                                                                                                                                                  NaN   \n",
       "\n",
       "                        condition               predicate  \n",
       "0  Progressive Supranuclear Palsy  in_clinical_trials_for  \n",
       "1                        Pruritus  in_clinical_trials_for  \n",
       "2                    Fibromyalgia  in_clinical_trials_for  \n",
       "3    Relapsing Multiple Sclerosis  in_clinical_trials_for  \n",
       "4                  Retinoblastoma  in_clinical_trials_for  \n",
       "5                         Healthy  in_clinical_trials_for  \n",
       "6       Pancreatic Adenocarcinoma  in_clinical_trials_for  \n",
       "7     Smoldering Multiple Myeloma  in_clinical_trials_for  \n",
       "8                          Asthma  in_clinical_trials_for  \n",
       "9                 Prostate Cancer  in_clinical_trials_for  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nct_id</th>\n",
       "      <th>primary_purpose</th>\n",
       "      <th>intervention_model</th>\n",
       "      <th>start_month_year</th>\n",
       "      <th>n_conditions</th>\n",
       "      <th>n_interventions</th>\n",
       "      <th>is_fda_regulated_drug</th>\n",
       "      <th>conditions</th>\n",
       "      <th>exp_only</th>\n",
       "      <th>ctr_only</th>\n",
       "      <th>both</th>\n",
       "      <th>condition</th>\n",
       "      <th>predicate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NCT00807209</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2008-12</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>?</td>\n",
       "      <td>Postoperative Pain|-0.092</td>\n",
       "      <td>High Dose SKY0402|Low Dose SKY0402</td>\n",
       "      <td>Placebo</td>\n",
       "      <td>Bupivacaine via epidural|Fentanyl via PCA</td>\n",
       "      <td>Postoperative Pain</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NCT05287724</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Crossover Assignment</td>\n",
       "      <td>2022-6-19</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>t</td>\n",
       "      <td>Pruritus|1.000|Skin Disorder|-0.091</td>\n",
       "      <td>N-acetyl cysteine|Placebo</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Skin Disorder</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NCT05624918</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Single Group Assignment</td>\n",
       "      <td>2024-4</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>f</td>\n",
       "      <td>Pancreatic Adenocarcinoma|1.000|Resectable Pancreatic Cancer|0.566</td>\n",
       "      <td>Gemcitabine|Nab paclitaxel|NovoTTF-200T(P)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Resectable Pancreatic Cancer</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NCT01003288</td>\n",
       "      <td>Prevention</td>\n",
       "      <td>Single Group Assignment</td>\n",
       "      <td>2009-10</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>?</td>\n",
       "      <td>Healthy|-0.400</td>\n",
       "      <td>Adjuvanted influenza H1N1split virion vaccine</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Healthy</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NCT06257693</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Single Group Assignment</td>\n",
       "      <td>2024-2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>t</td>\n",
       "      <td>Prostate Adenocarcinoma|0.033</td>\n",
       "      <td>enzalutamide</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Prostate Adenocarcinoma</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>NCT00003494</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Single Group Assignment</td>\n",
       "      <td>1996-3-26</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>t</td>\n",
       "      <td>Bronchial Alveolar; Tumor|0.567</td>\n",
       "      <td>Antineoplaston therapy (Atengenal + Astugenal)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Bronchial Alveolar; Tumor</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>NCT06096909</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2023-11-1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>f</td>\n",
       "      <td>Acute Coronary Syndrome|1.000|Non ST Segment Elevation Acute Coronary Syndrome|0.279</td>\n",
       "      <td>Tafolecimab</td>\n",
       "      <td>Cholesterol Absorption Inhibitor</td>\n",
       "      <td>Statin</td>\n",
       "      <td>Non ST Segment Elevation Acute Coronary Syndrome</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>NCT00012012</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Single Group Assignment</td>\n",
       "      <td>2001-8</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>?</td>\n",
       "      <td>Cervical Cancer|1.000|Radiation Toxicity|0.242</td>\n",
       "      <td>Amifostine trihydrate|Cisplatin|External beam radiation therapy|Intracavitary brachytherapy</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Radiation Toxicity</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>NCT05377203</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Crossover Assignment</td>\n",
       "      <td>2022-7-13</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>f</td>\n",
       "      <td>Hypertension|1.000|Arterial Hypertension|0.304</td>\n",
       "      <td>Dual combination of standard dose therapy→ Quadruple combination of half doses therapy|Quadruple combination of half doses therapy→Dual combination of standard dose therapy</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Arterial Hypertension</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>NCT05762107</td>\n",
       "      <td>Prevention</td>\n",
       "      <td>Crossover Assignment</td>\n",
       "      <td>2023-7-28</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>t</td>\n",
       "      <td>Type 1 Diabetes Mellitus With Hypoglycemia|0.733</td>\n",
       "      <td>Placebo|ZT-01, 15 mg|ZT-01, 22 mg|ZT-01, 7 mg</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Type 1 Diabetes Mellitus With Hypoglycemia</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        nct_id primary_purpose       intervention_model start_month_year  \\\n",
       "0  NCT00807209       Treatment      Parallel Assignment          2008-12   \n",
       "1  NCT05287724       Treatment     Crossover Assignment        2022-6-19   \n",
       "2  NCT05624918       Treatment  Single Group Assignment           2024-4   \n",
       "3  NCT01003288      Prevention  Single Group Assignment          2009-10   \n",
       "4  NCT06257693       Treatment  Single Group Assignment           2024-2   \n",
       "5  NCT00003494       Treatment  Single Group Assignment        1996-3-26   \n",
       "6  NCT06096909       Treatment      Parallel Assignment        2023-11-1   \n",
       "7  NCT00012012       Treatment  Single Group Assignment           2001-8   \n",
       "8  NCT05377203       Treatment     Crossover Assignment        2022-7-13   \n",
       "9  NCT05762107      Prevention     Crossover Assignment        2023-7-28   \n",
       "\n",
       "   n_conditions  n_interventions is_fda_regulated_drug  \\\n",
       "0             1                5                     ?   \n",
       "1             2                2                     t   \n",
       "2             2                3                     f   \n",
       "3             1                1                     ?   \n",
       "4             1                1                     t   \n",
       "5             1                1                     t   \n",
       "6             2                3                     f   \n",
       "7             2                4                     ?   \n",
       "8             2                2                     f   \n",
       "9             1                4                     t   \n",
       "\n",
       "                                                                             conditions  \\\n",
       "0                                                             Postoperative Pain|-0.092   \n",
       "1                                                   Pruritus|1.000|Skin Disorder|-0.091   \n",
       "2                    Pancreatic Adenocarcinoma|1.000|Resectable Pancreatic Cancer|0.566   \n",
       "3                                                                        Healthy|-0.400   \n",
       "4                                                         Prostate Adenocarcinoma|0.033   \n",
       "5                                                       Bronchial Alveolar; Tumor|0.567   \n",
       "6  Acute Coronary Syndrome|1.000|Non ST Segment Elevation Acute Coronary Syndrome|0.279   \n",
       "7                                        Cervical Cancer|1.000|Radiation Toxicity|0.242   \n",
       "8                                        Hypertension|1.000|Arterial Hypertension|0.304   \n",
       "9                                      Type 1 Diabetes Mellitus With Hypoglycemia|0.733   \n",
       "\n",
       "                                                                                                                                                                       exp_only  \\\n",
       "0                                                                                                                                            High Dose SKY0402|Low Dose SKY0402   \n",
       "1                                                                                                                                                     N-acetyl cysteine|Placebo   \n",
       "2                                                                                                                                    Gemcitabine|Nab paclitaxel|NovoTTF-200T(P)   \n",
       "3                                                                                                                                 Adjuvanted influenza H1N1split virion vaccine   \n",
       "4                                                                                                                                                                  enzalutamide   \n",
       "5                                                                                                                                Antineoplaston therapy (Atengenal + Astugenal)   \n",
       "6                                                                                                                                                                   Tafolecimab   \n",
       "7                                                                                   Amifostine trihydrate|Cisplatin|External beam radiation therapy|Intracavitary brachytherapy   \n",
       "8  Dual combination of standard dose therapy→ Quadruple combination of half doses therapy|Quadruple combination of half doses therapy→Dual combination of standard dose therapy   \n",
       "9                                                                                                                                 Placebo|ZT-01, 15 mg|ZT-01, 22 mg|ZT-01, 7 mg   \n",
       "\n",
       "                           ctr_only  \\\n",
       "0                           Placebo   \n",
       "1                               NaN   \n",
       "2                               NaN   \n",
       "3                               NaN   \n",
       "4                               NaN   \n",
       "5                               NaN   \n",
       "6  Cholesterol Absorption Inhibitor   \n",
       "7                               NaN   \n",
       "8                               NaN   \n",
       "9                               NaN   \n",
       "\n",
       "                                        both  \\\n",
       "0  Bupivacaine via epidural|Fentanyl via PCA   \n",
       "1                                        NaN   \n",
       "2                                        NaN   \n",
       "3                                        NaN   \n",
       "4                                        NaN   \n",
       "5                                        NaN   \n",
       "6                                     Statin   \n",
       "7                                        NaN   \n",
       "8                                        NaN   \n",
       "9                                        NaN   \n",
       "\n",
       "                                          condition  \\\n",
       "0                                Postoperative Pain   \n",
       "1                                     Skin Disorder   \n",
       "2                      Resectable Pancreatic Cancer   \n",
       "3                                           Healthy   \n",
       "4                           Prostate Adenocarcinoma   \n",
       "5                         Bronchial Alveolar; Tumor   \n",
       "6  Non ST Segment Elevation Acute Coronary Syndrome   \n",
       "7                                Radiation Toxicity   \n",
       "8                             Arterial Hypertension   \n",
       "9        Type 1 Diabetes Mellitus With Hypoglycemia   \n",
       "\n",
       "                          predicate  \n",
       "0  mentioned_in_clinical_trials_for  \n",
       "1  mentioned_in_clinical_trials_for  \n",
       "2  mentioned_in_clinical_trials_for  \n",
       "3  mentioned_in_clinical_trials_for  \n",
       "4  mentioned_in_clinical_trials_for  \n",
       "5  mentioned_in_clinical_trials_for  \n",
       "6  mentioned_in_clinical_trials_for  \n",
       "7  mentioned_in_clinical_trials_for  \n",
       "8  mentioned_in_clinical_trials_for  \n",
       "9  mentioned_in_clinical_trials_for  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "104829"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "131559"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#   ---    ----  low score conditions   ----    ---   #\n",
    "new_rows = []\n",
    "for l in high_score_conditions:\n",
    "    idx = l[0]\n",
    "    corresponding_row = trials_list.loc[idx].tolist()\n",
    "    new_row = corresponding_row + [l[1]]\n",
    "    new_rows.append(new_row)\n",
    "\n",
    "# first_length = len(new_rows[0])\n",
    "# Check if all inner lists have the same length\n",
    "# all(len(sublist) == first_length for sublist in new_rows)\n",
    "\n",
    "ct_high = pd.DataFrame(new_rows)\n",
    "cols = trials_list.columns.tolist() + [\"condition\"]\n",
    "ct_high.columns = cols\n",
    "ct_high[\"predicate\"] = \"in_clinical_trials_for\"\n",
    "\n",
    "#   ---    ----  high score conditions   ----    ---   #\n",
    "new_rows = []\n",
    "for l in low_score_conditions:\n",
    "    idx = l[0]\n",
    "    corresponding_row = trials_list.loc[idx].tolist()\n",
    "    new_row = corresponding_row + [l[1]]\n",
    "    new_rows.append(new_row)\n",
    "\n",
    "# first_length = len(new_rows[0])\n",
    "# Check if all inner lists have the same length\n",
    "# all(len(sublist) == first_length for sublist in new_rows)\n",
    "\n",
    "ct_low = pd.DataFrame(new_rows)\n",
    "cols = trials_list.columns.tolist() + [\"condition\"]\n",
    "ct_low.columns = cols\n",
    "ct_low[\"predicate\"] = \"mentioned_in_clinical_trials_for\"\n",
    "\n",
    "with pd.option_context('display.max_rows', None, 'display.max_columns', None, 'display.max_colwidth', None):  # more options can be specified also\n",
    "    display(ct_high[:10])\n",
    "    \n",
    "with pd.option_context('display.max_rows', None, 'display.max_columns', None, 'display.max_colwidth', None):  # more options can be specified also\n",
    "    display(ct_low[:10])\n",
    "len(ct_high)\n",
    "len(ct_low)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "99b35463-3b15-492b-b693-17d95ee152cd",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "236388\n",
      "544550\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nct_id</th>\n",
       "      <th>primary_purpose</th>\n",
       "      <th>intervention_model</th>\n",
       "      <th>start_month_year</th>\n",
       "      <th>is_fda_regulated_drug</th>\n",
       "      <th>condition</th>\n",
       "      <th>predicate</th>\n",
       "      <th>treatment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>544540</th>\n",
       "      <td>NCT01996384</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2013-11</td>\n",
       "      <td>?</td>\n",
       "      <td>Provoked Vestibulodynia</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Non-classical acupuncture</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544541</th>\n",
       "      <td>NCT01996384</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2013-11</td>\n",
       "      <td>?</td>\n",
       "      <td>Vulvar Vestibulitis</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Non-classical acupuncture</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544542</th>\n",
       "      <td>NCT01247064</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2010-10</td>\n",
       "      <td>?</td>\n",
       "      <td>Bronchiolitis, Viral</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Nebulized 0.9% Normal Saline</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544543</th>\n",
       "      <td>NCT01247064</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2010-10</td>\n",
       "      <td>?</td>\n",
       "      <td>Saline Solution, Hypertonic</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Nebulized 0.9% Normal Saline</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544544</th>\n",
       "      <td>NCT02576574</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2015-10-29</td>\n",
       "      <td>?</td>\n",
       "      <td>First Line Non-Small Cell Lung Cancer</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Carboplatin</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544545</th>\n",
       "      <td>NCT02576574</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2015-10-29</td>\n",
       "      <td>?</td>\n",
       "      <td>First Line Non-Small Cell Lung Cancer</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Gemcitabine</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544546</th>\n",
       "      <td>NCT02576574</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2015-10-29</td>\n",
       "      <td>?</td>\n",
       "      <td>First Line Non-Small Cell Lung Cancer</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Paclitaxel</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544547</th>\n",
       "      <td>NCT02576574</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2015-10-29</td>\n",
       "      <td>?</td>\n",
       "      <td>First Line Non-Small Cell Lung Cancer</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Pemetrexed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544548</th>\n",
       "      <td>NCT04264533</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2020-2-14</td>\n",
       "      <td>f</td>\n",
       "      <td>Pneumonia, Viral</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Sterile Water for Injection</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544549</th>\n",
       "      <td>NCT04264533</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2020-2-14</td>\n",
       "      <td>f</td>\n",
       "      <td>Pneumonia, Ventilator-Associated</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Sterile Water for Injection</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             nct_id primary_purpose   intervention_model start_month_year  \\\n",
       "544540  NCT01996384       Treatment  Parallel Assignment          2013-11   \n",
       "544541  NCT01996384       Treatment  Parallel Assignment          2013-11   \n",
       "544542  NCT01247064       Treatment  Parallel Assignment          2010-10   \n",
       "544543  NCT01247064       Treatment  Parallel Assignment          2010-10   \n",
       "544544  NCT02576574       Treatment  Parallel Assignment       2015-10-29   \n",
       "544545  NCT02576574       Treatment  Parallel Assignment       2015-10-29   \n",
       "544546  NCT02576574       Treatment  Parallel Assignment       2015-10-29   \n",
       "544547  NCT02576574       Treatment  Parallel Assignment       2015-10-29   \n",
       "544548  NCT04264533       Treatment  Parallel Assignment        2020-2-14   \n",
       "544549  NCT04264533       Treatment  Parallel Assignment        2020-2-14   \n",
       "\n",
       "       is_fda_regulated_drug                              condition  \\\n",
       "544540                     ?                Provoked Vestibulodynia   \n",
       "544541                     ?                    Vulvar Vestibulitis   \n",
       "544542                     ?                   Bronchiolitis, Viral   \n",
       "544543                     ?            Saline Solution, Hypertonic   \n",
       "544544                     ?  First Line Non-Small Cell Lung Cancer   \n",
       "544545                     ?  First Line Non-Small Cell Lung Cancer   \n",
       "544546                     ?  First Line Non-Small Cell Lung Cancer   \n",
       "544547                     ?  First Line Non-Small Cell Lung Cancer   \n",
       "544548                     f                       Pneumonia, Viral   \n",
       "544549                     f       Pneumonia, Ventilator-Associated   \n",
       "\n",
       "                               predicate                     treatment  \n",
       "544540  mentioned_in_clinical_trials_for     Non-classical acupuncture  \n",
       "544541  mentioned_in_clinical_trials_for     Non-classical acupuncture  \n",
       "544542  mentioned_in_clinical_trials_for  Nebulized 0.9% Normal Saline  \n",
       "544543  mentioned_in_clinical_trials_for  Nebulized 0.9% Normal Saline  \n",
       "544544  mentioned_in_clinical_trials_for                   Carboplatin  \n",
       "544545  mentioned_in_clinical_trials_for                   Gemcitabine  \n",
       "544546  mentioned_in_clinical_trials_for                    Paclitaxel  \n",
       "544547  mentioned_in_clinical_trials_for                    Pemetrexed  \n",
       "544548  mentioned_in_clinical_trials_for   Sterile Water for Injection  \n",
       "544549  mentioned_in_clinical_trials_for   Sterile Water for Injection  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ct = pd.concat([ct_high, ct_low], axis=0)\n",
    "print(len(ct))\n",
    "\n",
    "ct_exp_only = ct.copy()\n",
    "ct_exp_only = ct_exp_only.drop([\"ctr_only\", \"both\"], axis=1).reset_index(drop=True)\n",
    "ct_exp_only[\"exp_only_copy\"] = ct_exp_only[\"exp_only\"]\n",
    "ct_exp_only.exp_only_copy = ct_exp_only.exp_only_copy.str.split('|')\n",
    "ct_exp_only = ct_exp_only.explode('exp_only_copy')\n",
    "ct_exp_only = ct_exp_only.rename(columns={'exp_only_copy': 'treatment'})\n",
    "\n",
    "ct_ctr_only = ct.copy()\n",
    "ct_ctr_only = ct_ctr_only.drop([\"exp_only\", \"both\"], axis=1).reset_index(drop=True)\n",
    "ct_ctr_only[\"ctr_only_copy\"] = ct_ctr_only[\"ctr_only\"]\n",
    "ct_ctr_only.ctr_only_copy = ct_ctr_only.ctr_only_copy.str.split('|')\n",
    "ct_ctr_only = ct_ctr_only.explode('ctr_only_copy')\n",
    "ct_ctr_only = ct_ctr_only.rename(columns={'ctr_only_copy': 'treatment'})\n",
    "ct_ctr_only[\"predicate\"] = \"mentioned_in_clinical_trials_for\"\n",
    "\n",
    "ct = pd.concat([ct_exp_only, ct_ctr_only], axis=0)\n",
    "ct.dropna(subset=['treatment'], inplace=True)\n",
    "ct = ct[~ct['treatment'].isnull()]\n",
    "ct = ct.drop([\"conditions\", \"exp_only\", \"ctr_only\", \"n_conditions\", \"n_interventions\"], axis=1).reset_index(drop=True)\n",
    "print(len(ct))\n",
    "\n",
    "with pd.option_context('display.max_rows', None, 'display.max_columns', None, 'display.max_colwidth', None):  # more options can be specified also\n",
    "    display(ct[-10:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ef48257a-25c7-436f-8c10-dd2a7603f881",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "470185\n",
      "469833\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nct_id</th>\n",
       "      <th>primary_purpose</th>\n",
       "      <th>intervention_model</th>\n",
       "      <th>start_month_year</th>\n",
       "      <th>is_fda_regulated_drug</th>\n",
       "      <th>condition</th>\n",
       "      <th>predicate</th>\n",
       "      <th>treatment</th>\n",
       "      <th>treatment_lower</th>\n",
       "      <th>condition_lower</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NCT01049399</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2009-12</td>\n",
       "      <td>?</td>\n",
       "      <td>Progressive Supranuclear Palsy</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>tideglusib</td>\n",
       "      <td>tideglusib</td>\n",
       "      <td>progressive supranuclear palsy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NCT05287724</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Crossover Assignment</td>\n",
       "      <td>2022-6-19</td>\n",
       "      <td>t</td>\n",
       "      <td>Pruritus</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>N-acetyl cysteine</td>\n",
       "      <td>n-acetyl cysteine</td>\n",
       "      <td>pruritus</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NCT01432236</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Crossover Assignment</td>\n",
       "      <td>2011-10</td>\n",
       "      <td>?</td>\n",
       "      <td>Fibromyalgia</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>Pregabalin</td>\n",
       "      <td>pregabalin</td>\n",
       "      <td>fibromyalgia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NCT04626921</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Single Group Assignment</td>\n",
       "      <td>2020-10-22</td>\n",
       "      <td>t</td>\n",
       "      <td>Relapsing Multiple Sclerosis</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>CNM-Au8</td>\n",
       "      <td>cnm-au8</td>\n",
       "      <td>relapsing multiple sclerosis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>NCT04681417</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2021-3-25</td>\n",
       "      <td>f</td>\n",
       "      <td>Retinoblastoma</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>Melphalan or Melphalan + Topotecan</td>\n",
       "      <td>melphalan or melphalan + topotecan</td>\n",
       "      <td>retinoblastoma</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>NCT04036227</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2019-7-3</td>\n",
       "      <td>f</td>\n",
       "      <td>Healthy</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>GS-248</td>\n",
       "      <td>gs-248</td>\n",
       "      <td>healthy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>NCT05624918</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Single Group Assignment</td>\n",
       "      <td>2024-4</td>\n",
       "      <td>f</td>\n",
       "      <td>Pancreatic Adenocarcinoma</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>Gemcitabine</td>\n",
       "      <td>gemcitabine</td>\n",
       "      <td>pancreatic adenocarcinoma</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>NCT05624918</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Single Group Assignment</td>\n",
       "      <td>2024-4</td>\n",
       "      <td>f</td>\n",
       "      <td>Pancreatic Adenocarcinoma</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>Nab paclitaxel</td>\n",
       "      <td>nab paclitaxel</td>\n",
       "      <td>pancreatic adenocarcinoma</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>NCT05624918</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Single Group Assignment</td>\n",
       "      <td>2024-4</td>\n",
       "      <td>f</td>\n",
       "      <td>Pancreatic Adenocarcinoma</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>NovoTTF-200T(P)</td>\n",
       "      <td>novottf-200t(p)</td>\n",
       "      <td>pancreatic adenocarcinoma</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>NCT02886065</td>\n",
       "      <td>Prevention</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2017-3-7</td>\n",
       "      <td>t</td>\n",
       "      <td>Smoldering Multiple Myeloma</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>Citarinostat</td>\n",
       "      <td>citarinostat</td>\n",
       "      <td>smoldering multiple myeloma</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         nct_id primary_purpose       intervention_model start_month_year  \\\n",
       "0   NCT01049399       Treatment      Parallel Assignment          2009-12   \n",
       "1   NCT05287724       Treatment     Crossover Assignment        2022-6-19   \n",
       "3   NCT01432236       Treatment     Crossover Assignment          2011-10   \n",
       "4   NCT04626921       Treatment  Single Group Assignment       2020-10-22   \n",
       "5   NCT04681417       Treatment      Parallel Assignment        2021-3-25   \n",
       "6   NCT04036227       Treatment      Parallel Assignment         2019-7-3   \n",
       "7   NCT05624918       Treatment  Single Group Assignment           2024-4   \n",
       "8   NCT05624918       Treatment  Single Group Assignment           2024-4   \n",
       "9   NCT05624918       Treatment  Single Group Assignment           2024-4   \n",
       "10  NCT02886065      Prevention      Parallel Assignment         2017-3-7   \n",
       "\n",
       "   is_fda_regulated_drug                       condition  \\\n",
       "0                      ?  Progressive Supranuclear Palsy   \n",
       "1                      t                        Pruritus   \n",
       "3                      ?                    Fibromyalgia   \n",
       "4                      t    Relapsing Multiple Sclerosis   \n",
       "5                      f                  Retinoblastoma   \n",
       "6                      f                         Healthy   \n",
       "7                      f       Pancreatic Adenocarcinoma   \n",
       "8                      f       Pancreatic Adenocarcinoma   \n",
       "9                      f       Pancreatic Adenocarcinoma   \n",
       "10                     t     Smoldering Multiple Myeloma   \n",
       "\n",
       "                 predicate                           treatment  \\\n",
       "0   in_clinical_trials_for                          tideglusib   \n",
       "1   in_clinical_trials_for                   N-acetyl cysteine   \n",
       "3   in_clinical_trials_for                          Pregabalin   \n",
       "4   in_clinical_trials_for                             CNM-Au8   \n",
       "5   in_clinical_trials_for  Melphalan or Melphalan + Topotecan   \n",
       "6   in_clinical_trials_for                              GS-248   \n",
       "7   in_clinical_trials_for                         Gemcitabine   \n",
       "8   in_clinical_trials_for                      Nab paclitaxel   \n",
       "9   in_clinical_trials_for                     NovoTTF-200T(P)   \n",
       "10  in_clinical_trials_for                        Citarinostat   \n",
       "\n",
       "                       treatment_lower                 condition_lower  \n",
       "0                           tideglusib  progressive supranuclear palsy  \n",
       "1                    n-acetyl cysteine                        pruritus  \n",
       "3                           pregabalin                    fibromyalgia  \n",
       "4                              cnm-au8    relapsing multiple sclerosis  \n",
       "5   melphalan or melphalan + topotecan                  retinoblastoma  \n",
       "6                               gs-248                         healthy  \n",
       "7                          gemcitabine       pancreatic adenocarcinoma  \n",
       "8                       nab paclitaxel       pancreatic adenocarcinoma  \n",
       "9                      novottf-200t(p)       pancreatic adenocarcinoma  \n",
       "10                        citarinostat     smoldering multiple myeloma  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# List of interventions to block \n",
    "undesirable_interventions = [\n",
    "    \"placebo\", \"standard of care\", \"laboratory biomarker analysis\",\n",
    "    \"questionnaire\", \"standard treatment\",\n",
    "    \"data collection\", \"educational intervention\",\n",
    "    \"intervention group\", \"training\", \"management of therapy complications\", \n",
    "    \"contingency management\", \"active control\", \"experimental group\",\n",
    "    \" sham \", \"sham intervention\", \"active comparator\",\n",
    "    \"patient navigation\", \"self-management\", \"quality of life\",\n",
    "    \"treatment group\", \"study\", \"routine care\",\n",
    "]\n",
    "\n",
    "ct[\"treatment_lower\"] = ct[\"treatment\"].str.lower()\n",
    "# Create a boolean mask for each substring and combine them with logical OR\n",
    "mask = ct['treatment_lower'].str.contains('|'.join(undesirable_interventions))\n",
    "# Filter the DataFrame using the mask and replace matching rows with None\n",
    "ct.loc[mask, 'treatment_lower'] = None\n",
    "ct = ct[ct['treatment_lower'].notna()]\n",
    "ct = ct[~ct['treatment_lower'].isnull()]\n",
    "print(len(ct))\n",
    "\n",
    "ct[\"condition_lower\"] = ct[\"condition\"].str.lower()\n",
    "# Create a boolean mask for each substring and combine them with logical OR\n",
    "mask = ct['condition_lower'].str.contains('|'.join(undesirable_interventions))\n",
    "# Filter the DataFrame using the mask and replace matching rows with None\n",
    "ct.loc[mask, 'condition_lower'] = None\n",
    "ct = ct[ct['condition_lower'].notna()]\n",
    "ct = ct[~ct['condition_lower'].isnull()]\n",
    "print(len(ct))\n",
    "\n",
    "with pd.option_context('display.max_rows', None, 'display.max_columns', None, 'display.max_colwidth', None):  # more options can be specified also\n",
    "    display(ct[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "bad249c3-1ce7-4b20-9121-734a561f9c8a",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "nct_id                   object\n",
       "primary_purpose          object\n",
       "intervention_model       object\n",
       "start_month_year         object\n",
       "is_fda_regulated_drug    object\n",
       "condition                object\n",
       "predicate                object\n",
       "treatment                object\n",
       "treatment_lower          object\n",
       "condition_lower          object\n",
       "dtype: object"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# ct.to_csv(\"ct_check.tsv\", sep='\\t', index=False, header=True)\n",
    "ct.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "b8321118-5a96-46c0-a91a-046ca692a94e",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>condition_lower</th>\n",
       "      <th>treatment_lower</th>\n",
       "      <th>condition_outside_p_1</th>\n",
       "      <th>condition_outside_p_2</th>\n",
       "      <th>condition_outside_p_3</th>\n",
       "      <th>condition_outside_p_4</th>\n",
       "      <th>condition_outside_p_5</th>\n",
       "      <th>condition_outside_p_6</th>\n",
       "      <th>condition_outside_p_7</th>\n",
       "      <th>condition_outside_p_8</th>\n",
       "      <th>condition_outside_p_9</th>\n",
       "      <th>condition_outside_p_10</th>\n",
       "      <th>condition_outside_p_11</th>\n",
       "      <th>condition_outside_p_12</th>\n",
       "      <th>condition_inside_p_1</th>\n",
       "      <th>condition_inside_p_2</th>\n",
       "      <th>condition_inside_p_3</th>\n",
       "      <th>condition_inside_p_4</th>\n",
       "      <th>condition_inside_p_5</th>\n",
       "      <th>condition_inside_p_6</th>\n",
       "      <th>treatment_outside_p_1</th>\n",
       "      <th>treatment_outside_p_2</th>\n",
       "      <th>treatment_outside_p_3</th>\n",
       "      <th>treatment_outside_p_4</th>\n",
       "      <th>treatment_outside_p_5</th>\n",
       "      <th>treatment_outside_p_6</th>\n",
       "      <th>treatment_outside_p_7</th>\n",
       "      <th>treatment_outside_p_8</th>\n",
       "      <th>treatment_outside_p_9</th>\n",
       "      <th>treatment_outside_p_10</th>\n",
       "      <th>treatment_outside_p_11</th>\n",
       "      <th>treatment_outside_p_12</th>\n",
       "      <th>treatment_outside_p_13</th>\n",
       "      <th>treatment_outside_p_14</th>\n",
       "      <th>treatment_outside_p_15</th>\n",
       "      <th>treatment_outside_p_16</th>\n",
       "      <th>treatment_inside_p_1</th>\n",
       "      <th>treatment_inside_p_2</th>\n",
       "      <th>treatment_inside_p_3</th>\n",
       "      <th>treatment_inside_p_4</th>\n",
       "      <th>treatment_inside_p_5</th>\n",
       "      <th>treatment_inside_p_6</th>\n",
       "      <th>treatment_inside_p_7</th>\n",
       "      <th>treatment_inside_p_8</th>\n",
       "      <th>treatment_inside_p_9</th>\n",
       "      <th>condition_outside_b_1</th>\n",
       "      <th>condition_outside_b_2</th>\n",
       "      <th>condition_outside_b_3</th>\n",
       "      <th>condition_outside_b_4</th>\n",
       "      <th>condition_outside_b_5</th>\n",
       "      <th>condition_outside_b_6</th>\n",
       "      <th>condition_outside_b_7</th>\n",
       "      <th>condition_outside_b_8</th>\n",
       "      <th>condition_outside_b_9</th>\n",
       "      <th>condition_outside_b_10</th>\n",
       "      <th>condition_outside_b_11</th>\n",
       "      <th>condition_outside_b_12</th>\n",
       "      <th>condition_inside_b_1</th>\n",
       "      <th>treatment_outside_b_1</th>\n",
       "      <th>treatment_outside_b_2</th>\n",
       "      <th>treatment_outside_b_3</th>\n",
       "      <th>treatment_outside_b_4</th>\n",
       "      <th>treatment_outside_b_5</th>\n",
       "      <th>treatment_outside_b_6</th>\n",
       "      <th>treatment_outside_b_7</th>\n",
       "      <th>treatment_outside_b_8</th>\n",
       "      <th>treatment_outside_b_9</th>\n",
       "      <th>treatment_outside_b_10</th>\n",
       "      <th>treatment_outside_b_11</th>\n",
       "      <th>treatment_outside_b_12</th>\n",
       "      <th>treatment_outside_b_13</th>\n",
       "      <th>treatment_outside_b_14</th>\n",
       "      <th>treatment_outside_b_15</th>\n",
       "      <th>treatment_outside_b_16</th>\n",
       "      <th>treatment_inside_b_1</th>\n",
       "      <th>treatment_inside_b_2</th>\n",
       "      <th>treatment_inside_b_3</th>\n",
       "      <th>treatment_inside_b_4</th>\n",
       "      <th>treatment_inside_b_5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>progressive supranuclear palsy</td>\n",
       "      <td>tideglusib</td>\n",
       "      <td>progressive supranuclear palsy</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>tideglusib</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>progressive supranuclear palsy</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>tideglusib</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>pruritus</td>\n",
       "      <td>n-acetyl cysteine</td>\n",
       "      <td>pruritus</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>n-acetyl cysteine</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>pruritus</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>n-acetyl cysteine</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>fibromyalgia</td>\n",
       "      <td>pregabalin</td>\n",
       "      <td>fibromyalgia</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>pregabalin</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>fibromyalgia</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>pregabalin</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>relapsing multiple sclerosis</td>\n",
       "      <td>cnm-au8</td>\n",
       "      <td>relapsing multiple sclerosis</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>cnm-au8</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>relapsing multiple sclerosis</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>cnm-au8</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>retinoblastoma</td>\n",
       "      <td>melphalan or melphalan + topotecan</td>\n",
       "      <td>retinoblastoma</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>melphalan</td>\n",
       "      <td>melphalan</td>\n",
       "      <td>topotecan</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>retinoblastoma</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>melphalan</td>\n",
       "      <td>melphalan</td>\n",
       "      <td>topotecan</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>healthy</td>\n",
       "      <td>gs-248</td>\n",
       "      <td>healthy</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>gs-248</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>healthy</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>gs-248</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>pancreatic adenocarcinoma</td>\n",
       "      <td>gemcitabine</td>\n",
       "      <td>pancreatic adenocarcinoma</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>gemcitabine</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>pancreatic adenocarcinoma</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>gemcitabine</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>pancreatic adenocarcinoma</td>\n",
       "      <td>nab paclitaxel</td>\n",
       "      <td>pancreatic adenocarcinoma</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>nab paclitaxel</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>pancreatic adenocarcinoma</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>nab paclitaxel</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>pancreatic adenocarcinoma</td>\n",
       "      <td>novottf-200t(p)</td>\n",
       "      <td>pancreatic adenocarcinoma</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>novottf-200t</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>p</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>pancreatic adenocarcinoma</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>novottf-200t(p)</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>smoldering multiple myeloma</td>\n",
       "      <td>citarinostat</td>\n",
       "      <td>smoldering multiple myeloma</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>citarinostat</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>smoldering multiple myeloma</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>citarinostat</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   condition_lower                     treatment_lower  \\\n",
       "0   progressive supranuclear palsy                          tideglusib   \n",
       "1                         pruritus                   n-acetyl cysteine   \n",
       "3                     fibromyalgia                          pregabalin   \n",
       "4     relapsing multiple sclerosis                             cnm-au8   \n",
       "5                   retinoblastoma  melphalan or melphalan + topotecan   \n",
       "6                          healthy                              gs-248   \n",
       "7        pancreatic adenocarcinoma                         gemcitabine   \n",
       "8        pancreatic adenocarcinoma                      nab paclitaxel   \n",
       "9        pancreatic adenocarcinoma                     novottf-200t(p)   \n",
       "10     smoldering multiple myeloma                        citarinostat   \n",
       "\n",
       "             condition_outside_p_1 condition_outside_p_2  \\\n",
       "0   progressive supranuclear palsy                  None   \n",
       "1                         pruritus                  None   \n",
       "3                     fibromyalgia                  None   \n",
       "4     relapsing multiple sclerosis                  None   \n",
       "5                   retinoblastoma                  None   \n",
       "6                          healthy                  None   \n",
       "7        pancreatic adenocarcinoma                  None   \n",
       "8        pancreatic adenocarcinoma                  None   \n",
       "9        pancreatic adenocarcinoma                  None   \n",
       "10     smoldering multiple myeloma                  None   \n",
       "\n",
       "   condition_outside_p_3 condition_outside_p_4 condition_outside_p_5  \\\n",
       "0                   None                  None                  None   \n",
       "1                   None                  None                  None   \n",
       "3                   None                  None                  None   \n",
       "4                   None                  None                  None   \n",
       "5                   None                  None                  None   \n",
       "6                   None                  None                  None   \n",
       "7                   None                  None                  None   \n",
       "8                   None                  None                  None   \n",
       "9                   None                  None                  None   \n",
       "10                  None                  None                  None   \n",
       "\n",
       "   condition_outside_p_6 condition_outside_p_7 condition_outside_p_8  \\\n",
       "0                   None                  None                  None   \n",
       "1                   None                  None                  None   \n",
       "3                   None                  None                  None   \n",
       "4                   None                  None                  None   \n",
       "5                   None                  None                  None   \n",
       "6                   None                  None                  None   \n",
       "7                   None                  None                  None   \n",
       "8                   None                  None                  None   \n",
       "9                   None                  None                  None   \n",
       "10                  None                  None                  None   \n",
       "\n",
       "   condition_outside_p_9 condition_outside_p_10 condition_outside_p_11  \\\n",
       "0                   None                   None                   None   \n",
       "1                   None                   None                   None   \n",
       "3                   None                   None                   None   \n",
       "4                   None                   None                   None   \n",
       "5                   None                   None                   None   \n",
       "6                   None                   None                   None   \n",
       "7                   None                   None                   None   \n",
       "8                   None                   None                   None   \n",
       "9                   None                   None                   None   \n",
       "10                  None                   None                   None   \n",
       "\n",
       "   condition_outside_p_12 condition_inside_p_1 condition_inside_p_2  \\\n",
       "0                    None                 None                 None   \n",
       "1                    None                 None                 None   \n",
       "3                    None                 None                 None   \n",
       "4                    None                 None                 None   \n",
       "5                    None                 None                 None   \n",
       "6                    None                 None                 None   \n",
       "7                    None                 None                 None   \n",
       "8                    None                 None                 None   \n",
       "9                    None                 None                 None   \n",
       "10                   None                 None                 None   \n",
       "\n",
       "   condition_inside_p_3 condition_inside_p_4 condition_inside_p_5  \\\n",
       "0                  None                 None                 None   \n",
       "1                  None                 None                 None   \n",
       "3                  None                 None                 None   \n",
       "4                  None                 None                 None   \n",
       "5                  None                 None                 None   \n",
       "6                  None                 None                 None   \n",
       "7                  None                 None                 None   \n",
       "8                  None                 None                 None   \n",
       "9                  None                 None                 None   \n",
       "10                 None                 None                 None   \n",
       "\n",
       "   condition_inside_p_6 treatment_outside_p_1 treatment_outside_p_2  \\\n",
       "0                  None            tideglusib                  None   \n",
       "1                  None     n-acetyl cysteine                  None   \n",
       "3                  None            pregabalin                  None   \n",
       "4                  None               cnm-au8                  None   \n",
       "5                  None             melphalan             melphalan   \n",
       "6                  None                gs-248                  None   \n",
       "7                  None           gemcitabine                  None   \n",
       "8                  None        nab paclitaxel                  None   \n",
       "9                  None          novottf-200t                  None   \n",
       "10                 None          citarinostat                  None   \n",
       "\n",
       "   treatment_outside_p_3 treatment_outside_p_4 treatment_outside_p_5  \\\n",
       "0                   None                  None                  None   \n",
       "1                   None                  None                  None   \n",
       "3                   None                  None                  None   \n",
       "4                   None                  None                  None   \n",
       "5              topotecan                  None                  None   \n",
       "6                   None                  None                  None   \n",
       "7                   None                  None                  None   \n",
       "8                   None                  None                  None   \n",
       "9                   None                  None                  None   \n",
       "10                  None                  None                  None   \n",
       "\n",
       "   treatment_outside_p_6 treatment_outside_p_7 treatment_outside_p_8  \\\n",
       "0                   None                  None                  None   \n",
       "1                   None                  None                  None   \n",
       "3                   None                  None                  None   \n",
       "4                   None                  None                  None   \n",
       "5                   None                  None                  None   \n",
       "6                   None                  None                  None   \n",
       "7                   None                  None                  None   \n",
       "8                   None                  None                  None   \n",
       "9                   None                  None                  None   \n",
       "10                  None                  None                  None   \n",
       "\n",
       "   treatment_outside_p_9 treatment_outside_p_10 treatment_outside_p_11  \\\n",
       "0                   None                   None                   None   \n",
       "1                   None                   None                   None   \n",
       "3                   None                   None                   None   \n",
       "4                   None                   None                   None   \n",
       "5                   None                   None                   None   \n",
       "6                   None                   None                   None   \n",
       "7                   None                   None                   None   \n",
       "8                   None                   None                   None   \n",
       "9                   None                   None                   None   \n",
       "10                  None                   None                   None   \n",
       "\n",
       "   treatment_outside_p_12 treatment_outside_p_13 treatment_outside_p_14  \\\n",
       "0                    None                   None                   None   \n",
       "1                    None                   None                   None   \n",
       "3                    None                   None                   None   \n",
       "4                    None                   None                   None   \n",
       "5                    None                   None                   None   \n",
       "6                    None                   None                   None   \n",
       "7                    None                   None                   None   \n",
       "8                    None                   None                   None   \n",
       "9                    None                   None                   None   \n",
       "10                   None                   None                   None   \n",
       "\n",
       "   treatment_outside_p_15 treatment_outside_p_16 treatment_inside_p_1  \\\n",
       "0                    None                   None                 None   \n",
       "1                    None                   None                 None   \n",
       "3                    None                   None                 None   \n",
       "4                    None                   None                 None   \n",
       "5                    None                   None                 None   \n",
       "6                    None                   None                 None   \n",
       "7                    None                   None                 None   \n",
       "8                    None                   None                 None   \n",
       "9                    None                   None                    p   \n",
       "10                   None                   None                 None   \n",
       "\n",
       "   treatment_inside_p_2 treatment_inside_p_3 treatment_inside_p_4  \\\n",
       "0                  None                 None                 None   \n",
       "1                  None                 None                 None   \n",
       "3                  None                 None                 None   \n",
       "4                  None                 None                 None   \n",
       "5                  None                 None                 None   \n",
       "6                  None                 None                 None   \n",
       "7                  None                 None                 None   \n",
       "8                  None                 None                 None   \n",
       "9                  None                 None                 None   \n",
       "10                 None                 None                 None   \n",
       "\n",
       "   treatment_inside_p_5 treatment_inside_p_6 treatment_inside_p_7  \\\n",
       "0                  None                 None                 None   \n",
       "1                  None                 None                 None   \n",
       "3                  None                 None                 None   \n",
       "4                  None                 None                 None   \n",
       "5                  None                 None                 None   \n",
       "6                  None                 None                 None   \n",
       "7                  None                 None                 None   \n",
       "8                  None                 None                 None   \n",
       "9                  None                 None                 None   \n",
       "10                 None                 None                 None   \n",
       "\n",
       "   treatment_inside_p_8 treatment_inside_p_9           condition_outside_b_1  \\\n",
       "0                  None                 None  progressive supranuclear palsy   \n",
       "1                  None                 None                        pruritus   \n",
       "3                  None                 None                    fibromyalgia   \n",
       "4                  None                 None    relapsing multiple sclerosis   \n",
       "5                  None                 None                  retinoblastoma   \n",
       "6                  None                 None                         healthy   \n",
       "7                  None                 None       pancreatic adenocarcinoma   \n",
       "8                  None                 None       pancreatic adenocarcinoma   \n",
       "9                  None                 None       pancreatic adenocarcinoma   \n",
       "10                 None                 None     smoldering multiple myeloma   \n",
       "\n",
       "   condition_outside_b_2 condition_outside_b_3 condition_outside_b_4  \\\n",
       "0                   None                  None                  None   \n",
       "1                   None                  None                  None   \n",
       "3                   None                  None                  None   \n",
       "4                   None                  None                  None   \n",
       "5                   None                  None                  None   \n",
       "6                   None                  None                  None   \n",
       "7                   None                  None                  None   \n",
       "8                   None                  None                  None   \n",
       "9                   None                  None                  None   \n",
       "10                  None                  None                  None   \n",
       "\n",
       "   condition_outside_b_5 condition_outside_b_6 condition_outside_b_7  \\\n",
       "0                   None                  None                  None   \n",
       "1                   None                  None                  None   \n",
       "3                   None                  None                  None   \n",
       "4                   None                  None                  None   \n",
       "5                   None                  None                  None   \n",
       "6                   None                  None                  None   \n",
       "7                   None                  None                  None   \n",
       "8                   None                  None                  None   \n",
       "9                   None                  None                  None   \n",
       "10                  None                  None                  None   \n",
       "\n",
       "   condition_outside_b_8 condition_outside_b_9 condition_outside_b_10  \\\n",
       "0                   None                  None                   None   \n",
       "1                   None                  None                   None   \n",
       "3                   None                  None                   None   \n",
       "4                   None                  None                   None   \n",
       "5                   None                  None                   None   \n",
       "6                   None                  None                   None   \n",
       "7                   None                  None                   None   \n",
       "8                   None                  None                   None   \n",
       "9                   None                  None                   None   \n",
       "10                  None                  None                   None   \n",
       "\n",
       "   condition_outside_b_11 condition_outside_b_12 condition_inside_b_1  \\\n",
       "0                    None                   None                 None   \n",
       "1                    None                   None                 None   \n",
       "3                    None                   None                 None   \n",
       "4                    None                   None                 None   \n",
       "5                    None                   None                 None   \n",
       "6                    None                   None                 None   \n",
       "7                    None                   None                 None   \n",
       "8                    None                   None                 None   \n",
       "9                    None                   None                 None   \n",
       "10                   None                   None                 None   \n",
       "\n",
       "   treatment_outside_b_1 treatment_outside_b_2 treatment_outside_b_3  \\\n",
       "0             tideglusib                  None                  None   \n",
       "1      n-acetyl cysteine                  None                  None   \n",
       "3             pregabalin                  None                  None   \n",
       "4                cnm-au8                  None                  None   \n",
       "5              melphalan             melphalan             topotecan   \n",
       "6                 gs-248                  None                  None   \n",
       "7            gemcitabine                  None                  None   \n",
       "8         nab paclitaxel                  None                  None   \n",
       "9        novottf-200t(p)                  None                  None   \n",
       "10          citarinostat                  None                  None   \n",
       "\n",
       "   treatment_outside_b_4 treatment_outside_b_5 treatment_outside_b_6  \\\n",
       "0                   None                  None                  None   \n",
       "1                   None                  None                  None   \n",
       "3                   None                  None                  None   \n",
       "4                   None                  None                  None   \n",
       "5                   None                  None                  None   \n",
       "6                   None                  None                  None   \n",
       "7                   None                  None                  None   \n",
       "8                   None                  None                  None   \n",
       "9                   None                  None                  None   \n",
       "10                  None                  None                  None   \n",
       "\n",
       "   treatment_outside_b_7 treatment_outside_b_8 treatment_outside_b_9  \\\n",
       "0                   None                  None                  None   \n",
       "1                   None                  None                  None   \n",
       "3                   None                  None                  None   \n",
       "4                   None                  None                  None   \n",
       "5                   None                  None                  None   \n",
       "6                   None                  None                  None   \n",
       "7                   None                  None                  None   \n",
       "8                   None                  None                  None   \n",
       "9                   None                  None                  None   \n",
       "10                  None                  None                  None   \n",
       "\n",
       "   treatment_outside_b_10 treatment_outside_b_11 treatment_outside_b_12  \\\n",
       "0                    None                   None                   None   \n",
       "1                    None                   None                   None   \n",
       "3                    None                   None                   None   \n",
       "4                    None                   None                   None   \n",
       "5                    None                   None                   None   \n",
       "6                    None                   None                   None   \n",
       "7                    None                   None                   None   \n",
       "8                    None                   None                   None   \n",
       "9                    None                   None                   None   \n",
       "10                   None                   None                   None   \n",
       "\n",
       "   treatment_outside_b_13 treatment_outside_b_14 treatment_outside_b_15  \\\n",
       "0                    None                   None                   None   \n",
       "1                    None                   None                   None   \n",
       "3                    None                   None                   None   \n",
       "4                    None                   None                   None   \n",
       "5                    None                   None                   None   \n",
       "6                    None                   None                   None   \n",
       "7                    None                   None                   None   \n",
       "8                    None                   None                   None   \n",
       "9                    None                   None                   None   \n",
       "10                   None                   None                   None   \n",
       "\n",
       "   treatment_outside_b_16 treatment_inside_b_1 treatment_inside_b_2  \\\n",
       "0                    None                 None                 None   \n",
       "1                    None                 None                 None   \n",
       "3                    None                 None                 None   \n",
       "4                    None                 None                 None   \n",
       "5                    None                 None                 None   \n",
       "6                    None                 None                 None   \n",
       "7                    None                 None                 None   \n",
       "8                    None                 None                 None   \n",
       "9                    None                 None                 None   \n",
       "10                   None                 None                 None   \n",
       "\n",
       "   treatment_inside_b_3 treatment_inside_b_4 treatment_inside_b_5  \n",
       "0                  None                 None                 None  \n",
       "1                  None                 None                 None  \n",
       "3                  None                 None                 None  \n",
       "4                  None                 None                 None  \n",
       "5                  None                 None                 None  \n",
       "6                  None                 None                 None  \n",
       "7                  None                 None                 None  \n",
       "8                  None                 None                 None  \n",
       "9                  None                 None                 None  \n",
       "10                 None                 None                 None  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ct['condition_outside_p'] =  [re.sub(r\"\\((.*?)\\)\",'', str(x)) for x in ct['condition_lower']]\n",
    "ct['condition_inside_p'] = ct['condition_lower'].str.extract(r\"\\((.*?)\\)\", expand=True)\n",
    "\n",
    "ct['treatment_outside_p'] =  [re.sub(r\"\\((.*?)\\)\",'', str(x)) for x in ct['treatment_lower']]\n",
    "ct['treatment_inside_p'] = ct['treatment_lower'].str.extract(r\"\\((.*?)\\)\", expand=True)\n",
    "\n",
    "ct['condition_outside_b'] =  [re.sub(r'\\[([^\\]]+)\\]','', str(x)) for x in ct['condition_lower']]\n",
    "ct['condition_inside_b'] = ct['condition_lower'].str.extract(r'\\[([^\\]]+)\\]', expand=True)\n",
    "\n",
    "ct['treatment_outside_b'] =  [re.sub(r'\\[([^\\]]+)\\]','', str(x)) for x in ct['treatment_lower']]\n",
    "ct['treatment_inside_b'] = ct['treatment_lower'].str.extract(r'\\[([^\\]]+)\\]', expand=True)\n",
    "\n",
    "ct = ct.apply(lambda x: x.str.strip() if x.dtype == \"object\" else x) # strip leading and trailing whitespace\n",
    "ct = ct[~ct['condition_lower'].isnull()]\n",
    "ct = ct[~ct['treatment_lower'].isnull()]\n",
    "\n",
    "split_chars = [\",\", \"+\", \"/\", \" and \", \"&\", \" or \"]\n",
    "split_chars = [re.escape(char) for char in split_chars]\n",
    "\n",
    "# Function to split column based on list of characters\n",
    "def split_column(df, column, split_chars):\n",
    "    # Create a temporary dataframe to store split values\n",
    "    temp_df = df[column].str.split('|'.join(split_chars), expand=True)\n",
    "    # Rename columns of the temporary dataframe\n",
    "    temp_df.columns = [f'{column}_{i+1}' for i in range(temp_df.shape[1])]\n",
    "    return temp_df\n",
    "\n",
    "# Apply split_column function to the desired column\n",
    "split_con_outside_p = split_column(ct, 'condition_outside_p', split_chars)\n",
    "split_con_inside_p = split_column(ct, 'condition_inside_p', split_chars)\n",
    "split_trmnt_outside_p = split_column(ct, 'treatment_outside_p', split_chars)\n",
    "split_trmnt_inside_p = split_column(ct, 'treatment_inside_p', split_chars)\n",
    "\n",
    "split_con_outside_b = split_column(ct, 'condition_outside_b', split_chars)\n",
    "split_con_inside_b = split_column(ct, 'condition_inside_b', split_chars)\n",
    "split_trmnt_outside_b = split_column(ct, 'treatment_outside_b', split_chars)\n",
    "split_trmnt_inside_b = split_column(ct, 'treatment_inside_b', split_chars)\n",
    "\n",
    "# Concatenate the split dataframe with the original dataframe\n",
    "split_df = pd.concat([split_con_outside_p, split_con_inside_p, split_trmnt_outside_p, split_trmnt_inside_p,\n",
    "                     split_con_outside_b, split_con_inside_b, split_trmnt_outside_b, split_trmnt_inside_b], axis=1)\n",
    "split_df = split_df.fillna(np.nan).replace([np.nan], [None]) # replace NaN values in dataframe with None (just for consistency)\n",
    "split_df = pd.concat([ct[\"condition_lower\"], ct[\"treatment_lower\"], split_df], axis=1)\n",
    "\n",
    "split_df = split_df.apply(lambda x: x.str.strip() if x.dtype == \"object\" else x) # strip leading and trailing whitespace\n",
    "\n",
    "with pd.option_context('display.max_rows', None, 'display.max_columns', None, 'display.max_colwidth', None):  # more options can be specified also\n",
    "    display(split_df[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "1e9b1b15-8a5a-4c19-b4ab-36684ff90750",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>condition_lower</th>\n",
       "      <th>treatment_lower</th>\n",
       "      <th>condition_outside_p_1</th>\n",
       "      <th>condition_outside_p_2</th>\n",
       "      <th>condition_outside_p_3</th>\n",
       "      <th>condition_outside_p_4</th>\n",
       "      <th>condition_outside_p_5</th>\n",
       "      <th>condition_outside_p_6</th>\n",
       "      <th>condition_outside_p_7</th>\n",
       "      <th>condition_outside_p_8</th>\n",
       "      <th>condition_outside_p_9</th>\n",
       "      <th>condition_outside_p_10</th>\n",
       "      <th>condition_outside_p_11</th>\n",
       "      <th>condition_outside_p_12</th>\n",
       "      <th>condition_inside_p_1</th>\n",
       "      <th>condition_inside_p_2</th>\n",
       "      <th>condition_inside_p_3</th>\n",
       "      <th>condition_inside_p_4</th>\n",
       "      <th>condition_inside_p_5</th>\n",
       "      <th>condition_inside_p_6</th>\n",
       "      <th>treatment_outside_p_1</th>\n",
       "      <th>treatment_outside_p_2</th>\n",
       "      <th>treatment_outside_p_3</th>\n",
       "      <th>treatment_outside_p_4</th>\n",
       "      <th>treatment_outside_p_5</th>\n",
       "      <th>treatment_outside_p_6</th>\n",
       "      <th>treatment_outside_p_7</th>\n",
       "      <th>treatment_outside_p_8</th>\n",
       "      <th>treatment_outside_p_9</th>\n",
       "      <th>treatment_outside_p_10</th>\n",
       "      <th>treatment_outside_p_11</th>\n",
       "      <th>treatment_outside_p_12</th>\n",
       "      <th>treatment_outside_p_13</th>\n",
       "      <th>treatment_outside_p_14</th>\n",
       "      <th>treatment_outside_p_15</th>\n",
       "      <th>treatment_outside_p_16</th>\n",
       "      <th>treatment_inside_p_1</th>\n",
       "      <th>treatment_inside_p_2</th>\n",
       "      <th>treatment_inside_p_3</th>\n",
       "      <th>treatment_inside_p_4</th>\n",
       "      <th>treatment_inside_p_5</th>\n",
       "      <th>treatment_inside_p_6</th>\n",
       "      <th>treatment_inside_p_7</th>\n",
       "      <th>treatment_inside_p_8</th>\n",
       "      <th>treatment_inside_p_9</th>\n",
       "      <th>condition_outside_b_1</th>\n",
       "      <th>condition_outside_b_2</th>\n",
       "      <th>condition_outside_b_3</th>\n",
       "      <th>condition_outside_b_4</th>\n",
       "      <th>condition_outside_b_5</th>\n",
       "      <th>condition_outside_b_6</th>\n",
       "      <th>condition_outside_b_7</th>\n",
       "      <th>condition_outside_b_8</th>\n",
       "      <th>condition_outside_b_9</th>\n",
       "      <th>condition_outside_b_10</th>\n",
       "      <th>condition_outside_b_11</th>\n",
       "      <th>condition_outside_b_12</th>\n",
       "      <th>condition_inside_b_1</th>\n",
       "      <th>treatment_outside_b_1</th>\n",
       "      <th>treatment_outside_b_2</th>\n",
       "      <th>treatment_outside_b_3</th>\n",
       "      <th>treatment_outside_b_4</th>\n",
       "      <th>treatment_outside_b_5</th>\n",
       "      <th>treatment_outside_b_6</th>\n",
       "      <th>treatment_outside_b_7</th>\n",
       "      <th>treatment_outside_b_8</th>\n",
       "      <th>treatment_outside_b_9</th>\n",
       "      <th>treatment_outside_b_10</th>\n",
       "      <th>treatment_outside_b_11</th>\n",
       "      <th>treatment_outside_b_12</th>\n",
       "      <th>treatment_outside_b_13</th>\n",
       "      <th>treatment_outside_b_14</th>\n",
       "      <th>treatment_outside_b_15</th>\n",
       "      <th>treatment_outside_b_16</th>\n",
       "      <th>treatment_inside_b_1</th>\n",
       "      <th>treatment_inside_b_2</th>\n",
       "      <th>treatment_inside_b_3</th>\n",
       "      <th>treatment_inside_b_4</th>\n",
       "      <th>treatment_inside_b_5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>progressive supranuclear palsy</td>\n",
       "      <td>tideglusib</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>pruritus</td>\n",
       "      <td>n-acetyl cysteine</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>fibromyalgia</td>\n",
       "      <td>pregabalin</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>relapsing multiple sclerosis</td>\n",
       "      <td>cnm-au8</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>retinoblastoma</td>\n",
       "      <td>melphalan or melphalan + topotecan</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>melphalan</td>\n",
       "      <td>None</td>\n",
       "      <td>topotecan</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>healthy</td>\n",
       "      <td>gs-248</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>pancreatic adenocarcinoma</td>\n",
       "      <td>gemcitabine</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>pancreatic adenocarcinoma</td>\n",
       "      <td>nab paclitaxel</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>pancreatic adenocarcinoma</td>\n",
       "      <td>novottf-200t(p)</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>novottf-200t</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>p</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>smoldering multiple myeloma</td>\n",
       "      <td>citarinostat</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   condition_lower                     treatment_lower  \\\n",
       "0   progressive supranuclear palsy                          tideglusib   \n",
       "1                         pruritus                   n-acetyl cysteine   \n",
       "3                     fibromyalgia                          pregabalin   \n",
       "4     relapsing multiple sclerosis                             cnm-au8   \n",
       "5                   retinoblastoma  melphalan or melphalan + topotecan   \n",
       "6                          healthy                              gs-248   \n",
       "7        pancreatic adenocarcinoma                         gemcitabine   \n",
       "8        pancreatic adenocarcinoma                      nab paclitaxel   \n",
       "9        pancreatic adenocarcinoma                     novottf-200t(p)   \n",
       "10     smoldering multiple myeloma                        citarinostat   \n",
       "\n",
       "   condition_outside_p_1 condition_outside_p_2 condition_outside_p_3  \\\n",
       "0                   None                  None                  None   \n",
       "1                   None                  None                  None   \n",
       "3                   None                  None                  None   \n",
       "4                   None                  None                  None   \n",
       "5                   None                  None                  None   \n",
       "6                   None                  None                  None   \n",
       "7                   None                  None                  None   \n",
       "8                   None                  None                  None   \n",
       "9                   None                  None                  None   \n",
       "10                  None                  None                  None   \n",
       "\n",
       "   condition_outside_p_4 condition_outside_p_5 condition_outside_p_6  \\\n",
       "0                   None                  None                  None   \n",
       "1                   None                  None                  None   \n",
       "3                   None                  None                  None   \n",
       "4                   None                  None                  None   \n",
       "5                   None                  None                  None   \n",
       "6                   None                  None                  None   \n",
       "7                   None                  None                  None   \n",
       "8                   None                  None                  None   \n",
       "9                   None                  None                  None   \n",
       "10                  None                  None                  None   \n",
       "\n",
       "   condition_outside_p_7 condition_outside_p_8 condition_outside_p_9  \\\n",
       "0                   None                  None                  None   \n",
       "1                   None                  None                  None   \n",
       "3                   None                  None                  None   \n",
       "4                   None                  None                  None   \n",
       "5                   None                  None                  None   \n",
       "6                   None                  None                  None   \n",
       "7                   None                  None                  None   \n",
       "8                   None                  None                  None   \n",
       "9                   None                  None                  None   \n",
       "10                  None                  None                  None   \n",
       "\n",
       "   condition_outside_p_10 condition_outside_p_11 condition_outside_p_12  \\\n",
       "0                    None                   None                   None   \n",
       "1                    None                   None                   None   \n",
       "3                    None                   None                   None   \n",
       "4                    None                   None                   None   \n",
       "5                    None                   None                   None   \n",
       "6                    None                   None                   None   \n",
       "7                    None                   None                   None   \n",
       "8                    None                   None                   None   \n",
       "9                    None                   None                   None   \n",
       "10                   None                   None                   None   \n",
       "\n",
       "   condition_inside_p_1 condition_inside_p_2 condition_inside_p_3  \\\n",
       "0                  None                 None                 None   \n",
       "1                  None                 None                 None   \n",
       "3                  None                 None                 None   \n",
       "4                  None                 None                 None   \n",
       "5                  None                 None                 None   \n",
       "6                  None                 None                 None   \n",
       "7                  None                 None                 None   \n",
       "8                  None                 None                 None   \n",
       "9                  None                 None                 None   \n",
       "10                 None                 None                 None   \n",
       "\n",
       "   condition_inside_p_4 condition_inside_p_5 condition_inside_p_6  \\\n",
       "0                  None                 None                 None   \n",
       "1                  None                 None                 None   \n",
       "3                  None                 None                 None   \n",
       "4                  None                 None                 None   \n",
       "5                  None                 None                 None   \n",
       "6                  None                 None                 None   \n",
       "7                  None                 None                 None   \n",
       "8                  None                 None                 None   \n",
       "9                  None                 None                 None   \n",
       "10                 None                 None                 None   \n",
       "\n",
       "   treatment_outside_p_1 treatment_outside_p_2 treatment_outside_p_3  \\\n",
       "0                   None                  None                  None   \n",
       "1                   None                  None                  None   \n",
       "3                   None                  None                  None   \n",
       "4                   None                  None                  None   \n",
       "5              melphalan                  None             topotecan   \n",
       "6                   None                  None                  None   \n",
       "7                   None                  None                  None   \n",
       "8                   None                  None                  None   \n",
       "9           novottf-200t                  None                  None   \n",
       "10                  None                  None                  None   \n",
       "\n",
       "   treatment_outside_p_4 treatment_outside_p_5 treatment_outside_p_6  \\\n",
       "0                   None                  None                  None   \n",
       "1                   None                  None                  None   \n",
       "3                   None                  None                  None   \n",
       "4                   None                  None                  None   \n",
       "5                   None                  None                  None   \n",
       "6                   None                  None                  None   \n",
       "7                   None                  None                  None   \n",
       "8                   None                  None                  None   \n",
       "9                   None                  None                  None   \n",
       "10                  None                  None                  None   \n",
       "\n",
       "   treatment_outside_p_7 treatment_outside_p_8 treatment_outside_p_9  \\\n",
       "0                   None                  None                  None   \n",
       "1                   None                  None                  None   \n",
       "3                   None                  None                  None   \n",
       "4                   None                  None                  None   \n",
       "5                   None                  None                  None   \n",
       "6                   None                  None                  None   \n",
       "7                   None                  None                  None   \n",
       "8                   None                  None                  None   \n",
       "9                   None                  None                  None   \n",
       "10                  None                  None                  None   \n",
       "\n",
       "   treatment_outside_p_10 treatment_outside_p_11 treatment_outside_p_12  \\\n",
       "0                    None                   None                   None   \n",
       "1                    None                   None                   None   \n",
       "3                    None                   None                   None   \n",
       "4                    None                   None                   None   \n",
       "5                    None                   None                   None   \n",
       "6                    None                   None                   None   \n",
       "7                    None                   None                   None   \n",
       "8                    None                   None                   None   \n",
       "9                    None                   None                   None   \n",
       "10                   None                   None                   None   \n",
       "\n",
       "   treatment_outside_p_13 treatment_outside_p_14 treatment_outside_p_15  \\\n",
       "0                    None                   None                   None   \n",
       "1                    None                   None                   None   \n",
       "3                    None                   None                   None   \n",
       "4                    None                   None                   None   \n",
       "5                    None                   None                   None   \n",
       "6                    None                   None                   None   \n",
       "7                    None                   None                   None   \n",
       "8                    None                   None                   None   \n",
       "9                    None                   None                   None   \n",
       "10                   None                   None                   None   \n",
       "\n",
       "   treatment_outside_p_16 treatment_inside_p_1 treatment_inside_p_2  \\\n",
       "0                    None                 None                 None   \n",
       "1                    None                 None                 None   \n",
       "3                    None                 None                 None   \n",
       "4                    None                 None                 None   \n",
       "5                    None                 None                 None   \n",
       "6                    None                 None                 None   \n",
       "7                    None                 None                 None   \n",
       "8                    None                 None                 None   \n",
       "9                    None                    p                 None   \n",
       "10                   None                 None                 None   \n",
       "\n",
       "   treatment_inside_p_3 treatment_inside_p_4 treatment_inside_p_5  \\\n",
       "0                  None                 None                 None   \n",
       "1                  None                 None                 None   \n",
       "3                  None                 None                 None   \n",
       "4                  None                 None                 None   \n",
       "5                  None                 None                 None   \n",
       "6                  None                 None                 None   \n",
       "7                  None                 None                 None   \n",
       "8                  None                 None                 None   \n",
       "9                  None                 None                 None   \n",
       "10                 None                 None                 None   \n",
       "\n",
       "   treatment_inside_p_6 treatment_inside_p_7 treatment_inside_p_8  \\\n",
       "0                  None                 None                 None   \n",
       "1                  None                 None                 None   \n",
       "3                  None                 None                 None   \n",
       "4                  None                 None                 None   \n",
       "5                  None                 None                 None   \n",
       "6                  None                 None                 None   \n",
       "7                  None                 None                 None   \n",
       "8                  None                 None                 None   \n",
       "9                  None                 None                 None   \n",
       "10                 None                 None                 None   \n",
       "\n",
       "   treatment_inside_p_9 condition_outside_b_1 condition_outside_b_2  \\\n",
       "0                  None                  None                  None   \n",
       "1                  None                  None                  None   \n",
       "3                  None                  None                  None   \n",
       "4                  None                  None                  None   \n",
       "5                  None                  None                  None   \n",
       "6                  None                  None                  None   \n",
       "7                  None                  None                  None   \n",
       "8                  None                  None                  None   \n",
       "9                  None                  None                  None   \n",
       "10                 None                  None                  None   \n",
       "\n",
       "   condition_outside_b_3 condition_outside_b_4 condition_outside_b_5  \\\n",
       "0                   None                  None                  None   \n",
       "1                   None                  None                  None   \n",
       "3                   None                  None                  None   \n",
       "4                   None                  None                  None   \n",
       "5                   None                  None                  None   \n",
       "6                   None                  None                  None   \n",
       "7                   None                  None                  None   \n",
       "8                   None                  None                  None   \n",
       "9                   None                  None                  None   \n",
       "10                  None                  None                  None   \n",
       "\n",
       "   condition_outside_b_6 condition_outside_b_7 condition_outside_b_8  \\\n",
       "0                   None                  None                  None   \n",
       "1                   None                  None                  None   \n",
       "3                   None                  None                  None   \n",
       "4                   None                  None                  None   \n",
       "5                   None                  None                  None   \n",
       "6                   None                  None                  None   \n",
       "7                   None                  None                  None   \n",
       "8                   None                  None                  None   \n",
       "9                   None                  None                  None   \n",
       "10                  None                  None                  None   \n",
       "\n",
       "   condition_outside_b_9 condition_outside_b_10 condition_outside_b_11  \\\n",
       "0                   None                   None                   None   \n",
       "1                   None                   None                   None   \n",
       "3                   None                   None                   None   \n",
       "4                   None                   None                   None   \n",
       "5                   None                   None                   None   \n",
       "6                   None                   None                   None   \n",
       "7                   None                   None                   None   \n",
       "8                   None                   None                   None   \n",
       "9                   None                   None                   None   \n",
       "10                  None                   None                   None   \n",
       "\n",
       "   condition_outside_b_12 condition_inside_b_1 treatment_outside_b_1  \\\n",
       "0                    None                 None                  None   \n",
       "1                    None                 None                  None   \n",
       "3                    None                 None                  None   \n",
       "4                    None                 None                  None   \n",
       "5                    None                 None                  None   \n",
       "6                    None                 None                  None   \n",
       "7                    None                 None                  None   \n",
       "8                    None                 None                  None   \n",
       "9                    None                 None                  None   \n",
       "10                   None                 None                  None   \n",
       "\n",
       "   treatment_outside_b_2 treatment_outside_b_3 treatment_outside_b_4  \\\n",
       "0                   None                  None                  None   \n",
       "1                   None                  None                  None   \n",
       "3                   None                  None                  None   \n",
       "4                   None                  None                  None   \n",
       "5                   None                  None                  None   \n",
       "6                   None                  None                  None   \n",
       "7                   None                  None                  None   \n",
       "8                   None                  None                  None   \n",
       "9                   None                  None                  None   \n",
       "10                  None                  None                  None   \n",
       "\n",
       "   treatment_outside_b_5 treatment_outside_b_6 treatment_outside_b_7  \\\n",
       "0                   None                  None                  None   \n",
       "1                   None                  None                  None   \n",
       "3                   None                  None                  None   \n",
       "4                   None                  None                  None   \n",
       "5                   None                  None                  None   \n",
       "6                   None                  None                  None   \n",
       "7                   None                  None                  None   \n",
       "8                   None                  None                  None   \n",
       "9                   None                  None                  None   \n",
       "10                  None                  None                  None   \n",
       "\n",
       "   treatment_outside_b_8 treatment_outside_b_9 treatment_outside_b_10  \\\n",
       "0                   None                  None                   None   \n",
       "1                   None                  None                   None   \n",
       "3                   None                  None                   None   \n",
       "4                   None                  None                   None   \n",
       "5                   None                  None                   None   \n",
       "6                   None                  None                   None   \n",
       "7                   None                  None                   None   \n",
       "8                   None                  None                   None   \n",
       "9                   None                  None                   None   \n",
       "10                  None                  None                   None   \n",
       "\n",
       "   treatment_outside_b_11 treatment_outside_b_12 treatment_outside_b_13  \\\n",
       "0                    None                   None                   None   \n",
       "1                    None                   None                   None   \n",
       "3                    None                   None                   None   \n",
       "4                    None                   None                   None   \n",
       "5                    None                   None                   None   \n",
       "6                    None                   None                   None   \n",
       "7                    None                   None                   None   \n",
       "8                    None                   None                   None   \n",
       "9                    None                   None                   None   \n",
       "10                   None                   None                   None   \n",
       "\n",
       "   treatment_outside_b_14 treatment_outside_b_15 treatment_outside_b_16  \\\n",
       "0                    None                   None                   None   \n",
       "1                    None                   None                   None   \n",
       "3                    None                   None                   None   \n",
       "4                    None                   None                   None   \n",
       "5                    None                   None                   None   \n",
       "6                    None                   None                   None   \n",
       "7                    None                   None                   None   \n",
       "8                    None                   None                   None   \n",
       "9                    None                   None                   None   \n",
       "10                   None                   None                   None   \n",
       "\n",
       "   treatment_inside_b_1 treatment_inside_b_2 treatment_inside_b_3  \\\n",
       "0                  None                 None                 None   \n",
       "1                  None                 None                 None   \n",
       "3                  None                 None                 None   \n",
       "4                  None                 None                 None   \n",
       "5                  None                 None                 None   \n",
       "6                  None                 None                 None   \n",
       "7                  None                 None                 None   \n",
       "8                  None                 None                 None   \n",
       "9                  None                 None                 None   \n",
       "10                 None                 None                 None   \n",
       "\n",
       "   treatment_inside_b_4 treatment_inside_b_5  \n",
       "0                  None                 None  \n",
       "1                  None                 None  \n",
       "3                  None                 None  \n",
       "4                  None                 None  \n",
       "5                  None                 None  \n",
       "6                  None                 None  \n",
       "7                  None                 None  \n",
       "8                  None                 None  \n",
       "9                  None                 None  \n",
       "10                 None                 None  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "split_t = split_df.transpose()\n",
    "split_t = split_t.apply(lambda x: x.mask(x.duplicated(), None))\n",
    "split_df = split_t.transpose()\n",
    "split_df = split_df[~split_df['condition_lower'].isnull()]\n",
    "split_df = split_df[~split_df['treatment_lower'].isnull()]\n",
    "\n",
    "with pd.option_context('display.max_rows', None, 'display.max_columns', None, 'display.max_colwidth', None):  # more options can be specified also\n",
    "    display(split_df[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "f0018c36-8b29-4ce5-9146-34677e603aca",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "469833"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "469034"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ct)\n",
    "len(split_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "0c2e874a-4aa8-44cd-af75-e6b12a446bde",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>condition_lower</th>\n",
       "      <th>treatment_lower</th>\n",
       "      <th>condition_outside_p_1</th>\n",
       "      <th>condition_outside_p_2</th>\n",
       "      <th>condition_outside_p_3</th>\n",
       "      <th>condition_outside_p_4</th>\n",
       "      <th>condition_outside_p_5</th>\n",
       "      <th>condition_outside_p_6</th>\n",
       "      <th>condition_outside_p_7</th>\n",
       "      <th>condition_outside_p_8</th>\n",
       "      <th>...</th>\n",
       "      <th>treatment_outside_b_12</th>\n",
       "      <th>treatment_outside_b_13</th>\n",
       "      <th>treatment_outside_b_14</th>\n",
       "      <th>treatment_outside_b_15</th>\n",
       "      <th>treatment_outside_b_16</th>\n",
       "      <th>treatment_inside_b_1</th>\n",
       "      <th>treatment_inside_b_2</th>\n",
       "      <th>treatment_inside_b_3</th>\n",
       "      <th>treatment_inside_b_4</th>\n",
       "      <th>treatment_inside_b_5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>0 rows × 79 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [condition_lower, treatment_lower, condition_outside_p_1, condition_outside_p_2, condition_outside_p_3, condition_outside_p_4, condition_outside_p_5, condition_outside_p_6, condition_outside_p_7, condition_outside_p_8, condition_outside_p_9, condition_outside_p_10, condition_outside_p_11, condition_outside_p_12, condition_inside_p_1, condition_inside_p_2, condition_inside_p_3, condition_inside_p_4, condition_inside_p_5, condition_inside_p_6, treatment_outside_p_1, treatment_outside_p_2, treatment_outside_p_3, treatment_outside_p_4, treatment_outside_p_5, treatment_outside_p_6, treatment_outside_p_7, treatment_outside_p_8, treatment_outside_p_9, treatment_outside_p_10, treatment_outside_p_11, treatment_outside_p_12, treatment_outside_p_13, treatment_outside_p_14, treatment_outside_p_15, treatment_outside_p_16, treatment_inside_p_1, treatment_inside_p_2, treatment_inside_p_3, treatment_inside_p_4, treatment_inside_p_5, treatment_inside_p_6, treatment_inside_p_7, treatment_inside_p_8, treatment_inside_p_9, condition_outside_b_1, condition_outside_b_2, condition_outside_b_3, condition_outside_b_4, condition_outside_b_5, condition_outside_b_6, condition_outside_b_7, condition_outside_b_8, condition_outside_b_9, condition_outside_b_10, condition_outside_b_11, condition_outside_b_12, condition_inside_b_1, treatment_outside_b_1, treatment_outside_b_2, treatment_outside_b_3, treatment_outside_b_4, treatment_outside_b_5, treatment_outside_b_6, treatment_outside_b_7, treatment_outside_b_8, treatment_outside_b_9, treatment_outside_b_10, treatment_outside_b_11, treatment_outside_b_12, treatment_outside_b_13, treatment_outside_b_14, treatment_outside_b_15, treatment_outside_b_16, treatment_inside_b_1, treatment_inside_b_2, treatment_inside_b_3, treatment_inside_b_4, treatment_inside_b_5]\n",
       "Index: []\n",
       "\n",
       "[0 rows x 79 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test = split_df[split_df['treatment_lower'].isnull()]\n",
    "test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "afb9e91c-00de-4678-b069-b8f6b683030e",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['condition_lower',\n",
       " 'condition_outside_p_1',\n",
       " 'condition_outside_p_2',\n",
       " 'condition_outside_p_3',\n",
       " 'condition_outside_p_4']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "['treatment_lower',\n",
       " 'treatment_outside_p_1',\n",
       " 'treatment_outside_p_2',\n",
       " 'treatment_outside_p_3',\n",
       " 'treatment_outside_p_4']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "condition_cols = [ele for ele in split_df.columns if any([substr in ele for substr in ['condition']])] # find columns with \"condition\" in it \n",
    "treatment_cols = [ele for ele in split_df.columns if any([substr in ele for substr in ['treatment']])] # find columns with \"treatment\" in it \n",
    "condition_cols[:5]\n",
    "treatment_cols[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "c7a0e30d-a744-48f5-9968-e93e75826b74",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "split_df[\"condition_collections\"] = split_df[condition_cols].stack().groupby(level=0).apply(list).values.tolist()  # get the columns with conditions into a list in a single column, ignoring the None values\n",
    "split_df[\"treatment_collections\"] = split_df[treatment_cols].stack().groupby(level=0).apply(list).values.tolist()  # get the columns with treatments into a list in a single column, ignoring the None values\n",
    "\n",
    "# split_df\n",
    "# trmnts = split_df[treatment_cols].stack().groupby(level=0).apply(list).values.tolist()\n",
    "# len(trmnts)\n",
    "\n",
    "# conds = split_df[condition_cols].stack().groupby(level=0).apply(list).values.tolist()\n",
    "# len(conds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "345d4f27-1207-4315-99e8-18dfe39bfb59",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nct_id</th>\n",
       "      <th>primary_purpose</th>\n",
       "      <th>intervention_model</th>\n",
       "      <th>start_month_year</th>\n",
       "      <th>is_fda_regulated_drug</th>\n",
       "      <th>condition</th>\n",
       "      <th>predicate</th>\n",
       "      <th>treatment</th>\n",
       "      <th>condition_collections</th>\n",
       "      <th>treatment_collections</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>544540</th>\n",
       "      <td>NCT01996384</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2013-11</td>\n",
       "      <td>?</td>\n",
       "      <td>Provoked Vestibulodynia</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Non-classical acupuncture</td>\n",
       "      <td>[provoked vestibulodynia]</td>\n",
       "      <td>[non-classical acupuncture]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544541</th>\n",
       "      <td>NCT01996384</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2013-11</td>\n",
       "      <td>?</td>\n",
       "      <td>Vulvar Vestibulitis</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Non-classical acupuncture</td>\n",
       "      <td>[vulvar vestibulitis]</td>\n",
       "      <td>[non-classical acupuncture]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544542</th>\n",
       "      <td>NCT01247064</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2010-10</td>\n",
       "      <td>?</td>\n",
       "      <td>Bronchiolitis, Viral</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Nebulized 0.9% Normal Saline</td>\n",
       "      <td>[bronchiolitis, viral, bronchiolitis, viral]</td>\n",
       "      <td>[nebulized 0.9% normal saline]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544543</th>\n",
       "      <td>NCT01247064</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2010-10</td>\n",
       "      <td>?</td>\n",
       "      <td>Saline Solution, Hypertonic</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Nebulized 0.9% Normal Saline</td>\n",
       "      <td>[saline solution, hypertonic, saline solution, hypertonic]</td>\n",
       "      <td>[nebulized 0.9% normal saline]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544544</th>\n",
       "      <td>NCT02576574</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2015-10-29</td>\n",
       "      <td>?</td>\n",
       "      <td>First Line Non-Small Cell Lung Cancer</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Carboplatin</td>\n",
       "      <td>[first line non-small cell lung cancer]</td>\n",
       "      <td>[carboplatin]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544545</th>\n",
       "      <td>NCT02576574</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2015-10-29</td>\n",
       "      <td>?</td>\n",
       "      <td>First Line Non-Small Cell Lung Cancer</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Gemcitabine</td>\n",
       "      <td>[first line non-small cell lung cancer]</td>\n",
       "      <td>[gemcitabine]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544546</th>\n",
       "      <td>NCT02576574</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2015-10-29</td>\n",
       "      <td>?</td>\n",
       "      <td>First Line Non-Small Cell Lung Cancer</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Paclitaxel</td>\n",
       "      <td>[first line non-small cell lung cancer]</td>\n",
       "      <td>[paclitaxel]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544547</th>\n",
       "      <td>NCT02576574</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2015-10-29</td>\n",
       "      <td>?</td>\n",
       "      <td>First Line Non-Small Cell Lung Cancer</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Pemetrexed</td>\n",
       "      <td>[first line non-small cell lung cancer]</td>\n",
       "      <td>[pemetrexed]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544548</th>\n",
       "      <td>NCT04264533</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2020-2-14</td>\n",
       "      <td>f</td>\n",
       "      <td>Pneumonia, Viral</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Sterile Water for Injection</td>\n",
       "      <td>[pneumonia, viral, pneumonia, viral]</td>\n",
       "      <td>[sterile water for injection]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544549</th>\n",
       "      <td>NCT04264533</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2020-2-14</td>\n",
       "      <td>f</td>\n",
       "      <td>Pneumonia, Ventilator-Associated</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Sterile Water for Injection</td>\n",
       "      <td>[pneumonia, ventilator-associated, pneumonia, ventilator-associated]</td>\n",
       "      <td>[sterile water for injection]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             nct_id primary_purpose   intervention_model start_month_year  \\\n",
       "544540  NCT01996384       Treatment  Parallel Assignment          2013-11   \n",
       "544541  NCT01996384       Treatment  Parallel Assignment          2013-11   \n",
       "544542  NCT01247064       Treatment  Parallel Assignment          2010-10   \n",
       "544543  NCT01247064       Treatment  Parallel Assignment          2010-10   \n",
       "544544  NCT02576574       Treatment  Parallel Assignment       2015-10-29   \n",
       "544545  NCT02576574       Treatment  Parallel Assignment       2015-10-29   \n",
       "544546  NCT02576574       Treatment  Parallel Assignment       2015-10-29   \n",
       "544547  NCT02576574       Treatment  Parallel Assignment       2015-10-29   \n",
       "544548  NCT04264533       Treatment  Parallel Assignment        2020-2-14   \n",
       "544549  NCT04264533       Treatment  Parallel Assignment        2020-2-14   \n",
       "\n",
       "       is_fda_regulated_drug                              condition  \\\n",
       "544540                     ?                Provoked Vestibulodynia   \n",
       "544541                     ?                    Vulvar Vestibulitis   \n",
       "544542                     ?                   Bronchiolitis, Viral   \n",
       "544543                     ?            Saline Solution, Hypertonic   \n",
       "544544                     ?  First Line Non-Small Cell Lung Cancer   \n",
       "544545                     ?  First Line Non-Small Cell Lung Cancer   \n",
       "544546                     ?  First Line Non-Small Cell Lung Cancer   \n",
       "544547                     ?  First Line Non-Small Cell Lung Cancer   \n",
       "544548                     f                       Pneumonia, Viral   \n",
       "544549                     f       Pneumonia, Ventilator-Associated   \n",
       "\n",
       "                               predicate                     treatment  \\\n",
       "544540  mentioned_in_clinical_trials_for     Non-classical acupuncture   \n",
       "544541  mentioned_in_clinical_trials_for     Non-classical acupuncture   \n",
       "544542  mentioned_in_clinical_trials_for  Nebulized 0.9% Normal Saline   \n",
       "544543  mentioned_in_clinical_trials_for  Nebulized 0.9% Normal Saline   \n",
       "544544  mentioned_in_clinical_trials_for                   Carboplatin   \n",
       "544545  mentioned_in_clinical_trials_for                   Gemcitabine   \n",
       "544546  mentioned_in_clinical_trials_for                    Paclitaxel   \n",
       "544547  mentioned_in_clinical_trials_for                    Pemetrexed   \n",
       "544548  mentioned_in_clinical_trials_for   Sterile Water for Injection   \n",
       "544549  mentioned_in_clinical_trials_for   Sterile Water for Injection   \n",
       "\n",
       "                                                       condition_collections  \\\n",
       "544540                                             [provoked vestibulodynia]   \n",
       "544541                                                 [vulvar vestibulitis]   \n",
       "544542                          [bronchiolitis, viral, bronchiolitis, viral]   \n",
       "544543            [saline solution, hypertonic, saline solution, hypertonic]   \n",
       "544544                               [first line non-small cell lung cancer]   \n",
       "544545                               [first line non-small cell lung cancer]   \n",
       "544546                               [first line non-small cell lung cancer]   \n",
       "544547                               [first line non-small cell lung cancer]   \n",
       "544548                                  [pneumonia, viral, pneumonia, viral]   \n",
       "544549  [pneumonia, ventilator-associated, pneumonia, ventilator-associated]   \n",
       "\n",
       "                 treatment_collections  \n",
       "544540     [non-classical acupuncture]  \n",
       "544541     [non-classical acupuncture]  \n",
       "544542  [nebulized 0.9% normal saline]  \n",
       "544543  [nebulized 0.9% normal saline]  \n",
       "544544                   [carboplatin]  \n",
       "544545                   [gemcitabine]  \n",
       "544546                    [paclitaxel]  \n",
       "544547                    [pemetrexed]  \n",
       "544548   [sterile water for injection]  \n",
       "544549   [sterile water for injection]  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ct[\"condition_collections\"] = split_df[\"condition_collections\"]\n",
    "ct[\"treatment_collections\"] = split_df[\"treatment_collections\"]\n",
    "cols_to_keep = [element for element in ct.columns if not any(substring in element for substring in ['inside', 'outside', 'lower'])]\n",
    "ct = ct[cols_to_keep]\n",
    "with pd.option_context('display.max_rows', None, 'display.max_columns', None, 'display.max_colwidth', None):  # more options can be specified also\n",
    "    display(ct[-10:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "169c6491-4276-41d4-99ef-07df4875ae0c",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "795705"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ct = ct.explode('condition_collections')\n",
    "ct = ct.explode('treatment_collections')\n",
    "len(ct)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "2812b343-4772-4b77-93ae-090d7624568e",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nct_id</th>\n",
       "      <th>primary_purpose</th>\n",
       "      <th>intervention_model</th>\n",
       "      <th>start_month_year</th>\n",
       "      <th>is_fda_regulated_drug</th>\n",
       "      <th>condition</th>\n",
       "      <th>predicate</th>\n",
       "      <th>treatment</th>\n",
       "      <th>condition_collections</th>\n",
       "      <th>treatment_collections</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>544544</th>\n",
       "      <td>NCT02576574</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2015-10-29</td>\n",
       "      <td>?</td>\n",
       "      <td>First Line Non-Small Cell Lung Cancer</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Carboplatin</td>\n",
       "      <td>first line non-small cell lung cancer</td>\n",
       "      <td>carboplatin</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544545</th>\n",
       "      <td>NCT02576574</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2015-10-29</td>\n",
       "      <td>?</td>\n",
       "      <td>First Line Non-Small Cell Lung Cancer</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Gemcitabine</td>\n",
       "      <td>first line non-small cell lung cancer</td>\n",
       "      <td>gemcitabine</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544546</th>\n",
       "      <td>NCT02576574</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2015-10-29</td>\n",
       "      <td>?</td>\n",
       "      <td>First Line Non-Small Cell Lung Cancer</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Paclitaxel</td>\n",
       "      <td>first line non-small cell lung cancer</td>\n",
       "      <td>paclitaxel</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544547</th>\n",
       "      <td>NCT02576574</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2015-10-29</td>\n",
       "      <td>?</td>\n",
       "      <td>First Line Non-Small Cell Lung Cancer</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Pemetrexed</td>\n",
       "      <td>first line non-small cell lung cancer</td>\n",
       "      <td>pemetrexed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544548</th>\n",
       "      <td>NCT04264533</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2020-2-14</td>\n",
       "      <td>f</td>\n",
       "      <td>Pneumonia, Viral</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Sterile Water for Injection</td>\n",
       "      <td>pneumonia, viral</td>\n",
       "      <td>sterile water for injection</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544548</th>\n",
       "      <td>NCT04264533</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2020-2-14</td>\n",
       "      <td>f</td>\n",
       "      <td>Pneumonia, Viral</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Sterile Water for Injection</td>\n",
       "      <td>pneumonia</td>\n",
       "      <td>sterile water for injection</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544548</th>\n",
       "      <td>NCT04264533</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2020-2-14</td>\n",
       "      <td>f</td>\n",
       "      <td>Pneumonia, Viral</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Sterile Water for Injection</td>\n",
       "      <td>viral</td>\n",
       "      <td>sterile water for injection</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544549</th>\n",
       "      <td>NCT04264533</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2020-2-14</td>\n",
       "      <td>f</td>\n",
       "      <td>Pneumonia, Ventilator-Associated</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Sterile Water for Injection</td>\n",
       "      <td>pneumonia, ventilator-associated</td>\n",
       "      <td>sterile water for injection</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544549</th>\n",
       "      <td>NCT04264533</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2020-2-14</td>\n",
       "      <td>f</td>\n",
       "      <td>Pneumonia, Ventilator-Associated</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Sterile Water for Injection</td>\n",
       "      <td>pneumonia</td>\n",
       "      <td>sterile water for injection</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544549</th>\n",
       "      <td>NCT04264533</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2020-2-14</td>\n",
       "      <td>f</td>\n",
       "      <td>Pneumonia, Ventilator-Associated</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Sterile Water for Injection</td>\n",
       "      <td>ventilator-associated</td>\n",
       "      <td>sterile water for injection</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             nct_id primary_purpose   intervention_model start_month_year  \\\n",
       "544544  NCT02576574       Treatment  Parallel Assignment       2015-10-29   \n",
       "544545  NCT02576574       Treatment  Parallel Assignment       2015-10-29   \n",
       "544546  NCT02576574       Treatment  Parallel Assignment       2015-10-29   \n",
       "544547  NCT02576574       Treatment  Parallel Assignment       2015-10-29   \n",
       "544548  NCT04264533       Treatment  Parallel Assignment        2020-2-14   \n",
       "544548  NCT04264533       Treatment  Parallel Assignment        2020-2-14   \n",
       "544548  NCT04264533       Treatment  Parallel Assignment        2020-2-14   \n",
       "544549  NCT04264533       Treatment  Parallel Assignment        2020-2-14   \n",
       "544549  NCT04264533       Treatment  Parallel Assignment        2020-2-14   \n",
       "544549  NCT04264533       Treatment  Parallel Assignment        2020-2-14   \n",
       "\n",
       "       is_fda_regulated_drug                              condition  \\\n",
       "544544                     ?  First Line Non-Small Cell Lung Cancer   \n",
       "544545                     ?  First Line Non-Small Cell Lung Cancer   \n",
       "544546                     ?  First Line Non-Small Cell Lung Cancer   \n",
       "544547                     ?  First Line Non-Small Cell Lung Cancer   \n",
       "544548                     f                       Pneumonia, Viral   \n",
       "544548                     f                       Pneumonia, Viral   \n",
       "544548                     f                       Pneumonia, Viral   \n",
       "544549                     f       Pneumonia, Ventilator-Associated   \n",
       "544549                     f       Pneumonia, Ventilator-Associated   \n",
       "544549                     f       Pneumonia, Ventilator-Associated   \n",
       "\n",
       "                               predicate                    treatment  \\\n",
       "544544  mentioned_in_clinical_trials_for                  Carboplatin   \n",
       "544545  mentioned_in_clinical_trials_for                  Gemcitabine   \n",
       "544546  mentioned_in_clinical_trials_for                   Paclitaxel   \n",
       "544547  mentioned_in_clinical_trials_for                   Pemetrexed   \n",
       "544548  mentioned_in_clinical_trials_for  Sterile Water for Injection   \n",
       "544548  mentioned_in_clinical_trials_for  Sterile Water for Injection   \n",
       "544548  mentioned_in_clinical_trials_for  Sterile Water for Injection   \n",
       "544549  mentioned_in_clinical_trials_for  Sterile Water for Injection   \n",
       "544549  mentioned_in_clinical_trials_for  Sterile Water for Injection   \n",
       "544549  mentioned_in_clinical_trials_for  Sterile Water for Injection   \n",
       "\n",
       "                        condition_collections        treatment_collections  \n",
       "544544  first line non-small cell lung cancer                  carboplatin  \n",
       "544545  first line non-small cell lung cancer                  gemcitabine  \n",
       "544546  first line non-small cell lung cancer                   paclitaxel  \n",
       "544547  first line non-small cell lung cancer                   pemetrexed  \n",
       "544548                       pneumonia, viral  sterile water for injection  \n",
       "544548                              pneumonia  sterile water for injection  \n",
       "544548                                  viral  sterile water for injection  \n",
       "544549       pneumonia, ventilator-associated  sterile water for injection  \n",
       "544549                              pneumonia  sterile water for injection  \n",
       "544549                  ventilator-associated  sterile water for injection  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "with pd.option_context('display.max_rows', None, 'display.max_columns', None, 'display.max_colwidth', None):  # more options can be specified also\n",
    "    display(ct[-10:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "7d17e98b-0efb-4603-89b3-6e013b8408bd",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nct_id</th>\n",
       "      <th>primary_purpose</th>\n",
       "      <th>intervention_model</th>\n",
       "      <th>start_month_year</th>\n",
       "      <th>is_fda_regulated_drug</th>\n",
       "      <th>condition</th>\n",
       "      <th>predicate</th>\n",
       "      <th>treatment</th>\n",
       "      <th>condition_collections</th>\n",
       "      <th>treatment_collections</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NCT01049399</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2009-12</td>\n",
       "      <td>?</td>\n",
       "      <td>Progressive Supranuclear Palsy</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>tideglusib</td>\n",
       "      <td>progressive supranuclear palsy</td>\n",
       "      <td>tideglusib</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NCT05287724</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Crossover Assignment</td>\n",
       "      <td>2022-6-19</td>\n",
       "      <td>t</td>\n",
       "      <td>Pruritus</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>N-acetyl cysteine</td>\n",
       "      <td>pruritus</td>\n",
       "      <td>n-acetyl cysteine</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NCT01432236</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Crossover Assignment</td>\n",
       "      <td>2011-10</td>\n",
       "      <td>?</td>\n",
       "      <td>Fibromyalgia</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>Pregabalin</td>\n",
       "      <td>fibromyalgia</td>\n",
       "      <td>pregabalin</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NCT04626921</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Single Group Assignment</td>\n",
       "      <td>2020-10-22</td>\n",
       "      <td>t</td>\n",
       "      <td>Relapsing Multiple Sclerosis</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>CNM-Au8</td>\n",
       "      <td>relapsing multiple sclerosis</td>\n",
       "      <td>cnm-au8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>NCT04681417</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2021-3-25</td>\n",
       "      <td>f</td>\n",
       "      <td>Retinoblastoma</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>Melphalan or Melphalan + Topotecan</td>\n",
       "      <td>retinoblastoma</td>\n",
       "      <td>melphalan or melphalan + topotecan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544548</th>\n",
       "      <td>NCT04264533</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2020-2-14</td>\n",
       "      <td>f</td>\n",
       "      <td>Pneumonia, Viral</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Sterile Water for Injection</td>\n",
       "      <td>pneumonia</td>\n",
       "      <td>sterile water for injection</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544548</th>\n",
       "      <td>NCT04264533</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2020-2-14</td>\n",
       "      <td>f</td>\n",
       "      <td>Pneumonia, Viral</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Sterile Water for Injection</td>\n",
       "      <td>viral</td>\n",
       "      <td>sterile water for injection</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544549</th>\n",
       "      <td>NCT04264533</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2020-2-14</td>\n",
       "      <td>f</td>\n",
       "      <td>Pneumonia, Ventilator-Associated</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Sterile Water for Injection</td>\n",
       "      <td>pneumonia, ventilator-associated</td>\n",
       "      <td>sterile water for injection</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544549</th>\n",
       "      <td>NCT04264533</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2020-2-14</td>\n",
       "      <td>f</td>\n",
       "      <td>Pneumonia, Ventilator-Associated</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Sterile Water for Injection</td>\n",
       "      <td>pneumonia</td>\n",
       "      <td>sterile water for injection</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544549</th>\n",
       "      <td>NCT04264533</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2020-2-14</td>\n",
       "      <td>f</td>\n",
       "      <td>Pneumonia, Ventilator-Associated</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Sterile Water for Injection</td>\n",
       "      <td>ventilator-associated</td>\n",
       "      <td>sterile water for injection</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>750068 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             nct_id primary_purpose       intervention_model start_month_year  \\\n",
       "0       NCT01049399       Treatment      Parallel Assignment          2009-12   \n",
       "1       NCT05287724       Treatment     Crossover Assignment        2022-6-19   \n",
       "3       NCT01432236       Treatment     Crossover Assignment          2011-10   \n",
       "4       NCT04626921       Treatment  Single Group Assignment       2020-10-22   \n",
       "5       NCT04681417       Treatment      Parallel Assignment        2021-3-25   \n",
       "...             ...             ...                      ...              ...   \n",
       "544548  NCT04264533       Treatment      Parallel Assignment        2020-2-14   \n",
       "544548  NCT04264533       Treatment      Parallel Assignment        2020-2-14   \n",
       "544549  NCT04264533       Treatment      Parallel Assignment        2020-2-14   \n",
       "544549  NCT04264533       Treatment      Parallel Assignment        2020-2-14   \n",
       "544549  NCT04264533       Treatment      Parallel Assignment        2020-2-14   \n",
       "\n",
       "       is_fda_regulated_drug                         condition  \\\n",
       "0                          ?    Progressive Supranuclear Palsy   \n",
       "1                          t                          Pruritus   \n",
       "3                          ?                      Fibromyalgia   \n",
       "4                          t      Relapsing Multiple Sclerosis   \n",
       "5                          f                    Retinoblastoma   \n",
       "...                      ...                               ...   \n",
       "544548                     f                  Pneumonia, Viral   \n",
       "544548                     f                  Pneumonia, Viral   \n",
       "544549                     f  Pneumonia, Ventilator-Associated   \n",
       "544549                     f  Pneumonia, Ventilator-Associated   \n",
       "544549                     f  Pneumonia, Ventilator-Associated   \n",
       "\n",
       "                               predicate                           treatment  \\\n",
       "0                 in_clinical_trials_for                          tideglusib   \n",
       "1                 in_clinical_trials_for                   N-acetyl cysteine   \n",
       "3                 in_clinical_trials_for                          Pregabalin   \n",
       "4                 in_clinical_trials_for                             CNM-Au8   \n",
       "5                 in_clinical_trials_for  Melphalan or Melphalan + Topotecan   \n",
       "...                                  ...                                 ...   \n",
       "544548  mentioned_in_clinical_trials_for         Sterile Water for Injection   \n",
       "544548  mentioned_in_clinical_trials_for         Sterile Water for Injection   \n",
       "544549  mentioned_in_clinical_trials_for         Sterile Water for Injection   \n",
       "544549  mentioned_in_clinical_trials_for         Sterile Water for Injection   \n",
       "544549  mentioned_in_clinical_trials_for         Sterile Water for Injection   \n",
       "\n",
       "                   condition_collections               treatment_collections  \n",
       "0         progressive supranuclear palsy                          tideglusib  \n",
       "1                               pruritus                   n-acetyl cysteine  \n",
       "3                           fibromyalgia                          pregabalin  \n",
       "4           relapsing multiple sclerosis                             cnm-au8  \n",
       "5                         retinoblastoma  melphalan or melphalan + topotecan  \n",
       "...                                  ...                                 ...  \n",
       "544548                         pneumonia         sterile water for injection  \n",
       "544548                             viral         sterile water for injection  \n",
       "544549  pneumonia, ventilator-associated         sterile water for injection  \n",
       "544549                         pneumonia         sterile water for injection  \n",
       "544549             ventilator-associated         sterile water for injection  \n",
       "\n",
       "[750068 rows x 10 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ct['condition_collections'] = ct['condition_collections'].apply(lambda x: x if len(str(x)) > 3 else None)\n",
    "ct['treatment_collections'] = ct['treatment_collections'].apply(lambda x: x if len(str(x)) > 3 else None)\n",
    "ct = ct[~ct['condition_collections'].isnull()]\n",
    "ct = ct[~ct['treatment_collections'].isnull()]\n",
    "ct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "75633c5b-e411-47df-9988-542f5b606c45",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nct_id</th>\n",
       "      <th>primary_purpose</th>\n",
       "      <th>intervention_model</th>\n",
       "      <th>start_month_year</th>\n",
       "      <th>is_fda_regulated_drug</th>\n",
       "      <th>condition</th>\n",
       "      <th>predicate</th>\n",
       "      <th>treatment</th>\n",
       "      <th>condition_collections</th>\n",
       "      <th>treatment_collections</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6084</th>\n",
       "      <td>NCT06012760</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2024-4-1</td>\n",
       "      <td>f</td>\n",
       "      <td>Cardiac Surgery</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>Iron sucrose, Human Erythropoietin Injection, Vitamin C</td>\n",
       "      <td>cardiac surgery</td>\n",
       "      <td>human erythropoietin injection</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6084</th>\n",
       "      <td>NCT06012760</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2024-4-1</td>\n",
       "      <td>f</td>\n",
       "      <td>Cardiac Surgery</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>Iron sucrose, Human Erythropoietin Injection, Vitamin C</td>\n",
       "      <td>cardiac surgery</td>\n",
       "      <td>vitamin c</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6085</th>\n",
       "      <td>NCT03676309</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2017-9-1</td>\n",
       "      <td>f</td>\n",
       "      <td>Diabetes Mellitus, Type 2</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>Nutriceutical Oral Capsule,</td>\n",
       "      <td>diabetes mellitus, type 2</td>\n",
       "      <td>nutriceutical oral capsule,</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6085</th>\n",
       "      <td>NCT03676309</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2017-9-1</td>\n",
       "      <td>f</td>\n",
       "      <td>Diabetes Mellitus, Type 2</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>Nutriceutical Oral Capsule,</td>\n",
       "      <td>diabetes mellitus, type 2</td>\n",
       "      <td>nutriceutical oral capsule</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6085</th>\n",
       "      <td>NCT03676309</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2017-9-1</td>\n",
       "      <td>f</td>\n",
       "      <td>Diabetes Mellitus, Type 2</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>Nutriceutical Oral Capsule,</td>\n",
       "      <td>diabetes mellitus</td>\n",
       "      <td>nutriceutical oral capsule,</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6085</th>\n",
       "      <td>NCT03676309</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2017-9-1</td>\n",
       "      <td>f</td>\n",
       "      <td>Diabetes Mellitus, Type 2</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>Nutriceutical Oral Capsule,</td>\n",
       "      <td>diabetes mellitus</td>\n",
       "      <td>nutriceutical oral capsule</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6085</th>\n",
       "      <td>NCT03676309</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2017-9-1</td>\n",
       "      <td>f</td>\n",
       "      <td>Diabetes Mellitus, Type 2</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>Nutriceutical Oral Capsule,</td>\n",
       "      <td>type 2</td>\n",
       "      <td>nutriceutical oral capsule,</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6085</th>\n",
       "      <td>NCT03676309</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2017-9-1</td>\n",
       "      <td>f</td>\n",
       "      <td>Diabetes Mellitus, Type 2</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>Nutriceutical Oral Capsule,</td>\n",
       "      <td>type 2</td>\n",
       "      <td>nutriceutical oral capsule</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6086</th>\n",
       "      <td>NCT04015492</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Crossover Assignment</td>\n",
       "      <td>2019-8-8</td>\n",
       "      <td>t</td>\n",
       "      <td>Hemophilia A</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>Damoctocog-alfa-pegol (BAY94-9027, Jivi)</td>\n",
       "      <td>hemophilia a</td>\n",
       "      <td>damoctocog-alfa-pegol (bay94-9027, jivi)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6086</th>\n",
       "      <td>NCT04015492</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Crossover Assignment</td>\n",
       "      <td>2019-8-8</td>\n",
       "      <td>t</td>\n",
       "      <td>Hemophilia A</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>Damoctocog-alfa-pegol (BAY94-9027, Jivi)</td>\n",
       "      <td>hemophilia a</td>\n",
       "      <td>damoctocog-alfa-pegol</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           nct_id primary_purpose    intervention_model start_month_year  \\\n",
       "6084  NCT06012760       Treatment   Parallel Assignment         2024-4-1   \n",
       "6084  NCT06012760       Treatment   Parallel Assignment         2024-4-1   \n",
       "6085  NCT03676309       Treatment   Parallel Assignment         2017-9-1   \n",
       "6085  NCT03676309       Treatment   Parallel Assignment         2017-9-1   \n",
       "6085  NCT03676309       Treatment   Parallel Assignment         2017-9-1   \n",
       "6085  NCT03676309       Treatment   Parallel Assignment         2017-9-1   \n",
       "6085  NCT03676309       Treatment   Parallel Assignment         2017-9-1   \n",
       "6085  NCT03676309       Treatment   Parallel Assignment         2017-9-1   \n",
       "6086  NCT04015492       Treatment  Crossover Assignment         2019-8-8   \n",
       "6086  NCT04015492       Treatment  Crossover Assignment         2019-8-8   \n",
       "\n",
       "     is_fda_regulated_drug                  condition               predicate  \\\n",
       "6084                     f            Cardiac Surgery  in_clinical_trials_for   \n",
       "6084                     f            Cardiac Surgery  in_clinical_trials_for   \n",
       "6085                     f  Diabetes Mellitus, Type 2  in_clinical_trials_for   \n",
       "6085                     f  Diabetes Mellitus, Type 2  in_clinical_trials_for   \n",
       "6085                     f  Diabetes Mellitus, Type 2  in_clinical_trials_for   \n",
       "6085                     f  Diabetes Mellitus, Type 2  in_clinical_trials_for   \n",
       "6085                     f  Diabetes Mellitus, Type 2  in_clinical_trials_for   \n",
       "6085                     f  Diabetes Mellitus, Type 2  in_clinical_trials_for   \n",
       "6086                     t               Hemophilia A  in_clinical_trials_for   \n",
       "6086                     t               Hemophilia A  in_clinical_trials_for   \n",
       "\n",
       "                                                    treatment  \\\n",
       "6084  Iron sucrose, Human Erythropoietin Injection, Vitamin C   \n",
       "6084  Iron sucrose, Human Erythropoietin Injection, Vitamin C   \n",
       "6085                              Nutriceutical Oral Capsule,   \n",
       "6085                              Nutriceutical Oral Capsule,   \n",
       "6085                              Nutriceutical Oral Capsule,   \n",
       "6085                              Nutriceutical Oral Capsule,   \n",
       "6085                              Nutriceutical Oral Capsule,   \n",
       "6085                              Nutriceutical Oral Capsule,   \n",
       "6086                 Damoctocog-alfa-pegol (BAY94-9027, Jivi)   \n",
       "6086                 Damoctocog-alfa-pegol (BAY94-9027, Jivi)   \n",
       "\n",
       "          condition_collections                     treatment_collections  \n",
       "6084            cardiac surgery            human erythropoietin injection  \n",
       "6084            cardiac surgery                                 vitamin c  \n",
       "6085  diabetes mellitus, type 2               nutriceutical oral capsule,  \n",
       "6085  diabetes mellitus, type 2                nutriceutical oral capsule  \n",
       "6085          diabetes mellitus               nutriceutical oral capsule,  \n",
       "6085          diabetes mellitus                nutriceutical oral capsule  \n",
       "6085                     type 2               nutriceutical oral capsule,  \n",
       "6085                     type 2                nutriceutical oral capsule  \n",
       "6086               hemophilia a  damoctocog-alfa-pegol (bay94-9027, jivi)  \n",
       "6086               hemophilia a                     damoctocog-alfa-pegol  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "with pd.option_context('display.max_rows', None, 'display.max_columns', None, 'display.max_colwidth', None):  # more options can be specified also\n",
    "    display(ct[8700:8710])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5491b97c-7269-4854-a7b0-956b22fc71ae",
   "metadata": {
    "tags": []
   },
   "source": [
    "#### Map CURIEs from mapper to ct dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "2921a960-1747-40e5-8096-1ac11d5429ea",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "mapping_cache = pd.read_csv(\"mapping_cache.tsv\", sep='\\t', index_col=False, header=0, dtype=\"object\", on_bad_lines=\"skip\")\n",
    "mapping_cache = mapping_cache[mapping_cache['score'] != 'unscored'] \n",
    "mapping_cache[\"score\"] = pd.to_numeric(mapping_cache[\"score\"], errors='coerce')\n",
    "mapping_cache = mapping_cache.sort_values(by=['clintrial_term', 'score'], ascending=[False, False])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "4bf70ef0-5d10-4b79-b587-7e50b3df7a4e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "idx = mapping_cache.groupby(['clintrial_term', 'term_type'])['score'].transform(max) == mapping_cache['score']\n",
    "max_scores = mapping_cache[idx]\n",
    "max_scores = max_scores[max_scores.score >= 70]\n",
    "conditions_maxscores = max_scores[max_scores.term_type == \"condition\"]\n",
    "interventions_maxscores = max_scores[max_scores.term_type == \"intervention\"]\n",
    "alt_intervention_maxscores = max_scores[max_scores.term_type == \"alternate_intervention\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "2bc6bfff-74eb-4a68-8aab-3629266f5c82",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "conditions_dict = dict(zip(conditions_maxscores.clintrial_term, conditions_maxscores.mapping_tool_response))\n",
    "interventions_dict = dict(zip(interventions_maxscores.clintrial_term, interventions_maxscores.mapping_tool_response))\n",
    "alt_intervention_dict = dict(zip(alt_intervention_maxscores.clintrial_term, interventions_maxscores.mapping_tool_response))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "113b8a7d-9e22-47bc-8d09-bf585f2703fc",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nct_id</th>\n",
       "      <th>primary_purpose</th>\n",
       "      <th>intervention_model</th>\n",
       "      <th>start_month_year</th>\n",
       "      <th>is_fda_regulated_drug</th>\n",
       "      <th>condition</th>\n",
       "      <th>predicate</th>\n",
       "      <th>treatment</th>\n",
       "      <th>condition_collections</th>\n",
       "      <th>treatment_collections</th>\n",
       "      <th>condition_curie_info</th>\n",
       "      <th>treatment_curie_info</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NCT01049399</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2009-12</td>\n",
       "      <td>?</td>\n",
       "      <td>Progressive Supranuclear Palsy</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>tideglusib</td>\n",
       "      <td>progressive supranuclear palsy</td>\n",
       "      <td>tideglusib</td>\n",
       "      <td>{'mapped_name': 'Progressive supranuclear pals...</td>\n",
       "      <td>{'mapped_name': 'Tideglusib', 'mapped_curie': ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NCT05287724</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Crossover Assignment</td>\n",
       "      <td>2022-6-19</td>\n",
       "      <td>t</td>\n",
       "      <td>Pruritus</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>N-acetyl cysteine</td>\n",
       "      <td>pruritus</td>\n",
       "      <td>n-acetyl cysteine</td>\n",
       "      <td>{'mapped_name': 'Pruritus', 'mapped_curie': 'C...</td>\n",
       "      <td>{'mapped_name': 'acetylcysteine', 'mapped_curi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NCT01432236</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Crossover Assignment</td>\n",
       "      <td>2011-10</td>\n",
       "      <td>?</td>\n",
       "      <td>Fibromyalgia</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>Pregabalin</td>\n",
       "      <td>fibromyalgia</td>\n",
       "      <td>pregabalin</td>\n",
       "      <td>{'mapped_name': 'Fibromyalgia', 'mapped_curie'...</td>\n",
       "      <td>{'mapped_name': 'pregabalin', 'mapped_curie': ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NCT04626921</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Single Group Assignment</td>\n",
       "      <td>2020-10-22</td>\n",
       "      <td>t</td>\n",
       "      <td>Relapsing Multiple Sclerosis</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>CNM-Au8</td>\n",
       "      <td>relapsing multiple sclerosis</td>\n",
       "      <td>cnm-au8</td>\n",
       "      <td>{'mapped_name': 'Multiple Sclerosis', 'mapped_...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>NCT04681417</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2021-3-25</td>\n",
       "      <td>f</td>\n",
       "      <td>Retinoblastoma</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>Melphalan or Melphalan + Topotecan</td>\n",
       "      <td>retinoblastoma</td>\n",
       "      <td>melphalan or melphalan + topotecan</td>\n",
       "      <td>{'mapped_name': 'Retinoblastoma', 'mapped_curi...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544548</th>\n",
       "      <td>NCT04264533</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2020-2-14</td>\n",
       "      <td>f</td>\n",
       "      <td>Pneumonia, Viral</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Sterile Water for Injection</td>\n",
       "      <td>pneumonia</td>\n",
       "      <td>sterile water for injection</td>\n",
       "      <td>{'mapped_name': 'Pneumonia', 'mapped_curie': '...</td>\n",
       "      <td>{'mapped_name': 'Sterile Water for Injection',...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544548</th>\n",
       "      <td>NCT04264533</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2020-2-14</td>\n",
       "      <td>f</td>\n",
       "      <td>Pneumonia, Viral</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Sterile Water for Injection</td>\n",
       "      <td>viral</td>\n",
       "      <td>sterile water for injection</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{'mapped_name': 'Sterile Water for Injection',...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544549</th>\n",
       "      <td>NCT04264533</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2020-2-14</td>\n",
       "      <td>f</td>\n",
       "      <td>Pneumonia, Ventilator-Associated</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Sterile Water for Injection</td>\n",
       "      <td>pneumonia, ventilator-associated</td>\n",
       "      <td>sterile water for injection</td>\n",
       "      <td>{'mapped_name': 'Pneumonia, Ventilator-Associa...</td>\n",
       "      <td>{'mapped_name': 'Sterile Water for Injection',...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544549</th>\n",
       "      <td>NCT04264533</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2020-2-14</td>\n",
       "      <td>f</td>\n",
       "      <td>Pneumonia, Ventilator-Associated</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Sterile Water for Injection</td>\n",
       "      <td>pneumonia</td>\n",
       "      <td>sterile water for injection</td>\n",
       "      <td>{'mapped_name': 'Pneumonia', 'mapped_curie': '...</td>\n",
       "      <td>{'mapped_name': 'Sterile Water for Injection',...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544549</th>\n",
       "      <td>NCT04264533</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2020-2-14</td>\n",
       "      <td>f</td>\n",
       "      <td>Pneumonia, Ventilator-Associated</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Sterile Water for Injection</td>\n",
       "      <td>ventilator-associated</td>\n",
       "      <td>sterile water for injection</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{'mapped_name': 'Sterile Water for Injection',...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>750068 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             nct_id primary_purpose       intervention_model start_month_year  \\\n",
       "0       NCT01049399       Treatment      Parallel Assignment          2009-12   \n",
       "1       NCT05287724       Treatment     Crossover Assignment        2022-6-19   \n",
       "3       NCT01432236       Treatment     Crossover Assignment          2011-10   \n",
       "4       NCT04626921       Treatment  Single Group Assignment       2020-10-22   \n",
       "5       NCT04681417       Treatment      Parallel Assignment        2021-3-25   \n",
       "...             ...             ...                      ...              ...   \n",
       "544548  NCT04264533       Treatment      Parallel Assignment        2020-2-14   \n",
       "544548  NCT04264533       Treatment      Parallel Assignment        2020-2-14   \n",
       "544549  NCT04264533       Treatment      Parallel Assignment        2020-2-14   \n",
       "544549  NCT04264533       Treatment      Parallel Assignment        2020-2-14   \n",
       "544549  NCT04264533       Treatment      Parallel Assignment        2020-2-14   \n",
       "\n",
       "       is_fda_regulated_drug                         condition  \\\n",
       "0                          ?    Progressive Supranuclear Palsy   \n",
       "1                          t                          Pruritus   \n",
       "3                          ?                      Fibromyalgia   \n",
       "4                          t      Relapsing Multiple Sclerosis   \n",
       "5                          f                    Retinoblastoma   \n",
       "...                      ...                               ...   \n",
       "544548                     f                  Pneumonia, Viral   \n",
       "544548                     f                  Pneumonia, Viral   \n",
       "544549                     f  Pneumonia, Ventilator-Associated   \n",
       "544549                     f  Pneumonia, Ventilator-Associated   \n",
       "544549                     f  Pneumonia, Ventilator-Associated   \n",
       "\n",
       "                               predicate                           treatment  \\\n",
       "0                 in_clinical_trials_for                          tideglusib   \n",
       "1                 in_clinical_trials_for                   N-acetyl cysteine   \n",
       "3                 in_clinical_trials_for                          Pregabalin   \n",
       "4                 in_clinical_trials_for                             CNM-Au8   \n",
       "5                 in_clinical_trials_for  Melphalan or Melphalan + Topotecan   \n",
       "...                                  ...                                 ...   \n",
       "544548  mentioned_in_clinical_trials_for         Sterile Water for Injection   \n",
       "544548  mentioned_in_clinical_trials_for         Sterile Water for Injection   \n",
       "544549  mentioned_in_clinical_trials_for         Sterile Water for Injection   \n",
       "544549  mentioned_in_clinical_trials_for         Sterile Water for Injection   \n",
       "544549  mentioned_in_clinical_trials_for         Sterile Water for Injection   \n",
       "\n",
       "                   condition_collections               treatment_collections  \\\n",
       "0         progressive supranuclear palsy                          tideglusib   \n",
       "1                               pruritus                   n-acetyl cysteine   \n",
       "3                           fibromyalgia                          pregabalin   \n",
       "4           relapsing multiple sclerosis                             cnm-au8   \n",
       "5                         retinoblastoma  melphalan or melphalan + topotecan   \n",
       "...                                  ...                                 ...   \n",
       "544548                         pneumonia         sterile water for injection   \n",
       "544548                             viral         sterile water for injection   \n",
       "544549  pneumonia, ventilator-associated         sterile water for injection   \n",
       "544549                         pneumonia         sterile water for injection   \n",
       "544549             ventilator-associated         sterile water for injection   \n",
       "\n",
       "                                     condition_curie_info  \\\n",
       "0       {'mapped_name': 'Progressive supranuclear pals...   \n",
       "1       {'mapped_name': 'Pruritus', 'mapped_curie': 'C...   \n",
       "3       {'mapped_name': 'Fibromyalgia', 'mapped_curie'...   \n",
       "4       {'mapped_name': 'Multiple Sclerosis', 'mapped_...   \n",
       "5       {'mapped_name': 'Retinoblastoma', 'mapped_curi...   \n",
       "...                                                   ...   \n",
       "544548  {'mapped_name': 'Pneumonia', 'mapped_curie': '...   \n",
       "544548                                                NaN   \n",
       "544549  {'mapped_name': 'Pneumonia, Ventilator-Associa...   \n",
       "544549  {'mapped_name': 'Pneumonia', 'mapped_curie': '...   \n",
       "544549                                                NaN   \n",
       "\n",
       "                                     treatment_curie_info  \n",
       "0       {'mapped_name': 'Tideglusib', 'mapped_curie': ...  \n",
       "1       {'mapped_name': 'acetylcysteine', 'mapped_curi...  \n",
       "3       {'mapped_name': 'pregabalin', 'mapped_curie': ...  \n",
       "4                                                     NaN  \n",
       "5                                                     NaN  \n",
       "...                                                   ...  \n",
       "544548  {'mapped_name': 'Sterile Water for Injection',...  \n",
       "544548  {'mapped_name': 'Sterile Water for Injection',...  \n",
       "544549  {'mapped_name': 'Sterile Water for Injection',...  \n",
       "544549  {'mapped_name': 'Sterile Water for Injection',...  \n",
       "544549  {'mapped_name': 'Sterile Water for Injection',...  \n",
       "\n",
       "[750068 rows x 12 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ct['condition_curie_info'] = ct.condition_collections.map(conditions_dict)\n",
    "ct['treatment_curie_info'] = ct.treatment_collections.map(interventions_dict)\n",
    "ct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "85611741-9287-4f44-a763-9060e2b96588",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nct_id</th>\n",
       "      <th>primary_purpose</th>\n",
       "      <th>intervention_model</th>\n",
       "      <th>start_month_year</th>\n",
       "      <th>is_fda_regulated_drug</th>\n",
       "      <th>condition</th>\n",
       "      <th>predicate</th>\n",
       "      <th>treatment</th>\n",
       "      <th>condition_collections</th>\n",
       "      <th>treatment_collections</th>\n",
       "      <th>condition_curie_info</th>\n",
       "      <th>treatment_curie_info</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NCT01049399</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2009-12</td>\n",
       "      <td>?</td>\n",
       "      <td>Progressive Supranuclear Palsy</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>tideglusib</td>\n",
       "      <td>progressive supranuclear palsy</td>\n",
       "      <td>tideglusib</td>\n",
       "      <td>{'mapped_name': 'Progressive supranuclear pals...</td>\n",
       "      <td>{'mapped_name': 'Tideglusib', 'mapped_curie': ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NCT05287724</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Crossover Assignment</td>\n",
       "      <td>2022-6-19</td>\n",
       "      <td>t</td>\n",
       "      <td>Pruritus</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>N-acetyl cysteine</td>\n",
       "      <td>pruritus</td>\n",
       "      <td>n-acetyl cysteine</td>\n",
       "      <td>{'mapped_name': 'Pruritus', 'mapped_curie': 'C...</td>\n",
       "      <td>{'mapped_name': 'acetylcysteine', 'mapped_curi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NCT01432236</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Crossover Assignment</td>\n",
       "      <td>2011-10</td>\n",
       "      <td>?</td>\n",
       "      <td>Fibromyalgia</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>Pregabalin</td>\n",
       "      <td>fibromyalgia</td>\n",
       "      <td>pregabalin</td>\n",
       "      <td>{'mapped_name': 'Fibromyalgia', 'mapped_curie'...</td>\n",
       "      <td>{'mapped_name': 'pregabalin', 'mapped_curie': ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>NCT04681417</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2021-3-25</td>\n",
       "      <td>f</td>\n",
       "      <td>Retinoblastoma</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>Melphalan or Melphalan + Topotecan</td>\n",
       "      <td>retinoblastoma</td>\n",
       "      <td>melphalan</td>\n",
       "      <td>{'mapped_name': 'Retinoblastoma', 'mapped_curi...</td>\n",
       "      <td>{'mapped_name': 'melphalan', 'mapped_curie': '...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>NCT04681417</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2021-3-25</td>\n",
       "      <td>f</td>\n",
       "      <td>Retinoblastoma</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>Melphalan or Melphalan + Topotecan</td>\n",
       "      <td>retinoblastoma</td>\n",
       "      <td>topotecan</td>\n",
       "      <td>{'mapped_name': 'Retinoblastoma', 'mapped_curi...</td>\n",
       "      <td>{'mapped_name': 'topotecan', 'mapped_curie': '...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544547</th>\n",
       "      <td>NCT02576574</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2015-10-29</td>\n",
       "      <td>?</td>\n",
       "      <td>First Line Non-Small Cell Lung Cancer</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Pemetrexed</td>\n",
       "      <td>first line non-small cell lung cancer</td>\n",
       "      <td>pemetrexed</td>\n",
       "      <td>{'mapped_name': 'Non-Small Cell Lung Carcinoma...</td>\n",
       "      <td>{'mapped_name': 'pemetrexed', 'mapped_curie': ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544548</th>\n",
       "      <td>NCT04264533</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2020-2-14</td>\n",
       "      <td>f</td>\n",
       "      <td>Pneumonia, Viral</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Sterile Water for Injection</td>\n",
       "      <td>pneumonia, viral</td>\n",
       "      <td>sterile water for injection</td>\n",
       "      <td>{'mapped_name': 'Pneumonia, Viral', 'mapped_cu...</td>\n",
       "      <td>{'mapped_name': 'Sterile Water for Injection',...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544548</th>\n",
       "      <td>NCT04264533</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2020-2-14</td>\n",
       "      <td>f</td>\n",
       "      <td>Pneumonia, Viral</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Sterile Water for Injection</td>\n",
       "      <td>pneumonia</td>\n",
       "      <td>sterile water for injection</td>\n",
       "      <td>{'mapped_name': 'Pneumonia', 'mapped_curie': '...</td>\n",
       "      <td>{'mapped_name': 'Sterile Water for Injection',...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544549</th>\n",
       "      <td>NCT04264533</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2020-2-14</td>\n",
       "      <td>f</td>\n",
       "      <td>Pneumonia, Ventilator-Associated</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Sterile Water for Injection</td>\n",
       "      <td>pneumonia, ventilator-associated</td>\n",
       "      <td>sterile water for injection</td>\n",
       "      <td>{'mapped_name': 'Pneumonia, Ventilator-Associa...</td>\n",
       "      <td>{'mapped_name': 'Sterile Water for Injection',...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544549</th>\n",
       "      <td>NCT04264533</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2020-2-14</td>\n",
       "      <td>f</td>\n",
       "      <td>Pneumonia, Ventilator-Associated</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Sterile Water for Injection</td>\n",
       "      <td>pneumonia</td>\n",
       "      <td>sterile water for injection</td>\n",
       "      <td>{'mapped_name': 'Pneumonia', 'mapped_curie': '...</td>\n",
       "      <td>{'mapped_name': 'Sterile Water for Injection',...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>341720 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             nct_id primary_purpose    intervention_model start_month_year  \\\n",
       "0       NCT01049399       Treatment   Parallel Assignment          2009-12   \n",
       "1       NCT05287724       Treatment  Crossover Assignment        2022-6-19   \n",
       "3       NCT01432236       Treatment  Crossover Assignment          2011-10   \n",
       "5       NCT04681417       Treatment   Parallel Assignment        2021-3-25   \n",
       "5       NCT04681417       Treatment   Parallel Assignment        2021-3-25   \n",
       "...             ...             ...                   ...              ...   \n",
       "544547  NCT02576574       Treatment   Parallel Assignment       2015-10-29   \n",
       "544548  NCT04264533       Treatment   Parallel Assignment        2020-2-14   \n",
       "544548  NCT04264533       Treatment   Parallel Assignment        2020-2-14   \n",
       "544549  NCT04264533       Treatment   Parallel Assignment        2020-2-14   \n",
       "544549  NCT04264533       Treatment   Parallel Assignment        2020-2-14   \n",
       "\n",
       "       is_fda_regulated_drug                              condition  \\\n",
       "0                          ?         Progressive Supranuclear Palsy   \n",
       "1                          t                               Pruritus   \n",
       "3                          ?                           Fibromyalgia   \n",
       "5                          f                         Retinoblastoma   \n",
       "5                          f                         Retinoblastoma   \n",
       "...                      ...                                    ...   \n",
       "544547                     ?  First Line Non-Small Cell Lung Cancer   \n",
       "544548                     f                       Pneumonia, Viral   \n",
       "544548                     f                       Pneumonia, Viral   \n",
       "544549                     f       Pneumonia, Ventilator-Associated   \n",
       "544549                     f       Pneumonia, Ventilator-Associated   \n",
       "\n",
       "                               predicate                           treatment  \\\n",
       "0                 in_clinical_trials_for                          tideglusib   \n",
       "1                 in_clinical_trials_for                   N-acetyl cysteine   \n",
       "3                 in_clinical_trials_for                          Pregabalin   \n",
       "5                 in_clinical_trials_for  Melphalan or Melphalan + Topotecan   \n",
       "5                 in_clinical_trials_for  Melphalan or Melphalan + Topotecan   \n",
       "...                                  ...                                 ...   \n",
       "544547  mentioned_in_clinical_trials_for                          Pemetrexed   \n",
       "544548  mentioned_in_clinical_trials_for         Sterile Water for Injection   \n",
       "544548  mentioned_in_clinical_trials_for         Sterile Water for Injection   \n",
       "544549  mentioned_in_clinical_trials_for         Sterile Water for Injection   \n",
       "544549  mentioned_in_clinical_trials_for         Sterile Water for Injection   \n",
       "\n",
       "                        condition_collections        treatment_collections  \\\n",
       "0              progressive supranuclear palsy                   tideglusib   \n",
       "1                                    pruritus            n-acetyl cysteine   \n",
       "3                                fibromyalgia                   pregabalin   \n",
       "5                              retinoblastoma                    melphalan   \n",
       "5                              retinoblastoma                    topotecan   \n",
       "...                                       ...                          ...   \n",
       "544547  first line non-small cell lung cancer                   pemetrexed   \n",
       "544548                       pneumonia, viral  sterile water for injection   \n",
       "544548                              pneumonia  sterile water for injection   \n",
       "544549       pneumonia, ventilator-associated  sterile water for injection   \n",
       "544549                              pneumonia  sterile water for injection   \n",
       "\n",
       "                                     condition_curie_info  \\\n",
       "0       {'mapped_name': 'Progressive supranuclear pals...   \n",
       "1       {'mapped_name': 'Pruritus', 'mapped_curie': 'C...   \n",
       "3       {'mapped_name': 'Fibromyalgia', 'mapped_curie'...   \n",
       "5       {'mapped_name': 'Retinoblastoma', 'mapped_curi...   \n",
       "5       {'mapped_name': 'Retinoblastoma', 'mapped_curi...   \n",
       "...                                                   ...   \n",
       "544547  {'mapped_name': 'Non-Small Cell Lung Carcinoma...   \n",
       "544548  {'mapped_name': 'Pneumonia, Viral', 'mapped_cu...   \n",
       "544548  {'mapped_name': 'Pneumonia', 'mapped_curie': '...   \n",
       "544549  {'mapped_name': 'Pneumonia, Ventilator-Associa...   \n",
       "544549  {'mapped_name': 'Pneumonia', 'mapped_curie': '...   \n",
       "\n",
       "                                     treatment_curie_info  \n",
       "0       {'mapped_name': 'Tideglusib', 'mapped_curie': ...  \n",
       "1       {'mapped_name': 'acetylcysteine', 'mapped_curi...  \n",
       "3       {'mapped_name': 'pregabalin', 'mapped_curie': ...  \n",
       "5       {'mapped_name': 'melphalan', 'mapped_curie': '...  \n",
       "5       {'mapped_name': 'topotecan', 'mapped_curie': '...  \n",
       "...                                                   ...  \n",
       "544547  {'mapped_name': 'pemetrexed', 'mapped_curie': ...  \n",
       "544548  {'mapped_name': 'Sterile Water for Injection',...  \n",
       "544548  {'mapped_name': 'Sterile Water for Injection',...  \n",
       "544549  {'mapped_name': 'Sterile Water for Injection',...  \n",
       "544549  {'mapped_name': 'Sterile Water for Injection',...  \n",
       "\n",
       "[341720 rows x 12 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ct = ct.dropna(subset=[\"condition_curie_info\", \"treatment_curie_info\"])\n",
    "ct = ct[ct['condition'] != ct['treatment']]   # delete rows where the subject and object are =\n",
    "ct = ct[ct['condition_curie_info'] != ct['treatment_curie_info']]   # delete rows where the subject and object are =\n",
    "ct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "b4a80e96-429d-444c-8a7f-de5ec8fa949d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "336390"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nct_id</th>\n",
       "      <th>primary_purpose</th>\n",
       "      <th>intervention_model</th>\n",
       "      <th>start_month_year</th>\n",
       "      <th>is_fda_regulated_drug</th>\n",
       "      <th>condition</th>\n",
       "      <th>predicate</th>\n",
       "      <th>treatment</th>\n",
       "      <th>condition_collections</th>\n",
       "      <th>treatment_collections</th>\n",
       "      <th>condition_curie_info</th>\n",
       "      <th>treatment_curie_info</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NCT01049399</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2009-12</td>\n",
       "      <td>?</td>\n",
       "      <td>Progressive Supranuclear Palsy</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>tideglusib</td>\n",
       "      <td>progressive supranuclear palsy</td>\n",
       "      <td>tideglusib</td>\n",
       "      <td>{'mapped_name': 'Progressive supranuclear palsy', 'mapped_curie': 'C0038868', 'mapped_score': '5.18', 'mapped_semtypes': '[dsyn]'}</td>\n",
       "      <td>{'mapped_name': 'Tideglusib', 'mapped_curie': 'C3273375', 'mapped_score': '5.18', 'mapped_semtypes': '[orch,phsu]'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NCT05287724</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Crossover Assignment</td>\n",
       "      <td>2022-6-19</td>\n",
       "      <td>t</td>\n",
       "      <td>Pruritus</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>N-acetyl cysteine</td>\n",
       "      <td>pruritus</td>\n",
       "      <td>n-acetyl cysteine</td>\n",
       "      <td>{'mapped_name': 'Pruritus', 'mapped_curie': 'C0033774', 'mapped_score': '5.18', 'mapped_semtypes': '[fndg]'}</td>\n",
       "      <td>{'mapped_name': 'acetylcysteine', 'mapped_curie': 'C0001047', 'mapped_score': '5.18', 'mapped_semtypes': '[aapp,phsu]'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NCT01432236</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Crossover Assignment</td>\n",
       "      <td>2011-10</td>\n",
       "      <td>?</td>\n",
       "      <td>Fibromyalgia</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>Pregabalin</td>\n",
       "      <td>fibromyalgia</td>\n",
       "      <td>pregabalin</td>\n",
       "      <td>{'mapped_name': 'Fibromyalgia', 'mapped_curie': 'C0016053', 'mapped_score': '5.18', 'mapped_semtypes': '[dsyn]'}</td>\n",
       "      <td>{'mapped_name': 'pregabalin', 'mapped_curie': 'C0657912', 'mapped_score': '5.18', 'mapped_semtypes': '[aapp,phsu]'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>NCT04681417</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2021-3-25</td>\n",
       "      <td>f</td>\n",
       "      <td>Retinoblastoma</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>Melphalan or Melphalan + Topotecan</td>\n",
       "      <td>retinoblastoma</td>\n",
       "      <td>melphalan</td>\n",
       "      <td>{'mapped_name': 'Retinoblastoma', 'mapped_curie': 'C0035335', 'mapped_score': '5.18', 'mapped_semtypes': '[neop]'}</td>\n",
       "      <td>{'mapped_name': 'melphalan', 'mapped_curie': 'C0025241', 'mapped_score': '5.18', 'mapped_semtypes': '[aapp,phsu]'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>NCT04681417</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2021-3-25</td>\n",
       "      <td>f</td>\n",
       "      <td>Retinoblastoma</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>Melphalan or Melphalan + Topotecan</td>\n",
       "      <td>retinoblastoma</td>\n",
       "      <td>topotecan</td>\n",
       "      <td>{'mapped_name': 'Retinoblastoma', 'mapped_curie': 'C0035335', 'mapped_score': '5.18', 'mapped_semtypes': '[neop]'}</td>\n",
       "      <td>{'mapped_name': 'topotecan', 'mapped_curie': 'C0146224', 'mapped_score': '5.18', 'mapped_semtypes': '[orch,phsu]'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>NCT05624918</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Single Group Assignment</td>\n",
       "      <td>2024-4</td>\n",
       "      <td>f</td>\n",
       "      <td>Pancreatic Adenocarcinoma</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>Gemcitabine</td>\n",
       "      <td>pancreatic adenocarcinoma</td>\n",
       "      <td>gemcitabine</td>\n",
       "      <td>{'mapped_name': 'Adenocarcinoma of pancreas', 'mapped_curie': 'C0281361', 'mapped_score': '5.18', 'mapped_semtypes': '[neop]'}</td>\n",
       "      <td>{'mapped_name': 'gemcitabine', 'mapped_curie': 'C0045093', 'mapped_score': '5.18', 'mapped_semtypes': '[nnon,phsu]'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>NCT05624918</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Single Group Assignment</td>\n",
       "      <td>2024-4</td>\n",
       "      <td>f</td>\n",
       "      <td>Pancreatic Adenocarcinoma</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>NovoTTF-200T(P)</td>\n",
       "      <td>pancreatic adenocarcinoma</td>\n",
       "      <td>novottf-200t</td>\n",
       "      <td>{'mapped_name': 'Adenocarcinoma of pancreas', 'mapped_curie': 'C0281361', 'mapped_score': '5.18', 'mapped_semtypes': '[neop]'}</td>\n",
       "      <td>{'mapped_name': 'NovoTTF-200A Device', 'mapped_curie': 'UMLS:C4683819', 'mapped_score': 14.063435, 'mapped_semtypes': 'biolink:Device'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>NCT02886065</td>\n",
       "      <td>Prevention</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2017-3-7</td>\n",
       "      <td>t</td>\n",
       "      <td>Smoldering Multiple Myeloma</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>Citarinostat</td>\n",
       "      <td>smoldering multiple myeloma</td>\n",
       "      <td>citarinostat</td>\n",
       "      <td>{'mapped_name': 'Smoldering myeloma', 'mapped_curie': 'C1531608', 'mapped_score': '5.18', 'mapped_semtypes': '[neop]'}</td>\n",
       "      <td>{'mapped_name': 'Citarinostat', 'mapped_curie': 'C4724808', 'mapped_score': '5.18', 'mapped_semtypes': '[orch,phsu]'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>NCT02886065</td>\n",
       "      <td>Prevention</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2017-3-7</td>\n",
       "      <td>t</td>\n",
       "      <td>Smoldering Multiple Myeloma</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>Hiltonol</td>\n",
       "      <td>smoldering multiple myeloma</td>\n",
       "      <td>hiltonol</td>\n",
       "      <td>{'mapped_name': 'Smoldering myeloma', 'mapped_curie': 'C1531608', 'mapped_score': '5.18', 'mapped_semtypes': '[neop]'}</td>\n",
       "      <td>{'mapped_name': 'hiltonol', 'mapped_curie': 'C2698842', 'mapped_score': '5.18', 'mapped_semtypes': '[phsu]'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>NCT02886065</td>\n",
       "      <td>Prevention</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2017-3-7</td>\n",
       "      <td>t</td>\n",
       "      <td>Smoldering Multiple Myeloma</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>Lenalidomide</td>\n",
       "      <td>smoldering multiple myeloma</td>\n",
       "      <td>lenalidomide</td>\n",
       "      <td>{'mapped_name': 'Smoldering myeloma', 'mapped_curie': 'C1531608', 'mapped_score': '5.18', 'mapped_semtypes': '[neop]'}</td>\n",
       "      <td>{'mapped_name': 'lenalidomide', 'mapped_curie': 'C1144149', 'mapped_score': '5.18', 'mapped_semtypes': '[orch,phsu]'}</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         nct_id primary_purpose       intervention_model start_month_year  \\\n",
       "0   NCT01049399       Treatment      Parallel Assignment          2009-12   \n",
       "1   NCT05287724       Treatment     Crossover Assignment        2022-6-19   \n",
       "3   NCT01432236       Treatment     Crossover Assignment          2011-10   \n",
       "5   NCT04681417       Treatment      Parallel Assignment        2021-3-25   \n",
       "5   NCT04681417       Treatment      Parallel Assignment        2021-3-25   \n",
       "7   NCT05624918       Treatment  Single Group Assignment           2024-4   \n",
       "9   NCT05624918       Treatment  Single Group Assignment           2024-4   \n",
       "10  NCT02886065      Prevention      Parallel Assignment         2017-3-7   \n",
       "11  NCT02886065      Prevention      Parallel Assignment         2017-3-7   \n",
       "12  NCT02886065      Prevention      Parallel Assignment         2017-3-7   \n",
       "\n",
       "   is_fda_regulated_drug                       condition  \\\n",
       "0                      ?  Progressive Supranuclear Palsy   \n",
       "1                      t                        Pruritus   \n",
       "3                      ?                    Fibromyalgia   \n",
       "5                      f                  Retinoblastoma   \n",
       "5                      f                  Retinoblastoma   \n",
       "7                      f       Pancreatic Adenocarcinoma   \n",
       "9                      f       Pancreatic Adenocarcinoma   \n",
       "10                     t     Smoldering Multiple Myeloma   \n",
       "11                     t     Smoldering Multiple Myeloma   \n",
       "12                     t     Smoldering Multiple Myeloma   \n",
       "\n",
       "                 predicate                           treatment  \\\n",
       "0   in_clinical_trials_for                          tideglusib   \n",
       "1   in_clinical_trials_for                   N-acetyl cysteine   \n",
       "3   in_clinical_trials_for                          Pregabalin   \n",
       "5   in_clinical_trials_for  Melphalan or Melphalan + Topotecan   \n",
       "5   in_clinical_trials_for  Melphalan or Melphalan + Topotecan   \n",
       "7   in_clinical_trials_for                         Gemcitabine   \n",
       "9   in_clinical_trials_for                     NovoTTF-200T(P)   \n",
       "10  in_clinical_trials_for                        Citarinostat   \n",
       "11  in_clinical_trials_for                            Hiltonol   \n",
       "12  in_clinical_trials_for                        Lenalidomide   \n",
       "\n",
       "             condition_collections treatment_collections  \\\n",
       "0   progressive supranuclear palsy            tideglusib   \n",
       "1                         pruritus     n-acetyl cysteine   \n",
       "3                     fibromyalgia            pregabalin   \n",
       "5                   retinoblastoma             melphalan   \n",
       "5                   retinoblastoma             topotecan   \n",
       "7        pancreatic adenocarcinoma           gemcitabine   \n",
       "9        pancreatic adenocarcinoma          novottf-200t   \n",
       "10     smoldering multiple myeloma          citarinostat   \n",
       "11     smoldering multiple myeloma              hiltonol   \n",
       "12     smoldering multiple myeloma          lenalidomide   \n",
       "\n",
       "                                                                                                                  condition_curie_info  \\\n",
       "0   {'mapped_name': 'Progressive supranuclear palsy', 'mapped_curie': 'C0038868', 'mapped_score': '5.18', 'mapped_semtypes': '[dsyn]'}   \n",
       "1                         {'mapped_name': 'Pruritus', 'mapped_curie': 'C0033774', 'mapped_score': '5.18', 'mapped_semtypes': '[fndg]'}   \n",
       "3                     {'mapped_name': 'Fibromyalgia', 'mapped_curie': 'C0016053', 'mapped_score': '5.18', 'mapped_semtypes': '[dsyn]'}   \n",
       "5                   {'mapped_name': 'Retinoblastoma', 'mapped_curie': 'C0035335', 'mapped_score': '5.18', 'mapped_semtypes': '[neop]'}   \n",
       "5                   {'mapped_name': 'Retinoblastoma', 'mapped_curie': 'C0035335', 'mapped_score': '5.18', 'mapped_semtypes': '[neop]'}   \n",
       "7       {'mapped_name': 'Adenocarcinoma of pancreas', 'mapped_curie': 'C0281361', 'mapped_score': '5.18', 'mapped_semtypes': '[neop]'}   \n",
       "9       {'mapped_name': 'Adenocarcinoma of pancreas', 'mapped_curie': 'C0281361', 'mapped_score': '5.18', 'mapped_semtypes': '[neop]'}   \n",
       "10              {'mapped_name': 'Smoldering myeloma', 'mapped_curie': 'C1531608', 'mapped_score': '5.18', 'mapped_semtypes': '[neop]'}   \n",
       "11              {'mapped_name': 'Smoldering myeloma', 'mapped_curie': 'C1531608', 'mapped_score': '5.18', 'mapped_semtypes': '[neop]'}   \n",
       "12              {'mapped_name': 'Smoldering myeloma', 'mapped_curie': 'C1531608', 'mapped_score': '5.18', 'mapped_semtypes': '[neop]'}   \n",
       "\n",
       "                                                                                                                       treatment_curie_info  \n",
       "0                       {'mapped_name': 'Tideglusib', 'mapped_curie': 'C3273375', 'mapped_score': '5.18', 'mapped_semtypes': '[orch,phsu]'}  \n",
       "1                   {'mapped_name': 'acetylcysteine', 'mapped_curie': 'C0001047', 'mapped_score': '5.18', 'mapped_semtypes': '[aapp,phsu]'}  \n",
       "3                       {'mapped_name': 'pregabalin', 'mapped_curie': 'C0657912', 'mapped_score': '5.18', 'mapped_semtypes': '[aapp,phsu]'}  \n",
       "5                        {'mapped_name': 'melphalan', 'mapped_curie': 'C0025241', 'mapped_score': '5.18', 'mapped_semtypes': '[aapp,phsu]'}  \n",
       "5                        {'mapped_name': 'topotecan', 'mapped_curie': 'C0146224', 'mapped_score': '5.18', 'mapped_semtypes': '[orch,phsu]'}  \n",
       "7                      {'mapped_name': 'gemcitabine', 'mapped_curie': 'C0045093', 'mapped_score': '5.18', 'mapped_semtypes': '[nnon,phsu]'}  \n",
       "9   {'mapped_name': 'NovoTTF-200A Device', 'mapped_curie': 'UMLS:C4683819', 'mapped_score': 14.063435, 'mapped_semtypes': 'biolink:Device'}  \n",
       "10                    {'mapped_name': 'Citarinostat', 'mapped_curie': 'C4724808', 'mapped_score': '5.18', 'mapped_semtypes': '[orch,phsu]'}  \n",
       "11                             {'mapped_name': 'hiltonol', 'mapped_curie': 'C2698842', 'mapped_score': '5.18', 'mapped_semtypes': '[phsu]'}  \n",
       "12                    {'mapped_name': 'lenalidomide', 'mapped_curie': 'C1144149', 'mapped_score': '5.18', 'mapped_semtypes': '[orch,phsu]'}  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "conditions_to_exclude = [\"chronic\", \"viral\", \"vaccines\", \"virus diseases\", \"health volunteers\", \"healthy volunteers\",\n",
    "                         \"safety\", \"severe\",  \"intravenous\", \"injectable\",  \"liver\", \"healthy\", \n",
    "                         \"healthy men\", \"healthy women\", \"systemic\"]\n",
    "ct = ct[~ct['condition_collections'].isin(conditions_to_exclude)]\n",
    "len(ct)\n",
    "with pd.option_context('display.max_rows', None, 'display.max_columns', None, 'display.max_colwidth', None):  # more options can be specified also\n",
    "    display(ct[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "d04a5cec-97f6-4045-95b1-2fca14cdbe10",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # check ct dataframe before tacking on other trial-related information\n",
    "# ct.to_csv(\"ct_check.tsv\", sep='\\t', index=False, header=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ef2d161e-7cf1-4903-9c29-81559ada8cf3",
   "metadata": {},
   "source": [
    "### Add other trial information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "4cb2ba74-1c5a-4980-ac5e-a23464cedb64",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nct_id</th>\n",
       "      <th>primary_purpose</th>\n",
       "      <th>intervention_model</th>\n",
       "      <th>start_month_year</th>\n",
       "      <th>is_fda_regulated_drug</th>\n",
       "      <th>condition</th>\n",
       "      <th>predicate</th>\n",
       "      <th>treatment</th>\n",
       "      <th>condition_collections</th>\n",
       "      <th>treatment_collections</th>\n",
       "      <th>condition_curie_info</th>\n",
       "      <th>treatment_curie_info</th>\n",
       "      <th>study_url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NCT01049399</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2009-12</td>\n",
       "      <td>?</td>\n",
       "      <td>Progressive Supranuclear Palsy</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>tideglusib</td>\n",
       "      <td>progressive supranuclear palsy</td>\n",
       "      <td>tideglusib</td>\n",
       "      <td>{'mapped_name': 'Progressive supranuclear pals...</td>\n",
       "      <td>{'mapped_name': 'Tideglusib', 'mapped_curie': ...</td>\n",
       "      <td>https://www.clinicaltrials.gov/study/NCT01049399</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NCT05287724</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Crossover Assignment</td>\n",
       "      <td>2022-6-19</td>\n",
       "      <td>t</td>\n",
       "      <td>Pruritus</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>N-acetyl cysteine</td>\n",
       "      <td>pruritus</td>\n",
       "      <td>n-acetyl cysteine</td>\n",
       "      <td>{'mapped_name': 'Pruritus', 'mapped_curie': 'C...</td>\n",
       "      <td>{'mapped_name': 'acetylcysteine', 'mapped_curi...</td>\n",
       "      <td>https://www.clinicaltrials.gov/study/NCT05287724</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NCT01432236</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Crossover Assignment</td>\n",
       "      <td>2011-10</td>\n",
       "      <td>?</td>\n",
       "      <td>Fibromyalgia</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>Pregabalin</td>\n",
       "      <td>fibromyalgia</td>\n",
       "      <td>pregabalin</td>\n",
       "      <td>{'mapped_name': 'Fibromyalgia', 'mapped_curie'...</td>\n",
       "      <td>{'mapped_name': 'pregabalin', 'mapped_curie': ...</td>\n",
       "      <td>https://www.clinicaltrials.gov/study/NCT01432236</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>NCT04681417</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2021-3-25</td>\n",
       "      <td>f</td>\n",
       "      <td>Retinoblastoma</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>Melphalan or Melphalan + Topotecan</td>\n",
       "      <td>retinoblastoma</td>\n",
       "      <td>melphalan</td>\n",
       "      <td>{'mapped_name': 'Retinoblastoma', 'mapped_curi...</td>\n",
       "      <td>{'mapped_name': 'melphalan', 'mapped_curie': '...</td>\n",
       "      <td>https://www.clinicaltrials.gov/study/NCT04681417</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>NCT04681417</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2021-3-25</td>\n",
       "      <td>f</td>\n",
       "      <td>Retinoblastoma</td>\n",
       "      <td>in_clinical_trials_for</td>\n",
       "      <td>Melphalan or Melphalan + Topotecan</td>\n",
       "      <td>retinoblastoma</td>\n",
       "      <td>topotecan</td>\n",
       "      <td>{'mapped_name': 'Retinoblastoma', 'mapped_curi...</td>\n",
       "      <td>{'mapped_name': 'topotecan', 'mapped_curie': '...</td>\n",
       "      <td>https://www.clinicaltrials.gov/study/NCT04681417</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544547</th>\n",
       "      <td>NCT02576574</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2015-10-29</td>\n",
       "      <td>?</td>\n",
       "      <td>First Line Non-Small Cell Lung Cancer</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Pemetrexed</td>\n",
       "      <td>first line non-small cell lung cancer</td>\n",
       "      <td>pemetrexed</td>\n",
       "      <td>{'mapped_name': 'Non-Small Cell Lung Carcinoma...</td>\n",
       "      <td>{'mapped_name': 'pemetrexed', 'mapped_curie': ...</td>\n",
       "      <td>https://www.clinicaltrials.gov/study/NCT02576574</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544548</th>\n",
       "      <td>NCT04264533</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2020-2-14</td>\n",
       "      <td>f</td>\n",
       "      <td>Pneumonia, Viral</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Sterile Water for Injection</td>\n",
       "      <td>pneumonia, viral</td>\n",
       "      <td>sterile water for injection</td>\n",
       "      <td>{'mapped_name': 'Pneumonia, Viral', 'mapped_cu...</td>\n",
       "      <td>{'mapped_name': 'Sterile Water for Injection',...</td>\n",
       "      <td>https://www.clinicaltrials.gov/study/NCT04264533</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544548</th>\n",
       "      <td>NCT04264533</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2020-2-14</td>\n",
       "      <td>f</td>\n",
       "      <td>Pneumonia, Viral</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Sterile Water for Injection</td>\n",
       "      <td>pneumonia</td>\n",
       "      <td>sterile water for injection</td>\n",
       "      <td>{'mapped_name': 'Pneumonia', 'mapped_curie': '...</td>\n",
       "      <td>{'mapped_name': 'Sterile Water for Injection',...</td>\n",
       "      <td>https://www.clinicaltrials.gov/study/NCT04264533</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544549</th>\n",
       "      <td>NCT04264533</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2020-2-14</td>\n",
       "      <td>f</td>\n",
       "      <td>Pneumonia, Ventilator-Associated</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Sterile Water for Injection</td>\n",
       "      <td>pneumonia, ventilator-associated</td>\n",
       "      <td>sterile water for injection</td>\n",
       "      <td>{'mapped_name': 'Pneumonia, Ventilator-Associa...</td>\n",
       "      <td>{'mapped_name': 'Sterile Water for Injection',...</td>\n",
       "      <td>https://www.clinicaltrials.gov/study/NCT04264533</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544549</th>\n",
       "      <td>NCT04264533</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Parallel Assignment</td>\n",
       "      <td>2020-2-14</td>\n",
       "      <td>f</td>\n",
       "      <td>Pneumonia, Ventilator-Associated</td>\n",
       "      <td>mentioned_in_clinical_trials_for</td>\n",
       "      <td>Sterile Water for Injection</td>\n",
       "      <td>pneumonia</td>\n",
       "      <td>sterile water for injection</td>\n",
       "      <td>{'mapped_name': 'Pneumonia', 'mapped_curie': '...</td>\n",
       "      <td>{'mapped_name': 'Sterile Water for Injection',...</td>\n",
       "      <td>https://www.clinicaltrials.gov/study/NCT04264533</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>336390 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             nct_id primary_purpose    intervention_model start_month_year  \\\n",
       "0       NCT01049399       Treatment   Parallel Assignment          2009-12   \n",
       "1       NCT05287724       Treatment  Crossover Assignment        2022-6-19   \n",
       "3       NCT01432236       Treatment  Crossover Assignment          2011-10   \n",
       "5       NCT04681417       Treatment   Parallel Assignment        2021-3-25   \n",
       "5       NCT04681417       Treatment   Parallel Assignment        2021-3-25   \n",
       "...             ...             ...                   ...              ...   \n",
       "544547  NCT02576574       Treatment   Parallel Assignment       2015-10-29   \n",
       "544548  NCT04264533       Treatment   Parallel Assignment        2020-2-14   \n",
       "544548  NCT04264533       Treatment   Parallel Assignment        2020-2-14   \n",
       "544549  NCT04264533       Treatment   Parallel Assignment        2020-2-14   \n",
       "544549  NCT04264533       Treatment   Parallel Assignment        2020-2-14   \n",
       "\n",
       "       is_fda_regulated_drug                              condition  \\\n",
       "0                          ?         Progressive Supranuclear Palsy   \n",
       "1                          t                               Pruritus   \n",
       "3                          ?                           Fibromyalgia   \n",
       "5                          f                         Retinoblastoma   \n",
       "5                          f                         Retinoblastoma   \n",
       "...                      ...                                    ...   \n",
       "544547                     ?  First Line Non-Small Cell Lung Cancer   \n",
       "544548                     f                       Pneumonia, Viral   \n",
       "544548                     f                       Pneumonia, Viral   \n",
       "544549                     f       Pneumonia, Ventilator-Associated   \n",
       "544549                     f       Pneumonia, Ventilator-Associated   \n",
       "\n",
       "                               predicate                           treatment  \\\n",
       "0                 in_clinical_trials_for                          tideglusib   \n",
       "1                 in_clinical_trials_for                   N-acetyl cysteine   \n",
       "3                 in_clinical_trials_for                          Pregabalin   \n",
       "5                 in_clinical_trials_for  Melphalan or Melphalan + Topotecan   \n",
       "5                 in_clinical_trials_for  Melphalan or Melphalan + Topotecan   \n",
       "...                                  ...                                 ...   \n",
       "544547  mentioned_in_clinical_trials_for                          Pemetrexed   \n",
       "544548  mentioned_in_clinical_trials_for         Sterile Water for Injection   \n",
       "544548  mentioned_in_clinical_trials_for         Sterile Water for Injection   \n",
       "544549  mentioned_in_clinical_trials_for         Sterile Water for Injection   \n",
       "544549  mentioned_in_clinical_trials_for         Sterile Water for Injection   \n",
       "\n",
       "                        condition_collections        treatment_collections  \\\n",
       "0              progressive supranuclear palsy                   tideglusib   \n",
       "1                                    pruritus            n-acetyl cysteine   \n",
       "3                                fibromyalgia                   pregabalin   \n",
       "5                              retinoblastoma                    melphalan   \n",
       "5                              retinoblastoma                    topotecan   \n",
       "...                                       ...                          ...   \n",
       "544547  first line non-small cell lung cancer                   pemetrexed   \n",
       "544548                       pneumonia, viral  sterile water for injection   \n",
       "544548                              pneumonia  sterile water for injection   \n",
       "544549       pneumonia, ventilator-associated  sterile water for injection   \n",
       "544549                              pneumonia  sterile water for injection   \n",
       "\n",
       "                                     condition_curie_info  \\\n",
       "0       {'mapped_name': 'Progressive supranuclear pals...   \n",
       "1       {'mapped_name': 'Pruritus', 'mapped_curie': 'C...   \n",
       "3       {'mapped_name': 'Fibromyalgia', 'mapped_curie'...   \n",
       "5       {'mapped_name': 'Retinoblastoma', 'mapped_curi...   \n",
       "5       {'mapped_name': 'Retinoblastoma', 'mapped_curi...   \n",
       "...                                                   ...   \n",
       "544547  {'mapped_name': 'Non-Small Cell Lung Carcinoma...   \n",
       "544548  {'mapped_name': 'Pneumonia, Viral', 'mapped_cu...   \n",
       "544548  {'mapped_name': 'Pneumonia', 'mapped_curie': '...   \n",
       "544549  {'mapped_name': 'Pneumonia, Ventilator-Associa...   \n",
       "544549  {'mapped_name': 'Pneumonia', 'mapped_curie': '...   \n",
       "\n",
       "                                     treatment_curie_info  \\\n",
       "0       {'mapped_name': 'Tideglusib', 'mapped_curie': ...   \n",
       "1       {'mapped_name': 'acetylcysteine', 'mapped_curi...   \n",
       "3       {'mapped_name': 'pregabalin', 'mapped_curie': ...   \n",
       "5       {'mapped_name': 'melphalan', 'mapped_curie': '...   \n",
       "5       {'mapped_name': 'topotecan', 'mapped_curie': '...   \n",
       "...                                                   ...   \n",
       "544547  {'mapped_name': 'pemetrexed', 'mapped_curie': ...   \n",
       "544548  {'mapped_name': 'Sterile Water for Injection',...   \n",
       "544548  {'mapped_name': 'Sterile Water for Injection',...   \n",
       "544549  {'mapped_name': 'Sterile Water for Injection',...   \n",
       "544549  {'mapped_name': 'Sterile Water for Injection',...   \n",
       "\n",
       "                                               study_url  \n",
       "0       https://www.clinicaltrials.gov/study/NCT01049399  \n",
       "1       https://www.clinicaltrials.gov/study/NCT05287724  \n",
       "3       https://www.clinicaltrials.gov/study/NCT01432236  \n",
       "5       https://www.clinicaltrials.gov/study/NCT04681417  \n",
       "5       https://www.clinicaltrials.gov/study/NCT04681417  \n",
       "...                                                  ...  \n",
       "544547  https://www.clinicaltrials.gov/study/NCT02576574  \n",
       "544548  https://www.clinicaltrials.gov/study/NCT04264533  \n",
       "544548  https://www.clinicaltrials.gov/study/NCT04264533  \n",
       "544549  https://www.clinicaltrials.gov/study/NCT04264533  \n",
       "544549  https://www.clinicaltrials.gov/study/NCT04264533  \n",
       "\n",
       "[336390 rows x 13 columns]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ct = ct.copy()\n",
    "ct[\"study_url\"] = ct['nct_id'].apply(lambda x: f'https://www.clinicaltrials.gov/study/{x}')\n",
    "ct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "cd9a362e-ef21-4d25-b4d5-f2c7e5bd285e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# read in other tables \n",
    "data_extracted = \"/Users/kamhome/all_projects/ISB/Translator/Projects/ClinicalTrials/ETL_Python/data/6zy655l6br1i6q5o2h32k4h9wjx8\"\n",
    "study_references = pd.read_csv(data_extracted + '/study_references.txt', sep='|', dtype=object, index_col=False, header=0)\n",
    "studies = pd.read_csv(data_extracted + '/studies.txt', sep='|', dtype=object, index_col=False, header=0)\n",
    "calculated_values = pd.read_csv(data_extracted + '/calculated_values.txt', sep='|', dtype=object, index_col=False, header=0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "f5f189be-b350-4b9c-9c8f-992671680cf2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>nct_id</th>\n",
       "      <th>pmid</th>\n",
       "      <th>reference_type</th>\n",
       "      <th>citation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>87323472</td>\n",
       "      <td>NCT00624156</td>\n",
       "      <td>21678181</td>\n",
       "      <td>derived</td>\n",
       "      <td>Mosher CE, Duhamel KN, Lam J, Dickler M, Li Y,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>87323473</td>\n",
       "      <td>NCT05396586</td>\n",
       "      <td>35106729</td>\n",
       "      <td>background</td>\n",
       "      <td>Pahor A, Mester RE, Carrillo AA, Ghil E, Reime...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>87323474</td>\n",
       "      <td>NCT05396586</td>\n",
       "      <td>30367386</td>\n",
       "      <td>background</td>\n",
       "      <td>Pahor A, Stavropoulos T, Jaeggi SM, Seitz AR. ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>87323475</td>\n",
       "      <td>NCT05396586</td>\n",
       "      <td>17852284</td>\n",
       "      <td>background</td>\n",
       "      <td>Royle J, Lincoln NB. The Everyday Memory Quest...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>87323476</td>\n",
       "      <td>NCT05396586</td>\n",
       "      <td>34485810</td>\n",
       "      <td>background</td>\n",
       "      <td>Pahor A, Collins C, Smith RN, Moon A, Stavropo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>871607</th>\n",
       "      <td>87323467</td>\n",
       "      <td>NCT00624078</td>\n",
       "      <td>16197974</td>\n",
       "      <td>background</td>\n",
       "      <td>Vazquez H, Chavez-Haro A, Garcia-Ubbelohde W, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>871608</th>\n",
       "      <td>87323468</td>\n",
       "      <td>NCT00624078</td>\n",
       "      <td>6516334</td>\n",
       "      <td>background</td>\n",
       "      <td>Likes K, Banner W Jr, Chavez M. Centruroides e...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>871609</th>\n",
       "      <td>87323469</td>\n",
       "      <td>NCT00624078</td>\n",
       "      <td>17015284</td>\n",
       "      <td>background</td>\n",
       "      <td>Lai MW, Klein-Schwartz W, Rodgers GC, Abrams J...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>871610</th>\n",
       "      <td>87323470</td>\n",
       "      <td>NCT00624078</td>\n",
       "      <td>10533010</td>\n",
       "      <td>background</td>\n",
       "      <td>Gibly R, Williams M, Walter FG, McNally J, Con...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>871611</th>\n",
       "      <td>87323471</td>\n",
       "      <td>NCT00624078</td>\n",
       "      <td>16546632</td>\n",
       "      <td>background</td>\n",
       "      <td>Riley BD, LoVecchio F, Pizon AF. Lack of scorp...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>871612 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              id       nct_id      pmid reference_type  \\\n",
       "0       87323472  NCT00624156  21678181        derived   \n",
       "1       87323473  NCT05396586  35106729     background   \n",
       "2       87323474  NCT05396586  30367386     background   \n",
       "3       87323475  NCT05396586  17852284     background   \n",
       "4       87323476  NCT05396586  34485810     background   \n",
       "...          ...          ...       ...            ...   \n",
       "871607  87323467  NCT00624078  16197974     background   \n",
       "871608  87323468  NCT00624078   6516334     background   \n",
       "871609  87323469  NCT00624078  17015284     background   \n",
       "871610  87323470  NCT00624078  10533010     background   \n",
       "871611  87323471  NCT00624078  16546632     background   \n",
       "\n",
       "                                                 citation  \n",
       "0       Mosher CE, Duhamel KN, Lam J, Dickler M, Li Y,...  \n",
       "1       Pahor A, Mester RE, Carrillo AA, Ghil E, Reime...  \n",
       "2       Pahor A, Stavropoulos T, Jaeggi SM, Seitz AR. ...  \n",
       "3       Royle J, Lincoln NB. The Everyday Memory Quest...  \n",
       "4       Pahor A, Collins C, Smith RN, Moon A, Stavropo...  \n",
       "...                                                   ...  \n",
       "871607  Vazquez H, Chavez-Haro A, Garcia-Ubbelohde W, ...  \n",
       "871608  Likes K, Banner W Jr, Chavez M. Centruroides e...  \n",
       "871609  Lai MW, Klein-Schwartz W, Rodgers GC, Abrams J...  \n",
       "871610  Gibly R, Williams M, Walter FG, McNally J, Con...  \n",
       "871611  Riley BD, LoVecchio F, Pizon AF. Lack of scorp...  \n",
       "\n",
       "[871612 rows x 5 columns]"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>result_url</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nct_id</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>NCT00000112</th>\n",
       "      <td>[https://pubmed.ncbi.nlm.nih.gov/35235641, htt...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NCT00000113</th>\n",
       "      <td>[https://pubmed.ncbi.nlm.nih.gov/12657584, htt...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NCT00000114</th>\n",
       "      <td>[https://pubmed.ncbi.nlm.nih.gov/8512476, http...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NCT00000115</th>\n",
       "      <td>[https://pubmed.ncbi.nlm.nih.gov/8684794]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NCT00000116</th>\n",
       "      <td>[https://pubmed.ncbi.nlm.nih.gov/8512476, http...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NCT06366048</th>\n",
       "      <td>[https://pubmed.ncbi.nlm.nih.gov/31145929, htt...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NCT06366100</th>\n",
       "      <td>[https://pubmed.ncbi.nlm.nih.gov/34098412, htt...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NCT06366412</th>\n",
       "      <td>[https://pubmed.ncbi.nlm.nih.gov/34418401, htt...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NCT06366425</th>\n",
       "      <td>[https://pubmed.ncbi.nlm.nih.gov/18395077, htt...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NCT06366438</th>\n",
       "      <td>[https://pubmed.ncbi.nlm.nih.gov/32897527, htt...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>167130 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    result_url\n",
       "nct_id                                                        \n",
       "NCT00000112  [https://pubmed.ncbi.nlm.nih.gov/35235641, htt...\n",
       "NCT00000113  [https://pubmed.ncbi.nlm.nih.gov/12657584, htt...\n",
       "NCT00000114  [https://pubmed.ncbi.nlm.nih.gov/8512476, http...\n",
       "NCT00000115          [https://pubmed.ncbi.nlm.nih.gov/8684794]\n",
       "NCT00000116  [https://pubmed.ncbi.nlm.nih.gov/8512476, http...\n",
       "...                                                        ...\n",
       "NCT06366048  [https://pubmed.ncbi.nlm.nih.gov/31145929, htt...\n",
       "NCT06366100  [https://pubmed.ncbi.nlm.nih.gov/34098412, htt...\n",
       "NCT06366412  [https://pubmed.ncbi.nlm.nih.gov/34418401, htt...\n",
       "NCT06366425  [https://pubmed.ncbi.nlm.nih.gov/18395077, htt...\n",
       "NCT06366438  [https://pubmed.ncbi.nlm.nih.gov/32897527, htt...\n",
       "\n",
       "[167130 rows x 1 columns]"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "study_references\n",
    "study_references[\"result_url\"] = study_references['pmid'].apply(lambda x: f'https://pubmed.ncbi.nlm.nih.gov/{x}')\n",
    "study_results_agg = pd.DataFrame(study_references[[\"nct_id\", \"result_url\"]].groupby(\"nct_id\")[\"result_url\"].apply(list)) # get all pmids available for each study\n",
    "study_results_agg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "85a0cf47-aa14-490e-9d28-37fde4b84a8c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nct_id</th>\n",
       "      <th>nlm_download_date_description</th>\n",
       "      <th>study_first_submitted_date</th>\n",
       "      <th>results_first_submitted_date</th>\n",
       "      <th>disposition_first_submitted_date</th>\n",
       "      <th>last_update_submitted_date</th>\n",
       "      <th>study_first_submitted_qc_date</th>\n",
       "      <th>study_first_posted_date</th>\n",
       "      <th>study_first_posted_date_type</th>\n",
       "      <th>results_first_submitted_qc_date</th>\n",
       "      <th>...</th>\n",
       "      <th>last_update_posted_date</th>\n",
       "      <th>last_update_posted_date_type</th>\n",
       "      <th>start_date_type</th>\n",
       "      <th>start_date</th>\n",
       "      <th>verification_date</th>\n",
       "      <th>completion_date_type</th>\n",
       "      <th>completion_date</th>\n",
       "      <th>primary_completion_date_type</th>\n",
       "      <th>primary_completion_date</th>\n",
       "      <th>updated_at</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NCT00214500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2005-09-13</td>\n",
       "      <td>2018-08-10</td>\n",
       "      <td>2010-08-17</td>\n",
       "      <td>2018-10-01</td>\n",
       "      <td>2005-09-14</td>\n",
       "      <td>2005-09-22</td>\n",
       "      <td>Estimate</td>\n",
       "      <td>2018-08-10</td>\n",
       "      <td>...</td>\n",
       "      <td>2018-10-30</td>\n",
       "      <td>Actual</td>\n",
       "      <td>Actual</td>\n",
       "      <td>2006-01-02</td>\n",
       "      <td>2018-10-31</td>\n",
       "      <td>Actual</td>\n",
       "      <td>2008-01-29</td>\n",
       "      <td>Actual</td>\n",
       "      <td>2008-01-29</td>\n",
       "      <td>2024-04-13 15:18:37.689771</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NCT04002726</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019-06-19</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2022-08-18</td>\n",
       "      <td>2019-06-28</td>\n",
       "      <td>2019-07-01</td>\n",
       "      <td>Actual</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>2022-08-22</td>\n",
       "      <td>Actual</td>\n",
       "      <td>Actual</td>\n",
       "      <td>2019-07-08</td>\n",
       "      <td>2022-05-31</td>\n",
       "      <td>Actual</td>\n",
       "      <td>2019-07-12</td>\n",
       "      <td>Actual</td>\n",
       "      <td>2019-07-12</td>\n",
       "      <td>2024-04-13 15:18:38.732734</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NCT03597984</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018-05-31</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019-07-01</td>\n",
       "      <td>2018-07-22</td>\n",
       "      <td>2018-07-24</td>\n",
       "      <td>Actual</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>2019-07-05</td>\n",
       "      <td>Actual</td>\n",
       "      <td>Anticipated</td>\n",
       "      <td>2019-07-01</td>\n",
       "      <td>2019-07-31</td>\n",
       "      <td>Anticipated</td>\n",
       "      <td>2020-12-28</td>\n",
       "      <td>Actual</td>\n",
       "      <td>2019-07-01</td>\n",
       "      <td>2024-04-13 15:18:39.514159</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NCT03597906</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018-06-21</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2020-01-26</td>\n",
       "      <td>2018-07-22</td>\n",
       "      <td>2018-07-24</td>\n",
       "      <td>Actual</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>2020-01-28</td>\n",
       "      <td>Actual</td>\n",
       "      <td>Actual</td>\n",
       "      <td>2018-08-15</td>\n",
       "      <td>2020-01-31</td>\n",
       "      <td>Actual</td>\n",
       "      <td>2020-01-20</td>\n",
       "      <td>Actual</td>\n",
       "      <td>2019-11-15</td>\n",
       "      <td>2024-04-13 15:18:40.664541</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NCT03597828</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018-06-27</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2020-06-29</td>\n",
       "      <td>2018-07-21</td>\n",
       "      <td>2018-07-24</td>\n",
       "      <td>Actual</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>2020-06-30</td>\n",
       "      <td>Actual</td>\n",
       "      <td>Actual</td>\n",
       "      <td>2018-08-05</td>\n",
       "      <td>2020-06-30</td>\n",
       "      <td>Actual</td>\n",
       "      <td>2020-06-30</td>\n",
       "      <td>Actual</td>\n",
       "      <td>2020-06-30</td>\n",
       "      <td>2024-04-13 15:18:41.432351</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>491081</th>\n",
       "      <td>NCT03593460</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018-06-07</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018-12-31</td>\n",
       "      <td>2018-07-09</td>\n",
       "      <td>2018-07-20</td>\n",
       "      <td>Actual</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>2019-01-03</td>\n",
       "      <td>Actual</td>\n",
       "      <td>Anticipated</td>\n",
       "      <td>2019-01-01</td>\n",
       "      <td>2018-12-31</td>\n",
       "      <td>Anticipated</td>\n",
       "      <td>2019-12-31</td>\n",
       "      <td>Anticipated</td>\n",
       "      <td>2019-11-01</td>\n",
       "      <td>2024-04-13 15:18:27.19171</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>491082</th>\n",
       "      <td>NCT03024294</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2017-01-06</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018-07-12</td>\n",
       "      <td>2017-01-13</td>\n",
       "      <td>2017-01-18</td>\n",
       "      <td>Estimate</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>2018-07-16</td>\n",
       "      <td>Actual</td>\n",
       "      <td>Actual</td>\n",
       "      <td>2017-01-31</td>\n",
       "      <td>2018-07-31</td>\n",
       "      <td>Anticipated</td>\n",
       "      <td>2019-12-31</td>\n",
       "      <td>Anticipated</td>\n",
       "      <td>2019-12-31</td>\n",
       "      <td>2024-04-13 15:18:27.880336</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>491083</th>\n",
       "      <td>NCT03593382</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2012-09-16</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018-07-09</td>\n",
       "      <td>2018-07-09</td>\n",
       "      <td>2018-07-20</td>\n",
       "      <td>Actual</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>2018-07-20</td>\n",
       "      <td>Actual</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2011-11-30</td>\n",
       "      <td>2018-07-31</td>\n",
       "      <td>Actual</td>\n",
       "      <td>2013-12-31</td>\n",
       "      <td>Actual</td>\n",
       "      <td>2013-06-30</td>\n",
       "      <td>2024-04-13 15:18:28.57842</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>491084</th>\n",
       "      <td>NCT04407078</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2020-05-25</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2021-01-06</td>\n",
       "      <td>2020-05-25</td>\n",
       "      <td>2020-05-29</td>\n",
       "      <td>Actual</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>2021-01-08</td>\n",
       "      <td>Actual</td>\n",
       "      <td>Actual</td>\n",
       "      <td>2019-12-02</td>\n",
       "      <td>2021-01-31</td>\n",
       "      <td>Actual</td>\n",
       "      <td>2021-01-06</td>\n",
       "      <td>Actual</td>\n",
       "      <td>2021-01-06</td>\n",
       "      <td>2024-04-13 15:18:29.292283</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>491085</th>\n",
       "      <td>NCT04003116</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019-06-27</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2022-08-17</td>\n",
       "      <td>2019-06-27</td>\n",
       "      <td>2019-07-01</td>\n",
       "      <td>Actual</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>2022-08-18</td>\n",
       "      <td>Actual</td>\n",
       "      <td>Actual</td>\n",
       "      <td>2019-07-11</td>\n",
       "      <td>2022-08-31</td>\n",
       "      <td>Actual</td>\n",
       "      <td>2022-07-25</td>\n",
       "      <td>Actual</td>\n",
       "      <td>2022-04-24</td>\n",
       "      <td>2024-04-13 15:18:30.070101</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>491086 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             nct_id nlm_download_date_description study_first_submitted_date  \\\n",
       "0       NCT00214500                           NaN                 2005-09-13   \n",
       "1       NCT04002726                           NaN                 2019-06-19   \n",
       "2       NCT03597984                           NaN                 2018-05-31   \n",
       "3       NCT03597906                           NaN                 2018-06-21   \n",
       "4       NCT03597828                           NaN                 2018-06-27   \n",
       "...             ...                           ...                        ...   \n",
       "491081  NCT03593460                           NaN                 2018-06-07   \n",
       "491082  NCT03024294                           NaN                 2017-01-06   \n",
       "491083  NCT03593382                           NaN                 2012-09-16   \n",
       "491084  NCT04407078                           NaN                 2020-05-25   \n",
       "491085  NCT04003116                           NaN                 2019-06-27   \n",
       "\n",
       "       results_first_submitted_date disposition_first_submitted_date  \\\n",
       "0                        2018-08-10                       2010-08-17   \n",
       "1                               NaN                              NaN   \n",
       "2                               NaN                              NaN   \n",
       "3                               NaN                              NaN   \n",
       "4                               NaN                              NaN   \n",
       "...                             ...                              ...   \n",
       "491081                          NaN                              NaN   \n",
       "491082                          NaN                              NaN   \n",
       "491083                          NaN                              NaN   \n",
       "491084                          NaN                              NaN   \n",
       "491085                          NaN                              NaN   \n",
       "\n",
       "       last_update_submitted_date study_first_submitted_qc_date  \\\n",
       "0                      2018-10-01                    2005-09-14   \n",
       "1                      2022-08-18                    2019-06-28   \n",
       "2                      2019-07-01                    2018-07-22   \n",
       "3                      2020-01-26                    2018-07-22   \n",
       "4                      2020-06-29                    2018-07-21   \n",
       "...                           ...                           ...   \n",
       "491081                 2018-12-31                    2018-07-09   \n",
       "491082                 2018-07-12                    2017-01-13   \n",
       "491083                 2018-07-09                    2018-07-09   \n",
       "491084                 2021-01-06                    2020-05-25   \n",
       "491085                 2022-08-17                    2019-06-27   \n",
       "\n",
       "       study_first_posted_date study_first_posted_date_type  \\\n",
       "0                   2005-09-22                     Estimate   \n",
       "1                   2019-07-01                       Actual   \n",
       "2                   2018-07-24                       Actual   \n",
       "3                   2018-07-24                       Actual   \n",
       "4                   2018-07-24                       Actual   \n",
       "...                        ...                          ...   \n",
       "491081              2018-07-20                       Actual   \n",
       "491082              2017-01-18                     Estimate   \n",
       "491083              2018-07-20                       Actual   \n",
       "491084              2020-05-29                       Actual   \n",
       "491085              2019-07-01                       Actual   \n",
       "\n",
       "       results_first_submitted_qc_date  ... last_update_posted_date  \\\n",
       "0                           2018-08-10  ...              2018-10-30   \n",
       "1                                  NaN  ...              2022-08-22   \n",
       "2                                  NaN  ...              2019-07-05   \n",
       "3                                  NaN  ...              2020-01-28   \n",
       "4                                  NaN  ...              2020-06-30   \n",
       "...                                ...  ...                     ...   \n",
       "491081                             NaN  ...              2019-01-03   \n",
       "491082                             NaN  ...              2018-07-16   \n",
       "491083                             NaN  ...              2018-07-20   \n",
       "491084                             NaN  ...              2021-01-08   \n",
       "491085                             NaN  ...              2022-08-18   \n",
       "\n",
       "       last_update_posted_date_type start_date_type  start_date  \\\n",
       "0                            Actual          Actual  2006-01-02   \n",
       "1                            Actual          Actual  2019-07-08   \n",
       "2                            Actual     Anticipated  2019-07-01   \n",
       "3                            Actual          Actual  2018-08-15   \n",
       "4                            Actual          Actual  2018-08-05   \n",
       "...                             ...             ...         ...   \n",
       "491081                       Actual     Anticipated  2019-01-01   \n",
       "491082                       Actual          Actual  2017-01-31   \n",
       "491083                       Actual             NaN  2011-11-30   \n",
       "491084                       Actual          Actual  2019-12-02   \n",
       "491085                       Actual          Actual  2019-07-11   \n",
       "\n",
       "       verification_date completion_date_type completion_date  \\\n",
       "0             2018-10-31               Actual      2008-01-29   \n",
       "1             2022-05-31               Actual      2019-07-12   \n",
       "2             2019-07-31          Anticipated      2020-12-28   \n",
       "3             2020-01-31               Actual      2020-01-20   \n",
       "4             2020-06-30               Actual      2020-06-30   \n",
       "...                  ...                  ...             ...   \n",
       "491081        2018-12-31          Anticipated      2019-12-31   \n",
       "491082        2018-07-31          Anticipated      2019-12-31   \n",
       "491083        2018-07-31               Actual      2013-12-31   \n",
       "491084        2021-01-31               Actual      2021-01-06   \n",
       "491085        2022-08-31               Actual      2022-07-25   \n",
       "\n",
       "       primary_completion_date_type primary_completion_date  \\\n",
       "0                            Actual              2008-01-29   \n",
       "1                            Actual              2019-07-12   \n",
       "2                            Actual              2019-07-01   \n",
       "3                            Actual              2019-11-15   \n",
       "4                            Actual              2020-06-30   \n",
       "...                             ...                     ...   \n",
       "491081                  Anticipated              2019-11-01   \n",
       "491082                  Anticipated              2019-12-31   \n",
       "491083                       Actual              2013-06-30   \n",
       "491084                       Actual              2021-01-06   \n",
       "491085                       Actual              2022-04-24   \n",
       "\n",
       "                        updated_at  \n",
       "0       2024-04-13 15:18:37.689771  \n",
       "1       2024-04-13 15:18:38.732734  \n",
       "2       2024-04-13 15:18:39.514159  \n",
       "3       2024-04-13 15:18:40.664541  \n",
       "4       2024-04-13 15:18:41.432351  \n",
       "...                            ...  \n",
       "491081   2024-04-13 15:18:27.19171  \n",
       "491082  2024-04-13 15:18:27.880336  \n",
       "491083   2024-04-13 15:18:28.57842  \n",
       "491084  2024-04-13 15:18:29.292283  \n",
       "491085  2024-04-13 15:18:30.070101  \n",
       "\n",
       "[491086 rows x 26 columns]"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "ename": "NameError",
     "evalue": "name 'calculated_values' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[49], line 11\u001b[0m\n\u001b[1;32m      8\u001b[0m studies_dates \u001b[38;5;241m=\u001b[39m studies[cols]\n\u001b[1;32m      9\u001b[0m studies_dates\n\u001b[0;32m---> 11\u001b[0m all_dates \u001b[38;5;241m=\u001b[39m pd\u001b[38;5;241m.\u001b[39mmerge(studies_dates, calculated_values[[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnct_id\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mactual_duration\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mwere_results_reported\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmonths_to_report_results\u001b[39m\u001b[38;5;124m\"\u001b[39m]], how\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mleft\u001b[39m\u001b[38;5;124m'\u001b[39m, left_on\u001b[38;5;241m=\u001b[39m[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mnct_id\u001b[39m\u001b[38;5;124m'\u001b[39m], right_on \u001b[38;5;241m=\u001b[39m [\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mnct_id\u001b[39m\u001b[38;5;124m'\u001b[39m])\n\u001b[1;32m     12\u001b[0m all_dates\n\u001b[1;32m     13\u001b[0m \u001b[38;5;66;03m# all_dates.columns\u001b[39;00m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'calculated_values' is not defined"
     ]
    }
   ],
   "source": [
    "# attach relevant dates\n",
    "\n",
    "# list(studies.columns)\n",
    "cols = []\n",
    "cols.append(\"nct_id\")\n",
    "date_cols = [col for col in studies.columns if 'date' in col]\n",
    "cols.extend(date_cols)\n",
    "studies_dates = studies[cols]\n",
    "studies_dates\n",
    "\n",
    "all_dates = pd.merge(studies_dates, calculated_values[[\"nct_id\", \"actual_duration\", \"were_results_reported\", \"months_to_report_results\"]], how='left', left_on=['nct_id'], right_on = ['nct_id'])\n",
    "all_dates\n",
    "# all_dates.columns\n",
    "all_dates.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f96e8db1-a729-49cc-8f4f-bd23d339aa2f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "67ffdbc4-da85-4120-ba78-4eae5aafb869",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "61e85e60-27ce-4717-b6e3-65b29a5d069d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1bdb76d1-4d2e-48bd-99a1-0d9fdbcee97a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8c673e63-7712-455b-a628-612f508236c5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "69062483-4570-4e7e-a319-394cd7197ab0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "86d9b0ca-c3b3-43f5-bcad-e9cfbc63981d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "55d93929-b76e-4a31-9f25-b6c93c6fda1b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e62dcd7c-8978-4131-ad90-cbd51ec62c07",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "403eb673-321b-4861-86bc-55bfbfe0df72",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4eab196d-0b1f-4f1f-8edd-6328acd54516",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "271ba226-1e4d-4166-a3cd-c69e6588e5b5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "506f06a0-0902-4cd1-8416-8fd8c1238e5a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6a00041e-37c4-42ba-aa72-e9fca114e9dd",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e3dce317-6cc3-44e1-a971-7f2d62b29ca6",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "57e1cabb-1345-448b-8b70-a620f210ff13",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2a7e5ae3-af43-4f2c-b995-7a103d20267e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f783666f-ccae-4b8d-a194-31f8f19155a5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e056846f-84cf-4c8e-9d95-a1e18f973524",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b080b5f9-53d7-4d27-96fd-e90f73f1ff73",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e5bab417-9b4a-4259-9131-f8b1dfc3653b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "520a994d-efb9-4b2d-9b8e-128a852db654",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fa38d769-ca83-494f-bcab-c6672f7b9738",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ba2ee36d-1be0-4b06-876e-23d6474e7f53",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "df2e04f6-340d-4aac-86f8-88ef4f2ce337",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8957e9d5-9607-43d7-bea4-fd1367ef529a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1ceeed71-1ee3-44bf-b378-9205df209f13",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "20c8709a-8e74-45ba-8f26-bcf7178a4557",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "889cad35-b645-4ab5-ae02-71739dabb52a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "34b762b7-738b-4c86-b722-c2542b72251e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9019137e-10b3-4520-9cd5-5dc6a3616fa2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ec0d1068-b611-456c-828b-93cc359db1fd",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "43613388-d044-4052-b11a-545522ac0b10",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "be6c781b-48ec-412a-8118-958af288e6d4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "76c555e8-c290-4599-83a1-e915c5b89e70",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ae5f0cd-416d-43e7-a454-90f540323971",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "341bd81e-cf45-4798-a3f7-ab3081e5073f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db5ef8c5-bcaf-42fd-aa13-f086b2b095fe",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "00a2a502-9b61-4cda-b961-bc45dd1047a1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe8e008e-f455-4cf4-84f2-d93da844b235",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "682ca2f9-30eb-4562-8296-4171ff6c01ce",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10a25bdf-49ed-4815-9886-468ce6f918cc",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b560beeb-d5a2-43ec-9220-a36edf9303f4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "47a72ba4-f34f-43fa-a251-cc3cef73b434",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8ddb45bd-7451-4149-b206-74a548ee05d8",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5b033afd-e102-464e-b8ec-1faab36b5438",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c233b342-8a1b-4dea-8008-a6bb421b8525",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee65be23-641a-4e10-bfe4-33de27dc6b76",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2283d719-f1fa-4cbe-ad70-3841ec67a4d3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aeea18cb-79a8-4bbb-9cac-27d09ca6fe5f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0c84ec2a-0024-4a39-9b04-1b96069f5dae",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9dc28dbc-7c61-48bb-9ef2-f0d438fe9457",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "38fdbe1b-a96a-4c13-8a7c-6a88196509fa",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7a8695f2-bf08-4020-aca9-4076953cefc5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "366557d1-1b9b-4cb2-8c45-52445fa3bc97",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a489323-6900-42e3-9054-5e426a6d6ad6",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bb40ea93-2856-4b62-9312-dbe99a1a1071",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eea069c6-f70b-4e31-94ad-f0e62ff54678",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1136eb79-cb77-4e68-819f-2e3eb5e34508",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7bdebe0e-0ffa-47f6-8f18-99652b99f9f9",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}