3_quality_control/expression_QC_part2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Expression Quality Control (Part 2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is a template notebook for performing the final quality control on your organism's expression data. This requires a curated metadata sheet."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import itertools\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "\n",
    "from os import path\n",
    "from scipy import stats\n",
    "from tqdm.notebook import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.set_style('ticks')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "logTPM_file = path.join('..','data','raw_data','log_tpm.csv') # Enter log-TPM filename here\n",
    "all_metadata_file = path.join('..','data','interim','metadata_qc_part1_all.tsv') # Enter full metadata filename here\n",
    "metadata_file = path.join('..','data','interim','metadata_qc_part1_curated.tsv') # Enter curated metadata filename here"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load expression data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of genes: 4325\n",
      "Number of samples: 708\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DRX131730</th>\n",
       "      <th>DRX131731</th>\n",
       "      <th>ERX1295610</th>\n",
       "      <th>ERX1295611</th>\n",
       "      <th>ERX1295612</th>\n",
       "      <th>ERX1295613</th>\n",
       "      <th>ERX1295614</th>\n",
       "      <th>ERX1295615</th>\n",
       "      <th>ERX1295616</th>\n",
       "      <th>ERX1295617</th>\n",
       "      <th>...</th>\n",
       "      <th>SRX959208</th>\n",
       "      <th>SRX959209</th>\n",
       "      <th>SRX959210</th>\n",
       "      <th>SRX959211</th>\n",
       "      <th>SRX959212</th>\n",
       "      <th>SRX959213</th>\n",
       "      <th>SRX959214</th>\n",
       "      <th>SRX959215</th>\n",
       "      <th>SRX959216</th>\n",
       "      <th>SRX959217</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Geneid</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>BSU_00010</th>\n",
       "      <td>7.462530</td>\n",
       "      <td>7.098605</td>\n",
       "      <td>7.916877</td>\n",
       "      <td>8.869394</td>\n",
       "      <td>8.300513</td>\n",
       "      <td>8.855594</td>\n",
       "      <td>7.846917</td>\n",
       "      <td>8.943174</td>\n",
       "      <td>8.740763</td>\n",
       "      <td>9.189900</td>\n",
       "      <td>...</td>\n",
       "      <td>7.838464</td>\n",
       "      <td>7.417794</td>\n",
       "      <td>7.460627</td>\n",
       "      <td>7.154734</td>\n",
       "      <td>7.707032</td>\n",
       "      <td>7.275798</td>\n",
       "      <td>7.606982</td>\n",
       "      <td>7.544739</td>\n",
       "      <td>7.432504</td>\n",
       "      <td>7.536969</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BSU_00020</th>\n",
       "      <td>8.016003</td>\n",
       "      <td>7.392746</td>\n",
       "      <td>7.902305</td>\n",
       "      <td>8.416779</td>\n",
       "      <td>8.584681</td>\n",
       "      <td>8.537186</td>\n",
       "      <td>7.928895</td>\n",
       "      <td>8.562883</td>\n",
       "      <td>9.070726</td>\n",
       "      <td>9.386362</td>\n",
       "      <td>...</td>\n",
       "      <td>6.734199</td>\n",
       "      <td>6.899037</td>\n",
       "      <td>6.997465</td>\n",
       "      <td>6.675458</td>\n",
       "      <td>7.878843</td>\n",
       "      <td>7.673993</td>\n",
       "      <td>7.843933</td>\n",
       "      <td>7.641269</td>\n",
       "      <td>7.612722</td>\n",
       "      <td>7.851214</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BSU_00030</th>\n",
       "      <td>6.870758</td>\n",
       "      <td>7.033185</td>\n",
       "      <td>7.450087</td>\n",
       "      <td>7.973767</td>\n",
       "      <td>7.841053</td>\n",
       "      <td>7.731775</td>\n",
       "      <td>7.100132</td>\n",
       "      <td>7.902341</td>\n",
       "      <td>7.949342</td>\n",
       "      <td>8.503556</td>\n",
       "      <td>...</td>\n",
       "      <td>7.828215</td>\n",
       "      <td>6.908195</td>\n",
       "      <td>6.744848</td>\n",
       "      <td>6.685771</td>\n",
       "      <td>7.837414</td>\n",
       "      <td>8.261272</td>\n",
       "      <td>7.925421</td>\n",
       "      <td>7.768439</td>\n",
       "      <td>7.477140</td>\n",
       "      <td>7.614617</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BSU_00040</th>\n",
       "      <td>8.353873</td>\n",
       "      <td>8.225154</td>\n",
       "      <td>8.396466</td>\n",
       "      <td>8.879935</td>\n",
       "      <td>8.629102</td>\n",
       "      <td>8.916810</td>\n",
       "      <td>8.346213</td>\n",
       "      <td>8.995864</td>\n",
       "      <td>9.148182</td>\n",
       "      <td>9.310753</td>\n",
       "      <td>...</td>\n",
       "      <td>8.459266</td>\n",
       "      <td>8.174386</td>\n",
       "      <td>8.015089</td>\n",
       "      <td>7.770594</td>\n",
       "      <td>8.530265</td>\n",
       "      <td>8.515994</td>\n",
       "      <td>8.491431</td>\n",
       "      <td>8.368535</td>\n",
       "      <td>8.188117</td>\n",
       "      <td>8.354613</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BSU_00050</th>\n",
       "      <td>8.049120</td>\n",
       "      <td>7.748395</td>\n",
       "      <td>8.024609</td>\n",
       "      <td>8.443884</td>\n",
       "      <td>8.112961</td>\n",
       "      <td>8.355763</td>\n",
       "      <td>7.868489</td>\n",
       "      <td>8.430197</td>\n",
       "      <td>7.862666</td>\n",
       "      <td>8.770069</td>\n",
       "      <td>...</td>\n",
       "      <td>7.391560</td>\n",
       "      <td>6.870268</td>\n",
       "      <td>6.912665</td>\n",
       "      <td>6.928184</td>\n",
       "      <td>7.539349</td>\n",
       "      <td>7.699223</td>\n",
       "      <td>7.676687</td>\n",
       "      <td>7.218108</td>\n",
       "      <td>7.316390</td>\n",
       "      <td>7.342894</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 708 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           DRX131730  DRX131731  ERX1295610  ERX1295611  ERX1295612  \\\n",
       "Geneid                                                                \n",
       "BSU_00010   7.462530   7.098605    7.916877    8.869394    8.300513   \n",
       "BSU_00020   8.016003   7.392746    7.902305    8.416779    8.584681   \n",
       "BSU_00030   6.870758   7.033185    7.450087    7.973767    7.841053   \n",
       "BSU_00040   8.353873   8.225154    8.396466    8.879935    8.629102   \n",
       "BSU_00050   8.049120   7.748395    8.024609    8.443884    8.112961   \n",
       "\n",
       "           ERX1295613  ERX1295614  ERX1295615  ERX1295616  ERX1295617  ...  \\\n",
       "Geneid                                                                 ...   \n",
       "BSU_00010    8.855594    7.846917    8.943174    8.740763    9.189900  ...   \n",
       "BSU_00020    8.537186    7.928895    8.562883    9.070726    9.386362  ...   \n",
       "BSU_00030    7.731775    7.100132    7.902341    7.949342    8.503556  ...   \n",
       "BSU_00040    8.916810    8.346213    8.995864    9.148182    9.310753  ...   \n",
       "BSU_00050    8.355763    7.868489    8.430197    7.862666    8.770069  ...   \n",
       "\n",
       "           SRX959208  SRX959209  SRX959210  SRX959211  SRX959212  SRX959213  \\\n",
       "Geneid                                                                        \n",
       "BSU_00010   7.838464   7.417794   7.460627   7.154734   7.707032   7.275798   \n",
       "BSU_00020   6.734199   6.899037   6.997465   6.675458   7.878843   7.673993   \n",
       "BSU_00030   7.828215   6.908195   6.744848   6.685771   7.837414   8.261272   \n",
       "BSU_00040   8.459266   8.174386   8.015089   7.770594   8.530265   8.515994   \n",
       "BSU_00050   7.391560   6.870268   6.912665   6.928184   7.539349   7.699223   \n",
       "\n",
       "           SRX959214  SRX959215  SRX959216  SRX959217  \n",
       "Geneid                                                 \n",
       "BSU_00010   7.606982   7.544739   7.432504   7.536969  \n",
       "BSU_00020   7.843933   7.641269   7.612722   7.851214  \n",
       "BSU_00030   7.925421   7.768439   7.477140   7.614617  \n",
       "BSU_00040   8.491431   8.368535   8.188117   8.354613  \n",
       "BSU_00050   7.676687   7.218108   7.316390   7.342894  \n",
       "\n",
       "[5 rows x 708 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DF_log_tpm = pd.read_csv(logTPM_file,index_col=0).fillna(0)\n",
    "print('Number of genes:',DF_log_tpm.shape[0])\n",
    "print('Number of samples:',DF_log_tpm.shape[1])\n",
    "DF_log_tpm.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of samples with curated metadata: 387\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ReleaseDate</th>\n",
       "      <th>TaxID</th>\n",
       "      <th>ScientificName</th>\n",
       "      <th>species_tax_id</th>\n",
       "      <th>class_tax_id</th>\n",
       "      <th>strain_tax_id</th>\n",
       "      <th>species</th>\n",
       "      <th>LibraryLayout</th>\n",
       "      <th>Platform</th>\n",
       "      <th>Model</th>\n",
       "      <th>...</th>\n",
       "      <th>strain_description</th>\n",
       "      <th>base_media</th>\n",
       "      <th>carbon_source</th>\n",
       "      <th>treatment</th>\n",
       "      <th>temperature</th>\n",
       "      <th>pH</th>\n",
       "      <th>OD</th>\n",
       "      <th>growth_phase</th>\n",
       "      <th>notes</th>\n",
       "      <th>skip</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>DRX131730</th>\n",
       "      <td>2018-09-25 19:19:17</td>\n",
       "      <td>224308</td>\n",
       "      <td>Bacillus subtilis subsp. subtilis str. 168</td>\n",
       "      <td>1423</td>\n",
       "      <td>91061</td>\n",
       "      <td>224308</td>\n",
       "      <td>Bacillus subtilis</td>\n",
       "      <td>PAIRED</td>\n",
       "      <td>ILLUMINA</td>\n",
       "      <td>Illumina HiSeq 4000</td>\n",
       "      <td>...</td>\n",
       "      <td>168</td>\n",
       "      <td>DSM</td>\n",
       "      <td>glucose(2%)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>mid-exponential</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DRX131731</th>\n",
       "      <td>2018-09-25 19:19:17</td>\n",
       "      <td>224308</td>\n",
       "      <td>Bacillus subtilis subsp. subtilis str. 168</td>\n",
       "      <td>1423</td>\n",
       "      <td>91061</td>\n",
       "      <td>224308</td>\n",
       "      <td>Bacillus subtilis</td>\n",
       "      <td>PAIRED</td>\n",
       "      <td>ILLUMINA</td>\n",
       "      <td>Illumina HiSeq 4000</td>\n",
       "      <td>...</td>\n",
       "      <td>168 del_ylxR</td>\n",
       "      <td>DSM</td>\n",
       "      <td>glucose(2%)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>mid-exponential</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ERX1295610</th>\n",
       "      <td>2016-10-05 10:52:19</td>\n",
       "      <td>224308</td>\n",
       "      <td>Bacillus subtilis subsp. subtilis str. 168</td>\n",
       "      <td>1423</td>\n",
       "      <td>91061</td>\n",
       "      <td>224308</td>\n",
       "      <td>Bacillus subtilis</td>\n",
       "      <td>PAIRED</td>\n",
       "      <td>ILLUMINA</td>\n",
       "      <td>Illumina HiSeq 2500</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>No metadata</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ERX1295611</th>\n",
       "      <td>2016-10-05 10:52:19</td>\n",
       "      <td>224308</td>\n",
       "      <td>Bacillus subtilis subsp. subtilis str. 168</td>\n",
       "      <td>1423</td>\n",
       "      <td>91061</td>\n",
       "      <td>224308</td>\n",
       "      <td>Bacillus subtilis</td>\n",
       "      <td>PAIRED</td>\n",
       "      <td>ILLUMINA</td>\n",
       "      <td>Illumina HiSeq 2500</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>No metadata</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ERX1295612</th>\n",
       "      <td>2016-10-05 10:52:19</td>\n",
       "      <td>224308</td>\n",
       "      <td>Bacillus subtilis subsp. subtilis str. 168</td>\n",
       "      <td>1423</td>\n",
       "      <td>91061</td>\n",
       "      <td>224308</td>\n",
       "      <td>Bacillus subtilis</td>\n",
       "      <td>PAIRED</td>\n",
       "      <td>ILLUMINA</td>\n",
       "      <td>Illumina HiSeq 2500</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>No metadata</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 49 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                    ReleaseDate   TaxID  \\\n",
       "DRX131730   2018-09-25 19:19:17  224308   \n",
       "DRX131731   2018-09-25 19:19:17  224308   \n",
       "ERX1295610  2016-10-05 10:52:19  224308   \n",
       "ERX1295611  2016-10-05 10:52:19  224308   \n",
       "ERX1295612  2016-10-05 10:52:19  224308   \n",
       "\n",
       "                                        ScientificName  species_tax_id  \\\n",
       "DRX131730   Bacillus subtilis subsp. subtilis str. 168            1423   \n",
       "DRX131731   Bacillus subtilis subsp. subtilis str. 168            1423   \n",
       "ERX1295610  Bacillus subtilis subsp. subtilis str. 168            1423   \n",
       "ERX1295611  Bacillus subtilis subsp. subtilis str. 168            1423   \n",
       "ERX1295612  Bacillus subtilis subsp. subtilis str. 168            1423   \n",
       "\n",
       "            class_tax_id  strain_tax_id            species LibraryLayout  \\\n",
       "DRX131730          91061         224308  Bacillus subtilis        PAIRED   \n",
       "DRX131731          91061         224308  Bacillus subtilis        PAIRED   \n",
       "ERX1295610         91061         224308  Bacillus subtilis        PAIRED   \n",
       "ERX1295611         91061         224308  Bacillus subtilis        PAIRED   \n",
       "ERX1295612         91061         224308  Bacillus subtilis        PAIRED   \n",
       "\n",
       "            Platform                Model  ... strain_description base_media  \\\n",
       "DRX131730   ILLUMINA  Illumina HiSeq 4000  ...                168        DSM   \n",
       "DRX131731   ILLUMINA  Illumina HiSeq 4000  ...       168 del_ylxR        DSM   \n",
       "ERX1295610  ILLUMINA  Illumina HiSeq 2500  ...                NaN        NaN   \n",
       "ERX1295611  ILLUMINA  Illumina HiSeq 2500  ...                NaN        NaN   \n",
       "ERX1295612  ILLUMINA  Illumina HiSeq 2500  ...                NaN        NaN   \n",
       "\n",
       "           carbon_source  treatment temperature   pH  OD     growth_phase  \\\n",
       "DRX131730    glucose(2%)        NaN         NaN  7.0 NaN  mid-exponential   \n",
       "DRX131731    glucose(2%)        NaN         NaN  7.0 NaN  mid-exponential   \n",
       "ERX1295610           NaN        NaN         NaN  NaN NaN              NaN   \n",
       "ERX1295611           NaN        NaN         NaN  NaN NaN              NaN   \n",
       "ERX1295612           NaN        NaN         NaN  NaN NaN              NaN   \n",
       "\n",
       "                  notes  skip  \n",
       "DRX131730           NaN   NaN  \n",
       "DRX131731           NaN   NaN  \n",
       "ERX1295610  No metadata  True  \n",
       "ERX1295611  No metadata  True  \n",
       "ERX1295612  No metadata  True  \n",
       "\n",
       "[5 rows x 49 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DF_metadata = pd.read_csv(metadata_file,index_col=0,sep='\\t')\n",
    "print('Number of samples with curated metadata:',DF_metadata.shape[0])\n",
    "DF_metadata.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "DF_metadata_all = pd.read_csv(all_metadata_file,index_col=0,sep='\\t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Remove samples due to poor metadata"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After curation, some samples either did not have enough replicates or metadata to warrant inclusion in this database."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "New number of samples with curated metadata: 292\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ReleaseDate</th>\n",
       "      <th>TaxID</th>\n",
       "      <th>ScientificName</th>\n",
       "      <th>species_tax_id</th>\n",
       "      <th>class_tax_id</th>\n",
       "      <th>strain_tax_id</th>\n",
       "      <th>species</th>\n",
       "      <th>LibraryLayout</th>\n",
       "      <th>Platform</th>\n",
       "      <th>Model</th>\n",
       "      <th>...</th>\n",
       "      <th>strain_description</th>\n",
       "      <th>base_media</th>\n",
       "      <th>carbon_source</th>\n",
       "      <th>treatment</th>\n",
       "      <th>temperature</th>\n",
       "      <th>pH</th>\n",
       "      <th>OD</th>\n",
       "      <th>growth_phase</th>\n",
       "      <th>notes</th>\n",
       "      <th>skip</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>DRX131730</th>\n",
       "      <td>2018-09-25 19:19:17</td>\n",
       "      <td>224308</td>\n",
       "      <td>Bacillus subtilis subsp. subtilis str. 168</td>\n",
       "      <td>1423</td>\n",
       "      <td>91061</td>\n",
       "      <td>224308</td>\n",
       "      <td>Bacillus subtilis</td>\n",
       "      <td>PAIRED</td>\n",
       "      <td>ILLUMINA</td>\n",
       "      <td>Illumina HiSeq 4000</td>\n",
       "      <td>...</td>\n",
       "      <td>168</td>\n",
       "      <td>DSM</td>\n",
       "      <td>glucose(2%)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>mid-exponential</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DRX131731</th>\n",
       "      <td>2018-09-25 19:19:17</td>\n",
       "      <td>224308</td>\n",
       "      <td>Bacillus subtilis subsp. subtilis str. 168</td>\n",
       "      <td>1423</td>\n",
       "      <td>91061</td>\n",
       "      <td>224308</td>\n",
       "      <td>Bacillus subtilis</td>\n",
       "      <td>PAIRED</td>\n",
       "      <td>ILLUMINA</td>\n",
       "      <td>Illumina HiSeq 4000</td>\n",
       "      <td>...</td>\n",
       "      <td>168 del_ylxR</td>\n",
       "      <td>DSM</td>\n",
       "      <td>glucose(2%)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>mid-exponential</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ERX1819430</th>\n",
       "      <td>2017-01-18 14:21:19</td>\n",
       "      <td>1204342</td>\n",
       "      <td>Bacillus subtilis BEST7003</td>\n",
       "      <td>1423</td>\n",
       "      <td>91061</td>\n",
       "      <td>1204342</td>\n",
       "      <td>Bacillus subtilis</td>\n",
       "      <td>SINGLE</td>\n",
       "      <td>ILLUMINA</td>\n",
       "      <td>NextSeq 500</td>\n",
       "      <td>...</td>\n",
       "      <td>BEST7003 with spBeta</td>\n",
       "      <td>LB</td>\n",
       "      <td>NaN</td>\n",
       "      <td>MnCl2(.1mM);MgCl2(5mM)</td>\n",
       "      <td>37.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ERX1819431</th>\n",
       "      <td>2017-01-18 14:21:19</td>\n",
       "      <td>1204342</td>\n",
       "      <td>Bacillus subtilis BEST7003</td>\n",
       "      <td>1423</td>\n",
       "      <td>91061</td>\n",
       "      <td>1204342</td>\n",
       "      <td>Bacillus subtilis</td>\n",
       "      <td>SINGLE</td>\n",
       "      <td>ILLUMINA</td>\n",
       "      <td>NextSeq 500</td>\n",
       "      <td>...</td>\n",
       "      <td>BEST7003 with spBeta</td>\n",
       "      <td>LB</td>\n",
       "      <td>NaN</td>\n",
       "      <td>MnCl2(.1mM);MgCl2(5mM);SAIRGA(1uM)</td>\n",
       "      <td>37.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ERX1819432</th>\n",
       "      <td>2017-01-18 14:21:19</td>\n",
       "      <td>1204342</td>\n",
       "      <td>Bacillus subtilis BEST7003</td>\n",
       "      <td>1423</td>\n",
       "      <td>91061</td>\n",
       "      <td>1204342</td>\n",
       "      <td>Bacillus subtilis</td>\n",
       "      <td>SINGLE</td>\n",
       "      <td>ILLUMINA</td>\n",
       "      <td>NextSeq 500</td>\n",
       "      <td>...</td>\n",
       "      <td>BEST7003 with spBeta</td>\n",
       "      <td>LB</td>\n",
       "      <td>NaN</td>\n",
       "      <td>MnCl2(.1mM);MgCl2(5mM)</td>\n",
       "      <td>37.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 49 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                    ReleaseDate    TaxID  \\\n",
       "DRX131730   2018-09-25 19:19:17   224308   \n",
       "DRX131731   2018-09-25 19:19:17   224308   \n",
       "ERX1819430  2017-01-18 14:21:19  1204342   \n",
       "ERX1819431  2017-01-18 14:21:19  1204342   \n",
       "ERX1819432  2017-01-18 14:21:19  1204342   \n",
       "\n",
       "                                        ScientificName  species_tax_id  \\\n",
       "DRX131730   Bacillus subtilis subsp. subtilis str. 168            1423   \n",
       "DRX131731   Bacillus subtilis subsp. subtilis str. 168            1423   \n",
       "ERX1819430                  Bacillus subtilis BEST7003            1423   \n",
       "ERX1819431                  Bacillus subtilis BEST7003            1423   \n",
       "ERX1819432                  Bacillus subtilis BEST7003            1423   \n",
       "\n",
       "            class_tax_id  strain_tax_id            species LibraryLayout  \\\n",
       "DRX131730          91061         224308  Bacillus subtilis        PAIRED   \n",
       "DRX131731          91061         224308  Bacillus subtilis        PAIRED   \n",
       "ERX1819430         91061        1204342  Bacillus subtilis        SINGLE   \n",
       "ERX1819431         91061        1204342  Bacillus subtilis        SINGLE   \n",
       "ERX1819432         91061        1204342  Bacillus subtilis        SINGLE   \n",
       "\n",
       "            Platform                Model  ...    strain_description  \\\n",
       "DRX131730   ILLUMINA  Illumina HiSeq 4000  ...                   168   \n",
       "DRX131731   ILLUMINA  Illumina HiSeq 4000  ...          168 del_ylxR   \n",
       "ERX1819430  ILLUMINA          NextSeq 500  ...  BEST7003 with spBeta   \n",
       "ERX1819431  ILLUMINA          NextSeq 500  ...  BEST7003 with spBeta   \n",
       "ERX1819432  ILLUMINA          NextSeq 500  ...  BEST7003 with spBeta   \n",
       "\n",
       "           base_media carbon_source                           treatment  \\\n",
       "DRX131730         DSM   glucose(2%)                                 NaN   \n",
       "DRX131731         DSM   glucose(2%)                                 NaN   \n",
       "ERX1819430         LB           NaN              MnCl2(.1mM);MgCl2(5mM)   \n",
       "ERX1819431         LB           NaN  MnCl2(.1mM);MgCl2(5mM);SAIRGA(1uM)   \n",
       "ERX1819432         LB           NaN              MnCl2(.1mM);MgCl2(5mM)   \n",
       "\n",
       "           temperature   pH  OD     growth_phase notes skip  \n",
       "DRX131730          NaN  7.0 NaN  mid-exponential   NaN  NaN  \n",
       "DRX131731          NaN  7.0 NaN  mid-exponential   NaN  NaN  \n",
       "ERX1819430        37.0  NaN NaN              NaN   NaN  NaN  \n",
       "ERX1819431        37.0  NaN NaN              NaN   NaN  NaN  \n",
       "ERX1819432        37.0  NaN NaN              NaN   NaN  NaN  \n",
       "\n",
       "[5 rows x 49 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DF_metadata_passed_step4 = DF_metadata[~DF_metadata.skip.fillna(False)].copy()\n",
    "print('New number of samples with curated metadata:',DF_metadata_passed_step4.shape[0])\n",
    "DF_metadata_passed_step4.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Check curation\n",
    "Since manual curation is error-prone, we want to make sure that all samples have labels for their project and condition. In addition, there should only be one reference condition in each project, and it should be in the project itself.\n",
    "\n",
    "Any samples that fail these checks will be printed below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert(DF_metadata_passed_step4.project.notnull().all())\n",
    "assert(DF_metadata_passed_step4.condition.notnull().all())\n",
    "\n",
    "for name,group in DF_metadata_passed_step4.groupby('project'):\n",
    "    ref_cond = group.reference_condition.unique()\n",
    "    \n",
    "    # Ensure that there is only one reference condition per project\n",
    "    if not len(ref_cond) == 1:\n",
    "        print('Multiple reference conditions for:, name')\n",
    "    \n",
    "    # Ensure the reference condition is in fact in the project\n",
    "    ref_cond = ref_cond[0]\n",
    "    if not ref_cond in group.condition.tolist():\n",
    "        print('Reference condition not in project:', name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Next, make a new column called ``full_name`` that gives every experimental condition a unique, human-readable identifier."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "DF_metadata_passed_step4['full_name'] = DF_metadata_passed_step4['project'].str.cat(DF_metadata_passed_step4['condition'],sep=':')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Remove samples with only one replicate"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First, find sample names that have at least two replicates."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['nusA:no_nusA', 'nusA:nusA_induced', 'biofilm_time:8h',\n",
      "       'biofilm_time:12h', 'biofilm_time:16h'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "counts = DF_metadata_passed_step4.full_name.value_counts()\n",
    "keep_samples = counts[counts >= 2].index\n",
    "print(keep_samples[:5])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Only keep these samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "New number of samples with curated metadata: 273\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ReleaseDate</th>\n",
       "      <th>TaxID</th>\n",
       "      <th>ScientificName</th>\n",
       "      <th>species_tax_id</th>\n",
       "      <th>class_tax_id</th>\n",
       "      <th>strain_tax_id</th>\n",
       "      <th>species</th>\n",
       "      <th>LibraryLayout</th>\n",
       "      <th>Platform</th>\n",
       "      <th>Model</th>\n",
       "      <th>...</th>\n",
       "      <th>base_media</th>\n",
       "      <th>carbon_source</th>\n",
       "      <th>treatment</th>\n",
       "      <th>temperature</th>\n",
       "      <th>pH</th>\n",
       "      <th>OD</th>\n",
       "      <th>growth_phase</th>\n",
       "      <th>notes</th>\n",
       "      <th>skip</th>\n",
       "      <th>full_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ERX1819430</th>\n",
       "      <td>2017-01-18 14:21:19</td>\n",
       "      <td>1204342</td>\n",
       "      <td>Bacillus subtilis BEST7003</td>\n",
       "      <td>1423</td>\n",
       "      <td>91061</td>\n",
       "      <td>1204342</td>\n",
       "      <td>Bacillus subtilis</td>\n",
       "      <td>SINGLE</td>\n",
       "      <td>ILLUMINA</td>\n",
       "      <td>NextSeq 500</td>\n",
       "      <td>...</td>\n",
       "      <td>LB</td>\n",
       "      <td>NaN</td>\n",
       "      <td>MnCl2(.1mM);MgCl2(5mM)</td>\n",
       "      <td>37.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>spbeta:spbeta_nopep_20m</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ERX1819431</th>\n",
       "      <td>2017-01-18 14:21:19</td>\n",
       "      <td>1204342</td>\n",
       "      <td>Bacillus subtilis BEST7003</td>\n",
       "      <td>1423</td>\n",
       "      <td>91061</td>\n",
       "      <td>1204342</td>\n",
       "      <td>Bacillus subtilis</td>\n",
       "      <td>SINGLE</td>\n",
       "      <td>ILLUMINA</td>\n",
       "      <td>NextSeq 500</td>\n",
       "      <td>...</td>\n",
       "      <td>LB</td>\n",
       "      <td>NaN</td>\n",
       "      <td>MnCl2(.1mM);MgCl2(5mM);SAIRGA(1uM)</td>\n",
       "      <td>37.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>spbeta:spbeta_pep_20m</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ERX1819432</th>\n",
       "      <td>2017-01-18 14:21:19</td>\n",
       "      <td>1204342</td>\n",
       "      <td>Bacillus subtilis BEST7003</td>\n",
       "      <td>1423</td>\n",
       "      <td>91061</td>\n",
       "      <td>1204342</td>\n",
       "      <td>Bacillus subtilis</td>\n",
       "      <td>SINGLE</td>\n",
       "      <td>ILLUMINA</td>\n",
       "      <td>NextSeq 500</td>\n",
       "      <td>...</td>\n",
       "      <td>LB</td>\n",
       "      <td>NaN</td>\n",
       "      <td>MnCl2(.1mM);MgCl2(5mM)</td>\n",
       "      <td>37.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>spbeta:spbeta_nopep_20m</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ERX1819433</th>\n",
       "      <td>2017-01-18 14:21:19</td>\n",
       "      <td>1204342</td>\n",
       "      <td>Bacillus subtilis BEST7003</td>\n",
       "      <td>1423</td>\n",
       "      <td>91061</td>\n",
       "      <td>1204342</td>\n",
       "      <td>Bacillus subtilis</td>\n",
       "      <td>SINGLE</td>\n",
       "      <td>ILLUMINA</td>\n",
       "      <td>NextSeq 500</td>\n",
       "      <td>...</td>\n",
       "      <td>LB</td>\n",
       "      <td>NaN</td>\n",
       "      <td>MnCl2(.1mM);MgCl2(5mM);SAIRGA(1uM)</td>\n",
       "      <td>37.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>spbeta:spbeta_pep_20m</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ERX1819434</th>\n",
       "      <td>2017-01-18 14:21:19</td>\n",
       "      <td>1204342</td>\n",
       "      <td>Bacillus subtilis BEST7003</td>\n",
       "      <td>1423</td>\n",
       "      <td>91061</td>\n",
       "      <td>1204342</td>\n",
       "      <td>Bacillus subtilis</td>\n",
       "      <td>SINGLE</td>\n",
       "      <td>ILLUMINA</td>\n",
       "      <td>NextSeq 500</td>\n",
       "      <td>...</td>\n",
       "      <td>LB</td>\n",
       "      <td>NaN</td>\n",
       "      <td>MnCl2(.1mM);MgCl2(5mM);SAIRGA(1uM)</td>\n",
       "      <td>37.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>spbeta:phi3T_pep_10m</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 50 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                    ReleaseDate    TaxID              ScientificName  \\\n",
       "ERX1819430  2017-01-18 14:21:19  1204342  Bacillus subtilis BEST7003   \n",
       "ERX1819431  2017-01-18 14:21:19  1204342  Bacillus subtilis BEST7003   \n",
       "ERX1819432  2017-01-18 14:21:19  1204342  Bacillus subtilis BEST7003   \n",
       "ERX1819433  2017-01-18 14:21:19  1204342  Bacillus subtilis BEST7003   \n",
       "ERX1819434  2017-01-18 14:21:19  1204342  Bacillus subtilis BEST7003   \n",
       "\n",
       "            species_tax_id  class_tax_id  strain_tax_id            species  \\\n",
       "ERX1819430            1423         91061        1204342  Bacillus subtilis   \n",
       "ERX1819431            1423         91061        1204342  Bacillus subtilis   \n",
       "ERX1819432            1423         91061        1204342  Bacillus subtilis   \n",
       "ERX1819433            1423         91061        1204342  Bacillus subtilis   \n",
       "ERX1819434            1423         91061        1204342  Bacillus subtilis   \n",
       "\n",
       "           LibraryLayout  Platform        Model  ... base_media carbon_source  \\\n",
       "ERX1819430        SINGLE  ILLUMINA  NextSeq 500  ...         LB           NaN   \n",
       "ERX1819431        SINGLE  ILLUMINA  NextSeq 500  ...         LB           NaN   \n",
       "ERX1819432        SINGLE  ILLUMINA  NextSeq 500  ...         LB           NaN   \n",
       "ERX1819433        SINGLE  ILLUMINA  NextSeq 500  ...         LB           NaN   \n",
       "ERX1819434        SINGLE  ILLUMINA  NextSeq 500  ...         LB           NaN   \n",
       "\n",
       "                                     treatment  temperature  pH  OD  \\\n",
       "ERX1819430              MnCl2(.1mM);MgCl2(5mM)         37.0 NaN NaN   \n",
       "ERX1819431  MnCl2(.1mM);MgCl2(5mM);SAIRGA(1uM)         37.0 NaN NaN   \n",
       "ERX1819432              MnCl2(.1mM);MgCl2(5mM)         37.0 NaN NaN   \n",
       "ERX1819433  MnCl2(.1mM);MgCl2(5mM);SAIRGA(1uM)         37.0 NaN NaN   \n",
       "ERX1819434  MnCl2(.1mM);MgCl2(5mM);SAIRGA(1uM)         37.0 NaN NaN   \n",
       "\n",
       "           growth_phase notes skip                full_name  \n",
       "ERX1819430          NaN   NaN  NaN  spbeta:spbeta_nopep_20m  \n",
       "ERX1819431          NaN   NaN  NaN    spbeta:spbeta_pep_20m  \n",
       "ERX1819432          NaN   NaN  NaN  spbeta:spbeta_nopep_20m  \n",
       "ERX1819433          NaN   NaN  NaN    spbeta:spbeta_pep_20m  \n",
       "ERX1819434          NaN   NaN  NaN     spbeta:phi3T_pep_10m  \n",
       "\n",
       "[5 rows x 50 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DF_metadata_passed_step4 = DF_metadata_passed_step4[DF_metadata_passed_step4.full_name.isin(keep_samples)]\n",
    "print('New number of samples with curated metadata:',DF_metadata_passed_step4.shape[0])\n",
    "DF_metadata_passed_step4.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save this information to the full metadata dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "DF_metadata_all['passed_curation'] = DF_metadata_all.index.isin(DF_metadata_passed_step4.index)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check correlations between replicates"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Remove failed data from log_tpm files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "DF_log_tpm = DF_log_tpm[DF_metadata_passed_step4.index]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compute Pearson R Score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Biological replicates should have a Pearson R correlation above 0.95. For samples with more than 2 replicates, the replicates must have R >= 0.95 with at least one other replicate or it will be dropped. The correlation threshold can be changed below:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "rcutoff = 0.95"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following code computes correlations between all samples and collects correlations between replicates and non-replicates."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5cd0fd254c094f299e7ef9c0d2c8b4b9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/37128.0 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "rep_corrs = {}\n",
    "rand_corrs = {}\n",
    "\n",
    "num_comparisons = len(DF_metadata_passed_step4)*(len(DF_metadata_passed_step4)-1)/2\n",
    "\n",
    "for exp1,exp2 in tqdm(itertools.combinations(DF_metadata_passed_step4.index,2),total=num_comparisons):\n",
    "    if DF_metadata_passed_step4.loc[exp1,'full_name'] == DF_metadata_passed_step4.loc[exp2,'full_name']:\n",
    "        rep_corrs[(exp1,exp2)] = stats.pearsonr(DF_log_tpm[exp1],DF_log_tpm[exp2])[0]\n",
    "    else:\n",
    "        rand_corrs[(exp1,exp2)] = stats.pearsonr(DF_log_tpm[exp1],DF_log_tpm[exp2])[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Correlations can be plotted on a histogram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Median Pearson R between replicates: 0.98\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAFSCAYAAAAQBrOYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAABRbUlEQVR4nO3deVxO6f8/8NetRZms0R0mDKOi0oIIFZFoGYmECS00GaOSMWmQdUZZSxk0GIT5ZkmljC1LtlE02RkNqdDdaJFE6/X7o1/n454Wp7rb9H4+HvMY93XOfV3vc1fnfZ/rOue6BIwxBkIIIS1aq8YOgBBCSOOjZEAIIYSSASGEEEoGhBBCQMmAEEIIKBkQQggBJQPSQMLCwqCrq1vneq5fvw41NTVkZWVJIKqqBQYGwtLSsl7bIHUjqd8pUqbZJYPFixdDTU0Nampq0NDQwOjRo+Hn54f8/PzGDq1ehIWFccerpqaGYcOGwdXVFY8fP27s0OqdiYkJdu3aJVamq6uLy5cvo2PHjo0UVc2oqanh5MmTjR3GJ8nc3Bxnz55t7DAAfBqJqdklAwAYNmwYLl++jLNnz8LDwwMHDx6En59fvbVXXFyMxnw2T15eHpcvX8alS5ewY8cO5Ofn45tvvkFhYWGDxlFUVFShrKFjkJWVRZcuXSAQCBq0XdK0FBUVQU5ODoqKio0dyiejWSaD8hNC165dYWVlBSsrK8TExAAAGGP49ddfMWbMGAwYMABWVlaIiIgQe/+GDRtgZmaGAQMGwMTEBOvWrUNBQQG3vbyLICwsDGPGjIGWlhby8/MRHx+PKVOmQFdXFwMHDsTkyZPx999/c+87ffo0rKysoKmpCWNjY2zbtk0siZiYmOCXX36Bj48P9PT0YGRkhJ07d370eAUCAbp06QIlJSVoaWnBwcEBz58/x9OnT6t93z///ANXV1cMHDgQurq6sLOzw6NHjwAApaWl2Lp1K4yNjaGpqQkrKyuxb1lpaWlQU1NDVFQUZs6ciQEDBiA0NBSLFy/GN998g+DgYBgZGcHY2BgAIBKJsGDBAgwePBiDBw+Gi4sLkpOTq4wtJSUFc+fOxfDhw6Gjo4OJEyfi/Pnz3PYZM2bg+fPnWLduHXdVBFTeTVRfnzsAHD58GCNHjsSAAQPw7bffVuieOnr0KMzNzaGlpQUzMzPs2bMHpaWlXLsA4O7uDjU1NZiYmODt27fQ0NBAYmIiV4exsTHGjRvHvb569Sp0dHS4RPvmzRssW7YMBgYG0NXVhb29Pe7cuSMWR0JCAuzt7aGtrQ1DQ0MsX74ceXl5Yp/nihUrsGnTJgwZMgQGBgbw8/PjYq1KdfXGxcVBQ0MD169f5/b/v//7P+jp6SE1NZVr18fHB2vWrOF+N/7bbmFhIdavXw8jIyNoa2tj0qRJuHTpEre9/Gd+8eJFTJ48GZqamrh8+XKFb+Plf7fHjh2DiYkJdHR04O3tjcLCQhw4cADGxsYYMmQI1q5dW6v2r127BltbW2hra8PGxgb37t3jtnt7eyM/P5/7XQ0MDATwv9/NAQMGQF9fH/b29nj16lW1n3mjYc2Ml5cXc3FxEStbvXo109fXZ4wxtmnTJjZ27Fh28eJFlpKSwiIjI5m2tjY7f/48t39QUBC7ceMGS01NZRcuXGDGxsZs8+bN3PYtW7YwbW1t5ujoyO7evcsePXrEioqK2KBBg5ivry979uwZS0pKYpGRkSwpKYkxxtidO3eYuro6CwgIYE+ePGERERFMR0eH7du3j6t31KhRTF9fn4WEhLDk5GS2b98+pqqqyhISEqo83qNHjzIdHR3u9evXr9mCBQuYqqoq13Zl0tPTmb6+PnN1dWW3bt1iT548YeHh4ez+/fuMMcZ+++03pquryyIjI9mTJ0+Yv78/U1dX57anpqYyVVVVNmrUKPbHH3+wlJQU9vLlS+bl5cV0dHSYp6cne/ToEXv48CHLz89nY8eOZV5eXuzBgwcsKSmJ/fjjj2zkyJEsPz+/0uN48OABO3jwIHv48CFLTk5mv/zyC9PQ0OCOKTs7mxkZGbGAgACWkZHBMjIyGGOM/fnnn0xVVZVlZmbW6+e+ZcsWpqOjw+zt7dm9e/fYjRs3mLm5Ofvmm2+4fUJDQ9nw4cO5zycmJoYNGzaMhYSEMMYYy8zMZKqqquzQoUMsIyODi9nW1pbt2LGDMcZYcnIy09HRYVpaWtwxbtq0ic2aNYsxxlhpaSmbOnUqmzNnDrt16xZLTk5mmzdvZrq6ukwkEjHGGHv48CHT0dFhu3btYk+fPmWJiYlsypQpbP78+Vys9vb2TE9Pj/n7+7MnT56w6Oho1q9fP3b8+PEqPwM+9W7cuJEZGRmxnJwclpSUxLS1tVlYWJhYuzo6OmzVqlUsKSmJRUdHMz09PbZ7925uH09PT2Zra8vi4uJYSkoKCwkJYRoaGuzBgwdiP3NLS0t26dIllpKSwjIzMyv8TpX/zObNm8cePXrEYmNjmY6ODnNycmKLFy9mSUlJ7PTp06x///7s5MmTNW5/0qRJ7Nq1aywpKYk5OTmxcePGsdLSUlZQUMD27NnDtLW1ud/VvLw8lpGRwTQ0NNiuXbtYamoqe/ToETt06BD7999/q/zMG1OzTwa3bt1i+vr6zN3dnb19+5ZpaWmx+Ph4sfesWbOGzZ49u8o6Dx48yMaMGcO93rJlC+vfv7/YDy07O5upqqqy69evV1qHp6cnmzFjhljZli1bmKGhIfd61KhRbMGCBWL7mJqasq1bt1YZ29GjR5mqqirT0dFh2traTFVVlamqqjJXV9cq38NY2Qll5MiRrKCgoNLtI0aMYIGBgWJl9vb2bOHChYyx/yWDXbt2ie3j5eXFhgwZIlbv4cOHmampKSstLeXKiouLmb6+PouOjuaO48M/3MrY2tqKfRajRo1iO3fuFNvnv8mgvj73LVu2MHV1dfb8+XOuLD4+nqmqqrKnT58yxhgzNjZmx44dE3vfb7/9xsaPH8+9VlVVZX/88YfYPuvXr2dOTk6MMcYOHTrEnJ2dmb29PXdinjp1Khfb1atXmY6ODnv37p1YHV999RULDg5mjDG2aNEi5u3tLbb9/v37TFVVlb169YoxVvaznTJlitg+Dg4O7Mcff6zyM+BTb2FhIbOxsWHz5s1j1tbWzN3dXWx/e3t7NnbsWLHfja1bt3I/n2fPnjE1NTWxz5kxxubOncuWL1/OGPvfz/zDEzhjFX+ntmzZwrS0tFhubi5XNn/+/Aq/r/b29mzlypU1bj82NpbbfuPGDaaqqspevnxZaSyMMXb37l2mqqrK0tLSWHMg3dhXJrVx6dIl6Orqori4GMXFxRg9ejSWLVuGpKQkFBQUYPbs2WJ9ykVFRejevTv3+uTJk9i7dy9SUlKQn5+PkpKSCpfLQqEQnTt35l536NABNjY2cHZ2hoGBAQwMDGBmZoZu3boBAJ48ecJ1mZQbOHAggoKCkJeXBwUFBQDgujvKKSkpffTOGHl5eYSHh6O4uBg3btzA7t27sWrVqmrfc//+fejp6UFWVrbCtry8PGRkZGDgwIFi5Xp6eoiNjRUr09TUrPD+vn37itV77949pKWlQU9PT2y/d+/ecd0F/5Wfn4+goCBcuHAB//77L4qLi1FQUFDh8/mY+vzchUIh9/MFAG1tbbRq1Qr//PMP2rVrh5cvX2L58uVYuXIltw+f8SV9fX0cOHAARUVFuH79OoYMGYL3798jLi4Oo0ePxp07d7Bw4UIAZZ/tu3fvYGBgIFZHQUEB99neu3cPz549wx9//MFtL48hJSWF61ev7DPIzMysMk4+9crIyGDjxo2wtLREp06dsHfv3gr1aGtri/096urqIiAgAHl5ebh37x4YY7CwsBB7T2FhIYYOHSpWVtnv4n917doVbdu25V4rKiqiV69eYr+vioqK3HHXpP0PPz8lJSUAQGZmJpSVlSuNRV1dHcOGDYOlpSVGjBgBAwMDjBs3Dp06dfrocTSGZpkMBg0ahNWrV0NaWhpKSkqQkZEBUNbPDQDbtm0T+yMGAGnpskNNTEyEp6cn5s2bB0NDQ7Rr1w7nzp2rMADdpk2bCu2uXbsWs2bNQmxsLM6dO4fNmzdj69atMDQ05B17eRzlBALBR/ttBQIBevbsCQDo06cP/v33X3h6eiIkJIR3u3z9d2BWXl6+wj7//WxKS0uhrq6OzZs3V9i3ffv2lbbj5+eHS5cuwcvLCz179oS8vDy8vLwqHaSWhNp87tUpf+/KlStrfBfJwIEDUVhYiDt37iA+Ph4zZ87Eu3fv4OPjg7/++gvS0tIYMGAA107nzp1x4MCBCvWUJ7rS0lLY2trCwcGhwj5CoZD7d2WfQXWJi2+9iYmJKC0txZs3b5CVlYV27dpVe/wfYoxBIBDgyJEjFeKTk5MTe13Z7+J/lZ8LygkEgkrLyn9+NWn/w+3lfyfV/Q5JSUlh9+7dSExMxJUrV3DkyBFs2rQJ+/fvh7q6+kePpaE1y2QgLy/PnRw/1KdPH8jKyuLFixcVvkmVS0hIgFAoxLx587iyFy9e8G5bXV0d6urqcHFxwezZsxEeHg5DQ0P07t0bCQkJYvvevHkTysrK3B+tpDg4OOC3337D6dOnMXbs2Er36d+/PyIjI1FYWFjh6kBBQQFKSkq4efOm2OeUkJCAPn361DgeDQ0NREdHo2PHjrxPBAkJCbC2toaZmRmAsm+6KSkp6NWrF7ePjIwMSkpKqq2nPj93kUiEly9fomvXrgCA27dvo7S0FH369EHnzp2hpKSElJQUWFtbV1mHjIxMhRPGZ599Bg0NDRw+fBh5eXnQ0NBAcXExXr58iePHj0NHR4f7mWloaODVq1do1aoVVFRUKm2jf//+SEpKqvRvoi741JuamorVq1fDx8cHly5dwqJFi/D777+LnThv3brFnXSBsuShpKQEBQUF9OvXD4wx/PvvvxW+iTcESbVf1e+qQCCArq4udHV1MW/ePFhYWODEiRNNMhk0y7uJqqKgoAAnJyesW7cOR44cwbNnz/DgwQP8/vvvCA0NBQD06tULIpEIkZGRSE1NxcGDBxEVFfXRulNTU7FhwwYkJCTg+fPn+PPPP/Ho0SPu5Onk5IT4+HgEBgbi6dOniIyMxO7duzF79ux6OU5bW1sEBgZW+c1k+vTpyM/Ph4eHB27fvo1nz54hKioKDx48AAA4Oztj9+7diIqKwtOnTxEQEIAbN27A2dm5xvFYWVlBUVER3377LeLi4pCamor4+Hj4+vpWeUdRr169cObMGdy7dw+PHj3CokWLxO7oAoDu3bvj5s2bEIlEVXbp1OfnLicnBy8vLzx48AB//fUXVqxYgZEjR3IJy83NDTt37sSePXvw5MkT/P333wgPD8eOHTvEjuHatWv4999/8fr1a65cX18fkZGRGDRoEKSkpNC6dWtoa2sjMjIS+vr63H7Dhg2Dnp4evv32W1y8eBGpqan466+/sGXLFty4cQMAMGfOHNy+fRs+Pj64f/8+nj17hvPnz8PHx6dOx/+xektKSvDDDz9g8ODBmDp1KtasWYOXL18iKChIrJ6MjAz89NNPePLkCU6ePIldu3ZxVxtffPEFrKys4O3tjZMnTyI1NRV37tzBrl27cPr06TrFz4ek2u/evTsKCgpw5coVZGVl4d27d0hMTMQvv/yC27dv48WLF4iJicHLly9r9YWrITTLK4PqeHh4oHPnzti9ezdWrFjBffsoPzmYmJjA2dkZP//8MwoKCjB8+HC4ubmJ9ftWRl5eHsnJyXB3d0d2djY6d+4MKysrzJkzB0DZN7iAgAAEBgZix44dUFRUhIuLC+zt7evlOGfOnImQkBBER0fDysqqwnahUIj9+/dj3bp1mDVrFoCyPs/ysYaZM2fi7du3WL9+PTIzM/HFF18gMDCwVt9Y5OXlceDAAWzcuBHu7u548+YNlJSUMGTIkCqvFBYvXowlS5bg66+/Rrt27TBr1qwKycDNzQ0+Pj4YM2YMCgsLudtiP1Sfn3v37t1hYWEBV1dXZGdnY/jw4fjpp5+47ba2tpCXl8euXbuwceNGyMnJ4csvvxRr28vLC76+vhg5ciSEQiHOnTsHoCwZ/Prrr2Infn19fcTFxYmVCQQCBAcHw9/fH8uWLUNWVhYUFRWhp6fHXZGoq6tj//798Pf3h729PUpLS6GiooIxY8bU6fg/Vu/27duRkpKC48ePAwA6duwIPz8/uLi4YMSIERg0aBCAsi8LpaWlmDJlCgQCASZPnizW9bR27Vps374d69evh0gkQvv27aGlpYUhQ4bUKX6+JNG+np4epk6dCk9PT+Tk5OC7776Dubk5EhISsH//fuTm5qJr16749ttvMWHChHo8mtoTsI+NdhFCSC3NmDEDffv2rfNVCql/n1Q3ESGEkNqhZEAIIYS6iQghhNCVASGEEFAyIIQQgk/w1tKqqKuro3Xr1o0dBiHkE1NQUICHDx82dhh11mKSQevWrXHr1q3GDoMQ8onR1tZu7BAkgrqJCCGEUDIghJD65O3tDQMDA7E1tf38/DBu3DhYWVlh3rx5yM3N5bbt2LEDpqamMDMzE1tkp75RMiCEkHpkY2NTYWW94cOHIyoqCsePH0evXr24+aySkpIQHR2N6Oho7Ny5EytXrvzoZI2SQsmAEELq0eDBgytM5T5ixAhuZlcdHR2kp6cDAGJiYmBhYQFZWVmoqKigZ8+euH37doPE2WIGkAkhpD6UlJTAxsaGe21nZwc7Ozve7z969CjGjx8PoGza9A8HpIVCIUQikeSCrQYlA0IIqQMpKSmEhYXV6r3btm2DlJQUvvrqKwlHVXOUDAghpBGEhYXhwoUL2LNnD7fwj1Ao5LqMgLIrhQ9XlatPNGZACCENLDY2Fjt37sS2bdvElvM0MTFBdHQ0CgsLkZqaiuTkZG4J1PpGVwaEEFKPPD09ERcXh+zsbBgZGWH+/PkIDg5GYWEhHB0dAZQ9uLZq1Sr07dsX48ePh7m5OaSkpODj4wMpKakGibPFzFqqra1NTyATQiTuUzm3UDcRIYQQ6iYipKX5/0sTi7lxo+HjaGyDgit+EDdcWuAH8f/RlQEhhBBKBoQQQigZEEIIQQOOGXh7e+PChQtQVFREVFQUAMDDwwNPnz4FALx58wZt27ZFREQE0tLSYG5uji+++ALA/267AoC7d+/C29sb79+/h7GxMZYsWcI9sEEIIaR2GiwZ2NjYwN7eHl5eXlyZv78/929fX18oKChwr3v06IGIiIgK9axYsQKrV6+GtrY25syZg9jYWBgbG9dr7IQQ8qlrsG6iymbuK8cYwx9//CE233dlMjIykJeXBx0dHQgEAlhbWyMmJqY+wiWEkBalSYwZ3LhxA4qKiujVqxdXlpaWBmtra9jb2+PG/7/vTSQSQVlZmdtHWVm5wWb0I4SQT1mTeM4gKipK7KpASUkJ58+fR8eOHXH37l3MmzcP0dHRNa43NDQUoaGhANBgC0QQQkhz1OjJoLi4GGfOnBGbAlZWVhaysrIAAE1NTfTo0QNPnz6tMKNfenp6tTP6fTiv+KeyaDUhhNSHRu8munr1Knr37i3W/ZOVlcV9ky+fuU9FRQVKSkpQUFBAYmIiGGMIDw/H6NGjGyt0Qgj5ZDTYlUFlM/fZ2trixIkTsLCwENs3Pj4eW7ZsgbS0NFq1aoWVK1eiQ4cOAIDly5dzt5YaGRnByMiooQ6BEEI+WTRrKSEtDM1NVEZScxN9KueWRu8mIoQQ0vgoGRBCCKFkQAghhJIBIYQQUDIghBACSgaEEEJAyYAQQggoGRBCCEETmJuIEFJ/KnvAjJDK0JUBIYQQSgaEEEIoGRBCCAElA0IIIaBkQAghBJQMCCGEgJIBIYQQUDIghBACSgaEEEJAyYAQQggoGRBCCAElA0IIIaBkQAgh9crb2xsGBgawtLTkynJycuDo6IixY8fC0dERr1+/BgAwxrBmzRqYmprCysoK9+7da7A4KRkQQkg9srGxwc6dO8XKgoODYWBggNOnT8PAwADBwcEAgNjYWCQnJ+P06dNYvXo1VqxY0WBxUjIghJB6NHjwYLRv316sLCYmBtbW1gAAa2trnD17VqxcIBBAR0cHubm5yMjIaJA4KRkQQkgDy8zMhJKSEgCgS5cuyMzMBACIRCIoKytz+ykrK0MkEjVITLS4DSGk0kVwbtxo+Diao5KSEtjY2HCv7ezsYGdnx/v9AoEAAoGgPkKrEUoGhBBSB1JSUggLC6vRexQVFZGRkQElJSVkZGSgU6dOAAChUIj09HRuv/T0dAiFQonGW5UG6yaqbEQ9MDAQhoaGmDBhAiZMmICLFy9y23bs2AFTU1OYmZnh0qVLXHlsbCzMzMxgamrKDboQQkhzYmJigvDwcABAeHg4Ro8eLVbOGENiYiLatm3LdSfVtwa7MrCxsYG9vT28vLzEyh0cHODs7CxWlpSUhOjoaERHR0MkEsHR0RGnTp0CAKxatQq//fYbhEIhJk+eDBMTE3z55ZcNdRiEEFIjnp6eiIuLQ3Z2NoyMjDB//ny4uLjAw8MDR44cQbdu3eDv7w8AMDY2xsWLF2Fqagp5eXn8/PPPDRZngyWDwYMHIy0tjde+MTExsLCwgKysLFRUVNCzZ0/cvn0bANCzZ0+oqKgAACwsLBATE0PJgBDSZG3atKnS8r1791YoEwgEWL58eX2HVKlGv5vowIEDsLKygre3N/fgxX9H1IVCIUQiUZXlhBBC6qZRk8G0adNw5swZREREQElJCb6+vhKtPzQ0FDY2NrCxsUFJSYlE6yaEkE9Jo95N1LlzZ+7ftra2cHV1BVBxRF0kEnEj6lWVV+bDW7y0tbUlGjshhHxKGvXK4MMn686ePYu+ffsCKBtRj46ORmFhIVJTU5GcnIwBAwZAS0sLycnJSE1NRWFhIaKjo2FiYtJY4RNCyCejwa4MKhtRj4uLw8OHDwEA3bt3x6pVqwAAffv2xfjx42Fubg4pKSn4+PhASkoKAODj44PZs2ejpKQEkyZN4hIIIYSQ2hMwxlhjB9EQtLW1cevWrcYOg5AGVdmTxXx96k8gDwqu+OHccKn5QX8q55ZGv5uIEEJI46NkQAghhJIBIYQQSgaEEEJQh7uJioqKICMjI8lYCCGkXlQ2WEzE8boy2LdvHzdRHAD8+OOP0NbWhpmZGZ48eVJvwRFCCGkYvJJBSEgIN992fHw8/vjjD2zYsAH9+vWDn59fvQZICCGk/vHqJhKJRPj8888BAOfOncO4ceNgbm4ONTU1TJ8+vV4DJIQQUv94XRkoKChwa3RevXoVBgYGAABpaWkUFhbWX3SEEEIaBK8rg+HDh2PZsmXo378/UlJSYGRkBAB4/Pgxd8VACCGk+eJ1ZbB8+XLo6ekhKysLAQEB6NChAwDg/v37sLCwqM/4CCGENABeVwYKCgpYtmxZhXI3NzeJB0QIIaTh1eg5A5FIhKysLJSWloqVa2hoSDQoQgghDYtXMrh//z4WLVqEJ0+e4L+TnAoEAjx48KBegiOEEFJztXkomFcyWLZsGZSVlbF69WooKSlBIBDUKkBCCCGStW/fPgiFQpiZmQEoeyg4PDwcKioq2LZtG3r37s2rHl7J4J9//sGxY8fwxRdf1D5iQki9qsvaBXzr+9TXOGiOQkJC8PPPPwMQfyj49OnT8PPzw44dO3jVw+tuIlVVVbx69ar20RJCCKkXVT0UPH/+fCQmJvKuh1cyWLBgAdavX4+rV6/i1atXyMnJEfuPEEJI45DUQ8G8uokcHR0BAE5OTmLjBYwxGkAmhJBGJKmHgnklg3379tUuSkIIIfVq+fLl2Lx5M168eFGnh4J5JQN9ff1aBUkIIaR+SeqhYN4Pnb169QoHDhzAP//8AwDo27cvpk2bhs6dO9eoQUI+ZXW9A4fu4CG18erVK0RERCAlJQXu7u7o1KkTbt68CSUlJaioqPCqg9cA8s2bN2FqaoqoqCi0bt0arVu3RmRkJMaOHYu//vqrTgdBCCGk9u7evYtx48bh+PHjOHr0KN6+fQugbDDZ39+fdz28rgzWrVsHS0tLrFy5Eq1aleWP0tJSLF++HH5+fvi///u/mh8BIc0IfWMnTZWfnx9mzpwJNzc36OrqcuUjRoxAWFgY73p4XRk8ePAAjo6OXCIAgFatWsHBwQH379+vQdiEEEIk6d69e5g4cWKF8i5dutTo+TBeyaBt27ZIS0urUJ6WloZ27drxbowQQohkycnJ4fXr1xXKnzx5AkVFRd718EoG5ubmWLJkCSIjI5GamorU1FRERERg6dKltJ4BIYQ0otGjRyMoKEjsAbO0tDRs2LABY8eO5V0PrzGDRYsWgTGGH3/8ESUlJWVvlJbGtGnTsHDhQl4NeXt748KFC1BUVERUVBSAsr6u8+fPQ0ZGBj169MDatWvRrl07pKWlwdzcnJsLSVtbG6tWrQJQNlji7e2N9+/fw9jYGEuWLKGJ80ijkPRcQM0BjZ00PV5eXpgzZw6GDh2K9+/fY/r06cjMzISenh48PDx418MrGcjKymLp0qVYuHAhUlJSAAA9evSAvLw874ZsbGxgb28PLy8vrmz48OFYuHAhpKWlsX79euzYsQOLFi3i6o+IiKhQz4oVK7B69Wpoa2tjzpw5iI2NhbGxMe84CCHkU6KgoIDff/8d165dw/3791FaWgoNDQ0MGzasRvXw6iYqJy8vDzU1NaipqdUoEQDA4MGD0b59e7GyESNGQFq6LB/p6OggPT292joyMjKQl5cHHR0dCAQCWFtbIyYmpkZxEEJIQ9qzZw8sLCxgaWkJT09PFBQUIDU1Fba2tjA1NYWHh0eN5hD6r/DwcBQWFsLAwADOzs6YM2cOhg0bhsLCQoSHh/Oup8orA1dXV2zYsAEKCgpwdXWttpLt27fzbrAqR48exfjx47nXaWlpsLa2hoKCAjw8PDBo0CCIRCIoKytz+ygrK0MkEtW5bUIIqQ8ikQj79u3DiRMnICcnB3d3d0RHR+PixYtwcHCAhYUFfHx8cOTIEUyfPr1WbXh7e8PQ0LDCYPHbt2/h7e0Na2trXvVUmQw6duzI/btDhw712i+/bds2SElJ4auvvgIAKCkp4fz58+jYsSPu3r2LefPmITo6usb1hoaGIjQ0FAC4sQ5CPobWBSCSVFJSgvfv30NaWhrv379Hly5d8Oeff2Ljxo0AgIkTJyIoKKjWyaB8wtD/evnyJdq2bcu7niqTwdq1a7l/+/r61jA8/sLCwnDhwgXs2bOHOyBZWVnIysoCADQ1NdGjRw88ffoUQqFQrCspPT0dQqGwyrrt7OxgZ2cHoGwQmpDmiJJJ8yUUCuHk5IRRo0ahdevWGD58ODQ0NNCuXTuui7y2PRxWVlYAypYetre3h5SUFLettLQUL1684GYw5YPXALK3tzeWLFkCBQUFsfL8/HysXr1aLHHURGxsLHbu3In9+/eLjUFkZWWhffv2kJKSQmpqKpKTk6GiooIOHTpAQUEBiYmJ0NbWRnh4OGbMmFGrtgkhktHSk1VJSQlsbGy41x9+CX39+jViYmIQExODtm3bwt3dHZcuXZJIu+XLXD5+/BjGxsb47LPPuG0yMjLo3r275G8tDQ8Px/fff18hGbx//x4RERG8koGnpyfi4uKQnZ0NIyMjzJ8/H8HBwSgsLOTWSyi/hTQ+Ph5btmyBtLQ0WrVqhZUrV3LTsi5fvpy7tdTIyKhGmY+QT0VLvK21qZKSkqpy2oerV6/i888/R6dOnQAAY8eORUJCAnJzc1FcXAxpaemP9nBU5bvvvgMAdO/eHebm5mjdunXtDwIfSQY5OTlgjIExhtevX1e4DCl/boCPTZs2VSiztbWtdF8zMzMu6/2XlpYW95wCIYQ0Zd26dcOtW7fw7t07yMnJ4dq1a9DU1MSQIUNw6tQpWFhY4NixYzAxMal1G5VNRVEb1SaDoUOHQiAQQCAQVPqksUAgwPz58yUSCCGEfGq0tbVhZmaGiRMnQlpaGv369YOdnR1GjhyJBQsWwN/fH/369avyizEfhYWF2L59O6Kjo/HixQsUFxeLbee7EmW1yWDfvn1gjGHWrFkIDAwUe05ARkYG3bp1q9XlDSGEtBRubm4VFppRUVHBkSNHJFJ/QEAA/vjjD7i4uGDt2rX44Ycf8Pz5c0RHR8Pd3Z13PdUmg/IVzmJiYtCtWzea9oEQQpqYP/74AytWrICRkRH8/PwwevRo9OjRA71798bVq1cxdepUXvVUmQzu3buHfv36oVWrVsjJyUFOTk6VlWhoaNT4AAghhNRdZmYmvvzySwDAZ599htzcXACAoaEhNmzYwLueKpPBpEmTcOXKFSgqKmLSpEkQCARgjFXYTyAQ8O6TIoQQIlldu3ZFRkYGunXrhh49euDy5cvQ1NREYmIi5OTkeNdTZTKIiYnhboei+X8IkSy6NZRIiqmpKa5duwYdHR3MnDkTCxcuxKFDh5CRkQFnZ2fe9VSZDLp3717pvwkhhDQdHy4jMG7cOHTt2hUJCQno1asXRo0axbueKpNBfHw870oGDx7Me19CCCH1R1tbu1bT71SZDGbMmCE2TlB+J9F/XwP872MlhBAiWZs3b4aysjKmTZsmVv77779DJBLxXuCmyvUMrl27hqtXr+LatWvYsWMHvvjiC/j5+eHMmTM4c+YM/Pz80Lt3b2zbtq1OB0IIIaT2IiIi0L9//wrlGhoalS4QVhVeU1gHBARgyZIlGD58OFemoqICRUVFrF+/HiNHjuTdICGEEMnJzMzkbvb5UMeOHfHq1Sve9fBa6SwpKUlsUZlyQqEQT5484d0YIYQQyerWrRtuVDJNbHx8fKXn7arwSgZffvklgoKC8P79e67s/fv32Lp1K/ewAyGEkIZnZ2eHtWvX4tChQ0hJSUFKSgpCQ0Ph5+eHKVOm8K6H1xTWK1asgKurK4yMjKCmpgYA+Pvvv9GqVSvs2LGjdkdACCGkzpycnJCdnY01a9agqKgIQNnccTNnzsScOXN41yNglT1WXIn8/HwcP36c6xbq06cPLC0t0aZNm1qE3/C0tbVx69atxg6DNAP0QFj9aMwFbwYF8/uh3nCpeZBN5dySn5+PpKQkAGXn5w8Xu+GD15UBALRp04ZbvYcQQkjT0qZNGwwYMKDW7+edDNLT0xEfH4/MzMwKcxSVr1RGCCGk/rm6umLDhg1QUFCAq6trtftu376dV528kkFkZCR+/PFHSEtLV7iFSSAQUDIgzRZ1CZHm6MNb/z/8d13wSgZbtmyBk5MT3N3dxZa+JIQQ0vA+XHeezxr0fPC6tTQzMxO2traUCAgh5BPF68rAyMgIt27dgoqKSn3HQwgh5CM+Nk7wIYmOGQwfPhwbNmzA48ePoaamBmlp8beNHTuWd2CEEELqRlLjBB/ilQx8fHwAoNIHzGilM0IIaViSGif4EK9k8PDhQ4k3TAghRHLev3+PlJQUAECPHj1qtOQlUIPnDAghhDQ9hYWFWL9+PUJDQ1FUVATGGGRlZTFlyhQsWrQIrVu35lUP72Rw4cIF/Prrr0hKSoJAIMCXX36JOXPmwNjYuNYHQQghpG6WL1+OK1euYM2aNdDV1QUA/PXXX9i0aRPevn3Lu0uJVzI4fPgwVq5cCSsrK1hbWwMAbt68iXnz5mHFihWYPHly7Y6CEEJInZw8eRJBQUGVrjczf/58ySaDX3/9FYsXL4a9vT1XZmtrCw0NDfz666+8k4G3tzcuXLgARUVFREVFAQBycnKwYMECPH/+HN27d4e/vz/at28Pxhh++uknXLx4EXJycvD19YWGhgYA4NixY9wKa3PnzsXEiRN5tU8IIZ+aNm3aQCgUVigXCoU1Gjfg9dDZixcvYGhoWKHcyMgIz58/592YjY0Ndu7cKVYWHBwMAwMDnD59GgYGBggODgYAxMbGIjk5GadPn8bq1auxYsUKAGXJIygoCIcOHcLhw4cRFBSE169f846BEEI+Jfb29pWuN/PLL7+IfYH/GF5XBt26dcOVK1fQs2dPsfLLly+je/fuvBsbPHgw0tLSxMpiYmIQEhICALC2tsaMGTOwaNEixMTEwNraGgKBADo6OsjNzUVGRgbi4uIwfPhwdOjQAUDZMxCXLl2CpaUl7zgIIeRTcevWLcTFxVVYb6a4uBj5+fliD6hV9wAar2Tg5OSENWvW4P79+9wARUJCAiIiIrBs2bK6HAcyMzOhpKQEAOjSpQsyMzMBACKRSGzJNmVlZYhEogrlQqEQIpGoTjEQQkhz1bFjR5iZmYmVff755zWuh1cymDp1KhQVFbF7926cOXMGANC7d2/4+/tjzJgxNW60KgKBAAKBQGL1hYaGIjQ0FABQUlIisXoJIaSpkNQDaLxvLTU1NYWpqalEGv2QoqIiMjIyoKSkhIyMDG6KbKFQiPT0dG6/9PR0CIVCCIVCxMXFceUikQj6+vqV1m1nZ8ctyKOtrS3x2AkhpKm4c+cOUlNTMXLkSLRp0wb5+fmQlZWtMH1QVaodQM7KysLWrVuRl5dXYdubN2+wdetWZGdn1y7y/8/ExATh4eEAgPDwcIwePVqsnDGGxMREtG3bFkpKShgxYgQuX76M169f4/Xr17h8+TJGjBhRpxgIIaS5evXqFaZMmQJbW1ssXLiQ62r39fWFr68v73qqTQZ79+7F06dPoaCgUGFb27Zt8ezZM+zdu5d3Y56enpg6dSqePn0KIyMjHD58GC4uLrhy5QrGjh2Lq1evwsXFBQBgbGwMFRUVmJqaYtmyZVi+fDkAoEOHDvj2228xefJkTJ48GfPmzeMGkwkhpKVZu3YtFBUVcf36dbFbSceNG4crV67wrqfa64fz589jyZIlVW6fNGkS1qxZAw8PD16Nbdq0qdLyyhKKQCDgEsB/lScCQghp6a5du4Y9e/agffv2YuUqKip4+fIl73qqvTJITU1Fjx49qtyuoqJS4VZRQgghDef9+/eQkZGpUJ6dnc17XiLgI8lAWlq62sySnp7Oe3CCEEJaotzcXLi5uWHcuHEYP348/vrrL+Tk5MDR0RFjx46Fo6NjnR6cHTx4MI4dOyZWVlJSgl9//RVDhw7lXU+1yaB///7craSVOX36NPr168e7MUIIaWl++uknGBoa4uTJk4iIiECfPn2qnHmhNhYtWoRDhw7B0dERRUVF8PPzg7m5ORISErBw4ULe9VSbDL7++mvs3bsXe/fuFbtPv7i4GHv27EFISAi+/vrrWh8EIYR8yt68eYP4+HhujFNWVhbt2rXjZlgAymZeOHv2bK3b+PLLL3H8+HHo6upi+PDhKCgowLhx43Ds2LFqu/n/q9o+nrFjx2LOnDlYu3YtAgICuIpTUlLw7t07ODs7V3jyjRBC+Bo0qGLZjRsNH0d9SUtLQ6dOneDt7Y2HDx9CQ0MDS5YsqXLmhdrq0qUL3NzcxMrev3+P4OBg7g7Nj/loh/+CBQswevRoREZGIiUlBYwxDB48GFZWVhgwYEDtIieEkE9ESUkJbGxsuNcfPuxaXFyM+/fvY9myZdDW1saaNWsqdAnVZeaFrKws3L59G9LS0jAwMICUlBSKiopw4MABBAcHo6SkRHLJAAAGDBhAJ35CCKmElJQUwsLCKt2mrKwMZWVlbgaEcePGITg4uMqZF2oiISEB33zzDd68eQOBQAAtLS2sXbsW3333HYqLi/Htt99i0qRJvOujW4FIi1FZlwQh9alLly5QVlbGkydP0Lt3b1y7dg19+vRBnz59EB4eDhcXF7GZF2oiICAAI0aMwNy5cxEWFoY9e/bA1dUV8+bNw4QJE2p8tUHJgBBC6tGyZcvw/fffo6ioCCoqKli7di1KS0vh4eGBI0eOoFu3bvD3969xvQ8fPsT+/fvRt29fuLu7Y+/evfD09MT48eNrFSclA0IIqUf9+vWrtBupJlP5VOb169dc95K8vDzk5OTQv3//WtdHyYAQQpqp169fQ0pKCowxCAQC5OXlIScnR2wfvnO38UoG3t7eWLJkSYUJ6/Lz87F69WqJzadNCCGEPwsLC+7fjDGxOdvKE8SDBw941cUrGYSHh+P777+vkAzev3+PiIgISgaEENLA9u3bJ9H6qk0GOTk5YIyBMcZdjpQrLS3FhQsXoKioKNGACCGEfFxVi3rVVrXJYOjQodwDER9ejpQTCASYP3++RAMihBDS8KpNBvv27QNjDLNmzUJgYKDYfNkyMjLo1q0bhEJhvQdJCCGkflWbDMovQ2JiYtC1a1e0alXtvHaEEFIleuivaeM1gNy9e3e8e/cODx48QFZWFkpLS8W2jx07tl6CI4QQ0jB4JYOrV6/C09Ozwv2rAGp06xIhhJCmiVcy+OmnnzBy5EgsWLCAxggIIaSJOXHiBK5du4bMzMwKPTfbt2/nVQevZPD8+XNs27aNEgEhhDQxfn5+2LdvH4YMGQIlJaVaT4fNKxno6enh6dOnNVo1hxBCSP2LiIjAxo0bMW7cuDrVwysZTJ06FX5+fsjIyICqqiqkpcXfpqGhUacgCCGE1E5paalE1qLnlQzKl1NbtmxZhW00gEwIIY3Hzs4OkZGRdX4AmFcyiImJqVMjhBBC6kdubi6ioqJw5coVqKmpQUZGRmz70qVLedXD+zkDQgghTU9SUhLU1dUBAE+ePBHbVpPBZN7rGVy8eBEHDx5Eamoqdu3aha5du+Lw4cP4/PPPYWBgwLtBQgghkhMSEiKRenjNLxEZGQkPDw/07NkTaWlpKC4uBgCUlJRg586ddQrgyZMnmDBhAvefnp4e9uzZg8DAQBgaGnLlFy9e5N6zY8cOmJqawszMDJcuXapT+4QQ0pwlJCRw5+S64HVlsHPnTqxZswYWFhY4fPgwV66jo4MtW7bUKYDevXsjIiICQFlyMTIygqmpKcLCwuDg4ABnZ2ex/ZOSkhAdHY3o6GiIRCI4Ojri1KlTYtNrE0JISzFr1ixIS0tDV1cX+vr60NfXx4ABAyrc9fkxvPZ+9uwZdHR0KpS3adMGeXl5NWqwOteuXYOKikq1YxQxMTGwsLCArKwsVFRU0LNnT9y+fRu6uroSi4M0fzQpGmkp4uPjkZCQgLi4OMTGxmLr1q1cchgyZAi++eYbXvXwSgZKSkpITk6ucJKOj4+X6INo0dHRsLS05F4fOHAA4eHh0NTUxOLFi9G+fXuIRCJoa2tz+wiFQohEokrrCw0NRWhoKICyqw5CSNNXVSK/caNh42gu5OTkMGzYMAwbNgwAkJKSgm3btuH48eO4du0a72TAa8xgypQpWLNmDW7evAkAePnyJY4dO4b169dj2rRptTwEcYWFhTh37hz3FN20adNw5swZREREQElJCb6+vjWu087ODmFhYQgLC6NuJELIJykzMxMnTpzA8uXLMX78eHz11VdIS0uDq6sr9u7dy7seXlcGc+bMQV5eHpycnFBQUICZM2dCVlYWTk5O+Prrr2t9EB+KjY2FhoYGOnfuDADc/wHA1tYWrq6uAMquBNLT07ltIpGI5kwihLRYw4cPR6dOnWBnZ4dVq1ZBW1sbsrKyNa7no8mguLgYV65cgYODA1xdXZGUlATGGPr06YPPPvusVsFXJjo6WmxpzYyMDCgpKQEAzp49i759+wIATExMsHDhQjg6OkIkEiE5ORkDBgyQWByEENKcWFpa4saNG9i3bx/u37+PIUOGQF9fHxoaGpJ9zkBaWhrfffcd/vjjD3Ts2BFaWlp1Crwy+fn5uHr1KlatWsWVrV+/Hg8fPgRQ9tBb+ba+ffti/PjxMDc3h5SUFHx8fKgLiBDSYm3YsAFA2VjB9evXERcXh5CQEOTl5WHQoEHYtm0br3p4dROpq6sjJSUFn3/+ee0jrkabNm1w/fp1sbL169dXuf/cuXMxd+7ceomFEEKao88//xzZ2dnIyspCZmYm4uLiavQcFq9k8N1338HX1xdubm7Q0NCAvLy82PYOHTrUKGhCCCGS8euvvyIuLg43b95EYWEhNDU1MXjwYDg6OmLgwIG86+GVDMpvTfruu+/E+qAYYzRrKSGENKKzZ89CX18fM2fOxMCBA9GmTZta1cMrGezbt69WlRNCCKlf5c9S1RWvZKCvry+RxgghhEjeq1evcODAAfzzzz8AgC+//BLTp08Xu0X/Y3g9dAYAjx49wqpVqzB79mxkZGQAKLs8uX//fg3DJoQQIik3b96EqakpoqKi0Lp1a7Ru3RrHjx/H2LFj8ddff/Guh1cyuHz5MiZPngyRSIQ///wTBQUFAMpuZQoKCqrdERBCCKmzdevWwdLSEqdOncL69euxfv16nDp1ChYWFvDz8+NdD69kEBAQgMWLF2Pr1q1iq+jo6+vj9u3bNY+eEEKIRDx48ACOjo5o1ep/p/NWrVrBwcGhRj03vJLB48ePYWxsXKG8ffv2eP36Ne/GCCGESFbbtm2RlpZWoTwtLQ3t2rXjXQ+vZFA+W+h/3b9/H8rKyrwbI4QQIlnm5uZYsmQJIiMjkZqaitTUVERERGDp0qViU/x8DK+7iSwtLbF+/Xr4+/tDIBCguLgYcXFx8PPzg42NTa0PghBJoLULyIcGBbesX4hFixaBMYYff/yRm6pfWloa06ZNw8KFC3nXI2CMsY/tVFRUhMWLF+PEiRNgjKFVq1ZgjMHS0hK+vr7NYm4gbW1t3Lp1q7HDIPWAkkHLVdkaB3VJBjdcar5oAp9zS0lJCSZNmgShUIgdO3YgNTUVnp6eyMnJgYaGBtatW1ermUY/9O7dO6SkpAAAevToUWGmiI+p8srgxYsX6Nq1KwQCAWRkZLBx40Z4eHjg3r17KC0tRf/+/dGrV686BU8IIS3Bvn370KdPH25lyA0bNsDBwQEWFhbw8fHBkSNHMH369Dq1IS8vDzU1tVq/v8pkMHr0aFy+fBmKioqYOXMmgoKCoKKiAhUVlVo3RgghLU16ejouXLgAV1dX7NmzB4wx/Pnnn9i4cSMAYOLEiQgKCqp1MigoKMDevXvx559/IjMzE6WlpWLbjx8/zqueKpPBZ599huzsbCgqKiIuLg7FxcW1CpQQQj5lJSUlYmOndnZ2sLOz417//PPPWLRoEd6+fQsAyM7ORrt27bgF65WVlatcupePFStW4OzZsxg3bhx0dXVrtIbBh6pMBsOGDcOsWbPQu3dvAMC8efPEnjH4EM1dRAhpqaSkpBAWFlbptvPnz6NTp07Q1NSsME2/pMTExCAgIIBbA7m2qkwG69atw9GjR/Hs2TPEx8fjiy++qPGABCGEtGQJCQk4d+4cYmNjUVBQgLy8PPz000/Izc1FcXExpKWlkZ6eXqele+Xk5CRyi3+VySArKwvTp0/npqhevHhxjR5gIISQlm7hwoXc7Z3Xr1/H7t27sXHjRri5uXFTRhw7dgwmJia1bmP27NnYs2cPVq5cWesuIoDnAHJdGiCEECJu0aJFWLBgAfz9/dGvXz/Y2trWuq6rV6/ixo0buHTpEvr06cONRZTbvn07r3p4DSDHx8fTADJpEuiZAtJcDRkyBEOGDAEAqKio4MiRIxKpt2PHjjA1Na1zPbwGkBljNIBMCCFN0Nq1a6vclpWVxbseGkAmhJBmKCAgAO7u7lVuz87OhoODAyIjI3nVV2UykJOTw9dffw0ANIBMCCFNzG+//QZFRUXY29tX2JaTkwMHBwexaa0/htdEdSEhIfwjJIQQUu82b94MNzc3dOjQAZaWllx5bm4unJycUFpaWqMu/CqTwZo1a+Dp6Yk2bdpgzZo11VaydOlS3g0SQgipu1GjRmH16tX48ccf0b59exgaGuLNmzdwdHTE+/fvsX//fnTs2JF3fVUmg0ePHnF3ED169KjKCui2U1Jf6M4hQqpnbW2NnJwcuLm5wd/fH0FBQXj79i1CQkLQqVOnGtVVZTL4sGuIuokIIaRpcnBwQE5ODlxdXdGjRw+EhISgS5cuNa6H15hBQzAxMcFnn32GVq1acXN95OTkYMGCBXj+/Dm6d+8Of39/tG/fHowx/PTTT7h48SLk5OTg6+sLDQ2Nxj6EJquyb9iVzQNPCGk+XF1dxV5LS0tDQUEBPj4+YuV1fuis3Pv377Fz506cOnWKW2dTRUUF48aNg5OTE+Tk5PjG/lF79+4Vu7QJDg6GgYEBXFxcEBwcjODgYCxatAixsbFITk7G6dOncevWLaxYsQKHDx+WWByEENLU/Xc84MNB5NqoNhkUFxdj1qxZuHfvHgwNDTFy5EgwxpCUlIRffvkFsbGx2L9/f4XHnyUlJiaG66KytrbGjBkzsGjRIsTExMDa2hoCgQA6OjrIzc1FRkYGlJSU6iUOQghpaqp72Kw2qj2Lh4aG4tmzZzh27Bj69u0rtu3vv//GzJkzcejQoTqv0FPO2dkZAoGAmw88MzOTO8F36dIFmZmZAACRSCQ2S1/5fOCUDAghpHaqTQanTp2Cq6trhUQAAKqqqvjmm29w8uRJiSSD33//HUKhEJmZmXB0dOTWUSgnEAhqfOdSaGgoQkNDAYBbKJoQQkhF1T6elpSUhKFDh1a53cDAAI8fP5ZIIOXzeSsqKsLU1BS3b9+GoqIiMjIyAAAZGRnceIJQKER6ejr33qrmA7ezs0NYWBjCwsIgJSUlkTgJIeRTVG0yyM3NrfZe1U6dOuHNmzd1DiI/P59bKDo/Px9XrlxB3759YWJigvDwcABAeHg4Ro8eDQBcOWMMiYmJaNu2LXUREUJIHVTbTVRSUlLt4HCrVq0k0v2SmZmJefPmcW1aWlrCyMgIWlpa8PDwwJEjR9CtWzf4+/sDAIyNjXHx4kWYmppCXl4eP//8c51jIISQlqzaZMAYw6JFi6qcurqoqEgiQaioqFQ6s17Hjh2xd+/eCuUCgQDLly+XSNuEEEI+kgwmTpz40Qqsra0lFQuRAL5TONCDaISQD1WbDCR9HyshhJCmqclMR0FaNpqUjpDGxX/lA0IIIZ8sSgaEEEIoGRBCCKFkQAghBJQMCCGEgO4mIvWInmUgpPmgZEA4dPImpOWibiJCCCF0ZUAkoy7TYBBCGh9dGRBCCKFkQAghhJIBIYQQUDIghBACSgaEEEJAdxORWqA7ggj59NCVASGEELoyaM7oGzpp6Sr9G3Bp8DCq9PLlS/zwww/IzMyEQCDAlClTMGvWLOTk5GDBggV4/vw5unfvDn9/f7Rv375RY6UrA0IIqSdSUlJYvHgxTpw4gdDQUBw8eBBJSUkIDg6GgYEBTp8+DQMDAwQHBzd2qJQMCCGkvigpKUFDQwMAoKCggN69e0MkEiEmJgbW1tYAAGtra5w9e7YRoyxDyYAQQhpAWloaHjx4AG1tbWRmZkJJSQkA0KVLF2RmZjZydDRmQD6CxiUIqV5JSQlsbGy413Z2drCzsxPb5+3bt3Bzc8OPP/4IBQUFsW0CgQACgaBBYq0OJQNCCKkDKSkphIWFVbm9qKgIbm5usLKywtixYwEAioqKyMjIgJKSEjIyMtCpU6eGCrdKlAyaCfqGTkjzwxjDkiVL0Lt3bzg6OnLlJiYmCA8Ph4uLC8LDwzF69OhGjLIMJQNCCKknN2/eREREBFRVVTFhwgQAgKenJ1xcXODh4YEjR46gW7du8Pf3b9xA0QSSQVX34QYGBuLQoUPc5ZOnpyeMjY0BADt27MCRI0fQqlUrLF26FIaGho15CIQQUqlBgwbh0aNHlW7bu3dvA0dTvUZPBuX34WpoaCAvLw+TJk3C8OHDAQAODg5wdnYW2z8pKQnR0dGIjo6GSCSCo6MjTp06BSkpqcYInxBCPgmNfmtpVffhViUmJgYWFhaQlZWFiooKevbsidu3bzdUuIQQ8klq9CuDD314H25CQgIOHDiA8PBwaGpqYvHixWjfvj1EIhG0tbW59wiFwmqTR3NEg8WESNaDBxXL+vVr+Diaska/Mij33/twp02bhjNnziAiIgJKSkrw9fWtcZ2hoaGwsbGBjY0NSkpK6iFqQgj5NDSJZFDZfbidO3eGlJQUWrVqBVtbW9y5cwdA2ZVAeno6916RSAShUFhpvXZ2dggLC0NYWBiNKRBCSDUaPRlUdR9uRkYG9++zZ8+ib9++AMruz42OjkZhYSFSU1ORnJyMAQMGNHjchBDyKWn0MYOq7sONiorCw4cPAQDdu3fHqlWrAAB9+/bF+PHjYW5uDikpKfj4+NC3fkJaqAeGFQfYaCigdho9GVR1H275MwWVmTt3LubOnVufYRFCSIvS6MmAEEIkqbI7h8jHNfqYASGEkMZHyYAQQgglA0IIIZQMCCGEgJIBIYQQUDIghBACSgaEEEJAyYAQQgjoobNGR9NVE0KaAroyIIQQQsmAEEIIJQNCCCGgZEAIIQSUDAghhIDuJiKENBOVLWRDJIeuDAghhNCVQUOiZwoIIU0VJQNCSItEK6KJo24iQgghlAwIIYRQN1G9oLEBQkhzQ8mAENKk0C2kjYOSQR3RVQAh5FNAYwaEEEIoGRBCCKFuIkJII6Lxgaaj2V4ZxMbGwszMDKampggODm6QNgcNqvgfIYRUpzHOVbXRLJNBSUkJVq1ahZ07dyI6OhpRUVFISkpq7LAIIURMczpXNctuotu3b6Nnz55QUVEBAFhYWCAmJgZffvllI0dGyKejsi6cfpdu1Pt7PyXN6VzVLJOBSCSCsrIy91ooFOL27dsSbYO6gAghddUQ5ypJaZbJgK/Q0FCEhoYCAN6/fw9tbW3e7y0pKYGUlFR9hUbtN9G2W3r7H7b95bmK24vA72+I73v/u19jf/aampo1bv/9+/ewsbHhXtvZ2cHOzk7SodW7ZpkMhEIh0tPTudcikQhCobDCfnX5odjY2CAsLKzWMdZVS26/JR97Y7ffko+9Ptrne65qCprlALKWlhaSk5ORmpqKwsJCREdHw8TEpLHDIoQQMc3pXNUsrwykpaXh4+OD2bNno6SkBJMmTULfvn0bOyxCCBHTnM5VzTIZAICxsTGMjY3rrf7G7vNrye235GNv7PZb8rHXV/v1fa6SFAFjjDV2EIQQQhpXsxwzIIQQIlktPhl87FHx3377Debm5rCyssKsWbPw/PnzBm3/999/h5WVFSZMmIBp06ZJ9OlFvo/Jnzp1Cmpqarhz547E2ubTflhYGIYOHYoJEyZgwoQJOHz4cIO2DwAnTpyAubk5LCwssHDhwgZr++eff+aO28zMDIMk/ODLx9p/8eIFZsyYAWtra1hZWeHixYsN2v7z588xa9YsWFlZYcaMGWJ35NSVt7c3DAwMYGlpWel2xhjWrFkDU1NTWFlZ4d69exJru0ljLVhxcTEbPXo0S0lJYQUFBczKyoo9fvxYbJ9r166x/Px8xhhjBw4cYO7u7g3a/ps3b7h/nz17ljk5OTVY2+XtT58+ndna2rLbt29LpG2+7R89epStXLlSYm3WtP2nT5+yCRMmsJycHMYYY69evWqwtj+0b98+tnjxYom0zbf9pUuXsgMHDjDGGHv8+DEbNWpUg7Y/f/58FhYWxhhj7OrVq+z777+XWPtxcXHs7t27zMLCotLtFy5cYM7Ozqy0tJT99ddfbPLkyRJruylr0VcGHz4qLisryz0q/qGhQ4dCXl4eAKCjoyPRbyh82ldQUOD+/e7dOwgEggZrGwACAgIwZ84ctG7dWiLt1rT9+sKn/UOHDuHrr79G+/btAQCKiooN1vaHoqOjq/wWW1/tCwQC5OXlAQDevHkDJSWlBm3/n3/+wdChQwGU/Q1K8ndj8ODB3M+0MjExMbC2toZAIICOjg5yc3ORkZEhsfabqhadDCp7VFwkElW5/5EjR2BkZNTg7R84cABjxozB+vXrsXTp0gZr+969e0hPT8fIkSMl0mZN2weA06dPw8rKCm5ubnj58mWDtp+cnIynT59i6tSpmDJlCmJjYxus7XLPnz9HWload2JsqPa/++47HD9+HEZGRnBxcZHY7x3f9tXV1XH69GkAwJkzZ/D27VtkZ2dLLIaaxKesrFzteeFT0aKTQU1ERETg7t27mD17doO3/fXXX+Ps2bP4/vvvsW3btgZps7S0FL6+vvDy8mqQ9iozatQonDt3DsePH8ewYcMaPJaSkhI8e/YMISEh2LhxI5YtW4bc3NwGjSE6OhpmZmYNPkVDdHQ0Jk6ciNjYWAQHB+OHH35AaWlpg7X/ww8/ID4+HtbW1oiLi4NQKGzUaSpaghadDPg+Kn716lVs374d27Ztg6ysbIO3X87CwgJnz55tkLbfvn2Lv//+GzNnzoSJiQkSExMxd+5ciQ0i8zn2jh07cp+3ra2tRAfy+LQvFAphYmICGRkZqKiooFevXkhOTm6QtsudOHECFhYWdW6zpu0fOXIE48ePBwDo6uqioKBAYt/M+X72QUFBCA8Px4IFCwAA7dq1k0j7NY0vPT29yU4hIUktOhnweVT8/v378PHxwbZt2yTWZ1yT9j88+Vy4cAE9e/ZskLbbtm2L69ev49y5czh37hx0dHSwbds2aGlpNUj7AMT6ac+dO4c+ffpIpG2+7Y8ZMwZxcXEAgKysLCQnJ3NTEdd320BZv3lubi50dXXr3GZN2+/atSuuXbvGxVFQUIBOnTo1WPtZWVnclUhwcDAmTZokkbb5MDExQXh4OBhjSExMRNu2bSU6ZtJUNdsnkCWhqkfFAwICoKmpidGjR2PdunXIz8+Hu7s7gLI/ku3btzdY+/v378e1a9cgLS2Ndu3awc/Pr8Hark982g8JCcG5c+cgJSWF9u3bY+3atQ3avqGhIa5cuQJzc3NISUnhhx9+QMeOHRukbeB/t7VK6qaBmrS/ePFiLF26FHv27IFAIICvr6/E4uDTflxcHDZt2gSBQIBBgwZh+fLlEmkbADw9PREXF4fs7GwYGRlh/vz5KC4uBgBMmzYNxsbGuHjxIkxNTSEvL4+ff/5ZYm03ZfQEMiGEkJbdTUQIIaQMJQNCCCGUDAghhFAyIIQQAkoGhBBCQMmAkCYvLCxMIs8aXL9+HWpqasjKypJAVORTQ8mgBVm8eDHU1NSgpqYGDQ0NjB49Gn5+fsjPz2/s0OpFWFgYd7xqamoYNmwYXF1d8fjx48YOrd6ZmJhg165dYmW6urq4fPmyRJ6VIJ8eSgYtzLBhw3D58mWcPXsWHh4eOHjwoMQeZKtMcXExGvNRFnl5eVy+fBmXLl3Cjh07kJ+fj2+++QaFhYUNGkdRUVGFsoaOQVZWFl26dJH4Q2zk00DJoIUpPyF07doVVlZWsLKy4qYHZozh119/xZgxYzBgwABYWVkhIiJC7P0bNmyAmZkZBgwYABMTE6xbtw4FBQXc9sDAQFhaWiIsLAxjxoyBlpYW8vPzER8fjylTpkBXVxcDBw7E5MmT8ffff3PvK5+dVFNTE8bGxti2bZtYEjExMcEvv/wCHx8f6OnpwcjICDt37vzo8QoEAnTp0gVKSkrQ0tKCg4MDnj9/jqdPn1b7vn/++Qeurq4YOHAgdHV1YWdnh0ePHgEom8Rv69atMDY2hqamJqysrMTmjEpLS4OamhqioqIwc+ZMDBgwAKGhoVi8eDG++eYbBAcHw8jIiFsXVyQSYcGCBRg8eDAGDx4MFxeXaudASklJwdy5czF8+HDo6Ohg4sSJOH/+PLd9xowZeP78OdatW8ddFQGVdxPV1+dOmp8WPR0FAeTk5Lhvrf7+/jh58iR8fHzwxRdfIDExEcuWLUP79u25aazLH88XCoX4559/sHz5csjKysLDw4OrMy0tDVFRUQgICICMjAxat26Nb7/9FpMnT8aGDRtQVFSE+/fvc7NQ3r17F+7u7pg7dy6srKxw584dLF++HAoKCpgxYwZX7969ezF//nwcO3YMsbGxWLNmDXey5iM3NxdRUVEAyqZEqIpIJML06dOhp6eH3377DW3btsXt27e5uXL27duHXbt2YeXKldDU1ERkZCTmz5+PsLAw9OvXj6tn06ZN+OGHH/DTTz9BRkYGd+/eRVxcHBQUFLBz504wxvDu3TvMnDkTurq6CAkJgYyMDHbv3g1HR0ecOHGCW0vjQ/n5+TAyMoKHhwfk5ORw4sQJzJ8/HxEREejTpw8CAwMxYcIETJo0CdOmTavyOBvqcyfNROOtq0MampeXF3NxceFe37p1i+nr6zN3d3f29u1bpqWlxeLj48Xes2bNGjZ79uwq6zx48CAbM2YM93rLli2sf//+7N9//+XKsrOzmaqqKrt+/XqldXh6erIZM2aIlW3ZsoUZGhpyr0eNGsUWLFggto+pqSnbunVrlbEdPXqUqaqqMh0dHaatrc1UVVWZqqoqc3V1rfI9jDG2adMmNnLkSFZQUFDp9hEjRrDAwECxMnt7e7Zw4ULGGGOpqalMVVWV7dq1S2wfLy8vNmTIELF6Dx8+zExNTVlpaSlXVlxczPT19Vl0dDR3HDo6OtXGbGtrK/ZZjBo1iu3cuVNsnz///JOpqqqyzMxMxlj9fe6keaIrgxbm0qVL0NXVRXFxMYqLizF69GgsW7YMSUlJKCgowOzZs8X6lIuKitC9e3fu9cmTJ7F3716kpKQgPz8fJSUlFea5FwqF6Ny5M/e6Q4cOsLGxgbOzMwwMDGBgYAAzMzN069YNAPDkyROuy6TcwIEDERQUhLy8PG61t/LujnJKSkofvTNGXl4e4eHhKC4uxo0bN7B7926sWrWq2vfcv38fenp6lU5XnpeXh4yMDAwcOFCsXE9Pr8LiN5qamhXe37dvX7F67927h7S0NOjp6Ynt9+7dO6SmplYaX35+PoKCgnDhwgX8+++/KC4uRkFBQYXP52Pq83MnzQ8lgxZm0KBBWL16NaSlpaGkpAQZGRkAZV07ALBt2zbuJF2uvEslMTERnp6emDdvHgwNDdGuXTucO3euwgB0mzZtKrS7du1azJo1C7GxsTh37hw2b96MrVu3wtDQkHfs/+3aEQgEH11wRSAQcNN+9+nTB//++y88PT0REhLCu12+/jswW1kXz38/m9LSUqirq2Pz5s0V9q1qaUY/Pz9cunQJXl5e6NmzJ+Tl5eHl5VXpILUk1OZzJ80PJYMWRl5evtI1Efr06QNZWVm8ePECBgYGlb43ISEBQqEQ8+bN48pevHjBu211dXWoq6vDxcUFs2fPRnh4OAwNDdG7d28kJCSI7Xvz5k0oKyuLrQEtCQ4ODvjtt99w+vRpjB07ttJ9+vfvj8jISBQWFla4OlBQUICSkhJu3rwp9jklJCTUar0FDQ0NREdHo2PHjrwXb0lISIC1tTXMzMwAAAUFBUhJSUGvXr24fWRkZFBSUlJtPQ35uZOmj+4mIgDKTnJOTk5Yt24djhw5gmfPnuHBgwf4/fffERoaCgDo1asXRCIRIiMjkZqaioMHD3IDstVJTU3Fhg0bkJCQgOfPn+PPP//Eo0ePuJOnk5MT4uPjERgYiKdPnyIyMhK7d++ulyVGFRQUYGtri8DAwCq/3U6fPh35+fnw8PDA7du38ezZM0RFReHBgwcAAGdnZ+zevRtRUVF4+vQpAgICcOPGDTg7O9c4HisrKygqKuLbb79FXFwcUlNTER8fD19f3yrvKOrVqxfOnDmDe/fu4dGjR1i0aJHYHV0A0L17d9y8eRMikajKLp2G/NxJ00dXBoTj4eGBzp07Y/fu3VixYgUUFBTQr18/7uRgYmICZ2dn/PzzzygoKMDw4cPh5uaGlStXVluvvLw8kpOT4e7ujuzsbHTu3BlWVlaYM2cOgLJvxwEBAQgMDMSOHTugqKgIFxcX2Nvb18txzpw5EyEhIYiOjoaVlVWF7UKhEPv378e6deswa9YsAGX95uVjDTNnzsTbt2+xfv16ZGZm4osvvkBgYCDU1dVrHIu8vDwOHDiAjRs3wt3dHW/evIGSkhKGDBlS5ZXC4sWLsWTJEnz99ddo164dZs2aVSEZuLm5wcfHB2PGjEFhYSF3W+yHGvpzJ00bLW5DCCGEuokIIYRQMiCEEAJKBoQQQkDJgBBCCCgZEEIIASUDQgghoGRACCEElAwIIYSAkgEhhBAA/w9VwIu+eGtj1gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 360x360 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig,ax = plt.subplots(figsize=(5,5))\n",
    "ax2 = ax.twinx()\n",
    "ax2.hist(rep_corrs.values(),bins=50,range=(0.2,1),alpha=0.8,color='green',linewidth=0)\n",
    "ax.hist(rand_corrs.values(),bins=50,range=(0.2,1),alpha=0.8,color='blue',linewidth=0)\n",
    "ax.set_title('Pearson R correlation between experiments',fontsize=14)\n",
    "ax.set_xlabel('Pearson R correlation',fontsize=14)\n",
    "ax.set_ylabel('Different Conditions',fontsize=14)\n",
    "ax2.set_ylabel('Known Replicates',fontsize=14)\n",
    "\n",
    "med_corr = np.median([v for k,v in rep_corrs.items()])\n",
    "print('Median Pearson R between replicates: {:.2f}'.format(med_corr))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Remove samples without any high-correlation replicates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "dissimilar = []\n",
    "for idx, grp in DF_metadata_passed_step4.groupby('full_name'):\n",
    "    ident = np.identity(len(grp))\n",
    "    corrs = (DF_log_tpm[grp.index].corr() - ident).max()\n",
    "    dissimilar.extend(corrs[corrs<rcutoff].index)\n",
    "\n",
    "# Save this information in both the original metadata dataframe and the new metadata dataframe\n",
    "DF_metadata_all['passed_replicate_correlations'] = ~DF_metadata_all.index.isin(dissimilar)\n",
    "DF_metadata_passed_step4['passed_replicate_correlations'] = ~DF_metadata_passed_step4.index.isin(dissimilar)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# Samples that passed replicate correlations: 265\n"
     ]
    }
   ],
   "source": [
    "DF_metadata_final = DF_metadata_passed_step4[DF_metadata_passed_step4['passed_replicate_correlations']]\n",
    "print('# Samples that passed replicate correlations:',len(DF_metadata_final))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check that reference conditions still exist\n",
    "If a reference condition was removed due to poor replicate correlations, a new reference condition needs to be defined.\n",
    "\n",
    "Again, any samples that fail these checks will be printed below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "project_exprs = []\n",
    "for name,group in DF_metadata_final.groupby('project'):\n",
    "    \n",
    "    # Get reference condition\n",
    "    ref_cond = group.reference_condition.iloc[0]\n",
    "    \n",
    "    # Ensure the reference condition is still in the project\n",
    "    if ref_cond not in group.condition.tolist():\n",
    "        print('Reference condition missing from:', name)\n",
    "    \n",
    "    # Check that each project has at least two conditions (a reference and at least one test condition)\n",
    "    if len(group.condition.unique()) <= 1:\n",
    "        print('Only one condition in:', name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If necessary, choose a new condition for failed projects and re-run notebook."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Normalize dataset to reference conditions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "DF_log_tpm_final = DF_log_tpm[DF_metadata_final.index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "project_exprs = []\n",
    "for name,group in DF_metadata_final.groupby('project'):\n",
    "    \n",
    "    # Get reference condition\n",
    "    ref_cond = group.reference_condition.iloc[0]\n",
    "    \n",
    "    # Get reference condition sample ids\n",
    "    ref_samples = group[group.condition == ref_cond].index\n",
    "    \n",
    "    # Get reference condition expression\n",
    "    ref_expr = DF_log_tpm_final[ref_samples].mean(axis=1)\n",
    "    \n",
    "    # Subtract reference expression from project\n",
    "    project_exprs.append(DF_log_tpm_final[group.index].sub(ref_expr,axis=0))\n",
    "\n",
    "DF_log_tpm_norm = pd.concat(project_exprs,axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save final datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "logTPM_qc_file = path.join('..','data','processed_data','log_tpm.csv')\n",
    "logTPM_norm_file = path.join('..','data','processed_data','log_tpm_norm.csv')\n",
    "final_metadata_file = path.join('..','data','processed_data','metadata.tsv')\n",
    "final_metadata_all_file = path.join('..','data','interim','metadata_qc_part2_all.tsv')\n",
    "\n",
    "DF_log_tpm_final.to_csv(logTPM_qc_file)\n",
    "DF_log_tpm_norm.to_csv(logTPM_norm_file)\n",
    "DF_metadata_final.to_csv(final_metadata_file, sep='\\t')\n",
    "DF_metadata_all.to_csv(final_metadata_all_file, sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": true,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}