createAutoEncoderDataSet.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create Auto Encoder data Set\n",
    "Figure out how to create a data set we can use to train and auto encoder\n",
    "input RNA seq data output Gene essential Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Explore RNAseq_lRPKM_data.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO <ipython-input-1-4e35873a7bd7>:10 - <module>()] using logging configuration file:src/test/logging.test.ini.json\n"
     ]
    }
   ],
   "source": [
    "# from DEMETER2.dataFactory import DataFactory\n",
    "\n",
    "from   DEMETER2.lowRankMatrixFactorizationEasyOfUse \\\n",
    "            import LowRankMatrixFactorizationEasyOfUse as LrmfEoU\n",
    "\n",
    "import logging\n",
    "from   setupLogging import setupLogging\n",
    "configFilePath = setupLogging( default_path='src/test/logging.test.ini.json')\n",
    "logger = logging.getLogger(\"notebook\")\n",
    "logger.info(\"using logging configuration file:{}\".format(configFilePath))\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "dataRootDir = \"data/\"\n",
    "rnaSeqDataFile = dataRootDir + \"RNAseq_lRPKM_data.csv\"\n",
    "geneDepDataFile =  dataRootDir +\"D2_Achilles_gene_dep_scores.tsv\"\n",
    "autoEncoderDataDir = dataRootDir + \"autoEncoder/\"\n",
    "\n",
    "# Load the best low rank matrix model identified in evaluateRandomHoldOut.ipynb. \n",
    "# These are the genes and cell lines we want to predict\n",
    "dataFileName = \"D2_Achilles_gene_dep_scores.tsv\"\n",
    "numFeatures = 19\n",
    "geneFilterPercent = 0.25 \n",
    "holdOutPercent = 0.40 \n",
    "easyOfUse = LrmfEoU(dataRootDir, dataFileName, numFeatures, geneFilterPercent, holdOutPercent)\n",
    "resultsDict = easyOfUse.loadAll()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "geneDependencies.shape:(11193, 501)\n",
      "genes.shape:(11193, 19)\n",
      "cellLines.shape:(501, 19)\n"
     ]
    }
   ],
   "source": [
    "# clean tidy version of demeter data\n",
    "Y, R, geneDepCellLines, geneDepNames, = resultsDict[\"DEMETER2\"]\n",
    "geneDependencies = Y\n",
    "print(\"geneDependencies.shape:{}\".format(geneDependencies.shape))\n",
    "\n",
    "# trained model\n",
    "# scipy.optimize.OptimizeResult\n",
    "X, Theta, optimizeResult = resultsDict[\"LowRankMatrixFactorizationModel\"]\n",
    "genes = X\n",
    "print(\"genes.shape:{}\".format(genes.shape))\n",
    "cellLines = Theta\n",
    "print(\"cellLines.shape:{}\".format(cellLines.shape))\n",
    "\n",
    "# knockout logical filters. Use to select Y Train, Validations, and Test values\n",
    "RTrain, RValidation, RTest = resultsDict[\"filters\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "rawRnaSeqData = np.loadtxt(rnaSeqDataFile, dtype=str, delimiter=\",\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rawRnaSeqData.shape:(19185, 646)\n",
      "rawRnaSeqData[0,0:5]:\n",
      "['\"\"' '\"143B_BONE\"' '\"22RV1_PROSTATE\"' '\"2313287_STOMACH\"'\n",
      " '\"697_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE\"']\n",
      "\n",
      "rawRnaSeqData[0:5,0]:\n",
      "['\"\"' '\"MAP4K5 (11183)\"' '\"PAPD7 (11044)\"' '\"ZC3H11A (9877)\"'\n",
      " '\"MYH14 (79784)\"']\n",
      "\n",
      "rawRnaSeqData[0:3,0:3]:\n",
      "[['\"\"' '\"143B_BONE\"' '\"22RV1_PROSTATE\"']\n",
      " ['\"MAP4K5 (11183)\"' '1.4432662739991' '1.2328470024234']\n",
      " ['\"PAPD7 (11044)\"' '1.26557693849693' '1.22391368028673']]\n"
     ]
    }
   ],
   "source": [
    "print(\"rawRnaSeqData.shape:{}\".format(rawRnaSeqData.shape))\n",
    "print(\"rawRnaSeqData[0,0:5]:\\n{}\".format(rawRnaSeqData[0,0:5]))\n",
    "print(\"\\nrawRnaSeqData[0:5,0]:\\n{}\".format(rawRnaSeqData[0:5,0]))\n",
    "\n",
    "print(\"\\nrawRnaSeqData[0:3,0:3]:\\n{}\".format(rawRnaSeqData[0:3,0:3]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rnaSeqCellLines.shape:(645,)\n",
      "rnaSeqCellLines[0:5]\n",
      "['\"143B_BONE\"' '\"22RV1_PROSTATE\"' '\"2313287_STOMACH\"'\n",
      " '\"697_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE\"' '\"769P_KIDNEY\"']\n",
      "\n",
      " rnaSeqGeneNames.shape:(19184,)\n",
      "rnaSeqGeneNames[0:5]\n",
      "['\"MAP4K5 (11183)\"' '\"PAPD7 (11044)\"' '\"ZC3H11A (9877)\"' '\"MYH14 (79784)\"'\n",
      " '\"NUP88 (4927)\"']\n",
      "\n",
      " rnaSeqData.shape:(19184, 645)\n",
      "rnaSeqData[0:5]\n",
      "[['1.4432662739991' '1.2328470024234' '1.20384248842145' ...\n",
      "  '1.20172668065854' '1.18751226047194' '0.938953802655811']\n",
      " ['1.26557693849693' '1.22391368028673' '1.2689538389447' ...\n",
      "  '1.14762439126215' '1.09657878061321' '1.28856689126963']\n",
      " ['1.45496476968394' '1.70206442013166' '1.64423971388072' ...\n",
      "  '1.55541708571845' '1.91717301837013' '1.83327444802754']\n",
      " ['-1.32175484807296' '1.25319693365058' '1.36527284352511' ...\n",
      "  '-1.19592875111434' '1.03735477366346' '1.41274539149957']\n",
      " ['1.69737425782221' '1.51889305820117' '1.38944801835085' ...\n",
      "  '1.42935380562316' '1.11148599007619' '0.850009334029727']]\n"
     ]
    }
   ],
   "source": [
    "rnaSeqCellLines = rawRnaSeqData[0,1:]\n",
    "print(\"rnaSeqCellLines.shape:{}\".format(rnaSeqCellLines.shape))\n",
    "print(\"rnaSeqCellLines[0:5]\\n{}\".format(rnaSeqCellLines[0:5]))\n",
    "\n",
    "rnaSeqGeneNames = rawRnaSeqData[1:,0]\n",
    "print(\"\\n rnaSeqGeneNames.shape:{}\".format(rnaSeqGeneNames.shape))\n",
    "print(\"rnaSeqGeneNames[0:5]\\n{}\".format(rnaSeqGeneNames[0:5]))\n",
    "\n",
    "rnaSeqData = rawRnaSeqData[1:, 1:]\n",
    "print(\"\\n rnaSeqData.shape:{}\".format(rnaSeqData.shape))\n",
    "print(\"rnaSeqData[0:5]\\n{}\".format(rnaSeqData[0:5]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Explore D2_Achilles_gene_dep_scores.tsv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# numFeatures = 19\n",
    "# holdOutPercent = 0.40 \n",
    "# filterPercent = 0.25 \n",
    "# suffix = \"n_{}_geneFilterPercent_{}_holdOutPercent_{}\".format(numFeatures, filterPercent, holdOutPercent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def createDataPath(dataRootDir, numFeatures, filterPercent, holdOutPercent):\n",
    "#     #fmt = \"numFeatures_{}_n_{}_geneFilterPercent_{}_holdOutPercent_{}\"\n",
    "#     fmt = \"n_{}_geneFilterPercent_{}_holdOutPercent_{}\"\n",
    "#     ret = dataRootDir + fmt.format(numFeatures, filterPercent, holdOutPercent)\n",
    "#     return ret\n",
    "    \n",
    "# dataPath = createDataPath(dataRootDir, numFeatures, filterPercent, holdOutPercent)\n",
    "# print(dataPath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # quick hack dataFactory.loadAll() assume trained model files are in the same\n",
    "# # directory as the original tsv file\n",
    "# ! cp $dataPath/*.csv $dataRootDir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dataFactory = DataFactory(numFeatures)\n",
    "# RTrain, RValidation, RTest, X, Theta, Y, geneDepCellLines, geneDepNames = \\\n",
    "#     dataFactory.loadAll(geneDepDataFile, suffix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "geneDepCellLines.shape:(501,)\n",
      "geneDepCellLines[0:5]\n",
      "['\"143B_BONE\"' '\"22RV1_PROSTATE\"' '\"2313287_STOMACH\"'\n",
      " '\"697_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE\"' '\"769P_KIDNEY\"']\n",
      "\n",
      "geneDepNames.shape:(11193,)\n",
      "geneDepNames[0:5]\n",
      "['\"A1BG (1)\"' '\"NAT2 (10)\"' '\"ADA (100)\"' '\"CDH2 (1000)\"' '\"AKT3 (10000)\"']\n"
     ]
    }
   ],
   "source": [
    "rnaSeqCellLines = rawRnaSeqData[0,1:]\n",
    "print(\"geneDepCellLines.shape:{}\".format(geneDepCellLines.shape))\n",
    "print(\"geneDepCellLines[0:5]\\n{}\".format(geneDepCellLines[0:5]))\n",
    "\n",
    "rnaSeqGeneNames = rawRnaSeqData[1:,0]\n",
    "print(\"\\ngeneDepNames.shape:{}\".format(geneDepNames.shape))\n",
    "print(\"geneDepNames[0:5]\\n{}\".format(geneDepNames[0:5]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Select common cell lines and gene"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sortedCommonCellLines.shape:(486,)\n",
      "sortedCommonCellLines[0:5]:\n",
      "['\"143B_BONE\"' '\"22RV1_PROSTATE\"' '\"2313287_STOMACH\"'\n",
      " '\"697_HAEMATOPOIETIC_AND_LYMPHOID_TISSUE\"' '\"769P_KIDNEY\"']\n",
      "\n",
      "sortedCommonCellLines[-5:]:\n",
      "['\"YD38_UPPER_AERODIGESTIVE_TRACT\"' '\"YD8_UPPER_AERODIGESTIVE_TRACT\"'\n",
      " '\"YKG1_CENTRAL_NERVOUS_SYSTEM\"' '\"ZR751_BREAST\"' '\"ZR7530_BREAST\"']\n"
     ]
    }
   ],
   "source": [
    "sortedCommonCellLines = np.intersect1d(rnaSeqCellLines, geneDepCellLines)\n",
    "print(\"sortedCommonCellLines.shape:{}\".format(sortedCommonCellLines.shape))\n",
    "print(\"sortedCommonCellLines[0:5]:\\n{}\".format(sortedCommonCellLines[0:5]))\n",
    "print(\"\\nsortedCommonCellLines[-5:]:\\n{}\".format(sortedCommonCellLines[-5:]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sortedCommonGeneDepNames.shape:(10499,)\n",
      "sortedCommonGeneDepNames[0:5]:\n",
      "['\"A1BG (1)\"' '\"A1CF (29974)\"' '\"A2M (2)\"' '\"A2ML1 (144568)\"'\n",
      " '\"A4GALT (53947)\"']\n",
      "\n",
      " sortedCommonGeneDepNames[-5:]:\n",
      "['\"ZSCAN9 (7746)\"' '\"ZSWIM2 (151112)\"' '\"ZWILCH (55055)\"' '\"ZYX (7791)\"'\n",
      " '\"ZZEF1 (23140)\"']\n"
     ]
    }
   ],
   "source": [
    "sortedCommonGeneDepNames = np.intersect1d(rnaSeqGeneNames, geneDepNames)\n",
    "print(\"sortedCommonGeneDepNames.shape:{}\".format(sortedCommonGeneDepNames.shape))\n",
    "print(\"sortedCommonGeneDepNames[0:5]:\\n{}\".format(sortedCommonGeneDepNames[0:5]))\n",
    "print(\"\\n sortedCommonGeneDepNames[-5:]:\\n{}\".format(sortedCommonGeneDepNames[-5:]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test Pandas\n",
    "\n",
    "There may be a way to do this directly in numpy. I was wasting to much time hacking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rowNames.shape:(4,) rowNames:['a' 'b' 'c' 'd']\n",
      "colNames.shape:(4,) colNames:['w' 'x' 'y' 'z']\n"
     ]
    }
   ],
   "source": [
    "d = np.array([\n",
    "    [ '', 'a', 'b', 'c', 'd'    ],\n",
    "    ['w',   1,     2,    3,   4 ],\n",
    "    ['x',   5,     6,    7,   8 ],\n",
    "    ['y',   9,    10,   11,  12 ],\n",
    "    ['z',   13,   14,   15,  16 ] ]) #.astype('str')\n",
    "\n",
    "rowNames = d[0,1:]\n",
    "# print(rowNames)\n",
    "# print(rowNames.shape)\n",
    "# rowNames = rowNames.reshape(4) #np.reshape(rowNames, 4) #\n",
    "print(\"rowNames.shape:{} rowNames:{}\".format(rowNames.shape, rowNames))\n",
    "colNames = d[1:,0] # .reshape(colsName.size())\n",
    "print(\"colNames.shape:{} colNames:{}\".format(colNames.shape, colNames))\n",
    "\n",
    "# selectRows = rowNames == ['a', 'c']\n",
    "# selectRows = np.argwhere(rowNames in ['a', 'c']\n",
    "# print(\"selectRows.shape:{} selectRows:{}\".format(selectRows.shape, selectRows))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['1' '2' '3' '4']\n",
      " ['5' '6' '7' '8']\n",
      " ['9' '10' '11' '12']\n",
      " ['13' '14' '15' '16']]\n",
      "\n",
      "    a   b   c   d\n",
      "w   1   2   3   4\n",
      "x   5   6   7   8\n",
      "y   9  10  11  12\n",
      "z  13  14  15  16\n",
      "\n",
      "    a   c\n",
      "w   1   3\n",
      "z  13  15\n"
     ]
    }
   ],
   "source": [
    "cNames = d[0,1:]\n",
    "rNames = d[1:,0]\n",
    "data = d[1:,1:]\n",
    "print(data)\n",
    "print()\n",
    "df = pd.DataFrame(data, columns=cNames, index=rNames)\n",
    "print(df)\n",
    "print()\n",
    "print( df.loc[['w', 'z'], ['a', 'c']])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Select the common cell Lines and genes from the RNA Seq data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>\"143B_BONE\"</th>\n",
       "      <th>\"22RV1_PROSTATE\"</th>\n",
       "      <th>\"2313287_STOMACH\"</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>\"MAP4K5 (11183)\"</th>\n",
       "      <td>1.4432662739991</td>\n",
       "      <td>1.2328470024234</td>\n",
       "      <td>1.20384248842145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\"PAPD7 (11044)\"</th>\n",
       "      <td>1.26557693849693</td>\n",
       "      <td>1.22391368028673</td>\n",
       "      <td>1.2689538389447</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\"ZC3H11A (9877)\"</th>\n",
       "      <td>1.45496476968394</td>\n",
       "      <td>1.70206442013166</td>\n",
       "      <td>1.64423971388072</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\"MYH14 (79784)\"</th>\n",
       "      <td>-1.32175484807296</td>\n",
       "      <td>1.25319693365058</td>\n",
       "      <td>1.36527284352511</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        \"143B_BONE\"  \"22RV1_PROSTATE\" \"2313287_STOMACH\"\n",
       "\"MAP4K5 (11183)\"    1.4432662739991   1.2328470024234  1.20384248842145\n",
       "\"PAPD7 (11044)\"    1.26557693849693  1.22391368028673   1.2689538389447\n",
       "\"ZC3H11A (9877)\"   1.45496476968394  1.70206442013166  1.64423971388072\n",
       "\"MYH14 (79784)\"   -1.32175484807296  1.25319693365058  1.36527284352511"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rnaSeqDF = pd.DataFrame(rnaSeqData, \n",
    "                        columns=rnaSeqCellLines, \n",
    "                        index=rnaSeqGeneNames)\n",
    "rnaSeqDF.iloc[0:4, 0:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "commonRNASeqDF.shape:(10499, 486)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>\"143B_BONE\"</th>\n",
       "      <th>\"22RV1_PROSTATE\"</th>\n",
       "      <th>\"2313287_STOMACH\"</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>\"A1BG (1)\"</th>\n",
       "      <td>-0.3238095</td>\n",
       "      <td>-0.4397346</td>\n",
       "      <td>-2.149967</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\"A1CF (29974)\"</th>\n",
       "      <td>-2.14752</td>\n",
       "      <td>0.7524987</td>\n",
       "      <td>0.4075303</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\"A2M (2)\"</th>\n",
       "      <td>0.5297267</td>\n",
       "      <td>0.2989709</td>\n",
       "      <td>-2.054531</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               \"143B_BONE\" \"22RV1_PROSTATE\" \"2313287_STOMACH\"\n",
       "\"A1BG (1)\"      -0.3238095       -0.4397346         -2.149967\n",
       "\"A1CF (29974)\"    -2.14752        0.7524987         0.4075303\n",
       "\"A2M (2)\"        0.5297267        0.2989709         -2.054531"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "commonRNASeqDF = rnaSeqDF.loc[sortedCommonGeneDepNames, sortedCommonCellLines]\n",
    "print(\"commonRNASeqDF.shape:{}\".format(commonRNASeqDF.shape))\n",
    "commonRNASeqDF.iloc[0:3, 0:3]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Select Commone cell lines and genes from D2_Achilles_gene_dep_scores\n",
    "First imput any missing values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "imputedGeneDepData.shape:(11193, 501)\n"
     ]
    }
   ],
   "source": [
    "imputedGeneDepData = np.matmul(X, Theta.transpose())\n",
    "print(\"imputedGeneDepData.shape:{}\".format(imputedGeneDepData.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>\"143B_BONE\"</th>\n",
       "      <th>\"22RV1_PROSTATE\"</th>\n",
       "      <th>\"2313287_STOMACH\"</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>\"A1BG (1)\"</th>\n",
       "      <td>-0.033601</td>\n",
       "      <td>-0.109786</td>\n",
       "      <td>-0.114928</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\"NAT2 (10)\"</th>\n",
       "      <td>-0.039952</td>\n",
       "      <td>-0.087199</td>\n",
       "      <td>-0.077400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\"ADA (100)\"</th>\n",
       "      <td>0.089201</td>\n",
       "      <td>0.043217</td>\n",
       "      <td>0.056489</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\"CDH2 (1000)\"</th>\n",
       "      <td>-0.034279</td>\n",
       "      <td>0.061393</td>\n",
       "      <td>0.044192</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               \"143B_BONE\"  \"22RV1_PROSTATE\"  \"2313287_STOMACH\"\n",
       "\"A1BG (1)\"       -0.033601         -0.109786          -0.114928\n",
       "\"NAT2 (10)\"      -0.039952         -0.087199          -0.077400\n",
       "\"ADA (100)\"       0.089201          0.043217           0.056489\n",
       "\"CDH2 (1000)\"    -0.034279          0.061393           0.044192"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "genDepDF = pd.DataFrame(imputedGeneDepData, \n",
    "                        columns=geneDepCellLines, \n",
    "                        index=geneDepNames)\n",
    "genDepDF.iloc[0:4, 0:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "commonGeneDupDF.shape:(10499, 486)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>\"143B_BONE\"</th>\n",
       "      <th>\"22RV1_PROSTATE\"</th>\n",
       "      <th>\"2313287_STOMACH\"</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>\"A1BG (1)\"</th>\n",
       "      <td>-0.033601</td>\n",
       "      <td>-0.109786</td>\n",
       "      <td>-0.114928</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\"A1CF (29974)\"</th>\n",
       "      <td>0.130508</td>\n",
       "      <td>-0.045974</td>\n",
       "      <td>-0.042363</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\"A2M (2)\"</th>\n",
       "      <td>-0.050844</td>\n",
       "      <td>-0.082420</td>\n",
       "      <td>-0.045868</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                \"143B_BONE\"  \"22RV1_PROSTATE\"  \"2313287_STOMACH\"\n",
       "\"A1BG (1)\"        -0.033601         -0.109786          -0.114928\n",
       "\"A1CF (29974)\"     0.130508         -0.045974          -0.042363\n",
       "\"A2M (2)\"         -0.050844         -0.082420          -0.045868"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "commonGeneDepDF = genDepDF.loc[sortedCommonGeneDepNames, sortedCommonCellLines]\n",
    "print(\"commonGeneDupDF.shape:{}\".format(commonGeneDepDF.shape))\n",
    "commonGeneDepDF.iloc[0:3, 0:3]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save to disk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "autoEncoderDataDir:data/autoEncoder/\n"
     ]
    }
   ],
   "source": [
    "print( \"autoEncoderDataDir:{}\".format(autoEncoderDataDir))\n",
    "! mkdir $autoEncoderDataDir\n",
    "\n",
    "rnaSeqDFsavePath = autoEncoderDataDir + \"common_RNAseq_lRPKM_data.csv\"\n",
    "commonRNASeqDF.to_csv(rnaSeqDFsavePath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "geneDupDFsavePath = autoEncoderDataDir + \"common_D2_Achilles_gene_dep_scores.csv\"\n",
    "commonGeneDepDF.to_csv(geneDupDFsavePath)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}