Notebooks/A05_compile_metadata_DNA.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # A05_compile_metadata_DNA overall cmds ===========================================\n",
    "\n",
    "# qsub Scripts/A05_compile_DNA_metadata.sub # ‡\n",
    "\n",
    "# # ‡ fast enough to run interactively"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## alternatively, use below to run interactively"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # for interactive mode, just need to specify working dir & 3 environment variables below\n",
    "# # then comment out the \"%%bash / cat\" lines and run python code in-notebook\n",
    "\n",
    "\n",
    "# import os\n",
    "# os.chdir('../') # move to $dir_proj\n",
    "# os.environ['metadat_well'] = \"Metadata/A01c_well_filepath.csv\"\n",
    "# os.environ['ref_chromsizes'] = \"/u/project/cluo/chliu/Genomes/IGVF_hg38_pluslambda/chromsizes.tsv\"\n",
    "\n",
    "# # alternatively, loop through \"../snmCT_parameters.env\":\n",
    "# import os\n",
    "# import pandas as pd\n",
    "\n",
    "# envvar_needed = ['dir_proj', 'metadat_well', 'ref_chromsizes']\n",
    "# try:\n",
    "#     os.environ['metadat_well']\n",
    "# except KeyError:\n",
    "#     envspec = pd.read_csv(\"../snmCT_parameters.env\", sep = \"=\", comment=\"#\", header = None\n",
    "#                ).set_axis(['varname', 'varpath'], axis = 1\n",
    "#                ).query('varname in @envvar_needed')\n",
    "#     for index, row in envspec.iterrows():\n",
    "#         os.environ[row[\"varname\"]] = row[\"varpath\"]\n",
    "# os.chdir(os.environ['dir_proj'])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## A05a. DNA+RNA: fastp trimming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "cat > ../Scripts/A05a_trimming.py\n",
    "\n",
    "# A05a_trimming.py =============================================================\n",
    "\n",
    "\n",
    "\n",
    "# setup ========================================================================\n",
    "\n",
    "import os\n",
    "import re\n",
    "import pandas as pd\n",
    "import glob\n",
    "\n",
    "filepath_wellmetadat = os.environ['metadat_well']\n",
    "metadata_well = pd.read_csv(filepath_wellmetadat)\n",
    "\n",
    "def parse_fastp_report(filepath):\n",
    "    jsonfile = pd.read_json(filepath)\n",
    "    dict_out = {\n",
    "        'nreads_pretrim' : jsonfile['summary']['before_filtering']['total_reads'],\n",
    "        'percreads_passtrim' : jsonfile['summary']['after_filtering']['total_reads'] /\n",
    "              jsonfile['summary']['before_filtering']['total_reads'],\n",
    "        'q20_pretrim' : jsonfile['summary']['before_filtering']['q30_rate'],\n",
    "        'q20_posttrim' : jsonfile['summary']['after_filtering']['q30_rate'],\n",
    "        'r1_len' : jsonfile['summary']['after_filtering']['read1_mean_length'],\n",
    "        'r2_len' : jsonfile['summary']['after_filtering']['read2_mean_length'],\n",
    "        'gc_perc' : jsonfile['summary']['after_filtering']['gc_content']}\n",
    "    return(dict_out)\n",
    "\n",
    "\n",
    "\n",
    "# gather metadata ==============================================================\n",
    "\n",
    "print(\"\\n\\nfastp .json...\")\n",
    "\n",
    "filelist = metadata_well['A03a_json_fastp']\n",
    "boolean_fileexists = [os.path.exists(f) for f in filelist]\n",
    "list_fastp = [parse_fastp_report(f) for f in filelist[boolean_fileexists]]\n",
    "df_fastp = pd.DataFrame(list_fastp,\n",
    "                        index = metadata_well['wellprefix'][boolean_fileexists])\n",
    "\n",
    "# percent files missing\n",
    "print(\"number of target files: \" + str(len(filelist)))\n",
    "print(\"fraction files missing: \")\n",
    "print(round(1 - sum(boolean_fileexists)/len(boolean_fileexists), 3))\n",
    "boolean_filemissing = [not f for f in boolean_fileexists]\n",
    "if sum(boolean_filemissing) != 0:\n",
    "    print(\"missing \" + str(sum(boolean_filemissing)) + \" files:\")\n",
    "    print(filelist[boolean_filemissing].to_string())\n",
    "\n",
    "# column QC\n",
    "print(\"number of NAs per column:\")\n",
    "print(df_fastp.isna().sum().to_string())\n",
    "\n",
    "print(\"number of duplicated wells:\")\n",
    "ndupe = df_fastp.index.duplicated().sum()\n",
    "print(ndupe)\n",
    "\n",
    "# final export\n",
    "print(\"exporting Metadata/A05a_trimming.tsv of shape: {}\".format(*df_fastp.shape))\n",
    "df_fastp.to_csv(\"Metadata/A05a_trimming.tsv\", sep = '\\t')\n",
    "print(\"\\n\\n\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## A05b. DNA: bismark mapping rates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "cat > ../Scripts/A05b_DNA_maprate.py\n",
    "\n",
    "# A05b_DNA_maprate.py ==========================================================\n",
    "\n",
    "\n",
    "\n",
    "# setup ========================================================================\n",
    "\n",
    "import os\n",
    "import glob\n",
    "import itertools\n",
    "import re\n",
    "import pandas as pd\n",
    "\n",
    "filepath_wellmetadat = os.environ['metadat_well']\n",
    "metadata_well = pd.read_csv(filepath_wellmetadat)\n",
    "\n",
    "\n",
    "def parse_bismark_report(filepath):\n",
    "\n",
    "    \"\"\"\n",
    "    parse bismark.txt output\n",
    "    adapted from YAP @ https://github.com/lhqing/cemba_data to include PE & SE output\n",
    "    commented out term_dict lines of limited interest\n",
    "    note that paired-end metrics usually yield fragments, versus reads\n",
    "    \"\"\"\n",
    "\n",
    "    term_dict = {\n",
    "        'Sequence pairs analysed in total': f'TotalReadPairsIn',\n",
    "        'Sequences analysed in total': f'TotalReadsIn',\n",
    "        'Number of paired-end alignments with a unique best hit': f'UniqueMappedPairs',\n",
    "        'Number of alignments with a unique best hit from the different alignments': f'UniqueMappedReads',\n",
    "        'Mapping efficiency': f'MappingRate',\n",
    "#         'Sequence pairs with no alignments under any condition': f'UnmappedPairs',\n",
    "#         'Sequences with no alignments under any condition': f'UnmappedReads',\n",
    "#         'Sequences did not map uniquely': f'AmbigReads',\n",
    "#         'Sequence pairs did not map uniquely': f'AmbigPairs',\n",
    "#         'CT/GA/CT': f'ReadsOT',\n",
    "#         'GA/CT/CT': f'ReadsOB',\n",
    "#         'GA/CT/GA': f'ReadsCTOT',\n",
    "#         'CT/GA/GA': f'ReadsCTOB',\n",
    "#         'CT/CT': f'ReadsOT',\n",
    "#         'CT/GA': f'ReadsOB',\n",
    "#         'GA/CT': f'ReadsCTOT',\n",
    "#         'GA/GA': f'ReadsCTOB',\n",
    "#         'Total number of C\\'s analysed': f'TotalC',\n",
    "        'C methylated in CpG context': f'BismarkmCGRate',\n",
    "        'C methylated in CHG context': f'BismarkmCHGRate',\n",
    "        'C methylated in CHH context': f'BismarkmCHHRate',\n",
    "        'C methylated in unknown context (CN or CHN)' : f'BismarkmCNCHNRate',\n",
    "        'C methylated in Unknown context (CN or CHN)' : f'BismarkmCNCHNRate'\n",
    "        }\n",
    "\n",
    "    with open(filepath) as report:\n",
    "        report_dict = {}\n",
    "        for line in report:\n",
    "            try:\n",
    "                lhs, rhs = line.split(':')\n",
    "            except ValueError:\n",
    "                continue\n",
    "            try:\n",
    "                report_dict[term_dict[lhs]] = rhs.strip().split('\\t')[0].strip('%')\n",
    "            except KeyError:\n",
    "                pass\n",
    "            \n",
    "    return(report_dict)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# gather metadata ==============================================================\n",
    "\n",
    "\n",
    "# paired-end -------------------------------------------------------------------\n",
    "\n",
    "print(\"\\n\\nPE logs...\")\n",
    "filelist = metadata_well['A04a_txt_bismark_PE']\n",
    "boolean_fileexists = [os.path.exists(f) for f in filelist]\n",
    "list_bismark_PE = [parse_bismark_report(f) for f in filelist[boolean_fileexists]]\n",
    "df_bismark_PE = pd.DataFrame(list_bismark_PE,\n",
    "                        index = metadata_well['wellprefix'][boolean_fileexists])\n",
    "\n",
    "# percent files missing\n",
    "print(\"number of target files: \" + str(len(filelist)))\n",
    "print(\"fraction files missing: \")\n",
    "print(round(1 - sum(boolean_fileexists)/len(boolean_fileexists), 3))\n",
    "boolean_filemissing = [not f for f in boolean_fileexists]\n",
    "if sum(boolean_filemissing) != 0:\n",
    "    print(\"missing \" + str(sum(boolean_filemissing)) + \" files:\")\n",
    "    print(filelist[boolean_filemissing].to_string())\n",
    "\n",
    "# column QC\n",
    "print(\"number of NAs per column:\")\n",
    "print(df_bismark_PE.isna().sum().to_string())\n",
    "\n",
    "print(\"number of duplicated wells:\")\n",
    "ndupe = df_bismark_PE.index.duplicated().sum()\n",
    "print(ndupe)\n",
    "\n",
    "# final export\n",
    "print(\"exporting Metadata/A05b_DNA_maprate_PE.tsv of shape: {}\".format(*df_bismark_PE.shape))\n",
    "df_bismark_PE.to_csv(\"Metadata/A05b_DNA_maprate_PE.tsv\", sep = '\\t')\n",
    "print(\"\\n\\n\")\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# read 1 singletons from trimming ----------------------------------------------\n",
    "\n",
    "print(\"\\n\\nSE1trim logs...\")\n",
    "filelist = metadata_well['A04a_txt_bismark_SE1trim']\n",
    "boolean_fileexists = [os.path.exists(f) for f in filelist]\n",
    "list_bismark_SE1trim = [parse_bismark_report(f) for f in filelist[boolean_fileexists]]\n",
    "df_bismark_SE1trim = pd.DataFrame(list_bismark_SE1trim,\n",
    "                        index = metadata_well['wellprefix'][boolean_fileexists])\n",
    "\n",
    "# percent files missing\n",
    "print(\"number of target files: \" + str(len(filelist)))\n",
    "print(\"fraction files missing: \")\n",
    "print(round(1 - sum(boolean_fileexists)/len(boolean_fileexists), 3))\n",
    "boolean_filemissing = [not f for f in boolean_fileexists]\n",
    "if sum(boolean_filemissing) != 0:\n",
    "    print(\"missing \" + str(sum(boolean_filemissing)) + \" files:\")\n",
    "    print(filelist[boolean_filemissing].to_string())\n",
    "\n",
    "# column QC\n",
    "print(\"number of NAs per column:\")\n",
    "print(df_bismark_SE1trim.isna().sum().to_string())\n",
    "\n",
    "print(\"number of duplicated wells:\")\n",
    "ndupe = df_bismark_SE1trim.index.duplicated().sum()\n",
    "print(ndupe)\n",
    "\n",
    "# final export\n",
    "print(\"exporting Metadata/A05b_DNA_maprate_SE1trim.tsv of shape: {}\".format(*df_bismark_SE1trim.shape))\n",
    "df_bismark_SE1trim.to_csv(\"Metadata/A05b_DNA_maprate_SE1trim.tsv\", sep = '\\t')\n",
    "print(\"\\n\\n\")\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# read 2 singletons from trimming ----------------------------------------------\n",
    "\n",
    "print(\"\\n\\nSE2trim logs...\")\n",
    "filelist = metadata_well['A04a_txt_bismark_SE2trim']\n",
    "boolean_fileexists = [os.path.exists(f) for f in filelist]\n",
    "list_bismark_SE2trim = [parse_bismark_report(f) for f in filelist[boolean_fileexists]]\n",
    "df_bismark_SE2trim = pd.DataFrame(list_bismark_SE2trim,\n",
    "                                  index = metadata_well['wellprefix'][boolean_fileexists])\n",
    "\n",
    "# percent files missing\n",
    "print(\"number of target files: \" + str(len(filelist)))\n",
    "print(\"fraction files missing: \")\n",
    "print(round(1 - sum(boolean_fileexists)/len(boolean_fileexists), 3))\n",
    "boolean_filemissing = [not f for f in boolean_fileexists]\n",
    "if sum(boolean_filemissing) != 0:\n",
    "    print(\"missing \" + str(sum(boolean_filemissing)) + \" files:\")\n",
    "    print(filelist[boolean_filemissing].to_string())\n",
    "\n",
    "# column QC\n",
    "print(\"number of NAs per column:\")\n",
    "print(df_bismark_SE2trim.isna().sum().to_string())\n",
    "\n",
    "print(\"number of duplicated wells:\")\n",
    "ndupe = df_bismark_SE2trim.index.duplicated().sum()\n",
    "print(ndupe)\n",
    "\n",
    "# final export\n",
    "print(\"exporting Metadata/A05b_DNA_maprate_SE2trim.tsv of shape: {}\".format(*df_bismark_SE2trim.shape))\n",
    "df_bismark_SE2trim.to_csv(\"Metadata/A05b_DNA_maprate_SE2trim.tsv\", sep = '\\t')\n",
    "print(\"\\n\\n\")\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# read 1 singletons unampped in paired-end mode --------------------------------\n",
    "\n",
    "print(\"\\n\\nSE1unmap logs...\")\n",
    "filelist = metadata_well['A04a_txt_bismark_SE1unmap']\n",
    "boolean_fileexists = [os.path.exists(f) for f in filelist]\n",
    "list_bismark_SE1unmap = [parse_bismark_report(f) for f in filelist[boolean_fileexists]]\n",
    "df_bismark_SE1unmap = pd.DataFrame(list_bismark_SE1unmap,\n",
    "                                   index = metadata_well['wellprefix'][boolean_fileexists])\n",
    "\n",
    "# percent files missing\n",
    "print(\"number of target files: \" + str(len(filelist)))\n",
    "print(\"fraction files missing: \")\n",
    "print(round(1 - sum(boolean_fileexists)/len(boolean_fileexists), 3))\n",
    "boolean_filemissing = [not f for f in boolean_fileexists]\n",
    "if sum(boolean_filemissing) != 0:\n",
    "    print(\"missing \" + str(sum(boolean_filemissing)) + \" files:\")\n",
    "    print(filelist[boolean_filemissing].to_string())\n",
    "\n",
    "# column QC\n",
    "print(\"number of NAs per column:\")\n",
    "print(df_bismark_SE1unmap.isna().sum().to_string())\n",
    "\n",
    "print(\"number of duplicated wells:\")\n",
    "ndupe = df_bismark_SE1unmap.index.duplicated().sum()\n",
    "print(ndupe)\n",
    "\n",
    "# final export\n",
    "print(\"exporting Metadata/A05b_DNA_maprate_SE1unmap.tsv of shape: {}\".format(*df_bismark_SE1unmap.shape))\n",
    "df_bismark_SE1unmap.to_csv(\"Metadata/A05b_DNA_maprate_SE1unmap.tsv\", sep = '\\t')\n",
    "print(\"\\n\\n\")\n",
    "\n",
    "\n",
    "\n",
    "# read 2 singletons unampped in paired-end mode --------------------------------\n",
    "\n",
    "print(\"\\n\\nSE2unmap logs...\")\n",
    "filelist = metadata_well['A04a_txt_bismark_SE2unmap']\n",
    "boolean_fileexists = [os.path.exists(f) for f in filelist]\n",
    "list_bismark_SE2unmap = [parse_bismark_report(f) for f in filelist[boolean_fileexists]]\n",
    "df_bismark_SE2unmap = pd.DataFrame(list_bismark_SE2unmap,\n",
    "                                  index = metadata_well['wellprefix'][boolean_fileexists])\n",
    "\n",
    "\n",
    "# percent files missing\n",
    "print(\"number of target files: \" + str(len(filelist)))\n",
    "print(\"fraction files missing: \")\n",
    "print(round(1 - sum(boolean_fileexists)/len(boolean_fileexists), 3))\n",
    "boolean_filemissing = [not f for f in boolean_fileexists]\n",
    "if sum(boolean_filemissing) != 0:\n",
    "    print(\"missing \" + str(sum(boolean_filemissing)) + \" files:\")\n",
    "    print(filelist[boolean_filemissing].to_string())\n",
    "\n",
    "# column QC\n",
    "print(\"number of NAs per column:\")\n",
    "print(df_bismark_SE2unmap.isna().sum().to_string())\n",
    "\n",
    "print(\"number of duplicated wells:\")\n",
    "ndupe = df_bismark_SE2unmap.index.duplicated().sum()\n",
    "print(ndupe)\n",
    "\n",
    "# final export\n",
    "print(\"exporting Metadata/A05b_DNA_maprate_SE2unmap.tsv of shape: {}\".format(*df_bismark_SE2unmap.shape))\n",
    "df_bismark_SE2unmap.to_csv(\"Metadata/A05b_DNA_maprate_SE2unmap.tsv\", sep = '\\t')\n",
    "print(\"\\n\\n\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## A05c. DNA: picard deduplication "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "cat > ../Scripts/A05c_DNA_dedupe.py\n",
    "\n",
    "# A05c_DNA_dedupe.py ===========================================================\n",
    "\n",
    "# setup ========================================================================\n",
    "\n",
    "import os\n",
    "import glob\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "filepath_wellmetadat = os.environ['metadat_well']\n",
    "metadata_well = pd.read_csv(filepath_wellmetadat)\n",
    "\n",
    "nulltable = np.array([pd.NA, pd.NA, pd.NA]) \n",
    "\n",
    "def parse_picard_dedupe(filepath):\n",
    "    try:\n",
    "        data_dedupe = pd.read_csv(filepath, delimiter = \"\\t\",\n",
    "                         comment = \"#\", nrows = 1)[[\n",
    "                             'UNPAIRED_READS_EXAMINED', 'READ_PAIRS_EXAMINED', 'PERCENT_DUPLICATION'\n",
    "                         ]].transpose()[0]\n",
    "        return(data_dedupe)\n",
    "    except:\n",
    "        print(\"error reading file: \" + filepath)\n",
    "        return(nulltable)\n",
    "\n",
    "tidy_name_dict = {'PERCENT_DUPLICATION' : 'picard_perc_dupe',\n",
    "                  'READ_PAIRS_EXAMINED' : 'picard_npairsin',\n",
    "                  'UNPAIRED_READS_EXAMINED' : 'picard_nreadsin'}\n",
    "\n",
    "\n",
    "\n",
    "# gather metadata ==============================================================\n",
    "\n",
    "# paired-end -------------------------------------------------------------------\n",
    "\n",
    "print(\"\\n\\nPE logs...\")\n",
    "filelist = metadata_well['A04b_txt_picard_PE']\n",
    "boolean_fileexists = [os.path.exists(f) for f in filelist]\n",
    "list_picard_PE = [parse_picard_dedupe(f) for f in filelist[boolean_fileexists]]\n",
    "df_picard_PE = pd.DataFrame(list_picard_PE,\n",
    "                        index = metadata_well['wellprefix'][boolean_fileexists]\n",
    "                           ).rename(columns = tidy_name_dict)\n",
    "\n",
    "# percent files missing\n",
    "print(\"number of target files: \" + str(len(filelist)))\n",
    "print(\"fraction files missing: \")\n",
    "print(round(1 - sum(boolean_fileexists)/len(boolean_fileexists), 3))\n",
    "boolean_filemissing = [not f for f in boolean_fileexists]\n",
    "if sum(boolean_filemissing) != 0:\n",
    "    print(\"missing \" + str(sum(boolean_filemissing)) + \" files:\")\n",
    "    print(filelist[boolean_filemissing].to_string())\n",
    "\n",
    "# column QC\n",
    "print(\"number of NAs per column:\")\n",
    "print(df_picard_PE.isna().sum().to_string())\n",
    "\n",
    "print(\"number of duplicated wells:\")\n",
    "ndupe = df_picard_PE.index.duplicated().sum()\n",
    "print(ndupe)\n",
    "\n",
    "# final export\n",
    "print(\"exporting Metadata/A05c_DNA_picard_PE.tsv of shape: {}\".format(*df_picard_PE.shape))\n",
    "df_picard_PE.to_csv(\"Metadata/A05c_DNA_picard_PE.tsv\", sep = '\\t')\n",
    "print(\"\\n\\n\")\n",
    "\n",
    "\n",
    "\n",
    "# single-end -------------------------------------------------------------------\n",
    "\n",
    "print(\"\\n\\nSE logs...\")\n",
    "filelist = metadata_well['A04b_txt_picard_SE']\n",
    "boolean_fileexists = [os.path.exists(f) for f in filelist]\n",
    "list_picard_SE = [parse_picard_dedupe(f) for f in filelist[boolean_fileexists]]\n",
    "df_picard_SE = pd.DataFrame(list_picard_SE,\n",
    "                            index = metadata_well['wellprefix'][boolean_fileexists]\n",
    "                            ).rename(columns = tidy_name_dict)\n",
    "\n",
    "# percent files missing\n",
    "print(\"number of target files: \" + str(len(filelist)))\n",
    "print(\"fraction files missing: \")\n",
    "print(round(1 - sum(boolean_fileexists)/len(boolean_fileexists), 3))\n",
    "boolean_filemissing = [not f for f in boolean_fileexists]\n",
    "if sum(boolean_filemissing) != 0:\n",
    "    print(\"missing \" + str(sum(boolean_filemissing)) + \" files:\")\n",
    "    print(filelist[boolean_filemissing].to_string())\n",
    "\n",
    "# column QC\n",
    "print(\"number of NAs per column:\")\n",
    "print(df_picard_SE.isna().sum().to_string())\n",
    "\n",
    "print(\"number of duplicated wells:\")\n",
    "ndupe = df_picard_SE.index.duplicated().sum()\n",
    "print(ndupe)\n",
    "\n",
    "# final export\n",
    "print(\"exporting Metadata/A05c_DNA_picard_SE.tsv of shape: {}\".format(*df_picard_SE.shape))\n",
    "df_picard_SE.to_csv(\"Metadata/A05c_DNA_picard_SE.tsv\", sep = '\\t')\n",
    "print(\"\\n\\n\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## A05d. DNA: mC fractions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "cat > ../Scripts/A05d_DNA_global_mCfracs.py\n",
    "\n",
    "# A05d_DNA_global_mCfracs.py ==========================================================\n",
    "\n",
    "# setup ------------------------------------------------------------------------\n",
    "\n",
    "import os\n",
    "import glob\n",
    "import pandas as pd\n",
    "\n",
    "filepath_wellmetadat = os.environ['metadat_well']\n",
    "metadata_well = pd.read_csv(filepath_wellmetadat)\n",
    "\n",
    "\n",
    "# gather metadata --------------------------------------------------------------\n",
    "\n",
    "filelist=pd.Series([ \"Metadata/A04d_mCfrac_\" + str(i) + \".tsv\"\n",
    "                for i in pd.unique(metadata_well['batchnum']) ])\n",
    "boolean_fileexists = [os.path.exists(f) for f in filelist]\n",
    "list_mCfracs = [ pd.read_csv(f, delimiter=\"\\t\") for f in filelist[boolean_fileexists] ] \n",
    "df_mCfracs = pd.concat(list_mCfracs)\n",
    "df_mCfracs = df_mCfracs.rename(columns = {\"Well\" : \"wellprefix\"})\n",
    "df_mCfracs = df_mCfracs.set_index(\"wellprefix\")\n",
    "\n",
    "# percent files missing\n",
    "print(\"number of target files: \" + str(len(filelist)))\n",
    "print(\"fraction files missing: \")\n",
    "print(round(1 - sum(boolean_fileexists)/len(boolean_fileexists), 3))\n",
    "boolean_filemissing = [not f for f in boolean_fileexists]\n",
    "if sum(boolean_filemissing) != 0:\n",
    "    print(\"missing \" + str(sum(boolean_filemissing)) + \" files:\")\n",
    "    print(filelist[boolean_filemissing].to_string())\n",
    "\n",
    "# column QC\n",
    "print(\"number of NAs per column:\")\n",
    "print(df_mCfracs.isna().sum().to_string())\n",
    "\n",
    "print(\"number of duplicated wells:\")\n",
    "ndupe = df_mCfracs.index.duplicated().sum()\n",
    "print(ndupe)\n",
    "\n",
    "# final export\n",
    "print(\"exporting Metadata/A05d_DNA_global_mCfracs.tsv of shape: {}\".format(*df_mCfracs.shape))\n",
    "df_mCfracs.to_csv(\"Metadata/A05d_DNA_global_mCfracs.tsv\", sep = '\\t')\n",
    "print(\"\\n\\n\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## A05e. DNA: samtools stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "cat > ../Scripts/A05e_DNA_samtools.py\n",
    "\n",
    "# A05e_DNA_samtools.py =========================================================\n",
    "\n",
    "# setup ------------------------------------------------------------------------\n",
    "\n",
    "import os\n",
    "import glob\n",
    "import pandas as pd\n",
    "\n",
    "filepath_wellmetadat = os.environ['metadat_well']\n",
    "metadata_well = pd.read_csv(filepath_wellmetadat)\n",
    "\n",
    "def parse_samstats(filepath):\n",
    "\n",
    "    term_dict = {\n",
    "        'raw total sequences': f'FilteredSeqCount',\n",
    "        'error rate': f'ErrorRate',\n",
    "        'insert size average': f'InsertSizeAvg',\n",
    "        'insert size standard deviation': f'InsertSizeSD',\n",
    "        }\n",
    "\n",
    "    with open(filepath) as report:\n",
    "        report_dict = {}\n",
    "        for line in report:\n",
    "            try:\n",
    "                lhs, rhs = line.split(':')\n",
    "            except ValueError:\n",
    "                continue\n",
    "            try:\n",
    "                report_dict[term_dict[lhs]] = rhs.strip().split('\\t')[0]\n",
    "            except KeyError:\n",
    "                pass\n",
    "            \n",
    "    return(report_dict)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# gather metadata --------------------------------------------------------------\n",
    "\n",
    "# paired-end -------------------------------------------------------------------\n",
    "\n",
    "print(\"\\n\\npaired-end...\")\n",
    "filelist = metadata_well[\"A04e_txt_samstats_PE\"]\n",
    "boolean_fileexists = [os.path.exists(f) for f in filelist]\n",
    "list_samstats_PE = [ parse_samstats(f) for f in filelist[boolean_fileexists] ] \n",
    "df_samstats_PE = pd.DataFrame(list_samstats_PE,\n",
    "                            index = metadata_well['wellprefix'][boolean_fileexists])\n",
    "\n",
    "# percent files missing\n",
    "print(\"number of target files: \" + str(len(filelist)))\n",
    "print(\"fraction files missing: \")\n",
    "print(round(1 - sum(boolean_fileexists)/len(boolean_fileexists), 3))\n",
    "boolean_filemissing = [not f for f in boolean_fileexists]\n",
    "if sum(boolean_filemissing) != 0:\n",
    "    print(\"missing \" + str(sum(boolean_filemissing)) + \" files:\")\n",
    "    print(filelist[boolean_filemissing].to_string())\n",
    "\n",
    "# column QC\n",
    "print(\"number of NAs per column:\")\n",
    "print(df_samstats_PE.isna().sum().to_string())\n",
    "\n",
    "print(\"number of duplicated wells:\")\n",
    "ndupe = df_samstats_PE.index.duplicated().sum()\n",
    "print(ndupe)\n",
    "\n",
    "# final export\n",
    "print(\"exporting Metadata/A05e_DNA_samstats_PE.tsv of shape: {}\".format(*df_samstats_PE.shape))\n",
    "df_samstats_PE.to_csv(\"Metadata/A05e_DNA_samstats_PE.tsv\", sep = '\\t')\n",
    "print(\"\\n\\n\")\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# single-end -------------------------------------------------------------------\n",
    "\n",
    "print(\"\\n\\nsingle-end...\")\n",
    "filelist = metadata_well[\"A04e_txt_samstats_SE\"]\n",
    "boolean_fileexists = [os.path.exists(f) for f in filelist]\n",
    "list_samstats_SE = [ parse_samstats(f) for f in filelist[boolean_fileexists] ] \n",
    "df_samstats_SE = pd.DataFrame(list_samstats_SE,\n",
    "                            index = metadata_well['wellprefix'][boolean_fileexists]\n",
    "                               ).drop(['InsertSizeAvg', 'InsertSizeSD'], axis = 1)\n",
    "\n",
    "\n",
    "# percent files missing\n",
    "print(\"number of target files: \" + str(len(filelist)))\n",
    "print(\"fraction files missing: \")\n",
    "print(round(1 - sum(boolean_fileexists)/len(boolean_fileexists), 3))\n",
    "boolean_filemissing = [not f for f in boolean_fileexists]\n",
    "if sum(boolean_filemissing) != 0:\n",
    "    print(\"missing \" + str(sum(boolean_filemissing)) + \" files:\")\n",
    "    print(filelist[boolean_filemissing].to_string())\n",
    "\n",
    "# column QC\n",
    "print(\"number of NAs per column:\")\n",
    "print(df_samstats_SE.isna().sum().to_string())\n",
    "\n",
    "print(\"number of duplicated wells:\")\n",
    "ndupe = df_samstats_SE.index.duplicated().sum()\n",
    "print(ndupe)\n",
    "\n",
    "# final export\n",
    "print(\"exporting Metadata/A05e_DNA_samstats_SE.tsv of shape: {}\".format(*df_samstats_SE.shape))\n",
    "df_samstats_SE.to_csv(\"Metadata/A05e_DNA_samstats_SE.tsv\", sep = '\\t')\n",
    "print(\"\\n\\n\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## A05f. DNA Coverage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "cat > ../Scripts/A05f_DNA_cov.py\n",
    "\n",
    "# A05f_DNA_cov.py ==============================================================\n",
    "\n",
    "# setup ------------------------------------------------------------------------\n",
    "\n",
    "import os\n",
    "import glob\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "filepath_wellmetadat = os.environ['metadat_well']\n",
    "metadata_well = pd.read_csv(filepath_wellmetadat)\n",
    "\n",
    "target_chroms = [\"chr\" + str(i) for i in range(1, 99)]\n",
    "total_autosomal_bases = \\\n",
    "    pd.read_csv(os.environ['ref_chromsizes'],\n",
    "                sep = \"\\t\", header = None, index_col = 0)\n",
    "total_autosomal_bases = \\\n",
    "    total_autosomal_bases.loc[np.intersect1d(target_chroms, total_autosomal_bases.index), 1].sum()\n",
    "\n",
    "\n",
    "# gather metadata --------------------------------------------------------------\n",
    "\n",
    "\n",
    "# extract autosomal ------------------------------------------------------------\n",
    "\n",
    "target_chroms = [\"chr\" + str(i) for i in range(1, 99)]\n",
    "autosomal_chroms = \\\n",
    "    pd.read_csv(os.environ['ref_chromsizes'],\n",
    "                sep = \"\\t\", header = None, index_col = 0)\n",
    "autosomal_chroms = autosomal_chroms[autosomal_chroms.index.isin(target_chroms)]\n",
    "total_autosomal_bases = autosomal_chroms.sum()\n",
    "target_chroms = autosomal_chroms.index\n",
    "\n",
    "\n",
    "\n",
    "# gather metadata: base-lvl unique coverage levels -----------------------------\n",
    "\n",
    "print(\"processing autosomal num sites with at least 1-fold coverage.\")\n",
    "print(\"if any filenames printed below, potentially corrupt files:\")\n",
    "def parse_coverage_unique(filepath):\n",
    "    try:\n",
    "        percent_coverage = \\\n",
    "            pd.read_csv(filepath, delimiter = \"\\s+\", header = None, index_col=1)\n",
    "        percent_coverage = (\n",
    "            percent_coverage.loc[\n",
    "                np.intersect1d(target_chroms, percent_coverage.index), 0\n",
    "            ].sum() / total_autosomal_bases).to_list()[0]\n",
    "    except:\n",
    "        print(\"'\" + filepath + \"'\")\n",
    "        percent_coverage = np.nan\n",
    "    return(percent_coverage)\n",
    "\n",
    "filelist = metadata_well['A04f_txt_covnsites']\n",
    "boolean_fileexists = [os.path.exists(f) for f in filelist]\n",
    "list_unique = [parse_coverage_unique(file) for file in filelist[boolean_fileexists]]\n",
    "df_unique = pd.DataFrame(list_unique,\n",
    "                        index = metadata_well['wellprefix'][boolean_fileexists])\n",
    "df_unique.columns = [\"CoveragePerc1x\"]\n",
    "\n",
    "\n",
    "# percent files missing\n",
    "print(\"number of target files: \" + str(len(filelist)))\n",
    "print(\"fraction files missing: \")\n",
    "print(round(1 - sum(boolean_fileexists)/len(boolean_fileexists), 3))\n",
    "boolean_filemissing = [not f for f in boolean_fileexists]\n",
    "if sum(boolean_filemissing) != 0:\n",
    "    print(\"missing \" + str(sum(boolean_filemissing)) + \" files:\")\n",
    "    print(filelist[boolean_filemissing].to_string())\n",
    "\n",
    "# column QC\n",
    "print(\"number of NAs per column:\")\n",
    "print(df_unique.isna().sum().to_string())\n",
    "\n",
    "print(\"number of duplicated wells:\")\n",
    "ndupe = df_unique.index.duplicated().sum()\n",
    "print(ndupe)\n",
    "\n",
    "# final export\n",
    "print(\"exporting Metadata/A05f_DNA_cov_percent1x.tsv of shape: {}\".format(*df_unique.shape))\n",
    "df_unique.to_csv(\"Metadata/A05f_DNA_cov_percent1x.tsv\", sep = '\\t')\n",
    "print(\"\\n\\n\")\n",
    "\n",
    "\n",
    "\n",
    "# total coverage levels for chrX/chrY ------------------------------------------\n",
    "\n",
    "print(\"processing total coverage levels per chrom.\")\n",
    "print(\"if any filenames printed below, potentially corrupt files:\")\n",
    "def parse_coverage_total(filepath):\n",
    "    try:\n",
    "        total_cov_by_chr = pd.read_csv(filepath, delimiter = \"\\s+\", header = None, index_col=0)\n",
    "        if not(any(total_cov_by_chr.index==\"chrX\")) and (not any(total_cov_by_chr.index==\"chrY\")):\n",
    "            coverage_XdivY = np.nan\n",
    "        elif any(total_cov_by_chr.index==\"chrX\") and (not any(total_cov_by_chr.index==\"chrY\")):\n",
    "            coverage_XdivY = np.inf\n",
    "        else:\n",
    "            coverage_XdivY = total_cov_by_chr.loc['chrX', ] / total_cov_by_chr.loc['chrY', ]\n",
    "            coverage_XdivY = coverage_XdivY.tolist()[0]\n",
    "    except:\n",
    "        print(\"'\" + filepath + \"'\")\n",
    "        coverage_XdivY = np.nan\n",
    "    return(coverage_XdivY)\n",
    "\n",
    "filelist = metadata_well['A04f_txt_covtot']\n",
    "boolean_fileexists = [os.path.exists(f) for f in filelist]\n",
    "list_total = [parse_coverage_total(file) for file in filelist[boolean_fileexists]]\n",
    "df_total = pd.DataFrame(list_total,\n",
    "             index = metadata_well['wellprefix'][boolean_fileexists])\n",
    "df_total.columns = [\"CoverageXdivY\"]\n",
    "\n",
    "\n",
    "# percent files missing\n",
    "print(\"number of target files: \" + str(len(filelist)))\n",
    "print(\"fraction files missing: \")\n",
    "print(round(1 - sum(boolean_fileexists)/len(boolean_fileexists), 3))\n",
    "boolean_filemissing = [not f for f in boolean_fileexists]\n",
    "if sum(boolean_filemissing) != 0:\n",
    "    print(\"missing \" + str(sum(boolean_filemissing)) + \" files:\")\n",
    "    print(filelist[boolean_filemissing].to_string())\n",
    "\n",
    "# column QC\n",
    "print(\"number of NAs per column:\")\n",
    "print(df_total.isna().sum().to_string())\n",
    "\n",
    "print(\"number of duplicated wells:\")\n",
    "ndupe = df_total.index.duplicated().sum()\n",
    "print(ndupe)\n",
    "\n",
    "# final export\n",
    "print(\"exporting Metadata/A05f_DNA_cov_chrXdivY.tsv of shape: {}\".format(*df_total.shape))\n",
    "df_total.to_csv(\"Metadata/A05f_DNA_cov_chrXdivY.tsv\", sep = '\\t')\n",
    "print(\"\\n\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## A05. run helper script"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "cat > ../Scripts/A05_compile_DNA_metadata.sub\n",
    "\n",
    "#!/bin/bash\n",
    "#$ -cwd\n",
    "#$ -o sublogs/A05_compile_DNA.$JOB_ID.$TASK_ID\n",
    "#$ -j y\n",
    "#$ -l h_rt=6:00:00,h_data=12G\n",
    "#$ -N A05_compile_DNA\n",
    "#$ -t 1-6\n",
    "#$ -hold_jid A04a_bismark,A04b_filter_mC,A04c_make_allc,A04d_mCfracs,A04f_global_mC_stats,A04g_coverage_DNA\n",
    "\n",
    "\n",
    "\n",
    "echo \"Job $JOB_ID.$SGE_TASK_ID started on:   \" `hostname -s`\n",
    "echo \"Job $JOB_ID.$SGE_TASK_ID started on:   \" `date `\n",
    "echo \" \"\n",
    "\n",
    "\n",
    "\n",
    "# environment init -------------------------------------------------------------\n",
    "\n",
    ". /u/local/Modules/default/init/modules.sh # <--\n",
    "module load anaconda3 # <--\n",
    "conda activate snmCTseq # <--\n",
    "\n",
    "export $(cat snmCT_parameters.env | grep -v '^#' | xargs) # <--\n",
    "\n",
    "\n",
    "\n",
    "# run each helper script (A05*) ------------------------------------------------\n",
    "\n",
    "# note: in practice these can each be submitted interactively/as its own task,\n",
    "# as some of these scripts should be much lower resource than others;\n",
    "# however, this -t 1-6 job parallelization is just for tidyness\n",
    "\n",
    "echo \"metadata script # $SGE_TASK_ID running:\"\n",
    "\n",
    "case $SGE_TASK_ID in\n",
    "\n",
    "  1)\n",
    "    echo \"python Scripts/A05a_trimming.py\"\n",
    "    python Scripts/A05a_trimming.py\n",
    "    ;;\n",
    "\n",
    "  2)\n",
    "    echo \"python Scripts/A05b_DNA_maprate.py\"\n",
    "    python Scripts/A05b_DNA_maprate.py\n",
    "    ;;\n",
    "\n",
    "  3)\n",
    "    echo \"python Scripts/A05c_DNA_dedupe.py\"\n",
    "    python Scripts/A05c_DNA_dedupe.py\n",
    "    ;;\n",
    "\n",
    "  4)\n",
    "    echo \"python Scripts/A05d_DNA_global_mCfracs.py\"\n",
    "    python Scripts/A05d_DNA_global_mCfracs.py\n",
    "    ;;\n",
    "\n",
    "  5)\n",
    "    echo \"python Scripts/A05e_DNA_samtools.py\"\n",
    "    python Scripts/A05e_DNA_samtools.py\n",
    "    ;;\n",
    "\n",
    "  6)\n",
    "    echo \"python Scripts/A05f_DNA_cov.py\"\n",
    "    python Scripts/A05f_DNA_cov.py\n",
    "    ;;\n",
    "    \n",
    "  *)\n",
    "    ;;\n",
    "esac\n",
    "\n",
    "\n",
    "echo \"completed 'A05_compile_DNA_metadata.'\"\n",
    "\n",
    "echo \" \"\n",
    "echo \"Job $JOB_ID.$SGE_TASK_ID ended on:   \" `hostname -s`\n",
    "echo \"Job $JOB_ID.$SGE_TASK_ID ended on:   \" `date `\n",
    "echo \" \"\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}