Notebooks/A08_compile_final_metadata.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "## overall scripts to run\n",
    "# qsub Scripts/A08_compile_final_metadata.sub\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # again can submit single script, or run interactively\n",
    "# # for interactive mode, just need to specify working dir below\n",
    "# # then comment out the \"%%bash / cat\" lines and run python code in-notebook\n",
    "\n",
    "# envvar_needed = ['dir_proj']\n",
    "# try:\n",
    "#     os.environ['metadat_well']\n",
    "# except KeyError:\n",
    "#     envspec = pd.read_csv(\"../snmCT_parameters.env\", sep = \"=\", comment=\"#\", header = None\n",
    "#                ).set_axis(['varname', 'varpath'], axis = 1\n",
    "#                ).query('varname in @envvar_needed')\n",
    "#     for index, row in envspec.iterrows():\n",
    "#         os.environ[row[\"varname\"]] = row[\"varpath\"]\n",
    "# os.chdir(os.environ['dir_proj'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "cat > ../Scripts/A08_compile_final_metadata.sub\n",
    "\n",
    "#!/bin/bash\n",
    "#$ -cwd\n",
    "#$ -o sublogs/A08_compile_final.$JOB_ID\n",
    "#$ -j y\n",
    "#$ -N A08_compile_metadata\n",
    "#$ -l h_rt=0:30:00,h_data=4G\n",
    "#$ -hold_jid A05_compile_DNA,A07_compile_RNA\n",
    "\n",
    "\n",
    "\n",
    "echo \"Job $JOB_ID started on:   \" `hostname -s`\n",
    "echo \"Job $JOB_ID started on:   \" `date `\n",
    "echo \" \"\n",
    "\n",
    "\n",
    "\n",
    "# environment init -----------------------------------------------------------\n",
    "\n",
    ". /u/local/Modules/default/init/modules.sh # <--\n",
    "module load anaconda3 # <--\n",
    "conda activate snmCTseq # <--\n",
    "\n",
    "export $(cat snmCT_parameters.env | grep -v '^#' | xargs) # <--\n",
    "\n",
    "\n",
    "\n",
    "# run metadat scripts --------------------------------------------------------\n",
    "\n",
    "# - recommend running these interactively in python/Jupyter to check outputs\n",
    "# - have a distinct RNA/DNA script available in case one modality finishes first\n",
    "#   & the analyst wants a quick look at the data\n",
    "\n",
    "python Scripts/A08a_final_metadat_DNA.py\n",
    "python Scripts/A08b_final_metadat_RNA.py\n",
    "python Scripts/A08c_metadata_RNADNA.py\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "echo -e \"\\n\\n'A08_compile_all' completed.\\n\\n\"\n",
    "\n",
    "\n",
    "echo \"Job $JOB_ID ended on:   \" `hostname -s`\n",
    "echo \"Job $JOB_ID ended on:   \" `date `\n",
    "echo \" \""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## A08a. DNA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "cat > ../Scripts/A08a_final_metadat_DNA.py\n",
    "\n",
    "# A08b_final_metadat_DNA.py ====================================================\n",
    "# assumes no changes to script output names from A05/A07\n",
    "\n",
    "# setup ------------------------------------------------------------------------\n",
    "\n",
    "import pandas as pd\n",
    "import glob\n",
    "\n",
    "\n",
    "# load tables ------------------------------------------------------------------\n",
    "\n",
    "def read_tbl_wrapper(filepath, prefix = \"\"):\n",
    "    return(pd.read_csv(filepath, delimiter = \"\\t\", index_col = 0).add_prefix(prefix))\n",
    "\n",
    "df_trim_fastp = read_tbl_wrapper(\"Metadata/A05a_trimming.tsv\", \"Premap_\")\n",
    "\n",
    "df_bismark_pe = read_tbl_wrapper(\"Metadata/A05b_DNA_maprate_PE.tsv\", \"PE_\")\n",
    "df_bismark_se1trim = read_tbl_wrapper(\"Metadata/A05b_DNA_maprate_SE1trim.tsv\", \"SE1t_\")\n",
    "df_bismark_se2trim = read_tbl_wrapper(\"Metadata/A05b_DNA_maprate_SE2trim.tsv\", \"SE2t_\")\n",
    "df_bismark_se1unmap = read_tbl_wrapper(\"Metadata/A05b_DNA_maprate_SE1unmap.tsv\", \"SE1u_\")\n",
    "df_bismark_se2unmap = read_tbl_wrapper(\"Metadata/A05b_DNA_maprate_SE2unmap.tsv\", \"SE2u_\")\n",
    "\n",
    "df_picard_pe = read_tbl_wrapper(\"Metadata/A05c_DNA_picard_PE.tsv\", \"PE_\")\n",
    "df_picard_se = read_tbl_wrapper(\"Metadata/A05c_DNA_picard_SE.tsv\", \"SE_\")\n",
    "\n",
    "df_DNAfrac = read_tbl_wrapper(\"Metadata/A05d_DNA_global_mCfracs.tsv\", \"\")\n",
    "\n",
    "df_samstat_pe = read_tbl_wrapper(\"Metadata/A05e_DNA_samstats_PE.tsv\", \"PE_\")\n",
    "df_samstat_se = read_tbl_wrapper(\"Metadata/A05e_DNA_samstats_SE.tsv\", \"SE_\")\n",
    "\n",
    "df_coverage_1x = read_tbl_wrapper(\"Metadata/A05f_DNA_cov_chrXdivY.tsv\", \"\")\n",
    "df_coverage_sex = read_tbl_wrapper(\"Metadata/A05f_DNA_cov_percent1x.tsv\", \"\")\n",
    "\n",
    "\n",
    "# merge on rowname index (wellprefix) ------------------------------------------\n",
    "\n",
    "# SE1trim, SE2trim = singletons from trimming\n",
    "# SE1unmap, SE2unmap = singletons from PE-mapping \n",
    "\n",
    "metadata_DNA = pd.concat([df_trim_fastp,\n",
    "                         df_bismark_pe, df_bismark_se1trim, df_bismark_se2trim, df_bismark_se1unmap, df_bismark_se2unmap,\n",
    "                         df_picard_pe, df_picard_se,\n",
    "                         df_DNAfrac,\n",
    "                         df_samstat_pe, df_samstat_se,\n",
    "                         df_coverage_1x, df_coverage_sex], axis = 1)\n",
    "metadata_DNA = metadata_DNA.apply(pd.to_numeric, errors = \"coerce\")\n",
    "\n",
    "\n",
    "\n",
    "# few 'combined' PE & SE metadata stats ----------------------------------------\n",
    "# some attempts at combined/weighted metrics\n",
    "\n",
    "# notes: (i) most output above reports paired-end read-pairs/fragments versus reads, hence *2\n",
    "#        (ii) reads that are attempted to be remapped in \"stage-two\" SE mapping\n",
    "#            are subsets of \"PE_TotalReadsIn\"; thus can't simply sum(total mapped) / sum(reads in)\n",
    "\n",
    "# estimated total mapping rate (at read level) ----------------------------------------\n",
    "\n",
    "metadata_DNA['Combined_TotalReadsIn'] = \\\n",
    "    metadata_DNA['PE_TotalReadPairsIn']*2 + \\\n",
    "    metadata_DNA['SE1t_TotalReadsIn'] + \\\n",
    "    metadata_DNA['SE2t_TotalReadsIn']\n",
    "\n",
    "metadata_DNA['Combined_UniqueMappedReads'] = \\\n",
    "    metadata_DNA.filter(regex = \"UniqueMapped\", axis = 1\n",
    "    ).assign(PE_UniqueMappedPairs=metadata_DNA['PE_UniqueMappedPairs']*2\n",
    "    ).sum(axis = 1)\n",
    "\n",
    "metadata_DNA['Combined_ReadMappingRate'] = \\\n",
    "    metadata_DNA['Combined_UniqueMappedReads'] / metadata_DNA['Combined_TotalReadsIn']\n",
    "\n",
    "\n",
    "# percent of dedupe reads that pass filtering ----------------------------------------\n",
    "\n",
    "metadata_DNA['PE_Filtered_Rate'] = \\\n",
    "    metadata_DNA['PE_FilteredSeqCount'] / \\\n",
    "    (2*metadata_DNA['PE_picard_npairsin']*(1 - metadata_DNA['PE_picard_perc_dupe']))\n",
    "metadata_DNA['SE_Filtered_Rate'] = \\\n",
    "    metadata_DNA['SE_FilteredSeqCount'] / \\\n",
    "    (metadata_DNA['SE_picard_nreadsin']*(1 - metadata_DNA['SE_picard_perc_dupe']))\n",
    "metadata_DNA['Combined_Filtered_ReadCount'] = \\\n",
    "    metadata_DNA['PE_FilteredSeqCount'] + metadata_DNA['SE_FilteredSeqCount']\n",
    "metadata_DNA['Combined_Filtered_ReadPercent'] = \\\n",
    "    metadata_DNA['Combined_Filtered_ReadCount'] / \\\n",
    "    (metadata_DNA['PE_picard_npairsin']*(1 - metadata_DNA['PE_picard_perc_dupe'])*2 +\n",
    "    metadata_DNA['SE_picard_nreadsin'])\n",
    "\n",
    "\n",
    "# final total # fragments --------------------------------------------------------------\n",
    "# (whereas Combined_Filtered_ReadCount above in terms of reads)\n",
    "\n",
    "metadata_DNA['Combined_Filtered_FragmentCount'] = \\\n",
    "    metadata_DNA['PE_FilteredSeqCount']/2 + metadata_DNA['SE_FilteredSeqCount']\n",
    "\n",
    "\n",
    "# final DNA metadata -----------------------------------------------------------\n",
    "\n",
    "metadata_DNA.to_csv(\"Metadata/A08a_metadata_DNA.tsv\", sep = \"\\t\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## A08b. compile RNA metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "cat > ../Scripts/A08b_final_metadat_RNA.py\n",
    "\n",
    "# A08b_final_metadat_RNA.py ====================================================\n",
    "# assumes no changes to script output names from A06/A07\n",
    "\n",
    "# setup ------------------------------------------------------------------------\n",
    "\n",
    "import pandas as pd\n",
    "import os\n",
    "\n",
    "\n",
    "# load tables ------------------------------------------------------------------\n",
    "\n",
    "def read_tbl_wrapper(filepath, prefix = \"\"):\n",
    "    return(pd.read_csv(filepath, delimiter=\"\\t\", index_col = 0).add_prefix(prefix))\n",
    "\n",
    "if os.path.exists(\"Metadata/A07a_trimming.tsv\"):\n",
    "    df_trim_fastp = read_tbl_wrapper(\"Metadata/A07a_trimming.tsv\", \"Premap_\")\n",
    "else:\n",
    "    df_trim_fastp = read_tbl_wrapper(\"Metadata/A05a_trimming.tsv\", \"Premap_\")\n",
    "    \n",
    "\n",
    "df_star_pe = read_tbl_wrapper(\"Metadata/A07b_RNA_maprate_PE.tsv\", \"PE_\")\n",
    "df_star_se1 = read_tbl_wrapper(\"Metadata/A07b_RNA_maprate_SE1.tsv\", \"SE1_\")\n",
    "df_star_se2 = read_tbl_wrapper(\"Metadata/A07b_RNA_maprate_SE2.tsv\", \"SE2_\")\n",
    "\n",
    "df_dedupe_pe = read_tbl_wrapper(\"Metadata/A07c_RNA_picard_PE.tsv\", \"PE_\")\n",
    "df_dedupe_se1 = read_tbl_wrapper(\"Metadata/A07c_RNA_picard_SE1.tsv\", \"SE1_\")\n",
    "df_dedupe_se2 = read_tbl_wrapper(\"Metadata/A07c_RNA_picard_SE2.tsv\", \"SE2_\")\n",
    "\n",
    "df_featurecounts_gene = read_tbl_wrapper(\"Metadata/A07d_RNA_featcounts_gene.tsv\", \"Gene_\")\n",
    "df_featurecounts_exon = read_tbl_wrapper(\"Metadata/A07d_RNA_featcounts_exon.tsv\", \"Exon_\")\n",
    "\n",
    "df_samstat_pe = read_tbl_wrapper(\"Metadata/A07e_RNA_samstats_PE.tsv\", \"PE_\")\n",
    "df_samstat_se1 = read_tbl_wrapper(\"Metadata/A07e_RNA_samstats_SE1.tsv\", \"SE1_\")\n",
    "df_samstat_se2 = read_tbl_wrapper(\"Metadata/A07e_RNA_samstats_SE2.tsv\", \"SE2_\")\n",
    "\n",
    "df_picard_pe = read_tbl_wrapper(\"Metadata/A07f_RNA_picard_PE.tsv\", \"PE_\")\n",
    "df_picard_se1 = read_tbl_wrapper(\"Metadata/A07f_RNA_picard_SE1.tsv\", \"SE1_\")\n",
    "df_picard_se2 = read_tbl_wrapper(\"Metadata/A07f_RNA_picard_SE2.tsv\", \"SE2_\")\n",
    "\n",
    "\n",
    "metadata_rna = \\\n",
    "    pd.concat([df_trim_fastp,\n",
    "               df_star_pe, df_star_se1, df_star_se2,\n",
    "               df_dedupe_pe, df_dedupe_se1,  df_dedupe_se2,\n",
    "              df_featurecounts_gene, df_featurecounts_exon,\n",
    "              df_samstat_pe, df_samstat_se1, df_samstat_se2,\n",
    "              df_picard_pe, df_picard_se1, df_picard_se2],\n",
    "              axis = 1)\n",
    "metadata_rna = metadata_rna.apply(pd.to_numeric, errors='coerce')\n",
    "\n",
    "\n",
    "\n",
    "# few 'combined' PE & SE metadata stats ----------------------------------------\n",
    "# some attempts at combined/weighted metrics\n",
    "\n",
    "# notes: (i) most output above reports paired-end read-pairs/fragments versus reads, hence *2\n",
    "#       (ii) reads that are attempted to be remapped in \"stage-two\" SE mapping\n",
    "#            are subsets of \"PE_TotalReadsIn\"; thus can't simply sum(total mapped) / sum(reads in)\n",
    "#     & (iii) \"PE-mapping singletons\" can also be generated where R1 assigned DNA while R2 assigned RNA,\n",
    "#           or with STAR, R1 can map while R2 doesn't (e.g., -f 0x0048) and these are retained in PE.bam;\n",
    "#           thus # based solely on the mapper's output miscount mapping singletons as PE instead of SE\n",
    "#           & the single-end mapping & filtering rate interpretations are slightly off versus bismark\n",
    "#           [may write QC scripts based on the _annotations files to make this more accurate in the future]\n",
    "\n",
    "\n",
    "# estimated total mapping rate (at read level) ---------------------------------\n",
    "\n",
    "# total reads in approximate because uses the rounded value of % reads mapped\n",
    "# (& other caveats above); in practice better to use \"Combined_TotalReadsIn\" from mC calc in A07b\n",
    "\n",
    "metadata_rna['Combined_TotReadsIn'] = \\\n",
    "    metadata_rna['PE_NumReadsIn'] * (metadata_rna['PE_PercentReadsUniqueMapped']/100) * 2 + \\\n",
    "    metadata_rna['SE1_NumReadsIn'] + \\\n",
    "    metadata_rna['SE2_NumReadsIn']\n",
    "\n",
    "metadata_rna['Combined_NumReadsUniqueMapped'] = \\\n",
    "    metadata_rna['PE_NumReadsUniqueMapped']*2 + \\\n",
    "    metadata_rna['SE1_NumReadsUniqueMapped'] + \\\n",
    "    metadata_rna['SE2_NumReadsUniqueMapped']\n",
    "\n",
    "metadata_rna['Combined_MappingRate'] = \\\n",
    "    metadata_rna['Combined_NumReadsUniqueMapped'] / metadata_rna['Combined_TotReadsIn']\n",
    "\n",
    "# percent of mapped reads that pass filtering ----------------------------------\n",
    "# note: SE1_TotalReadsFiltered should = SE1_FilteredSeqCount\n",
    "\n",
    "metadata_rna['PE_Filtered_Rate'] = \\\n",
    "    metadata_rna['PE_FilteredSeqCount'] / \\\n",
    "    metadata_rna['PE_NumReadsUniqueMapped']\n",
    "metadata_rna['SE1_Filtered_Rate'] = \\\n",
    "    metadata_rna['SE1_FilteredSeqCount'] / \\\n",
    "    metadata_rna['SE1_NumReadsUniqueMapped']\n",
    "metadata_rna['SE2_Filtered_Rate'] = \\\n",
    "    metadata_rna['SE2_FilteredSeqCount'] / \\\n",
    "    metadata_rna['SE2_NumReadsUniqueMapped']\n",
    "\n",
    "metadata_rna['Combined_Filtered_ReadCount'] = \\\n",
    "    metadata_rna['PE_FilteredSeqCount']*2 + \\\n",
    "    metadata_rna['SE1_FilteredSeqCount'] + \\\n",
    "    metadata_rna['SE2_FilteredSeqCount']\n",
    "metadata_rna['Combined_Filtered_FragmentCount'] = \\\n",
    "    metadata_rna['PE_FilteredSeqCount'] + \\\n",
    "    metadata_rna['SE1_FilteredSeqCount'] + \\\n",
    "    metadata_rna['SE2_FilteredSeqCount']\n",
    "\n",
    "# final assigned reads/fragments -----------------------------------------------\n",
    "\n",
    "metadata_rna['Combined_AssignedReads_Exon'] = \\\n",
    "    metadata_rna['Exon_PE_Assigned']*2 + \\\n",
    "    metadata_rna['Exon_SE1_Assigned'] + \\\n",
    "    metadata_rna['Exon_SE2_Assigned']\n",
    "\n",
    "metadata_rna['Combined_AssignedFragments_Exon'] = \\\n",
    "    metadata_rna['Exon_PE_Assigned'] + \\\n",
    "    metadata_rna['Exon_SE1_Assigned'] + \\\n",
    "    metadata_rna['Exon_SE2_Assigned']\n",
    "\n",
    "metadata_rna['Combined_AssignedReads_Gene'] = \\\n",
    "    metadata_rna['Gene_PE_Assigned']*2 + \\\n",
    "    metadata_rna['Gene_SE1_Assigned'] + \\\n",
    "    metadata_rna['Gene_SE2_Assigned']\n",
    "\n",
    "metadata_rna['Combined_AssignedFragments_Gene'] = \\\n",
    "    metadata_rna['Gene_PE_Assigned'] + \\\n",
    "    metadata_rna['Gene_SE1_Assigned'] + \\\n",
    "    metadata_rna['Gene_SE2_Assigned']\n",
    "\n",
    "\n",
    "# final RNA metadata -----------------------------------------------------------\n",
    "\n",
    "metadata_rna.to_csv(\"Metadata/A08b_metadata_RNA.tsv\", sep = \"\\t\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## A07c. join DNA + RNA metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "cat > ../Scripts/A08c_metadata_RNADNA.py\n",
    "\n",
    "# A08c_metadata_RNADNA.py: compile A08a & A08b\n",
    "# this final metadata file is what's typically used for QC\n",
    "\n",
    "# setup ------------------------------------------------------------------------\n",
    "\n",
    "import glob\n",
    "import pandas as pd\n",
    "\n",
    "\n",
    "\n",
    "# read and merge ---------------------------------------------------------------\n",
    "\n",
    "def read_tbl_wrapper(filepath, prefix = \"\"):\n",
    "    return(pd.read_csv(filepath, delimiter=\"\\t\", index_col = 0).add_prefix(prefix))\n",
    "\n",
    "metadata_DNA = read_tbl_wrapper(\"Metadata/A08a_metadata_DNA.tsv\", \"DNA_\")\n",
    "metadata_RNA = read_tbl_wrapper(\"Metadata/A08b_metadata_RNA.tsv\", \"RNA_\")\n",
    "\n",
    "metadata_joined = \\\n",
    "    pd.concat([metadata_DNA,\n",
    "               metadata_RNA.drop(\n",
    "                   metadata_RNA.columns[metadata_RNA.columns.str.contains(\"Premap\")],\n",
    "                   axis = 1)],\n",
    "              axis = 1)\n",
    "\n",
    "# calculate few joint library size metrics\n",
    "metadata_joined['Joint_TotalReadCount'] = \\\n",
    "    metadata_joined[\"DNA_Combined_Filtered_ReadCount\"] + metadata_joined['RNA_Combined_Filtered_ReadCount']\n",
    "metadata_joined['Joint_PercentRead_DNA'] = \\\n",
    "    metadata_joined[\"DNA_Combined_Filtered_ReadCount\"] / metadata_joined['Joint_TotalReadCount']\n",
    "\n",
    "metadata_joined['Joint_TotalFragmentCount'] = \\\n",
    "    metadata_joined[\"DNA_Combined_Filtered_FragmentCount\"] + metadata_joined['RNA_Combined_Filtered_FragmentCount']\n",
    "metadata_joined['Joint_PercentFragment_DNA'] = \\\n",
    "    metadata_joined[\"DNA_Combined_Filtered_FragmentCount\"] / metadata_joined['Joint_TotalFragmentCount']\n",
    "\n",
    "# RNA mapping rate, excluding DNA\n",
    "metadata_joined['RNA_DNAAdj_ReadMappingRate'] = \\\n",
    "    metadata_joined['RNA_Combined_Filtered_ReadCount'] / \\\n",
    "    (metadata_joined['RNA_Combined_TotReadsIn'] - metadata_joined['DNA_Combined_Filtered_ReadCount'])\n",
    "\n",
    "# DNA mapping rate, excluding RNA\n",
    "metadata_joined['DNA_RNAAdj_ReadMappingRate'] = \\\n",
    "    metadata_joined['DNA_Combined_Filtered_ReadCount'] / \\\n",
    "    (metadata_joined['DNA_Combined_TotalReadsIn'] - metadata_joined['RNA_Combined_Filtered_ReadCount'])\n",
    "\n",
    "\n",
    "\n",
    "# final DNA + RNA metadata -----------------------------------------------------\n",
    "\n",
    "metadata_joined.to_csv(\"Metadata/A08c_metadata_RNADNA.tsv\", sep = \"\\t\")\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}