ATACseq Pipeline QC-Filtering.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ATACseq pipeline QC filtering"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This jupyter-notebook filters out the BAM files that falls below `LIBRARY_SIZE_THRESHOLD`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import subprocess, os, sys, signal, pip\n",
    "from os import listdir\n",
    "from os.path import isfile, join\n",
    "import multiprocessing\n",
    "import threading\n",
    "import time\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setting the folders and tools"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First we configure the __Folders__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Setting the output folder where all the results would be stored\n",
    "outputFolder = '../../Buenrostro_pipeline_MACS_Improved_MultiThread_WithTrimmo/Human_data_analysis/output/'\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Filtering low Library sized BAM files\n",
    "We filter out the cells for which the library size as estimated by picard is $<LIBRARY\\_SIZE\\_THRESHOLD$. We look at the stats generated during the duplicate check stat and decide based on the parameter value `ESTIMATED_LIBRARY_SIZE`. By default $LIBRARY\\_SIZE\\_THRESHOLD = 10000$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Filtering the BAM files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "LIBRARY_SIZE_THRESHOLD = 10000\n",
    "\n",
    "NoDupStatPath = outputFolder + 'Duplicates_removed'\n",
    "FilteredFilePath = outputFolder + 'Bam_Files_Filtered_on_Library_Size/'\n",
    "for root, folders, files in os.walk(NoDupStatPath):\n",
    "    files = [os.path.join(root, f) for f in files if (f.endswith('nodup_stats'))]\n",
    "LibrarySize = 0\n",
    "for f in files:\n",
    "    with open(f,'rt') as DupStatFile:\n",
    "        FLAG = 0\n",
    "        for line in DupStatFile:\n",
    "            if 'ESTIMATED_LIBRARY_SIZE' in line:\n",
    "                FLAG = 1\n",
    "            elif FLAG == 1:\n",
    "                LibrarySize = float(line.rsplit(None, 1)[-1])\n",
    "                FLAG = 0\n",
    "        if LibrarySize > LIBRARY_SIZE_THRESHOLD:        \n",
    "            TempCellName = os.path.basename(f).split('_')\n",
    "            CellName = TempCellName[0]\n",
    "            BamFiles = [join(NoDupStatPath, f) for f in listdir(NoDupStatPath) if (f.startswith(CellName) & f.endswith('nodup_sorted_cleaned.bam'))]\n",
    "            CopyFilteredCmd = 'cp ' + BamFiles[0] + ' ' + FilteredFilePath\n",
    "            subprocess.call(CopyFilteredCmd, shell=True)\n",
    "            BamFiles = [join(NoDupStatPath, f) for f in listdir(NoDupStatPath) if (f.startswith(CellName) & f.endswith('nodup_sorted_cleaned.bam.bai'))]\n",
    "            CopyFilteredCmd = 'cp ' + BamFiles[0] + ' ' + FilteredFilePath\n",
    "            subprocess.call(CopyFilteredCmd, shell=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Filtering the macs2 files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "LIBRARY_SIZE_THRESHOLD = 10000\n",
    "\n",
    "NoDupStatPath = outputFolder + 'Duplicates_removed'\n",
    "FilteredFilePath = outputFolder + 'Filtered_Macs2_files/'\n",
    "Macs2Files = outputFolder + 'Macs2_files'\n",
    "for root, folders, files in os.walk(NoDupStatPath):\n",
    "    files = [os.path.join(root, f) for f in files if (f.endswith('nodup_stats'))]\n",
    "LibrarySize = 0\n",
    "for f in files:\n",
    "    with open(f,'rt') as DupStatFile:\n",
    "        FLAG = 0\n",
    "        for line in DupStatFile:\n",
    "            if 'ESTIMATED_LIBRARY_SIZE' in line:\n",
    "                FLAG = 1\n",
    "            elif FLAG == 1:\n",
    "                LibrarySize = float(line.rsplit(None, 1)[-1])\n",
    "                FLAG = 0\n",
    "        if LibrarySize > LIBRARY_SIZE_THRESHOLD:        \n",
    "            TempCellName = os.path.basename(f).split('_')\n",
    "            CellName = TempCellName[0]\n",
    "            MacsFiles = [join(Macs2Files, f) for f in listdir(Macs2Files) if (f.startswith(CellName) & f.endswith('control_lambda.bdg'))]\n",
    "            CopyFilteredCmd = 'cp ' + MacsFiles[0] + ' ' + FilteredFilePath\n",
    "            subprocess.call(CopyFilteredCmd, shell=True)\n",
    "            MacsFiles = [join(Macs2Files, f) for f in listdir(Macs2Files) if (f.startswith(CellName) & f.endswith('peaks.narrowPeak'))]\n",
    "            CopyFilteredCmd = 'cp ' + MacsFiles[0] + ' ' + FilteredFilePath\n",
    "            subprocess.call(CopyFilteredCmd, shell=True)\n",
    "            MacsFiles = [join(Macs2Files, f) for f in listdir(Macs2Files) if (f.startswith(CellName) & f.endswith('peaks.xls'))]\n",
    "            CopyFilteredCmd = 'cp ' + MacsFiles[0] + ' ' + FilteredFilePath\n",
    "            subprocess.call(CopyFilteredCmd, shell=True)\n",
    "            MacsFiles = [join(Macs2Files, f) for f in listdir(Macs2Files) if (f.startswith(CellName) & f.endswith('summits.bed'))]\n",
    "            CopyFilteredCmd = 'cp ' + MacsFiles[0] + ' ' + FilteredFilePath            \n",
    "            subprocess.call(CopyFilteredCmd, shell=True)\n",
    "            #MacsFiles = [join(Macs2Files, f) for f in listdir(Macs2Files) if (f.startswith(CellName) & f.endswith('summits_shifted.bed'))]\n",
    "            #CopyFilteredCmd = 'cp ' + MacsFiles[0] + ' ' + FilteredFilePath            \n",
    "            #subprocess.call(CopyFilteredCmd, shell=True)\n",
    "            MacsFiles = [join(Macs2Files, f) for f in listdir(Macs2Files) if (f.startswith(CellName) & f.endswith('treat_pileup.bdg'))]\n",
    "            CopyFilteredCmd = 'cp ' + MacsFiles[0] + ' ' + FilteredFilePath            \n",
    "            subprocess.call(CopyFilteredCmd, shell=True)\n",
    "            MacsFiles = [join(Macs2Files, f) for f in listdir(Macs2Files) if (f.startswith(CellName) & f.endswith('treat_pileup.bw'))]\n",
    "            CopyFilteredCmd = 'cp ' + MacsFiles[0] + ' ' + FilteredFilePath            \n",
    "            subprocess.call(CopyFilteredCmd, shell=True)          "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}