Notebooks/A01_mergefastq_preptargets.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ## A01_mergefastq_preptargets overall commands\n",
    "\n",
    "# qsub Scripts/A01a_merge_lanes.sub # *\n",
    "# qsub Scripts/A01b_plate_metadata.sub # ‡\n",
    "\n",
    "\n",
    "# # * = job array based on \"platenum\"\n",
    "# # † = job array based on \"batchnum\" (two rows at a time)\n",
    "# # ‡ fast enough to run interactively\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# # if not done from script A00, setup project directory:\n",
    "# dir_proj=/u/project/cluo/chliu/Analyses/IGVF_iPSC_snmCTseq_YZCL51\n",
    "# mkdir $dir_proj; cd $dir_proj\n",
    "# mkdir fastq_demultip fastq_raw fastq_trimmed mapping_bismark mapping_star\n",
    "# mkdir Metadata Notebooks Scripts sublogs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "# before proceeding, also check naming convention of\n",
    "# the raw .fastq files in $dir_originalfastq: \n",
    "# example shown for a 32-plate IGVF pilot experiment\n",
    "\n",
    "dir_originalfastq=/u/project/cluo/Shared_Datasets/source_fastq/yzcl51/\n",
    "\n",
    "# 128 .fastq files --> 64 read pairs (R1 and R2)\n",
    "# 64/4 = 16 plates\n",
    "echo -e \"number of .fastq.gz files\"\n",
    "ls $dir_originalfastq/*.fastq.gz | wc -l\n",
    "echo -e \"\\n\\n\"\n",
    "\n",
    "# print .fastq.gz examples names\n",
    "echo -e \"example .fastq.gz names\"\n",
    "ls $dir_originalfastq/ | head\n",
    "echo -e \"\\n\\n\"\n",
    "\n",
    "# print unique plate names, num lanes/plate (usually 4, but new runs may have L001 to L008)\n",
    "# our lab's convention is date-project-platemetadata-plateindexid\n",
    "# (check that this final lane-merged file is unique for each plate!)\n",
    "echo -e \"Nlanes\\tplatename\"\n",
    "for fastqfile in $dir_originalfastq/*R1*;\n",
    "do\n",
    "    echo $(basename ${fastqfile%_L00[1-8]_*});\n",
    "done | uniq -c\n",
    "\n",
    "echo -e \"\\nNplates:\"\n",
    "for fastqfile in $dir_originalfastq/*R1*;\n",
    "do\n",
    "    echo $(basename ${fastqfile%_L00[1-8]_*});\n",
    "done | uniq -c | wc -l"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## (A01a) merge .fastq.gz by lane"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "cat > ../Scripts/A01a_merge_lanes.sub\n",
    "\n",
    "#!/bin/bash\n",
    "#$ -cwd\n",
    "#$ -o sublogs/A01a_merge_lanes.$JOB_ID.$TASK_ID\n",
    "#$ -j y\n",
    "#$ -l h_rt=8:00:00,h_data=16G\n",
    "#$ -N A01a_merge_lanes\n",
    "#$ -t 1-32\n",
    "\n",
    "\n",
    "\n",
    "echo \"Job $JOB_ID.$SGE_TASK_ID started on:   \" `hostname -s`\n",
    "echo \"Job $JOB_ID.$SGE_TASK_ID started on:   \" `date `\n",
    "echo \" \"\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# environment init -------------------------------------------------------------\n",
    "\n",
    ". /u/local/Modules/default/init/modules.sh # <--\n",
    "\n",
    "export $(cat snmCT_parameters.env | grep -v '^#' | xargs)  # <--\n",
    "\n",
    "\n",
    "\n",
    "# get list of plates, files ----------------------------------------------------\n",
    "\n",
    "if [[ ! -s fastq_raw ]]\n",
    "then\n",
    "    mkdir fastq_raw\n",
    "fi\n",
    "\n",
    "list_of_plates=(\n",
    "  $(for plateid in ${dir_originalfastq}/*R1*;\n",
    "    do\n",
    "    echo $(basename ${plateid%_L00[1-8]_*});\n",
    "    done | uniq | sort))\n",
    "target_plate=${list_of_plates[${SGE_TASK_ID} - 1]}\n",
    "\n",
    "\n",
    "# print array task and plate name\n",
    "# make sure $target_plate is uniquely identifiable &\n",
    "# doesn't group more than the four lanes typically excepected\n",
    "echo -e \"\\n\\ntarget plate number (SGE_TASK_ID):\" ${SGE_TASK_ID}\n",
    "echo \"target plate prefix:\" ${target_plate}\n",
    "\n",
    "\n",
    "\n",
    "# merge R1, then R2 files across lanes -----------------------------------------\n",
    "\n",
    "filesin_r1=($(ls ${dir_originalfastq}/*${target_plate}*R1*fastq.gz))\n",
    "filesin_r2=($(ls ${dir_originalfastq}/*${target_plate}*R2*fastq.gz))\n",
    "\n",
    "echo -e \"\\n\\nmerging Read 1 files:\"\n",
    "for file in ${filesin_r1[@]}\n",
    "do \n",
    "    du -h ${file}\n",
    "done\n",
    "cat ${filesin_r1[@]} > fastq_raw/${target_plate}_R1.fastq.gz\n",
    "\n",
    "echo -e \"\\n\\nmerging Read 2 files:\"\n",
    "for file in ${filesin_r2[@]}\n",
    "do \n",
    "    du -h ${file}\n",
    "done\n",
    "cat ${filesin_r2[@]} > fastq_raw/${target_plate}_R2.fastq.gz\n",
    "\n",
    "\n",
    "\n",
    "# check output files -----------------------------------------------------------\n",
    "\n",
    "echo -e \"\\n\\nchecking output file sizes.\"\n",
    "du -h fastq_raw/${target_plate}*fastq.gz\n",
    "\n",
    "echo -e \"\\n\\n'A01a_merge_lanes' completed.\\n\\n\"\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "echo \"Job $JOB_ID.$SGE_TASK_ID ended on:   \" `hostname -s`\n",
    "echo \"Job $JOB_ID.$SGE_TASK_ID ended on:   \" `date `\n",
    "echo \" \""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## (A01b) parse plate metadata "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "cat > ../Scripts/A01b_plate_metadata.sub\n",
    "\n",
    "#!/bin/bash\n",
    "#$ -cwd\n",
    "#$ -o sublogs/A01b_plate_metadata.$JOB_ID\n",
    "#$ -j y\n",
    "#$ -N A01b_plate_metadata\n",
    "#$ -l h_rt=0:15:00,h_data=8G\n",
    "#$ -hold_jid A01a_merge_lanes\n",
    "\n",
    "\n",
    "echo \"Job $JOB_ID started on:   \" `hostname -s`\n",
    "echo \"Job $JOB_ID started on:   \" `date `\n",
    "echo \" \"\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# environment init -------------------------------------------------------------\n",
    "\n",
    ". /u/local/Modules/default/init/modules.sh # <--\n",
    "module load anaconda3 # <--\n",
    "conda activate snmCTseq # <--\n",
    "\n",
    "export $(cat snmCT_parameters.env | grep -v '^#' | xargs) # <--\n",
    "\n",
    "\n",
    "\n",
    "# run metadata compilation -----------------------------------------------------\n",
    "\n",
    "# because the two scripts are so fast,\n",
    "# violating tidy convention and just running both here\n",
    "# (suggest running these in interactive mode anyway)\n",
    "\n",
    "python Scripts/A01b_plate_metadata.py # <-- * may need to be customized!!\n",
    "python Scripts/A01c_well_filepaths.py # <--\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "echo -e \"\\n\\n'A01b_plate_metadata' completed.\\n\\n\"\n",
    "\n",
    "\n",
    "\n",
    "echo \"Job $JOB_ID ended on:   \" `hostname -s`\n",
    "echo \"Job $JOB_ID ended on:   \" `date `\n",
    "echo \" \""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "cat > ../Scripts/A01b_plate_metadata.py\n",
    "\n",
    "\n",
    "\n",
    "# ==============================================================================\n",
    "# Scripts/A01b_plate_metadata.py\n",
    "# should parse list of lane-merged plates -->\n",
    "# extract plate-level metadata saved to $dir_proj/Metadata\n",
    "# ==============================================================================\n",
    "\n",
    "# recommend running interactively in python/Jupyter to check outputs,\n",
    "# the relevant metadata parameters very likely to change between studies\n",
    "\n",
    "\n",
    "\n",
    "# load packages ----------------------------------------------------------------\n",
    "\n",
    "import glob\n",
    "import sys\n",
    "import pandas as pd\n",
    "import os\n",
    "\n",
    "# # if running interactively, need to load some lines from snmCT_parameters.env\n",
    "# # or manually spec os.environ -- e.g., via os.environ['dir_proj'] = \"mydirectory\" or this below loop\n",
    "# # (check relative path of parameters.env file or change to absolute if below not working!)\n",
    "# envvar_needed = ['dir_proj', 'dir_originalfastq', 'metadat_plate']\n",
    "# try:\n",
    "#     os.environ['dir_proj']\n",
    "# except KeyError:\n",
    "#     envspec = pd.read_csv(\"../snmCT_parameters.env\", sep = \"=\", comment=\"#\", header = None\n",
    "#                ).set_axis(['varname', 'varpath'], axis = 1\n",
    "#                ).query('varname in @envvar_needed')\n",
    "#     for index, row in envspec.iterrows():\n",
    "#         os.environ[row[\"varname\"]] = row[\"varpath\"]\n",
    "# os.chdir(os.environ['dir_proj'])\n",
    "\n",
    "\n",
    "\n",
    "# check fastq.gz names ---------------------------------------------------------\n",
    "\n",
    "fastq_dir = os.environ['dir_originalfastq']\n",
    "filepaths_raw_fastq = glob.glob(fastq_dir + \"*fastq.gz\")\n",
    "print( filepaths_raw_fastq[0:4] )\n",
    "\n",
    "\n",
    "\n",
    "# data.frame of plate names ----------------------------------------------------\n",
    "\n",
    "# split before lane (L00[1-8]) to get unique plate names\n",
    "plates_df = pd.DataFrame(\n",
    "    {'plate' : pd.unique([filepath.split(\"/\")[-1].split(\"_L\")[0] for filepath in filepaths_raw_fastq])}\n",
    "    ).sort_values('plate').reset_index(drop = True)\n",
    "\n",
    "# .fastq name --> study specific metadata, may need customization # <--\n",
    "# usually separated by \"-\"; example presented here is for IGVF cell lines\n",
    "# if not custom may get \"IndexError: list index out of range; ValueError: Transform function failed\"\n",
    "plates_df['datepool'] = plates_df['plate'].transform(lambda platename: platename.split(\"-\")[0])\n",
    "plates_df['sample'] = plates_df['plate'].transform(lambda platename: platename.split(\"-\")[1])\n",
    "plates_df['sort'] = plates_df['plate'].transform(lambda platename: platename.split(\"-\")[2])\n",
    "plates_df['plateindex'] = plates_df['plate'].transform(lambda platename: platename.split(\"-\")[3])\n",
    "\n",
    "plates_df['line'] = plates_df['sample'].transform(lambda platename: platename.split(\"D\")[0])\n",
    "plates_df['time'] = plates_df['sample'].transform(lambda platename: platename.split(\"D\")[1])\n",
    "\n",
    "# number each plate, \"platenum\" used for batch submission later on\n",
    "# platenum indexed by 1-Nplates for compatibility with SGE (can't qsub -t 0)\n",
    "plates_df['platenum'] = plates_df.index.astype(int) + 1\n",
    "plates_df.index = plates_df.index.astype(int) + 1\n",
    "\n",
    "# export to \"Metadata/A01b_plate_metadata.csv\" by default\n",
    "print( plates_df.head() )\n",
    "print ( plates_df.shape )\n",
    "plates_df.to_csv(os.environ['metadat_plate'])\n",
    "print(\"metadat_plate created.\")\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## (A01c) expand plate --> all 384 wells --> final \"targets\" file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "cat > ../Scripts/A01c_well_filepaths.py\n",
    "\n",
    "# ==============================================================================\n",
    "# Scripts/A01c_well_filepaths.py\n",
    "# expands plate-level metadata (A01b) into well-level metadata\n",
    "# ==============================================================================\n",
    "\n",
    "# recommend running interactively in python/Jupyter to check outputs,\n",
    "# but shouldn't require any changes to defaults\n",
    "\n",
    "# load packages ----------------------------------------------------------------\n",
    "\n",
    "import itertools\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "# # if running interactively, need to load some lines from snmCT_parameters.env\n",
    "# # or manually spec os.environ -- e.g., via the below loop\n",
    "# # (use absolute versus relative path of parameters.env file if below not working!)\n",
    "# envvar_needed = ['dir_proj', 'dir_originalfastq', 'metadat_plate', 'metadat_well']\n",
    "# try:\n",
    "#     os.environ['metadat_well']\n",
    "# except KeyError:\n",
    "#     envspec = pd.read_csv(\"snmCT_parameters.env\", sep = \"=\", comment=\"#\", header = None\n",
    "#                ).set_axis(['varname', 'varpath'], axis = 1\n",
    "#                ).query('varname in @envvar_needed')\n",
    "#     for index, row in envspec.iterrows():\n",
    "#         os.environ[row[\"varname\"]] = row[\"varpath\"]\n",
    "# os.chdir(os.environ['dir_proj'])\n",
    "\n",
    "\n",
    "\n",
    "# expand A01b metadata by well -------------------------------------------------\n",
    "\n",
    "# load A01b\n",
    "plates_df = pd.read_csv(os.environ['metadat_plate'], index_col=0)\n",
    "\n",
    "# from pandas documentation\n",
    "def expand_grid(data_dict):\n",
    "    \"\"\"Create a dataframe from every combination of given values.\"\"\"\n",
    "    rows = itertools.product(*data_dict.values())\n",
    "    return pd.DataFrame.from_records(rows, columns=data_dict.keys())\n",
    "\n",
    "filepath_df = expand_grid({'plate': plates_df['plate'],\n",
    "    'row' : [chr(x) for x in range(65, 65+16)],\n",
    "    'col' : [str(x + 1) for x in range(24)]})\n",
    "filepath_df['well'] = filepath_df[['row', 'col']].agg(''.join, axis = 1)\n",
    "filepath_df['wellprefix'] = filepath_df['plate'] + \"_\" + filepath_df['well']\n",
    "\n",
    "filepath_df = pd.merge(filepath_df, plates_df, how = \"left\", on = \"plate\")\n",
    "\n",
    "\n",
    "\n",
    "# batch into sets of 24 for bismark, STAR processing steps ---------------------\n",
    "# (by default, one row at a time, incremented by platenum)\n",
    "\n",
    "# - alternatively, could make smaller batches of wells (e.g., n = 5) for compute\n",
    "#   environments that favor many small jobs versus a few long jobs,\n",
    "# - or two sets of batches e.g., filepath_df['batchnum_A04a_bismark']\n",
    "#   pulled by the sub scripts for the A04a script only\n",
    "\n",
    "nwellstot = filepath_df.shape[0]\n",
    "wells_per_batch = 24 # <-- can be changed\n",
    "filepath_df['batchnum'] =\\\n",
    "    pd.Series(range(0, np.ceil(nwellstot / wells_per_batch).astype(int))\n",
    "             ).repeat(wells_per_batch)[0:nwellstot].reset_index(drop = True) + 1\n",
    "\n",
    "print( \"number of total wells:\" )\n",
    "print( nwellstot )\n",
    "\n",
    "filepath_df.index = filepath_df.index.astype(int) + 1\n",
    "\n",
    "def basename(pathin):\n",
    "    return(pathin.split(\"/\")[-1])\n",
    "\n",
    "print( \"number of plates:\" )\n",
    "print( \"Nplates: \" + str( filepath_df['platenum'].max() ) )\n",
    "\n",
    "print( \"number of batches:\" )\n",
    "print( \"Nbatches: \" + str( filepath_df['batchnum'].max() ) )\n",
    "\n",
    "\n",
    "\n",
    "# then extensive file paths for sections A02-A06 -------------------------------\n",
    "# (inelegant, but useful for file checking/compiling info)\n",
    "\n",
    "# A02: demultiplexing \n",
    "# all in dir: fastq_demultip/\n",
    "\n",
    "filepath_df['A02a_fqgz_demultip_R1'] = \"fastq_demultip/\" + filepath_df[['plate', 'well']].agg('_'.join, axis = 1) + \"_indexed_R1.fastq.gz\"\n",
    "filepath_df['A02a_fqgz_demultip_R2'] = \"fastq_demultip/\" + filepath_df[['plate', 'well']].agg('_'.join, axis = 1) + \"_indexed_R2.fastq.gz\"\n",
    "\n",
    "filepath_df['A02a_txt_summary1'] = \"fastq_demultip/\" + filepath_df['plate'] + \"_summary_1.txt\"\n",
    "filepath_df['A02a_txt_summary2'] = \"fastq_demultip/\" + filepath_df['plate'] + \"_summary_2.txt\"\n",
    "\n",
    "\n",
    "\n",
    "# A03: trimming ----------------------------------------------------------------\n",
    "# all in dir: fastq_trimmed/\n",
    "\n",
    "filepath_df['A03a_fqgz_paired_R1'] = \"fastq_trimmed/\" + filepath_df['wellprefix'] + \"_paired_R1.fastq.gz\"\n",
    "filepath_df['A03a_fqgz_paired_R2'] = \"fastq_trimmed/\" + filepath_df['wellprefix'] + \"_paired_R2.fastq.gz\"\n",
    "\n",
    "filepath_df['A03a_fqgz_singletrim_R1'] = \"fastq_trimmed/\" + filepath_df['wellprefix'] + \"_singletrim_R1.fastq.gz\"\n",
    "filepath_df['A03a_fqgz_singletrim_R2'] = \"fastq_trimmed/\" + filepath_df['wellprefix'] + \"_singletrim_R2.fastq.gz\"\n",
    "\n",
    "filepath_df['A03a_json_fastp'] = \"fastq_trimmed/\" + filepath_df['wellprefix'] + \".json\"\n",
    "\n",
    "\n",
    "\n",
    "# A04: bismark -----------------------------------------------------------------\n",
    "\n",
    "filepath_df['A04a_dir_bismark'] = \"mapping_bismark/\" + filepath_df['wellprefix'] + \"/\"\n",
    "\n",
    "# (i) paired-end mapping outputs\n",
    "filepath_df['A04a_bam_bismark_PE'] = \\\n",
    "filepath_df['A04a_dir_bismark'] + filepath_df['A03a_fqgz_paired_R1'].apply(basename).str.replace(\".fastq.gz\", \"_bismark_bt2_pe.bam\", regex = True)\n",
    "filepath_df['A04a_fqgz_unmap_R1'] = \\\n",
    "filepath_df['A04a_dir_bismark'] + filepath_df['A03a_fqgz_paired_R1'].apply(basename) + \"_unmapped_reads_1.fq.gz\"\n",
    "filepath_df['A04a_fqgz_unmap_R2'] = \\\n",
    "filepath_df['A04a_dir_bismark'] + filepath_df['A03a_fqgz_paired_R2'].apply(basename) + \"_unmapped_reads_2.fq.gz\"\n",
    "\n",
    "# single-end mapping outputs\n",
    "filepath_df['A04a_bam_bismark_SE1trim'] = filepath_df['A04a_dir_bismark'] + filepath_df['A03a_fqgz_singletrim_R1'].apply(basename).str.replace(\".fastq.gz\", \"_bismark_bt2.bam\", regex = True)\n",
    "filepath_df['A04a_bam_bismark_SE2trim'] = filepath_df['A04a_dir_bismark'] + filepath_df['A03a_fqgz_singletrim_R2'].apply(basename).str.replace(\".fastq.gz\", \"_bismark_bt2.bam\", regex = True)\n",
    "\n",
    "filepath_df['A04a_bam_bismark_SE1unmap'] = filepath_df['A04a_fqgz_unmap_R1'].str.replace(\".fq.gz\", \"_bismark_bt2.bam\", regex = True)\n",
    "filepath_df['A04a_bam_bismark_SE2unmap'] = filepath_df['A04a_fqgz_unmap_R2'].str.replace(\".fq.gz\", \"_bismark_bt2.bam\", regex = True)\n",
    "\n",
    "# bismark logs\n",
    "filepath_df['A04a_txt_bismark_PE'] = filepath_df['A04a_dir_bismark'] +\\\n",
    "filepath_df['wellprefix'] + \"_paired_R1_bismark_bt2_PE_report.txt\"\n",
    "filepath_df['A04a_txt_bismark_SE1unmap'] = filepath_df['A04a_dir_bismark'] +\\\n",
    "filepath_df['wellprefix'] + \"_paired_R1.fastq.gz_unmapped_reads_1_bismark_bt2_SE_report.txt\"\n",
    "filepath_df['A04a_txt_bismark_SE2unmap'] = filepath_df['A04a_dir_bismark'] +\\\n",
    "filepath_df['wellprefix'] + \"_paired_R2.fastq.gz_unmapped_reads_2_bismark_bt2_SE_report.txt\"\n",
    "filepath_df['A04a_txt_bismark_SE1trim'] = filepath_df['A04a_dir_bismark'] +\\\n",
    "filepath_df['wellprefix'] + \"_singletrim_R1_bismark_bt2_SE_report.txt\"\n",
    "filepath_df['A04a_txt_bismark_SE2trim'] = filepath_df['A04a_dir_bismark'] +\\\n",
    "filepath_df['wellprefix'] + \"_singletrim_R2_bismark_bt2_SE_report.txt\"\n",
    "\n",
    "# (ii) picard de-duplication\n",
    "filepath_df['A04b_bam_dedupe_PE'] = filepath_df['A04a_dir_bismark'] + \"PE_dedupe.bam\"\n",
    "filepath_df['A04b_bam_merge_SE'] = filepath_df['A04a_dir_bismark'] + \"SE_merge.bam\"\n",
    "filepath_df['A04b_bam_mergesort_SE'] = filepath_df['A04a_dir_bismark'] + \"SE_mergesort.bam\"\n",
    "filepath_df['A04b_bam_mergesortdedupe_SE'] = filepath_df['A04a_dir_bismark'] + \"SE_mergesortdedupe.bam\"\n",
    "\n",
    "filepath_df['A04b_txt_picard_PE'] = filepath_df['A04a_dir_bismark'] + \"picard_PE.log\"\n",
    "filepath_df['A04b_txt_picard_SE'] = filepath_df['A04a_dir_bismark'] + \"picard_SE.log\"\n",
    "\n",
    "# (iii) read-level filtering\n",
    "filepath_df['A04b_sam_dedupeq10_PE'] = filepath_df['A04a_dir_bismark'] + \"PE.dedupe_q10.sam\"\n",
    "filepath_df['A04b_sam_dedupeq10_SE'] = filepath_df['A04a_dir_bismark'] + \"SE.dedupe_q10.sam\"\n",
    "\n",
    "filepath_df['A04b_bamfinal_PE'] = filepath_df['A04a_dir_bismark'] + \"PE_final.bam\"\n",
    "filepath_df['A04b_bamfinal_SE'] = filepath_df['A04a_dir_bismark'] + \"SE_final.bam\"\n",
    "\n",
    "# (iii) alignments to allc\n",
    "filepath_df['A04c_allc_final'] = filepath_df['A04a_dir_bismark'] + \"allc.tsv.gz\"\n",
    "filepath_df['A04c_allctbi_final'] = filepath_df['A04a_dir_bismark'] + \"allc.tsv.gz.tbi\"\n",
    "filepath_df['A04c_txt_allccheck'] = filepath_df['A04a_dir_bismark'] + \"allc_check.txt\"\n",
    "\n",
    "# sam stats for coverage, final counts\n",
    "filepath_df['A04e_txt_samstats_PE'] = filepath_df['A04a_dir_bismark'] + \"samstats_PE\"\n",
    "filepath_df['A04e_txt_samstats_SE'] = filepath_df['A04a_dir_bismark'] + \"samstats_SE\"\n",
    "\n",
    "filepath_df['A04f_txt_covtot'] = filepath_df['A04a_dir_bismark'] + \"total_cov_by_chr\"\n",
    "filepath_df['A04f_txt_covnsites'] = filepath_df['A04a_dir_bismark'] + \"nbases_cov_by_chr\"\n",
    "\n",
    "\n",
    "\n",
    "# A06: STAR mapping ------------------------------------------------------------\n",
    "\n",
    "filepath_df['A06a_dir_star'] = \"mapping_star/\" + filepath_df['wellprefix'] + \"/\"\n",
    "\n",
    "# paired-end mapping outputs (A06a)\n",
    "filepath_df['A06a_bam_star_PE'] = filepath_df['A06a_dir_star'] + \"PE.Aligned.out.bam\"\n",
    "filepath_df['A06a_bam_star_SE1'] = filepath_df['A06a_dir_star'] + \"SE1.Aligned.out.bam\"\n",
    "filepath_df['A06a_bam_star_SE2'] = filepath_df['A06a_dir_star'] + \"SE2.Aligned.out.bam\"\n",
    "\n",
    "filepath_df['A06a_fq_unmap_R1'] = filepath_df['A06a_dir_star'] + \"PE.Unmapped.out.mate1\"\n",
    "filepath_df['A06a_fq_unmap_R2'] = filepath_df['A06a_dir_star'] + \"PE.Unmapped.out.mate2\"\n",
    "\n",
    "filepath_df['A06a_txt_star_PE'] = filepath_df['A06a_dir_star'] + \"PE.Log.final.out\"\n",
    "filepath_df['A06a_txt_star_SE1'] = filepath_df['A06a_dir_star'] + \"SE1.Log.final.out\"\n",
    "filepath_df['A06a_txt_star_SE2'] = filepath_df['A06a_dir_star'] + \"SE2.Log.final.out\"\n",
    "\n",
    "# filtered outputs (A06c)\n",
    "filepath_df['A06b_bam_dedupe_PE'] = filepath_df['A06a_dir_star'] + \"star_dedupe_pe.log\"\n",
    "filepath_df['A06b_bam_dedupe_SE1'] = filepath_df['A06a_dir_star'] + \"star_dedupe_se1.log\"\n",
    "filepath_df['A06b_bam_dedupe_SE2'] = filepath_df['A06a_dir_star'] + \"star_dedupe_se2.log\"\n",
    "filepath_df['A06b_bam_starfilt_PE'] = filepath_df['A06a_dir_star'] + \"PE.Final.bam\"\n",
    "filepath_df['A06b_bam_starfilt_SE1'] = filepath_df['A06a_dir_star'] + \"SE1.Final.bam\"\n",
    "filepath_df['A06b_bam_starfilt_SE2'] = filepath_df['A06a_dir_star'] + \"SE2.Final.bam\"\n",
    "\n",
    "# samtools & picard output (A06e)\n",
    "filepath_df['A06e_txt_samtools_PE'] = filepath_df['A06a_dir_star'] + \"samstats_PE\"\n",
    "filepath_df['A06e_txt_samtools_SE1'] = filepath_df['A06a_dir_star'] + \"samstats_SE1\"\n",
    "filepath_df['A06e_txt_samtools_SE2'] = filepath_df['A06a_dir_star'] + \"samstats_SE2\"\n",
    "\n",
    "filepath_df['A06e_txt_picard_PE'] = filepath_df['A06a_dir_star'] + \"picard_PE\"\n",
    "filepath_df['A06e_txt_picard_SE1'] = filepath_df['A06a_dir_star'] + \"picard_SE1\"\n",
    "filepath_df['A06e_txt_picard_SE2'] = filepath_df['A06a_dir_star'] + \"picard_SE2\"\n",
    "\n",
    "\n",
    "\n",
    "# finally, export --------------------------------------------------------------\n",
    "\n",
    "print(filepath_df.shape)\n",
    "filepath_df.to_csv(os.environ['metadat_well'])\n",
    "print(\"metadat_well created.\")\n",
    "\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}