_run_wgs_qc

#!/bin/sh
#
# master interactive script for processing a novaseq run through bclconvert 
#

WGS_PRISM_BIN=/projects/2023_sequence_production/wgs_prism
cd $WGS_PRISM_BIN

ILLUMINA_PROJECT=2024_illumina_sequencing_g

SMK_CONFIG_DIR=config/slurm/eRI

SEQ_ROOT=/projects/$ILLUMINA_PROJECT/run_data
SEQ_BCLCONVERT_ROOT=/projects/$ILLUMINA_PROJECT/postprocessing/illumina

OUT_ROOT=""

# default walltime for fastqc in minutes
FASTQC_WALL=120

# export GBS_PRISM_BIN=/dataset/gseq_processing/active/bin/gbs_prism
# #BCLCONVERT_NODE=invbfopp10.agresearch.co.nz   # iramohio-01
# ADAPTERS_FILE=/stash/miniconda3/envs/bifo-essential/opt/fastqc-0.11.8/Configuration/adapter_list.txt # use the fastqc one
# CONTAMINANTS_FILE=/stash/miniconda3/envs/bifo-essential/opt/fastqc-0.11.8/Configuration/contaminant_list.txt  # use the fastqc one
# # custom files includes Ag GBS barcode file and potentially others
# CUSTOM_ADAPTERS_FILES=/dataset/gseq_processing/active/bin/gquery/database/t_BarcodePlates.csv
# CUSTOM_CONTAMINANTS_FILES=""


function get_run_opts() {

   echo " Running wgs_prism2
* note that you can paste into your terminal window by clicking your right mouse button
* at any stage you can press CTRL-C to exit the dialogs
"

   ####### get and check RUN
   while [ 1 ] 
   do
      echo "
please give the full name of wgs run you would like to process ( e.g. 210712_A01439_0006_AHC7MJDRXY (novaseq) ):

"
      read_answer_with_default $ARGRUN
      RUN=$answer

      if [ -z "$SEQ_ROOT/$RUN" ]
      then 

         echo "sorry can't find $RUN under $SEQ_ROOT"
         quit
      else 
         if [ -d "$SEQ_ROOT/$RUN" ]
         then
            break
         fi
      fi
   done

   echo "will process $SEQ_ROOT/$RUN"

   RUN_TYPE=$(python workflow/scripts/runinfo_to_machinetype.py $SEQ_ROOT/$RUN/RunInfo.xml) 
   mkdir -p $SEQ_BCLCONVERT_ROOT/$RUN_TYPE


####### find the sample sheet  - e.g. could be HNFW2DRXY.csv or SampleSheet.csv

   SAMPLE_SHEET=$SEQ_ROOT/$RUN/SampleSheet.csv
   if [ ! -f $SAMPLE_SHEET ]; then
      SAMPLE_SHEET=`ls $SEQ_ROOT/$RUN/*.csv 2>/dev/null`  
      if [ $? != 0 ]; then
         echo "sorry can't find the sample-sheet for this run under $SEQ_ROOT/$RUN
try /dataset/hiseq/active/sample-sheets. If it is there, please copy to the run folder $SEQ_ROOT/$RUN
"
         exit 1
      fi
      base=`basename $SAMPLE_SHEET .csv`
      echo $RUN | grep $base > /dev/null 2>&1
      if [ $? != 0 ]; then
         echo "sorry can't find the sample-sheet for this run under $SEQ_ROOT/$RUN
try /dataset/hiseq/active/sample-sheets. If it is there, please copy to the run folder $SEQ_ROOT/$RUN
"
         exit 1
      fi
   fi


   ######## confirm output folder ###########
   # set up output folder

   while [ 1 ]; do
      echo "

please specify the full path to the q/c output folder (or just press ENTER to use default , $SEQ_BCLCONVERT_ROOT/$RUN_TYPE/$RUN )"
      read_answer_with_default $SEQ_BCLCONVERT_ROOT/$RUN_TYPE/$RUN
      NEW_ROOT=$answer
      if [ -d $NEW_ROOT ]; then
         echo "warning - $NEW_ROOT already exists, use anyway ? (y/n, default=y)"
         read_answer_with_default y
         if [ $answer == "y" ]; then
            OUTPUT_ROOT=$NEW_ROOT
            break
         fi
      else
         mkdir -p $NEW_ROOT
         if [ -d $NEW_ROOT ]; then
            OUTPUT_ROOT=$NEW_ROOT
            break
         fi
      fi
   done


####### check whether we can find sequence data - if not confirm  bclconvert is needed 
   echo "checking sequence data (looking under  $OUTPUT_ROOT)....

"
   ls -lR $OUTPUT_ROOT/SampleSheet/bclconvert 2>/dev/null | grep "fastq.gz"   > /dev/null 2>&1 
   if [ $? != 0 ]
   then
      echo "could not find fastq.gz data in $OUTPUT_ROOT/SampleSheet/bclconvert.

if you just want to run downstream q/c, CTRL-C now and create symlinks to your data in:

$OUTPUT_ROOT/SampleSheet/bclconvert 

. . . but you probably want to run bclconvert  - OK to run that ? (y/n, default=y)"
      read_answer_with_default y 
      if [ "$answer" != "y" ]; then
         echo "OK continuing..."
         echo
      else
         run_bclconvert 
      fi
   else
      echo "found fastq.gz data: "
      ls -lR $OUTPUT_ROOT/SampleSheet/bclconvert | grep "fastq.gz"
      echo

   fi


   ####### check whether we can find lane q/c landmark - if not confirm 
   echo "checking for MultiQC summary report ... "
   echo

   ls $OUTPUT_ROOT/SampleSheet/multiqc/*.multiqc.html  > /dev/null 2>&1
   if [ $? != 0 ]; then 
      echo "could not find MultiQC summary report - OK to run lane q/c ? (y/n, default=y)"
      read_answer_with_default y
      if [ "$answer" != "y" ]; then
         echo "OK continuing..."
      else
         run_lane_qc
      fi
   else
      echo "found multiQC report: "
      ls $OUTPUT_ROOT/SampleSheet/multiqc/*.multiqc.html
      echo
   fi

}


function send_mail() {
   message="$1"
   echo "sending mail"
   echo "" | mutt -s "$message" vanstijnt , mccullocha, bairdh, perrybe, andersonr, andrewsa, henryh, frenchm, hicklandm
}


function read_answer_with_default() {
   if [ $INTERACTIVE == yes ]; then
      read answer
      echo "User response: $answer"
      if [ -z "$answer" ]; then
         answer=$@
         echo "Default response: $answer"
      fi
   else
      answer=$@
      echo "Default response: $answer"
   fi
}


function more_with_default() {
   if [ $INTERACTIVE == yes ]; then
      more $1
   else
      cat $1
   fi
}


function get_opts() {
   INTERACTIVE=no
   INTERACTIVE_OPT=""
   ARGRUN=""
   help_text="
This script is called by run_wgs_qc (or non-interactively by a cron job).
Usage :\n
"
   while getopts ":hir:" opt; do
   case $opt in
       h)
         echo -e $help_text
         exit 0
         ;;
       i)
         INTERACTIVE=yes
         INTERACTIVE_OPT="-i"
         ;;
       r)
         ARGRUN=$OPTARG
         ;;
       \?)
         echo "Invalid option: -$OPTARG" >&2
         exit 1
         ;;
       :)
         echo "Option -$OPTARG requires an argument." >&2
         exit 1
         ;;
     esac
   done

   shift $((OPTIND-1))

}



function run_bclconvert() {

   echo "

Checking run is completed (i.e. looking for $SEQ_ROOT/$RUN/RTAComplete.txt).

"
   if [ ! -f $SEQ_ROOT/$RUN/RTAComplete.txt ]; then
      echo "warning: landmark file $SEQ_ROOT/$RUN/RTAComplete.txt does not exist => this run has not completed sequencing (or uploading?) - are you SURE you want to continue !? (y/n default n)"
      read_answer_with_default n
      if [ $answer != "y" ]; then
         echo "OK quitting."
         exit 1
      else
         echo "OK will continue but note that output may be incomplete."
      fi
   fi

   bclconvert_phrase="" # not currently used - previously used to pass in bcl2fastq options 
   samplesheet_to_fastqnames_phrase=""

   # set up for bclconvert run
   mkdir -p $OUTPUT_ROOT/SampleSheet
   mkdir -p $OUTPUT_ROOT/logs

   if [ ! -d $OUTPUT_ROOT/SampleSheet ]; then
      echo "error: could not create bclconvert output folder $OUTPUT_ROOT/SampleSheet -- quitting."
      exit 1
   fi

   # Gather sample sheet from active 
   
   if [ ! -f $OUTPUT_ROOT/SampleSheet.csv ]; then
      echo "Copying $SAMPLE_SHEET to $OUTPUT_ROOT/SampleSheet.csv "
      cp $SAMPLE_SHEET  $OUTPUT_ROOT/SampleSheet.csv
   fi

  echo "
  Using sample sheet $OUTPUT_ROOT/SampleSheet.csv for bclconvert.
  Please review and edit this samples sheet making any necessary changes for batch processing. 

  If this is batched with indices of different lengths, you will need to do the following:

  1) Add OverrideCycles options under [Settings] such as,
  # 151 PE reads with 10bp i5 and no i7 index
  [Settings]
  OverrideCycles,Y151;I10;N10;Y151
  # 101 PE reads with 8bp i5 and i7 indices
  [Settings]
  OverrideCycles,Y101;I8N2;I8N2;Y101,,,,,,,,
  # 151 PE reads with 8 bp indices
  [Settings]
  OverrideCycles,Y151;I8N2;I8N2;Y151,,,,,,,,
  # 101 bp SE read with 10 bp i5 and 8bp i7
  [Settings]
  OverrideCycles,Y101;I10;I8N2,,,,,,,,

  2) You will need to remove all samples from the sample sheet which are not part of this batch.
  In reprocessing of this run (eg. other batches) you will be prompted to modify the original sample sheet again.


  If this run requires the removal of T-overhangs:

  [Settings]
  OverrideCycles,N1Y150;I10;I10;N1Y150

  Ref https://support.illumina.com/bulletins/2020/06/trimming-t-overhang-options-for-the-illumina-rna-library-prep-wo.html


  If this run has UMIs which need to be demultiplexed and captured:
  Note: if the indices in the sample sheet are supplied with N for the UMIs remove those and leave the index sequences.

  # 101 bp PE reads with 12 bp UMI and 8 bp i5 index and 8 bp i7 index
  [Settings],,,,,,,,,
  OverrideCycles,Y101;U12I8;I8;Y101,,,,,,,,
  CreateFastqForIndexReads,1,,,,,,,,
  TrimUMI,0,,,,,,,,
  # 101 bp PE reads with 20 bp UMI and 8 bp i7 index
  [Settings],,,,,,,,,
  OverrideCycles,Y101;U20;I8;Y101,,,,,,,,
  CreateFastqForIndexReads,1,,,,,,,,
  TrimUMI,0,,,,,,,,

  Ref: https://knowledge.illumina.com/software/on-premises-software/software-on-premises-software-reference_material-list/000007337"


   echo "Press Enter to review sample sheet:"
   read_answer_with_default ""

   vim $OUTPUT_ROOT/SampleSheet.csv
   echo "
If this is OK ? answer y (or just press enter):
   "
   read_answer_with_default y
   if [ $answer != "y" ]; then
      echo "
OK quitting.

(please edit $OUTPUT_ROOT/SampleSheet.csv and try again)"
      exit 1
   fi 

   
   ###### ensure output folder does not exist
   if [ -d $OUTPUT_ROOT/SampleSheet/bclconvert ]; then
      echo "
error: $OUTPUT_ROOT/SampleSheet/bclconvert already exists - please clean up and retry."
      exit 1
   fi

   # remove the make target if it exists
   rm -f $OUTPUT_ROOT/SampleSheet/SampleSheet.csv.bclconvert


   echo "
About to run bclconvert using:

snakemake --profile $SMK_CONFIG_DIR --snakefile workflow/1_bclconvert.smk --config OUT_ROOT=$OUTPUT_ROOT IN_ROOT=$SEQ_ROOT RUN=$RUN

OK ? (y/n, default=y):
"

   read_answer_with_default  y
   if [ $answer != "y" ]; then
      echo "OK quitting."
      exit 1
   fi
   echo "
Starting bclconvert...

"
   echo "snakemake --profile $SMK_CONFIG_DIR --snakefile workflow/1_bclconvert.smk --config OUT_ROOT=$OUTPUT_ROOT IN_ROOT=$SEQ_ROOT RUN=$RUN "
   source activate snakemake
   export PATH=/usr/bin/bcl-convert:$PATH # TODO move to config
   snakemake --profile $SMK_CONFIG_DIR --snakefile workflow/1_bclconvert.smk --config OUT_ROOT=$OUTPUT_ROOT IN_ROOT=$SEQ_ROOT RUN=$RUN > $OUTPUT_ROOT/logs/bclconvert.smk.log
   conda deactivate

   if [ $? != 0 ]
   then

      # bad code but might ignore that if we have sequence data
      ls -lR $OUTPUT_ROOT/SampleSheet/bclconvert 2>/dev/null | grep "fastq.gz"   > /dev/null 2>&1

      if [ $? == 0 ]; then
         echo "warning: bclconvert exit code was non-zero, but it did generate fastq data. "
      else
         if [ $INTERACTIVE != yes ]; then
            send_mail "Sorry bclconvert for $RUN exited with an error code and no fastq data was generated."
         fi
         echo "error: bclconvert exit code was non-zero and no sequence data generated. "
         echo "(suggest check $OUTPUT_ROOT/SampleSheet/bcl-convert.log)"
         exit 1
      fi

      echo "

bcl-convert has finished but received a non-zero exit code, though did generate fastq data so might be OK. 

Do you want to continue ? (y/n, default = y):
"
      read_answer_with_default y 
      if [ $answer != "y" ]; then
         echo "OK quitting"
         exit 1
      fi

   fi
   

   if [ $INTERACTIVE != yes ]; then
      send_mail "(bclconvert for $RUN has completed - fastq data is now available)"
   fi

   echo "

bclconvert completed.

   "
}



function run_lane_qc() {
   echo "

Finding sequence files.
"
   mkdir -p $OUTPUT_ROOT/SampleSheet
   mkdir -p $OUTPUT_ROOT/logs
   find $OUTPUT_ROOT/SampleSheet/bclconvert -name "*.fastq.gz" -size +1000c -print | grep -vi Undetermined > $OUTPUT_ROOT/SampleSheet/sequence_files.txt
   return_code=$?

   find $OUTPUT_ROOT/SampleSheet/bclconvert -name "*.fastq.gz" -type l -print | grep -vi Undetermined >> $OUTPUT_ROOT/SampleSheet/sequence_files.txt
   
   if [[ ( $return_code != 0 ) && (  $? != 0 ) ]]; then
      echo "error: oops could not find any .fastq.gz files ( or links to fastq files) under $OUTPUT_ROOT/SampleSheet/bclconvert - quiting."
      exit 1
   fi  

### lane QC ###
      echo "
About to run detailed sequencing run QC using:

snakemake --profile $SMK_CONFIG_DIR --snakefile workflow/2_run_QC.smk --config fastqc_walltime=$FASTQC_WALL OUT_ROOT=$OUTPUT_ROOT IN_ROOT=$SEQ_ROOT RUN=$RUN 

OK ? (y/n, default=y):
"

   read_answer_with_default  y
   if [ $answer == "y" ]; then
      echo "
Starting Run QC...
"
      echo "snakemake --profile $SMK_CONFIG_DIR --snakefile workflow/2_run_QC.smk --config fastqc_walltime=$FASTQC_WALL OUT_ROOT=$OUTPUT_ROOT IN_ROOT=$SEQ_ROOT RUN=$RUN "
      
      source activate snakemake
      snakemake --profile $SMK_CONFIG_DIR --snakefile workflow/2_run_QC.smk --config fastqc_walltime=$FASTQC_WALL OUT_ROOT=$OUTPUT_ROOT IN_ROOT=$SEQ_ROOT RUN=$RUN  | tee $OUTPUT_ROOT/logs/run_QC.smk.log
      conda deactivate

      echo "
Run QC completed... $(date)
Results: $OUTPUT_ROOT
Questions: ben.perry@agresearch.co.nz

"
      exit 0
   else
      echo "
Skipping detailed QC report...
"
      echo
   fi



### fastqc rule ###

      echo "
About to run fastqc using:

snakemake --profile $SMK_CONFIG_DIR --snakefile workflow/2_fastqc.smk --config fastqc_walltime=$FASTQC_WALL OUT_ROOT=$OUTPUT_ROOT IN_ROOT=$SEQ_ROOT RUN=$RUN 

OK ? (y/n, default=y):
"

   read_answer_with_default  y
   if [ $answer != "y" ]; then
      echo "OK quitting."
      exit 1
   fi
   echo "
Starting fastqc...
"
   echo "snakemake --profile $SMK_CONFIG_DIR --snakefile workflow/2_fastqc.smk --config fastqc_walltime=$FASTQC_WALL OUT_ROOT=$OUTPUT_ROOT IN_ROOT=$SEQ_ROOT RUN=$RUN "
   source activate snakemake
   snakemake --profile $SMK_CONFIG_DIR --snakefile workflow/2_fastqc.smk --config fastqc_walltime=$FASTQC_WALL OUT_ROOT=$OUTPUT_ROOT IN_ROOT=$SEQ_ROOT RUN=$RUN | tee $OUTPUT_ROOT/logs/fastqc.smk.log
   conda deactivate


### multiQC rule ###

      echo "
about to run multiQC using:

snakemake --profile $SMK_CONFIG_DIR --snakefile workflow/3_multiQC.smk --config OUT_ROOT=$OUTPUT_ROOT IN_ROOT=$SEQ_ROOT RUN=$RUN 

OK ? (y/n, default=y)
"

   read_answer_with_default  y
   if [ $answer != "y" ]; then
      echo "OK quitting"
      exit 1
   fi
   echo "
Starting MultiQC...

   "
   echo "snakemake --profile $SMK_CONFIG_DIR --snakefile workflow/3_multiQC.smk --config OUT_ROOT=$OUTPUT_ROOT IN_ROOT=$SEQ_ROOT RUN=$RUN" 
   source activate snakemake
   snakemake --profile $SMK_CONFIG_DIR --snakefile workflow/3_multiQC.smk --config OUT_ROOT=$OUTPUT_ROOT IN_ROOT=$SEQ_ROOT RUN=$RUN | tee $OUTPUT_ROOT/logs/multiqc.smk.log
   conda deactivate


echo "
Run QC completed... $(date)
Results: $OUTPUT_ROOT
Questions: ben.perry@agresearch.co.nz

"
}


get_opts "$@"
get_run_opts