add (sloppy-ish) shell scripts to create Step 7: blast unassembled reads

Author: gitliver

pandora.py

@@ -78,7 +78,8 @@ def get_arg():
       step 2: assembly, \
       step 3: blast contigs, \
       step 4: orf discovery, \
-      step 5: reporting (default: 12345 - i.e, steps 1 through 5).')
+      step 5: reporting (default: 12345 - i.e, steps 1 through 5), \
+      step 7: blast unassembled reads.')
 
     # Trinity default contig length is 200
     # Ioan: for detection of species, impose no bound on the contig length in assembly
@@ -230,7 +231,8 @@ def scan_main(args):
              '3': '-S {mypython} -N blst_{args.identifier} -V -cwd -o log.out -e log.err'.format(mypython=sys.executable, args=args),
              '4': '-S {mypython} -N orf_{args.identifier} -V -cwd -o log.out -e log.err'.format(mypython=sys.executable, args=args),
              '5': '-S {mypython} -N rep_{args.identifier} -V -cwd -o log.out -e log.err'.format(mypython=sys.executable, args=args),
-             '6': '-S {mypython} -N rep2_{args.identifier} -V -cwd -o log.out -e log.err'.format(mypython=sys.executable, args=args)
+             '6': '-S {mypython} -N rep2_{args.identifier} -V -cwd -o log.out -e log.err'.format(mypython=sys.executable, args=args),
+             '7': '-S /bin/bash -N blst_unass_{args.identifier} -V -cwd -o log.out -e log.err'.format(args=args)
     }
 
     # dict which maps each step to extra qsub params for the CUMC cluster
@@ -240,7 +242,8 @@ def scan_main(args):
              '3': ' -l mem=4G,time=8::',
              '4': ' -l mem=2G,time=2::',
              '5': ' -l mem=1G,time=1::',
-             '6': ' -l mem=1G,time=1::'
+             '6': ' -l mem=1G,time=1::',
+             '7': ' -l mem=1G,time=12::'
     }
 
     # dict which maps each step to the shell part of the command
@@ -250,7 +253,8 @@ def scan_main(args):
              '3': '{args.scripts}/scripts/blast_wrapper.py --scripts {args.scripts} --threshold {args.contigthreshold} --db {args.blastdb} --threads {args.blast_threads} --id {args.identifier} --filelength {args.blastchunk} --verbose {args.verbose} --noclean {args.noclean} --nosge {args.noSGE} --hpc {args.hpc} --btime {args.btime} --bmem {args.bmem}'.format(args=args),
              '4': '{args.scripts}/scripts/orf_discovery.py --scripts {args.scripts} --id {args.identifier} --threshold {args.orfthreshold} --db {args.pblastdb} --blast {args.orfblast} --verbose {args.verbose} --noclean {args.noclean}'.format(args=args),
              '5': '{args.scripts}/scripts/makereport.py --scripts {args.scripts} --id {args.identifier} --verbose {args.verbose} --blacklist {args.blacklist} --taxid2names {args.taxid2names} --hpc {args.hpc}'.format(args=args),
-             '6': '{args.scripts}/scripts/makereport.py --outputdir report_ifilter --input blast/ifilter.concat.txt --scripts {args.scripts} --id {args.identifier} --verbose {args.verbose} --blacklist {args.blacklist} --taxid2names {args.taxid2names} --hpc {args.hpc}'.format(args=args)
+             '6': '{args.scripts}/scripts/makereport.py --outputdir report_ifilter --input blast/ifilter.concat.txt --scripts {args.scripts} --id {args.identifier} --verbose {args.verbose} --blacklist {args.blacklist} --taxid2names {args.taxid2names} --hpc {args.hpc}'.format(args=args),
+             '7': '{args.scripts}/scripts/blast_unassembled_reads.sh assembly/reads2contigs.bam blast_unassembled_reads {args.scripts} {args.blastdb} {args.blacklist} {args.taxid2names} {args.scripts}/resources/blast.header'.format(args=args)
     }
 
     # start with job id set to zero string

resources/blast.header

@@ -0,0 +1 @@
+qseqid	sseqid	saccver	staxids	pident	nident	length	mismatch	gapopen	gaps	qstart	qend	qlen	qframe	qcovs	sstart	send	slen	sframe	sstrand	evalue	bitscore	stitle

scripts/blast.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# simple wrapper for blast
+
+echo [start]
+echo [pwd] `pwd`
+echo [date] `date`
+
+db=$1
+
+input=${SGE_TASK_ID}.fasta
+output=${SGE_TASK_ID}.result
+
+echo "input "${input}
+echo "output "${output}
+echo "db "${db}
+
+# restrict to top 10 hits
+blastn -num_alignments 10 -outfmt "6 qseqid sseqid saccver staxids pident nident length mismatch gapopen gaps qstart qend qlen qframe qcovs sstart send slen sframe sstrand evalue bitscore stitle" -query ${input} -db ${db} > ${output}
+
+echo [finish]
+echo [date] `date`

scripts/blast_unassembled_reads.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+bamfile=$1
+outputdir=$2
+scripts=$3
+blastdb=$4
+blacklist=$5
+taxid2names=$6
+header=$7
+
+# output directory
+mkdir -p ${outputdir}
+if ! cd ${outputdir}; then exit; fi
+mkdir -p logs
+
+# get unmapped reads
+samtools view -b -f 4 ../${bamfile} > unassembled.bam
+
+# transform to fasta and split
+samtools view unassembled.bam | cut -f1,10 > tmp
+split -d -l 1000 -a 3 tmp
+num=0
+for i in x*; do ((num++)); cat $i | tr ":" "_" | awk '{print ">"$1; print $2;}' > ${num}.fasta; done
+echo "num: "${num}
+# filecount=$( ls -1 *fasta | wc -l )
+
+# blast
+# message looks like this: Your job-array 3147839.1-2:1 ("wait") has been submitted
+jid=$( qsub -V -N bunassembled -e logs -o logs -cwd -l mem=10G,time=6:: -t 1-${num} ${scripts}/scripts/blast.sh ${blastdb} | cut -f3 -d' ' | cut -f1 -d'.' )
+echo "job id: "${jid}
+# pause
+qsub -V -b y -o logs -e logs -cwd -N wait -hold_jid ${jid} -sync y echo wait_here
+
+# get top hit, concatenate
+for i in *.result; do ${scripts}/scripts/filterblast.py ${i} ${i}.tophit ${scripts}; done
+cat *.result > all.concat.txt
+cat *.tophit > top.concat.txt
+head -1000 logs/* > all.logs.txt
+# clean up
+echo 'clean up'
+rm x*
+rm *.fasta
+rm *.result
+rm *.tophit
+rm -r logs
+# generate report
+${scripts}/scripts/makereport.py --header ${header} --scripts ${scripts} -i top.concat.txt --outputdir . --id 1 --verbose 1 --blacklist ${blacklist} --taxid2names ${taxid2names} --hpc 1

scripts/filterblast.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python
+
+import sys
+import os
+
+inp = sys.argv[1]
+oup = sys.argv[2]
+scripts = sys.argv[3]
+
+# need this to get local modules
+sys.path.append(scripts)
+from helpers import helpers as hp
+
+# filter top hit
+hp.tophitsfilter(inp, oup)