From edc5a4e329ee7d044c7211ec7c189a37a71d5ac4 Mon Sep 17 00:00:00 2001 From: esnaultc Date: Fri, 23 Aug 2019 12:01:07 -0400 Subject: [PATCH 01/15] add reports to rnaseq-wf --- include/WRAPPER_SLURM | 1 + workflows/rnaseq/Snakefile | 352 ++++++++++++------- workflows/rnaseq/config/rnaseq_patterns.yaml | 156 +++++--- workflows/rnaseq/downstream/rnaseq.Rmd | 2 +- 4 files changed, 343 insertions(+), 168 deletions(-) diff --git a/include/WRAPPER_SLURM b/include/WRAPPER_SLURM index 522af2cf..28fa899f 100755 --- a/include/WRAPPER_SLURM +++ b/include/WRAPPER_SLURM @@ -21,6 +21,7 @@ if [[ ! -e logs ]]; then mkdir -p logs; fi --use-conda \ --configfile config/config.yaml \ --latency-wait=300 \ + --report report.html ) > "Snakefile.log" 2>&1 SNAKE_PID=$! diff --git a/workflows/rnaseq/Snakefile b/workflows/rnaseq/Snakefile index c81acdfe..57b5ab30 100644 --- a/workflows/rnaseq/Snakefile +++ b/workflows/rnaseq/Snakefile @@ -10,6 +10,7 @@ from lcdblib.utils import utils from lib import common from lib import cluster_specific from lib.patterns_targets import RNASeqConfig +import copy # ---------------------------------------------------------------------------- # @@ -40,48 +41,92 @@ wildcard_constraints: def wrapper_for(path): return 'file:' + os.path.join('../..','wrappers', 'wrappers', path) + # ---------------------------------------------------------------------------- # RULES # ---------------------------------------------------------------------------- -# See "patterns and targets" in the documentation for what's going on here. -final_targets = utils.flatten(( - c.targets['bam'], - utils.flatten(c.targets['fastqc']), - utils.flatten(c.targets['libsizes']), - [c.targets['fastq_screen']], - [c.targets['libsizes_table']], - [c.targets['rrna_percentages_table']], - [c.targets['multiqc']], - utils.flatten(c.targets['featurecounts']), - utils.flatten(c.targets['rrna']), - utils.flatten(c.targets['markduplicates']), - utils.flatten(c.targets['salmon']), - utils.flatten(c.targets['dupradar']), - utils.flatten(c.targets['preseq']), - utils.flatten(c.targets['rseqc']), - utils.flatten(c.targets['collectrnaseqmetrics']), - utils.flatten(c.targets['bigwig']), - utils.flatten(c.targets['downstream']), -)) - -if 'merged_bigwigs' in config: - final_targets.extend(utils.flatten(c.targets['merged_bigwig'])) - -# Special case: all samples are single-end -if all(c.sampletable.iloc[:, 0].apply( - lambda x: not common.is_paired_end(c.sampletable, x)) -): - ALL_SE = True - final_targets = [i.replace('{n}', '1') for i in final_targets] -else: - ALL_SE = False +report: "report/workflow.rst" + +def targets(p): + # See "patterns and targets" in the documentation for what's going on here. + final_targets = utils.flatten(( + c.targets['bam'][p], + utils.flatten(c.targets['fastqc'][p]), + utils.flatten(c.targets['libsizes'][p]), + [c.targets['fastq_screen'][p]], + [c.targets['libsizes_table'][p]], + [c.targets['rrna_percentages_table'][p]], + [c.targets['multiqc'][p]], + utils.flatten(c.targets['featurecounts'][p]), + utils.flatten(c.targets['rrna'][p]), + utils.flatten(c.targets['markduplicates'][p]), + utils.flatten(c.targets['salmon'][p]), + utils.flatten(c.targets['dupradar'][p]), + utils.flatten(c.targets['preseq'][p]), + utils.flatten(c.targets['rseqc'][p]['bam_stat']), + utils.flatten(c.targets['collectrnaseqmetrics'][p]), + utils.flatten(c.targets['bigwig'][p]), + utils.flatten(c.targets['downstream'][p]), + )) + + if 'merged_bigwigs' in config: + final_targets.extend(utils.flatten(c.targets['merged_bigwig'][p])) + + # Special case: all samples are single-end + if all(c.sampletable.iloc[:, 0].apply( + lambda x: not common.is_paired_end(c.sampletable, x)) + ): + ALL_SE = True + final_targets = [i.replace('{n}', '1') for i in final_targets] + else: + ALL_SE = False + return (final_targets, ALL_SE, c.sampletable) + +final_targets, ALL_SE, c.sampletable = targets('pattern') + + +final_rst = copy.deepcopy(c.patterns) +rstpaths = [] +for key, val in final_rst.items(): + if type(val['pattern']) is not dict: + val['pattern'] = val['pattern'].replace('{', '_').replace('}', '_') + final_rst[key]['pattern'] = 'report/' + val['pattern'] + '.rst' + rstpaths.append(final_rst[key]['pattern']) + elif type(val['pattern']) is dict: + for nestkey, nestval in val['pattern'].items(): + nestval= nestval.replace('{', '_').replace('}', '_') + final_rst[key]['pattern'][nestkey] = 'report/' + nestval + '.rst' + rstpaths.append(final_rst[key]['pattern'][nestkey]) rule targets: """ Final targets to create """ - input: final_targets + input: + final_targets, + rstpaths + +rule report_rst: + """ + Create .rst containing captions for report + """ + input: + 'config/rnaseq_patterns.yaml' + output: + rstpaths + run: + for key, val in final_rst.items(): + if type(val['description']) is not dict: + descr = val['description'] + pat = val['pattern'] + shell("echo {descr} > {pat}") + elif type(val['description']) is dict: + for nestkey, nestval in val['description'].items(): + descr = val['description'][nestkey] + pat = val['pattern'][nestkey] + shell("echo '{descr}' > {pat}") + if 'orig_filename' in c.sampletable.columns: @@ -111,7 +156,7 @@ if 'orig_filename' in c.sampletable.columns: input: orig_for_sample output: - c.patterns['fastq'] + c.patterns['fastq']['pattern'] wildcard_constraints: n="\d+" run: @@ -120,7 +165,7 @@ if 'orig_filename' in c.sampletable.columns: rule symlink_targets: - input: c.targets['fastq'] + input: c.targets['fastq']['pattern'] def render_r1_r2(pattern): @@ -137,7 +182,7 @@ if 'Run' in c.sampletable.columns and sum(c.sampletable['Run'].str.startswith('S rule fastq_dump: output: - fastq=render_r1_r2(c.patterns['fastq']) + fastq=render_r1_r2(c.patterns['fastq']['pattern']) run: srr = _st.loc[wildcards.sample, 'Run'] @@ -180,11 +225,13 @@ rule cutadapt: Run cutadapt """ input: - fastq=common.fill_r1_r2(c.sampletable, c.patterns['fastq']) + fastq=common.fill_r1_r2(c.sampletable, c.patterns['fastq']['pattern']) output: - fastq=render_r1_r2(c.patterns['cutadapt']) + fastq=report(render_r1_r2(c.patterns['cutadapt']['pattern']), caption= + final_rst['cutadapt']['pattern'], + category="fastq") log: - render_r1_r2(c.patterns['cutadapt'])[0] + '.log' + render_r1_r2(c.patterns['cutadapt']['pattern'])[0] + '.log' run: paired = len(input) == 2 @@ -231,12 +278,12 @@ if config['aligner']['index'] == 'hisat2': Map reads with HISAT2 """ input: - fastq=common.fill_r1_r2(c.sampletable, c.patterns['cutadapt']), + fastq=common.fill_r1_r2(c.sampletable, c.patterns['cutadapt']['pattern']), index=[c.refdict[c.organism][config['aligner']['tag']]['hisat2']] output: - bam=c.patterns['bam'] + bam=c.patterns['bam']['pattern'] log: - c.patterns['bam'] + '.log' + c.patterns['bam']['pattern'] + '.log' params: # NOTE: see examples at # https://github.com/lcdb/lcdb-wf/tree/master/wrappers/wrappers/hisat2/align @@ -252,12 +299,12 @@ if config['aligner']['index'] == 'star': Map reads with STAR """ input: - fastq=common.fill_r1_r2(c.sampletable, c.patterns['cutadapt']), + fastq=common.fill_r1_r2(c.sampletable, c.patterns['cutadapt']['pattern']), index=[c.refdict[c.organism][config['aligner']['tag']]['star']] output: - bam=c.patterns['bam'] + bam=c.patterns['bam']['pattern'] log: - c.patterns['bam'] + '.log' + c.patterns['bam']['pattern'] + '.log' threads: 6 run: genomedir = os.path.dirname(input.index[0]) @@ -328,13 +375,13 @@ if config['aligner']['index'] == 'ngm': Map reads with NextGenMap """ input: - fastq=common.fill_r1_r2(c.sampletable, c.patterns['cutadapt']), + fastq=common.fill_r1_r2(c.sampletable, c.patterns['cutadapt']['pattern']), index=[c.refdict[c.organism][config['aligner']['tag']]['ngm']], fasta=[c.refdict[c.organism][config['aligner']['tag']]['ngm_fasta']] output: - bam=c.patterns['bam'] + bam=c.patterns['bam']['pattern'] log: - c.patterns['bam'] + '.log' + c.patterns['bam']['pattern'] + '.log' threads: 6 run: if common.is_paired_end(c.sampletable, wildcards.sample): @@ -382,12 +429,12 @@ rule rRNA: Map reads with bowtie2 to the rRNA reference """ input: - fastq=common.fill_r1_r2(c.sampletable, c.patterns['cutadapt'], r1_only=True), + fastq=common.fill_r1_r2(c.sampletable, c.patterns['cutadapt']['pattern'], r1_only=True), index=[c.refdict[c.organism][config['rrna']['tag']]['bowtie2']] output: - bam=c.patterns['rrna']['bam'] + bam=c.patterns['rrna']['pattern']['bam'] log: - c.patterns['rrna']['bam'] + '.log' + c.patterns['rrna']['pattern']['bam'] + '.log' params: # NOTE: we'd likely only want to report a single alignment for rRNA # screening @@ -405,7 +452,7 @@ rule fastq_count: input: fastq='{sample_dir}/{sample}/{sample}{suffix}.fastq.gz' output: - count='{sample_dir}/{sample}/{sample}{suffix}.fastq.gz.libsize' + count='{sample_dir}/{sample}/{sample}{suffix}.fastq.gz.libsize', shell: 'zcat {input} | echo $((`wc -l`/4)) > {output}' @@ -417,7 +464,8 @@ rule bam_count: input: bam='{sample_dir}/{sample}/{suffix}.bam' output: - count='{sample_dir}/{sample}/{suffix}.bam.libsize' + count='{sample_dir}/{sample}/{suffix}.bam.libsize', + shell: 'samtools view -c {input} > {output}' @@ -453,9 +501,11 @@ rule fastq_screen: **fastq_screen_references(), fastq=common.fill_r1_r2(c.sampletable, rules.cutadapt.output.fastq, r1_only=True), output: - txt=c.patterns['fastq_screen'] + txt=report(c.patterns['fastq_screen']['pattern'], + caption=final_rst['fastq_screen']['pattern'], + category="fastq screen") log: - c.patterns['fastq_screen'] + '.log' + c.patterns['fastq_screen']['pattern'] + '.log' params: subset=100000 script: wrapper_for('fastq_screen/wrapper.py') @@ -467,9 +517,11 @@ rule featurecounts: """ input: annotation=c.refdict[c.organism][config['gtf']['tag']]['gtf'], - bam=c.patterns['bam'] + bam=c.patterns['bam']['pattern'] output: - counts='{sample_dir}/{sample}/{sample}.cutadapt.bam.featurecounts.{stranded}.txt' + counts=report('{sample_dir}/{sample}/{sample}.cutadapt.bam.featurecounts.{stranded}.txt', + caption=final_rst['bam']['pattern'], + category='Feature count') log: '{sample_dir}/{sample}/{sample}.cutadapt.bam.featurecounts.{stranded}.txt.log' shell: @@ -499,17 +551,21 @@ rule rrna_libsizes_table: Aggregate rRNA counts into a table """ input: - rrna=c.targets['rrna']['libsize'], - fastq=c.targets['libsizes']['cutadapt'] + rrna=c.targets['rrna']['pattern']['libsize'], + fastq=c.targets['libsizes']['pattern']['cutadapt'] output: - json=c.patterns['rrna_percentages_yaml'], - tsv=c.patterns['rrna_percentages_table'] + json=report(c.patterns['rrna_percentages_yaml']['pattern'], + caption=final_rst['rrna_percentages_yaml']['pattern'], + category='Libsizes'), + tsv=report(c.patterns['rrna_percentages_table']['pattern'], + caption=final_rst['rrna_percentages_table']['pattern'], + category='Libsizes'), run: def rrna_sample(f): - return helpers.extract_wildcards(c.patterns['rrna']['libsize'], f)['sample'] + return helpers.extract_wildcards(c.patterns['rrna']['pattern']['libsize'], f)['sample'] def sample(f): - return helpers.extract_wildcards(c.patterns['libsizes']['cutadapt'], f)['sample'] + return helpers.extract_wildcards(c.patterns['libsizes']['pattern']['cutadapt'], f)['sample'] def million(f): return float(open(f).read()) / 1e6 @@ -552,8 +608,13 @@ rule libsizes_table: input: utils.flatten(c.targets['libsizes']) output: - json=c.patterns['libsizes_yaml'], - tsv=c.patterns['libsizes_table'] + json=report(c.patterns['libsizes_yaml']['pattern'], + caption=final_rst['libsizes_yaml']['pattern'], + category='Libsizes'), + tsv=report(c.patterns['libsizes_table']['pattern'], + caption=final_rst['libsizes_table']['pattern'], + category='Libsizes'), + run: def sample(f): return os.path.basename(os.path.dirname(f)) @@ -605,27 +666,27 @@ rule multiqc: # add outputs from those rules to the inputs here. input: files=( - utils.flatten(c.targets['fastqc']) + - utils.flatten(c.targets['libsizes_yaml']) + - utils.flatten(c.targets['rrna_percentages_yaml']) + - utils.flatten(c.targets['cutadapt']) + - utils.flatten(c.targets['featurecounts']) + - utils.flatten(c.targets['bam']) + - utils.flatten(c.targets['markduplicates']) + - utils.flatten(c.targets['salmon']) + - utils.flatten(c.targets['rseqc']) + - utils.flatten(c.targets['fastq_screen']) + - utils.flatten(c.targets['dupradar']) + - utils.flatten(c.targets['preseq']) + - utils.flatten(c.targets['collectrnaseqmetrics']) + utils.flatten(c.targets['fastqc']['pattern']) + + utils.flatten(c.targets['libsizes_yaml']['pattern']) + + utils.flatten(c.targets['rrna_percentages_yaml']['pattern']) + + utils.flatten(c.targets['cutadapt']['pattern']) + + utils.flatten(c.targets['featurecounts']['pattern']) + + utils.flatten(c.targets['bam']['pattern']) + + utils.flatten(c.targets['markduplicates']['pattern']) + + utils.flatten(c.targets['salmon']['pattern']) + + utils.flatten(c.targets['rseqc']['pattern']) + + utils.flatten(c.targets['fastq_screen']['pattern']) + + utils.flatten(c.targets['dupradar']['pattern']) + + utils.flatten(c.targets['preseq']['pattern']) + + utils.flatten(c.targets['collectrnaseqmetrics']['pattern']) ), config='config/multiqc_config.yaml' - output: c.targets['multiqc'] - log: c.targets['multiqc'][0] + '.log' + output: report(c.targets['multiqc']['pattern'], caption=final_rst['multiqc']['pattern'], category='QC') + log: c.targets['multiqc']['pattern'][0] + '.log' run: analysis_directory = set([os.path.dirname(i) for i in input]) - outdir = os.path.dirname(c.targets['multiqc'][0]) - basename = os.path.basename(c.targets['multiqc'][0]) + outdir = os.path.dirname(c.targets['multiqc']['pattern'][0]) + basename = os.path.basename(c.targets['multiqc']['pattern'][0]) shell( 'LC_ALL=en_US.UTF.8 LC_LANG=en_US.UTF-8 ' 'multiqc ' @@ -643,12 +704,14 @@ rule markduplicates: Mark or remove PCR duplicates with Picard MarkDuplicates """ input: - bam=c.patterns['bam'] + bam=c.patterns['bam']['pattern'] output: - bam=c.patterns['markduplicates']['bam'], - metrics=c.patterns['markduplicates']['metrics'] + bam=c.patterns['markduplicates']['pattern']['bam'], + metrics=c.patterns['markduplicates']['pattern']['metrics'], +# caption=final_rst['markduplicates']['pattern']['metrics'], +# category='Metrics') log: - c.patterns['markduplicates']['bam'] + '.log' + c.patterns['markduplicates']['pattern']['bam'] + '.log' params: # NOTE: Be careful with the memory here; make sure you have enough # and/or it matches the resources you're requesting in the cluster @@ -670,11 +733,15 @@ rule collectrnaseqmetrics: Calculate various RNA-seq QC metrics with Picarc CollectRnaSeqMetrics """ input: - bam=c.patterns['bam'], + bam=c.patterns['bam']['pattern'], refflat=c.refdict[c.organism][config['gtf']['tag']]['refflat'] output: - metrics=c.patterns['collectrnaseqmetrics']['metrics'], - pdf=c.patterns['collectrnaseqmetrics']['pdf'] + metrics=report(c.patterns['collectrnaseqmetrics']['pattern']['metrics'], + caption=final_rst['collectrnaseqmetrics']['pattern']['metrics'], + category='Metrics'), + pdf=report(c.patterns['collectrnaseqmetrics']['pattern']['pdf'], + caption=final_rst['collectrnaseqmetrics']['pattern']['pdf'], + category='Metrics'), params: # NOTE: Be careful with the memory here; make sure you have enough # and/or it matches the resources you're requesting in the cluster @@ -682,7 +749,7 @@ rule collectrnaseqmetrics: java_args='-Xmx20g' # java_args='-Xmx2g' # [TEST SETTINGS -1] log: - c.patterns['collectrnaseqmetrics']['metrics'] + '.log' + c.patterns['collectrnaseqmetrics']['pattern']['metrics'] + '.log' shell: 'picard ' '{params.java_args} ' @@ -710,9 +777,12 @@ rule preseq: Compute a library complexity curve with preseq """ input: - bam=c.patterns['bam'] + bam=c.patterns['bam']['pattern'] output: - c.patterns['preseq'] + report(c.patterns['preseq']['pattern'], + caption=final_rst['preseq']['pattern'], + category='Metrics'), + shell: 'preseq ' 'c_curve ' @@ -728,15 +798,32 @@ rule dupRadar: bam=rules.markduplicates.output.bam, annotation=c.refdict[c.organism][config['gtf']['tag']]['gtf'], output: - density_scatter=c.patterns['dupradar']['density_scatter'], - expression_histogram=c.patterns['dupradar']['expression_histogram'], - expression_boxplot=c.patterns['dupradar']['expression_boxplot'], - expression_barplot=c.patterns['dupradar']['expression_barplot'], - multimapping_histogram=c.patterns['dupradar']['multimapping_histogram'], - dataframe=c.patterns['dupradar']['dataframe'], - model=c.patterns['dupradar']['model'], - curve=c.patterns['dupradar']['curve'], - log: c.patterns['dupradar']['dataframe'] + '.log' + density_scatter=report(c.patterns['dupradar']['pattern']['density_scatter'], + caption=final_rst['dupradar']['pattern']['density_scatter'], + category='dupRadar'), + expression_histogram=report(c.patterns['dupradar']['pattern']['expression_histogram'], + caption=final_rst['dupradar']['pattern']['expression_histogram'], + category='dupRadar'), + expression_boxplot=report(c.patterns['dupradar']['pattern']['expression_boxplot'], + caption=final_rst['dupradar']['pattern']['expression_boxplot'], + category='dupRadar'), + expression_barplot=report(c.patterns['dupradar']['pattern']['expression_barplot'], + caption=final_rst['dupradar']['pattern']['expression_barplot'], + category='dupRadar'), + multimapping_histogram=report(c.patterns['dupradar']['pattern']['multimapping_histogram'], + caption=final_rst['dupradar']['pattern']['multimapping_histogram'], + category='dupRadar'), + dataframe=report(c.patterns['dupradar']['pattern']['dataframe'], + caption=final_rst['dupradar']['pattern']['dataframe'], + category='dupRadar'), + model=report(c.patterns['dupradar']['pattern']['model'], + caption=final_rst['dupradar']['pattern']['model'], + category='dupRadar'), + curve=report(c.patterns['dupradar']['pattern']['curve'], + caption=final_rst['dupradar']['pattern']['curve'], + category='dupRadar'), + + log: c.patterns['dupradar']['pattern']['dataframe'] + '.log' script: wrapper_for('dupradar/wrapper.py') @@ -746,15 +833,17 @@ rule salmon: Quantify reads coming from transcripts with Salmon """ input: - fastq=common.fill_r1_r2(c.sampletable, c.patterns['cutadapt']), + fastq=common.fill_r1_r2(c.sampletable, c.patterns['cutadapt']['pattern']), index=c.refdict[c.organism][config['salmon']['tag']]['salmon'], output: - c.patterns['salmon'] + report(c.patterns['salmon']['pattern'], + caption=final_rst['salmon']['pattern'], + category='quantification') params: index_dir=os.path.dirname(c.refdict[c.organism][config['salmon']['tag']]['salmon']), - outdir=os.path.dirname(c.patterns['salmon']) + outdir=os.path.dirname(c.patterns['salmon']['pattern']) log: - c.patterns['salmon'] + '.log' + c.patterns['salmon']['pattern'] + '.log' run: paired = len(input.fastq) == 2 if paired: @@ -792,6 +881,7 @@ rule salmon: '-r {input.fastq} ' '&> {log}' ) + shell('mv {params.outdir}/quant.sf {output}') rule rseqc_bam_stat: @@ -799,9 +889,11 @@ rule rseqc_bam_stat: Calculate various BAM stats with RSeQC """ input: - bam=c.patterns['bam'] + bam=c.patterns['bam']['pattern'] output: - txt=c.patterns['rseqc']['bam_stat'] + txt=report(c.patterns['rseqc']['pattern']['bam_stat'], + caption=final_rst['rseqc']['pattern']['bam_stat'], + category='QC') wrapper: wrapper_for('rseqc/bam_stat') @@ -810,12 +902,12 @@ rule bigwig_neg: Create a bigwig for negative-strand reads """ input: - bam=c.patterns['bam'], - bai=c.patterns['bam'] + '.bai', - output: c.patterns['bigwig']['neg'] + bam=c.patterns['bam']['pattern'], + bai=c.patterns['bam']['pattern'] + '.bai', + output: c.patterns['bigwig']['pattern']['neg'] threads: 8 log: - c.patterns['bigwig']['neg'] + '.log' + c.patterns['bigwig']['pattern']['neg'] + '.log' shell: # NOTE: adjust bamCoverage params as needed # Make sure the bigwig_pos rule below reflects the same changes. @@ -836,12 +928,12 @@ rule bigwig_pos: Create a bigwig for postive-strand reads. """ input: - bam=c.patterns['bam'], - bai=c.patterns['bam'] + '.bai', - output: c.patterns['bigwig']['pos'] + bam=c.patterns['bam']['pattern'], + bai=c.patterns['bam']['pattern'] + '.bai', + output: c.patterns['bigwig']['pattern']['pos'] threads: 8 log: - c.patterns['bigwig']['pos'] + '.log' + c.patterns['bigwig']['pattern']['pos'] + '.log' shell: # NOTE: adjust bamCoverage params as needed # Make sure the bigwig_neg rule above reflects the same changes. @@ -859,11 +951,16 @@ rule bigwig_pos: rule symlink_bigwigs: input: - pos=c.patterns['bigwig']['pos'], - neg=c.patterns['bigwig']['neg'], + pos=c.patterns['bigwig']['pattern']['pos'], + neg=c.patterns['bigwig']['pattern']['neg'], output: - sense=c.patterns['bigwig']['sense'], - antisense=c.patterns['bigwig']['antisense'], + sense=report(c.patterns['bigwig']['pattern']['sense'], + caption=final_rst['bigwig']['pattern']['sense'], + category='bigwig'), + antisense=report(c.patterns['bigwig']['pattern']['antisense'], + caption=final_rst['bigwig']['pattern']['antisense'], + category='bigwig'), + run: # NOTE: # In our test data, reads mapping to the positive strand correspond @@ -886,14 +983,15 @@ rule rnaseq_rmarkdown: Run and render the RMarkdown file that performs differential expression """ input: - featurecounts=utils.flatten(c.targets['featurecounts']), - salmon=utils.flatten(c.targets['salmon']), + featurecounts=utils.flatten(c.targets['featurecounts']['pattern']), + salmon=utils.flatten(c.targets['salmon']['pattern']), # NOTE: the Rmd will likely need heavy editing depending on the project. rmd='downstream/rnaseq.Rmd', sampletable=config['sampletable'] output: - 'downstream/rnaseq.html' + report('downstream/rnaseq.html', + caption=final_rst['downstream']['pattern']['rnaseq']) shell: 'Rscript -e ' '''"rmarkdown::render('{input.rmd}', 'knitrBootstrap::bootstrap_document')"''' @@ -904,11 +1002,11 @@ def bigwigs_to_merge(wc): antisense_labels = chunk.get('antisense', []) sense_labels = chunk.get('sense', []) sense_bigwigs = expand( - c.patterns['bigwig']['sense'], + c.patterns['bigwig']['pattern']['sense'], sample=sense_labels ) antisense_bigwigs = expand( - c.patterns['bigwig']['antisense'], + c.patterns['bigwig']['pattern']['antisense'], sample=antisense_labels) return sense_bigwigs + antisense_bigwigs @@ -922,9 +1020,11 @@ if 'merged_bigwigs' in config: bigwigs=bigwigs_to_merge, chromsizes=refdict[c.organism][config['aligner']['tag']]['chromsizes'], output: - c.patterns['merged_bigwig'] + report(c.patterns['merged_bigwig']['pattern'], caption=( + final_rst['merged_bigwig']['pattern']), + category='bigwig') log: - c.patterns['merged_bigwig'] + '.log' + c.patterns['merged_bigwig']['pattern'] + '.log' script: wrapper_for('average-bigwigs/wrapper.py') diff --git a/workflows/rnaseq/config/rnaseq_patterns.yaml b/workflows/rnaseq/config/rnaseq_patterns.yaml index 95916f81..a11a0ada 100644 --- a/workflows/rnaseq/config/rnaseq_patterns.yaml +++ b/workflows/rnaseq/config/rnaseq_patterns.yaml @@ -1,52 +1,126 @@ -fastq: 'data/rnaseq_samples/{sample}/{sample}_R{n}.fastq.gz' -cutadapt: 'data/rnaseq_samples/{sample}/{sample}_R{n}.cutadapt.fastq.gz' -bam: 'data/rnaseq_samples/{sample}/{sample}.cutadapt.bam' +fastq: + pattern: 'data/rnaseq_samples/{sample}/{sample}_R{n}.fastq.gz' + description: 'Raw sequence reads' +cutadapt: + pattern: 'data/rnaseq_samples/{sample}/{sample}_R{n}.cutadapt.fastq.gz' + description: 'Sequence reads trimmed of adaptors' +bam: + pattern: 'data/rnaseq_samples/{sample}/{sample}.cutadapt.bam' + description: 'Binary file containing sequence alignment data' fastqc: - raw: 'data/rnaseq_samples/{sample}/fastqc/{sample}_R1.fastq.gz_fastqc.zip' - cutadapt: 'data/rnaseq_samples/{sample}/fastqc/{sample}_R1.cutadapt.fastq.gz_fastqc.zip' - bam: 'data/rnaseq_samples/{sample}/fastqc/{sample}.cutadapt.bam_fastqc.zip' + pattern: + raw: 'data/rnaseq_samples/{sample}/fastqc/{sample}_R1.fastq.gz_fastqc.zip' + cutadapt: 'data/rnaseq_samples/{sample}/fastqc/{sample}_R1.cutadapt.fastq.gz_fastqc.zip' + bam: 'data/rnaseq_samples/{sample}/fastqc/{sample}.cutadapt.bam_fastqc.zip' + description: + raw: 'Quality control analysis of raw sequence reads' + cutadapt: 'Quality control analysis of raw sequence reads post adaptor trimming' + bam: 'Quality control analysis of aligned reads' libsizes: - fastq: 'data/rnaseq_samples/{sample}/{sample}_R1.fastq.gz.libsize' - cutadapt: 'data/rnaseq_samples/{sample}/{sample}_R1.cutadapt.fastq.gz.libsize' - bam: 'data/rnaseq_samples/{sample}/{sample}.cutadapt.bam.libsize' -fastq_screen: 'data/rnaseq_samples/{sample}/{sample}.cutadapt.screen.txt' + pattern: + fastq: 'data/rnaseq_samples/{sample}/{sample}_R1.fastq.gz.libsize' + cutadapt: 'data/rnaseq_samples/{sample}/{sample}_R1.cutadapt.fastq.gz.libsize' + bam: 'data/rnaseq_samples/{sample}/{sample}.cutadapt.bam.libsize' + description: + fastq: 'Sample library size using raw sequence reads' + cutadapt: 'Sample library size using sequence reads post adaptor trimming' + bam: 'Sample library size using aligned reads' +fastq_screen: + pattern: 'data/rnaseq_samples/{sample}/{sample}.cutadapt.screen.txt' + description: 'fastq screen statistics' featurecounts: - s0: 'data/rnaseq_samples/{sample}/{sample}.cutadapt.bam.featurecounts.s0.txt' - s1: 'data/rnaseq_samples/{sample}/{sample}.cutadapt.bam.featurecounts.s1.txt' - s2: 'data/rnaseq_samples/{sample}/{sample}.cutadapt.bam.featurecounts.s2.txt' -libsizes_table: 'data/rnaseq_aggregation/libsizes_table.tsv' -libsizes_yaml: 'data/rnaseq_aggregation/libsizes_table_mqc.yaml' -rrna_percentages_table: 'data/rnaseq_aggregation/rrna_percentages_table.tsv' -rrna_percentages_yaml: 'data/rnaseq_aggregation/rrna_percentages_table_mqc.yaml' + pattern: + s0: 'data/rnaseq_samples/{sample}/{sample}.cutadapt.bam.featurecounts.s0.txt' + s1: 'data/rnaseq_samples/{sample}/{sample}.cutadapt.bam.featurecounts.s1.txt' + s2: 'data/rnaseq_samples/{sample}/{sample}.cutadapt.bam.featurecounts.s2.txt' + description: + s0: 'Transcripts quantification using unstranded reads' + s1: 'Transcripts quantification using Read1 as sense reads' + s2: 'Transcripts quantification using Read2 as sense reads' +libsizes_table: + pattern: 'data/rnaseq_aggregation/libsizes_table.tsv' + description: 'Aggregated libraries size per chromosome' +libsizes_yaml: + pattern: 'data/rnaseq_aggregation/libsizes_table_mqc.yaml' + description: 'Config for aggregation of libraries size per chromosome' +rrna_percentages_table: + pattern: 'data/rnaseq_aggregation/rrna_percentages_table.tsv' + description: 'Aggregated rRNA percentage per library' +rrna_percentages_yaml: + pattern: 'data/rnaseq_aggregation/rrna_percentages_table_mqc.yaml' + description: 'Config for aggregation of rRNA percentage per library' rrna: - bam: 'data/rnaseq_samples/{sample}/rRNA/{sample}.cutadapt.rrna.bam' - libsize: 'data/rnaseq_samples/{sample}/rRNA/{sample}.cutadapt.rrna.bam.libsize' -multiqc: 'data/rnaseq_aggregation/multiqc.html' + pattern: + bam: 'data/rnaseq_samples/{sample}/rRNA/{sample}.cutadapt.rrna.bam' + libsize: 'data/rnaseq_samples/{sample}/rRNA/{sample}.cutadapt.rrna.bam.libsize' + description: + bam: 'Binary file containing sequence aligning to rRNA' + libsize: 'Sample library sizes aligning to rRNA' +multiqc: + pattern: 'data/rnaseq_aggregation/multiqc.html' + description: 'Aggregated Quality Control analysis' markduplicates: - bam: 'data/rnaseq_samples/{sample}/{sample}.cutadapt.markdups.bam' - metrics: 'data/rnaseq_samples/{sample}/{sample}.cutadapt.markdups.bam.metrics' + pattern: + bam: 'data/rnaseq_samples/{sample}/{sample}.cutadapt.markdups.bam' + metrics: 'data/rnaseq_samples/{sample}/{sample}.cutadapt.markdups.bam.metrics' + description: + bam: 'Binary file containing sequence alignment data for duplicate reads' + metrics: 'Metrics details of duplicate reads' collectrnaseqmetrics: - metrics: 'data/rnaseq_samples/{sample}/{sample}.collectrnaseqmetrics.metrics' - pdf: 'data/rnaseq_samples/{sample}/{sample}.collectrnaseqmetrics.pdf' + pattern: + metrics: 'data/rnaseq_samples/{sample}/{sample}.collectrnaseqmetrics.metrics' + pdf: 'data/rnaseq_samples/{sample}/{sample}.collectrnaseqmetrics.pdf' + description: + metrics: 'Details of RNAseq metrics' + pdf: 'Summary pdf of RNAseq metrics' dupradar: - density_scatter: 'data/rnaseq_samples/{sample}/dupradar/{sample}_density_scatter.png' - expression_histogram: 'data/rnaseq_samples/{sample}/dupradar/{sample}_expression_histogram.png' - expression_boxplot: 'data/rnaseq_samples/{sample}/dupradar/{sample}_expression_boxplot.png' - expression_barplot: 'data/rnaseq_samples/{sample}/dupradar/{sample}_expression_barplot.png' - multimapping_histogram: 'data/rnaseq_samples/{sample}/dupradar/{sample}_multimapping_histogram.png' - dataframe: 'data/rnaseq_samples/{sample}/dupradar/{sample}_dataframe.tsv' - model: 'data/rnaseq_samples/{sample}/dupradar/{sample}_model.txt' - curve: 'data/rnaseq_samples/{sample}/dupradar/{sample}_curve.txt' -preseq: 'data/rnaseq_samples/{sample}/{sample}_preseq_c_curve.txt' -salmon: 'data/rnaseq_samples/{sample}/{sample}.salmon/quant.sf' + pattern: + density_scatter: 'data/rnaseq_samples/{sample}/dupradar/{sample}_density_scatter.png' + expression_histogram: 'data/rnaseq_samples/{sample}/dupradar/{sample}_expression_histogram.png' + expression_boxplot: 'data/rnaseq_samples/{sample}/dupradar/{sample}_expression_boxplot.png' + expression_barplot: 'data/rnaseq_samples/{sample}/dupradar/{sample}_expression_barplot.png' + multimapping_histogram: 'data/rnaseq_samples/{sample}/dupradar/{sample}_multimapping_histogram.png' + dataframe: 'data/rnaseq_samples/{sample}/dupradar/{sample}_dataframe.tsv' + model: 'data/rnaseq_samples/{sample}/dupradar/{sample}_model.txt' + curve: 'data/rnaseq_samples/{sample}/dupradar/{sample}_curve.txt' + description: + density_scatter: 'Density scatterplot for Duplication rate quality control' + expression_histogram: 'Expression histogram for Duplication rate quality control' + expression_boxplot: 'Expression boxplot for Duplication rate quality control' + expression_barplot: 'Expression barplot for Duplication rate quality control' + multimapping_histogram: 'Histogram of multimapping for Duplication rate quality control' + dataframe: 'Results table for Duplication rate quality control' + model: 'Model used in Duplication rate quality control' + curve: 'Curve for Duplication rate quality control' +preseq: + pattern: 'data/rnaseq_samples/{sample}/{sample}_preseq_c_curve.txt' + description: 'Preseq' +salmon: + pattern: 'data/rnaseq_samples/{sample}/{sample}.salmon/{sample}_quant.sf' + description: 'Transcripts quantification using Salmon' rseqc: - bam_stat: 'data/rnaseq_samples/{sample}/rseqc/{sample}_bam_stat.txt' + pattern: + bam_stat: 'data/rnaseq_samples/{sample}/rseqc/{sample}_bam_stat.txt' + description: + bam_stat: 'RNAseq quality control analysis' bigwig: - pos: 'data/rnaseq_samples/{sample}/{sample}.cutadapt.bam.pos.bigwig' - neg: 'data/rnaseq_samples/{sample}/{sample}.cutadapt.bam.neg.bigwig' - sense: 'data/rnaseq_samples/{sample}/{sample}.cutadapt.bam.sense.bigwig' - antisense: 'data/rnaseq_samples/{sample}/{sample}.cutadapt.bam.antisense.bigwig' + pattern: + pos: 'data/rnaseq_samples/{sample}/{sample}.cutadapt.bam.pos.bigwig' + neg: 'data/rnaseq_samples/{sample}/{sample}.cutadapt.bam.neg.bigwig' + sense: 'data/rnaseq_samples/{sample}/{sample}.cutadapt.bam.sense.bigwig' + antisense: 'data/rnaseq_samples/{sample}/{sample}.cutadapt.bam.antisense.bigwig' + description: + pos: 'bigwig file for positive strand relative to sequence reads' + neg: 'bigwig file for negative strand relative to sequence reads' + sense: 'bigwig file for sense strand relative to the transcript' + antisense: 'bigwig file for antisense strand relative to the transcript' downstream: - rnaseq: 'downstream/rnaseq.html' + pattern: + rnaseq: 'downstream/rnaseq.html' + description: + rnaseq: 'Results summary' + patterns_by_aggregate: - merged_bigwig: 'data/rnaseq_aggregation/merged_bigwigs/{merged_bigwig_label}.bigwig' + merged_bigwig: + pattern: 'data/rnaseq_aggregation/merged_bigwigs/{merged_bigwig_label}.bigwig' + description: 'Merged sense and antisense bigwigs for unstranded libraries' diff --git a/workflows/rnaseq/downstream/rnaseq.Rmd b/workflows/rnaseq/downstream/rnaseq.Rmd index ef443aa4..d1c4fadf 100644 --- a/workflows/rnaseq/downstream/rnaseq.Rmd +++ b/workflows/rnaseq/downstream/rnaseq.Rmd @@ -121,7 +121,7 @@ colData$featurecounts.path <- sapply( # you've changed anything there you will need to change it here as well. colData$salmon.path <- sapply( colData$samplename, - function (x) file.path('..', 'data', 'rnaseq_samples', x, paste0(x, '.salmon'), 'quant.sf') + function (x) file.path('..', 'data', 'rnaseq_samples', x, paste0(x, '.salmon'), paste0(x, '_quant.sf')) ) # NOTE: Factor columns. From 59e95c99d15e23de0a0a53feb7ddc3997d056b88 Mon Sep 17 00:00:00 2001 From: esnaultc Date: Mon, 26 Aug 2019 13:27:39 -0400 Subject: [PATCH 02/15] fix rnaseq_pattern index --- workflows/rnaseq/Snakefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/workflows/rnaseq/Snakefile b/workflows/rnaseq/Snakefile index 57b5ab30..189fd854 100644 --- a/workflows/rnaseq/Snakefile +++ b/workflows/rnaseq/Snakefile @@ -682,11 +682,11 @@ rule multiqc: ), config='config/multiqc_config.yaml' output: report(c.targets['multiqc']['pattern'], caption=final_rst['multiqc']['pattern'], category='QC') - log: c.targets['multiqc']['pattern'][0] + '.log' + log: c.targets['multiqc']['pattern'] + '.log' run: analysis_directory = set([os.path.dirname(i) for i in input]) - outdir = os.path.dirname(c.targets['multiqc']['pattern'][0]) - basename = os.path.basename(c.targets['multiqc']['pattern'][0]) + outdir = os.path.dirname(c.targets['multiqc']['pattern']) + basename = os.path.basename(c.targets['multiqc']['pattern']) shell( 'LC_ALL=en_US.UTF.8 LC_LANG=en_US.UTF-8 ' 'multiqc ' From 5ab2ee1f0b29e24d37686e065e4b17126768d85b Mon Sep 17 00:00:00 2001 From: Ryan Dale Date: Sun, 15 Sep 2019 10:37:43 -0400 Subject: [PATCH 03/15] add support for pulling out nested patterns and descriptions --- lib/patterns_targets.py | 119 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 118 insertions(+), 1 deletion(-) diff --git a/lib/patterns_targets.py b/lib/patterns_targets.py index 7d986b79..3ba098ae 100644 --- a/lib/patterns_targets.py +++ b/lib/patterns_targets.py @@ -9,6 +9,7 @@ from . import common from . import chipseq from . import helpers +from . import utils HERE = os.path.abspath(os.path.dirname(__file__)) @@ -61,7 +62,112 @@ def __init__(self, config, patterns, workdir=None): self.samples, self.sampletable = common.get_sampletable(self.config) self.refdict, self.conversion_kwargs = common.references_dict(self.config) self.organism = self.config['organism'] - self.patterns = yaml.load(open(patterns), Loader=yaml.FullLoader) + + # Patterns have one of three structures: + # + # Structure 1 (simple): + # + # salmon: + # pattern: 'data/rnaseq_samples/{sample}/{sample}.salmon/{sample}_quant.sf' + # description: 'Transcripts quantification using Salmon' + # + # Structure 2 (nested 1 deep): + # + # rseqc: + # bam_stat: + # pattern: 'data/rnaseq_samples/{sample}/rseqc/{sample}_bam_stat.txt' + # description: 'RNAseq quality control analysis' + # infer_experiment: + # pattern: 'data/rnaseq_samples/{sample}/rseqc/{sample}_infer_experiment.txt' + # description: 'Infer layout and strandedness of experiment' + # + # + # Structure 3 (nested 2 deep): + # + # patterns_by_peaks: + # peaks: + # macs2: + # pattern: 'data/chipseq_peaks/macs2/{macs2_run}/peaks.bed' + # description: 'BED file of peaks from macs2 peak-caller' + # + # + # Our job here is to create three objects: + # + # patterns = { + # 'salmon': 'data/rnaseq_samples/{sample}/{sample}.salmon/{sample}_quant.sf', + # 'rseqc': { + # 'bam_stat': 'data/rnaseq_samples/{sample}/rseqc/{sample}_bam_stat.txt', + # 'infer_experiment': 'data/rnaseq_samples/{sample}/rseqc/{sample}_infer_experiment.txt', + # }, + # 'patterns_by_peaks': { + # 'peaks': { + # 'macs2': 'data/chipseq_peaks/macs2/{macs2_run}/peaks.bed' + # } + # } + # } + # + # rst_files = { + # 'salmon': 'reports/data/rnaseq_samples/sample/sample.salmon/sample_quant.sf.rst', + # 'rseqc': { + # 'bam_stat': 'reports/data/rnaseq_samples/sample/rseqc/sample_bam_stat.txt', + # 'infer_experiment': 'reports/data/rnaseq_samples/sample/rseqc/sample_infer_experiment.txt.rst', + # }, + # 'patterns_by_peaks': { + # 'peaks': { + # 'macs2': 'reports/data/chipseq_peaks/macs2/macs2_run/peaks.bed.rst' + # } + # } + # + # descriptions = { + # 'salmon': 'Transcripts quantification using Salmon' + # 'rseqc': { + # 'bam_stat': 'RNAseq quality control analysis', + # 'infer_experiment': 'Infer layout and strandedness of experiment', + # }, + # 'patterns_by_peaks': { + # 'peaks': { + # 'macs2': 'BED file of peaks from macs2 peak-caller' + # } + # } + # } + # + # + _patterns = {} + _descriptions = {} + _rst_files = {} + + def pattern_to_rst_file(p): + return os.path.join('reports', p.replace('{', '').replace('}', '')) + '.rst' + + loaded_patterns = yaml.load(open(patterns), Loader=yaml.FullLoader) + for k, v in loaded_patterns.items(): + if 'pattern' in v: # simple + _patterns[k] = v['pattern'] + _rst_files[k] = pattern_to_rst_file(v['pattern']) + _descriptions[k] = v['description'] + + else: # nested + _patterns[k] = {} + _descriptions[k] = {} + _rst_files[k] = {} + for k2, v2 in v.items(): + if 'pattern' in v2: + _patterns[k][k2] = v2['pattern'] + _rst_files[k][k2] = pattern_to_rst_file(v2['pattern']) + _descriptions[k][k2] = v2['description'] + else: + _patterns[k][k2] = {} + _descriptions[k][k2] = {} + _rst_files[k][k2] = {} + for k3, v3 in v2.items(): + _patterns[k][k2][k3] = v3['pattern'] + _rst_files[k][k2][k3] = pattern_to_rst_file(v3['pattern']) + _descriptions[k][k2][k3] = v3['description'] + + self.patterns = _patterns + self.rst_files = _rst_files + self.descriptions = _descriptions + self.is_paired = helpers.detect_layout(self.sampletable) == 'PE' if self.is_paired: self.n = [1, 2] @@ -92,9 +198,14 @@ def __init__(self, config, patterns, workdir=None): SeqConfig.__init__(self, config, patterns, workdir) self.fill = dict(sample=self.samples, n=self.n) + + # The merged bigwigs have different placeholders and therefore must be + # filled separately. They also only should be included if + # merged_bigwigs have been configured. self.patterns_by_aggregation = self.patterns.pop('patterns_by_aggregate', None) self.targets = helpers.fill_patterns(self.patterns, self.fill, zip) + # Then the aggregation if self.patterns_by_aggregation is not None and 'merged_bigwigs' in self.config: self.fill_by_aggregation = dict( @@ -107,6 +218,9 @@ def __init__(self, config, patterns, workdir=None): self.targets.update(self.targets_by_aggregation) self.patterns.update(self.patterns_by_aggregation) + self.rst_files.update(self.rst_files.pop('patterns_by_aggregate')) + self.descriptions.update(self.descriptions.pop('patterns_by_aggregate')) + class ChIPSeqConfig(SeqConfig): def __init__(self, config, patterns, workdir=None): @@ -170,6 +284,9 @@ def __init__(self, config, patterns, workdir=None): self.targets.update(self.targets_by_aggregation) self.patterns.update(self.patterns_by_aggregation) + self.rst_files.update(self.rst_files.pop('patterns_by_aggregate')) + self.descriptions.update(self.descriptions.pop('patterns_by_aggregate')) + # Then the peaks... # # Note: when adding support for new peak callers, add them here. From 1683f6c07fcfe6b00e44abdefcc5d56357029d03 Mon Sep 17 00:00:00 2001 From: Ryan Dale Date: Sun, 15 Sep 2019 10:39:03 -0400 Subject: [PATCH 04/15] use new patterns_targets functionality in rnaseq snakefile --- workflows/rnaseq/Snakefile | 213 ++++++++++++++++++------------------- 1 file changed, 106 insertions(+), 107 deletions(-) diff --git a/workflows/rnaseq/Snakefile b/workflows/rnaseq/Snakefile index 7e1c3c01..62b90d60 100644 --- a/workflows/rnaseq/Snakefile +++ b/workflows/rnaseq/Snakefile @@ -47,7 +47,7 @@ def wrapper_for(path): # RULES # ---------------------------------------------------------------------------- -report: "report/workflow.rst" +report: "workflow.rst" # See "patterns and targets" in the documentation for what's going on here. final_targets = utils.flatten(( @@ -86,28 +86,27 @@ rule targets: Final targets to create """ input: - final_targets, - rstpaths + final_targets, utils.flatten(c.rst_files) + rule report_rst: """ Create .rst containing captions for report """ - input: + input: 'config/rnaseq_patterns.yaml' - output: - rstpaths + output: + utils.flatten(c.rst_files) run: - for key, val in final_rst.items(): - if type(val['description']) is not dict: - descr = val['description'] - pat = val['pattern'] - shell("echo {descr} > {pat}") - elif type(val['description']) is dict: - for nestkey, nestval in val['description'].items(): - descr = val['description'][nestkey] - pat = val['pattern'][nestkey] - shell("echo '{descr}' > {pat}") + + for k, v in c.rst_files.items(): + if isinstance(v, str): + with open(v, 'w') as fout: + fout.write(c.descriptions[k] + '\n') + else: + for k2, v2 in v.items(): + with open(v2, 'w') as fout: + fout.write(c.descriptions[k][k2]) if 'orig_filename' in c.sampletable.columns: @@ -142,7 +141,7 @@ if 'orig_filename' in c.sampletable.columns: rule symlink_targets: - input: c.targets['fastq']['pattern'] + input: c.targets['fastq'] if 'Run' in c.sampletable.columns and sum(c.sampletable['Run'].str.startswith('SRR')) > 0: @@ -153,7 +152,7 @@ if 'Run' in c.sampletable.columns and sum(c.sampletable['Run'].str.startswith('S rule fastq_dump: output: - fastq=render_r1_r2(c.patterns['fastq']['pattern']) + fastq=render_r1_r2(c.patterns['fastq']) run: srr = _st.loc[wildcards.sample, 'Run'] @@ -195,8 +194,8 @@ rule cutadapt: input: fastq=render_r1_r2(c.patterns['fastq']) output: - fastq=report(render_r1_r2(c.patterns['cutadapt']['pattern']), caption= - final_rst['cutadapt']['pattern'], + fastq=report(render_r1_r2(c.patterns['cutadapt']), caption= + c.rst_files['cutadapt'], category="fastq") log: render_r1_r2(c.patterns['cutadapt'])[0] + '.log' @@ -248,10 +247,10 @@ if config['aligner']['index'] == 'hisat2': Map reads with HISAT2 """ input: - fastq=common.fill_r1_r2(c.sampletable, c.patterns['cutadapt']['pattern']), + fastq=render_r1_r2(c.patterns['cutadapt']), index=[c.refdict[c.organism][config['aligner']['tag']]['hisat2']] output: - bam=c.patterns['bam']['pattern'] + bam=c.patterns['bam'] log: c.patterns['bam'] + '.log' threads: 6 @@ -288,12 +287,12 @@ if config['aligner']['index'] == 'star': Map reads with STAR """ input: - fastq=common.fill_r1_r2(c.sampletable, c.patterns['cutadapt']['pattern']), + fastq=render_r1_r2(c.patterns['cutadapt']), index=[c.refdict[c.organism][config['aligner']['tag']]['star']] output: - bam=c.patterns['bam']['pattern'] + bam=c.patterns['bam'] log: - c.patterns['bam']['pattern'] + '.log' + c.patterns['bam'] + '.log' threads: 6 run: genomedir = os.path.dirname(input.index[0]) @@ -335,7 +334,7 @@ rule rRNA: fastq=r1_only(c.patterns['cutadapt']), index=[c.refdict[c.organism][config['rrna']['tag']]['bowtie2']] output: - bam=c.patterns['rrna']['pattern']['bam'] + bam=c.patterns['rrna']['bam'] log: c.patterns['rrna']['bam'] + '.log' threads: 6 @@ -417,11 +416,11 @@ rule fastq_screen: **fastq_screen_references(), fastq=r1_only(rules.cutadapt.output.fastq), output: - txt=report(c.patterns['fastq_screen']['pattern'], - caption=final_rst['fastq_screen']['pattern'], + txt=report(c.patterns['fastq_screen'], + caption=c.rst_files['fastq_screen'], category="fastq screen") log: - c.patterns['fastq_screen']['pattern'] + '.log' + c.patterns['fastq_screen'] + '.log' params: subset=100000 script: wrapper_for('fastq_screen/wrapper.py') @@ -464,21 +463,21 @@ rule rrna_libsizes_table: Aggregate rRNA counts into a table """ input: - rrna=c.targets['rrna']['pattern']['libsize'], - fastq=c.targets['libsizes']['pattern']['cutadapt'] + rrna=c.targets['rrna']['libsize'], + fastq=c.targets['libsizes']['cutadapt'] output: - json=report(c.patterns['rrna_percentages_yaml']['pattern'], - caption=final_rst['rrna_percentages_yaml']['pattern'], + json=report(c.patterns['rrna_percentages_yaml'], + caption=c.rst_files['rrna_percentages_yaml'], category='Libsizes'), - tsv=report(c.patterns['rrna_percentages_table']['pattern'], - caption=final_rst['rrna_percentages_table']['pattern'], + tsv=report(c.patterns['rrna_percentages_table'], + caption=c.rst_files['rrna_percentages_table'], category='Libsizes'), run: def rrna_sample(f): - return helpers.extract_wildcards(c.patterns['rrna']['pattern']['libsize'], f)['sample'] + return helpers.extract_wildcards(c.patterns['rrna']['libsize'], f)['sample'] def sample(f): - return helpers.extract_wildcards(c.patterns['libsizes']['pattern']['cutadapt'], f)['sample'] + return helpers.extract_wildcards(c.patterns['libsizes']['cutadapt'], f)['sample'] def million(f): return float(open(f).read()) / 1e6 @@ -521,11 +520,11 @@ rule libsizes_table: input: utils.flatten(c.targets['libsizes']) output: - json=report(c.patterns['libsizes_yaml']['pattern'], - caption=final_rst['libsizes_yaml']['pattern'], + json=report(c.patterns['libsizes_yaml'], + caption=c.rst_files['libsizes_yaml'], category='Libsizes'), - tsv=report(c.patterns['libsizes_table']['pattern'], - caption=final_rst['libsizes_table']['pattern'], + tsv=report(c.patterns['libsizes_table'], + caption=c.rst_files['libsizes_table'], category='Libsizes'), run: @@ -579,27 +578,27 @@ rule multiqc: # add outputs from those rules to the inputs here. input: files=( - utils.flatten(c.targets['fastqc']['pattern']) + - utils.flatten(c.targets['libsizes_yaml']['pattern']) + - utils.flatten(c.targets['rrna_percentages_yaml']['pattern']) + - utils.flatten(c.targets['cutadapt']['pattern']) + - utils.flatten(c.targets['featurecounts']['pattern']) + - utils.flatten(c.targets['bam']['pattern']) + - utils.flatten(c.targets['markduplicates']['pattern']) + - utils.flatten(c.targets['salmon']['pattern']) + - utils.flatten(c.targets['rseqc']['pattern']) + - utils.flatten(c.targets['fastq_screen']['pattern']) + - utils.flatten(c.targets['dupradar']['pattern']) + - utils.flatten(c.targets['preseq']['pattern']) + - utils.flatten(c.targets['collectrnaseqmetrics']['pattern']) + utils.flatten(c.targets['fastqc']) + + utils.flatten(c.targets['libsizes_yaml']) + + utils.flatten(c.targets['rrna_percentages_yaml']) + + utils.flatten(c.targets['cutadapt']) + + utils.flatten(c.targets['featurecounts']) + + utils.flatten(c.targets['bam']) + + utils.flatten(c.targets['markduplicates']) + + utils.flatten(c.targets['salmon']) + + utils.flatten(c.targets['rseqc']) + + utils.flatten(c.targets['fastq_screen']) + + utils.flatten(c.targets['dupradar']) + + utils.flatten(c.targets['preseq']) + + utils.flatten(c.targets['collectrnaseqmetrics']) ), config='config/multiqc_config.yaml' - output: report(c.targets['multiqc']['pattern'], caption=final_rst['multiqc']['pattern'], category='QC') - log: c.targets['multiqc']['pattern'] + '.log' + output: report(c.patterns['multiqc'], caption=c.rst_files['multiqc'], category='QC') + log: c.patterns['multiqc'] + '.log' run: analysis_directory = set([os.path.dirname(i) for i in input]) - outdir = os.path.dirname(c.targets['multiqc']['pattern']) - basename = os.path.basename(c.targets['multiqc']['pattern']) + outdir = os.path.dirname(c.patterns['multiqc']) + basename = os.path.basename(c.patterns['multiqc']) shell( 'LC_ALL=en_US.UTF.8 LC_LANG=en_US.UTF-8 ' 'multiqc ' @@ -618,14 +617,14 @@ rule markduplicates: Mark or remove PCR duplicates with Picard MarkDuplicates """ input: - bam=c.patterns['bam']['pattern'] + bam=c.patterns['bam'] output: - bam=c.patterns['markduplicates']['pattern']['bam'], - metrics=c.patterns['markduplicates']['pattern']['metrics'], -# caption=final_rst['markduplicates']['pattern']['metrics'], + bam=c.patterns['markduplicates']['bam'], + metrics=c.patterns['markduplicates']['metrics'], +# caption=c.rst_files['markduplicates']['metrics'], # category='Metrics') log: - c.patterns['markduplicates']['pattern']['bam'] + '.log' + c.patterns['markduplicates']['bam'] + '.log' params: # NOTE: Be careful with the memory here; make sure you have enough # and/or it matches the resources you're requesting in the cluster @@ -647,14 +646,14 @@ rule collectrnaseqmetrics: Calculate various RNA-seq QC metrics with Picarc CollectRnaSeqMetrics """ input: - bam=c.patterns['bam']['pattern'], + bam=c.patterns['bam'], refflat=c.refdict[c.organism][config['gtf']['tag']]['refflat'] output: - metrics=report(c.patterns['collectrnaseqmetrics']['pattern']['metrics'], - caption=final_rst['collectrnaseqmetrics']['pattern']['metrics'], + metrics=report(c.patterns['collectrnaseqmetrics']['metrics'], + caption=c.rst_files['collectrnaseqmetrics']['metrics'], category='Metrics'), - pdf=report(c.patterns['collectrnaseqmetrics']['pattern']['pdf'], - caption=final_rst['collectrnaseqmetrics']['pattern']['pdf'], + pdf=report(c.patterns['collectrnaseqmetrics']['pdf'], + caption=c.rst_files['collectrnaseqmetrics']['pdf'], category='Metrics'), params: # NOTE: Be careful with the memory here; make sure you have enough @@ -663,7 +662,7 @@ rule collectrnaseqmetrics: java_args='-Xmx20g' # java_args='-Xmx2g' # [TEST SETTINGS -1] log: - c.patterns['collectrnaseqmetrics']['pattern']['metrics'] + '.log' + c.patterns['collectrnaseqmetrics']['metrics'] + '.log' shell: 'picard ' '{params.java_args} ' @@ -691,10 +690,10 @@ rule preseq: Compute a library complexity curve with preseq """ input: - bam=c.patterns['bam']['pattern'] + bam=c.patterns['bam'] output: - report(c.patterns['preseq']['pattern'], - caption=final_rst['preseq']['pattern'], + report(c.patterns['preseq'], + caption=c.rst_files['preseq'], category='Metrics'), shell: @@ -712,32 +711,32 @@ rule dupRadar: bam=rules.markduplicates.output.bam, annotation=c.refdict[c.organism][config['gtf']['tag']]['gtf'], output: - density_scatter=report(c.patterns['dupradar']['pattern']['density_scatter'], - caption=final_rst['dupradar']['pattern']['density_scatter'], + density_scatter=report(c.patterns['dupradar']['density_scatter'], + caption=c.rst_files['dupradar']['density_scatter'], category='dupRadar'), - expression_histogram=report(c.patterns['dupradar']['pattern']['expression_histogram'], - caption=final_rst['dupradar']['pattern']['expression_histogram'], + expression_histogram=report(c.patterns['dupradar']['expression_histogram'], + caption=c.rst_files['dupradar']['expression_histogram'], category='dupRadar'), - expression_boxplot=report(c.patterns['dupradar']['pattern']['expression_boxplot'], - caption=final_rst['dupradar']['pattern']['expression_boxplot'], + expression_boxplot=report(c.patterns['dupradar']['expression_boxplot'], + caption=c.rst_files['dupradar']['expression_boxplot'], category='dupRadar'), - expression_barplot=report(c.patterns['dupradar']['pattern']['expression_barplot'], - caption=final_rst['dupradar']['pattern']['expression_barplot'], + expression_barplot=report(c.patterns['dupradar']['expression_barplot'], + caption=c.rst_files['dupradar']['expression_barplot'], category='dupRadar'), - multimapping_histogram=report(c.patterns['dupradar']['pattern']['multimapping_histogram'], - caption=final_rst['dupradar']['pattern']['multimapping_histogram'], + multimapping_histogram=report(c.patterns['dupradar']['multimapping_histogram'], + caption=c.rst_files['dupradar']['multimapping_histogram'], category='dupRadar'), - dataframe=report(c.patterns['dupradar']['pattern']['dataframe'], - caption=final_rst['dupradar']['pattern']['dataframe'], + dataframe=report(c.patterns['dupradar']['dataframe'], + caption=c.rst_files['dupradar']['dataframe'], category='dupRadar'), - model=report(c.patterns['dupradar']['pattern']['model'], - caption=final_rst['dupradar']['pattern']['model'], + model=report(c.patterns['dupradar']['model'], + caption=c.rst_files['dupradar']['model'], category='dupRadar'), - curve=report(c.patterns['dupradar']['pattern']['curve'], - caption=final_rst['dupradar']['pattern']['curve'], + curve=report(c.patterns['dupradar']['curve'], + caption=c.rst_files['dupradar']['curve'], category='dupRadar'), - log: c.patterns['dupradar']['pattern']['dataframe'] + '.log' + log: c.patterns['dupradar']['dataframe'] + '.log' script: wrapper_for('dupradar/wrapper.py') @@ -747,15 +746,15 @@ rule salmon: Quantify reads coming from transcripts with Salmon """ input: - fastq=common.fill_r1_r2(c.sampletable, c.patterns['cutadapt']['pattern']), + fastq=common.fill_r1_r2(c.sampletable, c.patterns['cutadapt']), index=c.refdict[c.organism][config['salmon']['tag']]['salmon'], output: - report(c.patterns['salmon']['pattern'], - caption=final_rst['salmon']['pattern'], + report(c.patterns['salmon'], + caption=c.rst_files['salmon'], category='quantification') params: index_dir=os.path.dirname(c.refdict[c.organism][config['salmon']['tag']]['salmon']), - outdir=os.path.dirname(c.patterns['salmon']['pattern']) + outdir=os.path.dirname(c.patterns['salmon']) log: c.patterns['salmon'] + '.log' threads: 6 @@ -807,7 +806,7 @@ rule rseqc_bam_stat: Calculate various BAM stats with RSeQC """ input: - bam=c.patterns['bam']['pattern'] + bam=c.patterns['bam'] output: txt=c.patterns['rseqc']['bam_stat'] shell: @@ -835,12 +834,12 @@ rule bigwig_neg: Create a bigwig for negative-strand reads """ input: - bam=c.patterns['bam']['pattern'], - bai=c.patterns['bam']['pattern'] + '.bai', - output: c.patterns['bigwig']['pattern']['neg'] + bam=c.patterns['bam'], + bai=c.patterns['bam'] + '.bai', + output: c.patterns['bigwig']['neg'] threads: 8 log: - c.patterns['bigwig']['pattern']['neg'] + '.log' + c.patterns['bigwig']['neg'] + '.log' shell: # NOTE: adjust bamCoverage params as needed # Make sure the bigwig_pos rule below reflects the same changes. @@ -865,12 +864,12 @@ rule bigwig_pos: Create a bigwig for postive-strand reads. """ input: - bam=c.patterns['bam']['pattern'], - bai=c.patterns['bam']['pattern'] + '.bai', - output: c.patterns['bigwig']['pattern']['pos'] + bam=c.patterns['bam'], + bai=c.patterns['bam'] + '.bai', + output: c.patterns['bigwig']['pos'] threads: 8 log: - c.patterns['bigwig']['pattern']['pos'] + '.log' + c.patterns['bigwig']['pos'] + '.log' shell: # NOTE: adjust bamCoverage params as needed # Make sure the bigwig_neg rule above reflects the same changes. @@ -895,8 +894,8 @@ rule rnaseq_rmarkdown: Run and render the RMarkdown file that performs differential expression """ input: - featurecounts=utils.flatten(c.targets['featurecounts']['pattern']), - salmon=utils.flatten(c.targets['salmon']['pattern']), + featurecounts=utils.flatten(c.targets['featurecounts']), + salmon=utils.flatten(c.targets['salmon']), # NOTE: the Rmd will likely need heavy editing depending on the project. rmd='downstream/rnaseq.Rmd', @@ -935,11 +934,11 @@ if 'merged_bigwigs' in config: bigwigs=bigwigs_to_merge, chromsizes=refdict[c.organism][config['aligner']['tag']]['chromsizes'], output: - report(c.patterns['merged_bigwig']['pattern'], caption=( - final_rst['merged_bigwig']['pattern']), + report(c.patterns['merged_bigwig'], caption=( + c.rst_files['merged_bigwig']), category='bigwig') log: - c.patterns['merged_bigwig']['pattern'] + '.log' + c.patterns['merged_bigwig'] + '.log' script: wrapper_for('average-bigwigs/wrapper.py') From a80878e949e2c389cd6aa5874ada37dd141207d3 Mon Sep 17 00:00:00 2001 From: Ryan Dale Date: Sun, 15 Sep 2019 10:39:31 -0400 Subject: [PATCH 05/15] chipseq snakefile supports reporting --- workflows/chipseq/Snakefile | 26 ++++ .../chipseq/config/chipseq_patterns.yaml | 141 +++++++++++++----- 2 files changed, 132 insertions(+), 35 deletions(-) diff --git a/workflows/chipseq/Snakefile b/workflows/chipseq/Snakefile index c6ab2da7..d818b43c 100644 --- a/workflows/chipseq/Snakefile +++ b/workflows/chipseq/Snakefile @@ -41,6 +41,7 @@ c = ChIPSeqConfig( wildcard_constraints: n = '[1,2]' +report: "workflow.rst" def wrapper_for(path): return 'file:' + os.path.join('../..','wrappers', 'wrappers', path) @@ -50,6 +51,7 @@ def wrapper_for(path): # RULES # ---------------------------------------------------------------------------- # See "patterns and targets" in the documentation for what's going on here. + final_targets = utils.flatten(( c.targets['bam'], utils.flatten(c.targets['fastqc']), @@ -77,6 +79,30 @@ rule targets: input: final_targets +rule report_rst: + """ + Create .rst containing captions for report + """ + input: + 'config/chipseq_patterns.yaml' + output: + utils.flatten(c.rst_files) + run: + + for k, v in c.rst_files.items(): + if isinstance(v, str): + with open(v, 'w') as fout: + fout.write(c.descriptions[k] + '\n') + else: + for k2, v2 in v.items(): + if isinstance(v2, str): + with open(v2, 'w') as fout: + fout.write(c.descriptions[k][k2]) + else: + for k3, v3 in v2.items(): + with open(v3, 'w') as fout: + fout.write(c.descriptions[k][k2][k3]) + if 'orig_filename' in c.sampletable.columns: # Convert the sampletable to be indexed by the first column, for diff --git a/workflows/chipseq/config/chipseq_patterns.yaml b/workflows/chipseq/config/chipseq_patterns.yaml index 94130b54..36b1d23c 100644 --- a/workflows/chipseq/config/chipseq_patterns.yaml +++ b/workflows/chipseq/config/chipseq_patterns.yaml @@ -1,56 +1,127 @@ patterns_by_sample: - fastq: 'data/chipseq_samples/{sample}/{sample}_R1.fastq.gz' - cutadapt: 'data/chipseq_samples/{sample}/{sample}_R1.cutadapt.fastq.gz' - bam: 'data/chipseq_samples/{sample}/{sample}.cutadapt.bam' + fastq: + pattern: 'data/chipseq_samples/{sample}/{sample}_R1.fastq.gz' + description: 'Original FASTQ file' + cutadapt: + pattern: 'data/chipseq_samples/{sample}/{sample}_R1.cutadapt.fastq.gz' + description: 'Trimmed reads from cutadapt' + + bam: + pattern: 'data/chipseq_samples/{sample}/{sample}.cutadapt.bam' + description: 'Sorted, aligned BAM' fastqc: - raw: 'data/chipseq_samples/{sample}/fastqc/{sample}_R1.fastq.gz_fastqc.zip' - cutadapt: 'data/chipseq_samples/{sample}/fastqc/{sample}_R1.cutadapt.fastq.gz_fastqc.zip' - bam: 'data/chipseq_samples/{sample}/fastqc/{sample}.cutadapt.unique.nodups.bam_fastqc.zip' + raw: + pattern: 'data/chipseq_samples/{sample}/fastqc/{sample}_R1.fastq.gz_fastqc.zip' + description: 'Quality control analysis of raw sequence reads' + cutadapt: + pattern: 'data/chipseq_samples/{sample}/fastqc/{sample}_R1.cutadapt.fastq.gz_fastqc.zip' + description: 'Quality control analysis of raw sequence reads post adaptor trimming' + bam: + pattern: 'data/chipseq_samples/{sample}/fastqc/{sample}.cutadapt.unique.nodups.bam_fastqc.zip' + description: 'Quality control analysis of aligned reads' libsizes: - fastq: 'data/chipseq_samples/{sample}/{sample}_R1.fastq.gz.libsize' - cutadapt: 'data/chipseq_samples/{sample}/{sample}_R1.cutadapt.fastq.gz.libsize' - bam: 'data/chipseq_samples/{sample}/{sample}.cutadapt.bam.libsize' - unique: 'data/chipseq_samples/{sample}/{sample}.cutadapt.unique.bam.libsize' - nodups: 'data/chipseq_samples/{sample}/{sample}.cutadapt.unique.nodups.bam.libsize' - - fastq_screen: 'data/chipseq_samples/{sample}/{sample}.cutadapt.screen.txt' - libsizes_table: 'data/chipseq_aggregation/libsizes_table.tsv' - libsizes_yaml: 'data/chipseq_aggregation/libsizes_table_mqc.yaml' - multiqc: 'data/chipseq_aggregation/multiqc.html' - unique: 'data/chipseq_samples/{sample}/{sample}.cutadapt.unique.bam' + fastq: + pattern: 'data/chipseq_samples/{sample}/{sample}_R1.fastq.gz.libsize' + description: 'Sample library size using raw sequence reads' + cutadapt: + pattern: 'data/chipseq_samples/{sample}/{sample}_R1.cutadapt.fastq.gz.libsize' + description: 'Sample library size using sequence reads post adaptor trimming' + bam: + pattern: 'data/chipseq_samples/{sample}/{sample}.cutadapt.bam.libsize' + description: 'Sample library size using aligned reads' + unique: + pattern: 'data/chipseq_samples/{sample}/{sample}.cutadapt.unique.bam.libsize' + description: 'Sample library size using unique aligned reads' + nodups: + pattern: 'data/chipseq_samples/{sample}/{sample}.cutadapt.unique.nodups.bam.libsize' + description: 'Sample library size using unique, non-duplicate reads' + + fastq_screen: + pattern: 'data/chipseq_samples/{sample}/{sample}.cutadapt.screen.txt' + description: 'fastq screen statistics' + libsizes_table: + pattern: 'data/chipseq_aggregation/libsizes_table.tsv' + description: 'TSV of library sizes' + libsizes_yaml: + pattern: 'data/chipseq_aggregation/libsizes_table_mqc.yaml' + description: 'YAML file of library sizes, for MultiQC' + multiqc: + pattern: 'data/chipseq_aggregation/multiqc.html' + description: 'MultiQC output that aggregates many stages' + + unique: + pattern: 'data/chipseq_samples/{sample}/{sample}.cutadapt.unique.bam' + description: 'BAM file with multimappers removed' markduplicates: - bam: 'data/chipseq_samples/{sample}/{sample}.cutadapt.unique.nodups.bam' - metrics: 'data/chipseq_samples/{sample}/{sample}.cutadapt.unique.nodups.bam.metrics' + bam: + pattern: 'data/chipseq_samples/{sample}/{sample}.cutadapt.unique.nodups.bam' + description: 'BAM file with multimappers removed and duplicates removed' + + metrics: + pattern: 'data/chipseq_samples/{sample}/{sample}.cutadapt.unique.nodups.bam.metrics' + description: 'Metrics file from MarkDuplicates' - merged_techreps: 'data/chipseq_merged/{label}/{label}.cutadapt.unique.nodups.merged.bam' + merged_techreps: + pattern: 'data/chipseq_merged/{label}/{label}.cutadapt.unique.nodups.merged.bam' + description: 'BAM file of merged technical replicates' - bigwig: 'data/chipseq_merged/{label}/{label}.cutadapt.unique.nodups.bam.bigwig' + bigwig: + pattern: 'data/chipseq_merged/{label}/{label}.cutadapt.unique.nodups.bam.bigwig' + description: 'bigwig of merged technical replicates' fingerprint: - plot: 'data/chipseq_aggregation/fingerprints/{ip_label}/{ip_label}_fingerprint.png' - raw_counts: 'data/chipseq_aggregation/fingerprints/{ip_label}/{ip_label}_fingerprint.tab' - metrics: 'data/chipseq_aggregation/fingerprints/{ip_label}/{ip_label}_fingerprint.metrics' + plot: + pattern: 'data/chipseq_aggregation/fingerprints/{ip_label}/{ip_label}_fingerprint.png' + description: 'Fingerprint plot' + raw_counts: + pattern: 'data/chipseq_aggregation/fingerprints/{ip_label}/{ip_label}_fingerprint.tab' + description: 'TSV of results from fingerprint analysis' + metrics: + pattern: 'data/chipseq_aggregation/fingerprints/{ip_label}/{ip_label}_fingerprint.metrics' + description: 'Metrics file from fingerprint analysis' multibigwigsummary: - npz: 'data/chipseq_aggregation/deeptools/multibigwigsummary_matrix.npz' - tab: 'data/chipseq_aggregation/deeptools/multibigwigsummary.tab' + npz: + pattern: 'data/chipseq_aggregation/deeptools/multibigwigsummary_matrix.npz' + description: 'Compressed numpy array from multibigwigsummary output' + tab: + pattern: 'data/chipseq_aggregation/deeptools/multibigwigsummary.tab' + description: 'TSV of multibigwigsummary results' plotcorrelation: - tab: 'data/chipseq_aggregation/deeptools/plotcorrelation.tab' - heatmap: 'data/chipseq_aggregation/deeptools/correlation_heatmap.png' + tab: + pattern: 'data/chipseq_aggregation/deeptools/plotcorrelation.tab' + description: 'TSV of correlation results' + heatmap: + pattern: 'data/chipseq_aggregation/deeptools/correlation_heatmap.png' + description: 'Heatmap of correlations between samples' patterns_by_peaks: peaks: - macs2: 'data/chipseq_peaks/macs2/{macs2_run}/peaks.bed' - spp: 'data/chipseq_peaks/spp/{spp_run}/peaks.bed' - sicer: 'data/chipseq_peaks/sicer/{sicer_run}/peaks.bed' + macs2: + pattern: 'data/chipseq_peaks/macs2/{macs2_run}/peaks.bed' + description: 'BED file of peaks from macs2 peak-caller' + spp: + pattern: 'data/chipseq_peaks/spp/{spp_run}/peaks.bed' + description: 'BED file of peaks from spp peak-caller' + sicer: + pattern: 'data/chipseq_peaks/sicer/{sicer_run}/peaks.bed' + description: 'BED file of domains from SICER domain caller' bigbed: - macs2: 'data/chipseq_peaks/macs2/{macs2_run}/peaks.bigbed' - spp: 'data/chipseq_peaks/spp/{spp_run}/peaks.bigbed' - sicer: 'data/chipseq_peaks/sicer/{sicer_run}/peaks.bigbed' + macs2: + pattern: 'data/chipseq_peaks/macs2/{macs2_run}/peaks.bigbed' + description: 'bigBed file of peaks from macs2 peak-caller' + spp: + pattern: 'data/chipseq_peaks/spp/{spp_run}/peaks.bigbed' + description: 'bigBed file of peaks from spp peak-caller' + sicer: + pattern: 'data/chipseq_peaks/sicer/{sicer_run}/peaks.bigbed' + description: 'bigBed file of domains from SICER domain caller' patterns_by_aggregate: - merged_bigwig: 'data/chipseq_aggregation/merged_bigwigs/{merged_bigwig_label}.bigwig' + merged_bigwig: + pattern: 'data/chipseq_aggregation/merged_bigwigs/{merged_bigwig_label}.bigwig' + description: 'Merged bigwigs as specified in config' From b69a9477e010f37562148a56bdd95df1c7f52997 Mon Sep 17 00:00:00 2001 From: Ryan Dale Date: Sun, 15 Sep 2019 10:40:10 -0400 Subject: [PATCH 06/15] add workflow.rst for rnaseq and chipseq --- workflows/chipseq/workflow.rst | 4 +++ workflows/rnaseq/workflow.rst | 54 ++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 workflows/chipseq/workflow.rst create mode 100644 workflows/rnaseq/workflow.rst diff --git a/workflows/chipseq/workflow.rst b/workflows/chipseq/workflow.rst new file mode 100644 index 00000000..0240ddcf --- /dev/null +++ b/workflows/chipseq/workflow.rst @@ -0,0 +1,4 @@ +lcdb-wf is a collection of snakemake workflows and tools for common high-throughput sequencing analysis, along with associated infrastructure. + +See docs at https://lcdb.github.io/lcdb-wf. + diff --git a/workflows/rnaseq/workflow.rst b/workflows/rnaseq/workflow.rst new file mode 100644 index 00000000..a7b0d8c0 --- /dev/null +++ b/workflows/rnaseq/workflow.rst @@ -0,0 +1,54 @@ +lcdb-wf is a collection of snakemake workflows and tools for common high-throughput sequencing analysis, along with associated infrastructure. + +See docs at https://lcdb.github.io/lcdb-wf. + +RNASEQ workflow + +This workflow is used for RNA-seq and RNA-seq-like analysis (like euRNA-seq, RIP-seq or small RNA-seq). + +This workflow can use references created by the references workflow with no need to run the references workflow separately. This workflow performs the following tasks: + +- Build a HISAT2 index +- Build a salmon transcriptome index +- Download a GTF annotation +- Convert the GTF to refflat format +- Trim reads with cutadapt +- Align with HISAT2 +- Run FastQC on raw, trimmed, and aligned reads +- Align reads to rRNA using bowtie2 to evaluate rRNA contamination +- Count reads in genes with featureCounts +- Run dupRadar and preseq to assess library complexity +- Check for evidence of cross-contamination using fastq_screen on multiple configured genomes +- Assess transcript coverage with Picard CollectRnaSeqMetrics +- Build bigWigs (optionally strand-specific) created from BAM files +- Optionally merge bigWigs as defined by config +- Aggregate QC results using MultiQC. Includes custom tables for library sizes and rRNA contamination +- Run various QC and differential expression. This is performed in an RMarkdown file that runs a standard DESeq2 differential expression analysis along with diagnostic plots, exported tables of differentially expressed genes for each contrast, and downstream GO analysis using clusterProfiler. This file is run and rendered into an output HTML file. +- Construct and upload a track hub of scaled coverage bigWigs for each sample that can be viewed in UCSC Genome Browser + + +Configurations used: + +{% for items in snakemake.config %} + {% if items != 'references'%} + - {{ items }} : {{snakemake.config[items]}} + {% elif items == 'references' %} + - References: + + {% for sublista in snakemake.config[items] %} + - {{sublista}} : + + {% for sublistb in snakemake.config[items][sublista] %} + - {{ sublistb }} : + + {% for sublistc in snakemake.config[items][sublista][sublistb] %} + + - {{sublistc}} : {{snakemake.config[items][sublista][sublistb][sublistc]}} + + {% endfor %} + {% endfor %} + {% endfor %} + {% endif %} +{% endfor %} + + From 09e7f2b999b7630a75da17b11d367c619631e336 Mon Sep 17 00:00:00 2001 From: Ryan Dale Date: Sun, 15 Sep 2019 10:47:57 -0400 Subject: [PATCH 07/15] add map_nested_dicts and extract_nested util funcs --- lib/utils.py | 86 +++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 69 insertions(+), 17 deletions(-) diff --git a/lib/utils.py b/lib/utils.py index 97e352c1..5132cbce 100644 --- a/lib/utils.py +++ b/lib/utils.py @@ -1,7 +1,7 @@ import os import contextlib import collections -from collections.abc import Iterable +from collections.abc import Iterable, Mapping from snakemake.shell import shell @@ -50,23 +50,56 @@ def gen(): return results -def test_flatten(): - assert sorted(flatten({ - 'a': { - 'b': { - 'c': ['a', 'b', 'c'], - }, - }, - 'x': ['e', 'f', 'g'], - 'y': { - 'z': 'd' - }, - })) == ['a', 'b', 'c', 'd', 'e', 'f', 'g'] +def map_nested_dicts(d, func): + """ + Apply `func` to all values of a nested dictionary - assert flatten('a', True) == 'a' - assert flatten(['a'], True) == 'a' - assert flatten('a') == ['a'] - assert flatten(['a']) == ['a'] + Parameters + ---------- + d : dict + + func : callable + + Examples + -------- + + >>> d = {'a': {'b': {'target': 1, 'ignore': 2}}, 'c': {'target': 3}} + >>> res = map_nested_dicts(d, lambda x: x < 3) + >>> assert res == {'a': {'b': {'target': True, 'ignore': True}}, 'c': {'target': False}} + + """ + if isinstance(d, Mapping): + return {k: map_nested_dicts(v, func) for k, v in d.items()} + else: + return func(d) + + +def extract_nested(d, key): + """ + From a nested dict, keep all nesting the same EXCEPT for the leaf dicts, + from which only the provided key will be returned. + + Parameters + ---------- + d : dict + + key : str or hashable type + Key to extract. Effectively collapses leaf dictionaries containing this + key into just the value. + + Examples + -------- + + >>> d = {'a': {'b': {'target': 1, 'ignore': 2}}, 'c': {'target': 3}} + >>> result = extract_nested(d, 'target') + >>> assert result == {'a': {'b': 1}, 'c': 3}, result + """ + if not isinstance(d, Mapping): + return d + if key in d: + return d[key] + else: + return {k: extract_nested(v, key) for k,v in d.items()} def updatecopy(orig, update_with, keys=None, override=False): @@ -186,3 +219,22 @@ def make_relative_symlink(target, linkname): if not os.path.exists(linkdir): shell('mkdir -p {linkdir}') shell('cd {linkdir}; ln -sf {relative_target} {linkbase}') + + +def test_flatten(): + assert sorted(flatten({ + 'a': { + 'b': { + 'c': ['a', 'b', 'c'], + }, + }, + 'x': ['e', 'f', 'g'], + 'y': { + 'z': 'd' + }, + })) == ['a', 'b', 'c', 'd', 'e', 'f', 'g'] + + assert flatten('a', True) == 'a' + assert flatten(['a'], True) == 'a' + assert flatten('a') == ['a'] + assert flatten(['a']) == ['a'] From e32c33716a850574363d4e0e767f837b341f0d33 Mon Sep 17 00:00:00 2001 From: Ryan Dale Date: Sun, 15 Sep 2019 10:51:59 -0400 Subject: [PATCH 08/15] patterns_targets uses new utils funcs --- lib/patterns_targets.py | 43 ++++++++++------------------------------- 1 file changed, 10 insertions(+), 33 deletions(-) diff --git a/lib/patterns_targets.py b/lib/patterns_targets.py index 3ba098ae..35620e88 100644 --- a/lib/patterns_targets.py +++ b/lib/patterns_targets.py @@ -132,43 +132,20 @@ def __init__(self, config, patterns, workdir=None): # } # # - _patterns = {} - _descriptions = {} - _rst_files = {} def pattern_to_rst_file(p): - return os.path.join('reports', p.replace('{', '').replace('}', '')) + '.rst' + """ + Convert filename pattern containing wildcards into an RST filename + """ + return os.path.join("reports", p.replace("{", "").replace("}", "")) + ".rst" loaded_patterns = yaml.load(open(patterns), Loader=yaml.FullLoader) - for k, v in loaded_patterns.items(): - if 'pattern' in v: # simple - _patterns[k] = v['pattern'] - _rst_files[k] = pattern_to_rst_file(v['pattern']) - _descriptions[k] = v['description'] - - else: # nested - _patterns[k] = {} - _descriptions[k] = {} - _rst_files[k] = {} - for k2, v2 in v.items(): - if 'pattern' in v2: - _patterns[k][k2] = v2['pattern'] - _rst_files[k][k2] = pattern_to_rst_file(v2['pattern']) - _descriptions[k][k2] = v2['description'] - else: - _patterns[k][k2] = {} - _descriptions[k][k2] = {} - _rst_files[k][k2] = {} - for k3, v3 in v2.items(): - _patterns[k][k2][k3] = v3['pattern'] - _rst_files[k][k2][k3] = pattern_to_rst_file(v3['pattern']) - _descriptions[k][k2][k3] = v3['description'] - - self.patterns = _patterns - self.rst_files = _rst_files - self.descriptions = _descriptions - - self.is_paired = helpers.detect_layout(self.sampletable) == 'PE' + + self.patterns = utils.extract_nested(loaded_patterns, "pattern") + self.descriptions = utils.extract_nested(loaded_patterns, "description") + self.rst_files = utils.map_nested_dicts(self.patterns, pattern_to_rst_file) + + self.is_paired = helpers.detect_layout(self.sampletable) == "PE" if self.is_paired: self.n = [1, 2] else: From 58ab78c192fbe9e15fb387c1902813c083ca495a Mon Sep 17 00:00:00 2001 From: Ryan Dale Date: Sun, 15 Sep 2019 13:17:58 -0400 Subject: [PATCH 09/15] mv pattern_to_rst to utils; update dicts based on sub-dicts --- lib/utils.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lib/utils.py b/lib/utils.py index 5132cbce..64768168 100644 --- a/lib/utils.py +++ b/lib/utils.py @@ -3,6 +3,11 @@ import collections from collections.abc import Iterable, Mapping from snakemake.shell import shell +from snakemake.io import expand + + +def render_r1_r2(pattern): + return expand(pattern, sample='{sample}', n=[1,2]) @contextlib.contextmanager @@ -102,6 +107,13 @@ def extract_nested(d, key): return {k: extract_nested(v, key) for k,v in d.items()} +def pattern_to_rst_file(p): + """ + Convert filename pattern containing wildcards into an RST filename + """ + return os.path.join("reports", p.replace("{", "").replace("}", "")) + ".rst" + + def updatecopy(orig, update_with, keys=None, override=False): """ Update a copy of a dictionary, with a bit more control than the built-in From be4e1e429ae5ca0fe9efc3e4be0270b4ede6b168 Mon Sep 17 00:00:00 2001 From: Ryan Dale Date: Sun, 15 Sep 2019 13:18:19 -0400 Subject: [PATCH 10/15] with prev commit --- lib/patterns_targets.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/patterns_targets.py b/lib/patterns_targets.py index 35620e88..c6f6a299 100644 --- a/lib/patterns_targets.py +++ b/lib/patterns_targets.py @@ -133,17 +133,13 @@ def __init__(self, config, patterns, workdir=None): # # - def pattern_to_rst_file(p): - """ - Convert filename pattern containing wildcards into an RST filename - """ - return os.path.join("reports", p.replace("{", "").replace("}", "")) + ".rst" loaded_patterns = yaml.load(open(patterns), Loader=yaml.FullLoader) + self._loaded_patterns = loaded_patterns self.patterns = utils.extract_nested(loaded_patterns, "pattern") self.descriptions = utils.extract_nested(loaded_patterns, "description") - self.rst_files = utils.map_nested_dicts(self.patterns, pattern_to_rst_file) + self.rst_files = utils.map_nested_dicts(self.patterns, utils.pattern_to_rst_file) self.is_paired = helpers.detect_layout(self.sampletable) == "PE" if self.is_paired: @@ -247,6 +243,8 @@ def __init__(self, config, patterns, workdir=None): self.targets.update(self.targets_by_sample) self.patterns.update(self.patterns_by_sample) + self.descriptions.update(self.descriptions['patterns_by_sample']) + self.rst_files.update(self.rst_files['patterns_by_sample']) # Then the aggregation... self.patterns_by_aggregation = self.patterns.pop('patterns_by_aggregate', None) @@ -319,3 +317,5 @@ def __init__(self, config, patterns, workdir=None): self.targets.update(self.targets_for_peaks) self.patterns.update(self.patterns_by_peaks) + self.descriptions.update(self.descriptions['patterns_by_peaks']) + self.rst_files.update(self.rst_files['patterns_by_peaks']) From f2525b5608fd59ae57ef98738cc01611b96733be Mon Sep 17 00:00:00 2001 From: Ryan Dale Date: Sun, 15 Sep 2019 13:18:55 -0400 Subject: [PATCH 11/15] add more functionality to map_nested_dicts --- lib/utils.py | 74 +++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 70 insertions(+), 4 deletions(-) diff --git a/lib/utils.py b/lib/utils.py index 64768168..4d9e72c6 100644 --- a/lib/utils.py +++ b/lib/utils.py @@ -55,7 +55,7 @@ def gen(): return results -def map_nested_dicts(d, func): +def map_nested_dicts(d, func, stop_condition=None): """ Apply `func` to all values of a nested dictionary @@ -64,21 +64,68 @@ def map_nested_dicts(d, func): d : dict func : callable + Function to apply to values of d, or, if `stop_condition` is provided, + function to apply to remainder when `stop_condition(d)` is True. + + stop_condition : callable + Mechanism for stopping recursion at a particular level and sending the + results at that point to `func`. Function should accept a dict as its + only input argument. Examples -------- - >>> d = {'a': {'b': {'target': 1, 'ignore': 2}}, 'c': {'target': 3}} + Convert leaf values into boolean indicating if they are less than three: + + >>> d = {'a': {'b': {'target': 1, 'nontarget': 2}}, 'c': {'target': 3}} >>> res = map_nested_dicts(d, lambda x: x < 3) - >>> assert res == {'a': {'b': {'target': True, 'ignore': True}}, 'c': {'target': False}} + >>> assert res == {'a': {'b': {'target': True, 'nontarget': True}}, 'c': {'target': False}} + + + This function will sum values of provided dictionaries + + >>> def sum_values(x): + ... if isinstance(x, Mapping): + ... return sum(x.values()) + + Since we don't specify a stopping condition which would send a dict to + `sum_values`, only the leaf integers get sent and the `sum_values` will + return None for those: + + >>> res = map_nested_dicts(d, sum_values) + >>> assert res == {'a': {'b': {'target': None, 'nontarget': None}}, 'c': {'target': None}}, res + + Here the stopping condition is whether "target" is in the keys, and if so, + the dict for which that is true is sent to `sum_values`: + + + >>> def stop1(x): + ... return isinstance(x, Mapping) and 'target' in x.keys() + + >>> res = map_nested_dicts(d, sum_values, stop_condition=stop1) + >>> assert res == {'a': {'b': 3}, 'c': 3}, res + + + Now if we only send dicts with "nontarget" in the keys, values in `b` are + summed but values in `c` are not because nothing there satisfied the + stopping condition: + + >>> def stop2(x): + ... return isinstance(x, Mapping) and 'nontarget' in x.keys() + + >>> res = map_nested_dicts(d, sum_values, stop_condition=stop2) + >>> assert res == {'a': {'b': 3}, 'c': {'target': None}}, res """ + if stop_condition and stop_condition(d): + return func(d) if isinstance(d, Mapping): - return {k: map_nested_dicts(v, func) for k, v in d.items()} + return {k: map_nested_dicts(v, func, stop_condition) for k, v in d.items()} else: return func(d) + def extract_nested(d, key): """ From a nested dict, keep all nesting the same EXCEPT for the leaf dicts, @@ -114,6 +161,25 @@ def pattern_to_rst_file(p): return os.path.join("reports", p.replace("{", "").replace("}", "")) + ".rst" +def write_out_rsts(full_patterns): + """ + Given the full patterns dictionary (containing patterns and descriptions), + write out a corresponding rst file containing the contents of the + description. + + Returns None; the side effect is to create all the necessary rst files. + """ + def stop_condition(x): + return isinstance(x, Mapping) and 'pattern' in x and 'description' in x + + def writer(x): + rst = pattern_to_rst_file(x['pattern']) + desc = x['description'] + with open(rst, 'w') as fout: + fout.write(desc + '\n') + + map_nested_dicts(full_patterns, func=writer, stop_condition=stop_condition) + def updatecopy(orig, update_with, keys=None, override=False): """ Update a copy of a dictionary, with a bit more control than the built-in From 39cc5e7e56be5896fcd8c86328d39fbe3148ef5c Mon Sep 17 00:00:00 2001 From: Ryan Dale Date: Sun, 15 Sep 2019 13:19:22 -0400 Subject: [PATCH 12/15] simplify and add report outputs --- workflows/chipseq/Snakefile | 33 ++++++++++----------------------- 1 file changed, 10 insertions(+), 23 deletions(-) diff --git a/workflows/chipseq/Snakefile b/workflows/chipseq/Snakefile index d818b43c..6e74d0cf 100644 --- a/workflows/chipseq/Snakefile +++ b/workflows/chipseq/Snakefile @@ -9,6 +9,7 @@ import numpy as np import pybedtools from lib import common, cluster_specific, utils, helpers, aligners, chipseq from lib.patterns_targets import ChIPSeqConfig +from lib.utils import render_r1_r2 # ---------------------------------------------------------------------------- # @@ -76,7 +77,7 @@ rule targets: """ Final targets to create """ - input: final_targets + input: final_targets, utils.flatten(c.rst_files) rule report_rst: @@ -86,22 +87,10 @@ rule report_rst: input: 'config/chipseq_patterns.yaml' output: - utils.flatten(c.rst_files) + list(set(utils.flatten(c.rst_files))) run: + utils.write_out_rsts(c._loaded_patterns) - for k, v in c.rst_files.items(): - if isinstance(v, str): - with open(v, 'w') as fout: - fout.write(c.descriptions[k] + '\n') - else: - for k2, v2 in v.items(): - if isinstance(v2, str): - with open(v2, 'w') as fout: - fout.write(c.descriptions[k][k2]) - else: - for k3, v3 in v2.items(): - with open(v3, 'w') as fout: - fout.write(c.descriptions[k][k2][k3]) if 'orig_filename' in c.sampletable.columns: @@ -192,8 +181,6 @@ if 'Run' in c.sampletable.columns and sum(c.sampletable['Run'].str.startswith('S ) -def render_r1_r2(pattern): - return expand(pattern, sample='{sample}', n=[1,2]) rule cutadapt: """ @@ -437,7 +424,7 @@ rule multiqc: ), config='config/multiqc_config.yaml' output: - c.targets['multiqc'] + report(c.targets['multiqc'], caption=c.rst_files['multiqc']) log: c.targets['multiqc'][0] + '.log' run: @@ -553,7 +540,7 @@ rule fingerprint: bais=lambda wc: expand(c.patterns['merged_techreps'] + '.bai', label=wc.ip_label), control_bais=lambda wc: expand(c.patterns['merged_techreps'] + '.bai', label=chipseq.merged_input_for_ip(c.sampletable, wc.ip_label)), output: - plot=c.patterns['fingerprint']['plot'], + plot=report(c.patterns['fingerprint']['plot']), raw_counts=c.patterns['fingerprint']['raw_counts'], metrics=c.patterns['fingerprint']['metrics'] threads: 8 @@ -595,7 +582,7 @@ rule sicer: ), chromsizes=refdict[c.organism][config['aligner']['tag']]['chromsizes'], output: - bed=c.patterns['peaks']['sicer'] + bed=report(c.patterns['peaks']['sicer']) log: c.patterns['peaks']['sicer'] + '.log' params: @@ -620,7 +607,7 @@ rule macs2: ), chromsizes=refdict[c.organism][config['aligner']['tag']]['chromsizes'], output: - bed=c.patterns['peaks']['macs2'] + bed=report(c.patterns['peaks']['macs2']) log: c.patterns['peaks']['macs2'] + '.log' params: @@ -646,7 +633,7 @@ rule spp: ), chromsizes=refdict[c.organism][config['aligner']['tag']]['chromsizes'], output: - bed=c.patterns['peaks']['spp'], + bed=report(c.patterns['peaks']['spp']), enrichment_estimates=c.patterns['peaks']['spp'] + '.est.wig', smoothed_enrichment_mle=c.patterns['peaks']['spp'] + '.mle.wig', rdata=c.patterns['peaks']['spp'] + '.RData' @@ -739,7 +726,7 @@ rule plotcorrelation: input: c.targets['multibigwigsummary']['npz'] output: - heatmap=c.targets['plotcorrelation']['heatmap'], + heatmap=report(c.targets['plotcorrelation']['heatmap']), tab=c.targets['plotcorrelation']['tab'] shell: 'plotCorrelation ' From a53398131a4c6496ae4a4f63ae35eba868966906 Mon Sep 17 00:00:00 2001 From: Ryan Dale Date: Sun, 15 Sep 2019 13:19:38 -0400 Subject: [PATCH 13/15] simplify rst creation --- workflows/rnaseq/Snakefile | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/workflows/rnaseq/Snakefile b/workflows/rnaseq/Snakefile index 62b90d60..7360d6c9 100644 --- a/workflows/rnaseq/Snakefile +++ b/workflows/rnaseq/Snakefile @@ -98,15 +98,7 @@ rule report_rst: output: utils.flatten(c.rst_files) run: - - for k, v in c.rst_files.items(): - if isinstance(v, str): - with open(v, 'w') as fout: - fout.write(c.descriptions[k] + '\n') - else: - for k2, v2 in v.items(): - with open(v2, 'w') as fout: - fout.write(c.descriptions[k][k2]) + utils.write_out_rsts(c._loaded_patterns) if 'orig_filename' in c.sampletable.columns: From 69a7122377268faf0c6ade406e9262bca83a1553 Mon Sep 17 00:00:00 2001 From: Ryan Dale Date: Sun, 15 Sep 2019 13:21:50 -0400 Subject: [PATCH 14/15] simplify snakefiles by factoring out functions --- lib/utils.py | 8 ++++++++ workflows/chipseq/Snakefile | 5 +---- workflows/rnaseq/Snakefile | 13 ++----------- 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/lib/utils.py b/lib/utils.py index 4d9e72c6..b41229d9 100644 --- a/lib/utils.py +++ b/lib/utils.py @@ -6,10 +6,18 @@ from snakemake.io import expand +def wrapper_for(path): + return 'file:' + os.path.join('../..','wrappers', 'wrappers', path) + + def render_r1_r2(pattern): return expand(pattern, sample='{sample}', n=[1,2]) +def r1_only(pattern): + return expand(pattern, sample='{sample}', n=1) + + @contextlib.contextmanager def temp_env(env): """ diff --git a/workflows/chipseq/Snakefile b/workflows/chipseq/Snakefile index 6e74d0cf..30e1bc8d 100644 --- a/workflows/chipseq/Snakefile +++ b/workflows/chipseq/Snakefile @@ -9,7 +9,7 @@ import numpy as np import pybedtools from lib import common, cluster_specific, utils, helpers, aligners, chipseq from lib.patterns_targets import ChIPSeqConfig -from lib.utils import render_r1_r2 +from lib.utils import render_r1_r2, r1_only, wrapper_for # ---------------------------------------------------------------------------- # @@ -44,9 +44,6 @@ wildcard_constraints: report: "workflow.rst" -def wrapper_for(path): - return 'file:' + os.path.join('../..','wrappers', 'wrappers', path) - # ---------------------------------------------------------------------------- # RULES diff --git a/workflows/rnaseq/Snakefile b/workflows/rnaseq/Snakefile index 7360d6c9..14348303 100644 --- a/workflows/rnaseq/Snakefile +++ b/workflows/rnaseq/Snakefile @@ -8,6 +8,8 @@ import tempfile import pandas as pd from lib import common, cluster_specific, utils, helpers, aligners from lib.patterns_targets import RNASeqConfig +from lib.utils import render_r1_r2, r1_only, wrapper_for + import copy # ---------------------------------------------------------------------------- @@ -39,10 +41,6 @@ wildcard_constraints: n = '[1,2]' -def wrapper_for(path): - return 'file:' + os.path.join('../..','wrappers', 'wrappers', path) - - # ---------------------------------------------------------------------------- # RULES # ---------------------------------------------------------------------------- @@ -74,13 +72,6 @@ if 'merged_bigwigs' in config: final_targets.extend(utils.flatten(c.targets['merged_bigwig'])) -def render_r1_r2(pattern, r1_only=False): - return expand(pattern, sample='{sample}', n=c.n) - -def r1_only(pattern): - return expand(pattern, sample='{sample}', n=1) - - rule targets: """ Final targets to create From da65e6330b573ede9dd6122f6267f65abe1337b4 Mon Sep 17 00:00:00 2001 From: Ryan Dale Date: Sun, 15 Sep 2019 13:22:14 -0400 Subject: [PATCH 15/15] add reporting to tests --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5f12fcb0..ea7c773f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -74,6 +74,7 @@ variables: cd workflows/chipseq source activate lcdb-wf-test ./run_test.sh --use-conda -j2 -k -p -r + ./run_test.sh --report python chipseq_trackhub.py config/config.yaml config/hub_config.yaml chipseq-regression-step: &chipseq-regression-step @@ -103,6 +104,7 @@ variables: cd workflows/rnaseq source activate lcdb-wf-test ./run_test.sh --use-conda -j2 -k -p -r + ./run_test.sh --report python rnaseq_trackhub.py config/config.yaml config/hub_config.yaml rnaseq-star-step: &rnaseq-star-step