add UTILS_PIPELINE.groovy and semi-automate pipeline level tests and …

…snapshots
nf-core · Sep 22, 2024 · b837413 · b837413
1 parent 63e4153
commit b837413
Show file tree

Hide file tree

Showing 12 changed files with 3,629 additions and 542 deletions.
diff --git a/tests/bismark_emseq_clip_r1.nf.test b/tests/bismark_emseq_clip_r1.nf.test
@@ -14,64 +14,29 @@ nextflow_pipeline {
         }
 
         then {
+            // stable_name: All files + folders in ${params.outdir}/ with a stable name
+            def stable_name = UTILS.getAllFilesFromDir(params.outdir, true, UTILS.exclusionRegexesForUnstableFileNames)
+
+            // stable_path: All files in ${params.outdir}/ with stable content
+            def stable_path = UTILS.getAllFilesFromDir(params.outdir, false, UTILS.exclusionRegexesForUnstableFileNames + UTILS_PIPELINE.exclusionRegexesForUnstableFileContents)
+
             assertAll(
-                { assert workflow.success },
+                { assert workflow.success},
                 { assert snapshot(
-                    path("$outputDir/bismark/deduplicated/Ecoli.deduplicated.sorted.bam"),
-                    path("$outputDir/bismark/deduplicated/SRR389222_sub1.deduplicated.sorted.bam"),
-                    path("$outputDir/bismark/deduplicated/SRR389222_sub2.deduplicated.sorted.bam"),
-                    path("$outputDir/bismark/deduplicated/SRR389222_sub3.deduplicated.sorted.bam"),
-                    path("$outputDir/bismark/methylation_calls/bedGraph/Ecoli_1_val_1_bismark_bt2_pe.deduplicated.bedGraph.gz"),
-                    path("$outputDir/bismark/methylation_calls/bedGraph/SRR389222_sub1_trimmed_bismark_bt2.deduplicated.bedGraph.gz"),
-                    path("$outputDir/bismark/methylation_calls/bedGraph/SRR389222_sub2_trimmed_bismark_bt2.deduplicated.bedGraph.gz"),
-                    path("$outputDir/bismark/methylation_calls/bedGraph/SRR389222_sub3_trimmed_bismark_bt2.deduplicated.bedGraph.gz"),
-                    path("$outputDir/bismark/methylation_calls/mbias/Ecoli_1_val_1_bismark_bt2_pe.deduplicated.M-bias.txt"),
-                    path("$outputDir/bismark/methylation_calls/mbias/SRR389222_sub1_trimmed_bismark_bt2.deduplicated.M-bias.txt"),
-                    path("$outputDir/bismark/methylation_calls/mbias/SRR389222_sub2_trimmed_bismark_bt2.deduplicated.M-bias.txt"),
-                    path("$outputDir/bismark/methylation_calls/mbias/SRR389222_sub3_trimmed_bismark_bt2.deduplicated.M-bias.txt"),
-                    path("$outputDir/bismark/methylation_calls/methylation_coverage/Ecoli_1_val_1_bismark_bt2_pe.deduplicated.bismark.cov.gz"),
-                    path("$outputDir/bismark/methylation_calls/methylation_coverage/SRR389222_sub1_trimmed_bismark_bt2.deduplicated.bismark.cov.gz"),
-                    path("$outputDir/bismark/methylation_calls/methylation_coverage/SRR389222_sub2_trimmed_bismark_bt2.deduplicated.bismark.cov.gz"),
-                    path("$outputDir/bismark/methylation_calls/methylation_coverage/SRR389222_sub3_trimmed_bismark_bt2.deduplicated.bismark.cov.gz"),
-                    path("$outputDir/bismark/methylation_calls/methylation_calls/").list(),
-                    path("$outputDir/bismark/methylation_calls/splitting_report/Ecoli_1_val_1_bismark_bt2_pe.deduplicated_splitting_report.txt"),
-                    path("$outputDir/bismark/methylation_calls/splitting_report/SRR389222_sub1_trimmed_bismark_bt2.deduplicated_splitting_report.txt"),
-                    path("$outputDir/bismark/methylation_calls/splitting_report/SRR389222_sub2_trimmed_bismark_bt2.deduplicated_splitting_report.txt"),
-                    path("$outputDir/bismark/methylation_calls/splitting_report/SRR389222_sub3_trimmed_bismark_bt2.deduplicated_splitting_report.txt"),
-                    path("$outputDir/qualimap/Ecoli/genome_results.txt"),
-                    path("$outputDir/qualimap/SRR389222_sub1/genome_results.txt"),
-                    path("$outputDir/qualimap/SRR389222_sub2/genome_results.txt"),
-                    path("$outputDir/qualimap/SRR389222_sub3/genome_results.txt")
-                ).match("Bismark em_seq_clip_r1") },
+                    // Number of tasks
+                    workflow.trace.tasks().size(),
 
-                { assert new File("$outputDir/bismark/alignments/logs/Ecoli_1_val_1_bismark_bt2_PE_report.txt").exists() },
-                { assert new File("$outputDir/bismark/alignments/logs/SRR389222_sub1_trimmed_bismark_bt2_SE_report.txt").exists() },
-                { assert new File("$outputDir/bismark/alignments/logs/SRR389222_sub2_trimmed_bismark_bt2_SE_report.txt").exists() },
-                { assert new File("$outputDir/bismark/alignments/logs/SRR389222_sub3_trimmed_bismark_bt2_SE_report.txt").exists() },
-                { assert new File("$outputDir/bismark/deduplicated/logs/Ecoli_1_val_1_bismark_bt2_pe.deduplication_report.txt").exists() },
-                { assert new File("$outputDir/bismark/deduplicated/logs/SRR389222_sub1_trimmed_bismark_bt2.deduplication_report.txt").exists() },
-                { assert new File("$outputDir/bismark/deduplicated/logs/SRR389222_sub2_trimmed_bismark_bt2.deduplication_report.txt").exists() },
-                { assert new File("$outputDir/bismark/deduplicated/logs/SRR389222_sub3_trimmed_bismark_bt2.deduplication_report.txt").exists() },
-                { assert new File("$outputDir/bismark/reports/Ecoli_1_val_1_bismark_bt2_PE_report.html").exists() },
-                { assert new File("$outputDir/bismark/reports/SRR389222_sub1_trimmed_bismark_bt2_SE_report.html").exists() },
-                { assert new File("$outputDir/bismark/reports/SRR389222_sub2_trimmed_bismark_bt2_SE_report.html").exists() },
-                { assert new File("$outputDir/bismark/reports/SRR389222_sub3_trimmed_bismark_bt2_SE_report.html").exists() },
-                { assert new File("$outputDir/bismark/summary/bismark_summary_report.html").exists() },
-                { assert new File("$outputDir/bismark/summary/bismark_summary_report.txt").exists() },
-                { assert new File("$outputDir/fastqc/Ecoli_1_fastqc.html").exists() },
-                { assert new File("$outputDir/fastqc/Ecoli_2_fastqc.html").exists() },
-                { assert new File("$outputDir/fastqc/SRR389222_sub1_fastqc.html").exists() },
-                { assert new File("$outputDir/fastqc/SRR389222_sub2_fastqc.html").exists() },
-                { assert new File("$outputDir/fastqc/SRR389222_sub3_fastqc.html").exists() },
-                { assert new File("$outputDir/multiqc/bismark/multiqc_report.html").exists() },
-                { assert new File("$outputDir/trimgalore/logs/Ecoli_1.fastq.gz_trimming_report.txt").exists() },
-                { assert new File("$outputDir/trimgalore/logs/Ecoli_2.fastq.gz_trimming_report.txt").exists() },
-                { assert new File("$outputDir/trimgalore/logs/SRR389222_sub1.fastq.gz_trimming_report.txt").exists() },
-                { assert new File("$outputDir/trimgalore/logs/SRR389222_sub2.fastq.gz_trimming_report.txt").exists() },
-                { assert new File("$outputDir/trimgalore/logs/SRR389222_sub3.fastq.gz_trimming_report.txt").exists() },
+                    // pipeline versions.yml file for multiqc from which Nextflow version is removed because we tests pipelines on multiple Nextflow versions
+                    UTILS.removeNextflowVersion("$outputDir/pipeline_info/nf_core_pipeline_software_mqc_versions.yml"),
 
-                { assert snapshot(UTILS.removeNextflowVersion("$outputDir")).match("software_versions") },
+                    // All stable path name
+                    stable_name*.name,
+
+                    // All files with stable contents
+                    stable_path
+                ).match() }
             )
+
         }
     }