update modules/subworkflows and fix tests

conf/modules.config

@@ -18,16 +18,4 @@ process {
         saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
     ]
 
-    withName: FASTQC {
-        ext.args = '--quiet'
-    }
-    withName: 'MULTIQC' {
-        ext.args   = { params.multiqc_title ? "--title \"$params.multiqc_title\"" : '' }
-        publishDir = [
-            path: { "${params.outdir}/multiqc" },
-            mode: params.publish_dir_mode,
-            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
-        ]
-    }
-
 }

modules.json

@@ -8,7 +8,7 @@
                     "custom/sratoolsncbisettings": {
                         "branch": "master",
                         "git_sha": "666652151335353eef2fcd58880bcef5bc2928e1",
-                        "installed_by": ["modules"]
+                        "installed_by": ["fastq_download_prefetch_fasterqdump_sratools", "modules"]
                     },
                     "sratools/fasterqdump": {
                         "branch": "master",

nf-test.config

@@ -1,10 +1,19 @@
 config {
-    // Location of nf-tests
+    // Location for all nf-tests
     testsDir "."
 
-    // nf-test directory used to create temporary files for each test
-    workDir System.getenv("NFT_WORKDIR") ?: ".nf-test"
+    // nf-test directory including temporary files for each test
+    workDir ".nf-test"
 
-    // Location of an optional nextflow.config file specific for executing pipeline tests
-    configFile "tests/nextflow.config"
+    // Location of an optional nextflow.config file specific for executing tests
+    configFile "conf/test.config"
+
+    // Run all test with defined profile(s) from the main nextflow.config
+    profile "test"
+
+    // Include plugins
+    plugins {
+        load "[email protected]"
+        load "[email protected]"
+    }
 }

subworkflows/local/utils_nfcore_fetchngs_pipeline/main.nf

@@ -27,12 +27,12 @@ workflow PIPELINE_INITIALISATION {
 
     take:
     version             // boolean: Display version and exit
-    help                // boolean: Display help text
     validate_params     // boolean: Boolean whether to validate parameters against the schema at runtime
     monochrome_logs     // boolean: Do not use coloured log outputs
     nextflow_cli_args   //   array: List of positional nextflow CLI args
     outdir              //  string: The output directory where the results will be saved
     input               //  string: File containing SRA/ENA/GEO/DDBJ identifiers one per line to download their associated metadata and FastQ files
+    ena_metadata_fields // string: Comma-separated list of ENA metadata fields to fetch before downloading data
 
     main:
 

tests/.nftignore

@@ -0,0 +1,2 @@
+pipeline_info/*.{html,json,txt,yml}
+samplesheet/samplesheet.csv

tests/main.nf.test

@@ -14,67 +14,26 @@ nextflow_pipeline {
         }
 
         then {
-            assert workflow.success
-
+            // stable_name: All files + folders in ${params.outdir}/ with a stable name
+            def stable_name = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}'])
+            // stable_path: All files in ${params.outdir}/ with stable content
+            def stable_path = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore')
             assertAll(
-                { assert new File("$outputDir/samplesheet/samplesheet.csv").readLines().size() == 15 },
-                { assert new File("$outputDir/samplesheet/samplesheet.csv").readLines()*.split(',')[0].take(4) == ['"sample"', '"fastq_1"', '"fastq_2"', '"run_accession"'] },
-                { assert new File("$outputDir/samplesheet/samplesheet.csv").readLines()*.split(',').collect { it[0] } == ['"sample"', '"DRX024467"', '"DRX026011"', '"ERX1234253"', '"SRX10940790"', '"SRX11047067"', '"SRX17709227"', '"SRX17709228"', '"SRX6725035"', '"SRX9315476"', '"SRX9504942"', '"SRX9504942"', '"SRX9504942"', '"SRX9504942"', '"SRX9626017"'] },
-                { assert new File("$outputDir/samplesheet/samplesheet.csv").text.contains('Illumina HiSeq 2500') },
-                { assert new File("$outputDir/custom/user-settings.mkfg").exists() },
-                { assert new File("$outputDir/fastq/md5/DRX024467_DRR026872.fastq.gz.md5").exists() },
-                { assert new File("$outputDir/fastq/md5/DRX026011_DRR028935_1.fastq.gz.md5").exists() },
-                { assert new File("$outputDir/fastq/md5/DRX026011_DRR028935_2.fastq.gz.md5").exists() },
-                { assert new File("$outputDir/fastq/md5/ERX1234253_ERR1160846.fastq.gz.md5").exists() },
-                { assert new File("$outputDir/fastq/md5/SRX17709227_SRR21711856.fastq.gz.md5").exists() },
-                { assert new File("$outputDir/fastq/md5/SRX17709228_SRR21711855.fastq.gz.md5").exists() },
-                { assert new File("$outputDir/fastq/md5/SRX6725035_SRR9984183.fastq.gz.md5").exists() },
-                { assert new File("$outputDir/fastq/md5/SRX9504942_SRR13055517_1.fastq.gz.md5").exists() },
-                { assert new File("$outputDir/fastq/md5/SRX9504942_SRR13055517_2.fastq.gz.md5").exists() },
-                { assert new File("$outputDir/fastq/md5/SRX9504942_SRR13055518_1.fastq.gz.md5").exists() },
-                { assert new File("$outputDir/fastq/md5/SRX9504942_SRR13055518_2.fastq.gz.md5").exists() },
-                { assert new File("$outputDir/fastq/md5/SRX9504942_SRR13055519_1.fastq.gz.md5").exists() },
-                { assert new File("$outputDir/fastq/md5/SRX9504942_SRR13055519_2.fastq.gz.md5").exists() },
-                { assert new File("$outputDir/fastq/md5/SRX9504942_SRR13055520_1.fastq.gz.md5").exists() },
-                { assert new File("$outputDir/fastq/md5/SRX9504942_SRR13055520_2.fastq.gz.md5").exists() },
-                { assert new File("$outputDir/fastq/md5/SRX9626017_SRR13191702_1.fastq.gz.md5").exists() },
-                { assert new File("$outputDir/fastq/md5/SRX9626017_SRR13191702_2.fastq.gz.md5").exists() },
-                { assert new File("$outputDir/fastq/DRX024467_DRR026872.fastq.gz").exists() },
-                { assert new File("$outputDir/fastq/DRX026011_DRR028935_1.fastq.gz").exists() },
-                { assert new File("$outputDir/fastq/DRX026011_DRR028935_2.fastq.gz").exists() },
-                { assert new File("$outputDir/fastq/ERX1234253_ERR1160846.fastq.gz").exists() },
-                { assert new File("$outputDir/fastq/SRX10940790_SRR14593545_1.fastq.gz").exists() },
-                { assert new File("$outputDir/fastq/SRX10940790_SRR14593545_2.fastq.gz").exists() },
-                { assert new File("$outputDir/fastq/SRX11047067_SRR14709033.fastq.gz").exists() },
-                { assert new File("$outputDir/fastq/SRX17709227_SRR21711856.fastq.gz").exists() },
-                { assert new File("$outputDir/fastq/SRX17709228_SRR21711855.fastq.gz").exists() },
-                { assert new File("$outputDir/fastq/SRX6725035_SRR9984183.fastq.gz").exists() },
-                { assert new File("$outputDir/fastq/SRX9315476_SRR12848126_1.fastq.gz").exists() },
-                { assert new File("$outputDir/fastq/SRX9315476_SRR12848126_2.fastq.gz").exists() },
-                { assert new File("$outputDir/fastq/SRX9504942_SRR13055517_1.fastq.gz").exists() },
-                { assert new File("$outputDir/fastq/SRX9504942_SRR13055517_2.fastq.gz").exists() },
-                { assert new File("$outputDir/fastq/SRX9504942_SRR13055518_1.fastq.gz").exists() },
-                { assert new File("$outputDir/fastq/SRX9504942_SRR13055518_2.fastq.gz").exists() },
-                { assert new File("$outputDir/fastq/SRX9504942_SRR13055519_1.fastq.gz").exists() },
-                { assert new File("$outputDir/fastq/SRX9504942_SRR13055519_2.fastq.gz").exists() },
-                { assert new File("$outputDir/fastq/SRX9504942_SRR13055520_1.fastq.gz").exists() },
-                { assert new File("$outputDir/fastq/SRX9504942_SRR13055520_2.fastq.gz").exists() },
-                { assert new File("$outputDir/fastq/SRX9626017_SRR13191702_1.fastq.gz").exists() },
-                { assert new File("$outputDir/fastq/SRX9626017_SRR13191702_2.fastq.gz").exists() },
-                { assert new File("$outputDir/metadata/DRR026872.runinfo_ftp.tsv").exists() },
-                { assert new File("$outputDir/metadata/DRR028935.runinfo_ftp.tsv").exists() },
-                { assert new File("$outputDir/metadata/ERR1160846.runinfo_ftp.tsv").exists() },
-                { assert new File("$outputDir/metadata/GSE214215.runinfo_ftp.tsv").exists() },
-                { assert new File("$outputDir/metadata/GSM4907283.runinfo_ftp.tsv").exists() },
-                { assert new File("$outputDir/metadata/SRR12848126.runinfo_ftp.tsv").exists() },
-                { assert new File("$outputDir/metadata/SRR13191702.runinfo_ftp.tsv").exists() },
-                { assert new File("$outputDir/metadata/SRR14593545.runinfo_ftp.tsv").exists() },
-                { assert new File("$outputDir/metadata/SRR14709033.runinfo_ftp.tsv").exists() },
-                { assert new File("$outputDir/metadata/SRR9984183.runinfo_ftp.tsv").exists() },
-                { assert new File("$outputDir/pipeline_info/nf_core_fetchngs_software_mqc_versions.yml").exists() },
-                { assert new File("$outputDir/samplesheet/id_mappings.csv").exists() },
-                { assert new File("$outputDir/samplesheet/multiqc_config.yml").exists() },
-                { assert new File("$outputDir/samplesheet/samplesheet.csv").exists() }
+                { assert workflow.success},
+                { assert snapshot(
+                    // Number of successful tasks
+                    workflow.trace.succeeded().size(),
+                    // pipeline versions.yml file for multiqc from which Nextflow version is removed because we tests pipelines on multiple Nextflow versions
+                    removeNextflowVersion("$outputDir/pipeline_info/nf_core_fetchngs_software_mqc_versions.yml"),
+                    // All stable path name, with a relative path
+                    stable_name,
+                    // All stable path
+                    stable_path
+                    // new File("$outputDir/samplesheet/samplesheet.csv").readLines().size(),
+                    // new File("$outputDir/samplesheet/samplesheet.csv").readLines()*.split(',')[0].take(4),
+                    // new File("$outputDir/samplesheet/samplesheet.csv").readLines()*.split(',').collect { it[0] },
+                    // new File("$outputDir/samplesheet/samplesheet.csv").text.contains('Illumina HiSeq 2500')
+                ).match()}
             )
         }
     }