Merge branch 'master' into add-json-support

CHANGELOG.md

@@ -1,5 +1,17 @@
 # nextflow-io/nf-validation: Changelog
 
+# Version 1.1.3 - Asahikawa
+
+## Improvements
+
+- Added support for double quotes (`"`) in CSV and TSV samplesheets ([#134](https://github.com/nextflow-io/nf-validation/pull/134))
+
+# Version 1.1.2 - Wakayama
+
+## Bug fixes
+
+- Fixed an issue with inputs using `file-path-pattern` where only one file was found (`Path` casting to `ArrayList` error) ([#132](https://github.com/nextflow-io/nf-validation/pull/132))
+
 # Version 1.1.1 - Shoyu
 
 ## Bug fixes

parameters_meta_schema.json

@@ -0,0 +1,155 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema",
+  "$id": "https://nextflow.io",
+  "title": "Nextflow Schema Meta-schema",
+  "description": "Meta-schema to validate Nextflow parameter schema files",
+  "type": "object",
+  "properties": {
+    "$schema": {
+      "title": "schema",
+      "type": "string",
+      "minLength": 1
+    },
+    "$id": {
+      "title": "ID URI",
+      "type": "string",
+      "minLength": 1
+    },
+    "title": {
+      "title": "Title",
+      "type": "string",
+      "minLength": 1
+    },
+    "description": {
+      "title": "Description",
+      "type": "string",
+      "minLength": 1
+    },
+    "type": {
+      "title": "Top level type",
+      "type": "string",
+      "const": "object"
+    },
+    "definitions": {
+      "title": "Parameter groups",
+      "type": "object",
+      "patternProperties": {
+        "^.*$": {
+          "type": "object",
+          "required": [
+            "title",
+            "type",
+            "properties"
+          ],
+          "properties": {
+            "title": {
+              "type": "string",
+              "minLength": 1
+            },
+            "type": {
+              "const": "object"
+            },
+            "fa_icon": {
+              "type": "string",
+              "pattern": "^fa"
+            },
+            "description": {
+              "type": "string"
+            },
+            "required": {
+              "type": "array"
+            },
+            "properties": {
+              "type": "object",
+              "patternProperties": {
+                "^.*$": {
+                    "type": "object",
+                    "required": [
+                      "type"
+                    ],
+                    "properties": {
+                      "type": {
+                        "type": "string",
+                        "enum": ["string", "boolean", "integer", "number"]
+                      },
+                      "format": {
+                        "type": "string",
+                        "enum": ["file-path", "directory-path", "path", "file-path-pattern"]
+                      },
+                      "exists": {
+                        "type": "boolean"
+                      },
+                      "mimetype": {
+                        "type": "string",
+                        "pattern": ".+/.+"
+                      },
+                      "pattern": {
+                        "type": "string",
+                        "minLength": 1
+                      },
+                      "schema": {
+                        "type": "string",
+                        "minLength": 1
+                      },
+                      "description": {
+                        "type": "string"
+                      },
+                      "help_text": {
+                        "type": "string"
+                      },
+                      "fa_icon": {
+                        "type": "string",
+                        "pattern": "^fa"
+                      },
+                      "errorMessage": {
+                        "type": "string",
+                        "minLength": 1
+                      },
+                      "hidden": {
+                        "type": "boolean"
+                      },
+                      "minLength": {
+                        "type": "integer"
+                      },
+                      "maxLength": {
+                        "type": "integer"
+                      },
+                      "minimum": {
+                        "type": "integer"
+                      },
+                      "maximum": {
+                        "type": "integer"
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+    },
+    "allOf": {
+      "title": "Combine definition groups",
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": [
+          "$ref"
+        ],
+        "properties": {
+          "$ref": {
+            "type": "string",
+            "pattern": "^#/definitions/"
+          }
+        }
+      }
+    }
+  },
+  "required": [
+    "$schema",
+    "$id",
+    "title",
+    "description",
+    "type"
+  ]
+}

plugins/nf-validation/src/main/nextflow/validation/FormatValidators/FilePathPatternValidator.groovy

@@ -15,7 +15,7 @@ public class FilePathPatternValidator implements FormatValidator {
             log.debug("S3 paths are not supported by 'FilePathPatternValidator': '${subject}'")
             return Optional.empty()
         }
-        ArrayList files = Nextflow.file(subject) as ArrayList
+        ArrayList files = Nextflow.files(subject)
         ArrayList errors = []
 
         if(files.size() == 0) {
@@ -26,7 +26,7 @@ public class FilePathPatternValidator implements FormatValidator {
                 errors.add("'${file.toString()}' is not a file, but a directory" as String)
             }
         }
-        if(errors.size > 0) {
+        if(errors.size() > 0) {
             return Optional.of(errors.join('\n'))
         }
         return Optional.empty()

plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy

@@ -88,7 +88,7 @@ class SamplesheetConverter {
         }
         else {
             Path fileSamplesheet = Nextflow.file(samplesheetFile) as Path
-            samplesheetList = fileSamplesheet.splitCsv(header:containsHeader ?: ["empty"], strip:true, sep:delimiter, quote:'"')
+            samplesheetList = fileSamplesheet.splitCsv(header:containsHeader ?: ["empty"], strip:true, sep:delimiter, quote:'\"')
         }
 
         // Field checks + returning the channels

plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy

@@ -208,7 +208,7 @@ class SchemaValidator extends PluginExtensionPoint {
             }
         }
         else {
-            fileContent = samplesheetFile.splitCsv(header:containsHeader ?: ["empty"], strip:true, sep:delimiter)
+            fileContent = samplesheetFile.splitCsv(header:containsHeader ?: ["empty"], strip:true, sep:delimiter, quote:'\"')
         }
         def List<Map<String,String>> fileContentCasted = castToType(fileContent, types)
         if (validateFile(false, samplesheetFile.toString(), fileContentCasted, schemaFile.toString(), baseDir, s3PathCheck)) {
@@ -454,7 +454,7 @@ class SchemaValidator extends PluginExtensionPoint {
                         }
                     }
                     else {
-                        fileContent = file_path.splitCsv(header:containsHeader ?: ["empty"], strip:true, sep:delimiter)
+                        fileContent = file_path.splitCsv(header:containsHeader ?: ["empty"], strip:true, sep:delimiter, quote:'\"')
                     }
                     def List<Map<String,String>> fileContentCasted = castToType(fileContent, types)
                     if (validateFile(useMonochromeLogs, key, fileContentCasted, schema_name, baseDir, s3PathCheck)) {

plugins/nf-validation/src/resources/META-INF/MANIFEST.MF

@@ -1,6 +1,6 @@
 Manifest-Version: 1.0
 Plugin-Id: nf-validation
-Plugin-Version: 1.1.1
+Plugin-Version: 1.1.3
 Plugin-Class: nextflow.validation.ValidationPlugin
 Plugin-Provider: nextflow
 Plugin-Requires: >=22.10.0

plugins/nf-validation/src/test/nextflow/validation/PluginExtensionMethodsTest.groovy

@@ -511,6 +511,50 @@ class PluginExtensionMethodsTest extends Dsl2Spec{
         !stdout
     }
 
+    def 'correct validation of file-path-pattern - glob' () {
+        given:
+        def schema = Path.of('src/testResources/nextflow_schema_file_path_pattern.json').toAbsolutePath().toString()
+        def  SCRIPT_TEXT = """
+            params.glob = 'src/testResources/*.csv'
+            include { validateParameters } from 'plugin/nf-validation'
+            
+            validateParameters(parameters_schema: '$schema')
+        """
+
+        when:
+        dsl_eval(SCRIPT_TEXT)
+        def stdout = capture
+                .toString()
+                .readLines()
+                .findResults {it.contains('WARN nextflow.validation.SchemaValidator') || it.startsWith('* --') ? it : null }
+
+        then:
+        noExceptionThrown()
+        !stdout
+    }
+
+    def 'correct validation of file-path-pattern - single file' () {
+        given:
+        def schema = Path.of('src/testResources/nextflow_schema_file_path_pattern.json').toAbsolutePath().toString()
+        def  SCRIPT_TEXT = """
+            params.glob = 'src/testResources/correct.csv'
+            include { validateParameters } from 'plugin/nf-validation'
+            
+            validateParameters(parameters_schema: '$schema')
+        """
+
+        when:
+        dsl_eval(SCRIPT_TEXT)
+        def stdout = capture
+                .toString()
+                .readLines()
+                .findResults {it.contains('WARN nextflow.validation.SchemaValidator') || it.startsWith('* --') ? it : null }
+
+        then:
+        noExceptionThrown()
+        !stdout
+    }
+
     def 'correct validation of numbers with lenient mode' () {
         given:
         def schema = Path.of('src/testResources/nextflow_schema.json').toAbsolutePath().toString()

plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy

@@ -83,7 +83,34 @@ class SamplesheetConverterTest extends Dsl2Spec{
         stdout.contains("[[string1:extraField, string2:extraField, integer1:10, integer2:10, boolean1:true, boolean2:true], string1, 25, false, ${this.getRootString()}/src/testResources/test.txt, ${this.getRootString()}/src/testResources/testDir, ${this.getRootString()}/src/testResources/testDir, unique3, 1, itDoesExist]" as String)
     }
 
-        def 'should work fine - TSV' () {
+    def 'should work fine - quoted CSV' () {
+        given:
+        def SCRIPT_TEXT = '''
+            include { fromSamplesheet } from 'plugin/nf-validation'
+
+            params.input = 'src/testResources/correct_quoted.csv'
+
+            workflow {
+                Channel.fromSamplesheet("input", parameters_schema:"src/testResources/nextflow_schema_with_samplesheet_converter.json").view().first().map {println(it[0].getClass())}
+            }
+        '''
+
+        when:
+        dsl_eval(SCRIPT_TEXT)
+        def stdout = capture
+                .toString()
+                .readLines()
+                .findResults {it.startsWith('[[') ? it : null }
+
+        then:
+        noExceptionThrown()
+        stdout.contains("[[string1:fullField, string2:fullField, integer1:10, integer2:10, boolean1:true, boolean2:true], string1, 25.12, false, ${this.getRootString()}/src/testResources/test.txt, ${this.getRootString()}/src/testResources/testDir, ${this.getRootString()}/src/testResources/test.txt, unique1, 1, itDoesExist]" as String)
+        stdout.contains("[[string1:value, string2:value, integer1:0, integer2:0, boolean1:true, boolean2:true], string1, 25.08, false, [], [], [], [], [], itDoesExist]")
+        stdout.contains("[[string1:dependentRequired, string2:dependentRequired, integer1:10, integer2:10, boolean1:true, boolean2:true], string1, 25, false, [], [], [], unique2, 1, itDoesExist]")
+        stdout.contains("[[string1:extraField, string2:extraField, integer1:10, integer2:10, boolean1:true, boolean2:true], string1, 25, false, ${this.getRootString()}/src/testResources/test.txt, ${this.getRootString()}/src/testResources/testDir, ${this.getRootString()}/src/testResources/testDir, unique3, 1, itDoesExist]" as String)
+    }
+
+    def 'should work fine - TSV' () {
         given:
         def SCRIPT_TEXT = '''
             include { fromSamplesheet } from 'plugin/nf-validation'

plugins/nf-validation/src/testResources/correct_quoted.csv

@@ -0,0 +1,5 @@
+field_1,field_2,field_3,field_4,field_5,field_6,field_7,field_8,field_9,field_10,field_11
+"fullField",10,"true","string1",25.12,false,"src/testResources/test.txt",src/testResources/testDir,src/testResources/test.txt,unique1,1
+,,,string1,"25.08",false,"",,,,,
+"dependentRequired",10,true,string1,25,"false",,,,"unique2",1
+"extraField",10,true,string1,25,false,src/testResources/test.txt,"src/testResources/testDir",src/testResources/testDir,unique3,1

plugins/nf-validation/src/testResources/nextflow_schema_file_path_pattern.json

@@ -0,0 +1,25 @@
+{
+    "$schema": "http://json-schema.org/draft-07/schema",
+    "$id": "https://raw.githubusercontent.com/nf-core/testpipeline/master/nextflow_schema.json",
+    "title": "nf-core/testpipeline pipeline parameters",
+    "description": "this is a test",
+    "type": "object",
+    "definitions": {
+        "file_patterns": {
+            "title": "Input/output options",
+            "type": "object",
+            "fa_icon": "fas fa-terminal",
+            "properties": {
+                "glob": {
+                    "type": "string",
+                    "format": "file-path-pattern"
+                }
+            }
+        }
+    },
+    "allOf": [
+        {
+            "$ref": "#/definitions/file_patterns"
+        }
+    ]
+}