Merge pull request #74 from nextflow-io/2.2.0-dev

Release 2.2.0

CHANGELOG.md

@@ -1,5 +1,23 @@
 # nextflow-io/nf-schema: Changelog
 
+# Version 2.2.0 - Kitakata
+
+## New features
+
+1. Added a new configuration option `validation.failUnrecognisedHeaders`. This is the analogue to `failUnrecognisedParams`, but for samplesheet headers. The default is `false` which means that unrecognized headers throw a warning instead of an error.
+2. Added a new configuration option `validation.summary.hideParams`. This option takes a list of parameter names to hide from the parameters summary created by `paramsSummaryMap()` and `paramsSummaryLog()`
+
+## Bug fixes
+
+1. Fixed a bug in `samplesheetToList` that caused output mixing when the function was used more than once in channel operators.
+2. Added a missing depencency for email format validation.
+3. All path formats (with exception to `file-path-pattern`) will now give a proper error message when a `file-path-pattern` has been used.
+
+## Improvements
+
+1. Improved the `exists` keyword documentation with a warning about an edge case.
+2. Updated the error messages. Custom error messages provided in the JSON schema will now be appended to the original error messages instead of overwriting them.
+
 # Version 2.1.2
 
 ## Bug fixes

README.md

@@ -25,7 +25,7 @@ Declare the plugin in your Nextflow pipeline configuration file:
 
 ```groovy title="nextflow.config"
 plugins {
-  id 'nf-schema@2.1.2'
+  id 'nf-schema@2.2.0'
 }
 ```
 

docs/configuration/configuration.md

@@ -54,6 +54,14 @@ By default the `validateParameters()` function will only give a warning if an un
 validation.failUnrecognisedParams = <true|false> // default: false
 ```
 
+## failUnrecognisedHeaders
+
+By default the `samplesheetToList()` function will only give a warning if an unrecognised header is present in the samplesheet. This usually indicates that a typo has been made and can be easily overlooked when the plugin only emits a warning. You can turn this warning into an error with the `failUnrecognisedHeaders` option.
+
+```groovy
+validation.failUnrecognisedHeaders = <true|false> // default: false
+```
+
 ## showHiddenParams
 
 !!! deprecated
@@ -218,3 +226,11 @@ validation.summary.afterText = "Please cite the pipeline owners when using this
 !!! info
 
     All color values (like `\033[0;31m`, which means the color red) will be filtered out when `validation.monochromeLogs` is set to `true`
+
+### hideParams
+
+Takes a list of parameter names to exclude from the parameters summary created by `paramsSummaryMap()` and `paramsSummaryLog()`
+
+```groovy
+validation.summary.hideParams = ["param1", "nested.param"] // default: []
+```

docs/nextflow_schema/nextflow_schema_specification.md

@@ -231,7 +231,7 @@ If validation fails, an error message is printed to the terminal, so that the en
 However, these messages are not always very clear - especially to newcomers.
 
 To improve this experience, pipeline developers can set a custom `errorMessage` for a given parameter in a the schema.
-If validation fails, this `errorMessage` is printed instead, and the raw JSON schema validation message goes to the Nextflow debug log output.
+If validation fails, this `errorMessage` is printed after the original error message to guide the pipeline users to an easier solution.
 
 For example, instead of printing:
 
@@ -252,7 +252,7 @@ We can set
 and get:
 
 ```
-* --input (samples.yml): File name must end in '.csv' cannot contain spaces
+* --input (samples.yml): "samples.yml" does not match regular expression [^\S+\.csv$] (File name must end in '.csv' cannot contain spaces)
 ```
 
 ### `deprecated`
@@ -389,7 +389,11 @@ Example usage is as follows:
 
 !!! note
 
-    If the parameter is an S3 URL path, this validation is ignored.
+    If the parameter is an S3, Azure or Google Cloud URI path, this validation is ignored.
+
+!!! warning
+
+    Make sure to only use the `exists` keyword in combination with any file path format. Using `exists` on a normal string will assume that it's a file and will probably fail unexpectedly.
 
 ### `mimetype`
 

plugins/nf-schema/build.gradle

@@ -56,6 +56,7 @@ dependencies {
     compileOnly 'org.pf4j:pf4j:3.4.1'
     implementation 'org.json:json:20240303'
     implementation 'dev.harrel:json-schema:1.5.0'
+    implementation 'com.sanctionco.jmail:jmail:1.6.3' // Needed for e-mail format validation
 
     // test configuration
     testImplementation "io.nextflow:nextflow:$nextflowVersion"

plugins/nf-schema/src/main/nextflow/validation/CustomEvaluators/FormatDirectoryPathEvaluator.groovy

@@ -33,6 +33,9 @@ class FormatDirectoryPathEvaluator implements Evaluator {
 
         // Actual validation logic
         def Path file = Nextflow.file(value) as Path
+        if (file instanceof List) {
+            return Evaluator.Result.failure("'${value}' is not a directory, but a file path pattern" as String)
+        }
         if (file.exists() && !file.isDirectory()) {
             return Evaluator.Result.failure("'${value}' is not a directory, but a file" as String)
         }

plugins/nf-schema/src/main/nextflow/validation/CustomEvaluators/FormatFilePathEvaluator.groovy

@@ -33,6 +33,9 @@ class FormatFilePathEvaluator implements Evaluator {
 
         // Actual validation logic
         def Path file = Nextflow.file(value) as Path
+        if (file instanceof List) {
+            return Evaluator.Result.failure("'${value}' is not a file, but a file path pattern" as String)
+        }
         if (file.exists() && file.isDirectory()) {
             return Evaluator.Result.failure("'${value}' is not a file, but a directory" as String)
         }

plugins/nf-schema/src/main/nextflow/validation/CustomEvaluators/FormatPathEvaluator.groovy

@@ -33,6 +33,9 @@ class FormatPathEvaluator implements Evaluator {
 
         // Actual validation logic
         def Path file = Nextflow.file(value) as Path
+        if (file instanceof List) {
+            return Evaluator.Result.failure("'${value}' is not a path, but a file path pattern" as String)
+        }
         return Evaluator.Result.success()
     }
 }

plugins/nf-schema/src/main/nextflow/validation/JsonSchemaValidator.groovy

@@ -22,9 +22,9 @@ import java.util.regex.Matcher
 @CompileStatic
 public class JsonSchemaValidator {
 
-    private static ValidatorFactory validator
-    private static Pattern uriPattern = Pattern.compile('^#/(\\d*)?/?(.*)$')
-    private static ValidationConfig config
+    private ValidatorFactory validator
+    private Pattern uriPattern = Pattern.compile('^#/(\\d*)?/?(.*)$')
+    private ValidationConfig config
 
     JsonSchemaValidator(ValidationConfig config) {
         this.validator = new ValidatorFactory()
@@ -34,14 +34,14 @@ public class JsonSchemaValidator {
         this.config = config
     }
 
-    private static List<String> validateObject(JsonNode input, String validationType, Object rawJson, String schemaString) {
+    private List<String> validateObject(JsonNode input, String validationType, Object rawJson, String schemaString) {
         def JSONObject schema = new JSONObject(schemaString)
         def String draft = Utils.getValueFromJson("#/\$schema", schema)
         if(draft != "https://json-schema.org/draft/2020-12/schema") {
             log.error("""Failed to load the meta schema:
     The used schema draft (${draft}) is not correct, please use \"https://json-schema.org/draft/2020-12/schema\" instead.
         - If you are a pipeline developer, check our migration guide for more information: https://nextflow-io.github.io/nf-schema/latest/migration_guide/
-        - If you are a pipeline user, pin the previous version of the plugin (1.1.3) to avoid this error: https://www.nextflow.io/docs/latest/plugins.html#using-plugins, i.e. set `plugins {
+        - If you are a pipeline user, revert back to nf-validation to avoid this error: https://www.nextflow.io/docs/latest/plugins.html#using-plugins, i.e. set `plugins {
     id '[email protected]'
 }` in your `nextflow.config` file
             """)
@@ -50,11 +50,11 @@ public class JsonSchemaValidator {
 
         def Validator.Result result = this.validator.validate(schema, input)
         def List<String> errors = []
-        for (error : result.getErrors()) {
+        result.getErrors().each { error ->
             def String errorString = error.getError()
             // Skip double error in the parameter schema
             if (errorString.startsWith("Value does not match against the schemas at indexes") && validationType == "parameter") {
-                continue
+                return
             }
 
             def String instanceLocation = error.getInstanceLocation()
@@ -68,48 +68,47 @@ public class JsonSchemaValidator {
             }
 
             // Change some error messages to make them more clear
-            if (customError == "") {
-                def String keyword = error.getKeyword()
-                if (keyword == "required") {
-                    def Matcher matcher = errorString =~ ~/\[\[([^\[\]]*)\]\]$/
-                    def String missingKeywords = matcher.findAll().flatten().last()
-                    customError = "Missing required ${validationType}(s): ${missingKeywords}"
-                }
+            def String keyword = error.getKeyword()
+            if (keyword == "required") {
+                def Matcher matcher = errorString =~ ~/\[\[([^\[\]]*)\]\]$/
+                def String missingKeywords = matcher.findAll().flatten().last()
+                errorString = "Missing required ${validationType}(s): ${missingKeywords}"
             }
 
             def List<String> locationList = instanceLocation.split("/").findAll { it != "" } as List
 
+            def String printableError = "${validationType == 'field' ? '->' : '*'} ${errorString}" as String
             if (locationList.size() > 0 && Utils.isInteger(locationList[0]) && validationType == "field") {
                 def Integer entryInteger = locationList[0] as Integer
                 def String entryString = "Entry ${entryInteger + 1}" as String
-                def String fieldError = ""
+                def String fieldError = "${errorString}" as String
                 if(locationList.size() > 1) {
-                    fieldError = "Error for ${validationType} '${locationList[1..-1].join("/")}' (${value}): ${customError ?: errorString}"
-                } else {
-                    fieldError = "${customError ?: errorString}" as String
+                    fieldError = "Error for ${validationType} '${locationList[1..-1].join("/")}' (${value}): ${errorString}"
                 }
-                errors.add("-> ${entryString}: ${fieldError}" as String)
+                printableError = "-> ${entryString}: ${fieldError}" as String
             } else if (validationType == "parameter") {
                 def String fieldName = locationList.join(".")
                 if(fieldName != "") {
-                    errors.add("* --${fieldName} (${value}): ${customError ?: errorString}" as String)
-                } else {
-                    errors.add("* ${customError ?: errorString}" as String)
+                    printableError = "* --${fieldName} (${value}): ${errorString}" as String
                 }
-            } else {
-                errors.add("-> ${customError ?: errorString}" as String)
             }
 
+            if(customError != "") {
+                printableError = printableError + " (${customError})"
+            }
+
+            errors.add(printableError)
+
         }
         return errors
     }
 
-    public static List<String> validate(JSONArray input, String schemaString) {
+    public List<String> validate(JSONArray input, String schemaString) {
         def JsonNode jsonInput = new OrgJsonNode.Factory().wrap(input)
         return this.validateObject(jsonInput, "field", input, schemaString)
     }
 
-    public static List<String> validate(JSONObject input, String schemaString) {
+    public List<String> validate(JSONObject input, String schemaString) {
         def JsonNode jsonInput = new OrgJsonNode.Factory().wrap(input)
         return this.validateObject(jsonInput, "parameter", input, schemaString)
     }