From 73e58d69991889657c69a2a365333b31678dcb20 Mon Sep 17 00:00:00 2001
From: Arthur Gymer <24782660+awgymer@users.noreply.github.com>
Date: Thu, 26 Oct 2023 02:19:44 +0100
Subject: [PATCH 1/9] feat: add support for simple arrays to samplesheet
 validation

---
 .../validation/SamplesheetConverter.groovy    | 55 +++++++++++--------
 .../validation/SchemaValidator.groovy         | 28 +++++++---
 2 files changed, 53 insertions(+), 30 deletions(-)

diff --git a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy
index 3118bd80..8218662b 100644
--- a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy
+++ b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy
@@ -97,10 +97,11 @@ class SamplesheetConverter {
         def Boolean headerCheck = true
         this.rows = []
         resetCount()
+
         def List outputs = samplesheetList.collect { Map<String,String> fullRow ->
             increaseCount()
 
-            Map<String,String> row = fullRow.findAll { it.value != "" }
+            Map<String,Object> row = fullRow.findAll { it.value != "" }
             def Set rowKeys = containsHeader ? row.keySet() : ["empty"].toSet()
             def String yamlInfo = fileType == "yaml" ? " for entry ${this.getCount()}." : ""
 
@@ -128,7 +129,7 @@ class SamplesheetConverter {
 
             for( Map.Entry<String, Map> field : schemaFields ){
                 def String key = containsHeader ? field.key : "empty"
-                def String input = row[key]
+                def Object input = row[key]
 
                 // Check if the field is deprecated
                 if(field['value']['deprecated']){
@@ -159,7 +160,7 @@ class SamplesheetConverter {
                     if(input in booleanUniques[key] && input){
                         this.errors << addSample("The '${key}' value needs to be unique. '${input}' was found at least twice in the samplesheet.".toString())
                     }
-                    booleanUniques[key].add(input)
+                    booleanUniques[key].add(input as String)
                 }
                 else if(unique && uniqueIsList) {
                     def Map<String,String> newMap = (Map) row.subMap((List) [key] + (List) unique)
@@ -176,20 +177,20 @@ class SamplesheetConverter {
                 def List<String> metaNames = field['value']['meta'] as List<String>
                 if(metaNames) {
                     for(name : metaNames) {
-                        meta[name] = (input != '' && input) ? 
-                                castToType(input, field) : 
-                            field['value']['default'] != null ? 
-                                castToType(field['value']['default'] as String, field) : 
+                        meta[name] = (input != '' && input) ?
+                                castToNFType(input, field) :
+                            field['value']['default'] != null ?
+                                castToNFType(field['value']['default'], field) :
                                 null
                     }
                 }
                 else {
-                    def inputFile = (input != '' && input) ? 
-                            castToType(input, field) : 
-                        field['value']['default'] != null ? 
-                            castToType(field['value']['default'] as String, field) : 
+                    def inputVal = (input != '' && input) ?
+                            castToNFType(input, field) :
+                        field['value']['default'] != null ?
+                            castToNFType(field['value']['default'], field) :
                             []
-                    output.add(inputFile)
+                    output.add(inputVal)
                 }
             }
             // Add meta to the output when a meta field has been created
@@ -253,26 +254,34 @@ class SamplesheetConverter {
     }
 
     // Function to transform an input field from the samplesheet to its desired type
-    private static castToType(
-        String input,
+    private static castToNFType(
+        Object input,
         Map.Entry<String, Map> field
     ) {
         def String type = field['value']['type']
         def String key = field.key
 
+        if (type == "array") {
+            def Map.Entry<String, Map> subfield = (Map.Entry<String, Map>) Map.entry(field.key, field['value']['items'])
+            log.debug "subfield = $subfield"
+            def ArrayList result = input.collect{ castToNFType(it, subfield) } as ArrayList
+            return result
+        }
+
+        def String inputStr = input as String
         // Convert string values
         if(type == "string" || !type) {
-            def String result = input as String
+            def String result = inputStr as String
             
             // Check and convert to the desired format
             def String format = field['value']['format']
             if(format) {
                 if(format == "file-path-pattern") {
-                    def ArrayList inputFiles = Nextflow.file(input) as ArrayList
+                    def ArrayList inputFiles = Nextflow.file(inputStr) as ArrayList
                     return inputFiles
                 }
                 if(format.contains("path")) {
-                    def Path inputFile = Nextflow.file(input) as Path
+                    def Path inputFile = Nextflow.file(inputStr) as Path
                     return inputFile
                 }
             }
@@ -285,7 +294,7 @@ class SamplesheetConverter {
         // Convert number values
         else if(type == "number") {
             try {
-                def int result = input as int
+                def int result = inputStr as int
                 return result
             }
             catch (NumberFormatException e) {
@@ -293,28 +302,28 @@ class SamplesheetConverter {
             }
 
             try {
-                def float result = input as float
+                def float result = inputStr as float
                 return result
             }
             catch (NumberFormatException e) {
-                log.debug("Could not convert ${input} to a float. Trying to convert to a double.")
+                log.debug("Could not convert ${inputStr} to a float. Trying to convert to a double.")
             }
             
-            def double result = input as double
+            def double result = inputStr as double
             return result
         }
 
         // Convert integer values
         else if(type == "integer") {
 
-            def int result = input as int
+            def int result = inputStr as int
             return result
         }
 
         // Convert boolean values
         else if(type == "boolean") {
 
-            if(input.toLowerCase() == "true") {
+            if(inputStr.toLowerCase() == "true") {
                 return true
             }
             return false
diff --git a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy
index a19715b8..290da553 100644
--- a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy
+++ b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy
@@ -184,6 +184,7 @@ class SchemaValidator extends PluginExtensionPoint {
         def String fileType = SamplesheetConverter.getFileType(samplesheetFile)
         def String delimiter = fileType == "csv" ? "," : fileType == "tsv" ? "\t" : null
         def List<Map<String,String>> fileContent
+        def List<Map<String,String>> fileContentCasted
         def Boolean s3PathCheck = params.validationS3PathCheck ? params.validationS3PathCheck : false
         def Map types = variableTypes(schemaFile.toString(), baseDir)
         def Boolean containsHeader = !(types.keySet().size() == 1 && types.keySet()[0] == "")
@@ -192,7 +193,15 @@ class SchemaValidator extends PluginExtensionPoint {
             types = ["empty": types[""]]
         }
         if(fileType == "yaml"){
-            fileContent = new Yaml().load((samplesheetFile.text)).collect {
+            fileContentCasted = new Yaml().load((samplesheetFile.text)).collect {
+                if(containsHeader) {
+                    return it as Map
+                }
+                return ["empty": it] as Map
+            }
+        }
+        else if(fileType == "json"){
+            fileContent = new JsonSlurper().parseText(samplesheetFile.text).collect {
                 if(containsHeader) {
                     return it as Map
                 }
@@ -210,7 +219,6 @@ class SchemaValidator extends PluginExtensionPoint {
         else {
             fileContent = samplesheetFile.splitCsv(header:containsHeader ?: ["empty"], strip:true, sep:delimiter, quote:'\"')
         }
-        def List<Map<String,String>> fileContentCasted = castToType(fileContent, types)
         if (validateFile(false, samplesheetFile.toString(), fileContentCasted, schemaFile.toString(), baseDir, s3PathCheck)) {
             log.debug "Validation passed: '$samplesheetFile' with '$schemaFile'"
         }
@@ -430,6 +438,7 @@ class SchemaValidator extends PluginExtensionPoint {
                     def String fileType = SamplesheetConverter.getFileType(file_path)
                     def String delimiter = fileType == "csv" ? "," : fileType == "tsv" ? "\t" : null
                     def List<Map<String,String>> fileContent
+                    def List<Map<String,String>> fileContentCasted
                     def Map types = variableTypes(schema_name, baseDir)
                     def Boolean containsHeader = !(types.keySet().size() == 1 && types.keySet()[0] == "")
 
@@ -438,7 +447,7 @@ class SchemaValidator extends PluginExtensionPoint {
                     }
 
                     if(fileType == "yaml"){
-                        fileContent = new Yaml().load(file_path.text).collect {
+                        fileContentCasted = new Yaml().load(file_path.text).collect {
                             if(containsHeader) {
                                 return it as Map
                             }
@@ -454,9 +463,8 @@ class SchemaValidator extends PluginExtensionPoint {
                         }
                     }
                     else {
-                        fileContent = file_path.splitCsv(header:containsHeader ?: ["empty"], strip:true, sep:delimiter, quote:'\"')
+                        fileContent = file_path.splitCsv(header:containsHeader ?: ["empty"], strip:true, sep:delimiter)
                     }
-                    def List<Map<String,String>> fileContentCasted = castToType(fileContent, types)
                     if (validateFile(useMonochromeLogs, key, fileContentCasted, schema_name, baseDir, s3PathCheck)) {
                         log.debug "Validation passed: '$key': '$file_path' with '$schema_name'"
                     }
@@ -591,9 +599,12 @@ class SchemaValidator extends PluginExtensionPoint {
         pathsToCheck.each { String fieldName ->
             for (int i=0; i < arrayJSON.size(); i++) {
                 def JSONObject entry = arrayJSON.getJSONObject(i)
-                if ( entry.has(fieldName) ) {
+                if ( entry.has(fieldName) && entry[fieldName] instanceof ArrayList ) {
                     pathExists(entry[fieldName].toString(), " Entry ${(i+1).toString()} - ${fieldName.toString()}", s3PathCheck)
                 }
+                else if ( entry.has(fieldName) ) {
+                    entry[fieldName].collect{ pathExists(it.toString(), " Entry ${(i+1).toString()} - ${fieldName.toString()}", s3PathCheck) }
+                }
             }
         }
 
@@ -651,7 +662,10 @@ class SchemaValidator extends PluginExtensionPoint {
             def Map properties = (Map) group.value['properties']
             for (p in properties) {
                 def String key = (String) p.key
-                def Map property = properties[key] as Map
+                def Map<String,Object> property = properties[key] as Map
+                if(property.containsKey('items')){
+                    property = property.items as Map
+                }
                 if (property.containsKey('exists') && property.containsKey('format')) {
                     if (property['exists'] && (property['format'] == 'file-path' || property['format'] == 'directory-path' || property['format'] == 'path') ) {
                         exists.push(key)

From bccdd9f8aa9d5067a20485d76954d67f2b487019 Mon Sep 17 00:00:00 2001
From: Arthur Gymer <24782660+awgymer@users.noreply.github.com>
Date: Thu, 26 Oct 2023 02:40:07 +0100
Subject: [PATCH 2/9] fix: correct logic for array type fields when checking
 for file existence

---
 .../src/main/nextflow/validation/SchemaValidator.groovy     | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy
index 290da553..8aaa7022 100644
--- a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy
+++ b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy
@@ -599,11 +599,11 @@ class SchemaValidator extends PluginExtensionPoint {
         pathsToCheck.each { String fieldName ->
             for (int i=0; i < arrayJSON.size(); i++) {
                 def JSONObject entry = arrayJSON.getJSONObject(i)
-                if ( entry.has(fieldName) && entry[fieldName] instanceof ArrayList ) {
-                    pathExists(entry[fieldName].toString(), " Entry ${(i+1).toString()} - ${fieldName.toString()}", s3PathCheck)
+                if ( entry.has(fieldName) && entry[fieldName] instanceof JSONArray ) {
+                    entry[fieldName].collect{ pathExists(it.toString(), " Entry ${(i+1).toString()} - ${fieldName.toString()}", s3PathCheck) }
                 }
                 else if ( entry.has(fieldName) ) {
-                    entry[fieldName].collect{ pathExists(it.toString(), " Entry ${(i+1).toString()} - ${fieldName.toString()}", s3PathCheck) }
+                    pathExists(entry[fieldName].toString(), " Entry ${(i+1).toString()} - ${fieldName.toString()}", s3PathCheck)
                 }
             }
         }

From 54e3fa4c15ac672963b3849101195a937c0f83f9 Mon Sep 17 00:00:00 2001
From: Arthur Gymer <24782660+awgymer@users.noreply.github.com>
Date: Thu, 21 Dec 2023 13:34:30 +0000
Subject: [PATCH 3/9] fix: some mistakes with fileContentCasted variable during
 rebase

---
 .../validation/SamplesheetConverter.groovy         |  2 ++
 .../nextflow/validation/SchemaValidator.groovy     | 14 ++++----------
 .../validation/SamplesheetConverterTest.groovy     |  1 -
 3 files changed, 6 insertions(+), 11 deletions(-)

diff --git a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy
index 8218662b..0f90b691 100644
--- a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy
+++ b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy
@@ -261,6 +261,8 @@ class SamplesheetConverter {
         def String type = field['value']['type']
         def String key = field.key
 
+        // Recursively call this function for each item in the array if the field is an array-type
+        // The returned values are collected into a single array
         if (type == "array") {
             def Map.Entry<String, Map> subfield = (Map.Entry<String, Map>) Map.entry(field.key, field['value']['items'])
             log.debug "subfield = $subfield"
diff --git a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy
index 8aaa7022..3ff44a3f 100644
--- a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy
+++ b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy
@@ -201,15 +201,7 @@ class SchemaValidator extends PluginExtensionPoint {
             }
         }
         else if(fileType == "json"){
-            fileContent = new JsonSlurper().parseText(samplesheetFile.text).collect {
-                if(containsHeader) {
-                    return it as Map
-                }
-                return ["empty": it] as Map
-            }
-        }
-        else if(fileType == "json"){
-            fileContent = new JsonSlurper().parseText(samplesheetFile.text).collect {
+            fileContentCasted = new JsonSlurper().parseText(samplesheetFile.text).collect {
                 if(containsHeader) {
                     return it as Map
                 }
@@ -218,6 +210,7 @@ class SchemaValidator extends PluginExtensionPoint {
         }
         else {
             fileContent = samplesheetFile.splitCsv(header:containsHeader ?: ["empty"], strip:true, sep:delimiter, quote:'\"')
+            fileContentCasted = castToType(fileContent, types)
         }
         if (validateFile(false, samplesheetFile.toString(), fileContentCasted, schemaFile.toString(), baseDir, s3PathCheck)) {
             log.debug "Validation passed: '$samplesheetFile' with '$schemaFile'"
@@ -455,7 +448,7 @@ class SchemaValidator extends PluginExtensionPoint {
                         }
                     }
                     else if(fileType == "json"){
-                        fileContent = new JsonSlurper().parseText(file_path.text).collect {
+                        fileContentCasted = new JsonSlurper().parseText(file_path.text).collect {
                             if(containsHeader) {
                                 return it as Map
                             }
@@ -464,6 +457,7 @@ class SchemaValidator extends PluginExtensionPoint {
                     }
                     else {
                         fileContent = file_path.splitCsv(header:containsHeader ?: ["empty"], strip:true, sep:delimiter)
+                        fileContentCasted = castToType(fileContent, types)
                     }
                     if (validateFile(useMonochromeLogs, key, fileContentCasted, schema_name, baseDir, s3PathCheck)) {
                         log.debug "Validation passed: '$key': '$file_path' with '$schema_name'"
diff --git a/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy b/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy
index 444ca15b..8c9f54c4 100644
--- a/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy
+++ b/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy
@@ -266,7 +266,6 @@ class SamplesheetConverterTest extends Dsl2Spec{
         stdout.contains("[test_2]")
     }
 
-
     def 'extra field' () {
         given:
         def SCRIPT_TEXT = '''

From 40ffe278c2defd3d17ecfee3e2326aab4b3b58d8 Mon Sep 17 00:00:00 2001
From: Arthur Gymer <24782660+awgymer@users.noreply.github.com>
Date: Thu, 21 Dec 2023 17:39:13 +0000
Subject: [PATCH 4/9] fix: do the same entryInfo process for json as for yaml

---
 .../src/main/nextflow/validation/SamplesheetConverter.groovy  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy
index 0f90b691..9f222f46 100644
--- a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy
+++ b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy
@@ -103,13 +103,13 @@ class SamplesheetConverter {
 
             Map<String,Object> row = fullRow.findAll { it.value != "" }
             def Set rowKeys = containsHeader ? row.keySet() : ["empty"].toSet()
-            def String yamlInfo = fileType == "yaml" ? " for entry ${this.getCount()}." : ""
+            def String entryInfo = fileType in ["yaml", "json"] ? " for entry ${this.getCount()}." : ""
 
             // Check the header (CSV/TSV) or present fields (YAML)
             if(headerCheck) {
                 def unexpectedFields = containsHeader ? rowKeys - allFields : []
                 if(unexpectedFields.size() > 0) {
-                    this.warnings << "The samplesheet contains following unchecked field(s): ${unexpectedFields}${yamlInfo}".toString()
+                    this.warnings << "The samplesheet contains following unchecked field(s): ${unexpectedFields}${entryInfo}".toString()
                 }
 
                 if(fileType != 'yaml'){

From 8f02b26477928819e54e412681e5473544592fd9 Mon Sep 17 00:00:00 2001
From: Arthur Gymer <24782660+awgymer@users.noreply.github.com>
Date: Thu, 21 Dec 2023 17:40:47 +0000
Subject: [PATCH 5/9] fix: make tests use correct input file for yaml and json
 tests. Ensure that boolean values in json/yaml are not treated as null before
 casting

---
 .../src/main/nextflow/validation/SamplesheetConverter.groovy  | 4 ++--
 .../test/nextflow/validation/SamplesheetConverterTest.groovy  | 4 ++--
 plugins/nf-validation/src/testResources/schema_input.json     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy
index 9f222f46..566153ac 100644
--- a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy
+++ b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy
@@ -177,7 +177,7 @@ class SamplesheetConverter {
                 def List<String> metaNames = field['value']['meta'] as List<String>
                 if(metaNames) {
                     for(name : metaNames) {
-                        meta[name] = (input != '' && input) ?
+                        meta[name] = (input != '' && input != null) ?
                                 castToNFType(input, field) :
                             field['value']['default'] != null ?
                                 castToNFType(field['value']['default'], field) :
@@ -185,7 +185,7 @@ class SamplesheetConverter {
                     }
                 }
                 else {
-                    def inputVal = (input != '' && input) ?
+                    def inputVal = (input != '' && input != null) ?
                             castToNFType(input, field) :
                         field['value']['default'] != null ?
                             castToNFType(field['value']['default'], field) :
diff --git a/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy b/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy
index 8c9f54c4..cdec98c0 100644
--- a/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy
+++ b/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy
@@ -115,7 +115,7 @@ class SamplesheetConverterTest extends Dsl2Spec{
         def SCRIPT_TEXT = '''
             include { fromSamplesheet } from 'plugin/nf-validation'
 
-            params.input = 'src/testResources/correct.csv'
+            params.input = 'src/testResources/correct.tsv'
 
             workflow {
                 Channel.fromSamplesheet("input", parameters_schema:"src/testResources/nextflow_schema_with_samplesheet_converter.json").view()
@@ -142,7 +142,7 @@ class SamplesheetConverterTest extends Dsl2Spec{
         def SCRIPT_TEXT = '''
             include { fromSamplesheet } from 'plugin/nf-validation'
 
-            params.input = 'src/testResources/correct.csv'
+            params.input = 'src/testResources/correct.yaml'
 
             workflow {
                 Channel.fromSamplesheet("input", parameters_schema:"src/testResources/nextflow_schema_with_samplesheet_converter.json").view()
diff --git a/plugins/nf-validation/src/testResources/schema_input.json b/plugins/nf-validation/src/testResources/schema_input.json
index d955d512..b896f136 100644
--- a/plugins/nf-validation/src/testResources/schema_input.json
+++ b/plugins/nf-validation/src/testResources/schema_input.json
@@ -47,7 +47,7 @@
                 "type": "string",
                 "format": "path",
                 "exists": true
-                },
+            },
             "field_10": {
                 "type": "string",
                 "unique": true

From 0644b531a11787ccc056f84b0220ab2a80904677 Mon Sep 17 00:00:00 2001
From: Arthur Gymer <24782660+awgymer@users.noreply.github.com>
Date: Thu, 21 Dec 2023 17:47:46 +0000
Subject: [PATCH 6/9] fix: put quote parameter back to match master

---
 .../src/main/nextflow/validation/SchemaValidator.groovy         | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy
index 3ff44a3f..cdb56718 100644
--- a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy
+++ b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy
@@ -456,7 +456,7 @@ class SchemaValidator extends PluginExtensionPoint {
                         }
                     }
                     else {
-                        fileContent = file_path.splitCsv(header:containsHeader ?: ["empty"], strip:true, sep:delimiter)
+                        fileContent = file_path.splitCsv(header:containsHeader ?: ["empty"], strip:true, sep:delimiter, quote:'\"')
                         fileContentCasted = castToType(fileContent, types)
                     }
                     if (validateFile(useMonochromeLogs, key, fileContentCasted, schema_name, baseDir, s3PathCheck)) {

From 90d205303b8f26fb8250a12c8b2d940649a87468 Mon Sep 17 00:00:00 2001
From: Arthur Gymer <24782660+awgymer@users.noreply.github.com>
Date: Thu, 21 Dec 2023 17:48:48 +0000
Subject: [PATCH 7/9] feat: add tests for arrays in YAML/JSON. Add new test
 data.

---
 .../SamplesheetConverterTest.groovy           |  90 ++++++++++++++++
 .../src/testResources/correct_arrays.json     |  72 +++++++++++++
 .../src/testResources/correct_arrays.yaml     |  45 ++++++++
 .../src/testResources/error_arrays.yaml       |  35 ++++++
 ...ema_with_samplesheet_converter_arrays.json |  28 +++++
 .../schema_input_with_arrays.json             | 102 ++++++++++++++++++
 .../src/testResources/testDir2/testFile2.txt  |   0
 7 files changed, 372 insertions(+)
 create mode 100644 plugins/nf-validation/src/testResources/correct_arrays.json
 create mode 100644 plugins/nf-validation/src/testResources/correct_arrays.yaml
 create mode 100644 plugins/nf-validation/src/testResources/error_arrays.yaml
 create mode 100644 plugins/nf-validation/src/testResources/nextflow_schema_with_samplesheet_converter_arrays.json
 create mode 100644 plugins/nf-validation/src/testResources/schema_input_with_arrays.json
 create mode 100644 plugins/nf-validation/src/testResources/testDir2/testFile2.txt

diff --git a/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy b/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy
index cdec98c0..e5161c73 100644
--- a/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy
+++ b/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy
@@ -191,6 +191,96 @@ class SamplesheetConverterTest extends Dsl2Spec{
         stdout.contains("[[string1:extraField, string2:extraField, integer1:10, integer2:10, boolean1:true, boolean2:true], string1, 25, false, ${this.getRootString()}/src/testResources/test.txt, ${this.getRootString()}/src/testResources/testDir, ${this.getRootString()}/src/testResources/testDir, unique3, 1, itDoesExist]" as String)
     }
 
+    def 'arrays should work fine - YAML' () {
+        given:
+        def SCRIPT_TEXT = '''
+            include { fromSamplesheet } from 'plugin/nf-validation'
+
+            params.input = 'src/testResources/correct_arrays.yaml'
+
+            workflow {
+                Channel.fromSamplesheet("input", parameters_schema:"src/testResources/nextflow_schema_with_samplesheet_converter_arrays.json").view()
+            }
+        '''
+
+        when:
+        dsl_eval(SCRIPT_TEXT)
+        def stdout = capture
+                .toString()
+                .readLines()
+                .findResults {it.startsWith('[[') ? it : null }
+
+        then:
+        noExceptionThrown()
+        stdout.contains("[[${this.getRootString()}/src/testResources/testDir/testFile.txt, ${this.getRootString()}/src/testResources/testDir2/testFile2.txt], [${this.getRootString()}/src/testResources/testDir, ${this.getRootString()}/src/testResources/testDir2], [${this.getRootString()}/src/testResources/testDir, ${this.getRootString()}/src/testResources/testDir2/testFile2.txt], [string1, string2], [25, 26], [25, 26.5], [false, true], [1, 2, 3], [true], [${this.getRootString()}/src/testResources/testDir/testFile.txt], [[${this.getRootString()}/src/testResources/testDir/testFile.txt]]]" as String)
+        stdout.contains("[[], [], [], [string1, string2], [25, 26], [25, 26.5], [], [1, 2, 3], [false, true, false], [${this.getRootString()}/src/testResources/testDir/testFile.txt], [[${this.getRootString()}/src/testResources/testDir/testFile.txt]]]" as String)
+        stdout.contains("[[], [], [], [string1, string2], [25, 26], [25, 26.5], [], [1, 2, 3], [false, true, false], [${this.getRootString()}/src/testResources/testDir/testFile.txt], [[${this.getRootString()}/src/testResources/testDir/testFile.txt], [${this.getRootString()}/src/testResources/testDir/testFile.txt, ${this.getRootString()}/src/testResources/testDir2/testFile2.txt]]]" as String)
+    }
+
+    def 'arrays should work fine - JSON' () {
+        given:
+        def SCRIPT_TEXT = '''
+            include { fromSamplesheet } from 'plugin/nf-validation'
+
+            params.input = 'src/testResources/correct_arrays.json'
+
+            workflow {
+                Channel.fromSamplesheet("input", parameters_schema:"src/testResources/nextflow_schema_with_samplesheet_converter_arrays.json").view()
+            }
+        '''
+
+        when:
+        dsl_eval(SCRIPT_TEXT)
+        def stdout = capture
+                .toString()
+                .readLines()
+                .findResults {it.startsWith('[[') ? it : null }
+
+        then:
+        noExceptionThrown()
+        stdout.contains("[[${this.getRootString()}/src/testResources/testDir/testFile.txt, ${this.getRootString()}/src/testResources/testDir2/testFile2.txt], [${this.getRootString()}/src/testResources/testDir, ${this.getRootString()}/src/testResources/testDir2], [${this.getRootString()}/src/testResources/testDir, ${this.getRootString()}/src/testResources/testDir2/testFile2.txt], [string1, string2], [25, 26], [25, 26.5], [false, true], [1, 2, 3], [true], [${this.getRootString()}/src/testResources/testDir/testFile.txt], [[${this.getRootString()}/src/testResources/testDir/testFile.txt]]]" as String)
+        stdout.contains("[[], [], [], [string1, string2], [25, 26], [25, 26.5], [], [1, 2, 3], [false, true, false], [${this.getRootString()}/src/testResources/testDir/testFile.txt], [[${this.getRootString()}/src/testResources/testDir/testFile.txt]]]" as String)
+        stdout.contains("[[], [], [], [string1, string2], [25, 26], [25, 26.5], [], [1, 2, 3], [false, true, false], [${this.getRootString()}/src/testResources/testDir/testFile.txt], [[${this.getRootString()}/src/testResources/testDir/testFile.txt], [${this.getRootString()}/src/testResources/testDir/testFile.txt, ${this.getRootString()}/src/testResources/testDir2/testFile2.txt]]]" as String)
+    }
+
+    def 'array errors before channel conversion - YAML' () {
+        given:
+        def SCRIPT_TEXT = '''
+            include { fromSamplesheet } from 'plugin/nf-validation'
+
+            params.input = 'src/testResources/error_arrays.yaml'
+
+            workflow {
+                Channel.fromSamplesheet("input", parameters_schema:"src/testResources/nextflow_schema_with_samplesheet_converter_arrays.json").view()
+            }
+        '''
+
+        when:
+        dsl_eval(SCRIPT_TEXT)
+        def stdout = capture
+                .toString()
+                .readLines()
+                .findResults {it.startsWith('[[') ? it : null }
+
+        then:
+        def error = thrown(SchemaValidationException)
+        def errorMessages = error.message.readLines()
+        errorMessages[0] == "\033[0;31mThe following errors have been detected:"
+        errorMessages[2] == "* -- Entry 1 - field_3: the file or directory 'src/testResources/testDir3' does not exist."
+        errorMessages[3] == "* -- Entry 1 - field_3: the file or directory 'src/testResources/testDir2/testFile3.txt' does not exist."
+        errorMessages[4] == "* -- Entry 1 - field_2: the file or directory 'src/testResources/testDir3' does not exist."
+        errorMessages[5] == "* -- Entry 1 - field_1: the file or directory 'src/testResources/testDir/testFile.fasta' does not exist."
+        errorMessages[6] == "* -- Entry 1 - field_1: the file or directory 'src/testResources/testDir2/testFile3.txt' does not exist."
+        errorMessages[7] == '* -- Entry 1 - field_4: array items are not unique (["string2","string2","string1"])'
+        errorMessages[8] == '* -- Entry 1 - field_1: string [src/testResources/testDir/testFile.fasta] does not match pattern ^.*\\.txt$ (["src/testResources/testDir/testFile.fasta","src/testResources/testDir2/testFile3.txt"])'
+        errorMessages[9] == "* -- Entry 1 - field_5: expected maximum item count: 3, found: 4 ([25,25,27,28])"
+        errorMessages[10] == "* -- Entry 1 - field_6: array items are not unique ([25,25])"
+        errorMessages[11] == "* -- Entry 2: Missing required value: field_4"
+        errorMessages[12] == "* -- Entry 2 - field_5: expected minimum item count: 2, found: 1 ([25])"
+        errorMessages[13] == "* -- Entry 3 - field_4: expected type: JSONArray, found: String (abc)"
+        !stdout
+    }
+
     def 'no header - CSV' () {
         given:
         def SCRIPT_TEXT = '''
diff --git a/plugins/nf-validation/src/testResources/correct_arrays.json b/plugins/nf-validation/src/testResources/correct_arrays.json
new file mode 100644
index 00000000..93dffd31
--- /dev/null
+++ b/plugins/nf-validation/src/testResources/correct_arrays.json
@@ -0,0 +1,72 @@
+[
+    {
+      "field_1": [
+        "src/testResources/testDir/testFile.txt",
+        "src/testResources/testDir2/testFile2.txt"
+      ],
+      "field_2": [
+        "src/testResources/testDir",
+        "src/testResources/testDir2"
+      ],
+      "field_3": [
+        "src/testResources/testDir",
+        "src/testResources/testDir2/testFile2.txt"
+      ],
+      "field_4": [
+        "string1",
+        "string2"
+      ],
+      "field_5": [
+        25,
+        26
+      ],
+      "field_6": [
+        25,
+        26.5
+      ],
+      "field_7": [
+        false,
+        true
+      ],
+      "field_9": [
+        true
+      ]
+    },
+    {
+      "field_4": [
+        "string1",
+        "string2"
+      ],
+      "field_5": [
+        25,
+        26
+      ],
+      "field_6": [
+        25,
+        26.5
+      ]
+    },
+    {
+      "field_4": [
+        "string1",
+        "string2"
+      ],
+      "field_5": [
+        25,
+        26
+      ],
+      "field_6": [
+        25,
+        26.5
+      ],
+      "field_11": [
+        [
+          "src/testResources/testDir/testFile.txt"
+        ],
+        [
+          "src/testResources/testDir/testFile.txt",
+          "src/testResources/testDir2/testFile2.txt"
+        ]
+      ]
+    }
+  ]
\ No newline at end of file
diff --git a/plugins/nf-validation/src/testResources/correct_arrays.yaml b/plugins/nf-validation/src/testResources/correct_arrays.yaml
new file mode 100644
index 00000000..b4654a1c
--- /dev/null
+++ b/plugins/nf-validation/src/testResources/correct_arrays.yaml
@@ -0,0 +1,45 @@
+- field_1:
+    - src/testResources/testDir/testFile.txt
+    - src/testResources/testDir2/testFile2.txt
+  field_2:
+    - src/testResources/testDir
+    - src/testResources/testDir2
+  field_3:
+    - src/testResources/testDir
+    - src/testResources/testDir2/testFile2.txt
+  field_4:
+    - string1
+    - string2
+  field_5:
+    - 25
+    - 26
+  field_6:
+    - 25
+    - 26.5
+  field_7:
+    - false
+    - true
+  field_9:
+    - true
+- field_4:
+    - string1
+    - string2
+  field_5:
+    - 25
+    - 26
+  field_6:
+    - 25
+    - 26.5
+- field_4:
+    - string1
+    - string2
+  field_5:
+    - 25
+    - 26
+  field_6:
+    - 25
+    - 26.5
+  field_11:
+    - - src/testResources/testDir/testFile.txt
+    - - src/testResources/testDir/testFile.txt
+      - src/testResources/testDir2/testFile2.txt
\ No newline at end of file
diff --git a/plugins/nf-validation/src/testResources/error_arrays.yaml b/plugins/nf-validation/src/testResources/error_arrays.yaml
new file mode 100644
index 00000000..78309dab
--- /dev/null
+++ b/plugins/nf-validation/src/testResources/error_arrays.yaml
@@ -0,0 +1,35 @@
+- field_1:
+    - src/testResources/testDir/testFile.fasta
+    - src/testResources/testDir2/testFile3.txt
+  field_2:
+    - src/testResources/testDir
+    - src/testResources/testDir3
+  field_3:
+    - src/testResources/testDir3
+    - src/testResources/testDir2/testFile3.txt
+  field_4:
+    - string2
+    - string2
+    - string1
+  field_5:
+    - 25
+    - 25
+    - 27
+    - 28
+  field_6:
+    - 25
+    - 25.0
+  field_7:
+    - null
+    - true
+  field_9:
+    - true
+- field_5:
+    - 25
+  field_6:
+    - 25
+    - 26.5
+- field_4: 'abc'
+  field_6:
+    - 25
+    - 26.5
\ No newline at end of file
diff --git a/plugins/nf-validation/src/testResources/nextflow_schema_with_samplesheet_converter_arrays.json b/plugins/nf-validation/src/testResources/nextflow_schema_with_samplesheet_converter_arrays.json
new file mode 100644
index 00000000..c1e90205
--- /dev/null
+++ b/plugins/nf-validation/src/testResources/nextflow_schema_with_samplesheet_converter_arrays.json
@@ -0,0 +1,28 @@
+{
+    "$schema": "http://json-schema.org/draft-07/schema",
+    "$id": "https://raw.githubusercontent.com/nf-core/testpipeline/master/nextflow_schema.json",
+    "title": "nf-core/testpipeline pipeline parameters",
+    "description": "this is a test",
+    "type": "object",
+    "definitions": {
+        "input_output_options": {
+            "title": "Input/output options",
+            "type": "object",
+            "fa_icon": "fas fa-terminal",
+            "description": "Define where the pipeline should find input data and save output data.",
+            "required": ["input"],
+            "properties": {
+                "input": {
+                    "type": "string",
+                    "format": "file-path",
+                    "mimetype": "text/csv",
+                    "pattern": "^\\S+\\.csv$",
+                    "schema": "src/testResources/schema_input_with_arrays.json",
+                    "description": "Path to comma-separated file containing information about the samples in the experiment.",
+                    "help_text": "You will need to create a design file with information about the samples in your experiment before running the pipeline. Use this parameter to specify its location. It has to be a comma-separated file with 3 columns, and a header row. See [usage docs](https://nf-co.re/testpipeline/usage#samplesheet-input).",
+                    "fa_icon": "fas fa-file-csv"
+                }
+            }
+        }
+    }
+}
diff --git a/plugins/nf-validation/src/testResources/schema_input_with_arrays.json b/plugins/nf-validation/src/testResources/schema_input_with_arrays.json
new file mode 100644
index 00000000..35e41743
--- /dev/null
+++ b/plugins/nf-validation/src/testResources/schema_input_with_arrays.json
@@ -0,0 +1,102 @@
+{
+    "$schema": "http://json-schema.org/draft-07/schema",
+    "$id": "https://raw.githubusercontent.com/nextflow-io/nf-validation/master/plugins/nf-validation/src/testResources/schema_input.json",
+    "title": "Samplesheet validation schema",
+    "description": "Schema for the samplesheet used in this pipeline",
+    "type": "array",
+    "items": {
+        "type": "object",
+        "properties": {
+            "field_1": {
+                "type": "array",
+                "items": {
+                    "type": "string",
+                    "format": "file-path",
+                    "pattern": "^.*\\.txt$",
+                    "exists": true
+                }
+            },
+            "field_2": {
+                "type": "array",
+                "items": {
+                    "type": "string",
+                    "format": "directory-path",
+                    "exists": true
+                }
+            },
+            "field_3": {
+                "type": "array",
+                "items": {
+                    "type": "string",
+                    "format": "path",
+                    "exists": true
+                }
+            },
+            "field_4": {
+                "type": "array",
+                "items": {
+                    "type": "string"
+                },
+                "uniqueItems": true
+            },
+            "field_5": {
+                "type": "array",
+                "items": {
+                    "type": "integer"
+                },
+                "minItems": 2,
+                "maxItems": 3
+            },
+            "field_6": {
+                "type": "array",
+                "items": {
+                    "type": "number"
+                },
+                "uniqueItems": true
+            },
+            "field_7": {
+                "type": "array",
+                "items": {
+                    "type": "boolean"
+                }
+            },
+            "field_8": {
+                "type": "array",
+                "items": {
+                    "type": "string"
+                },
+                "default": ["1", "2", "3"] 
+            },
+            "field_9": {
+                "type": "array",
+                "items": {
+                    "type": "boolean"
+                },
+                "default": [false, true, false] 
+            },
+            "field_10": {
+                "type": "array",
+                "items": {
+                    "type": "string",
+                    "format": "file-path",
+                    "pattern": "^.*\\.txt$",
+                    "exists": true
+                },
+                "default": ["src/testResources/testDir/testFile.txt"] 
+            },
+            "field_11": {
+                "type": "array",
+                "items": {
+                    "type": "array",
+                    "items": {
+                        "format": "file-path",
+                        "pattern": "^.*\\.txt$",
+                        "exists": true
+                    }
+                },
+                "default": [["src/testResources/testDir/testFile.txt"]] 
+            }
+        },
+        "required": ["field_4", "field_6"]
+    }
+}
\ No newline at end of file
diff --git a/plugins/nf-validation/src/testResources/testDir2/testFile2.txt b/plugins/nf-validation/src/testResources/testDir2/testFile2.txt
new file mode 100644
index 00000000..e69de29b

From 69f6c8f360c5222a63d542bf48d363b1594d6533 Mon Sep 17 00:00:00 2001
From: Arthur Gymer <24782660+awgymer@users.noreply.github.com>
Date: Thu, 21 Dec 2023 18:12:42 +0000
Subject: [PATCH 8/9] feat: demonstrate that array fields work in meta

---
 .../SamplesheetConverterTest.groovy           |  12 +-
 .../src/testResources/correct_arrays.json     | 143 +++++++++---------
 .../src/testResources/correct_arrays.yaml     |   6 +
 .../schema_input_with_arrays.json             |   7 +
 4 files changed, 94 insertions(+), 74 deletions(-)

diff --git a/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy b/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy
index e5161c73..d363e1ce 100644
--- a/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy
+++ b/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy
@@ -212,9 +212,9 @@ class SamplesheetConverterTest extends Dsl2Spec{
 
         then:
         noExceptionThrown()
-        stdout.contains("[[${this.getRootString()}/src/testResources/testDir/testFile.txt, ${this.getRootString()}/src/testResources/testDir2/testFile2.txt], [${this.getRootString()}/src/testResources/testDir, ${this.getRootString()}/src/testResources/testDir2], [${this.getRootString()}/src/testResources/testDir, ${this.getRootString()}/src/testResources/testDir2/testFile2.txt], [string1, string2], [25, 26], [25, 26.5], [false, true], [1, 2, 3], [true], [${this.getRootString()}/src/testResources/testDir/testFile.txt], [[${this.getRootString()}/src/testResources/testDir/testFile.txt]]]" as String)
-        stdout.contains("[[], [], [], [string1, string2], [25, 26], [25, 26.5], [], [1, 2, 3], [false, true, false], [${this.getRootString()}/src/testResources/testDir/testFile.txt], [[${this.getRootString()}/src/testResources/testDir/testFile.txt]]]" as String)
-        stdout.contains("[[], [], [], [string1, string2], [25, 26], [25, 26.5], [], [1, 2, 3], [false, true, false], [${this.getRootString()}/src/testResources/testDir/testFile.txt], [[${this.getRootString()}/src/testResources/testDir/testFile.txt], [${this.getRootString()}/src/testResources/testDir/testFile.txt, ${this.getRootString()}/src/testResources/testDir2/testFile2.txt]]]" as String)
+        stdout.contains("[[array_meta:null], [${this.getRootString()}/src/testResources/testDir/testFile.txt, ${this.getRootString()}/src/testResources/testDir2/testFile2.txt], [${this.getRootString()}/src/testResources/testDir, ${this.getRootString()}/src/testResources/testDir2], [${this.getRootString()}/src/testResources/testDir, ${this.getRootString()}/src/testResources/testDir2/testFile2.txt], [string1, string2], [25, 26], [25, 26.5], [false, true], [1, 2, 3], [true], [${this.getRootString()}/src/testResources/testDir/testFile.txt], [[${this.getRootString()}/src/testResources/testDir/testFile.txt]]]" as String)
+        stdout.contains("[[array_meta:[look, an, array, in, meta]], [], [], [], [string1, string2], [25, 26], [25, 26.5], [], [1, 2, 3], [false, true, false], [${this.getRootString()}/src/testResources/testDir/testFile.txt], [[${this.getRootString()}/src/testResources/testDir/testFile.txt]]]" as String)
+        stdout.contains("[[array_meta:null], [], [], [], [string1, string2], [25, 26], [25, 26.5], [], [1, 2, 3], [false, true, false], [${this.getRootString()}/src/testResources/testDir/testFile.txt], [[${this.getRootString()}/src/testResources/testDir/testFile.txt], [${this.getRootString()}/src/testResources/testDir/testFile.txt, ${this.getRootString()}/src/testResources/testDir2/testFile2.txt]]]" as String)
     }
 
     def 'arrays should work fine - JSON' () {
@@ -238,9 +238,9 @@ class SamplesheetConverterTest extends Dsl2Spec{
 
         then:
         noExceptionThrown()
-        stdout.contains("[[${this.getRootString()}/src/testResources/testDir/testFile.txt, ${this.getRootString()}/src/testResources/testDir2/testFile2.txt], [${this.getRootString()}/src/testResources/testDir, ${this.getRootString()}/src/testResources/testDir2], [${this.getRootString()}/src/testResources/testDir, ${this.getRootString()}/src/testResources/testDir2/testFile2.txt], [string1, string2], [25, 26], [25, 26.5], [false, true], [1, 2, 3], [true], [${this.getRootString()}/src/testResources/testDir/testFile.txt], [[${this.getRootString()}/src/testResources/testDir/testFile.txt]]]" as String)
-        stdout.contains("[[], [], [], [string1, string2], [25, 26], [25, 26.5], [], [1, 2, 3], [false, true, false], [${this.getRootString()}/src/testResources/testDir/testFile.txt], [[${this.getRootString()}/src/testResources/testDir/testFile.txt]]]" as String)
-        stdout.contains("[[], [], [], [string1, string2], [25, 26], [25, 26.5], [], [1, 2, 3], [false, true, false], [${this.getRootString()}/src/testResources/testDir/testFile.txt], [[${this.getRootString()}/src/testResources/testDir/testFile.txt], [${this.getRootString()}/src/testResources/testDir/testFile.txt, ${this.getRootString()}/src/testResources/testDir2/testFile2.txt]]]" as String)
+        stdout.contains("[[array_meta:null], [${this.getRootString()}/src/testResources/testDir/testFile.txt, ${this.getRootString()}/src/testResources/testDir2/testFile2.txt], [${this.getRootString()}/src/testResources/testDir, ${this.getRootString()}/src/testResources/testDir2], [${this.getRootString()}/src/testResources/testDir, ${this.getRootString()}/src/testResources/testDir2/testFile2.txt], [string1, string2], [25, 26], [25, 26.5], [false, true], [1, 2, 3], [true], [${this.getRootString()}/src/testResources/testDir/testFile.txt], [[${this.getRootString()}/src/testResources/testDir/testFile.txt]]]" as String)
+        stdout.contains("[[array_meta:[look, an, array, in, meta]], [], [], [], [string1, string2], [25, 26], [25, 26.5], [], [1, 2, 3], [false, true, false], [${this.getRootString()}/src/testResources/testDir/testFile.txt], [[${this.getRootString()}/src/testResources/testDir/testFile.txt]]]" as String)
+        stdout.contains("[[array_meta:null], [], [], [], [string1, string2], [25, 26], [25, 26.5], [], [1, 2, 3], [false, true, false], [${this.getRootString()}/src/testResources/testDir/testFile.txt], [[${this.getRootString()}/src/testResources/testDir/testFile.txt], [${this.getRootString()}/src/testResources/testDir/testFile.txt, ${this.getRootString()}/src/testResources/testDir2/testFile2.txt]]]" as String)
     }
 
     def 'array errors before channel conversion - YAML' () {
diff --git a/plugins/nf-validation/src/testResources/correct_arrays.json b/plugins/nf-validation/src/testResources/correct_arrays.json
index 93dffd31..d7ab9a23 100644
--- a/plugins/nf-validation/src/testResources/correct_arrays.json
+++ b/plugins/nf-validation/src/testResources/correct_arrays.json
@@ -1,72 +1,79 @@
 [
-    {
-      "field_1": [
-        "src/testResources/testDir/testFile.txt",
-        "src/testResources/testDir2/testFile2.txt"
+  {
+    "field_1": [
+      "src/testResources/testDir/testFile.txt",
+      "src/testResources/testDir2/testFile2.txt"
+    ],
+    "field_2": [
+      "src/testResources/testDir",
+      "src/testResources/testDir2"
+    ],
+    "field_3": [
+      "src/testResources/testDir",
+      "src/testResources/testDir2/testFile2.txt"
+    ],
+    "field_4": [
+      "string1",
+      "string2"
+    ],
+    "field_5": [
+      25,
+      26
+    ],
+    "field_6": [
+      25,
+      26.5
+    ],
+    "field_7": [
+      false,
+      true
+    ],
+    "field_9": [
+      true
+    ]
+  },
+  {
+    "field_4": [
+      "string1",
+      "string2"
+    ],
+    "field_5": [
+      25,
+      26
+    ],
+    "field_6": [
+      25,
+      26.5
+    ],
+    "field_12": [
+      "look",
+      "an",
+      "array",
+      "in",
+      "meta"
+    ]
+  },
+  {
+    "field_4": [
+      "string1",
+      "string2"
+    ],
+    "field_5": [
+      25,
+      26
+    ],
+    "field_6": [
+      25,
+      26.5
+    ],
+    "field_11": [
+      [
+        "src/testResources/testDir/testFile.txt"
       ],
-      "field_2": [
-        "src/testResources/testDir",
-        "src/testResources/testDir2"
-      ],
-      "field_3": [
-        "src/testResources/testDir",
+      [
+        "src/testResources/testDir/testFile.txt",
         "src/testResources/testDir2/testFile2.txt"
-      ],
-      "field_4": [
-        "string1",
-        "string2"
-      ],
-      "field_5": [
-        25,
-        26
-      ],
-      "field_6": [
-        25,
-        26.5
-      ],
-      "field_7": [
-        false,
-        true
-      ],
-      "field_9": [
-        true
-      ]
-    },
-    {
-      "field_4": [
-        "string1",
-        "string2"
-      ],
-      "field_5": [
-        25,
-        26
-      ],
-      "field_6": [
-        25,
-        26.5
-      ]
-    },
-    {
-      "field_4": [
-        "string1",
-        "string2"
-      ],
-      "field_5": [
-        25,
-        26
-      ],
-      "field_6": [
-        25,
-        26.5
-      ],
-      "field_11": [
-        [
-          "src/testResources/testDir/testFile.txt"
-        ],
-        [
-          "src/testResources/testDir/testFile.txt",
-          "src/testResources/testDir2/testFile2.txt"
-        ]
       ]
-    }
-  ]
\ No newline at end of file
+    ]
+  }
+]
\ No newline at end of file
diff --git a/plugins/nf-validation/src/testResources/correct_arrays.yaml b/plugins/nf-validation/src/testResources/correct_arrays.yaml
index b4654a1c..5ffc5a3d 100644
--- a/plugins/nf-validation/src/testResources/correct_arrays.yaml
+++ b/plugins/nf-validation/src/testResources/correct_arrays.yaml
@@ -30,6 +30,12 @@
   field_6:
     - 25
     - 26.5
+  field_12:
+    - look
+    - an
+    - array
+    - in
+    - meta
 - field_4:
     - string1
     - string2
diff --git a/plugins/nf-validation/src/testResources/schema_input_with_arrays.json b/plugins/nf-validation/src/testResources/schema_input_with_arrays.json
index 35e41743..06ebba89 100644
--- a/plugins/nf-validation/src/testResources/schema_input_with_arrays.json
+++ b/plugins/nf-validation/src/testResources/schema_input_with_arrays.json
@@ -95,6 +95,13 @@
                     }
                 },
                 "default": [["src/testResources/testDir/testFile.txt"]] 
+            },
+            "field_12": {
+                "type": "array",
+                "items": {
+                    "type": "string"
+                },
+                "meta": ["array_meta"]
             }
         },
         "required": ["field_4", "field_6"]

From 8bc3d3af88a743c2734412442b7e427f01908a89 Mon Sep 17 00:00:00 2001
From: Arthur Gymer <24782660+awgymer@users.noreply.github.com>
Date: Thu, 4 Jan 2024 14:10:12 +0000
Subject: [PATCH 9/9] feat: add error if 'arrays' used in samplesheet schema
 but a csv/tsv samplesheet is passed. Add tests for this behaviour

---
 .../validation/SchemaValidator.groovy         | 14 ++++-
 .../PluginExtensionMethodsTest.groovy         | 26 ++++++++-
 .../SamplesheetConverterTest.groovy           | 53 +++++++++++++++++++
 3 files changed, 90 insertions(+), 3 deletions(-)

diff --git a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy
index cdb56718..0ba33227 100644
--- a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy
+++ b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy
@@ -187,6 +187,11 @@ class SchemaValidator extends PluginExtensionPoint {
         def List<Map<String,String>> fileContentCasted
         def Boolean s3PathCheck = params.validationS3PathCheck ? params.validationS3PathCheck : false
         def Map types = variableTypes(schemaFile.toString(), baseDir)
+        if (types.find{ it.value == "array" } as Boolean && fileType in ["csv", "tsv"]){
+            def msg = "Using \"type\": \"array\" in schema with a \".$fileType\" samplesheet is not supported\n"
+            log.error("ERROR: Validation of pipeline parameters failed!")
+            throw new SchemaValidationException(msg, [])
+        }
         def Boolean containsHeader = !(types.keySet().size() == 1 && types.keySet()[0] == "")
 
         if(!containsHeader){
@@ -433,6 +438,11 @@ class SchemaValidator extends PluginExtensionPoint {
                     def List<Map<String,String>> fileContent
                     def List<Map<String,String>> fileContentCasted
                     def Map types = variableTypes(schema_name, baseDir)
+                    if (types.find{ it.value == "array" } as Boolean && fileType in ["csv", "tsv"]){
+                        def msg = "${colors.red}Using {\"type\": \"array\"} in schema with a \".$fileType\" samplesheet is not supported${colors.reset}\n"
+                        log.error("ERROR: Validation of pipeline parameters failed!")
+                        throw new SchemaValidationException(msg, [])
+                    }
                     def Boolean containsHeader = !(types.keySet().size() == 1 && types.keySet()[0] == "")
 
                     if(!containsHeader){
@@ -556,6 +566,8 @@ class SchemaValidator extends PluginExtensionPoint {
         Boolean monochrome_logs, String paramName, Object fileContent, String schemaFilename, String baseDir, Boolean s3PathCheck = false
     
     ) {
+        // declare this once for the method
+        def colors = logColours(monochrome_logs)
 
         // Load the schema
         def String schema_string = Files.readString( Path.of(getSchemaPath(baseDir, schemaFilename)) )
@@ -612,13 +624,11 @@ class SchemaValidator extends PluginExtensionPoint {
             validator.performValidation(schema, arrayJSON);
             if (this.hasErrors()) {
                 // Needed for custom errors such as pathExists() errors
-                def colors = logColours(monochrome_logs)
                 def msg = "${colors.red}The following errors have been detected:\n\n" + this.getErrors().join('\n').trim() + "\n${colors.reset}\n"
                 log.error("ERROR: Validation of '$paramName' file failed!")
                 throw new SchemaValidationException(msg, this.getErrors())
             }
         } catch (ValidationException e) {
-            def colors = logColours(monochrome_logs)
             JSONObject exceptionJSON = (JSONObject) e.toJSON()
             JSONObject objectJSON = new JSONObject();
             objectJSON.put("objects",arrayJSON);            
diff --git a/plugins/nf-validation/src/test/nextflow/validation/PluginExtensionMethodsTest.groovy b/plugins/nf-validation/src/test/nextflow/validation/PluginExtensionMethodsTest.groovy
index 5942d951..774f0d98 100644
--- a/plugins/nf-validation/src/test/nextflow/validation/PluginExtensionMethodsTest.groovy
+++ b/plugins/nf-validation/src/test/nextflow/validation/PluginExtensionMethodsTest.groovy
@@ -867,4 +867,28 @@ class PluginExtensionMethodsTest extends Dsl2Spec{
         error.message == '''The following errors have been detected:\n\n* -- Entry 1: Missing required value: sample\n* -- Entry 2: Missing required value: sample\n\n'''
         !stdout
     }
-}
+
+    def 'should fail because of arrays with csv' () {
+        given:
+        def schema = Path.of('src/testResources/nextflow_schema_with_samplesheet_converter_arrays.json').toAbsolutePath().toString()
+        def  SCRIPT_TEXT = """
+            params.monochrome_logs = true
+            params.input = 'src/testResources/correct.csv'
+            include { validateParameters } from 'plugin/nf-validation'
+            
+            validateParameters(parameters_schema: '$schema', monochrome_logs: params.monochrome_logs)
+        """
+
+        when:
+        dsl_eval(SCRIPT_TEXT)
+        def stdout = capture
+                .toString()
+                .readLines()
+                .findResults {it.contains('WARN nextflow.validation.SchemaValidator') || it.startsWith('* --') ? it : null }
+
+        then:
+        def error = thrown(SchemaValidationException)
+        error.message == '''Using {"type": "array"} in schema with a ".csv" samplesheet is not supported\n'''
+        !stdout
+    }
+}
\ No newline at end of file
diff --git a/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy b/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy
index d363e1ce..3d309cc7 100644
--- a/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy
+++ b/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy
@@ -281,6 +281,59 @@ class SamplesheetConverterTest extends Dsl2Spec{
         !stdout
     }
 
+    def 'array errors samplesheet format - CSV' () {
+        given:
+        def SCRIPT_TEXT = '''
+            include { fromSamplesheet } from 'plugin/nf-validation'
+
+            params.input = 'src/testResources/correct.csv'
+
+            workflow {
+                Channel.fromSamplesheet("input", parameters_schema:"src/testResources/nextflow_schema_with_samplesheet_converter_arrays.json").view()
+            }
+        '''
+
+        when:
+        dsl_eval(SCRIPT_TEXT)
+        def stdout = capture
+                .toString()
+                .readLines()
+                .findResults {it.startsWith('[[') ? it : null }
+
+        then:
+        def error = thrown(SchemaValidationException)
+        def errorMessages = error.message.readLines()
+        errorMessages[0] == 'Using "type": "array" in schema with a ".csv" samplesheet is not supported'
+        !stdout
+    }
+
+    def 'array errors samplesheet format - TSV' () {
+        given:
+        def SCRIPT_TEXT = '''
+            include { fromSamplesheet } from 'plugin/nf-validation'
+
+            params.input = 'src/testResources/correct.tsv'
+
+            workflow {
+                Channel.fromSamplesheet("input", parameters_schema:"src/testResources/nextflow_schema_with_samplesheet_converter_arrays.json").view()
+            }
+        '''
+
+        when:
+        dsl_eval(SCRIPT_TEXT)
+        def stdout = capture
+                .toString()
+                .readLines()
+                .findResults {it.startsWith('[[') ? it : null }
+
+        then:
+        def error = thrown(SchemaValidationException)
+        def errorMessages = error.message.readLines()
+        errorMessages[0] == 'Using "type": "array" in schema with a ".tsv" samplesheet is not supported'
+        !stdout
+    }
+
+
     def 'no header - CSV' () {
         given:
         def SCRIPT_TEXT = '''