diff --git a/docs/workflow.md b/docs/workflow.md index 8410778ebc..10d7359adb 100644 --- a/docs/workflow.md +++ b/docs/workflow.md @@ -53,13 +53,7 @@ params { } ``` -The following types can be used for parameters: - -- {ref}`stdlib-types-boolean` -- {ref}`stdlib-types-float` -- {ref}`stdlib-types-integer` -- {ref}`stdlib-types-path` -- {ref}`stdlib-types-string` +All {ref}`standard types ` except for the dataflow types (`Channel` and `Value`) can be used for parameters. Parameters can be used in the entry workflow: @@ -75,7 +69,16 @@ As a best practice, parameters should only be referenced in the entry workflow o The default value can be overridden by the command line, params file, or config file. Parameters from multiple sources are resolved in the order described in {ref}`cli-params`. Parameters specified on the command line are converted to the appropriate type based on the corresponding type annotation. -A parameter that doesn't specify a default value is a *required* param. If a required param is not given a value at runtime, the run will fail. +A parameter that doesn't specify a default value is a *required* parameter. If a required parameter is not given a value at runtime, the run will fail. + +:::{versionadded} 26.04.0 +::: + +Parameters with a collection type (i.e., `List`, `Set`, or `Bag`) can be supplied a file path instead of a literal collection. The file must be CSV, JSON, or YAML. Nextflow will parse the file contents and assign the resuling collection to the parameter. An error is thrown if the file contents do not match the parameter type. + +:::{note} +When supplying a CSV file to a collection parameter, the CSV file must contain a header row and must use a comma (`,`) as the column separator. +::: (workflow-params-legacy)= diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ParamsDsl.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ParamsDsl.groovy index bd79682a20..a4e28d48ab 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/ParamsDsl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/ParamsDsl.groovy @@ -18,13 +18,22 @@ package nextflow.script import java.nio.file.Path +import groovy.json.JsonSlurper +import groovy.yaml.YamlSlurper import groovy.transform.Canonical import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import nextflow.Session import nextflow.file.FileHelper import nextflow.exception.ScriptRuntimeException +import nextflow.script.types.Bag import nextflow.script.types.Types +import nextflow.splitter.CsvSplitter +import nextflow.util.ArrayBag +import nextflow.util.Duration +import nextflow.util.MemoryUnit +import org.codehaus.groovy.runtime.typehandling.DefaultTypeTransformation +import org.codehaus.groovy.runtime.typehandling.GroovyCastException /** * Implements the DSL for defining workflow params * @@ -102,6 +111,18 @@ class ParamsDsl { if( str.isBigDecimal() ) return str.toBigDecimal() } + if( decl.type == Duration ) { + return Duration.of(str) + } + + if( decl.type == MemoryUnit ) { + return MemoryUnit.of(str) + } + + if( Collection.class.isAssignableFrom(decl.type) ) { + return resolveFromFile(decl.name, decl.type, FileHelper.asPath(str)) + } + if( decl.type == Path ) { return FileHelper.asPath(str) } @@ -113,12 +134,41 @@ class ParamsDsl { if( value == null ) return null - if( decl.type == Path && value instanceof CharSequence ) - return FileHelper.asPath(value.toString()) + if( value !instanceof CharSequence ) + return value + + final str = value.toString() + + if( Collection.class.isAssignableFrom(decl.type) ) + return resolveFromFile(decl.name, decl.type, FileHelper.asPath(str)) + + if( decl.type == Path ) + return FileHelper.asPath(str) return value } + private Object resolveFromFile(String name, Class type, Path file) { + final ext = file.getExtension() + final value = switch( ext ) { + case 'csv' -> new CsvSplitter().options(header: true, sep: ',').target(file).list() + case 'json' -> new JsonSlurper().parse(file) + case 'yaml' -> new YamlSlurper().parse(file) + case 'yml' -> new YamlSlurper().parse(file) + default -> throw new ScriptRuntimeException("Unrecognized file format '${ext}' for input file '${file}' supplied for parameter `${name}` -- should be CSV, JSON, or YAML") + } + + try { + if( Bag.class.isAssignableFrom(type) && value instanceof Collection ) + return new ArrayBag(value) + return DefaultTypeTransformation.castToType(value, type) + } + catch( GroovyCastException e ) { + final actualType = value.getClass() + throw new ScriptRuntimeException("Parameter `${name}` with type ${Types.getName(type)} cannot be assigned to contents of '${file}' [${Types.getName(actualType)}]") + } + } + private boolean isAssignableFrom(Class target, Class source) { if( target == Float.class ) return Number.class.isAssignableFrom(source) diff --git a/modules/nextflow/src/test/groovy/nextflow/script/ParamsDslTest.groovy b/modules/nextflow/src/test/groovy/nextflow/script/ParamsDslTest.groovy index 910457d698..67a4245aa6 100644 --- a/modules/nextflow/src/test/groovy/nextflow/script/ParamsDslTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/script/ParamsDslTest.groovy @@ -1,10 +1,12 @@ package nextflow.script +import java.nio.file.Files import java.nio.file.Path import nextflow.Session import nextflow.file.FileHelper import nextflow.exception.ScriptRuntimeException +import nextflow.script.types.Bag import spock.lang.Specification import spock.lang.Unroll /** @@ -128,4 +130,187 @@ class ParamsDslTest extends Specification { DEF_VALUE << [ 100i, 100l, 100g ] } + def 'should load collection param from CSV file'() { + given: + def csvFile = Files.createTempFile('test', '.csv') + csvFile.text = '''\ + id,name,value + 1,sample1,100 + 2,sample2,200 + 3,sample3,300 + '''.stripIndent() + def cliParams = [samples: csvFile.toString()] + def session = new Session() + session.init(null, null, cliParams, [:]) + + when: + def dsl = new ParamsDsl() + dsl.declare('samples', List, false) + dsl.apply(session) + + then: + def samples = session.binding.getParams().samples + samples instanceof List + samples.size() == 3 + samples[0].id == '1' + samples[0].name == 'sample1' + samples[0].value == '100' + samples[1].id == '2' + samples[2].id == '3' + + cleanup: + csvFile?.delete() + } + + def 'should load collection param from JSON file'() { + given: + def jsonFile = Files.createTempFile('test', '.json') + jsonFile.text = '''\ + [ + {"id": 1, "name": "sample1", "value": 100}, + {"id": 2, "name": "sample2", "value": 200}, + {"id": 3, "name": "sample3", "value": 300} + ] + '''.stripIndent() + def cliParams = [ + samplesList: jsonFile.toString(), + samplesBag: jsonFile.toString(), + samplesSet: jsonFile.toString() + ] + def session = new Session() + session.init(null, null, cliParams, [:]) + + when: + def dsl = new ParamsDsl() + dsl.declare('samplesList', List, false) + dsl.declare('samplesBag', Bag, false) + dsl.declare('samplesSet', Set, false) + dsl.apply(session) + + then: + def samplesList = session.binding.getParams().samplesList + samplesList instanceof List + samplesList.size() == 3 + samplesList[0].id == 1 + samplesList[0].name == 'sample1' + samplesList[0].value == 100 + samplesList[1].id == 2 + samplesList[2].id == 3 + + def samplesBag = session.binding.getParams().samplesBag + samplesBag instanceof Bag + samplesBag.size() == 3 + + def samplesSet = session.binding.getParams().samplesSet + samplesSet instanceof Set + samplesSet.size() == 3 + + cleanup: + jsonFile?.delete() + } + + def 'should load collection param from YAML file'() { + given: + def yamlFile = Files.createTempFile('test', '.yml') + yamlFile.text = '''\ + - id: 1 + name: sample1 + value: 100 + - id: 2 + name: sample2 + value: 200 + - id: 3 + name: sample3 + value: 300 + '''.stripIndent() + def cliParams = [samples: yamlFile.toString()] + def session = new Session() + session.init(null, null, cliParams, [:]) + + when: + def dsl = new ParamsDsl() + dsl.declare('samples', List, false) + dsl.apply(session) + + then: + def samples = session.binding.getParams().samples + samples instanceof List + samples.size() == 3 + samples[0].id == 1 + samples[0].name == 'sample1' + samples[0].value == 100 + samples[1].id == 2 + samples[2].id == 3 + + cleanup: + yamlFile?.delete() + } + + def 'should load collection param from file specified in config'() { + given: + def jsonFile = Files.createTempFile('test', '.json') + jsonFile.text = '[{"x": 1}, {"x": 2}]' + def configParams = [items: jsonFile.toString()] + def session = new Session() + session.init(null, null, [:], configParams) + + when: + def dsl = new ParamsDsl() + dsl.declare('items', List, false) + dsl.apply(session) + + then: + def items = session.binding.getParams().items + items instanceof List + items.size() == 2 + items[0].x == 1 + items[1].x == 2 + + cleanup: + jsonFile?.delete() + } + + def 'should report error for unrecognized file format'() { + given: + def txtFile = Files.createTempFile('test', '.txt') + txtFile.text = 'some text' + def cliParams = [items: txtFile.toString()] + def session = new Session() + session.init(null, null, cliParams, [:]) + + when: + def dsl = new ParamsDsl() + dsl.declare('items', List, false) + dsl.apply(session) + + then: + def e = thrown(ScriptRuntimeException) + e.message.contains("Unrecognized file format 'txt'") + e.message.contains("supplied for parameter `items` -- should be CSV, JSON, or YAML") + + cleanup: + txtFile?.delete() + } + + def 'should report error for invalid file content type'() { + given: + def jsonFile = Files.createTempFile('test', '.json') + jsonFile.text = '{"not": "a list"}' + def cliParams = [items: jsonFile.toString()] + def session = new Session() + session.init(null, null, cliParams, [:]) + + when: + def dsl = new ParamsDsl() + dsl.declare('items', List, false) + dsl.apply(session) + + then: + def e = thrown(ScriptRuntimeException) + e.message.contains('Parameter `items` with type List cannot be assigned to contents of') + + cleanup: + jsonFile?.delete() + } + }