diff --git a/docs/conda.md b/docs/conda.md index 522b4b9900..03077c7021 100644 --- a/docs/conda.md +++ b/docs/conda.md @@ -163,10 +163,43 @@ https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-13.2.0-h77fa898_7.cond # .. and so on ``` -To use a Conda lock file with Nextflow, set the `conda` directive to the path of the lock file. +To use a Conda lock file with Nextflow, set the `conda` directive to the path of the lock file: + +```nextflow +process hello { + conda '/path/to/spec-file.lock' + + script: + """ + your_command --here + """ +} +``` + +:::{versionchanged} 26.01.0-edge +Conda lock files are now detected by the presence of the `@EXPLICIT` marker in the first 20 lines of the file. +They can have any file extension (e.g., `.lock`, `.txt`, or no extension at all). +::: :::{note} -Conda lock files must be a text file with the `.txt` extension. +Remote URLs are only supported for Conda environment YAML files (`.yml`, `.yaml`), not for lock files. +Lock files should be bundled with your pipeline as local files. This is intentional, as lock files are considered part of the module/pipeline bundle and should be versioned alongside your code rather than fetched on-the-fly. + +For example, you can use a remote YAML environment file: +```nextflow +process hello { + conda 'https://example.com/my-env.yml' + // ... +} +``` + +But lock files must be local: +```nextflow +process hello { + conda '/path/to/spec-file.lock' + // ... +} +``` ::: ### Use existing Conda environments diff --git a/modules/nextflow/src/main/groovy/nextflow/conda/CondaCache.groovy b/modules/nextflow/src/main/groovy/nextflow/conda/CondaCache.groovy index c019d32029..e0f04f4efe 100644 --- a/modules/nextflow/src/main/groovy/nextflow/conda/CondaCache.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/conda/CondaCache.groovy @@ -17,6 +17,7 @@ package nextflow.conda import java.nio.file.FileSystems +import java.nio.file.Files import java.nio.file.NoSuchFileException import java.nio.file.Path import java.nio.file.Paths @@ -166,6 +167,36 @@ class CondaCache { str.endsWith('.txt') && !str.contains('\n') } + /** + * Check if the given file is a conda lock file. + * A conda lock file contains "@EXPLICIT" marker in the first 20 lines. + * This method reads only the first 20 lines to avoid loading the entire file into memory. + * + * @param path The file path to check + * @return true if this is a conda lock file, false otherwise + */ + @PackageScope + boolean isLockFile(Path path) { + if( path == null || !Files.exists(path) ) + return false + try { + path.withReader { reader -> + String line + int count = 0 + while( count < 20 && (line = reader.readLine()) != null ) { + if( line.trim() == '@EXPLICIT' ) + return true + count++ + } + return false + } + } + catch( Exception e ) { + log.debug "Error checking if file is lock file: $path", e + return false + } + } + /** * Get the path on the file system where store a Conda environment * @@ -178,8 +209,14 @@ class CondaCache { String content String name = 'env' - // check if it's a remote uri + + // check if it's a remote HTTP/HTTPS URL if( isYamlUriPath(condaEnv) ) { + // Remote URLs are only supported for YAML environment files + if( !isYamlFilePath(condaEnv) ) { + throw new IllegalArgumentException("Remote Conda lock files are not supported. Only environment YAML files (.yml, .yaml) can be specified as remote URLs: $condaEnv") + } + // Use the URL itself as the content for hashing - conda will fetch it content = condaEnv } // check if it's a YAML file @@ -210,13 +247,26 @@ class CondaCache { } // it's interpreted as user provided prefix directory else if( condaEnv.contains('/') ) { + // check if it's a file path that might be a lock file final prefix = condaEnv as Path - if( !prefix.isDirectory() ) + if( prefix.isDirectory() ) { + if( prefix.fileSystem != FileSystems.default ) + throw new IllegalArgumentException("Conda prefix path must be a POSIX file path: $prefix") + return prefix + } + // it could be a file with a non-standard extension (e.g., .lock or no extension) + if( Files.exists(prefix) ) { + // if it's a lock file, read content for hashing (like YAML/text files); otherwise treat as invalid + if( !isLockFile(prefix) ) { + throw new IllegalArgumentException("Conda environment file is not a valid lock file (missing @EXPLICIT marker): $condaEnv") + } + // Note: file is read twice - once by isLockFile (first 20 lines only) and once here for full content. + // This is intentional: isLockFile is memory-efficient for large files, while hashing needs full content. + content = prefix.text + } + else { throw new IllegalArgumentException("Conda prefix path does not exist or is not a directory: $prefix") - if( prefix.fileSystem != FileSystems.default ) - throw new IllegalArgumentException("Conda prefix path must be a POSIX file path: $prefix") - - return prefix + } } else if( condaEnv.contains('\n') ) { throw new IllegalArgumentException("Invalid Conda environment definition: $condaEnv") @@ -279,15 +329,28 @@ class CondaCache { String opts = createOptions ? "$createOptions " : '' def cmd + // Check if it's a YAML file (by extension) - can be local path or HTTP/HTTPS URL if( isYamlFilePath(condaEnv) ) { + // For remote URLs, pass directly to conda; for local files, use absolute path final target = isYamlUriPath(condaEnv) ? condaEnv : Escape.path(makeAbsolute(condaEnv)) final yesOpt = binaryName=="mamba" || binaryName == "micromamba" ? '--yes ' : '' cmd = "${binaryName} env create ${yesOpt}--prefix ${Escape.path(prefixPath)} --file ${target}" } + // Check if it's a text file (by extension) - legacy support for .txt lock files else if( isTextFilePath(condaEnv) ) { cmd = "${binaryName} create ${opts}--yes --quiet --prefix ${Escape.path(prefixPath)} --file ${Escape.path(makeAbsolute(condaEnv))}" } - + // Check if it's a file path with non-standard extension that might be a lock file + else if( condaEnv.contains('/') ) { + final localPath = makeAbsolute(condaEnv) + if( Files.exists(localPath) && isLockFile(localPath) ) { + cmd = "${binaryName} create ${opts}--yes --quiet --prefix ${Escape.path(prefixPath)} --file ${Escape.path(localPath)}" + } + else { + throw new IllegalArgumentException("Conda environment file is not a valid lock file: $condaEnv") + } + } + // Otherwise treat as package name(s) else { final channelsOpt = channels.collect(it -> "-c $it ").join('') cmd = "${binaryName} create ${opts}--yes --quiet --prefix ${Escape.path(prefixPath)} ${channelsOpt}$condaEnv" diff --git a/modules/nextflow/src/test/groovy/nextflow/conda/CondaCacheTest.groovy b/modules/nextflow/src/test/groovy/nextflow/conda/CondaCacheTest.groovy index d8a0286056..a559919741 100644 --- a/modules/nextflow/src/test/groovy/nextflow/conda/CondaCacheTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/conda/CondaCacheTest.groovy @@ -17,6 +17,7 @@ package nextflow.conda import java.nio.file.Files +import java.nio.file.Path import java.nio.file.Paths import nextflow.SysEnv @@ -59,6 +60,54 @@ class CondaCacheTest extends Specification { cache.isTextFilePath('foo/bar/env.txt') } + def 'should detect lock file content' () { + + given: + def cache = new CondaCache() + def folder = Files.createTempDirectory('test') + + expect: + // Valid lock file content - @EXPLICIT marker present + isLockFileContent(cache, folder, '@EXPLICIT\nhttps://conda.anaconda.org/conda-forge/linux-64/package-1.0.tar.bz2') + isLockFileContent(cache, folder, '# This file may be used to create an environment\n@EXPLICIT\nhttps://url') + isLockFileContent(cache, folder, '# comment\n# another comment\n@EXPLICIT\nhttps://url') + // With spaces/indentation + isLockFileContent(cache, folder, ' @EXPLICIT \nhttps://url') + + // Invalid lock file content - no @EXPLICIT marker + !isLockFileContent(cache, folder, 'foo=1.0') + !isLockFileContent(cache, folder, 'channels:\n - conda-forge\ndependencies:\n - bwa') + !isLockFileContent(cache, folder, '') + !cache.isLockFile(null) + // @EXPLICIT after 20 lines should not be detected + !isLockFileContent(cache, folder, (1..25).collect { "# line $it" }.join('\n') + '\n@EXPLICIT') + + cleanup: + folder?.deleteDir() + } + + private boolean isLockFileContent(CondaCache cache, Path folder, String content) { + def file = folder.resolve("test-${System.nanoTime()}.lock") + file.text = content + return cache.isLockFile(file) + } + + def 'should detect yaml uri path' () { + + given: + def cache = new CondaCache() + + expect: + // HTTP/HTTPS protocols for YAML files + cache.isYamlUriPath('http://example.com/env.yml') + cache.isYamlUriPath('https://example.com/env.yaml') + cache.isYamlUriPath('https://foo.com/some/path/environment.yml') + // Not YAML URIs + !cache.isYamlUriPath('foo.yml') + !cache.isYamlUriPath('/path/to/env.yml') + !cache.isYamlUriPath('s3://bucket/path/to/env.yml') + !cache.isYamlUriPath('file:///path/to/env.yml') + } def 'should create conda env prefix path for a string env' () { @@ -75,20 +124,38 @@ class CondaCacheTest extends Specification { prefix.toString() == '/conda/envs/env-eaeb133f4ca62c95e9c0eec7ef8d553b' } - def 'should create conda env prefix path for remote uri' () { + def 'should create conda env prefix path for remote yaml uri' () { given: - def ENV = 'https://foo.com/lock-file.yml' + def ENV = 'https://foo.com/environment.yml' def cache = Spy(CondaCache) def BASE = Paths.get('/conda/envs') when: def prefix = cache.condaPrefixPath(ENV) then: - 0 * cache.isYamlFilePath(ENV) + // Remote YAML URIs use the URL itself as the content for hashing 1 * cache.isYamlUriPath(ENV) + 1 * cache.isYamlFilePath(ENV) 1 * cache.getCacheDir() >> BASE - prefix.toString() == '/conda/envs/env-12c863103deed9425ce8012323f948fc' + // Hash is based on the URL string, not file content + prefix.toString().startsWith('/conda/envs/env-') + } + + def 'should reject remote non-yaml uri' () { + + given: + def ENV = 'https://foo.com/condalock' + def cache = Spy(CondaCache) + + when: + cache.condaPrefixPath(ENV) + + then: + 1 * cache.isYamlUriPath(ENV) + 1 * cache.isYamlFilePath(ENV) + def e = thrown(IllegalArgumentException) + e.message.contains('Remote Conda lock files are not supported') } def 'should create conda env prefix path for a yaml env file' () { @@ -192,6 +259,84 @@ class CondaCacheTest extends Specification { folder?.deleteDir() } + def 'should create conda env prefix path for a lock file with .lock extension' () { + + given: + def folder = Files.createTempDirectory('test') + def cache = Spy(CondaCache) + def BASE = Paths.get('/conda/envs') + def ENV = folder.resolve('env.lock') + ENV.text = ''' + # This file may be used to create an environment using: + # $ conda create --name --file + @EXPLICIT + https://conda.anaconda.org/conda-forge/linux-64/package-1.0.tar.bz2 + ''' + .stripIndent(true) + + when: + def prefix = cache.condaPrefixPath(ENV.toString()) + then: + 1 * cache.isYamlFilePath(ENV.toString()) + 1 * cache.isTextFilePath(ENV.toString()) + 1 * cache.isLockFile(_) >> true + 1 * cache.getCacheDir() >> BASE + prefix.toString().startsWith("/conda/envs/env-") + + cleanup: + folder?.deleteDir() + } + + def 'should create conda env prefix path for a lock file with no extension' () { + + given: + def folder = Files.createTempDirectory('test') + def cache = Spy(CondaCache) + def BASE = Paths.get('/conda/envs') + def ENV = folder.resolve('condalock') + ENV.text = ''' + @EXPLICIT + https://conda.anaconda.org/conda-forge/linux-64/package-1.0.tar.bz2 + ''' + .stripIndent(true) + + when: + def prefix = cache.condaPrefixPath(ENV.toString()) + then: + 1 * cache.isYamlFilePath(ENV.toString()) + 1 * cache.isTextFilePath(ENV.toString()) + 1 * cache.isLockFile(_) >> true + 1 * cache.getCacheDir() >> BASE + prefix.toString().startsWith("/conda/envs/env-") + + cleanup: + folder?.deleteDir() + } + + def 'should reject file with non-standard extension that is not a lock file' () { + + given: + def folder = Files.createTempDirectory('test') + def cache = Spy(CondaCache) + def ENV = folder.resolve('env.lock') + ENV.text = ''' + # This is not a valid lock file + bwa=1.0 + samtools=1.15 + ''' + .stripIndent(true) + + when: + cache.condaPrefixPath(ENV.toString()) + + then: + def e = thrown(IllegalArgumentException) + e.message.contains('not a valid lock file') + + cleanup: + folder?.deleteDir() + } + def 'should create a conda environment' () { given: def ENV = 'bwa=1.1.1' @@ -264,9 +409,9 @@ class CondaCacheTest extends Specification { result == PREFIX } - def 'should create a conda environment using mamba and remote lock file' () { + def 'should create a conda environment using mamba and remote yaml file' () { given: - def ENV = 'http://foo.com/some/file-lock.yml' + def ENV = 'http://foo.com/some/env.yml' def PREFIX = Files.createTempDirectory('foo') def cache = Spy(new CondaCache(useMamba: true)) @@ -282,15 +427,18 @@ class CondaCacheTest extends Specification { PREFIX.deleteDir() result = cache.createLocalCondaEnv0(ENV, PREFIX) then: + // isYamlFilePath returns true for .yml files, so we enter YAML branch 1 * cache.isYamlFilePath(ENV) - 0 * cache.makeAbsolute(_) - 1 * cache.runCommand("mamba env create --yes --prefix $PREFIX --file $ENV") >> null + // isYamlUriPath is called to determine if URL should be passed directly + 1 * cache.isYamlUriPath(ENV) + // URL is passed directly to mamba - no local staging + 1 * cache.runCommand({ it.contains('mamba env create') && it.contains('--yes') && it.contains('--file') && it.contains(ENV) }) >> null result == PREFIX } - def 'should create a conda environment using micromamba and remote lock file' () { + def 'should create a conda environment using micromamba and remote yaml file' () { given: - def ENV = 'http://foo.com/some/file-lock.yml' + def ENV = 'http://foo.com/some/env.yml' def PREFIX = Files.createTempDirectory('foo') def cache = Spy(new CondaCache(useMicromamba: true)) @@ -306,9 +454,12 @@ class CondaCacheTest extends Specification { PREFIX.deleteDir() result = cache.createLocalCondaEnv0(ENV, PREFIX) then: + // isYamlFilePath returns true for .yml files, so we enter YAML branch 1 * cache.isYamlFilePath(ENV) - 0 * cache.makeAbsolute(_) - 1 * cache.runCommand("micromamba env create --yes --prefix $PREFIX --file $ENV") >> null + // isYamlUriPath is called to determine if URL should be passed directly + 1 * cache.isYamlUriPath(ENV) + // URL is passed directly to micromamba - no local staging + 1 * cache.runCommand({ it.contains('micromamba env create') && it.contains('--yes') && it.contains('--file') && it.contains(ENV) }) >> null result == PREFIX } @@ -454,6 +605,60 @@ class CondaCacheTest extends Specification { } + def 'should create a conda env with a lock file with .lock extension' () { + + given: + def folder = Files.createTempDirectory('test') + def ENV = folder.resolve('env.lock').toString() + folder.resolve('env.lock').text = ''' + @EXPLICIT + https://conda.anaconda.org/conda-forge/linux-64/package-1.0.tar.bz2 + ''' + .stripIndent(true) + def PREFIX = Paths.get('/conda/envs/my-env') + and: + def cache = Spy(new CondaCache(createOptions: '--this --that')) + + when: + def result = cache.createLocalCondaEnv0(ENV, PREFIX) + then: + 1 * cache.isYamlFilePath(ENV) + 1 * cache.isTextFilePath(ENV) + 1 * cache.isLockFile(_) >> true + 1 * cache.runCommand({ it.contains('--file') && it.contains('env.lock') }) >> null + result == PREFIX + + cleanup: + folder?.deleteDir() + } + + def 'should create a conda env with a lock file with no extension' () { + + given: + def folder = Files.createTempDirectory('test') + def ENV = folder.resolve('condalock').toString() + folder.resolve('condalock').text = ''' + @EXPLICIT + https://conda.anaconda.org/conda-forge/linux-64/package-1.0.tar.bz2 + ''' + .stripIndent(true) + def PREFIX = Paths.get('/conda/envs/my-env') + and: + def cache = Spy(new CondaCache()) + + when: + def result = cache.createLocalCondaEnv0(ENV, PREFIX) + then: + 1 * cache.isYamlFilePath(ENV) + 1 * cache.isTextFilePath(ENV) + 1 * cache.isLockFile(_) >> true + 1 * cache.runCommand({ it.contains('--file') && it.contains('condalock') }) >> null + result == PREFIX + + cleanup: + folder?.deleteDir() + } + def 'should get options from the config' () { when: