diff --git a/modules/nextflow/src/main/groovy/nextflow/conda/CondaCache.groovy b/modules/nextflow/src/main/groovy/nextflow/conda/CondaCache.groovy index c019d32029..7f0df4c833 100644 --- a/modules/nextflow/src/main/groovy/nextflow/conda/CondaCache.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/conda/CondaCache.groovy @@ -23,6 +23,7 @@ import java.nio.file.Paths import java.util.concurrent.ConcurrentHashMap import groovy.transform.CompileStatic +import groovy.transform.Memoized import groovy.transform.PackageScope import groovy.util.logging.Slf4j import groovyx.gpars.dataflow.DataflowVariable @@ -162,8 +163,53 @@ class CondaCache { (str.endsWith('.yml') || str.endsWith('.yaml')) && !str.contains('\n') } - boolean isTextFilePath(String str) { - str.endsWith('.txt') && !str.contains('\n') + /** + * Check if the given string is a path to a conda explicit file + * by verifying it contains the @EXPLICIT marker in the first 20 lines + * + * @param str The conda environment string + * @return {@code true} if it's a path to an explicit file, {@code false} otherwise + */ + @PackageScope + @Memoized // <-- annotate as "Memoized" to avoid parsing multiple time the same file + boolean isExplicitFile(String str) { + if( str.contains('\n') ) + return false + try { + final path = str as Path + if( !path.exists() ) + return false + return containsExplicitMarker(path) + } + catch( Exception e ) { + log.debug "Failed to check explicit file: $str - ${e.message}" + return false + } + } + + /** + * Check if a file contains the @EXPLICIT marker in the first 20 lines + * + * @param path The file path to check + * @return {@code true} if the marker is found, {@code false} otherwise + */ + private boolean containsExplicitMarker(Path path) { + try { + try(final reader = path.newReader()) { + for( int i = 0; i < 20; i++ ) { + final line = reader.readLine() + if( line == null ) + break + if( line.trim() == '@EXPLICIT' ) + return true + } + return false + } + } + catch( Exception e ) { + log.debug "Failed to check for @EXPLICIT marker in file: $path - ${e.message}" + return false + } } /** @@ -196,16 +242,14 @@ class CondaCache { throw new IllegalArgumentException("Error parsing Conda environment YAML file: $condaEnv -- Check the log file for details", e) } } - else if( isTextFilePath(condaEnv) ) { + // check if it's a conda explicit file (contains @EXPLICIT marker) + else if( isExplicitFile(condaEnv) ) { try { final path = condaEnv as Path content = path.text } - catch( NoSuchFileException e ) { - throw new IllegalArgumentException("Conda environment file does not exist: $condaEnv") - } catch( Exception e ) { - throw new IllegalArgumentException("Error parsing Conda environment text file: $condaEnv -- Check the log file for details", e) + throw new IllegalArgumentException("Error reading Conda explicit file: $condaEnv -- Check the log file for details", e) } } // it's interpreted as user provided prefix directory @@ -284,7 +328,7 @@ class CondaCache { final yesOpt = binaryName=="mamba" || binaryName == "micromamba" ? '--yes ' : '' cmd = "${binaryName} env create ${yesOpt}--prefix ${Escape.path(prefixPath)} --file ${target}" } - else if( isTextFilePath(condaEnv) ) { + else if( isExplicitFile(condaEnv) ) { cmd = "${binaryName} create ${opts}--yes --quiet --prefix ${Escape.path(prefixPath)} --file ${Escape.path(makeAbsolute(condaEnv))}" } diff --git a/modules/nextflow/src/test/groovy/nextflow/conda/CondaCacheTest.groovy b/modules/nextflow/src/test/groovy/nextflow/conda/CondaCacheTest.groovy index d8a0286056..0d287634a7 100644 --- a/modules/nextflow/src/test/groovy/nextflow/conda/CondaCacheTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/conda/CondaCacheTest.groovy @@ -46,17 +46,49 @@ class CondaCacheTest extends Specification { cache.isYamlFilePath('env.yaml') } - def 'should text file' () { + def 'should detect explicit file by content' () { given: + def folder = Files.createTempDirectory('test') def cache = new CondaCache() + // File with @EXPLICIT marker + def explicitFile = folder.resolve('packages.txt') + explicitFile.text = '''\ + # This file was created by conda + @EXPLICIT + https://repo.anaconda.com/pkgs/main/linux-64/bwa-0.7.17.tar.bz2 + '''.stripIndent() + + // File without @EXPLICIT marker + def regularFile = folder.resolve('regular.txt') + regularFile.text = '''\ + bwa=0.7.17 + samtools=1.9 + '''.stripIndent() + + // YAML file (should not match) + def yamlFile = folder.resolve('env.yaml') + yamlFile.text = '''\ + dependencies: + - bwa=0.7.17 + '''.stripIndent() + expect: - !cache.isTextFilePath('foo=1.0') - !cache.isTextFilePath('env.yaml') - !cache.isTextFilePath('foo.txt\nbar.txt') - cache.isTextFilePath('env.txt') - cache.isTextFilePath('foo/bar/env.txt') + // Non-existent file + !cache.isExplicitFile('foo=1.0') + !cache.isExplicitFile('nonexistent.txt') + // String with newline + !cache.isExplicitFile('foo.txt\nbar.txt') + // File with @EXPLICIT marker + cache.isExplicitFile(explicitFile.toString()) + // File without @EXPLICIT marker + !cache.isExplicitFile(regularFile.toString()) + // YAML file + !cache.isExplicitFile(yamlFile.toString()) + + cleanup: + folder?.deleteDir() } @@ -147,27 +179,28 @@ class CondaCacheTest extends Specification { } - def 'should create conda env prefix path for a text env file' () { + def 'should create conda env prefix path for a conda explicit file' () { given: def folder = Files.createTempDirectory('test') def cache = Spy(CondaCache) def BASE = Paths.get('/conda/envs') def ENV = folder.resolve('bar.txt') - ENV.text = ''' - star=2.5.4a - bwa=0.7.15 - multiqc=1.2.3 - ''' - .stripIndent(true) // https://issues.apache.org/jira/browse/GROOVY-9423 + ENV.text = '''\ + # This file was created by conda + @EXPLICIT + https://repo.anaconda.com/pkgs/main/linux-64/star-2.5.4a.tar.bz2 + https://repo.anaconda.com/pkgs/main/linux-64/bwa-0.7.15.tar.bz2 + https://repo.anaconda.com/pkgs/main/linux-64/multiqc-1.2.3.tar.bz2 + '''.stripIndent() when: def prefix = cache.condaPrefixPath(ENV.toString()) then: 1 * cache.isYamlFilePath(ENV.toString()) - 1 * cache.isTextFilePath(ENV.toString()) + 1 * cache.isExplicitFile(ENV.toString()) 1 * cache.getCacheDir() >> BASE - prefix.toString() == "/conda/envs/env-85371202d8820331ff19ae89c0595497" + prefix.toString() == "/conda/envs/env-24d602a5eecba868858ab48a41e2c9bd" cleanup: folder?.deleteDir() @@ -323,7 +356,7 @@ class CondaCacheTest extends Specification { def result = cache.createLocalCondaEnv0(ENV,PREFIX) then: 1 * cache.isYamlFilePath(ENV) - 1 * cache.isTextFilePath(ENV) + 1 * cache.isExplicitFile(ENV) 0 * cache.makeAbsolute(_) 1 * cache.runCommand( "conda create --this --that --yes --quiet --prefix $PREFIX $ENV" ) >> null result == PREFIX @@ -340,7 +373,7 @@ class CondaCacheTest extends Specification { def result = cache.createLocalCondaEnv0(ENV, PREFIX) then: 1 * cache.isYamlFilePath(ENV) - 1 * cache.isTextFilePath(ENV) + 1 * cache.isExplicitFile(ENV) 0 * cache.makeAbsolute(_) 1 * cache.runCommand("mamba create --this --that --yes --quiet --prefix $PREFIX $ENV") >> null result == PREFIX @@ -357,7 +390,7 @@ class CondaCacheTest extends Specification { def result = cache.createLocalCondaEnv0(ENV, PREFIX) then: 1 * cache.isYamlFilePath(ENV) - 1 * cache.isTextFilePath(ENV) + 1 * cache.isExplicitFile(ENV) 0 * cache.makeAbsolute(_) 1 * cache.runCommand("micromamba create --this --that --yes --quiet --prefix $PREFIX $ENV") >> null result == PREFIX @@ -374,7 +407,7 @@ class CondaCacheTest extends Specification { def result = cache.createLocalCondaEnv0(ENV, PREFIX) then: 1 * cache.isYamlFilePath(ENV) - 1 * cache.isTextFilePath(ENV) + 1 * cache.isExplicitFile(ENV) 0 * cache.makeAbsolute(_) 1 * cache.runCommand("conda create --yes --quiet --prefix /foo/bar -c bioconda -c defaults bwa=1.1.1") >> null result == PREFIX @@ -391,7 +424,7 @@ class CondaCacheTest extends Specification { def result = cache.createLocalCondaEnv0(ENV, PREFIX) then: 1 * cache.isYamlFilePath(ENV) - 0 * cache.isTextFilePath(ENV) + 0 * cache.isExplicitFile(ENV) 1 * cache.makeAbsolute(ENV) >> Paths.get('/usr/base').resolve(ENV) 1 * cache.runCommand( "conda env create --prefix $PREFIX --file /usr/base/foo.yml" ) >> null result == PREFIX @@ -409,17 +442,29 @@ class CondaCacheTest extends Specification { def result = cache.createLocalCondaEnv0(ENV, PREFIX) then: 1 * cache.isYamlFilePath(ENV) - 0 * cache.isTextFilePath(ENV) + 0 * cache.isExplicitFile(ENV) 1 * cache.makeAbsolute(ENV) >> Paths.get('/usr/base').resolve(ENV) 1 * cache.runCommand( "micromamba env create --yes --prefix $PREFIX --file /usr/base/foo.yml" ) >> null result == PREFIX } - def 'should create a conda env with a text file' () { + def 'should create a conda env with an explicit file' () { given: - def ENV = 'foo.txt' + def folder = Files.createTempDirectory('test') + def envFile = folder.resolve('explicit.txt') + envFile.text = '''\ + # This file may be used to create an environment using: + # $ conda create --name --file + # platform: linux-64 + @EXPLICIT + https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2#d7c89558ba9fa0495403155b64376d81 + https://conda.anaconda.org/conda-forge/linux-64/libgomp-15.2.0-h767d61c_7.conda#f7b4d76975aac7e5d9e6ad13845f92fe + https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2#73aaf86a425cc6e73fcf236a5a46396d + https://conda.anaconda.org/conda-forge/linux-64/libgcc-15.2.0-h767d61c_7.conda#c0374badb3a5d4b1372db28d19462c53 + '''.stripIndent() + def ENV = envFile.toString() def PREFIX = Paths.get('/conda/envs/my-env') and: def cache = Spy(new CondaCache(createOptions: '--this --that')) @@ -428,17 +473,30 @@ class CondaCacheTest extends Specification { def result = cache.createLocalCondaEnv0(ENV, PREFIX) then: 1 * cache.isYamlFilePath(ENV) - 1 * cache.isTextFilePath(ENV) - 1 * cache.makeAbsolute(ENV) >> Paths.get('/usr/base').resolve(ENV) - 1 * cache.runCommand( "conda create --this --that --yes --quiet --prefix $PREFIX --file /usr/base/foo.txt" ) >> null + 1 * cache.isExplicitFile(ENV) + 1 * cache.makeAbsolute(ENV) >> envFile.toAbsolutePath() + 1 * cache.runCommand( "conda create --this --that --yes --quiet --prefix $PREFIX --file ${envFile.toAbsolutePath()}" ) >> null result == PREFIX + cleanup: + folder?.deleteDir() } - def 'should create a conda env with a text file - using micromamba' () { + def 'should create a conda env with an explicit file - using micromamba' () { given: - def ENV = 'foo.txt' + def folder = Files.createTempDirectory('test') + def envFile = folder.resolve('explicit.txt') + envFile.text = '''\ + # This file may be used to create an environment using: + # $ conda create --name --file + # platform: linux-64 + @EXPLICIT + https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2#d7c89558ba9fa0495403155b64376d81 + https://conda.anaconda.org/conda-forge/linux-64/libgomp-15.2.0-h767d61c_7.conda#f7b4d76975aac7e5d9e6ad13845f92fe + https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2#73aaf86a425cc6e73fcf236a5a46396d + '''.stripIndent() + def ENV = envFile.toString() def PREFIX = Paths.get('/conda/envs/my-env') and: def cache = Spy(new CondaCache(useMicromamba: true, createOptions: '--this --that')) @@ -447,11 +505,13 @@ class CondaCacheTest extends Specification { def result = cache.createLocalCondaEnv0(ENV, PREFIX) then: 1 * cache.isYamlFilePath(ENV) - 1 * cache.isTextFilePath(ENV) - 1 * cache.makeAbsolute(ENV) >> Paths.get('/usr/base').resolve(ENV) - 1 * cache.runCommand( "micromamba create --this --that --yes --quiet --prefix $PREFIX --file /usr/base/foo.txt" ) >> null + 1 * cache.isExplicitFile(ENV) + 1 * cache.makeAbsolute(ENV) >> envFile.toAbsolutePath() + 1 * cache.runCommand( "micromamba create --this --that --yes --quiet --prefix $PREFIX --file ${envFile.toAbsolutePath()}" ) >> null result == PREFIX + cleanup: + folder?.deleteDir() } def 'should get options from the config' () {