Skip to content

Commit 4583a3d

Browse files
alberto-mirandaalberto-miranda
committed
feat(wave): Support fetching local files in FUSION_CONTAINER_CONFIG_URL
This change allows `FUSION_CONTAINER_CONFIG_URL` to accept a local filesystem path (an absolute path or a `file://` URI) in addition to an http(s) URL. This lets engineers iterate on Fusion manifests without publishing them to a reachable URL on every change. Signed-off-by: Alberto Miranda <alberto.miranda@seqera.io>
1 parent 5e2ef57 commit 4583a3d

4 files changed

Lines changed: 95 additions & 16 deletions

File tree

‎modules/nextflow/src/main/groovy/nextflow/fusion/FusionConfig.groovy‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -132,8 +132,8 @@ class FusionConfig implements ConfigScope {
132132

133133
boolean snapshotsEnabled() { snapshots }
134134

135-
URL containerConfigUrl() {
136-
this.containerConfigUrl ? new URL(this.containerConfigUrl) : null
135+
URI containerConfigURI() {
136+
containerConfigUrl ? new URI(containerConfigUrl) : null
137137
}
138138

139139
boolean privileged() {
@@ -156,7 +156,7 @@ class FusionConfig implements ConfigScope {
156156
this.targetVersion = opts.targetVersion as String
157157

158158
if( containerConfigUrl && !validProtocol(containerConfigUrl))
159-
throw new IllegalArgumentException("Fusion container config URL should start with 'http:' or 'https:' protocol prefix - offending value: $containerConfigUrl")
159+
throw new IllegalArgumentException("Fusion container config URL should be an http(s) URL, a file: URI, or an absolute file path - offending value: $containerConfigUrl")
160160
}
161161

162162
static private String parseTags(Object value) {
@@ -170,7 +170,7 @@ class FusionConfig implements ConfigScope {
170170
}
171171

172172
protected boolean validProtocol(String url) {
173-
url.startsWith('http://') || url.startsWith('https://') || url.startsWith('file:/')
173+
url.startsWith('http://') || url.startsWith('https://') || url.startsWith('file:') || url.startsWith('/')
174174
}
175175

176176
static FusionConfig getConfig() {

‎modules/nextflow/src/test/groovy/nextflow/fusion/FusionConfigTest.groovy‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,23 +40,35 @@ class FusionConfigTest extends Specification {
4040
}
4141

4242
@Unroll
43-
def 'should create container config url' () {
43+
def 'should create container config uri' () {
4444
when:
4545
def opts = new FusionConfig(OPTS, ENV)
4646
then:
47-
opts.containerConfigUrl() == (EXPECTED ? new URL(EXPECTED) : null)
47+
opts.containerConfigURI() == (EXPECTED ? new URI(EXPECTED) : null)
4848

4949
where:
5050
OPTS | ENV | EXPECTED
5151
[:] | [:] | null
5252
[containerConfigUrl:'http://foo.com'] | [:] | 'http://foo.com'
5353
[containerConfigUrl:'https://bar.com'] | [:] | 'https://bar.com'
5454
[containerConfigUrl:'file:///some/file']| [:] | 'file:///some/file'
55+
[containerConfigUrl:'/some/file'] | [:] | '/some/file'
5556
[:] | [FUSION_CONTAINER_CONFIG_URL:'http://bar.com'] | 'http://bar.com'
5657
[containerConfigUrl:'http://foo.com'] | [FUSION_CONTAINER_CONFIG_URL:'http://bar.com'] | 'http://foo.com'
5758

5859
}
5960

61+
@Unroll
62+
def 'should reject invalid container config url' () {
63+
when:
64+
new FusionConfig([containerConfigUrl: VALUE])
65+
then:
66+
thrown(IllegalArgumentException)
67+
68+
where:
69+
VALUE << ['ftp://foo.com/x.json', 'relative/path.json', 'gs://bucket/x.json']
70+
}
71+
6072
@Unroll
6173
def 'should get export aws key' () {
6274
expect:

‎plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveClient.groovy‎

Lines changed: 31 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import java.net.http.HttpClient
2222
import java.net.http.HttpRequest
2323
import java.net.http.HttpResponse
2424
import java.nio.file.Path
25+
import java.nio.file.Paths
2526
import java.time.Duration
2627
import java.time.Instant
2728
import java.time.OffsetDateTime
@@ -403,37 +404,57 @@ class WaveClient {
403404
: new URL(DEFAULT_S5CMD_AMD64_URL)
404405
}
405406

406-
protected static URL replaceFusionArch(URL url, String platform) {
407+
protected static URI replaceFusionArch(URI uri, String platform) {
407408
final isArm = platform.tokenize('/')?.contains('arm64')
408409
final targetArch = isArm ? 'arm64' : 'amd64'
409-
final replaced = url.toString().replaceAll(/(?<=[-_])(amd64|arm64)(?=\.)/, targetArch)
410-
return replaced != url.toString() ? new URL(replaced) : url
410+
final original = uri.toString()
411+
final replaced = original.replaceAll(/(?<=[-_])(amd64|arm64)(?=\.)/, targetArch)
412+
return replaced != original ? new URI(replaced) : uri
411413
}
412414

413415
ContainerConfig resolveContainerConfig(String platform = DEFAULT_DOCKER_PLATFORM) {
414-
final urls = new ArrayList<URL>(config.containerConfigUrl())
416+
final uris = new ArrayList<URI>(config.containerConfigUrl().collect { it.toURI() })
415417
final platforms = platform ? platform.tokenize(',') : List.of(DEFAULT_DOCKER_PLATFORM)
416418
if( fusion.enabled() ) {
417-
final customUrl = fusion.containerConfigUrl()
419+
final customUri = fusion.containerConfigURI()
418420
for( String p : platforms ) {
419-
final fusionUrl = customUrl ? replaceFusionArch(customUrl, p.trim()) : defaultFusionUrl(p.trim())
420-
urls.add(fusionUrl)
421+
final fusionUri = customUri ? replaceFusionArch(customUri, p.trim()) : defaultFusionUrl(p.trim()).toURI()
422+
uris.add(fusionUri)
421423
}
422424
}
423425
if( awsFargate ) {
424426
final s5cmdUrl = s5cmdConfigUrl ?: defaultS5cmdUrl(platform)
425-
urls.add(s5cmdUrl)
427+
uris.add(s5cmdUrl.toURI())
426428
}
427-
if( !urls )
429+
if( !uris )
428430
return null
429431
def result = new ContainerConfig()
430-
for( URL it : urls ) {
432+
for( URI it : uris ) {
431433
// append each config to the other - the last has priority
432434
result += fetchContainerConfig(it)
433435
}
434436
return result
435437
}
436438

439+
@Memoized
440+
synchronized protected ContainerConfig fetchContainerConfig(URI configURI) {
441+
log.debug "Wave fetch container config: $configURI"
442+
final scheme = configURI.scheme
443+
if( scheme == 'http' || scheme == 'https' )
444+
return fetchContainerConfig(configURI.toURL())
445+
if( scheme == 'file' )
446+
return fetchContainerConfig(Paths.get(configURI))
447+
if( scheme == null )
448+
return fetchContainerConfig(Paths.get(configURI.path ?: configURI.toString()))
449+
throw new IllegalArgumentException("Unsupported container config URI scheme: $scheme")
450+
}
451+
452+
@Memoized
453+
synchronized protected ContainerConfig fetchContainerConfig(Path configPath) {
454+
log.debug "Wave read local container config: $configPath"
455+
return jsonToContainerConfig(configPath.text)
456+
}
457+
437458
@Memoized
438459
synchronized protected ContainerConfig fetchContainerConfig(URL configUrl) {
439460
log.debug "Wave request container config: $configUrl"

‎plugins/nf-wave/src/test/io/seqera/wave/plugin/WaveClientTest.groovy‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -954,6 +954,52 @@ class WaveClientTest extends Specification {
954954
| CONFIG_RESP.get('combined')
955955
}
956956

957+
def 'should resolve container config from local file' () {
958+
given:
959+
def folder = Files.createTempDirectory('wave-local-manifest')
960+
def manifest = folder.resolve('manifest.json')
961+
manifest.text = JsonOutput.toJson([entrypoint: ['entry.sh']])
962+
and:
963+
def session = Mock(Session) { getConfig() >> [fusion: [enabled: true, containerConfigUrl: CONFIG_VALUE.call(manifest)]] }
964+
def client = new WaveClient(session)
965+
966+
expect:
967+
client.resolveContainerConfig() == new ContainerConfig(entrypoint: ['entry.sh'])
968+
969+
cleanup:
970+
folder?.deleteDir()
971+
972+
where:
973+
_ | CONFIG_VALUE
974+
_ | { Path p -> p.toUri().toString() } // file:// URI
975+
_ | { Path p -> p.toAbsolutePath().toString() } // bare absolute path
976+
}
977+
978+
def 'should reject unsupported scheme for container config' () {
979+
given:
980+
def session = Mock(Session) { getConfig() >> [:] }
981+
def client = new WaveClient(session)
982+
983+
when:
984+
client.fetchContainerConfig(new URI('ftp://example.com/manifest.json'))
985+
986+
then:
987+
thrown(IllegalArgumentException)
988+
}
989+
990+
@Unroll
991+
def 'should replace fusion arch in URI' () {
992+
expect:
993+
WaveClient.replaceFusionArch(new URI(URI_VALUE), PLATFORM).toString() == EXPECTED
994+
995+
where:
996+
URI_VALUE | PLATFORM | EXPECTED
997+
'https://fusionfs.seqera.io/releases/v2.5-amd64.json' | 'linux/arm64' | 'https://fusionfs.seqera.io/releases/v2.5-arm64.json'
998+
'https://fusionfs.seqera.io/releases/v2.5-arm64.json' | 'linux/amd64' | 'https://fusionfs.seqera.io/releases/v2.5-amd64.json'
999+
'file:///tmp/local-manifest.json' | 'linux/arm64' | 'file:///tmp/local-manifest.json'
1000+
'/tmp/local-manifest.json' | 'linux/arm64' | '/tmp/local-manifest.json'
1001+
}
1002+
9571003
@Unroll
9581004
def 'should get fusion default url' () {
9591005
given:

0 commit comments

Comments
 (0)