Fix incorrect AWS region when specifying a S3 regional endpoint (#6530)

Signed-off-by: jorgee <[email protected]>
Signed-off-by: Jorge Ejarque <[email protected]>
Co-authored-by: Paolo Di Tommaso <[email protected]>

Author: jorgee

plugins/nf-amazon/src/main/nextflow/cloud/aws/config/AwsConfig.groovy

@@ -93,12 +93,29 @@ class AwsConfig implements ConfigScope {
 
     AwsBatchConfig getBatchConfig() { batch }
 
+    @Deprecated
     String getS3GlobalRegion() {
         return !region || !s3Config.endpoint || s3Config.endpoint.contains(".amazonaws.com")
             ? Region.US_EAST_1.id()         // always use US_EAST_1 as global region for AWS endpoints
             : region                        // for custom endpoint use the config provided region
     }
 
+    /**
+     *  Resolves the region used for S3 evaluating the region resolved from config and a possible region defined in the endpoint.
+     *  Fallback to the global region US_EAST_1 when no region is found.
+     *
+     *  Preference:
+     *      1. endpoint region
+     *      2. config region
+     *      3. US_EAST_1
+     *
+     *  @returns Resolved region.
+     **/
+    String resolveS3Region() {
+        final epRegion = client.getEndpointRegion()
+        return epRegion ?: this.region ?: Region.US_EAST_1.id()
+    }
+
     static protected String getAwsProfile0(Map env, Map<String,Object> config) {
 
         final profile = config?.profile as String

plugins/nf-amazon/src/main/nextflow/cloud/aws/config/AwsS3Config.groovy

@@ -19,6 +19,7 @@ package nextflow.cloud.aws.config
 
 import static nextflow.cloud.aws.util.AwsHelper.*
 
+import software.amazon.awssdk.regions.Region
 import software.amazon.awssdk.services.s3.model.ObjectCannedACL
 import groovy.transform.CompileStatic
 import groovy.util.logging.Slf4j
@@ -288,6 +289,35 @@ class AwsS3Config implements ConfigScope {
         endpoint && !endpoint.endsWith(".amazonaws.com")
     }
 
+    /**
+     * Looks for the region defined in endpoints such as https://xxx.<region>.amazonaws.com
+     * @returns Region defined in the endpoint. Null if no endpoint or custom endpoint is defined,
+     * or when URI region subdomain doesn't match with a region (global or multi-region access point)
+     */
+    String getEndpointRegion(){
+        if( !endpoint || isCustomEndpoint() )
+            return null
+
+        try {
+            String host = URI.create(endpoint).getHost()
+            final hostDomains = host.split('\\.')
+            if (hostDomains.size() < 3) {
+                log.debug("Region subdomain doesn't exist in endpoint '${endpoint}'")
+                return null
+            }
+            final region = hostDomains[hostDomains.size()-3]
+            if (!Region.regions().contains(Region.of(region))){
+                log.debug("Region '${region}' extracted from endpoint '${endpoint}' is not valid")
+                return null
+            }
+            return region
+
+        } catch (Exception e){
+            log.debug("Exception getting region from endpoint: '${endpoint}' - ${e.message}")
+            return null
+        }
+    }
+
     Map<String,String> getAwsClientConfig() {
         return [
             connection_timeout: connectionTimeout?.toString(),

plugins/nf-amazon/src/main/nextflow/cloud/aws/nio/S3FileSystemProvider.java

@@ -740,7 +740,7 @@ protected S3FileSystem createFileSystem(URI uri, AwsConfig awsConfig) {
         // when enabling that flag, it overrides S3 endpoints with AWS global endpoint
         // see https://github.com/nextflow-io/nextflow/pull/5779
 		final boolean global = bucketName!=null && !awsConfig.getS3Config().isCustomEndpoint();
-		final AwsClientFactory factory = new AwsClientFactory(awsConfig, awsConfig.getS3GlobalRegion());
+		final AwsClientFactory factory = new AwsClientFactory(awsConfig, awsConfig.resolveS3Region());
 		final S3Client client = new S3Client(factory, props, global);
 
 		// set the client acl

plugins/nf-amazon/src/test/nextflow/cloud/aws/config/AwsConfigTest.groovy

@@ -17,6 +17,8 @@
 
 package nextflow.cloud.aws.config
 
+import software.amazon.awssdk.regions.Region
+
 import java.nio.file.Files
 
 import nextflow.SysEnv
@@ -115,4 +117,18 @@ class AwsConfigTest extends Specification {
         [max_error_retry: '2', foo: 1]  | [:]                   | [max_error_retry: '2', foo: 1]
         [:]                             | [:]                   | [max_error_retry: '5']
     }
+
+    @Unroll
+    def 'should resolve S3 region' () {
+        expect:
+        new AwsConfig(CONFIG).resolveS3Region() == REGION
+
+        where:
+        CONFIG                                                                                  | REGION
+        [:]                                                                                     | Region.US_EAST_1.id()
+        [client: [endpoint: "http://custom.endpoint.com"]]                                      | Region.US_EAST_1.id()
+        [region: "eu-south-1", client: [endpoint: "http://custom.endpoint.com"]]                | Region.EU_SOUTH_1.id()
+        [region: "eu-south-1", client: [endpoint: "https://s3.eu-west-1.amazonaws.com"]]        | Region.EU_WEST_1.id()
+        [region: "eu-south-1", client: [endpoint: "https://bucket.s3-global.amazonaws.com"]]    | Region.EU_SOUTH_1.id()
+    }
 }

plugins/nf-amazon/src/test/nextflow/cloud/aws/config/AwsS3ConfigTest.groovy

@@ -17,6 +17,7 @@
 
 package nextflow.cloud.aws.config
 
+import software.amazon.awssdk.regions.Region
 import software.amazon.awssdk.services.s3.model.ObjectCannedACL
 import nextflow.SysEnv
 import spock.lang.Specification
@@ -189,4 +190,17 @@ class AwsS3ConfigTest extends Specification {
         [ maxDownloadHeapMemory: '50 MB', minimumPartSize: '6 MB']  | "Configuration option `aws.client.maxDownloadHeapMemory` must be at least 10 times `aws.client.minimumPartSize`"
     }
 
+    @Unroll
+    def 'should get region from endpoint' () {
+        expect:
+        new AwsS3Config(CONFIG).getEndpointRegion() == REGION
+
+        where:
+        CONFIG                                                                                  | REGION
+        [:]                                                                                     | null
+        [endpoint: "http://custom.endpoint.com"]                                                | null
+        [endpoint: "https://s3.eu-west-1.amazonaws.com"]                                        | Region.EU_WEST_1.id()
+        [endpoint: "https://bucket.s3-global.amazonaws.com"]                                    | null
+    }
+
 }