From 85c3f1505bd9ee42a7b04d65e30f513b6869276e Mon Sep 17 00:00:00 2001 From: Eran Rosenberg Date: Sun, 30 Nov 2025 12:19:44 -0500 Subject: [PATCH 1/8] Updated checksum check w/ SDK V2 new method instead of deprecated one --- src/hpc-server/hpc-integration-impl/pom.xml | 9 +- .../s3/v2/impl/HpcS3Connection.java | 592 +++++++++--------- .../hpc-integration-beans-configuration.xml | 4 +- src/hpc-server/pom.xml | 6 - 4 files changed, 288 insertions(+), 323 deletions(-) diff --git a/src/hpc-server/hpc-integration-impl/pom.xml b/src/hpc-server/hpc-integration-impl/pom.xml index 75c631d798..5756ddb193 100644 --- a/src/hpc-server/hpc-integration-impl/pom.xml +++ b/src/hpc-server/hpc-integration-impl/pom.xml @@ -29,6 +29,11 @@ hpc-integration-api ${project.version} + + ${project.groupId} + hpc-domain-model + ${project.version} + org.slf4j slf4j-api @@ -125,10 +130,6 @@ software.amazon.awssdk aws-core - - software.amazon.awssdk.crt - aws-crt - com.google.apis google-api-services-drive diff --git a/src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/s3/v2/impl/HpcS3Connection.java b/src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/s3/v2/impl/HpcS3Connection.java index 7eefae16c7..7888cdf5b9 100644 --- a/src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/s3/v2/impl/HpcS3Connection.java +++ b/src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/s3/v2/impl/HpcS3Connection.java @@ -10,335 +10,305 @@ */ package gov.nih.nci.hpc.integration.s3.v2.impl; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.HashSet; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Value; - import gov.nih.nci.hpc.domain.datatransfer.HpcS3Account; import gov.nih.nci.hpc.domain.error.HpcErrorType; import gov.nih.nci.hpc.domain.user.HpcIntegratedSystem; import gov.nih.nci.hpc.domain.user.HpcIntegratedSystemAccount; import gov.nih.nci.hpc.exception.HpcException; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.core.exception.SdkException; -import software.amazon.awssdk.crt.CrtRuntimeException; -import software.amazon.awssdk.crt.Log; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.S3Configuration; -import software.amazon.awssdk.services.s3.S3CrtAsyncClientBuilder; import software.amazon.awssdk.services.s3.presigner.S3Presigner; import software.amazon.awssdk.transfer.s3.S3TransferManager; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + /** * HPC S3 Connection. * * @author Eran Rosenberg */ public class HpcS3Connection { - // ---------------------------------------------------------------------// - // Constants - // ---------------------------------------------------------------------// - - // 5GB in bytes - private static final long FIVE_GB = 5368709120L; - - // Google Storage S3 URL. - private static final String GOOGLE_STORAGE_URL = "https://storage.googleapis.com"; - - // ---------------------------------------------------------------------// - // Instance members - // ---------------------------------------------------------------------// - - // A list of S3 3rd party providers that require connection w/ path-style - // enabled. - private Set pathStyleAccessEnabledProviders = new HashSet<>(); - - // The multipart upload minimum part size. - @Value("${hpc.integration.s3.minimumUploadPartSize}") - private Long minimumUploadPartSize = null; - - // The multipart upload threshold. - @Value("${hpc.integration.s3.multipartUploadThreshold}") - private Long multipartUploadThreshold = null; - - // The CRT log file (Optional). - @Value("${hpc.integration.s3.crtLogFile:#{null}}") - private String crtLogFile = null; - - // AWS CRT to trust all certs config. - @Value("${hpc.integration.s3.trustAllCerts:false}") - private Boolean trustAllCerts = null; - - // The executor service to be used by AWSTransferManager - private ExecutorService executorService = null; - - // The logger instance. - private final Logger logger = LoggerFactory.getLogger(getClass().getName()); - - // ---------------------------------------------------------------------// - // Constructors - // ---------------------------------------------------------------------// - - /** - * Constructor for Spring Dependency Injection. - * - * @param pathStyleAccessEnabledProviders A list of S3 3rd party providers that - * require connection w/ path-style - * enabled. - * @param awsTransferManagerThreadPoolSize The thread pool size to be used for - * AWS transfer manager - */ - private HpcS3Connection(String pathStyleAccessEnabledProviders, int awsTransferManagerThreadPoolSize) { - for (String s3Provider : pathStyleAccessEnabledProviders.split(",")) { - this.pathStyleAccessEnabledProviders.add(HpcIntegratedSystem.fromValue(s3Provider)); - } - - // Instantiate the executor service for AWS transfer manager. - executorService = Executors.newFixedThreadPool(awsTransferManagerThreadPoolSize, - Executors.defaultThreadFactory()); - } - - // ---------------------------------------------------------------------// - // Methods - // ---------------------------------------------------------------------// - - /** - * Authenticate a (system) data transfer account to S3 (AWS or 3rd Party - * Provider) - * - * @param dataTransferAccount A data transfer account to authenticate. - * @param s3URLorRegion The S3 URL if authenticating with a 3rd party S3 - * Provider (Cleversafe, Cloudian, etc), or Region if - * authenticating w/ AWS. - * @return An authenticated TransferManager object, or null if authentication - * failed. - * @throws HpcException if authentication failed - */ - public Object authenticate(HpcIntegratedSystemAccount dataTransferAccount, String s3URLorRegion) - throws HpcException { - if (dataTransferAccount.getIntegratedSystem().equals(HpcIntegratedSystem.AWS)) { - return authenticateAWS(dataTransferAccount.getUsername(), dataTransferAccount.getPassword(), s3URLorRegion); - } else { - // Determine if this S3 provider require path-style enabled. - boolean pathStyleAccessEnabled = pathStyleAccessEnabledProviders - .contains(dataTransferAccount.getIntegratedSystem()); - - return authenticateS3Provider(dataTransferAccount.getUsername(), dataTransferAccount.getPassword(), - s3URLorRegion, pathStyleAccessEnabled, dataTransferAccount.getIntegratedSystem()); - } - } - - /** - * Authenticate a (user) S3 account (AWS or 3rd Party Provider) - * - * @param s3Account AWS S3 account. - * @return TransferManager - * @throws HpcException if authentication failed - */ - public Object authenticate(HpcS3Account s3Account) throws HpcException { - if (!StringUtils.isEmpty(s3Account.getRegion())) { - return authenticateAWS(s3Account.getAccessKey(), s3Account.getSecretKey(), s3Account.getRegion()); - - } else { - // Default S3 provider require path-style enabled to true if not provided by the - // user. - boolean pathStyleAccessEnabled = Optional.ofNullable(s3Account.getPathStyleAccessEnabled()).orElse(true); - - return authenticateS3Provider(s3Account.getAccessKey(), s3Account.getSecretKey(), s3Account.getUrl(), - pathStyleAccessEnabled, HpcIntegratedSystem.USER_S_3_PROVIDER); - } - } - - /** - * Get S3 Transfer Manager from an authenticated token. - * - * @param authenticatedToken An authenticated token. - * @return A transfer manager object. - * @throws HpcException on invalid authentication token. - */ - public S3TransferManager getTransferManager(Object authenticatedToken) throws HpcException { - if (!(authenticatedToken instanceof HpcS3)) { - throw new HpcException("Invalid S3 authentication token", HpcErrorType.INVALID_REQUEST_INPUT); - } - - return ((HpcS3) authenticatedToken).transferManager; - } - - /** - * Get S3 Client from an authenticated token. - * - * @param authenticatedToken An authenticated token. - * @return A S3 client object. - * @throws HpcException on invalid authentication token. - */ - public S3AsyncClient getClient(Object authenticatedToken) throws HpcException { - if (!(authenticatedToken instanceof HpcS3)) { - throw new HpcException("Invalid S3 authentication token", HpcErrorType.INVALID_REQUEST_INPUT); - } - - return ((HpcS3) authenticatedToken).client; - } - - /** - * Get S3 Presigner from an authenticated token. - * - * @param authenticatedToken An authenticated token. - * @return A S3 presigner object. - * @throws HpcException on invalid authentication token. - */ - public S3Presigner getPresigner(Object authenticatedToken) throws HpcException { - if (!(authenticatedToken instanceof HpcS3)) { - throw new HpcException("Invalid S3 authentication token", HpcErrorType.INVALID_REQUEST_INPUT); - } - - return ((HpcS3) authenticatedToken).presigner; - } - - /** - * Get S3 Provider from an authenticated token. - * - * @param authenticatedToken An authenticated token. - * @return A transfer manager object. - * @throws HpcException on invalid authentication token. - */ - public HpcIntegratedSystem getS3Provider(Object authenticatedToken) throws HpcException { - if (!(authenticatedToken instanceof HpcS3)) { - return null; - } - - return ((HpcS3) authenticatedToken).provider; - } - - // ---------------------------------------------------------------------// - // Helper Methods - // ---------------------------------------------------------------------// - - private class HpcS3 { - private S3TransferManager transferManager = null; - private S3AsyncClient client = null; - private S3Presigner presigner = null; - private HpcIntegratedSystem provider = null; - } - - /** - * Authenticate a 'S3 3rd Party Provider' account. - * - * @param username The S3 account user name. - * @param password The S3 account password. - * @param url The S3 3rd party provider URL. - * @param pathStyleAccessEnabled true if the S3 3rd Party provider supports path - * style access. - * @param s3Provider The 3rd party provider. - * @return HpcS3 instance - * @throws HpcException if authentication failed - */ - private Object authenticateS3Provider(String username, String password, String url, boolean pathStyleAccessEnabled, - HpcIntegratedSystem s3Provider) throws HpcException { - // Create the credential provider based on the configured credentials. - AwsBasicCredentials s3ProviderCredentials = AwsBasicCredentials.create(username, password); - StaticCredentialsProvider s3ProviderCredentialsProvider = StaticCredentialsProvider - .create(s3ProviderCredentials); - - // Create URI to the S3 provider endpoint - URI uri = null; - try { - uri = new URI(url); - - } catch (URISyntaxException e) { - throw new HpcException("Invalid URL: " + url, HpcErrorType.DATA_TRANSFER_ERROR, e); - } - - HpcS3 s3 = new HpcS3(); - s3.provider = s3Provider; - - try { - // If configured, start the AWS CRT logger. - if (!StringUtils.isEmpty(crtLogFile)) { - Log.initLoggingToFile(Log.LogLevel.Trace, crtLogFile); - } - - // Instantiate a S3 async client. - S3CrtAsyncClientBuilder crtAsyncClientBuilder = S3AsyncClient.crtBuilder() - .credentialsProvider(s3ProviderCredentialsProvider).forcePathStyle(pathStyleAccessEnabled) - .endpointOverride(uri).minimumPartSizeInBytes(minimumUploadPartSize) - .thresholdInBytes(url.equalsIgnoreCase(GOOGLE_STORAGE_URL) ? FIVE_GB : multipartUploadThreshold).maxConcurrency(1); - - if (trustAllCerts) { - crtAsyncClientBuilder.httpConfiguration(builder -> builder.trustAllCertificatesEnabled(true)); - logger.warn("hpc.integration.s3.trustAllCerts property is set to true. CRT cert validation is off"); - } - - s3.client = crtAsyncClientBuilder.build(); - - // Instantiate the S3 transfer manager. - s3.transferManager = S3TransferManager.builder().s3Client(s3.client).executor(executorService).build(); - - // Instantiate the S3 presigner. - s3.presigner = S3Presigner.builder().credentialsProvider(s3ProviderCredentialsProvider) - .endpointOverride(uri).serviceConfiguration( - S3Configuration.builder().pathStyleAccessEnabled(pathStyleAccessEnabled).build()) - .build(); - - return s3; - - } catch (SdkException | CrtRuntimeException e) { - throw new HpcException( - "[S3] Failed to authenticate S3 Provider: " + s3Provider.value() + "] - " + e.getMessage(), - HpcErrorType.DATA_TRANSFER_ERROR, e); - } - } - - /** - * Authenticate an AWS S3 account. - * - * @param accessKey The AWS account access key. - * @param secretKey The AWS account secret key. - * @param region The AWS account region. - * @return TransferManager - * @throws HpcException if authentication failed - */ - private Object authenticateAWS(String accessKey, String secretKey, String region) throws HpcException { - // Create the credential provider based on provided AWS S3 account. - AwsBasicCredentials awsCredentials = AwsBasicCredentials.create(accessKey, secretKey); - StaticCredentialsProvider awsCredentialsProvider = StaticCredentialsProvider.create(awsCredentials); - - HpcS3 s3 = new HpcS3(); - s3.provider = HpcIntegratedSystem.AWS; - - try { - // If configured, start the AWS CRT logger. - if (!StringUtils.isEmpty(crtLogFile)) { - Log.initLoggingToFile(Log.LogLevel.Trace, crtLogFile); - } - - // Instantiate a S3 async client. - s3.client = S3AsyncClient.crtBuilder().credentialsProvider(awsCredentialsProvider).region(Region.of(region)) - .minimumPartSizeInBytes(minimumUploadPartSize) - .thresholdInBytes(multipartUploadThreshold).maxConcurrency(1).build(); - - // Instantiate the S3 transfer manager. - s3.transferManager = S3TransferManager.builder().s3Client(s3.client).executor(executorService).build(); - - // Instantiate the S3 presigner. - s3.presigner = S3Presigner.builder().credentialsProvider(awsCredentialsProvider).region(Region.of(region)) - .build(); - - return s3; - - } catch (SdkException | CrtRuntimeException e) { - throw new HpcException("[S3] Failed to authenticate S3 in region " + region + "] - " + e.getMessage(), - HpcErrorType.DATA_TRANSFER_ERROR, e); - } - } + // ---------------------------------------------------------------------// + // Constants + // ---------------------------------------------------------------------// + + // 5GB in bytes + private static final long FIVE_GB = 5368709120L; + + // Google Storage S3 URL. + private static final String GOOGLE_STORAGE_URL = "https://storage.googleapis.com"; + + // ---------------------------------------------------------------------// + // Instance members + // ---------------------------------------------------------------------// + + // A list of S3 3rd party providers that require connection w/ path-style + // enabled. + private Set pathStyleAccessEnabledProviders = new HashSet<>(); + + // The multipart upload minimum part size. + @Value("${hpc.integration.s3.minimumUploadPartSize}") + private Long minimumUploadPartSize = null; + + // The multipart upload threshold. + @Value("${hpc.integration.s3.multipartUploadThreshold}") + private Long multipartUploadThreshold = null; + + // The executor service to be used by AWSTransferManager + private ExecutorService executorService = null; + + // The logger instance. + private final Logger logger = LoggerFactory.getLogger(getClass().getName()); + + // ---------------------------------------------------------------------// + // Constructors + // ---------------------------------------------------------------------// + + /** + * Constructor for Spring Dependency Injection. + * + * @param pathStyleAccessEnabledProviders A list of S3 3rd party providers that + * require connection w/ path-style + * enabled. + * @param awsTransferManagerThreadPoolSize The thread pool size to be used for + * AWS transfer manager + */ + private HpcS3Connection(String pathStyleAccessEnabledProviders, int awsTransferManagerThreadPoolSize) { + for (String s3Provider : pathStyleAccessEnabledProviders.split(",")) { + this.pathStyleAccessEnabledProviders.add(HpcIntegratedSystem.fromValue(s3Provider)); + } + + // Instantiate the executor service for AWS transfer manager. + executorService = Executors.newFixedThreadPool(awsTransferManagerThreadPoolSize, + Executors.defaultThreadFactory()); + } + + // ---------------------------------------------------------------------// + // Methods + // ---------------------------------------------------------------------// + + /** + * Authenticate a (system) data transfer account to S3 (AWS or 3rd Party + * Provider) + * + * @param dataTransferAccount A data transfer account to authenticate. + * @param s3URLorRegion The S3 URL if authenticating with a 3rd party S3 + * Provider (Cleversafe, Cloudian, etc), or Region if + * authenticating w/ AWS. + * @return An authenticated TransferManager object, or null if authentication + * failed. + * @throws HpcException if authentication failed + */ + public Object authenticate(HpcIntegratedSystemAccount dataTransferAccount, String s3URLorRegion) + throws HpcException { + if (dataTransferAccount.getIntegratedSystem().equals(HpcIntegratedSystem.AWS)) { + return authenticateAWS(dataTransferAccount.getUsername(), dataTransferAccount.getPassword(), s3URLorRegion); + } else { + // Determine if this S3 provider require path-style enabled. + boolean pathStyleAccessEnabled = pathStyleAccessEnabledProviders + .contains(dataTransferAccount.getIntegratedSystem()); + + return authenticateS3Provider(dataTransferAccount.getUsername(), dataTransferAccount.getPassword(), + s3URLorRegion, pathStyleAccessEnabled, dataTransferAccount.getIntegratedSystem()); + } + } + + /** + * Authenticate a (user) S3 account (AWS or 3rd Party Provider) + * + * @param s3Account AWS S3 account. + * @return TransferManager + * @throws HpcException if authentication failed + */ + public Object authenticate(HpcS3Account s3Account) throws HpcException { + if (!StringUtils.isEmpty(s3Account.getRegion())) { + return authenticateAWS(s3Account.getAccessKey(), s3Account.getSecretKey(), s3Account.getRegion()); + + } else { + // Default S3 provider require path-style enabled to true if not provided by the + // user. + boolean pathStyleAccessEnabled = Optional.ofNullable(s3Account.getPathStyleAccessEnabled()).orElse(true); + + return authenticateS3Provider(s3Account.getAccessKey(), s3Account.getSecretKey(), s3Account.getUrl(), + pathStyleAccessEnabled, HpcIntegratedSystem.USER_S_3_PROVIDER); + } + } + + /** + * Get S3 Transfer Manager from an authenticated token. + * + * @param authenticatedToken An authenticated token. + * @return A transfer manager object. + * @throws HpcException on invalid authentication token. + */ + public S3TransferManager getTransferManager(Object authenticatedToken) throws HpcException { + if (!(authenticatedToken instanceof HpcS3)) { + throw new HpcException("Invalid S3 authentication token", HpcErrorType.INVALID_REQUEST_INPUT); + } + + return ((HpcS3) authenticatedToken).transferManager; + } + + /** + * Get S3 Client from an authenticated token. + * + * @param authenticatedToken An authenticated token. + * @return A S3 client object. + * @throws HpcException on invalid authentication token. + */ + public S3AsyncClient getClient(Object authenticatedToken) throws HpcException { + if (!(authenticatedToken instanceof HpcS3)) { + throw new HpcException("Invalid S3 authentication token", HpcErrorType.INVALID_REQUEST_INPUT); + } + + return ((HpcS3) authenticatedToken).client; + } + + /** + * Get S3 Presigner from an authenticated token. + * + * @param authenticatedToken An authenticated token. + * @return A S3 presigner object. + * @throws HpcException on invalid authentication token. + */ + public S3Presigner getPresigner(Object authenticatedToken) throws HpcException { + if (!(authenticatedToken instanceof HpcS3)) { + throw new HpcException("Invalid S3 authentication token", HpcErrorType.INVALID_REQUEST_INPUT); + } + + return ((HpcS3) authenticatedToken).presigner; + } + + /** + * Get S3 Provider from an authenticated token. + * + * @param authenticatedToken An authenticated token. + * @return A transfer manager object. + * @throws HpcException on invalid authentication token. + */ + public HpcIntegratedSystem getS3Provider(Object authenticatedToken) throws HpcException { + if (!(authenticatedToken instanceof HpcS3)) { + return null; + } + + return ((HpcS3) authenticatedToken).provider; + } + + // ---------------------------------------------------------------------// + // Helper Methods + // ---------------------------------------------------------------------// + + private class HpcS3 { + private S3TransferManager transferManager = null; + private S3AsyncClient client = null; + private S3Presigner presigner = null; + private HpcIntegratedSystem provider = null; + } + + /** + * Authenticate a 'S3 3rd Party Provider' account. + * + * @param username The S3 account user name. + * @param password The S3 account password. + * @param url The S3 3rd party provider URL. + * @param pathStyleAccessEnabled true if the S3 3rd Party provider supports path + * style access. + * @param s3Provider The 3rd party provider. + * @return HpcS3 instance + * @throws HpcException if authentication failed + */ + private Object authenticateS3Provider(String username, String password, String url, boolean pathStyleAccessEnabled, + HpcIntegratedSystem s3Provider) throws HpcException { + // Create the credential provider based on the configured credentials. + AwsBasicCredentials s3ProviderCredentials = AwsBasicCredentials.create(username, password); + StaticCredentialsProvider s3ProviderCredentialsProvider = StaticCredentialsProvider + .create(s3ProviderCredentials); + + // Create URI to the S3 provider endpoint + URI uri = null; + try { + uri = new URI(url); + + } catch (URISyntaxException e) { + throw new HpcException("Invalid URL: " + url, HpcErrorType.DATA_TRANSFER_ERROR, e); + } + + HpcS3 s3 = new HpcS3(); + s3.provider = s3Provider; + + try { + // Instantiate a S3 async client. + s3.client = S3AsyncClient.builder() + .credentialsProvider(s3ProviderCredentialsProvider).forcePathStyle(pathStyleAccessEnabled) + .endpointOverride(uri).multipartConfiguration(mpConfigBuilder -> mpConfigBuilder.minimumPartSizeInBytes(minimumUploadPartSize) + .thresholdInBytes(url.equalsIgnoreCase(GOOGLE_STORAGE_URL) ? FIVE_GB : multipartUploadThreshold)).build(); + // Instantiate the S3 transfer manager. + s3.transferManager = S3TransferManager.builder().s3Client(s3.client).executor(executorService).build(); + + // Instantiate the S3 presigner. + s3.presigner = S3Presigner.builder().credentialsProvider(s3ProviderCredentialsProvider) + .endpointOverride(uri).serviceConfiguration( + S3Configuration.builder().pathStyleAccessEnabled(pathStyleAccessEnabled).build()) + .build(); + + return s3; + + } catch (SdkException e) { + throw new HpcException( + "[S3] Failed to authenticate S3 Provider: " + s3Provider.value() + "] - " + e.getMessage(), + HpcErrorType.DATA_TRANSFER_ERROR, e); + } + } + + /** + * Authenticate an AWS S3 account. + * + * @param accessKey The AWS account access key. + * @param secretKey The AWS account secret key. + * @param region The AWS account region. + * @return TransferManager + * @throws HpcException if authentication failed + */ + private Object authenticateAWS(String accessKey, String secretKey, String region) throws HpcException { + // Create the credential provider based on provided AWS S3 account. + AwsBasicCredentials awsCredentials = AwsBasicCredentials.create(accessKey, secretKey); + StaticCredentialsProvider awsCredentialsProvider = StaticCredentialsProvider.create(awsCredentials); + + HpcS3 s3 = new HpcS3(); + s3.provider = HpcIntegratedSystem.AWS; + + try { + // Instantiate a S3 async client. + s3.client = S3AsyncClient.builder().credentialsProvider(awsCredentialsProvider).region(Region.of(region)). + multipartConfiguration(mpConfigBuilder -> mpConfigBuilder.minimumPartSizeInBytes(minimumUploadPartSize) + .thresholdInBytes(multipartUploadThreshold)).build(); + + // Instantiate the S3 transfer manager. + s3.transferManager = S3TransferManager.builder().s3Client(s3.client).executor(executorService).build(); + + // Instantiate the S3 presigner. + s3.presigner = S3Presigner.builder().credentialsProvider(awsCredentialsProvider).region(Region.of(region)) + .build(); + + return s3; + + } catch (SdkException e) { + throw new HpcException("[S3] Failed to authenticate S3 in region " + region + "] - " + e.getMessage(), + HpcErrorType.DATA_TRANSFER_ERROR, e); + } + } } diff --git a/src/hpc-server/hpc-integration-impl/src/main/resources/META-INF/spring/hpc-integration-beans-configuration.xml b/src/hpc-server/hpc-integration-impl/src/main/resources/META-INF/spring/hpc-integration-beans-configuration.xml index e592d70591..a937a0e89e 100644 --- a/src/hpc-server/hpc-integration-impl/src/main/resources/META-INF/spring/hpc-integration-beans-configuration.xml +++ b/src/hpc-server/hpc-integration-impl/src/main/resources/META-INF/spring/hpc-integration-beans-configuration.xml @@ -54,14 +54,14 @@ class="gov.nih.nci.hpc.integration.globus.impl.HpcDataTransferProxyImpl" /> - + + class="gov.nih.nci.hpc.integration.s3.v2.impl.HpcDataTransferProxyImpl" /> 1.9.6 1.12.420 2.32.31 - 0.38.11 2.18.1 4.3.3.0-RELEASE 0.12.5 @@ -326,11 +325,6 @@ pom import - - software.amazon.awssdk.crt - aws-crt - ${amazon-aws-crt.version} - org.irods.jargon jargon-core From 014fda89f228d0cab44928d04fdbf83237823ebc Mon Sep 17 00:00:00 2001 From: Eran Rosenberg Date: Sun, 30 Nov 2025 12:21:36 -0500 Subject: [PATCH 2/8] IDEA files - ignore --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index 4c174d0e1d..96300d16f2 100644 --- a/.gitignore +++ b/.gitignore @@ -1051,3 +1051,8 @@ src/hpc-server/hpc-integration-api/target/hpc-integration-api-3.14.0.jar src/hpc-server/hpc-integration-api/target/classes/META-INF/MANIFEST.MF src/hpc-server/hpc-integration-api/target/classes/META-INF/maven/gov.nih.nci.hpc/hpc-integration-api/pom.properties src/hpc-server/hpc-integration-api/target/classes/META-INF/maven/gov.nih.nci.hpc/hpc-integration-api/pom.xml +src/hpc-server/.idea/copilot.data.migration.agent.xml +src/hpc-server/.idea/copilot.data.migration.ask.xml +src/hpc-server/.idea/copilot.data.migration.ask2agent.xml +src/hpc-server/.idea/copilot.data.migration.edit.xml +src/hpc-server/.idea/copilotDiffState.xml From ec6d110c4f2e98af30239eaf28cf2fcf6899f951 Mon Sep 17 00:00:00 2001 From: Eran Rosenberg Date: Sun, 30 Nov 2025 13:07:56 -0500 Subject: [PATCH 3/8] Updated checksum check w/ SDK V2 new method instead of deprecated one --- .../hpc/integration/s3/v2/impl/HpcDataTransferProxyImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/s3/v2/impl/HpcDataTransferProxyImpl.java b/src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/s3/v2/impl/HpcDataTransferProxyImpl.java index 06defc19df..dcec2892d1 100644 --- a/src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/s3/v2/impl/HpcDataTransferProxyImpl.java +++ b/src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/s3/v2/impl/HpcDataTransferProxyImpl.java @@ -729,7 +729,7 @@ private HpcDataObjectUploadResponse uploadDataObject(Object authenticatedToken, uploadResponse.setDataTransferType(HpcDataTransferType.S_3); uploadResponse.setDataTransferStarted(dataTransferStarted); uploadResponse.setDataTransferCompleted(dataTransferCompleted); - uploadResponse.setDataTransferRequestId(String.valueOf(fileUpload.hashCode())); + uploadResponse.setDataTransferRequestId(String.valueOf(fileUpload.completionFuture().hashCode())); uploadResponse.setSourceSize(sourceFile.length()); uploadResponse.setDataTransferMethod(HpcDataTransferUploadMethod.SYNC); if (archiveType.equals(HpcArchiveType.ARCHIVE)) { From e0dc29a3f6edeabbc1e2e0f382f3710f865307f0 Mon Sep 17 00:00:00 2001 From: Eran Rosenberg Date: Sun, 30 Nov 2025 14:51:21 -0500 Subject: [PATCH 4/8] Update src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/s3/v2/impl/HpcS3Connection.java Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../hpc/integration/s3/v2/impl/HpcS3Connection.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/s3/v2/impl/HpcS3Connection.java b/src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/s3/v2/impl/HpcS3Connection.java index 7888cdf5b9..34ab75ef02 100644 --- a/src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/s3/v2/impl/HpcS3Connection.java +++ b/src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/s3/v2/impl/HpcS3Connection.java @@ -293,9 +293,13 @@ private Object authenticateAWS(String accessKey, String secretKey, String region try { // Instantiate a S3 async client. - s3.client = S3AsyncClient.builder().credentialsProvider(awsCredentialsProvider).region(Region.of(region)). - multipartConfiguration(mpConfigBuilder -> mpConfigBuilder.minimumPartSizeInBytes(minimumUploadPartSize) - .thresholdInBytes(multipartUploadThreshold)).build(); + s3.client = S3AsyncClient.builder() + .credentialsProvider(awsCredentialsProvider) + .region(Region.of(region)) + .multipartConfiguration(mpConfigBuilder -> mpConfigBuilder + .minimumPartSizeInBytes(minimumUploadPartSize) + .thresholdInBytes(multipartUploadThreshold)) + .build(); // Instantiate the S3 transfer manager. s3.transferManager = S3TransferManager.builder().s3Client(s3.client).executor(executorService).build(); From e1efb4d1a2da297e298bea668bb3be4c8e5f6d2f Mon Sep 17 00:00:00 2001 From: Eran Rosenberg Date: Sun, 30 Nov 2025 14:56:36 -0500 Subject: [PATCH 5/8] formatting --- .../hpc/integration/s3/v2/impl/HpcS3Connection.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/s3/v2/impl/HpcS3Connection.java b/src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/s3/v2/impl/HpcS3Connection.java index 34ab75ef02..ca6f11f3e3 100644 --- a/src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/s3/v2/impl/HpcS3Connection.java +++ b/src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/s3/v2/impl/HpcS3Connection.java @@ -253,9 +253,13 @@ private Object authenticateS3Provider(String username, String password, String u try { // Instantiate a S3 async client. s3.client = S3AsyncClient.builder() - .credentialsProvider(s3ProviderCredentialsProvider).forcePathStyle(pathStyleAccessEnabled) - .endpointOverride(uri).multipartConfiguration(mpConfigBuilder -> mpConfigBuilder.minimumPartSizeInBytes(minimumUploadPartSize) - .thresholdInBytes(url.equalsIgnoreCase(GOOGLE_STORAGE_URL) ? FIVE_GB : multipartUploadThreshold)).build(); + .credentialsProvider(s3ProviderCredentialsProvider). + forcePathStyle(pathStyleAccessEnabled) + .endpointOverride(uri). + multipartConfiguration(mpConfigBuilder -> mpConfigBuilder + .minimumPartSizeInBytes(minimumUploadPartSize) + .thresholdInBytes(url.equalsIgnoreCase(GOOGLE_STORAGE_URL) ? FIVE_GB : multipartUploadThreshold)) + .build(); // Instantiate the S3 transfer manager. s3.transferManager = S3TransferManager.builder().s3Client(s3.client).executor(executorService).build(); From a7797895ab0c730b638ede5c1cee16ba4bb5d8e6 Mon Sep 17 00:00:00 2001 From: Eran Rosenberg Date: Sun, 30 Nov 2025 14:58:43 -0500 Subject: [PATCH 6/8] Update src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/s3/v2/impl/HpcS3Connection.java Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../nci/hpc/integration/s3/v2/impl/HpcS3Connection.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/s3/v2/impl/HpcS3Connection.java b/src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/s3/v2/impl/HpcS3Connection.java index ca6f11f3e3..2de8d90aba 100644 --- a/src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/s3/v2/impl/HpcS3Connection.java +++ b/src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/s3/v2/impl/HpcS3Connection.java @@ -253,10 +253,10 @@ private Object authenticateS3Provider(String username, String password, String u try { // Instantiate a S3 async client. s3.client = S3AsyncClient.builder() - .credentialsProvider(s3ProviderCredentialsProvider). - forcePathStyle(pathStyleAccessEnabled) - .endpointOverride(uri). - multipartConfiguration(mpConfigBuilder -> mpConfigBuilder + .credentialsProvider(s3ProviderCredentialsProvider) + .forcePathStyle(pathStyleAccessEnabled) + .endpointOverride(uri) + .multipartConfiguration(mpConfigBuilder -> mpConfigBuilder .minimumPartSizeInBytes(minimumUploadPartSize) .thresholdInBytes(url.equalsIgnoreCase(GOOGLE_STORAGE_URL) ? FIVE_GB : multipartUploadThreshold)) .build(); From fcabe83c3709c9115d953ce3410fd1b3d396dcd6 Mon Sep 17 00:00:00 2001 From: Eran Rosenberg Date: Sun, 7 Dec 2025 15:02:27 -0500 Subject: [PATCH 7/8] Start of V3 version of S3 proxy impl --- .../s3/v3/impl/HpcDataTransferProxyImpl.java | 1220 +++++++++++++++++ .../s3/v3/impl/HpcS3Connection.java | 322 +++++ .../s3/v3/impl/HpcS3ProgressListener.java | 129 ++ 3 files changed, 1671 insertions(+) create mode 100644 src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/s3/v3/impl/HpcDataTransferProxyImpl.java create mode 100644 src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/s3/v3/impl/HpcS3Connection.java create mode 100644 src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/s3/v3/impl/HpcS3ProgressListener.java diff --git a/src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/s3/v3/impl/HpcDataTransferProxyImpl.java b/src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/s3/v3/impl/HpcDataTransferProxyImpl.java new file mode 100644 index 0000000000..80d8ead7ae --- /dev/null +++ b/src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/s3/v3/impl/HpcDataTransferProxyImpl.java @@ -0,0 +1,1220 @@ +package gov.nih.nci.hpc.integration.s3.v3.impl; + +import static gov.nih.nci.hpc.integration.HpcDataTransferProxy.getArchiveDestinationLocation; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.UnsupportedEncodingException; +import java.net.URI; +import java.net.URL; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.Executor; + +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; + +import gov.nih.nci.hpc.domain.datamanagement.HpcPathAttributes; +import gov.nih.nci.hpc.domain.datatransfer.HpcArchive; +import gov.nih.nci.hpc.domain.datatransfer.HpcArchiveObjectMetadata; +import gov.nih.nci.hpc.domain.datatransfer.HpcArchiveType; +import gov.nih.nci.hpc.domain.datatransfer.HpcDataObjectDownloadRequest; +import gov.nih.nci.hpc.domain.datatransfer.HpcDataTransferType; +import gov.nih.nci.hpc.domain.datatransfer.HpcDataTransferUploadMethod; +import gov.nih.nci.hpc.domain.datatransfer.HpcDataTransferUploadStatus; +import gov.nih.nci.hpc.domain.datatransfer.HpcDeepArchiveStatus; +import gov.nih.nci.hpc.domain.datatransfer.HpcDirectoryScanItem; +import gov.nih.nci.hpc.domain.datatransfer.HpcFileLocation; +import gov.nih.nci.hpc.domain.datatransfer.HpcMultipartUpload; +import gov.nih.nci.hpc.domain.datatransfer.HpcS3Account; +import gov.nih.nci.hpc.domain.datatransfer.HpcS3DownloadDestination; +import gov.nih.nci.hpc.domain.datatransfer.HpcSetArchiveObjectMetadataResponse; +import gov.nih.nci.hpc.domain.datatransfer.HpcStreamingUploadSource; +import gov.nih.nci.hpc.domain.datatransfer.HpcUploadPartETag; +import gov.nih.nci.hpc.domain.datatransfer.HpcUploadPartURL; +import gov.nih.nci.hpc.domain.error.HpcErrorType; +import gov.nih.nci.hpc.domain.metadata.HpcMetadataEntry; +import gov.nih.nci.hpc.domain.model.HpcDataObjectUploadRequest; +import gov.nih.nci.hpc.domain.model.HpcDataObjectUploadResponse; +import gov.nih.nci.hpc.domain.user.HpcIntegratedSystemAccount; +import gov.nih.nci.hpc.exception.HpcException; +import gov.nih.nci.hpc.integration.HpcDataTransferProgressListener; +import gov.nih.nci.hpc.integration.HpcDataTransferProxy; +import software.amazon.awssdk.awscore.AwsRequestOverrideConfiguration; +import software.amazon.awssdk.core.async.AsyncRequestBody; +import software.amazon.awssdk.core.async.BlockingInputStreamAsyncRequestBody; +import software.amazon.awssdk.core.exception.SdkException; +import software.amazon.awssdk.core.exception.SdkServiceException; +import software.amazon.awssdk.services.s3.model.BucketLifecycleConfiguration; +import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload; +import software.amazon.awssdk.services.s3.model.CompletedPart; +import software.amazon.awssdk.services.s3.model.CopyObjectRequest; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.ExpirationStatus; +import software.amazon.awssdk.services.s3.model.GetBucketLifecycleConfigurationResponse; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GlacierJobParameters; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectResponse; +import software.amazon.awssdk.services.s3.model.LifecycleRule; +import software.amazon.awssdk.services.s3.model.ListObjectsRequest; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; +import software.amazon.awssdk.services.s3.model.MetadataDirective; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.RestoreObjectRequest; +import software.amazon.awssdk.services.s3.model.RestoreRequest; +import software.amazon.awssdk.services.s3.model.S3Object; +import software.amazon.awssdk.services.s3.model.Tier; +import software.amazon.awssdk.services.s3.model.Transition; +import software.amazon.awssdk.services.s3.model.TransitionStorageClass; +import software.amazon.awssdk.services.s3.model.UploadPartRequest; +import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest; +import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest; +import software.amazon.awssdk.services.s3.presigner.model.PresignedPutObjectRequest; +import software.amazon.awssdk.services.s3.presigner.model.PutObjectPresignRequest; +import software.amazon.awssdk.services.s3.presigner.model.UploadPartPresignRequest; +import software.amazon.awssdk.transfer.s3.model.CompletedCopy; +import software.amazon.awssdk.transfer.s3.model.Copy; +import software.amazon.awssdk.transfer.s3.model.CopyRequest; +import software.amazon.awssdk.transfer.s3.model.DownloadFileRequest; +import software.amazon.awssdk.transfer.s3.model.FileDownload; +import software.amazon.awssdk.transfer.s3.model.FileUpload; +import software.amazon.awssdk.transfer.s3.model.Upload; +import software.amazon.awssdk.transfer.s3.model.UploadFileRequest; + +/** + * HPC Data Transfer Proxy S3 Implementation. + * + * @author Eran Rosenberg + */ +public class HpcDataTransferProxyImpl implements HpcDataTransferProxy { + // ---------------------------------------------------------------------// + // Constants + // ---------------------------------------------------------------------// + + // The expiration of streaming data request from 3rd Party S3 Archive + // (Cleversafe, Cloudian, etc) to AWS S3. + private static final int S3_STREAM_EXPIRATION = 96; + + // Cloudian tiering info header required when adding tiering rule + private static final String CLOUDIAN_TIERING_INFO_HEADER = "x-gmt-tieringinfo"; + + // Number of days the restored data object will be available. + @Value("${hpc.integration.s3.tieringEndpoint}") + private String tieringEndpoint = null; + + // Number of days the restored data object will be available. + @Value("${hpc.integration.s3.restoreNumDays}") + private int restoreNumDays = 2; + + // ---------------------------------------------------------------------// + // Instance members + // ---------------------------------------------------------------------// + + // The S3 connection instance. + @Autowired + private HpcS3Connection s3Connection = null; + + // The S3 executor. + @Autowired + @Qualifier("hpcS3Executor") + Executor s3Executor = null; + + // Date formatter to format files last-modified date + private DateFormat dateFormat = new SimpleDateFormat("MM-dd-yyyy HH:mm:ss"); + + // The logger instance. + private final Logger logger = LoggerFactory.getLogger(getClass().getName()); + + // ---------------------------------------------------------------------// + // Constructors + // ---------------------------------------------------------------------// + + /** Constructor for spring injection. */ + private HpcDataTransferProxyImpl() { + } + + // ---------------------------------------------------------------------// + // Methods + // ---------------------------------------------------------------------// + + // ---------------------------------------------------------------------// + // HpcDataTransferProxy Interface Implementation + // ---------------------------------------------------------------------// + + @Override + public Object authenticate(HpcIntegratedSystemAccount dataTransferAccount, String urlOrRegion, + String encryptionAlgorithm, String encryptionKey) throws HpcException { + return s3Connection.authenticate(dataTransferAccount, urlOrRegion); + } + + @Override + public Object authenticate(HpcS3Account s3Account) throws HpcException { + return s3Connection.authenticate(s3Account); + } + + @Override + public HpcDataObjectUploadResponse uploadDataObject(Object authenticatedToken, + HpcDataObjectUploadRequest uploadRequest, HpcArchive baseArchiveDestination, + Integer uploadRequestURLExpiration, HpcDataTransferProgressListener progressListener, + List metadataEntries, Boolean encryptedTransfer, String storageClass) + throws HpcException { + if (uploadRequest.getGlobusUploadSource() != null) { + throw new HpcException("Invalid upload source", HpcErrorType.UNEXPECTED_ERROR); + } + + // Calculate the archive destination. + HpcFileLocation archiveDestinationLocation = getArchiveDestinationLocation( + baseArchiveDestination.getFileLocation(), uploadRequest.getPath(), uploadRequest.getCallerObjectId(), + baseArchiveDestination.getType(), this, authenticatedToken); + + if (uploadRequest.getGenerateUploadRequestURL()) { + int uploadParts = Optional.ofNullable(uploadRequest.getUploadParts()).orElse(1); + if (uploadParts == 1) { + // Generate an upload request URL for the caller to use to upload directly. + return generateUploadRequestURL(authenticatedToken, archiveDestinationLocation, + uploadRequestURLExpiration, metadataEntries, uploadRequest.getUploadRequestURLChecksum(), + storageClass, Optional.ofNullable(uploadRequest.getUploadCompletion()).orElse(false)); + } else { + return generateMultipartUploadRequestURLs(authenticatedToken, archiveDestinationLocation, + uploadRequestURLExpiration, uploadParts, metadataEntries, storageClass); + } + } else if (uploadRequest.getSourceFile() != null) { + // Upload a file + return uploadDataObject(authenticatedToken, uploadRequest.getSourceFile(), archiveDestinationLocation, + progressListener, baseArchiveDestination.getType(), metadataEntries, storageClass); + } else { + // Upload by streaming from AWS, 3rd Party S3 Provider, Google Drive or Google + // Cloud Storage source. + return uploadDataObject(authenticatedToken, uploadRequest.getS3UploadSource(), + uploadRequest.getGoogleDriveUploadSource(), uploadRequest.getGoogleCloudStorageUploadSource(), + archiveDestinationLocation, baseArchiveDestination, uploadRequest.getSourceSize(), progressListener, + metadataEntries, storageClass); + } + } + + @Override + public String downloadDataObject(Object authenticatedToken, HpcDataObjectDownloadRequest downloadRequest, + HpcArchive baseArchiveDestination, HpcDataTransferProgressListener progressListener, + Boolean encryptedTransfer) throws HpcException { + if (downloadRequest.getArchiveLocation() == null) { + throw new HpcException("Null archive location", HpcErrorType.UNEXPECTED_ERROR); + } + + if (downloadRequest.getFileDestination() != null) { + // This is a download request to a local file. + return downloadDataObject(authenticatedToken, downloadRequest.getArchiveLocation(), + downloadRequest.getFileDestination(), progressListener); + } else { + // This is a download to S3 destination (either AWS or 3rd Party Provider). + return downloadDataObject(authenticatedToken, downloadRequest.getArchiveLocation(), + downloadRequest.getArchiveLocationURL(), baseArchiveDestination, downloadRequest.getS3Destination(), + progressListener, downloadRequest.getSize()); + } + } + + @Override + public String generateDownloadRequestURL(Object authenticatedToken, HpcFileLocation archiveSourceLocation, + HpcArchive baseArchiveDestination, Integer downloadRequestURLExpiration) throws HpcException { + if (archiveSourceLocation == null) { + throw new HpcException("Null archive location", HpcErrorType.UNEXPECTED_ERROR); + } + + try { + GetObjectRequest getObjectRequest = GetObjectRequest.builder() + .bucket(archiveSourceLocation.getFileContainerId()).key(archiveSourceLocation.getFileId()).build(); + + GetObjectPresignRequest getObjectPresignRequest = GetObjectPresignRequest.builder() + .signatureDuration(Duration.ofHours(downloadRequestURLExpiration)) + .getObjectRequest(getObjectRequest).build(); + + PresignedGetObjectRequest presignedGetObjectRequest = s3Connection.getPresigner(authenticatedToken) + .presignGetObject(getObjectPresignRequest); + return presignedGetObjectRequest.url().toString(); + + } catch (SdkException e) { + throw new HpcException( + "[S3] Failed to generate presigned download URL" + archiveSourceLocation.getFileContainerId() + ":" + + archiveSourceLocation.getFileId() + " - " + e.getMessage(), + HpcErrorType.DATA_TRANSFER_ERROR, e); + } + } + + @Override + public HpcSetArchiveObjectMetadataResponse setDataObjectMetadata(Object authenticatedToken, + HpcFileLocation fileLocation, HpcArchive baseArchiveDestination, List metadataEntries, + String storageClass) throws HpcException { + HpcSetArchiveObjectMetadataResponse response = new HpcSetArchiveObjectMetadataResponse(); + + // Check if the metadata was already set on the data-object in the S3 archive. + try { + HeadObjectRequest headObjectRequest = HeadObjectRequest.builder().bucket(fileLocation.getFileContainerId()) + .key(fileLocation.getFileId()).build(); + HeadObjectResponse headObjectResponse = s3Connection.getClient(authenticatedToken) + .headObject(headObjectRequest).join(); + + Map s3Metadata = headObjectResponse.metadata(); + boolean metadataAlreadySet = true; + for (HpcMetadataEntry metadataEntry : metadataEntries) { + if (!s3Metadata.containsKey(metadataEntry.getAttribute())) { + metadataAlreadySet = false; + break; + } + } + + if (metadataAlreadySet) { + logger.info("System metadata in S3 archive already set for [{}]. No need to copy-object in archive", + fileLocation.getFileId()); + response.setChecksum(headObjectResponse.eTag().replace("\"", "")); + response.setMetadataAdded(false); + return response; + } + + } catch (CompletionException e) { + throw new HpcException( + "[S3] Failed to get object metadata: " + fileLocation.getFileContainerId() + ":" + + fileLocation.getFileId() + " - " + e.getCause().getMessage(), + HpcErrorType.DATA_TRANSFER_ERROR, e.getCause()); + } + + // We set S3 metadata by copying the data-object to itself w/ attached metadata. + CopyObjectRequest copyObjectRequest = CopyObjectRequest.builder() + .sourceBucket(fileLocation.getFileContainerId()).sourceKey(fileLocation.getFileId()) + .destinationBucket(fileLocation.getFileContainerId()).destinationKey(fileLocation.getFileId()) + .storageClass(storageClass).metadata(toS3Metadata(metadataEntries)) + .metadataDirective(MetadataDirective.REPLACE).build(); + + CopyRequest copyRequest = CopyRequest.builder().copyObjectRequest(copyObjectRequest).build(); + + try { + Copy copy = s3Connection.getTransferManager(authenticatedToken).copy(copyRequest); + + CompletedCopy completedCopy = copy.completionFuture().join(); + response.setChecksum(completedCopy.response().copyObjectResult().eTag().replace("\"", "")); + response.setMetadataAdded(true); + return response; + + } catch (CompletionException e) { + throw new HpcException("[S3] Failed to copy file: " + copyRequest, HpcErrorType.DATA_TRANSFER_ERROR, + s3Connection.getS3Provider(authenticatedToken), e.getCause()); + } + + } + + @Override + public void deleteDataObject(Object authenticatedToken, HpcFileLocation fileLocation, + HpcArchive baseArchiveDestination) throws HpcException { + + // Create a S3 delete request. + DeleteObjectRequest deleteRequest = DeleteObjectRequest.builder().bucket(fileLocation.getFileContainerId()) + .key(fileLocation.getFileId()).build(); + + try { + s3Connection.getClient(authenticatedToken).deleteObject(deleteRequest).join(); + + } catch (CompletionException e) { + throw new HpcException("[S3] Failed to delete file: " + deleteRequest, HpcErrorType.DATA_TRANSFER_ERROR, + s3Connection.getS3Provider(authenticatedToken), e.getCause()); + } + } + + @Override + public HpcPathAttributes getPathAttributes(Object authenticatedToken, HpcFileLocation fileLocation, boolean getSize) + throws HpcException { + + HpcPathAttributes pathAttributes = new HpcPathAttributes(); + HeadObjectResponse headObjectResponse = null; + + // Look for the file. + try { + pathAttributes.setIsAccessible(true); + HeadObjectRequest headObjectRequest = HeadObjectRequest.builder().bucket(fileLocation.getFileContainerId()) + .key(fileLocation.getFileId()).build(); + headObjectResponse = s3Connection.getClient(authenticatedToken).headObject(headObjectRequest) + .exceptionally(e -> { + Throwable cause = e.getCause(); + if (cause instanceof SdkServiceException) { + if (((SdkServiceException) cause).statusCode() == 403) { + pathAttributes.setIsAccessible(false); + } + } else if (!(cause instanceof NoSuchKeyException)) { + logger.error("[S3] Failed to get head object request: " + cause.getClass().toString() + + " * " + cause.getMessage(), cause); + } + return null; + }).join(); + + } catch (CompletionException e) { + throw new HpcException("[S3] Failed to get head object request: " + e.getCause().getMessage(), + HpcErrorType.DATA_TRANSFER_ERROR, s3Connection.getS3Provider(authenticatedToken), e.getCause()); + + } + + if (!pathAttributes.getIsAccessible()) { + return pathAttributes; + } + if (headObjectResponse != null) { + // This is a file. + pathAttributes.setIsFile(true); + pathAttributes.setIsDirectory(false); + pathAttributes.setExists(true); + + } else { + pathAttributes.setIsFile(false); + + // Check if this is a directory. + boolean directoryExists = isDirectory(authenticatedToken, fileLocation); + pathAttributes.setIsDirectory(directoryExists); + pathAttributes.setExists(directoryExists); + } + + // Optionally get the file size. We currently don't support getting file size + // for a directory. + if (getSize && headObjectResponse != null) { + pathAttributes.setSize(headObjectResponse.contentLength()); + } + + return pathAttributes; + } + + @Override + public List scanDirectory(Object authenticatedToken, HpcFileLocation directoryLocation) + throws HpcException { + List directoryScanItems = new ArrayList<>(); + + try { + ListObjectsV2Request listObjectsRequest = ListObjectsV2Request.builder() + .bucket(directoryLocation.getFileContainerId()).prefix(directoryLocation.getFileId()).build(); + + ListObjectsV2Response listObjectsResponse = s3Connection.getClient(authenticatedToken) + .listObjectsV2(listObjectsRequest).join(); + + List s3Objects = listObjectsResponse.contents(); + + // Paginate through all results. + while (listObjectsResponse.isTruncated()) { + String continuationToken = listObjectsResponse.nextContinuationToken(); + + listObjectsRequest = listObjectsRequest.toBuilder().continuationToken(continuationToken).build(); + listObjectsResponse = s3Connection.getClient(authenticatedToken).listObjectsV2(listObjectsRequest) + .join(); + + if (continuationToken.equals(listObjectsResponse.nextContinuationToken())) { + // Pagination over list objects is not working w/ Cleversafe storage, we keep + // getting the same set of results. This code is to protect against infinite + // loop. + break; + } + + s3Objects.addAll(listObjectsResponse.contents()); + } + + s3Objects.forEach(s3Object -> { + if (s3Object.size() > 0) { + HpcDirectoryScanItem directoryScanItem = new HpcDirectoryScanItem(); + directoryScanItem.setFilePath(s3Object.key()); + directoryScanItem.setFileName(FilenameUtils.getName(s3Object.key())); + directoryScanItem.setLastModified(dateFormat.format(Date.from(s3Object.lastModified()))); + directoryScanItems.add(directoryScanItem); + } + }); + + return directoryScanItems; + + } catch (CompletionException e) { + throw new HpcException("[S3] Failed to list objects: " + e.getCause().getMessage(), + HpcErrorType.DATA_TRANSFER_ERROR, s3Connection.getS3Provider(authenticatedToken), e.getCause()); + } + } + + @Override + public String completeMultipartUpload(Object authenticatedToken, HpcFileLocation archiveLocation, + String multipartUploadId, List uploadPartETags) throws HpcException { + + // Create AWS part ETags from the HPC model. + List parts = new ArrayList<>(); + uploadPartETags.forEach(uploadPartETag -> parts.add(CompletedPart.builder() + .partNumber(uploadPartETag.getPartNumber()).eTag(uploadPartETag.getETag()).build())); + CompletedMultipartUpload completedMultipartUpload = CompletedMultipartUpload.builder().parts(parts).build(); + + CompleteMultipartUploadRequest completeMultipartUploadRequest = CompleteMultipartUploadRequest.builder() + .bucket(archiveLocation.getFileContainerId()).key(archiveLocation.getFileId()) + .uploadId(multipartUploadId).multipartUpload(completedMultipartUpload).build(); + + try { + return s3Connection.getClient(authenticatedToken).completeMultipartUpload(completeMultipartUploadRequest) + .join().eTag(); + + } catch (CompletionException e) { + throw new HpcException( + "[S3] Failed to complete a multipart upload to " + archiveLocation.getFileContainerId() + ":" + + archiveLocation.getFileId() + ". multi-part-upload-id = " + multipartUploadId + + ", number-of-parts = " + uploadPartETags.size() + " - " + e.getMessage(), + HpcErrorType.DATA_TRANSFER_ERROR, s3Connection.getS3Provider(authenticatedToken), e.getCause()); + } + } + + @Override + public void restoreDataObject(Object authenticatedToken, HpcFileLocation archiveLocation) throws HpcException { + try { + // Create and submit a request to restore an object from Glacier. + RestoreRequest restoreRequest = RestoreRequest.builder().days(restoreNumDays) + .glacierJobParameters(GlacierJobParameters.builder().tier(Tier.STANDARD).build()).build(); + + RestoreObjectRequest restoreObjectRequest = RestoreObjectRequest.builder() + .bucket(archiveLocation.getFileContainerId()).key(archiveLocation.getFileId()) + .restoreRequest(restoreRequest).build(); + s3Connection.getClient(authenticatedToken).restoreObject(restoreObjectRequest).join(); + + } catch (CompletionException e) { + throw new HpcException( + "[S3] Failed to restore data object" + archiveLocation.getFileContainerId() + ":" + + archiveLocation.getFileId() + " - " + e.getCause().getMessage(), + HpcErrorType.DATA_TRANSFER_ERROR, s3Connection.getS3Provider(authenticatedToken), e.getCause()); + } + } + + @Override + public boolean existsTieringPolicy(Object authenticatedToken, HpcFileLocation archiveLocation) throws HpcException { + try { + // Retrieve the configuration. + GetBucketLifecycleConfigurationResponse bucketLifeCycleConfigurationResponse = s3Connection + .getClient(authenticatedToken) + .getBucketLifecycleConfiguration(builder -> builder.bucket(archiveLocation.getFileContainerId())) + .join(); + + if (bucketLifeCycleConfigurationResponse != null) { + for (LifecycleRule rule : bucketLifeCycleConfigurationResponse.rules()) { + // Look through filter prefix applied to lifecycle policy + boolean hasTransition = false; + + if (rule.hasTransitions()) { + for (Transition transition : rule.transitions()) { + if (!StringUtils.isEmpty(transition.storageClassAsString())) { + hasTransition = true; + } + } + } + + if (hasTransition && rule.filter() != null && rule.filter().prefix() != null) { + if (archiveLocation.getFileId().contains(rule.filter().prefix())) { + return true; + } + } else if (hasTransition) { + // This is a transition without prefix applies to entire bucket. + return true; + } + } + } + + return false; + + } catch (CompletionException e) { + throw new HpcException( + "[S3] Failed to retrieve life cycle policy on bucket: " + archiveLocation.getFileContainerId() + + "- " + e.getCause().getMessage(), + HpcErrorType.DATA_TRANSFER_ERROR, s3Connection.getS3Provider(authenticatedToken), e.getCause()); + } + } + + @Override + public HpcArchiveObjectMetadata getDataObjectMetadata(Object authenticatedToken, HpcFileLocation fileLocation) + throws HpcException { + HpcArchiveObjectMetadata objectMetadata = new HpcArchiveObjectMetadata(); + String s3ObjectName = fileLocation.getFileContainerId() + ":" + fileLocation.getFileId(); + + // Get metadata for the data-object in the S3 archive. + try { + HeadObjectRequest headObjectRequest = HeadObjectRequest.builder().bucket(fileLocation.getFileContainerId()) + .key(fileLocation.getFileId()).build(); + HeadObjectResponse headObjectResponse = s3Connection.getClient(authenticatedToken) + .headObject(headObjectRequest).join(); + + // x-amz-storage-class is not returned for standard S3 object + logger.info("[S3] Storage class [{}] - {}", s3ObjectName, headObjectResponse.storageClass()); + + if (headObjectResponse.storageClass() != null) { + objectMetadata.setDeepArchiveStatus( + HpcDeepArchiveStatus.fromValue(headObjectResponse.storageClassAsString())); + logger.info("[S3] Deep Archive Status [{}] - {}", s3ObjectName, objectMetadata.getDeepArchiveStatus()); + } + + // Check the restoration status of the object. + String restoreHeader = headObjectResponse.restore(); + logger.info("[S3] Restore Header [{}] - {}", s3ObjectName, restoreHeader); + + if (StringUtils.isEmpty(restoreHeader)) { + // the x-amz-restore header is not present on the response from the service + // (e.g. no restore request has been received). + objectMetadata.setRestorationStatus("not in progress"); + + } else if (restoreHeader.contains("ongoing-request=\"true\"")) { + // the x-amz-restore header is present and has a value of true (e.g. a restore + // request was received and is currently ongoing). + objectMetadata.setRestorationStatus("in progress"); + + } else if (restoreHeader.contains("ongoing-request=\"false\"")) { + // the x-amz-restore header is present and has a value of false (e.g the object + // has been restored and can currently be read from S3). + objectMetadata.setRestorationStatus("success"); + } + logger.info("[S3] Restoration Status [{}] - {}", s3ObjectName, objectMetadata.getRestorationStatus()); + + objectMetadata.setChecksum(headObjectResponse.eTag().replace("\"", "")); + logger.info("[S3] Checksum [{}] - {}", s3ObjectName, objectMetadata.getChecksum()); + + } catch (CompletionException e) { + throw new HpcException( + "[S3] Failed to get object metadata [" + s3ObjectName + "] - " + e.getCause().getMessage(), + HpcErrorType.DATA_TRANSFER_ERROR, s3Connection.getS3Provider(authenticatedToken), e.getCause()); + } + + return objectMetadata; + } + + @SuppressWarnings("deprecation") + @Override + public synchronized void setTieringPolicy(Object authenticatedToken, HpcFileLocation archiveLocation, String prefix, + String tieringBucket, String tieringProtocol) throws HpcException { + + try { + // Create a list of life cycle rules + List lifeCycleRules = new ArrayList<>(); + + // Add the new rule. + Transition transition = Transition.builder().days(0).storageClass(TransitionStorageClass.GLACIER).build(); + lifeCycleRules.add(LifecycleRule.builder().id(prefix).transitions(transition) + .filter(builder -> builder.prefix(prefix)).status(ExpirationStatus.ENABLED).build()); + + // Retrieve the configuration + GetBucketLifecycleConfigurationResponse bucketLifeCycleConfigurationResponse = s3Connection + .getClient(authenticatedToken) + .getBucketLifecycleConfiguration(builder -> builder.bucket(archiveLocation.getFileContainerId())) + .join(); + + // Add the existing rules to the list. + if (bucketLifeCycleConfigurationResponse != null) { + for (LifecycleRule lifeCycleRule : bucketLifeCycleConfigurationResponse.rules()) { + // Rules existing in Cloudian is retrieved with the prefix + // set to the same value as filter. + // Removing since it fails if this value is provided. + lifeCycleRules.add(lifeCycleRule.toBuilder().prefix(null).build()); + } + } + + // Add Cloudian custom tiering header, no impact to AWS S3 requests. + String customHeader = tieringProtocol + "|EndPoint:" + + URLEncoder.encode(tieringEndpoint, StandardCharsets.UTF_8.toString()) + ",TieringBucket:" + + tieringBucket; + String encodedCustomHeader = URLEncoder.encode(customHeader, StandardCharsets.UTF_8.toString()); + + BucketLifecycleConfiguration lifeCycleConfiguration = BucketLifecycleConfiguration.builder() + .rules(lifeCycleRules).build(); + AwsRequestOverrideConfiguration requestOverrideConfiguration = AwsRequestOverrideConfiguration.builder() + .putHeader(CLOUDIAN_TIERING_INFO_HEADER, encodedCustomHeader).build(); + + s3Connection.getClient(authenticatedToken) + .putBucketLifecycleConfiguration(builder -> builder.bucket(archiveLocation.getFileContainerId()) + .lifecycleConfiguration(lifeCycleConfiguration) + .overrideConfiguration(requestOverrideConfiguration)) + .join(); + + } catch (UnsupportedEncodingException e) { + throw new HpcException( + "[S3] Failed to add a new rule to life cycle policy on bucket " + + archiveLocation.getFileContainerId() + ":" + prefix + " - " + e.getMessage(), + HpcErrorType.DATA_TRANSFER_ERROR, s3Connection.getS3Provider(authenticatedToken), e); + + } catch (CompletionException e) { + throw new HpcException( + "[S3] Failed to add a new rule to life cycle policy on bucket " + + archiveLocation.getFileContainerId() + ":" + prefix + " - " + e.getCause().getMessage(), + HpcErrorType.DATA_TRANSFER_ERROR, s3Connection.getS3Provider(authenticatedToken), e.getCause()); + } + } + + @Override + public void shutdown(Object authenticatedToken) throws HpcException { + try { + s3Connection.getTransferManager(authenticatedToken).close(); + s3Connection.getPresigner(authenticatedToken).close(); + s3Connection.getClient(authenticatedToken).close(); + + } catch (Exception e) { + throw new HpcException("[S3] Failed to shutdown AWS TransferManager/Client/Presigner: " + e.getMessage(), + HpcErrorType.DATA_TRANSFER_ERROR, e); + } + } + + // ---------------------------------------------------------------------// + // Helper Methods + // ---------------------------------------------------------------------// + + /** + * Upload a data object file. + * + * @param authenticatedToken An authenticated token. + * @param sourceFile The file to upload. + * @param archiveDestinationLocation The archive destination location. + * @param progressListener (Optional) a progress listener for async + * notification on transfer completion. + * @param archiveType The archive type. + * @param metadataEntries The metadata entries to attach to the + * data-object in S3 archive. + * @param storageClass (Optional) The storage class to upload the + * file. + * @return A data object upload response. + * @throws HpcException on data transfer system failure. + */ + private HpcDataObjectUploadResponse uploadDataObject(Object authenticatedToken, File sourceFile, + HpcFileLocation archiveDestinationLocation, HpcDataTransferProgressListener progressListener, + HpcArchiveType archiveType, List metadataEntries, String storageClass) + throws HpcException { + // Create a S3 upload file request. + HpcS3ProgressListener listener = new HpcS3ProgressListener(progressListener, + "upload staged file [" + sourceFile.getAbsolutePath() + "] to " + + archiveDestinationLocation.getFileContainerId() + ":" + + archiveDestinationLocation.getFileId()); + + UploadFileRequest uploadFileRequest = UploadFileRequest.builder() + .putObjectRequest(b -> b.bucket(archiveDestinationLocation.getFileContainerId()) + .key(archiveDestinationLocation.getFileId()).metadata(toS3Metadata(metadataEntries)) + .storageClass(storageClass)) + .addTransferListener(listener).source(sourceFile).build(); + + // Upload the data. + FileUpload fileUpload = null; + Calendar dataTransferStarted = Calendar.getInstance(); + Calendar dataTransferCompleted = null; + try { + fileUpload = s3Connection.getTransferManager(authenticatedToken).uploadFile(uploadFileRequest); + progressListener.setCompletableFuture(fileUpload.completionFuture()); + fileUpload.completionFuture().join(); + + dataTransferCompleted = Calendar.getInstance(); + + } catch (CompletionException e) { + throw new HpcException("[S3] Failed to upload file.", HpcErrorType.DATA_TRANSFER_ERROR, + s3Connection.getS3Provider(authenticatedToken), e.getCause()); + + } + + // Upload completed. Create and populate the response object. + HpcDataObjectUploadResponse uploadResponse = new HpcDataObjectUploadResponse(); + uploadResponse.setArchiveLocation(archiveDestinationLocation); + uploadResponse.setDataTransferType(HpcDataTransferType.S_3); + uploadResponse.setDataTransferStarted(dataTransferStarted); + uploadResponse.setDataTransferCompleted(dataTransferCompleted); + uploadResponse.setDataTransferRequestId(String.valueOf(fileUpload.completionFuture().hashCode())); + uploadResponse.setSourceSize(sourceFile.length()); + uploadResponse.setDataTransferMethod(HpcDataTransferUploadMethod.SYNC); + if (archiveType.equals(HpcArchiveType.ARCHIVE)) { + uploadResponse.setDataTransferStatus(HpcDataTransferUploadStatus.ARCHIVED); + } else { + uploadResponse.setDataTransferStatus(HpcDataTransferUploadStatus.IN_TEMPORARY_ARCHIVE); + } + + return uploadResponse; + } + + /** + * Upload a data object file from AWS, 3rd Party S3 Provider or Google Drive + * source. + * + * @param authenticatedToken An authenticated token. + * @param s3UploadSource The S3 upload source (AWS or 3rd party + * provider) + * @param googleDriveUploadSource The Google Drive upload source. + * @param googleCloudStorageUploadSource The Google Cloud Storage upload source. + * @param archiveDestinationLocation The archive destination location. + * @param baseArchiveDestination The archive's base destination + * location. + * @param size the size of the file to upload. + * @param progressListener (Optional) a progress listener for + * async notification on transfer + * completion. + * @param metadataEntries The metadata entries to attach to the + * data-object in S3 archive. + * @param storageClass (Optional) The storage class to upload + * the file. + * @return A data object upload response. + * @throws HpcException on data transfer system failure. + */ + private HpcDataObjectUploadResponse uploadDataObject(Object authenticatedToken, + HpcStreamingUploadSource s3UploadSource, HpcStreamingUploadSource googleDriveUploadSource, + HpcStreamingUploadSource googleCloudStorageUploadSource, HpcFileLocation archiveDestinationLocation, + HpcArchive baseArchiveDestination, Long size, HpcDataTransferProgressListener progressListener, + List metadataEntries, String storageClass) throws HpcException { + + if (progressListener == null) { + throw new HpcException( + "[S3] No progress listener provided for a upload from AWS S3 / S3 Provider / Google Drive / Google Cloud Storage", + HpcErrorType.UNEXPECTED_ERROR); + } + if (size == null) { + throw new HpcException( + "[S3] File size not provided for an upload from AWS / S3 Provider / Google Drive / Google Cloud Storage", + HpcErrorType.UNEXPECTED_ERROR); + } + + HpcDataTransferUploadMethod uploadMethod = null; + String sourceURL = null; + HpcFileLocation sourceLocation = null; + + if (s3UploadSource != null) { // Upload by streaming from AWS or S3 Provider. + uploadMethod = HpcDataTransferUploadMethod.S_3; + sourceLocation = s3UploadSource.getSourceLocation(); + + // If not provided, generate a download pre-signed URL for the requested data + // file from AWS // (using the provided S3 account). + sourceURL = StringUtils.isEmpty(s3UploadSource.getSourceURL()) + ? generateDownloadRequestURL(s3Connection.authenticate(s3UploadSource.getAccount()), sourceLocation, + baseArchiveDestination, S3_STREAM_EXPIRATION) + : s3UploadSource.getSourceURL(); + + } else if (googleDriveUploadSource != null) { // Upload by streaming from Google Drive + uploadMethod = HpcDataTransferUploadMethod.GOOGLE_DRIVE; + sourceLocation = googleDriveUploadSource.getSourceLocation(); + + } else if (googleCloudStorageUploadSource != null) { // Upload by streaming from Google Cloud Storage + uploadMethod = HpcDataTransferUploadMethod.GOOGLE_CLOUD_STORAGE; + sourceLocation = googleCloudStorageUploadSource.getSourceLocation(); + + } else { + throw new HpcException("Unexpected upload source", HpcErrorType.UNEXPECTED_ERROR); + } + + final String url = sourceURL; + final String sourceDestinationLogMessage = "upload from " + sourceLocation.getFileContainerId() + ":" + + sourceLocation.getFileId(); + Calendar dataTransferStarted = Calendar.getInstance(); + + CompletableFuture s3TransferManagerUploadFuture = CompletableFuture.runAsync(() -> { + try { + // Open a connection to the input stream of the file to be uploaded. + InputStream sourceInputStream = null; + if (googleDriveUploadSource != null) { + sourceInputStream = googleDriveUploadSource.getSourceInputStream(); + } else if (googleCloudStorageUploadSource != null) { + sourceInputStream = googleCloudStorageUploadSource.getSourceInputStream(); + } else { + sourceInputStream = new URL(url).openStream(); + } + + HpcS3ProgressListener listener = new HpcS3ProgressListener(progressListener, + sourceDestinationLogMessage); + + // Create a S3 upload request. + BlockingInputStreamAsyncRequestBody body = AsyncRequestBody.forBlockingInputStream(size); + Upload streamUpload = s3Connection.getTransferManager(authenticatedToken) + .upload(builder -> builder + .putObjectRequest( + request -> request.bucket(archiveDestinationLocation.getFileContainerId()) + .key(archiveDestinationLocation.getFileId()) + .metadata(toS3Metadata(metadataEntries)).storageClass(storageClass)) + .requestBody(body).addTransferListener(listener)); + + // Stream the data. + body.writeInputStream(sourceInputStream); + progressListener.setCompletableFuture(streamUpload.completionFuture()); + streamUpload.completionFuture().join(); + + } catch (CompletionException | HpcException | IOException e) { + logger.error("[S3] Failed to upload from AWS S3 destination: " + e.getCause().getMessage(), e); + progressListener.transferFailed(e.getCause().getMessage()); + + } + + }, s3Executor); + + // Create and populate the response object. + HpcDataObjectUploadResponse uploadResponse = new HpcDataObjectUploadResponse(); + uploadResponse.setArchiveLocation(archiveDestinationLocation); + uploadResponse.setDataTransferType(HpcDataTransferType.S_3); + uploadResponse.setDataTransferStarted(dataTransferStarted); + uploadResponse.setUploadSource(sourceLocation); + uploadResponse.setDataTransferRequestId(String.valueOf(s3TransferManagerUploadFuture.hashCode())); + uploadResponse.setDataTransferStatus(HpcDataTransferUploadStatus.STREAMING_IN_PROGRESS); + uploadResponse.setSourceURL(sourceURL); + uploadResponse.setSourceSize(size); + uploadResponse.setDataTransferMethod(uploadMethod); + + return uploadResponse; + } + + /** + * Generate an upload request URL. + * + * @param authenticatedToken An authenticated token. + * @param archiveDestinationLocation The archive destination location. + * @param uploadRequestURLExpiration The URL expiration (in hours). + * @param metadataEntries The metadata entries to attach to the + * data-object in S3 archive. + * @param uploadRequestURLChecksum An optional user provided checksum value to + * attach to the generated url. + * @param storageClass (Optional) The storage class to upload the + * file. + * @param uploadCompletion An indicator whether the user will call an + * API to complete the registration once the + * file is uploaded via the generated URL. + * @return A data object upload response containing the upload request URL. + * @throws HpcException on data transfer system failure. + */ + private HpcDataObjectUploadResponse generateUploadRequestURL(Object authenticatedToken, + HpcFileLocation archiveDestinationLocation, int uploadRequestURLExpiration, + List metadataEntries, String uploadRequestURLChecksum, String storageClass, + boolean uploadCompletion) throws HpcException { + PutObjectRequest objectRequest = PutObjectRequest.builder() + .bucket(archiveDestinationLocation.getFileContainerId()).key(archiveDestinationLocation.getFileId()) + // .metadata(toS3Metadata(metadataEntries)) - TODO: setting metadata on the URL + // cause Cloudian upload w/ URL to fail. + .storageClass(storageClass).contentMD5(uploadRequestURLChecksum).build(); + PutObjectPresignRequest presignRequest = PutObjectPresignRequest.builder() + .signatureDuration(Duration.ofHours(uploadRequestURLExpiration)).putObjectRequest(objectRequest) + .build(); + + PresignedPutObjectRequest presignedRequest = null; + URL url = null; + + // Generate the upload pre-signed upload URL. + try { + presignedRequest = s3Connection.getPresigner(authenticatedToken).presignPutObject(presignRequest); + url = presignedRequest.url(); + + } catch (SdkException e) { + throw new HpcException("[S3] Failed to create a pre-signed URL", HpcErrorType.DATA_TRANSFER_ERROR, + s3Connection.getS3Provider(authenticatedToken), e); + } + + // Create and populate the response object. + HpcDataObjectUploadResponse uploadResponse = new HpcDataObjectUploadResponse(); + uploadResponse.setArchiveLocation(archiveDestinationLocation); + uploadResponse.setDataTransferType(HpcDataTransferType.S_3); + uploadResponse.setDataTransferStarted(Calendar.getInstance()); + uploadResponse.setDataTransferCompleted(null); + uploadResponse.setDataTransferRequestId(String.valueOf(presignedRequest.hashCode())); + uploadResponse.setUploadRequestURL(url.toString()); + uploadResponse.setDataTransferStatus(HpcDataTransferUploadStatus.URL_GENERATED); + uploadResponse + .setDataTransferMethod(uploadCompletion ? HpcDataTransferUploadMethod.URL_SINGLE_PART_WITH_COMPLETION + : HpcDataTransferUploadMethod.URL_SINGLE_PART); + + return uploadResponse; + } + + /** + * Generate upload request multipart URLs. + * + * @param authenticatedToken An authenticated token. + * @param archiveDestinationLocation The archive destination location. + * @param uploadRequestURLExpiration The URL expiration (in hours). + * @param uploadParts How many parts to generate upload URLs for. + * @param metadataEntries The metadata entries to attach to the + * data-object in S3 archive. + * @param storageClass (Optional) The storage class to upload the + * file. + * @return A data object upload response containing the upload request URL. + * @throws HpcException on data transfer system failure. + */ + private HpcDataObjectUploadResponse generateMultipartUploadRequestURLs(Object authenticatedToken, + HpcFileLocation archiveDestinationLocation, int uploadRequestURLExpiration, int uploadParts, + List metadataEntries, String storageClass) throws HpcException { + // Initiate the multipart upload. + HpcMultipartUpload multipartUpload = new HpcMultipartUpload(); + CreateMultipartUploadRequest createMultipartUploadRequest = CreateMultipartUploadRequest.builder() + .bucket(archiveDestinationLocation.getFileContainerId()).key(archiveDestinationLocation.getFileId()) + .metadata(toS3Metadata(metadataEntries)).storageClass(storageClass).build(); + + try { + multipartUpload.setId(s3Connection.getClient(authenticatedToken) + .createMultipartUpload(createMultipartUploadRequest).join().uploadId()); + + } catch (CompletionException e) { + throw new HpcException("[S3] Failed to create a multipart upload: " + createMultipartUploadRequest, + HpcErrorType.DATA_TRANSFER_ERROR, s3Connection.getS3Provider(authenticatedToken), e.getCause()); + } + + // Generate the parts pre-signed upload URLs. + UploadPartRequest uploadPartRequest = UploadPartRequest.builder() + .bucket(archiveDestinationLocation.getFileContainerId()).key(archiveDestinationLocation.getFileId()) + .uploadId(multipartUpload.getId()).build(); + for (int partNumber = 1; partNumber <= uploadParts; partNumber++) { + HpcUploadPartURL uploadPartURL = new HpcUploadPartURL(); + uploadPartURL.setPartNumber(partNumber); + + // Create a UploadPartPresignRequest to specify the signature duration + UploadPartPresignRequest uploadPartPresignRequest = UploadPartPresignRequest.builder() + .signatureDuration(Duration.ofHours(uploadRequestURLExpiration)) + .uploadPartRequest(uploadPartRequest.toBuilder().partNumber(partNumber).build()).build(); + + try { + uploadPartURL.setPartUploadRequestURL(s3Connection.getPresigner(authenticatedToken) + .presignUploadPart(uploadPartPresignRequest).url().toString()); + + } catch (SdkException e) { + throw new HpcException("[S3] Failed to create a pre-signed URL for part: " + partNumber, + HpcErrorType.DATA_TRANSFER_ERROR, s3Connection.getS3Provider(authenticatedToken), e); + } + + multipartUpload.getParts().add(uploadPartURL); + } + + // Create and populate the response object. + HpcDataObjectUploadResponse uploadResponse = new HpcDataObjectUploadResponse(); + uploadResponse.setArchiveLocation(archiveDestinationLocation); + uploadResponse.setDataTransferType(HpcDataTransferType.S_3); + uploadResponse.setDataTransferStarted(Calendar.getInstance()); + uploadResponse.setDataTransferCompleted(null); + uploadResponse.setDataTransferRequestId(String.valueOf(createMultipartUploadRequest.hashCode())); + uploadResponse.setMultipartUpload(multipartUpload); + uploadResponse.setDataTransferStatus(HpcDataTransferUploadStatus.URL_GENERATED); + uploadResponse.setDataTransferMethod(HpcDataTransferUploadMethod.URL_MULTI_PART); + + return uploadResponse; + } + + /** + * Convert HPC metadata entries into S3 metadata object + * + * @param metadataEntries The metadata entries to convert + * @return A S3 metadata object + */ + private Map toS3Metadata(List metadataEntries) { + + Map objectMetadata = new HashMap<>(); + if (metadataEntries != null) { + metadataEntries.forEach( + metadataEntry -> objectMetadata.put(metadataEntry.getAttribute(), metadataEntry.getValue())); + } + + return objectMetadata; + } + + /** + * Download a data object to a local file. + * + * @param authenticatedToken An authenticated token. + * @param archiveLocation The data object archive location. + * @param destinationLocation The local file destination. + * @param progressListener (Optional) a progress listener for async + * notification on transfer completion. + * @return A data transfer request Id. + * @throws HpcException on data transfer system failure. + */ + private String downloadDataObject(Object authenticatedToken, HpcFileLocation archiveLocation, + File destinationLocation, HpcDataTransferProgressListener progressListener) throws HpcException { + // Create a S3 download request. + DownloadFileRequest.Builder downloadFileRequestBuilder = DownloadFileRequest.builder() + .getObjectRequest(b -> b.bucket(archiveLocation.getFileContainerId()).key(archiveLocation.getFileId())) + .destination(destinationLocation); + if (progressListener != null) { + downloadFileRequestBuilder.addTransferListener(new HpcS3ProgressListener(progressListener, + "download from " + archiveLocation.getFileContainerId() + ":" + archiveLocation.getFileId())); + } + + FileDownload downloadFile = null; + try { + downloadFile = s3Connection.getTransferManager(authenticatedToken) + .downloadFile(downloadFileRequestBuilder.build()); + + if (progressListener == null) { + // Download synchronously. + downloadFile.completionFuture().join(); + } else { + progressListener.setCompletableFuture(downloadFile.completionFuture()); + } + + } catch (CompletionException | SdkException e) { + throw new HpcException("[S3] Failed to download file: [" + e.getCause().getMessage() + "]", + HpcErrorType.DATA_TRANSFER_ERROR, s3Connection.getS3Provider(authenticatedToken), e.getCause()); + + } + + return String.valueOf(downloadFile.hashCode()); + } + + /** + * Download a data object to S3 destination (either AWS or 3rd Party Provider). + * + * @param authenticatedToken An authenticated token. + * @param archiveLocation The data object archive location. + * @param archiveLocationURL (Optional) The data object archive location + * URL. + * @param baseArchiveDestination The archive's base destination location. + * @param s3Destination The S3 destination. + * @param progressListener (Optional) a progress listener for async + * notification on transfer completion. + * @param fileSize The size of the file to download. + * @return A data transfer request Id. + * @throws HpcException on data transfer failure. + */ + private String downloadDataObject(Object authenticatedToken, HpcFileLocation archiveLocation, + String archiveLocationURL, HpcArchive baseArchiveDestination, HpcS3DownloadDestination s3Destination, + HpcDataTransferProgressListener progressListener, long fileSize) throws HpcException { + // Authenticate the S3 account. + Object s3AccountAuthenticatedToken = s3Connection.authenticate(s3Destination.getAccount()); + + // Confirm the S3 bucket is accessible. + boolean s3BucketAccessible = true; + try { + s3BucketAccessible = getPathAttributes(s3AccountAuthenticatedToken, s3Destination.getDestinationLocation(), + false).getIsAccessible(); + } catch (HpcException e) { + s3BucketAccessible = false; + logger.error("Failed to get S3 path attributes: " + e.getMessage(), e); + } + if (!s3BucketAccessible) { + throw new HpcException( + "Failed to access AWS S3 bucket: " + s3Destination.getDestinationLocation().getFileContainerId(), + HpcErrorType.INVALID_REQUEST_INPUT); + } + + String sourceURL = null; + long size = fileSize; + if (StringUtils.isEmpty(archiveLocationURL)) { + // Downloading from S3 archive -> S3 destination. + sourceURL = generateDownloadRequestURL(authenticatedToken, archiveLocation, baseArchiveDestination, + S3_STREAM_EXPIRATION); + if (size == 0) { + size = getPathAttributes(authenticatedToken, archiveLocation, true).getSize(); + } + } else { + // Downloading from POSIX archive -> S3 destination. + sourceURL = archiveLocationURL; + if (size == 0) { + try { + size = Files.size(Paths.get(URI.create(archiveLocationURL))); + } catch (IOException e) { + throw new HpcException( + "Failed to determine data object size in a POSIX archive: " + archiveLocationURL, + HpcErrorType.UNEXPECTED_ERROR); + } + } + } + + // Use AWS transfer manager to download the file. + return downloadDataObject(s3AccountAuthenticatedToken, sourceURL, s3Destination.getDestinationLocation(), size, + progressListener); + } + + /** + * Download a data object to a user's AWS / S3 Provider by using AWS transfer + * Manager. + * + * @param s3AccountAuthenticatedToken An authenticated token to the user's AWS + * S3 account. + * @param sourceURL The download source URL. + * @param destinationLocation The destination location. + * @param fileSize The size of the file to download. + * @param progressListener A progress listener for async notification + * on transfer completion. + * @return A data transfer request Id. + * @throws HpcException on data transfer failure. + */ + private String downloadDataObject(Object s3AccountAuthenticatedToken, String sourceURL, + HpcFileLocation destinationLocation, long fileSize, HpcDataTransferProgressListener progressListener) + throws HpcException { + if (progressListener == null) { + throw new HpcException("[S3] No progress listener provided for a download to AWS S3 destination", + HpcErrorType.UNEXPECTED_ERROR); + } + + CompletableFuture s3TransferManagerDownloadFuture = CompletableFuture.runAsync(() -> { + try { + // Create source URL and open a connection to it. + InputStream sourceInputStream = new URL(sourceURL).openStream(); + String sourceDestinationLogMessage = "download to " + destinationLocation.getFileContainerId() + ":" + + destinationLocation.getFileId(); + HpcS3ProgressListener listener = new HpcS3ProgressListener(progressListener, + sourceDestinationLogMessage); + + // Create a S3 upload request. + BlockingInputStreamAsyncRequestBody body = AsyncRequestBody.forBlockingInputStream(fileSize); + Upload streamUpload = s3Connection.getTransferManager(s3AccountAuthenticatedToken) + .upload(builder -> builder + .putObjectRequest(request -> request.bucket(destinationLocation.getFileContainerId()) + .key(destinationLocation.getFileId())) + .requestBody(body).addTransferListener(listener)); + + logger.info("S3 download Archive->AWS/S3 Provider [{}] started. Source size - {} bytes", + sourceDestinationLogMessage, fileSize); + + // Stream the data. + body.writeInputStream(sourceInputStream); + progressListener.setCompletableFuture(streamUpload.completionFuture()); + streamUpload.completionFuture().join(); + + } catch (CompletionException | HpcException | IOException e) { + logger.error("[S3] Failed to download to S3 destination: " + e.getCause().getMessage(), e); + progressListener.transferFailed(e.getCause().getMessage()); + + } + + }, s3Executor); + + return String.valueOf(s3TransferManagerDownloadFuture.hashCode()); + } + + /** + * Check if a path is a directory on S3 bucket. + * + * @param authenticatedToken An authenticated token to S3. + * @param fileLocation the file location. + * @return true if it's a directory, or false otherwise. + * @throws HpcException on failure invoke AWS S3 api. + */ + private boolean isDirectory(Object authenticatedToken, HpcFileLocation fileLocation) throws HpcException { + try { + try { // Check if this is a directory. Use V2 listObjects API. + ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request.builder() + .bucket(fileLocation.getFileContainerId()).prefix(fileLocation.getFileId() + "/").build(); + + ListObjectsV2Response listObjectsV2Response = s3Connection.getClient(authenticatedToken) + .listObjectsV2(listObjectsV2Request).join(); + + return listObjectsV2Response.keyCount() > 0 || !listObjectsV2Response.contents().isEmpty(); + + } catch (CompletionException e) { + if (e.getCause() instanceof SdkServiceException + && ((SdkServiceException) e.getCause()).statusCode() == 400) { // V2 not supported. Use V1 + // listObjects API. + ListObjectsRequest listObjectsRequest = ListObjectsRequest.builder() + .bucket(fileLocation.getFileContainerId()).prefix(fileLocation.getFileId()).build(); + + return !s3Connection.getClient(authenticatedToken).listObjects(listObjectsRequest).join().contents() + .isEmpty(); + } else { + throw e; + } + } + + } catch (CompletionException e) { + throw new HpcException("[S3] Failed to list object: " + e.getCause().getMessage(), + HpcErrorType.DATA_TRANSFER_ERROR, s3Connection.getS3Provider(authenticatedToken), e.getCause()); + } + } +} diff --git a/src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/s3/v3/impl/HpcS3Connection.java b/src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/s3/v3/impl/HpcS3Connection.java new file mode 100644 index 0000000000..9f3f8684a6 --- /dev/null +++ b/src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/s3/v3/impl/HpcS3Connection.java @@ -0,0 +1,322 @@ +/** + * HpcS3Connection.java + * + *

+ * Copyright SVG, Inc. Copyright Leidos Biomedical Research, Inc + * + *

+ * Distributed under the OSI-approved BSD 3-Clause License. See + * http://ncip.github.com/HPC/LICENSE.txt for details. + */ +package gov.nih.nci.hpc.integration.s3.v3.impl; + +import gov.nih.nci.hpc.domain.datatransfer.HpcS3Account; +import gov.nih.nci.hpc.domain.error.HpcErrorType; +import gov.nih.nci.hpc.domain.user.HpcIntegratedSystem; +import gov.nih.nci.hpc.domain.user.HpcIntegratedSystemAccount; +import gov.nih.nci.hpc.exception.HpcException; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.exception.SdkException; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.S3Configuration; +import software.amazon.awssdk.services.s3.presigner.S3Presigner; +import software.amazon.awssdk.transfer.s3.S3TransferManager; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * HPC S3 Connection. + * + * @author Eran Rosenberg + */ +public class HpcS3Connection { + // ---------------------------------------------------------------------// + // Constants + // ---------------------------------------------------------------------// + + // 5GB in bytes + private static final long FIVE_GB = 5368709120L; + + // Google Storage S3 URL. + private static final String GOOGLE_STORAGE_URL = "https://storage.googleapis.com"; + + // ---------------------------------------------------------------------// + // Instance members + // ---------------------------------------------------------------------// + + // A list of S3 3rd party providers that require connection w/ path-style + // enabled. + private Set pathStyleAccessEnabledProviders = new HashSet<>(); + + // The multipart upload minimum part size. + @Value("${hpc.integration.s3.minimumUploadPartSize}") + private Long minimumUploadPartSize = null; + + // The multipart upload threshold. + @Value("${hpc.integration.s3.multipartUploadThreshold}") + private Long multipartUploadThreshold = null; + + // The executor service to be used by AWSTransferManager + private ExecutorService executorService = null; + + // The logger instance. + private final Logger logger = LoggerFactory.getLogger(getClass().getName()); + + // ---------------------------------------------------------------------// + // Constructors + // ---------------------------------------------------------------------// + + /** + * Constructor for Spring Dependency Injection. + * + * @param pathStyleAccessEnabledProviders A list of S3 3rd party providers that + * require connection w/ path-style + * enabled. + * @param awsTransferManagerThreadPoolSize The thread pool size to be used for + * AWS transfer manager + */ + private HpcS3Connection(String pathStyleAccessEnabledProviders, int awsTransferManagerThreadPoolSize) { + for (String s3Provider : pathStyleAccessEnabledProviders.split(",")) { + this.pathStyleAccessEnabledProviders.add(HpcIntegratedSystem.fromValue(s3Provider)); + } + + // Instantiate the executor service for AWS transfer manager. + executorService = Executors.newFixedThreadPool(awsTransferManagerThreadPoolSize, + Executors.defaultThreadFactory()); + } + + // ---------------------------------------------------------------------// + // Methods + // ---------------------------------------------------------------------// + + /** + * Authenticate a (system) data transfer account to S3 (AWS or 3rd Party + * Provider) + * + * @param dataTransferAccount A data transfer account to authenticate. + * @param s3URLorRegion The S3 URL if authenticating with a 3rd party S3 + * Provider (Cleversafe, Cloudian, etc), or Region if + * authenticating w/ AWS. + * @return An authenticated TransferManager object, or null if authentication + * failed. + * @throws HpcException if authentication failed + */ + public Object authenticate(HpcIntegratedSystemAccount dataTransferAccount, String s3URLorRegion) + throws HpcException { + if (dataTransferAccount.getIntegratedSystem().equals(HpcIntegratedSystem.AWS)) { + return authenticateAWS(dataTransferAccount.getUsername(), dataTransferAccount.getPassword(), s3URLorRegion); + } else { + // Determine if this S3 provider require path-style enabled. + boolean pathStyleAccessEnabled = pathStyleAccessEnabledProviders + .contains(dataTransferAccount.getIntegratedSystem()); + + return authenticateS3Provider(dataTransferAccount.getUsername(), dataTransferAccount.getPassword(), + s3URLorRegion, pathStyleAccessEnabled, dataTransferAccount.getIntegratedSystem()); + } + } + + /** + * Authenticate a (user) S3 account (AWS or 3rd Party Provider) + * + * @param s3Account AWS S3 account. + * @return TransferManager + * @throws HpcException if authentication failed + */ + public Object authenticate(HpcS3Account s3Account) throws HpcException { + if (!StringUtils.isEmpty(s3Account.getRegion())) { + return authenticateAWS(s3Account.getAccessKey(), s3Account.getSecretKey(), s3Account.getRegion()); + + } else { + // Default S3 provider require path-style enabled to true if not provided by the + // user. + boolean pathStyleAccessEnabled = Optional.ofNullable(s3Account.getPathStyleAccessEnabled()).orElse(true); + + return authenticateS3Provider(s3Account.getAccessKey(), s3Account.getSecretKey(), s3Account.getUrl(), + pathStyleAccessEnabled, HpcIntegratedSystem.USER_S_3_PROVIDER); + } + } + + /** + * Get S3 Transfer Manager from an authenticated token. + * + * @param authenticatedToken An authenticated token. + * @return A transfer manager object. + * @throws HpcException on invalid authentication token. + */ + public S3TransferManager getTransferManager(Object authenticatedToken) throws HpcException { + if (!(authenticatedToken instanceof HpcS3)) { + throw new HpcException("Invalid S3 authentication token", HpcErrorType.INVALID_REQUEST_INPUT); + } + + return ((HpcS3) authenticatedToken).transferManager; + } + + /** + * Get S3 Client from an authenticated token. + * + * @param authenticatedToken An authenticated token. + * @return A S3 client object. + * @throws HpcException on invalid authentication token. + */ + public S3AsyncClient getClient(Object authenticatedToken) throws HpcException { + if (!(authenticatedToken instanceof HpcS3)) { + throw new HpcException("Invalid S3 authentication token", HpcErrorType.INVALID_REQUEST_INPUT); + } + + return ((HpcS3) authenticatedToken).client; + } + + /** + * Get S3 Presigner from an authenticated token. + * + * @param authenticatedToken An authenticated token. + * @return A S3 presigner object. + * @throws HpcException on invalid authentication token. + */ + public S3Presigner getPresigner(Object authenticatedToken) throws HpcException { + if (!(authenticatedToken instanceof HpcS3)) { + throw new HpcException("Invalid S3 authentication token", HpcErrorType.INVALID_REQUEST_INPUT); + } + + return ((HpcS3) authenticatedToken).presigner; + } + + /** + * Get S3 Provider from an authenticated token. + * + * @param authenticatedToken An authenticated token. + * @return A transfer manager object. + * @throws HpcException on invalid authentication token. + */ + public HpcIntegratedSystem getS3Provider(Object authenticatedToken) throws HpcException { + if (!(authenticatedToken instanceof HpcS3)) { + return null; + } + + return ((HpcS3) authenticatedToken).provider; + } + + // ---------------------------------------------------------------------// + // Helper Methods + // ---------------------------------------------------------------------// + + private class HpcS3 { + private S3TransferManager transferManager = null; + private S3AsyncClient client = null; + private S3Presigner presigner = null; + private HpcIntegratedSystem provider = null; + } + + /** + * Authenticate a 'S3 3rd Party Provider' account. + * + * @param username The S3 account user name. + * @param password The S3 account password. + * @param url The S3 3rd party provider URL. + * @param pathStyleAccessEnabled true if the S3 3rd Party provider supports path + * style access. + * @param s3Provider The 3rd party provider. + * @return HpcS3 instance + * @throws HpcException if authentication failed + */ + private Object authenticateS3Provider(String username, String password, String url, boolean pathStyleAccessEnabled, + HpcIntegratedSystem s3Provider) throws HpcException { + // Create the credential provider based on the configured credentials. + AwsBasicCredentials s3ProviderCredentials = AwsBasicCredentials.create(username, password); + StaticCredentialsProvider s3ProviderCredentialsProvider = StaticCredentialsProvider + .create(s3ProviderCredentials); + + // Create URI to the S3 provider endpoint + URI uri = null; + try { + uri = new URI(url); + + } catch (URISyntaxException e) { + throw new HpcException("Invalid URL: " + url, HpcErrorType.DATA_TRANSFER_ERROR, e); + } + + HpcS3 s3 = new HpcS3(); + s3.provider = s3Provider; + + try { + // Instantiate a S3 async client. + s3.client = S3AsyncClient.builder() + .credentialsProvider(s3ProviderCredentialsProvider) + .forcePathStyle(pathStyleAccessEnabled) + .endpointOverride(uri) + .multipartConfiguration(mpConfigBuilder -> mpConfigBuilder + .minimumPartSizeInBytes(minimumUploadPartSize) + .thresholdInBytes(url.equalsIgnoreCase(GOOGLE_STORAGE_URL) ? FIVE_GB : multipartUploadThreshold)) + .build(); + // Instantiate the S3 transfer manager. + s3.transferManager = S3TransferManager.builder().s3Client(s3.client).executor(executorService).build(); + + // Instantiate the S3 presigner. + s3.presigner = S3Presigner.builder().credentialsProvider(s3ProviderCredentialsProvider) + .endpointOverride(uri).serviceConfiguration( + S3Configuration.builder().pathStyleAccessEnabled(pathStyleAccessEnabled).build()) + .build(); + + return s3; + + } catch (SdkException e) { + throw new HpcException( + "[S3] Failed to authenticate S3 Provider: " + s3Provider.value() + "] - " + e.getMessage(), + HpcErrorType.DATA_TRANSFER_ERROR, e); + } + } + + /** + * Authenticate an AWS S3 account. + * + * @param accessKey The AWS account access key. + * @param secretKey The AWS account secret key. + * @param region The AWS account region. + * @return TransferManager + * @throws HpcException if authentication failed + */ + private Object authenticateAWS(String accessKey, String secretKey, String region) throws HpcException { + // Create the credential provider based on provided AWS S3 account. + AwsBasicCredentials awsCredentials = AwsBasicCredentials.create(accessKey, secretKey); + StaticCredentialsProvider awsCredentialsProvider = StaticCredentialsProvider.create(awsCredentials); + + HpcS3 s3 = new HpcS3(); + s3.provider = HpcIntegratedSystem.AWS; + + try { + // Instantiate a S3 async client. + s3.client = S3AsyncClient.builder() + .credentialsProvider(awsCredentialsProvider) + .region(Region.of(region)) + .multipartConfiguration(mpConfigBuilder -> mpConfigBuilder + .minimumPartSizeInBytes(minimumUploadPartSize) + .thresholdInBytes(multipartUploadThreshold)) + .build(); + + // Instantiate the S3 transfer manager. + s3.transferManager = S3TransferManager.builder().s3Client(s3.client).executor(executorService).build(); + + // Instantiate the S3 presigner. + s3.presigner = S3Presigner.builder().credentialsProvider(awsCredentialsProvider).region(Region.of(region)) + .build(); + + return s3; + + } catch (SdkException e) { + throw new HpcException("[S3] Failed to authenticate S3 in region " + region + "] - " + e.getMessage(), + HpcErrorType.DATA_TRANSFER_ERROR, e); + } + } +} diff --git a/src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/s3/v3/impl/HpcS3ProgressListener.java b/src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/s3/v3/impl/HpcS3ProgressListener.java new file mode 100644 index 0000000000..6020a62c91 --- /dev/null +++ b/src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/s3/v3/impl/HpcS3ProgressListener.java @@ -0,0 +1,129 @@ +/** + * HpcS3ProgressListener.java + * + *

Copyright SVG, Inc. Copyright Leidos Biomedical Research, Inc + * + *

Distributed under the OSI-approved BSD 3-Clause License. See + * http://ncip.github.com/HPC/LICENSE.txt for details. + */ +package gov.nih.nci.hpc.integration.s3.v3.impl; + +import java.util.concurrent.atomic.AtomicLong; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import gov.nih.nci.hpc.domain.error.HpcErrorType; +import gov.nih.nci.hpc.exception.HpcException; +import gov.nih.nci.hpc.integration.HpcDataTransferProgressListener; +import software.amazon.awssdk.transfer.s3.progress.TransferListener; + +/** + * HPC S3 Progress Listener. + * + * @author Eran Rosenberg + */ +public class HpcS3ProgressListener implements TransferListener { + // ---------------------------------------------------------------------// + // Constants + // ---------------------------------------------------------------------// + + // The transfer progress report rate (in bytes). Log and send transfer progress + // notifications every 100MB. + private static final long TRANSFER_PROGRESS_REPORTING_RATE = 1024L * 1024 * 100; + private static final long MB = 1024L * 1024; + + // ---------------------------------------------------------------------// + // Instance members + // ---------------------------------------------------------------------// + + // HPC progress listener + private HpcDataTransferProgressListener progressListener = null; + + // Bytes transferred and logged. + private AtomicLong bytesTransferred = new AtomicLong(0); + private long bytesTransferredReported = 0; + + // Transfer source and/or destination (for logging purposed) + private String transferSourceDestination = null; + + // Logger + private final Logger logger = LoggerFactory.getLogger(getClass().getName()); + + // ---------------------------------------------------------------------// + // Constructors + // ---------------------------------------------------------------------// + + /** + * Constructor. + * + * @param progressListener The HPC progress listener. + * @param transferSourceDestination The transfer source and destination (for + * logging progress). + * @throws HpcException if no progress listener provided. + */ + public HpcS3ProgressListener(HpcDataTransferProgressListener progressListener, String transferSourceDestination) + throws HpcException { + if (progressListener == null) { + throw new HpcException("Null progress listener", HpcErrorType.UNEXPECTED_ERROR); + } + this.progressListener = progressListener; + this.transferSourceDestination = transferSourceDestination; + } + + /** + * Default Constructor. + * + * @throws HpcException Constructor is disabled. + */ + @SuppressWarnings("unused") + private HpcS3ProgressListener() throws HpcException { + throw new HpcException("Constructor Disabled", HpcErrorType.UNEXPECTED_ERROR); + } + + // ---------------------------------------------------------------------// + // Methods + // ---------------------------------------------------------------------// + + // ---------------------------------------------------------------------// + // ProgressListener Interface Implementation + // ---------------------------------------------------------------------// + + @Override + public void transferInitiated(TransferListener.Context.TransferInitiated context) { + bytesTransferred.getAndSet(context.progressSnapshot().transferredBytes()); + bytesTransferredReported = bytesTransferred.get(); + logger.info("S3 transfer [{}] started. {} bytes transferred so far", transferSourceDestination, + bytesTransferredReported); + } + + @Override + public void bytesTransferred(TransferListener.Context.BytesTransferred context) { + bytesTransferred.getAndSet(context.progressSnapshot().transferredBytes()); + + if (bytesTransferred.get() - bytesTransferredReported >= TRANSFER_PROGRESS_REPORTING_RATE) { + bytesTransferredReported = bytesTransferred.get(); + logger.info("S3 transfer [{}] in progress. {}MB transferred so far", transferSourceDestination, + bytesTransferredReported / MB); + + progressListener.transferProgressed(bytesTransferredReported); + } + } + + @Override + public void transferComplete(TransferListener.Context.TransferComplete context) { + bytesTransferred.getAndSet(context.progressSnapshot().transferredBytes()); + + logger.info("S3 transfer [{}] completed. {} bytes transferred", transferSourceDestination, bytesTransferred.get()); + progressListener.transferCompleted(bytesTransferred.get()); + } + + @Override + public void transferFailed(TransferListener.Context.TransferFailed context) { + bytesTransferred.getAndSet(context.progressSnapshot().transferredBytes()); + + logger.error("S3 transfer [{}] failed. {}MB transferred.", transferSourceDestination, + bytesTransferred.get() / MB, context.exception()); + progressListener.transferFailed("S3 transfer failed: " + context.exception().getMessage()); + } +} From 2b1a4100fd29c23a5a4e2765a5f6c4109eb3cf58 Mon Sep 17 00:00:00 2001 From: Eran Rosenberg Date: Sun, 7 Dec 2025 15:20:47 -0500 Subject: [PATCH 8/8] Revert "Start of V3 version of S3 proxy impl" This reverts commit fcabe83c3709c9115d953ce3410fd1b3d396dcd6. --- .../s3/v3/impl/HpcDataTransferProxyImpl.java | 1220 ----------------- .../s3/v3/impl/HpcS3Connection.java | 322 ----- .../s3/v3/impl/HpcS3ProgressListener.java | 129 -- 3 files changed, 1671 deletions(-) delete mode 100644 src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/s3/v3/impl/HpcDataTransferProxyImpl.java delete mode 100644 src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/s3/v3/impl/HpcS3Connection.java delete mode 100644 src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/s3/v3/impl/HpcS3ProgressListener.java diff --git a/src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/s3/v3/impl/HpcDataTransferProxyImpl.java b/src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/s3/v3/impl/HpcDataTransferProxyImpl.java deleted file mode 100644 index 80d8ead7ae..0000000000 --- a/src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/s3/v3/impl/HpcDataTransferProxyImpl.java +++ /dev/null @@ -1,1220 +0,0 @@ -package gov.nih.nci.hpc.integration.s3.v3.impl; - -import static gov.nih.nci.hpc.integration.HpcDataTransferProxy.getArchiveDestinationLocation; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.UnsupportedEncodingException; -import java.net.URI; -import java.net.URL; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.text.DateFormat; -import java.text.SimpleDateFormat; -import java.time.Duration; -import java.util.ArrayList; -import java.util.Calendar; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionException; -import java.util.concurrent.Executor; - -import org.apache.commons.io.FilenameUtils; -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.beans.factory.annotation.Value; - -import gov.nih.nci.hpc.domain.datamanagement.HpcPathAttributes; -import gov.nih.nci.hpc.domain.datatransfer.HpcArchive; -import gov.nih.nci.hpc.domain.datatransfer.HpcArchiveObjectMetadata; -import gov.nih.nci.hpc.domain.datatransfer.HpcArchiveType; -import gov.nih.nci.hpc.domain.datatransfer.HpcDataObjectDownloadRequest; -import gov.nih.nci.hpc.domain.datatransfer.HpcDataTransferType; -import gov.nih.nci.hpc.domain.datatransfer.HpcDataTransferUploadMethod; -import gov.nih.nci.hpc.domain.datatransfer.HpcDataTransferUploadStatus; -import gov.nih.nci.hpc.domain.datatransfer.HpcDeepArchiveStatus; -import gov.nih.nci.hpc.domain.datatransfer.HpcDirectoryScanItem; -import gov.nih.nci.hpc.domain.datatransfer.HpcFileLocation; -import gov.nih.nci.hpc.domain.datatransfer.HpcMultipartUpload; -import gov.nih.nci.hpc.domain.datatransfer.HpcS3Account; -import gov.nih.nci.hpc.domain.datatransfer.HpcS3DownloadDestination; -import gov.nih.nci.hpc.domain.datatransfer.HpcSetArchiveObjectMetadataResponse; -import gov.nih.nci.hpc.domain.datatransfer.HpcStreamingUploadSource; -import gov.nih.nci.hpc.domain.datatransfer.HpcUploadPartETag; -import gov.nih.nci.hpc.domain.datatransfer.HpcUploadPartURL; -import gov.nih.nci.hpc.domain.error.HpcErrorType; -import gov.nih.nci.hpc.domain.metadata.HpcMetadataEntry; -import gov.nih.nci.hpc.domain.model.HpcDataObjectUploadRequest; -import gov.nih.nci.hpc.domain.model.HpcDataObjectUploadResponse; -import gov.nih.nci.hpc.domain.user.HpcIntegratedSystemAccount; -import gov.nih.nci.hpc.exception.HpcException; -import gov.nih.nci.hpc.integration.HpcDataTransferProgressListener; -import gov.nih.nci.hpc.integration.HpcDataTransferProxy; -import software.amazon.awssdk.awscore.AwsRequestOverrideConfiguration; -import software.amazon.awssdk.core.async.AsyncRequestBody; -import software.amazon.awssdk.core.async.BlockingInputStreamAsyncRequestBody; -import software.amazon.awssdk.core.exception.SdkException; -import software.amazon.awssdk.core.exception.SdkServiceException; -import software.amazon.awssdk.services.s3.model.BucketLifecycleConfiguration; -import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest; -import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload; -import software.amazon.awssdk.services.s3.model.CompletedPart; -import software.amazon.awssdk.services.s3.model.CopyObjectRequest; -import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; -import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; -import software.amazon.awssdk.services.s3.model.ExpirationStatus; -import software.amazon.awssdk.services.s3.model.GetBucketLifecycleConfigurationResponse; -import software.amazon.awssdk.services.s3.model.GetObjectRequest; -import software.amazon.awssdk.services.s3.model.GlacierJobParameters; -import software.amazon.awssdk.services.s3.model.HeadObjectRequest; -import software.amazon.awssdk.services.s3.model.HeadObjectResponse; -import software.amazon.awssdk.services.s3.model.LifecycleRule; -import software.amazon.awssdk.services.s3.model.ListObjectsRequest; -import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; -import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; -import software.amazon.awssdk.services.s3.model.MetadataDirective; -import software.amazon.awssdk.services.s3.model.NoSuchKeyException; -import software.amazon.awssdk.services.s3.model.PutObjectRequest; -import software.amazon.awssdk.services.s3.model.RestoreObjectRequest; -import software.amazon.awssdk.services.s3.model.RestoreRequest; -import software.amazon.awssdk.services.s3.model.S3Object; -import software.amazon.awssdk.services.s3.model.Tier; -import software.amazon.awssdk.services.s3.model.Transition; -import software.amazon.awssdk.services.s3.model.TransitionStorageClass; -import software.amazon.awssdk.services.s3.model.UploadPartRequest; -import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest; -import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest; -import software.amazon.awssdk.services.s3.presigner.model.PresignedPutObjectRequest; -import software.amazon.awssdk.services.s3.presigner.model.PutObjectPresignRequest; -import software.amazon.awssdk.services.s3.presigner.model.UploadPartPresignRequest; -import software.amazon.awssdk.transfer.s3.model.CompletedCopy; -import software.amazon.awssdk.transfer.s3.model.Copy; -import software.amazon.awssdk.transfer.s3.model.CopyRequest; -import software.amazon.awssdk.transfer.s3.model.DownloadFileRequest; -import software.amazon.awssdk.transfer.s3.model.FileDownload; -import software.amazon.awssdk.transfer.s3.model.FileUpload; -import software.amazon.awssdk.transfer.s3.model.Upload; -import software.amazon.awssdk.transfer.s3.model.UploadFileRequest; - -/** - * HPC Data Transfer Proxy S3 Implementation. - * - * @author Eran Rosenberg - */ -public class HpcDataTransferProxyImpl implements HpcDataTransferProxy { - // ---------------------------------------------------------------------// - // Constants - // ---------------------------------------------------------------------// - - // The expiration of streaming data request from 3rd Party S3 Archive - // (Cleversafe, Cloudian, etc) to AWS S3. - private static final int S3_STREAM_EXPIRATION = 96; - - // Cloudian tiering info header required when adding tiering rule - private static final String CLOUDIAN_TIERING_INFO_HEADER = "x-gmt-tieringinfo"; - - // Number of days the restored data object will be available. - @Value("${hpc.integration.s3.tieringEndpoint}") - private String tieringEndpoint = null; - - // Number of days the restored data object will be available. - @Value("${hpc.integration.s3.restoreNumDays}") - private int restoreNumDays = 2; - - // ---------------------------------------------------------------------// - // Instance members - // ---------------------------------------------------------------------// - - // The S3 connection instance. - @Autowired - private HpcS3Connection s3Connection = null; - - // The S3 executor. - @Autowired - @Qualifier("hpcS3Executor") - Executor s3Executor = null; - - // Date formatter to format files last-modified date - private DateFormat dateFormat = new SimpleDateFormat("MM-dd-yyyy HH:mm:ss"); - - // The logger instance. - private final Logger logger = LoggerFactory.getLogger(getClass().getName()); - - // ---------------------------------------------------------------------// - // Constructors - // ---------------------------------------------------------------------// - - /** Constructor for spring injection. */ - private HpcDataTransferProxyImpl() { - } - - // ---------------------------------------------------------------------// - // Methods - // ---------------------------------------------------------------------// - - // ---------------------------------------------------------------------// - // HpcDataTransferProxy Interface Implementation - // ---------------------------------------------------------------------// - - @Override - public Object authenticate(HpcIntegratedSystemAccount dataTransferAccount, String urlOrRegion, - String encryptionAlgorithm, String encryptionKey) throws HpcException { - return s3Connection.authenticate(dataTransferAccount, urlOrRegion); - } - - @Override - public Object authenticate(HpcS3Account s3Account) throws HpcException { - return s3Connection.authenticate(s3Account); - } - - @Override - public HpcDataObjectUploadResponse uploadDataObject(Object authenticatedToken, - HpcDataObjectUploadRequest uploadRequest, HpcArchive baseArchiveDestination, - Integer uploadRequestURLExpiration, HpcDataTransferProgressListener progressListener, - List metadataEntries, Boolean encryptedTransfer, String storageClass) - throws HpcException { - if (uploadRequest.getGlobusUploadSource() != null) { - throw new HpcException("Invalid upload source", HpcErrorType.UNEXPECTED_ERROR); - } - - // Calculate the archive destination. - HpcFileLocation archiveDestinationLocation = getArchiveDestinationLocation( - baseArchiveDestination.getFileLocation(), uploadRequest.getPath(), uploadRequest.getCallerObjectId(), - baseArchiveDestination.getType(), this, authenticatedToken); - - if (uploadRequest.getGenerateUploadRequestURL()) { - int uploadParts = Optional.ofNullable(uploadRequest.getUploadParts()).orElse(1); - if (uploadParts == 1) { - // Generate an upload request URL for the caller to use to upload directly. - return generateUploadRequestURL(authenticatedToken, archiveDestinationLocation, - uploadRequestURLExpiration, metadataEntries, uploadRequest.getUploadRequestURLChecksum(), - storageClass, Optional.ofNullable(uploadRequest.getUploadCompletion()).orElse(false)); - } else { - return generateMultipartUploadRequestURLs(authenticatedToken, archiveDestinationLocation, - uploadRequestURLExpiration, uploadParts, metadataEntries, storageClass); - } - } else if (uploadRequest.getSourceFile() != null) { - // Upload a file - return uploadDataObject(authenticatedToken, uploadRequest.getSourceFile(), archiveDestinationLocation, - progressListener, baseArchiveDestination.getType(), metadataEntries, storageClass); - } else { - // Upload by streaming from AWS, 3rd Party S3 Provider, Google Drive or Google - // Cloud Storage source. - return uploadDataObject(authenticatedToken, uploadRequest.getS3UploadSource(), - uploadRequest.getGoogleDriveUploadSource(), uploadRequest.getGoogleCloudStorageUploadSource(), - archiveDestinationLocation, baseArchiveDestination, uploadRequest.getSourceSize(), progressListener, - metadataEntries, storageClass); - } - } - - @Override - public String downloadDataObject(Object authenticatedToken, HpcDataObjectDownloadRequest downloadRequest, - HpcArchive baseArchiveDestination, HpcDataTransferProgressListener progressListener, - Boolean encryptedTransfer) throws HpcException { - if (downloadRequest.getArchiveLocation() == null) { - throw new HpcException("Null archive location", HpcErrorType.UNEXPECTED_ERROR); - } - - if (downloadRequest.getFileDestination() != null) { - // This is a download request to a local file. - return downloadDataObject(authenticatedToken, downloadRequest.getArchiveLocation(), - downloadRequest.getFileDestination(), progressListener); - } else { - // This is a download to S3 destination (either AWS or 3rd Party Provider). - return downloadDataObject(authenticatedToken, downloadRequest.getArchiveLocation(), - downloadRequest.getArchiveLocationURL(), baseArchiveDestination, downloadRequest.getS3Destination(), - progressListener, downloadRequest.getSize()); - } - } - - @Override - public String generateDownloadRequestURL(Object authenticatedToken, HpcFileLocation archiveSourceLocation, - HpcArchive baseArchiveDestination, Integer downloadRequestURLExpiration) throws HpcException { - if (archiveSourceLocation == null) { - throw new HpcException("Null archive location", HpcErrorType.UNEXPECTED_ERROR); - } - - try { - GetObjectRequest getObjectRequest = GetObjectRequest.builder() - .bucket(archiveSourceLocation.getFileContainerId()).key(archiveSourceLocation.getFileId()).build(); - - GetObjectPresignRequest getObjectPresignRequest = GetObjectPresignRequest.builder() - .signatureDuration(Duration.ofHours(downloadRequestURLExpiration)) - .getObjectRequest(getObjectRequest).build(); - - PresignedGetObjectRequest presignedGetObjectRequest = s3Connection.getPresigner(authenticatedToken) - .presignGetObject(getObjectPresignRequest); - return presignedGetObjectRequest.url().toString(); - - } catch (SdkException e) { - throw new HpcException( - "[S3] Failed to generate presigned download URL" + archiveSourceLocation.getFileContainerId() + ":" - + archiveSourceLocation.getFileId() + " - " + e.getMessage(), - HpcErrorType.DATA_TRANSFER_ERROR, e); - } - } - - @Override - public HpcSetArchiveObjectMetadataResponse setDataObjectMetadata(Object authenticatedToken, - HpcFileLocation fileLocation, HpcArchive baseArchiveDestination, List metadataEntries, - String storageClass) throws HpcException { - HpcSetArchiveObjectMetadataResponse response = new HpcSetArchiveObjectMetadataResponse(); - - // Check if the metadata was already set on the data-object in the S3 archive. - try { - HeadObjectRequest headObjectRequest = HeadObjectRequest.builder().bucket(fileLocation.getFileContainerId()) - .key(fileLocation.getFileId()).build(); - HeadObjectResponse headObjectResponse = s3Connection.getClient(authenticatedToken) - .headObject(headObjectRequest).join(); - - Map s3Metadata = headObjectResponse.metadata(); - boolean metadataAlreadySet = true; - for (HpcMetadataEntry metadataEntry : metadataEntries) { - if (!s3Metadata.containsKey(metadataEntry.getAttribute())) { - metadataAlreadySet = false; - break; - } - } - - if (metadataAlreadySet) { - logger.info("System metadata in S3 archive already set for [{}]. No need to copy-object in archive", - fileLocation.getFileId()); - response.setChecksum(headObjectResponse.eTag().replace("\"", "")); - response.setMetadataAdded(false); - return response; - } - - } catch (CompletionException e) { - throw new HpcException( - "[S3] Failed to get object metadata: " + fileLocation.getFileContainerId() + ":" - + fileLocation.getFileId() + " - " + e.getCause().getMessage(), - HpcErrorType.DATA_TRANSFER_ERROR, e.getCause()); - } - - // We set S3 metadata by copying the data-object to itself w/ attached metadata. - CopyObjectRequest copyObjectRequest = CopyObjectRequest.builder() - .sourceBucket(fileLocation.getFileContainerId()).sourceKey(fileLocation.getFileId()) - .destinationBucket(fileLocation.getFileContainerId()).destinationKey(fileLocation.getFileId()) - .storageClass(storageClass).metadata(toS3Metadata(metadataEntries)) - .metadataDirective(MetadataDirective.REPLACE).build(); - - CopyRequest copyRequest = CopyRequest.builder().copyObjectRequest(copyObjectRequest).build(); - - try { - Copy copy = s3Connection.getTransferManager(authenticatedToken).copy(copyRequest); - - CompletedCopy completedCopy = copy.completionFuture().join(); - response.setChecksum(completedCopy.response().copyObjectResult().eTag().replace("\"", "")); - response.setMetadataAdded(true); - return response; - - } catch (CompletionException e) { - throw new HpcException("[S3] Failed to copy file: " + copyRequest, HpcErrorType.DATA_TRANSFER_ERROR, - s3Connection.getS3Provider(authenticatedToken), e.getCause()); - } - - } - - @Override - public void deleteDataObject(Object authenticatedToken, HpcFileLocation fileLocation, - HpcArchive baseArchiveDestination) throws HpcException { - - // Create a S3 delete request. - DeleteObjectRequest deleteRequest = DeleteObjectRequest.builder().bucket(fileLocation.getFileContainerId()) - .key(fileLocation.getFileId()).build(); - - try { - s3Connection.getClient(authenticatedToken).deleteObject(deleteRequest).join(); - - } catch (CompletionException e) { - throw new HpcException("[S3] Failed to delete file: " + deleteRequest, HpcErrorType.DATA_TRANSFER_ERROR, - s3Connection.getS3Provider(authenticatedToken), e.getCause()); - } - } - - @Override - public HpcPathAttributes getPathAttributes(Object authenticatedToken, HpcFileLocation fileLocation, boolean getSize) - throws HpcException { - - HpcPathAttributes pathAttributes = new HpcPathAttributes(); - HeadObjectResponse headObjectResponse = null; - - // Look for the file. - try { - pathAttributes.setIsAccessible(true); - HeadObjectRequest headObjectRequest = HeadObjectRequest.builder().bucket(fileLocation.getFileContainerId()) - .key(fileLocation.getFileId()).build(); - headObjectResponse = s3Connection.getClient(authenticatedToken).headObject(headObjectRequest) - .exceptionally(e -> { - Throwable cause = e.getCause(); - if (cause instanceof SdkServiceException) { - if (((SdkServiceException) cause).statusCode() == 403) { - pathAttributes.setIsAccessible(false); - } - } else if (!(cause instanceof NoSuchKeyException)) { - logger.error("[S3] Failed to get head object request: " + cause.getClass().toString() - + " * " + cause.getMessage(), cause); - } - return null; - }).join(); - - } catch (CompletionException e) { - throw new HpcException("[S3] Failed to get head object request: " + e.getCause().getMessage(), - HpcErrorType.DATA_TRANSFER_ERROR, s3Connection.getS3Provider(authenticatedToken), e.getCause()); - - } - - if (!pathAttributes.getIsAccessible()) { - return pathAttributes; - } - if (headObjectResponse != null) { - // This is a file. - pathAttributes.setIsFile(true); - pathAttributes.setIsDirectory(false); - pathAttributes.setExists(true); - - } else { - pathAttributes.setIsFile(false); - - // Check if this is a directory. - boolean directoryExists = isDirectory(authenticatedToken, fileLocation); - pathAttributes.setIsDirectory(directoryExists); - pathAttributes.setExists(directoryExists); - } - - // Optionally get the file size. We currently don't support getting file size - // for a directory. - if (getSize && headObjectResponse != null) { - pathAttributes.setSize(headObjectResponse.contentLength()); - } - - return pathAttributes; - } - - @Override - public List scanDirectory(Object authenticatedToken, HpcFileLocation directoryLocation) - throws HpcException { - List directoryScanItems = new ArrayList<>(); - - try { - ListObjectsV2Request listObjectsRequest = ListObjectsV2Request.builder() - .bucket(directoryLocation.getFileContainerId()).prefix(directoryLocation.getFileId()).build(); - - ListObjectsV2Response listObjectsResponse = s3Connection.getClient(authenticatedToken) - .listObjectsV2(listObjectsRequest).join(); - - List s3Objects = listObjectsResponse.contents(); - - // Paginate through all results. - while (listObjectsResponse.isTruncated()) { - String continuationToken = listObjectsResponse.nextContinuationToken(); - - listObjectsRequest = listObjectsRequest.toBuilder().continuationToken(continuationToken).build(); - listObjectsResponse = s3Connection.getClient(authenticatedToken).listObjectsV2(listObjectsRequest) - .join(); - - if (continuationToken.equals(listObjectsResponse.nextContinuationToken())) { - // Pagination over list objects is not working w/ Cleversafe storage, we keep - // getting the same set of results. This code is to protect against infinite - // loop. - break; - } - - s3Objects.addAll(listObjectsResponse.contents()); - } - - s3Objects.forEach(s3Object -> { - if (s3Object.size() > 0) { - HpcDirectoryScanItem directoryScanItem = new HpcDirectoryScanItem(); - directoryScanItem.setFilePath(s3Object.key()); - directoryScanItem.setFileName(FilenameUtils.getName(s3Object.key())); - directoryScanItem.setLastModified(dateFormat.format(Date.from(s3Object.lastModified()))); - directoryScanItems.add(directoryScanItem); - } - }); - - return directoryScanItems; - - } catch (CompletionException e) { - throw new HpcException("[S3] Failed to list objects: " + e.getCause().getMessage(), - HpcErrorType.DATA_TRANSFER_ERROR, s3Connection.getS3Provider(authenticatedToken), e.getCause()); - } - } - - @Override - public String completeMultipartUpload(Object authenticatedToken, HpcFileLocation archiveLocation, - String multipartUploadId, List uploadPartETags) throws HpcException { - - // Create AWS part ETags from the HPC model. - List parts = new ArrayList<>(); - uploadPartETags.forEach(uploadPartETag -> parts.add(CompletedPart.builder() - .partNumber(uploadPartETag.getPartNumber()).eTag(uploadPartETag.getETag()).build())); - CompletedMultipartUpload completedMultipartUpload = CompletedMultipartUpload.builder().parts(parts).build(); - - CompleteMultipartUploadRequest completeMultipartUploadRequest = CompleteMultipartUploadRequest.builder() - .bucket(archiveLocation.getFileContainerId()).key(archiveLocation.getFileId()) - .uploadId(multipartUploadId).multipartUpload(completedMultipartUpload).build(); - - try { - return s3Connection.getClient(authenticatedToken).completeMultipartUpload(completeMultipartUploadRequest) - .join().eTag(); - - } catch (CompletionException e) { - throw new HpcException( - "[S3] Failed to complete a multipart upload to " + archiveLocation.getFileContainerId() + ":" - + archiveLocation.getFileId() + ". multi-part-upload-id = " + multipartUploadId - + ", number-of-parts = " + uploadPartETags.size() + " - " + e.getMessage(), - HpcErrorType.DATA_TRANSFER_ERROR, s3Connection.getS3Provider(authenticatedToken), e.getCause()); - } - } - - @Override - public void restoreDataObject(Object authenticatedToken, HpcFileLocation archiveLocation) throws HpcException { - try { - // Create and submit a request to restore an object from Glacier. - RestoreRequest restoreRequest = RestoreRequest.builder().days(restoreNumDays) - .glacierJobParameters(GlacierJobParameters.builder().tier(Tier.STANDARD).build()).build(); - - RestoreObjectRequest restoreObjectRequest = RestoreObjectRequest.builder() - .bucket(archiveLocation.getFileContainerId()).key(archiveLocation.getFileId()) - .restoreRequest(restoreRequest).build(); - s3Connection.getClient(authenticatedToken).restoreObject(restoreObjectRequest).join(); - - } catch (CompletionException e) { - throw new HpcException( - "[S3] Failed to restore data object" + archiveLocation.getFileContainerId() + ":" - + archiveLocation.getFileId() + " - " + e.getCause().getMessage(), - HpcErrorType.DATA_TRANSFER_ERROR, s3Connection.getS3Provider(authenticatedToken), e.getCause()); - } - } - - @Override - public boolean existsTieringPolicy(Object authenticatedToken, HpcFileLocation archiveLocation) throws HpcException { - try { - // Retrieve the configuration. - GetBucketLifecycleConfigurationResponse bucketLifeCycleConfigurationResponse = s3Connection - .getClient(authenticatedToken) - .getBucketLifecycleConfiguration(builder -> builder.bucket(archiveLocation.getFileContainerId())) - .join(); - - if (bucketLifeCycleConfigurationResponse != null) { - for (LifecycleRule rule : bucketLifeCycleConfigurationResponse.rules()) { - // Look through filter prefix applied to lifecycle policy - boolean hasTransition = false; - - if (rule.hasTransitions()) { - for (Transition transition : rule.transitions()) { - if (!StringUtils.isEmpty(transition.storageClassAsString())) { - hasTransition = true; - } - } - } - - if (hasTransition && rule.filter() != null && rule.filter().prefix() != null) { - if (archiveLocation.getFileId().contains(rule.filter().prefix())) { - return true; - } - } else if (hasTransition) { - // This is a transition without prefix applies to entire bucket. - return true; - } - } - } - - return false; - - } catch (CompletionException e) { - throw new HpcException( - "[S3] Failed to retrieve life cycle policy on bucket: " + archiveLocation.getFileContainerId() - + "- " + e.getCause().getMessage(), - HpcErrorType.DATA_TRANSFER_ERROR, s3Connection.getS3Provider(authenticatedToken), e.getCause()); - } - } - - @Override - public HpcArchiveObjectMetadata getDataObjectMetadata(Object authenticatedToken, HpcFileLocation fileLocation) - throws HpcException { - HpcArchiveObjectMetadata objectMetadata = new HpcArchiveObjectMetadata(); - String s3ObjectName = fileLocation.getFileContainerId() + ":" + fileLocation.getFileId(); - - // Get metadata for the data-object in the S3 archive. - try { - HeadObjectRequest headObjectRequest = HeadObjectRequest.builder().bucket(fileLocation.getFileContainerId()) - .key(fileLocation.getFileId()).build(); - HeadObjectResponse headObjectResponse = s3Connection.getClient(authenticatedToken) - .headObject(headObjectRequest).join(); - - // x-amz-storage-class is not returned for standard S3 object - logger.info("[S3] Storage class [{}] - {}", s3ObjectName, headObjectResponse.storageClass()); - - if (headObjectResponse.storageClass() != null) { - objectMetadata.setDeepArchiveStatus( - HpcDeepArchiveStatus.fromValue(headObjectResponse.storageClassAsString())); - logger.info("[S3] Deep Archive Status [{}] - {}", s3ObjectName, objectMetadata.getDeepArchiveStatus()); - } - - // Check the restoration status of the object. - String restoreHeader = headObjectResponse.restore(); - logger.info("[S3] Restore Header [{}] - {}", s3ObjectName, restoreHeader); - - if (StringUtils.isEmpty(restoreHeader)) { - // the x-amz-restore header is not present on the response from the service - // (e.g. no restore request has been received). - objectMetadata.setRestorationStatus("not in progress"); - - } else if (restoreHeader.contains("ongoing-request=\"true\"")) { - // the x-amz-restore header is present and has a value of true (e.g. a restore - // request was received and is currently ongoing). - objectMetadata.setRestorationStatus("in progress"); - - } else if (restoreHeader.contains("ongoing-request=\"false\"")) { - // the x-amz-restore header is present and has a value of false (e.g the object - // has been restored and can currently be read from S3). - objectMetadata.setRestorationStatus("success"); - } - logger.info("[S3] Restoration Status [{}] - {}", s3ObjectName, objectMetadata.getRestorationStatus()); - - objectMetadata.setChecksum(headObjectResponse.eTag().replace("\"", "")); - logger.info("[S3] Checksum [{}] - {}", s3ObjectName, objectMetadata.getChecksum()); - - } catch (CompletionException e) { - throw new HpcException( - "[S3] Failed to get object metadata [" + s3ObjectName + "] - " + e.getCause().getMessage(), - HpcErrorType.DATA_TRANSFER_ERROR, s3Connection.getS3Provider(authenticatedToken), e.getCause()); - } - - return objectMetadata; - } - - @SuppressWarnings("deprecation") - @Override - public synchronized void setTieringPolicy(Object authenticatedToken, HpcFileLocation archiveLocation, String prefix, - String tieringBucket, String tieringProtocol) throws HpcException { - - try { - // Create a list of life cycle rules - List lifeCycleRules = new ArrayList<>(); - - // Add the new rule. - Transition transition = Transition.builder().days(0).storageClass(TransitionStorageClass.GLACIER).build(); - lifeCycleRules.add(LifecycleRule.builder().id(prefix).transitions(transition) - .filter(builder -> builder.prefix(prefix)).status(ExpirationStatus.ENABLED).build()); - - // Retrieve the configuration - GetBucketLifecycleConfigurationResponse bucketLifeCycleConfigurationResponse = s3Connection - .getClient(authenticatedToken) - .getBucketLifecycleConfiguration(builder -> builder.bucket(archiveLocation.getFileContainerId())) - .join(); - - // Add the existing rules to the list. - if (bucketLifeCycleConfigurationResponse != null) { - for (LifecycleRule lifeCycleRule : bucketLifeCycleConfigurationResponse.rules()) { - // Rules existing in Cloudian is retrieved with the prefix - // set to the same value as filter. - // Removing since it fails if this value is provided. - lifeCycleRules.add(lifeCycleRule.toBuilder().prefix(null).build()); - } - } - - // Add Cloudian custom tiering header, no impact to AWS S3 requests. - String customHeader = tieringProtocol + "|EndPoint:" - + URLEncoder.encode(tieringEndpoint, StandardCharsets.UTF_8.toString()) + ",TieringBucket:" - + tieringBucket; - String encodedCustomHeader = URLEncoder.encode(customHeader, StandardCharsets.UTF_8.toString()); - - BucketLifecycleConfiguration lifeCycleConfiguration = BucketLifecycleConfiguration.builder() - .rules(lifeCycleRules).build(); - AwsRequestOverrideConfiguration requestOverrideConfiguration = AwsRequestOverrideConfiguration.builder() - .putHeader(CLOUDIAN_TIERING_INFO_HEADER, encodedCustomHeader).build(); - - s3Connection.getClient(authenticatedToken) - .putBucketLifecycleConfiguration(builder -> builder.bucket(archiveLocation.getFileContainerId()) - .lifecycleConfiguration(lifeCycleConfiguration) - .overrideConfiguration(requestOverrideConfiguration)) - .join(); - - } catch (UnsupportedEncodingException e) { - throw new HpcException( - "[S3] Failed to add a new rule to life cycle policy on bucket " - + archiveLocation.getFileContainerId() + ":" + prefix + " - " + e.getMessage(), - HpcErrorType.DATA_TRANSFER_ERROR, s3Connection.getS3Provider(authenticatedToken), e); - - } catch (CompletionException e) { - throw new HpcException( - "[S3] Failed to add a new rule to life cycle policy on bucket " - + archiveLocation.getFileContainerId() + ":" + prefix + " - " + e.getCause().getMessage(), - HpcErrorType.DATA_TRANSFER_ERROR, s3Connection.getS3Provider(authenticatedToken), e.getCause()); - } - } - - @Override - public void shutdown(Object authenticatedToken) throws HpcException { - try { - s3Connection.getTransferManager(authenticatedToken).close(); - s3Connection.getPresigner(authenticatedToken).close(); - s3Connection.getClient(authenticatedToken).close(); - - } catch (Exception e) { - throw new HpcException("[S3] Failed to shutdown AWS TransferManager/Client/Presigner: " + e.getMessage(), - HpcErrorType.DATA_TRANSFER_ERROR, e); - } - } - - // ---------------------------------------------------------------------// - // Helper Methods - // ---------------------------------------------------------------------// - - /** - * Upload a data object file. - * - * @param authenticatedToken An authenticated token. - * @param sourceFile The file to upload. - * @param archiveDestinationLocation The archive destination location. - * @param progressListener (Optional) a progress listener for async - * notification on transfer completion. - * @param archiveType The archive type. - * @param metadataEntries The metadata entries to attach to the - * data-object in S3 archive. - * @param storageClass (Optional) The storage class to upload the - * file. - * @return A data object upload response. - * @throws HpcException on data transfer system failure. - */ - private HpcDataObjectUploadResponse uploadDataObject(Object authenticatedToken, File sourceFile, - HpcFileLocation archiveDestinationLocation, HpcDataTransferProgressListener progressListener, - HpcArchiveType archiveType, List metadataEntries, String storageClass) - throws HpcException { - // Create a S3 upload file request. - HpcS3ProgressListener listener = new HpcS3ProgressListener(progressListener, - "upload staged file [" + sourceFile.getAbsolutePath() + "] to " - + archiveDestinationLocation.getFileContainerId() + ":" - + archiveDestinationLocation.getFileId()); - - UploadFileRequest uploadFileRequest = UploadFileRequest.builder() - .putObjectRequest(b -> b.bucket(archiveDestinationLocation.getFileContainerId()) - .key(archiveDestinationLocation.getFileId()).metadata(toS3Metadata(metadataEntries)) - .storageClass(storageClass)) - .addTransferListener(listener).source(sourceFile).build(); - - // Upload the data. - FileUpload fileUpload = null; - Calendar dataTransferStarted = Calendar.getInstance(); - Calendar dataTransferCompleted = null; - try { - fileUpload = s3Connection.getTransferManager(authenticatedToken).uploadFile(uploadFileRequest); - progressListener.setCompletableFuture(fileUpload.completionFuture()); - fileUpload.completionFuture().join(); - - dataTransferCompleted = Calendar.getInstance(); - - } catch (CompletionException e) { - throw new HpcException("[S3] Failed to upload file.", HpcErrorType.DATA_TRANSFER_ERROR, - s3Connection.getS3Provider(authenticatedToken), e.getCause()); - - } - - // Upload completed. Create and populate the response object. - HpcDataObjectUploadResponse uploadResponse = new HpcDataObjectUploadResponse(); - uploadResponse.setArchiveLocation(archiveDestinationLocation); - uploadResponse.setDataTransferType(HpcDataTransferType.S_3); - uploadResponse.setDataTransferStarted(dataTransferStarted); - uploadResponse.setDataTransferCompleted(dataTransferCompleted); - uploadResponse.setDataTransferRequestId(String.valueOf(fileUpload.completionFuture().hashCode())); - uploadResponse.setSourceSize(sourceFile.length()); - uploadResponse.setDataTransferMethod(HpcDataTransferUploadMethod.SYNC); - if (archiveType.equals(HpcArchiveType.ARCHIVE)) { - uploadResponse.setDataTransferStatus(HpcDataTransferUploadStatus.ARCHIVED); - } else { - uploadResponse.setDataTransferStatus(HpcDataTransferUploadStatus.IN_TEMPORARY_ARCHIVE); - } - - return uploadResponse; - } - - /** - * Upload a data object file from AWS, 3rd Party S3 Provider or Google Drive - * source. - * - * @param authenticatedToken An authenticated token. - * @param s3UploadSource The S3 upload source (AWS or 3rd party - * provider) - * @param googleDriveUploadSource The Google Drive upload source. - * @param googleCloudStorageUploadSource The Google Cloud Storage upload source. - * @param archiveDestinationLocation The archive destination location. - * @param baseArchiveDestination The archive's base destination - * location. - * @param size the size of the file to upload. - * @param progressListener (Optional) a progress listener for - * async notification on transfer - * completion. - * @param metadataEntries The metadata entries to attach to the - * data-object in S3 archive. - * @param storageClass (Optional) The storage class to upload - * the file. - * @return A data object upload response. - * @throws HpcException on data transfer system failure. - */ - private HpcDataObjectUploadResponse uploadDataObject(Object authenticatedToken, - HpcStreamingUploadSource s3UploadSource, HpcStreamingUploadSource googleDriveUploadSource, - HpcStreamingUploadSource googleCloudStorageUploadSource, HpcFileLocation archiveDestinationLocation, - HpcArchive baseArchiveDestination, Long size, HpcDataTransferProgressListener progressListener, - List metadataEntries, String storageClass) throws HpcException { - - if (progressListener == null) { - throw new HpcException( - "[S3] No progress listener provided for a upload from AWS S3 / S3 Provider / Google Drive / Google Cloud Storage", - HpcErrorType.UNEXPECTED_ERROR); - } - if (size == null) { - throw new HpcException( - "[S3] File size not provided for an upload from AWS / S3 Provider / Google Drive / Google Cloud Storage", - HpcErrorType.UNEXPECTED_ERROR); - } - - HpcDataTransferUploadMethod uploadMethod = null; - String sourceURL = null; - HpcFileLocation sourceLocation = null; - - if (s3UploadSource != null) { // Upload by streaming from AWS or S3 Provider. - uploadMethod = HpcDataTransferUploadMethod.S_3; - sourceLocation = s3UploadSource.getSourceLocation(); - - // If not provided, generate a download pre-signed URL for the requested data - // file from AWS // (using the provided S3 account). - sourceURL = StringUtils.isEmpty(s3UploadSource.getSourceURL()) - ? generateDownloadRequestURL(s3Connection.authenticate(s3UploadSource.getAccount()), sourceLocation, - baseArchiveDestination, S3_STREAM_EXPIRATION) - : s3UploadSource.getSourceURL(); - - } else if (googleDriveUploadSource != null) { // Upload by streaming from Google Drive - uploadMethod = HpcDataTransferUploadMethod.GOOGLE_DRIVE; - sourceLocation = googleDriveUploadSource.getSourceLocation(); - - } else if (googleCloudStorageUploadSource != null) { // Upload by streaming from Google Cloud Storage - uploadMethod = HpcDataTransferUploadMethod.GOOGLE_CLOUD_STORAGE; - sourceLocation = googleCloudStorageUploadSource.getSourceLocation(); - - } else { - throw new HpcException("Unexpected upload source", HpcErrorType.UNEXPECTED_ERROR); - } - - final String url = sourceURL; - final String sourceDestinationLogMessage = "upload from " + sourceLocation.getFileContainerId() + ":" - + sourceLocation.getFileId(); - Calendar dataTransferStarted = Calendar.getInstance(); - - CompletableFuture s3TransferManagerUploadFuture = CompletableFuture.runAsync(() -> { - try { - // Open a connection to the input stream of the file to be uploaded. - InputStream sourceInputStream = null; - if (googleDriveUploadSource != null) { - sourceInputStream = googleDriveUploadSource.getSourceInputStream(); - } else if (googleCloudStorageUploadSource != null) { - sourceInputStream = googleCloudStorageUploadSource.getSourceInputStream(); - } else { - sourceInputStream = new URL(url).openStream(); - } - - HpcS3ProgressListener listener = new HpcS3ProgressListener(progressListener, - sourceDestinationLogMessage); - - // Create a S3 upload request. - BlockingInputStreamAsyncRequestBody body = AsyncRequestBody.forBlockingInputStream(size); - Upload streamUpload = s3Connection.getTransferManager(authenticatedToken) - .upload(builder -> builder - .putObjectRequest( - request -> request.bucket(archiveDestinationLocation.getFileContainerId()) - .key(archiveDestinationLocation.getFileId()) - .metadata(toS3Metadata(metadataEntries)).storageClass(storageClass)) - .requestBody(body).addTransferListener(listener)); - - // Stream the data. - body.writeInputStream(sourceInputStream); - progressListener.setCompletableFuture(streamUpload.completionFuture()); - streamUpload.completionFuture().join(); - - } catch (CompletionException | HpcException | IOException e) { - logger.error("[S3] Failed to upload from AWS S3 destination: " + e.getCause().getMessage(), e); - progressListener.transferFailed(e.getCause().getMessage()); - - } - - }, s3Executor); - - // Create and populate the response object. - HpcDataObjectUploadResponse uploadResponse = new HpcDataObjectUploadResponse(); - uploadResponse.setArchiveLocation(archiveDestinationLocation); - uploadResponse.setDataTransferType(HpcDataTransferType.S_3); - uploadResponse.setDataTransferStarted(dataTransferStarted); - uploadResponse.setUploadSource(sourceLocation); - uploadResponse.setDataTransferRequestId(String.valueOf(s3TransferManagerUploadFuture.hashCode())); - uploadResponse.setDataTransferStatus(HpcDataTransferUploadStatus.STREAMING_IN_PROGRESS); - uploadResponse.setSourceURL(sourceURL); - uploadResponse.setSourceSize(size); - uploadResponse.setDataTransferMethod(uploadMethod); - - return uploadResponse; - } - - /** - * Generate an upload request URL. - * - * @param authenticatedToken An authenticated token. - * @param archiveDestinationLocation The archive destination location. - * @param uploadRequestURLExpiration The URL expiration (in hours). - * @param metadataEntries The metadata entries to attach to the - * data-object in S3 archive. - * @param uploadRequestURLChecksum An optional user provided checksum value to - * attach to the generated url. - * @param storageClass (Optional) The storage class to upload the - * file. - * @param uploadCompletion An indicator whether the user will call an - * API to complete the registration once the - * file is uploaded via the generated URL. - * @return A data object upload response containing the upload request URL. - * @throws HpcException on data transfer system failure. - */ - private HpcDataObjectUploadResponse generateUploadRequestURL(Object authenticatedToken, - HpcFileLocation archiveDestinationLocation, int uploadRequestURLExpiration, - List metadataEntries, String uploadRequestURLChecksum, String storageClass, - boolean uploadCompletion) throws HpcException { - PutObjectRequest objectRequest = PutObjectRequest.builder() - .bucket(archiveDestinationLocation.getFileContainerId()).key(archiveDestinationLocation.getFileId()) - // .metadata(toS3Metadata(metadataEntries)) - TODO: setting metadata on the URL - // cause Cloudian upload w/ URL to fail. - .storageClass(storageClass).contentMD5(uploadRequestURLChecksum).build(); - PutObjectPresignRequest presignRequest = PutObjectPresignRequest.builder() - .signatureDuration(Duration.ofHours(uploadRequestURLExpiration)).putObjectRequest(objectRequest) - .build(); - - PresignedPutObjectRequest presignedRequest = null; - URL url = null; - - // Generate the upload pre-signed upload URL. - try { - presignedRequest = s3Connection.getPresigner(authenticatedToken).presignPutObject(presignRequest); - url = presignedRequest.url(); - - } catch (SdkException e) { - throw new HpcException("[S3] Failed to create a pre-signed URL", HpcErrorType.DATA_TRANSFER_ERROR, - s3Connection.getS3Provider(authenticatedToken), e); - } - - // Create and populate the response object. - HpcDataObjectUploadResponse uploadResponse = new HpcDataObjectUploadResponse(); - uploadResponse.setArchiveLocation(archiveDestinationLocation); - uploadResponse.setDataTransferType(HpcDataTransferType.S_3); - uploadResponse.setDataTransferStarted(Calendar.getInstance()); - uploadResponse.setDataTransferCompleted(null); - uploadResponse.setDataTransferRequestId(String.valueOf(presignedRequest.hashCode())); - uploadResponse.setUploadRequestURL(url.toString()); - uploadResponse.setDataTransferStatus(HpcDataTransferUploadStatus.URL_GENERATED); - uploadResponse - .setDataTransferMethod(uploadCompletion ? HpcDataTransferUploadMethod.URL_SINGLE_PART_WITH_COMPLETION - : HpcDataTransferUploadMethod.URL_SINGLE_PART); - - return uploadResponse; - } - - /** - * Generate upload request multipart URLs. - * - * @param authenticatedToken An authenticated token. - * @param archiveDestinationLocation The archive destination location. - * @param uploadRequestURLExpiration The URL expiration (in hours). - * @param uploadParts How many parts to generate upload URLs for. - * @param metadataEntries The metadata entries to attach to the - * data-object in S3 archive. - * @param storageClass (Optional) The storage class to upload the - * file. - * @return A data object upload response containing the upload request URL. - * @throws HpcException on data transfer system failure. - */ - private HpcDataObjectUploadResponse generateMultipartUploadRequestURLs(Object authenticatedToken, - HpcFileLocation archiveDestinationLocation, int uploadRequestURLExpiration, int uploadParts, - List metadataEntries, String storageClass) throws HpcException { - // Initiate the multipart upload. - HpcMultipartUpload multipartUpload = new HpcMultipartUpload(); - CreateMultipartUploadRequest createMultipartUploadRequest = CreateMultipartUploadRequest.builder() - .bucket(archiveDestinationLocation.getFileContainerId()).key(archiveDestinationLocation.getFileId()) - .metadata(toS3Metadata(metadataEntries)).storageClass(storageClass).build(); - - try { - multipartUpload.setId(s3Connection.getClient(authenticatedToken) - .createMultipartUpload(createMultipartUploadRequest).join().uploadId()); - - } catch (CompletionException e) { - throw new HpcException("[S3] Failed to create a multipart upload: " + createMultipartUploadRequest, - HpcErrorType.DATA_TRANSFER_ERROR, s3Connection.getS3Provider(authenticatedToken), e.getCause()); - } - - // Generate the parts pre-signed upload URLs. - UploadPartRequest uploadPartRequest = UploadPartRequest.builder() - .bucket(archiveDestinationLocation.getFileContainerId()).key(archiveDestinationLocation.getFileId()) - .uploadId(multipartUpload.getId()).build(); - for (int partNumber = 1; partNumber <= uploadParts; partNumber++) { - HpcUploadPartURL uploadPartURL = new HpcUploadPartURL(); - uploadPartURL.setPartNumber(partNumber); - - // Create a UploadPartPresignRequest to specify the signature duration - UploadPartPresignRequest uploadPartPresignRequest = UploadPartPresignRequest.builder() - .signatureDuration(Duration.ofHours(uploadRequestURLExpiration)) - .uploadPartRequest(uploadPartRequest.toBuilder().partNumber(partNumber).build()).build(); - - try { - uploadPartURL.setPartUploadRequestURL(s3Connection.getPresigner(authenticatedToken) - .presignUploadPart(uploadPartPresignRequest).url().toString()); - - } catch (SdkException e) { - throw new HpcException("[S3] Failed to create a pre-signed URL for part: " + partNumber, - HpcErrorType.DATA_TRANSFER_ERROR, s3Connection.getS3Provider(authenticatedToken), e); - } - - multipartUpload.getParts().add(uploadPartURL); - } - - // Create and populate the response object. - HpcDataObjectUploadResponse uploadResponse = new HpcDataObjectUploadResponse(); - uploadResponse.setArchiveLocation(archiveDestinationLocation); - uploadResponse.setDataTransferType(HpcDataTransferType.S_3); - uploadResponse.setDataTransferStarted(Calendar.getInstance()); - uploadResponse.setDataTransferCompleted(null); - uploadResponse.setDataTransferRequestId(String.valueOf(createMultipartUploadRequest.hashCode())); - uploadResponse.setMultipartUpload(multipartUpload); - uploadResponse.setDataTransferStatus(HpcDataTransferUploadStatus.URL_GENERATED); - uploadResponse.setDataTransferMethod(HpcDataTransferUploadMethod.URL_MULTI_PART); - - return uploadResponse; - } - - /** - * Convert HPC metadata entries into S3 metadata object - * - * @param metadataEntries The metadata entries to convert - * @return A S3 metadata object - */ - private Map toS3Metadata(List metadataEntries) { - - Map objectMetadata = new HashMap<>(); - if (metadataEntries != null) { - metadataEntries.forEach( - metadataEntry -> objectMetadata.put(metadataEntry.getAttribute(), metadataEntry.getValue())); - } - - return objectMetadata; - } - - /** - * Download a data object to a local file. - * - * @param authenticatedToken An authenticated token. - * @param archiveLocation The data object archive location. - * @param destinationLocation The local file destination. - * @param progressListener (Optional) a progress listener for async - * notification on transfer completion. - * @return A data transfer request Id. - * @throws HpcException on data transfer system failure. - */ - private String downloadDataObject(Object authenticatedToken, HpcFileLocation archiveLocation, - File destinationLocation, HpcDataTransferProgressListener progressListener) throws HpcException { - // Create a S3 download request. - DownloadFileRequest.Builder downloadFileRequestBuilder = DownloadFileRequest.builder() - .getObjectRequest(b -> b.bucket(archiveLocation.getFileContainerId()).key(archiveLocation.getFileId())) - .destination(destinationLocation); - if (progressListener != null) { - downloadFileRequestBuilder.addTransferListener(new HpcS3ProgressListener(progressListener, - "download from " + archiveLocation.getFileContainerId() + ":" + archiveLocation.getFileId())); - } - - FileDownload downloadFile = null; - try { - downloadFile = s3Connection.getTransferManager(authenticatedToken) - .downloadFile(downloadFileRequestBuilder.build()); - - if (progressListener == null) { - // Download synchronously. - downloadFile.completionFuture().join(); - } else { - progressListener.setCompletableFuture(downloadFile.completionFuture()); - } - - } catch (CompletionException | SdkException e) { - throw new HpcException("[S3] Failed to download file: [" + e.getCause().getMessage() + "]", - HpcErrorType.DATA_TRANSFER_ERROR, s3Connection.getS3Provider(authenticatedToken), e.getCause()); - - } - - return String.valueOf(downloadFile.hashCode()); - } - - /** - * Download a data object to S3 destination (either AWS or 3rd Party Provider). - * - * @param authenticatedToken An authenticated token. - * @param archiveLocation The data object archive location. - * @param archiveLocationURL (Optional) The data object archive location - * URL. - * @param baseArchiveDestination The archive's base destination location. - * @param s3Destination The S3 destination. - * @param progressListener (Optional) a progress listener for async - * notification on transfer completion. - * @param fileSize The size of the file to download. - * @return A data transfer request Id. - * @throws HpcException on data transfer failure. - */ - private String downloadDataObject(Object authenticatedToken, HpcFileLocation archiveLocation, - String archiveLocationURL, HpcArchive baseArchiveDestination, HpcS3DownloadDestination s3Destination, - HpcDataTransferProgressListener progressListener, long fileSize) throws HpcException { - // Authenticate the S3 account. - Object s3AccountAuthenticatedToken = s3Connection.authenticate(s3Destination.getAccount()); - - // Confirm the S3 bucket is accessible. - boolean s3BucketAccessible = true; - try { - s3BucketAccessible = getPathAttributes(s3AccountAuthenticatedToken, s3Destination.getDestinationLocation(), - false).getIsAccessible(); - } catch (HpcException e) { - s3BucketAccessible = false; - logger.error("Failed to get S3 path attributes: " + e.getMessage(), e); - } - if (!s3BucketAccessible) { - throw new HpcException( - "Failed to access AWS S3 bucket: " + s3Destination.getDestinationLocation().getFileContainerId(), - HpcErrorType.INVALID_REQUEST_INPUT); - } - - String sourceURL = null; - long size = fileSize; - if (StringUtils.isEmpty(archiveLocationURL)) { - // Downloading from S3 archive -> S3 destination. - sourceURL = generateDownloadRequestURL(authenticatedToken, archiveLocation, baseArchiveDestination, - S3_STREAM_EXPIRATION); - if (size == 0) { - size = getPathAttributes(authenticatedToken, archiveLocation, true).getSize(); - } - } else { - // Downloading from POSIX archive -> S3 destination. - sourceURL = archiveLocationURL; - if (size == 0) { - try { - size = Files.size(Paths.get(URI.create(archiveLocationURL))); - } catch (IOException e) { - throw new HpcException( - "Failed to determine data object size in a POSIX archive: " + archiveLocationURL, - HpcErrorType.UNEXPECTED_ERROR); - } - } - } - - // Use AWS transfer manager to download the file. - return downloadDataObject(s3AccountAuthenticatedToken, sourceURL, s3Destination.getDestinationLocation(), size, - progressListener); - } - - /** - * Download a data object to a user's AWS / S3 Provider by using AWS transfer - * Manager. - * - * @param s3AccountAuthenticatedToken An authenticated token to the user's AWS - * S3 account. - * @param sourceURL The download source URL. - * @param destinationLocation The destination location. - * @param fileSize The size of the file to download. - * @param progressListener A progress listener for async notification - * on transfer completion. - * @return A data transfer request Id. - * @throws HpcException on data transfer failure. - */ - private String downloadDataObject(Object s3AccountAuthenticatedToken, String sourceURL, - HpcFileLocation destinationLocation, long fileSize, HpcDataTransferProgressListener progressListener) - throws HpcException { - if (progressListener == null) { - throw new HpcException("[S3] No progress listener provided for a download to AWS S3 destination", - HpcErrorType.UNEXPECTED_ERROR); - } - - CompletableFuture s3TransferManagerDownloadFuture = CompletableFuture.runAsync(() -> { - try { - // Create source URL and open a connection to it. - InputStream sourceInputStream = new URL(sourceURL).openStream(); - String sourceDestinationLogMessage = "download to " + destinationLocation.getFileContainerId() + ":" - + destinationLocation.getFileId(); - HpcS3ProgressListener listener = new HpcS3ProgressListener(progressListener, - sourceDestinationLogMessage); - - // Create a S3 upload request. - BlockingInputStreamAsyncRequestBody body = AsyncRequestBody.forBlockingInputStream(fileSize); - Upload streamUpload = s3Connection.getTransferManager(s3AccountAuthenticatedToken) - .upload(builder -> builder - .putObjectRequest(request -> request.bucket(destinationLocation.getFileContainerId()) - .key(destinationLocation.getFileId())) - .requestBody(body).addTransferListener(listener)); - - logger.info("S3 download Archive->AWS/S3 Provider [{}] started. Source size - {} bytes", - sourceDestinationLogMessage, fileSize); - - // Stream the data. - body.writeInputStream(sourceInputStream); - progressListener.setCompletableFuture(streamUpload.completionFuture()); - streamUpload.completionFuture().join(); - - } catch (CompletionException | HpcException | IOException e) { - logger.error("[S3] Failed to download to S3 destination: " + e.getCause().getMessage(), e); - progressListener.transferFailed(e.getCause().getMessage()); - - } - - }, s3Executor); - - return String.valueOf(s3TransferManagerDownloadFuture.hashCode()); - } - - /** - * Check if a path is a directory on S3 bucket. - * - * @param authenticatedToken An authenticated token to S3. - * @param fileLocation the file location. - * @return true if it's a directory, or false otherwise. - * @throws HpcException on failure invoke AWS S3 api. - */ - private boolean isDirectory(Object authenticatedToken, HpcFileLocation fileLocation) throws HpcException { - try { - try { // Check if this is a directory. Use V2 listObjects API. - ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request.builder() - .bucket(fileLocation.getFileContainerId()).prefix(fileLocation.getFileId() + "/").build(); - - ListObjectsV2Response listObjectsV2Response = s3Connection.getClient(authenticatedToken) - .listObjectsV2(listObjectsV2Request).join(); - - return listObjectsV2Response.keyCount() > 0 || !listObjectsV2Response.contents().isEmpty(); - - } catch (CompletionException e) { - if (e.getCause() instanceof SdkServiceException - && ((SdkServiceException) e.getCause()).statusCode() == 400) { // V2 not supported. Use V1 - // listObjects API. - ListObjectsRequest listObjectsRequest = ListObjectsRequest.builder() - .bucket(fileLocation.getFileContainerId()).prefix(fileLocation.getFileId()).build(); - - return !s3Connection.getClient(authenticatedToken).listObjects(listObjectsRequest).join().contents() - .isEmpty(); - } else { - throw e; - } - } - - } catch (CompletionException e) { - throw new HpcException("[S3] Failed to list object: " + e.getCause().getMessage(), - HpcErrorType.DATA_TRANSFER_ERROR, s3Connection.getS3Provider(authenticatedToken), e.getCause()); - } - } -} diff --git a/src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/s3/v3/impl/HpcS3Connection.java b/src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/s3/v3/impl/HpcS3Connection.java deleted file mode 100644 index 9f3f8684a6..0000000000 --- a/src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/s3/v3/impl/HpcS3Connection.java +++ /dev/null @@ -1,322 +0,0 @@ -/** - * HpcS3Connection.java - * - *

- * Copyright SVG, Inc. Copyright Leidos Biomedical Research, Inc - * - *

- * Distributed under the OSI-approved BSD 3-Clause License. See - * http://ncip.github.com/HPC/LICENSE.txt for details. - */ -package gov.nih.nci.hpc.integration.s3.v3.impl; - -import gov.nih.nci.hpc.domain.datatransfer.HpcS3Account; -import gov.nih.nci.hpc.domain.error.HpcErrorType; -import gov.nih.nci.hpc.domain.user.HpcIntegratedSystem; -import gov.nih.nci.hpc.domain.user.HpcIntegratedSystemAccount; -import gov.nih.nci.hpc.exception.HpcException; -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Value; -import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; -import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; -import software.amazon.awssdk.core.exception.SdkException; -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.s3.S3AsyncClient; -import software.amazon.awssdk.services.s3.S3Configuration; -import software.amazon.awssdk.services.s3.presigner.S3Presigner; -import software.amazon.awssdk.transfer.s3.S3TransferManager; - -import java.net.URI; -import java.net.URISyntaxException; -import java.util.HashSet; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -/** - * HPC S3 Connection. - * - * @author Eran Rosenberg - */ -public class HpcS3Connection { - // ---------------------------------------------------------------------// - // Constants - // ---------------------------------------------------------------------// - - // 5GB in bytes - private static final long FIVE_GB = 5368709120L; - - // Google Storage S3 URL. - private static final String GOOGLE_STORAGE_URL = "https://storage.googleapis.com"; - - // ---------------------------------------------------------------------// - // Instance members - // ---------------------------------------------------------------------// - - // A list of S3 3rd party providers that require connection w/ path-style - // enabled. - private Set pathStyleAccessEnabledProviders = new HashSet<>(); - - // The multipart upload minimum part size. - @Value("${hpc.integration.s3.minimumUploadPartSize}") - private Long minimumUploadPartSize = null; - - // The multipart upload threshold. - @Value("${hpc.integration.s3.multipartUploadThreshold}") - private Long multipartUploadThreshold = null; - - // The executor service to be used by AWSTransferManager - private ExecutorService executorService = null; - - // The logger instance. - private final Logger logger = LoggerFactory.getLogger(getClass().getName()); - - // ---------------------------------------------------------------------// - // Constructors - // ---------------------------------------------------------------------// - - /** - * Constructor for Spring Dependency Injection. - * - * @param pathStyleAccessEnabledProviders A list of S3 3rd party providers that - * require connection w/ path-style - * enabled. - * @param awsTransferManagerThreadPoolSize The thread pool size to be used for - * AWS transfer manager - */ - private HpcS3Connection(String pathStyleAccessEnabledProviders, int awsTransferManagerThreadPoolSize) { - for (String s3Provider : pathStyleAccessEnabledProviders.split(",")) { - this.pathStyleAccessEnabledProviders.add(HpcIntegratedSystem.fromValue(s3Provider)); - } - - // Instantiate the executor service for AWS transfer manager. - executorService = Executors.newFixedThreadPool(awsTransferManagerThreadPoolSize, - Executors.defaultThreadFactory()); - } - - // ---------------------------------------------------------------------// - // Methods - // ---------------------------------------------------------------------// - - /** - * Authenticate a (system) data transfer account to S3 (AWS or 3rd Party - * Provider) - * - * @param dataTransferAccount A data transfer account to authenticate. - * @param s3URLorRegion The S3 URL if authenticating with a 3rd party S3 - * Provider (Cleversafe, Cloudian, etc), or Region if - * authenticating w/ AWS. - * @return An authenticated TransferManager object, or null if authentication - * failed. - * @throws HpcException if authentication failed - */ - public Object authenticate(HpcIntegratedSystemAccount dataTransferAccount, String s3URLorRegion) - throws HpcException { - if (dataTransferAccount.getIntegratedSystem().equals(HpcIntegratedSystem.AWS)) { - return authenticateAWS(dataTransferAccount.getUsername(), dataTransferAccount.getPassword(), s3URLorRegion); - } else { - // Determine if this S3 provider require path-style enabled. - boolean pathStyleAccessEnabled = pathStyleAccessEnabledProviders - .contains(dataTransferAccount.getIntegratedSystem()); - - return authenticateS3Provider(dataTransferAccount.getUsername(), dataTransferAccount.getPassword(), - s3URLorRegion, pathStyleAccessEnabled, dataTransferAccount.getIntegratedSystem()); - } - } - - /** - * Authenticate a (user) S3 account (AWS or 3rd Party Provider) - * - * @param s3Account AWS S3 account. - * @return TransferManager - * @throws HpcException if authentication failed - */ - public Object authenticate(HpcS3Account s3Account) throws HpcException { - if (!StringUtils.isEmpty(s3Account.getRegion())) { - return authenticateAWS(s3Account.getAccessKey(), s3Account.getSecretKey(), s3Account.getRegion()); - - } else { - // Default S3 provider require path-style enabled to true if not provided by the - // user. - boolean pathStyleAccessEnabled = Optional.ofNullable(s3Account.getPathStyleAccessEnabled()).orElse(true); - - return authenticateS3Provider(s3Account.getAccessKey(), s3Account.getSecretKey(), s3Account.getUrl(), - pathStyleAccessEnabled, HpcIntegratedSystem.USER_S_3_PROVIDER); - } - } - - /** - * Get S3 Transfer Manager from an authenticated token. - * - * @param authenticatedToken An authenticated token. - * @return A transfer manager object. - * @throws HpcException on invalid authentication token. - */ - public S3TransferManager getTransferManager(Object authenticatedToken) throws HpcException { - if (!(authenticatedToken instanceof HpcS3)) { - throw new HpcException("Invalid S3 authentication token", HpcErrorType.INVALID_REQUEST_INPUT); - } - - return ((HpcS3) authenticatedToken).transferManager; - } - - /** - * Get S3 Client from an authenticated token. - * - * @param authenticatedToken An authenticated token. - * @return A S3 client object. - * @throws HpcException on invalid authentication token. - */ - public S3AsyncClient getClient(Object authenticatedToken) throws HpcException { - if (!(authenticatedToken instanceof HpcS3)) { - throw new HpcException("Invalid S3 authentication token", HpcErrorType.INVALID_REQUEST_INPUT); - } - - return ((HpcS3) authenticatedToken).client; - } - - /** - * Get S3 Presigner from an authenticated token. - * - * @param authenticatedToken An authenticated token. - * @return A S3 presigner object. - * @throws HpcException on invalid authentication token. - */ - public S3Presigner getPresigner(Object authenticatedToken) throws HpcException { - if (!(authenticatedToken instanceof HpcS3)) { - throw new HpcException("Invalid S3 authentication token", HpcErrorType.INVALID_REQUEST_INPUT); - } - - return ((HpcS3) authenticatedToken).presigner; - } - - /** - * Get S3 Provider from an authenticated token. - * - * @param authenticatedToken An authenticated token. - * @return A transfer manager object. - * @throws HpcException on invalid authentication token. - */ - public HpcIntegratedSystem getS3Provider(Object authenticatedToken) throws HpcException { - if (!(authenticatedToken instanceof HpcS3)) { - return null; - } - - return ((HpcS3) authenticatedToken).provider; - } - - // ---------------------------------------------------------------------// - // Helper Methods - // ---------------------------------------------------------------------// - - private class HpcS3 { - private S3TransferManager transferManager = null; - private S3AsyncClient client = null; - private S3Presigner presigner = null; - private HpcIntegratedSystem provider = null; - } - - /** - * Authenticate a 'S3 3rd Party Provider' account. - * - * @param username The S3 account user name. - * @param password The S3 account password. - * @param url The S3 3rd party provider URL. - * @param pathStyleAccessEnabled true if the S3 3rd Party provider supports path - * style access. - * @param s3Provider The 3rd party provider. - * @return HpcS3 instance - * @throws HpcException if authentication failed - */ - private Object authenticateS3Provider(String username, String password, String url, boolean pathStyleAccessEnabled, - HpcIntegratedSystem s3Provider) throws HpcException { - // Create the credential provider based on the configured credentials. - AwsBasicCredentials s3ProviderCredentials = AwsBasicCredentials.create(username, password); - StaticCredentialsProvider s3ProviderCredentialsProvider = StaticCredentialsProvider - .create(s3ProviderCredentials); - - // Create URI to the S3 provider endpoint - URI uri = null; - try { - uri = new URI(url); - - } catch (URISyntaxException e) { - throw new HpcException("Invalid URL: " + url, HpcErrorType.DATA_TRANSFER_ERROR, e); - } - - HpcS3 s3 = new HpcS3(); - s3.provider = s3Provider; - - try { - // Instantiate a S3 async client. - s3.client = S3AsyncClient.builder() - .credentialsProvider(s3ProviderCredentialsProvider) - .forcePathStyle(pathStyleAccessEnabled) - .endpointOverride(uri) - .multipartConfiguration(mpConfigBuilder -> mpConfigBuilder - .minimumPartSizeInBytes(minimumUploadPartSize) - .thresholdInBytes(url.equalsIgnoreCase(GOOGLE_STORAGE_URL) ? FIVE_GB : multipartUploadThreshold)) - .build(); - // Instantiate the S3 transfer manager. - s3.transferManager = S3TransferManager.builder().s3Client(s3.client).executor(executorService).build(); - - // Instantiate the S3 presigner. - s3.presigner = S3Presigner.builder().credentialsProvider(s3ProviderCredentialsProvider) - .endpointOverride(uri).serviceConfiguration( - S3Configuration.builder().pathStyleAccessEnabled(pathStyleAccessEnabled).build()) - .build(); - - return s3; - - } catch (SdkException e) { - throw new HpcException( - "[S3] Failed to authenticate S3 Provider: " + s3Provider.value() + "] - " + e.getMessage(), - HpcErrorType.DATA_TRANSFER_ERROR, e); - } - } - - /** - * Authenticate an AWS S3 account. - * - * @param accessKey The AWS account access key. - * @param secretKey The AWS account secret key. - * @param region The AWS account region. - * @return TransferManager - * @throws HpcException if authentication failed - */ - private Object authenticateAWS(String accessKey, String secretKey, String region) throws HpcException { - // Create the credential provider based on provided AWS S3 account. - AwsBasicCredentials awsCredentials = AwsBasicCredentials.create(accessKey, secretKey); - StaticCredentialsProvider awsCredentialsProvider = StaticCredentialsProvider.create(awsCredentials); - - HpcS3 s3 = new HpcS3(); - s3.provider = HpcIntegratedSystem.AWS; - - try { - // Instantiate a S3 async client. - s3.client = S3AsyncClient.builder() - .credentialsProvider(awsCredentialsProvider) - .region(Region.of(region)) - .multipartConfiguration(mpConfigBuilder -> mpConfigBuilder - .minimumPartSizeInBytes(minimumUploadPartSize) - .thresholdInBytes(multipartUploadThreshold)) - .build(); - - // Instantiate the S3 transfer manager. - s3.transferManager = S3TransferManager.builder().s3Client(s3.client).executor(executorService).build(); - - // Instantiate the S3 presigner. - s3.presigner = S3Presigner.builder().credentialsProvider(awsCredentialsProvider).region(Region.of(region)) - .build(); - - return s3; - - } catch (SdkException e) { - throw new HpcException("[S3] Failed to authenticate S3 in region " + region + "] - " + e.getMessage(), - HpcErrorType.DATA_TRANSFER_ERROR, e); - } - } -} diff --git a/src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/s3/v3/impl/HpcS3ProgressListener.java b/src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/s3/v3/impl/HpcS3ProgressListener.java deleted file mode 100644 index 6020a62c91..0000000000 --- a/src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/s3/v3/impl/HpcS3ProgressListener.java +++ /dev/null @@ -1,129 +0,0 @@ -/** - * HpcS3ProgressListener.java - * - *

Copyright SVG, Inc. Copyright Leidos Biomedical Research, Inc - * - *

Distributed under the OSI-approved BSD 3-Clause License. See - * http://ncip.github.com/HPC/LICENSE.txt for details. - */ -package gov.nih.nci.hpc.integration.s3.v3.impl; - -import java.util.concurrent.atomic.AtomicLong; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import gov.nih.nci.hpc.domain.error.HpcErrorType; -import gov.nih.nci.hpc.exception.HpcException; -import gov.nih.nci.hpc.integration.HpcDataTransferProgressListener; -import software.amazon.awssdk.transfer.s3.progress.TransferListener; - -/** - * HPC S3 Progress Listener. - * - * @author Eran Rosenberg - */ -public class HpcS3ProgressListener implements TransferListener { - // ---------------------------------------------------------------------// - // Constants - // ---------------------------------------------------------------------// - - // The transfer progress report rate (in bytes). Log and send transfer progress - // notifications every 100MB. - private static final long TRANSFER_PROGRESS_REPORTING_RATE = 1024L * 1024 * 100; - private static final long MB = 1024L * 1024; - - // ---------------------------------------------------------------------// - // Instance members - // ---------------------------------------------------------------------// - - // HPC progress listener - private HpcDataTransferProgressListener progressListener = null; - - // Bytes transferred and logged. - private AtomicLong bytesTransferred = new AtomicLong(0); - private long bytesTransferredReported = 0; - - // Transfer source and/or destination (for logging purposed) - private String transferSourceDestination = null; - - // Logger - private final Logger logger = LoggerFactory.getLogger(getClass().getName()); - - // ---------------------------------------------------------------------// - // Constructors - // ---------------------------------------------------------------------// - - /** - * Constructor. - * - * @param progressListener The HPC progress listener. - * @param transferSourceDestination The transfer source and destination (for - * logging progress). - * @throws HpcException if no progress listener provided. - */ - public HpcS3ProgressListener(HpcDataTransferProgressListener progressListener, String transferSourceDestination) - throws HpcException { - if (progressListener == null) { - throw new HpcException("Null progress listener", HpcErrorType.UNEXPECTED_ERROR); - } - this.progressListener = progressListener; - this.transferSourceDestination = transferSourceDestination; - } - - /** - * Default Constructor. - * - * @throws HpcException Constructor is disabled. - */ - @SuppressWarnings("unused") - private HpcS3ProgressListener() throws HpcException { - throw new HpcException("Constructor Disabled", HpcErrorType.UNEXPECTED_ERROR); - } - - // ---------------------------------------------------------------------// - // Methods - // ---------------------------------------------------------------------// - - // ---------------------------------------------------------------------// - // ProgressListener Interface Implementation - // ---------------------------------------------------------------------// - - @Override - public void transferInitiated(TransferListener.Context.TransferInitiated context) { - bytesTransferred.getAndSet(context.progressSnapshot().transferredBytes()); - bytesTransferredReported = bytesTransferred.get(); - logger.info("S3 transfer [{}] started. {} bytes transferred so far", transferSourceDestination, - bytesTransferredReported); - } - - @Override - public void bytesTransferred(TransferListener.Context.BytesTransferred context) { - bytesTransferred.getAndSet(context.progressSnapshot().transferredBytes()); - - if (bytesTransferred.get() - bytesTransferredReported >= TRANSFER_PROGRESS_REPORTING_RATE) { - bytesTransferredReported = bytesTransferred.get(); - logger.info("S3 transfer [{}] in progress. {}MB transferred so far", transferSourceDestination, - bytesTransferredReported / MB); - - progressListener.transferProgressed(bytesTransferredReported); - } - } - - @Override - public void transferComplete(TransferListener.Context.TransferComplete context) { - bytesTransferred.getAndSet(context.progressSnapshot().transferredBytes()); - - logger.info("S3 transfer [{}] completed. {} bytes transferred", transferSourceDestination, bytesTransferred.get()); - progressListener.transferCompleted(bytesTransferred.get()); - } - - @Override - public void transferFailed(TransferListener.Context.TransferFailed context) { - bytesTransferred.getAndSet(context.progressSnapshot().transferredBytes()); - - logger.error("S3 transfer [{}] failed. {}MB transferred.", transferSourceDestination, - bytesTransferred.get() / MB, context.exception()); - progressListener.transferFailed("S3 transfer failed: " + context.exception().getMessage()); - } -}