diff --git a/doc/development/HPCDATAMGM-2088 - External Archive Auto-Tiering.docx b/doc/development/HPCDATAMGM-2088 - External Archive Auto-Tiering.docx index 6687479b6e..b76036a741 100644 Binary files a/doc/development/HPCDATAMGM-2088 - External Archive Auto-Tiering.docx and b/doc/development/HPCDATAMGM-2088 - External Archive Auto-Tiering.docx differ diff --git a/src/hpc-cli/pom.xml b/src/hpc-cli/pom.xml index 3bb1abb9d2..164c7a1feb 100644 --- a/src/hpc-cli/pom.xml +++ b/src/hpc-cli/pom.xml @@ -5,7 +5,7 @@ gov.nih.nci.hpc hpc - 3.23.0 + 3.24.0 gov.nih.nci.hpc.cli @@ -48,7 +48,7 @@ gov.nih.nci.hpc hpc-dto - 3.23.0 + 3.24.0 org.springframework diff --git a/src/hpc-cli/src/main/java/gov/nih/nci/hpc/cli/commands/HpcBannerProvider.java b/src/hpc-cli/src/main/java/gov/nih/nci/hpc/cli/commands/HpcBannerProvider.java index 80a36001be..41bf4fb326 100644 --- a/src/hpc-cli/src/main/java/gov/nih/nci/hpc/cli/commands/HpcBannerProvider.java +++ b/src/hpc-cli/src/main/java/gov/nih/nci/hpc/cli/commands/HpcBannerProvider.java @@ -44,7 +44,7 @@ public String getBanner() { } public String getVersion() { - return "3.23.0"; + return "3.24.0"; } public String getWelcomeMessage() { diff --git a/src/hpc-server/hpc-app-service-api/pom.xml b/src/hpc-server/hpc-app-service-api/pom.xml index 853a3a38df..486037ff7f 100644 --- a/src/hpc-server/hpc-app-service-api/pom.xml +++ b/src/hpc-server/hpc-app-service-api/pom.xml @@ -19,7 +19,7 @@ gov.nih.nci.hpc hpc-server - 3.23.0 + 3.24.0 hpc-app-service-api jar diff --git a/src/hpc-server/hpc-app-service-api/src/main/java/gov/nih/nci/hpc/service/HpcDataManagementService.java b/src/hpc-server/hpc-app-service-api/src/main/java/gov/nih/nci/hpc/service/HpcDataManagementService.java index bffaa8d4c0..622282f9fe 100644 --- a/src/hpc-server/hpc-app-service-api/src/main/java/gov/nih/nci/hpc/service/HpcDataManagementService.java +++ b/src/hpc-server/hpc-app-service-api/src/main/java/gov/nih/nci/hpc/service/HpcDataManagementService.java @@ -594,6 +594,15 @@ public List getRegistrationResults(String u */ public HpcDataManagementConfiguration getDataManagementConfiguration(String id); + /** + * Get all Data Management Configurations that have auto-tiering enabled. + * A configuration is considered to have auto-tiering enabled if it has a + * non-null s3AutoTieringArchiveConfigurationId. + * + * @return List of data management configurations with auto-tiering enabled. + */ + public List getAutoTieringDataManagementConfigurations(); + /** * Add a data object registration request result to the DB. * diff --git a/src/hpc-server/hpc-app-service-api/src/main/java/gov/nih/nci/hpc/service/HpcDataMigrationService.java b/src/hpc-server/hpc-app-service-api/src/main/java/gov/nih/nci/hpc/service/HpcDataMigrationService.java index 9b50ae5fa0..542cb9dbbb 100644 --- a/src/hpc-server/hpc-app-service-api/src/main/java/gov/nih/nci/hpc/service/HpcDataMigrationService.java +++ b/src/hpc-server/hpc-app-service-api/src/main/java/gov/nih/nci/hpc/service/HpcDataMigrationService.java @@ -8,282 +8,318 @@ */ package gov.nih.nci.hpc.service; -import java.util.List; - import gov.nih.nci.hpc.domain.datamigration.HpcDataMigrationResult; import gov.nih.nci.hpc.domain.datamigration.HpcDataMigrationStatus; import gov.nih.nci.hpc.domain.datamigration.HpcDataMigrationType; +import gov.nih.nci.hpc.domain.datatransfer.HpcFileLocation; import gov.nih.nci.hpc.domain.model.HpcDataMigrationTask; import gov.nih.nci.hpc.domain.model.HpcDataMigrationTaskResult; import gov.nih.nci.hpc.domain.model.HpcDataMigrationTaskStatus; import gov.nih.nci.hpc.exception.HpcException; +import java.util.List; +import java.util.Map; + /** * HPC Data Migration Application Service Interface. * * @author Eran Rosenberg */ public interface HpcDataMigrationService { - /** - * Create a data object migration task. - * - * @param path The data object path. - * @param userId The user Id requested the - * migration. - * @param configurationId The The data object configuration - * ID. - * @param fromS3ArchiveConfigurationId The migration source S3 archive - * configuration ID. - * @param toS3ArchiveConfigurationId The migration target S3 archive - * configuration ID. - * @param collectionMigrationTaskId (Optional) The collection migration - * task ID that is associated w/ this - * data object migration task - * @param alignArchivePath If true, the file is moved within - * its current archive to align w/ the - * iRODs path. - * @param size The data object size. - * @param retryTaskId The previous task ID if this is a - * retry request. - * @param retryUserId The user retrying the request if - * this is a retry request. - * @param metadataUpdateRequest True to indicate this is a metadata - * update request. - * @param metadataFromArchiveFileContainerId (Optional) The metadata migration - * from archive file container ID. - * @param metadataToArchiveFileContainerId (Optional) The metadata migration - * to archive file container ID. - * - * @return A migration task ID. - * @throws HpcException on service failure. - */ - public HpcDataMigrationTask createDataObjectMigrationTask(String path, String userId, String configurationId, - String fromS3ArchiveConfigurationId, String toS3ArchiveConfigurationId, String collectionMigrationTaskId, - boolean alignArchivePath, long size, String retryTaskId, String retryUserId, boolean metadataUpdateRequest, - String metadataFromArchiveFileContainerId, String metadataToArchiveFileContainerId) throws HpcException; + /** + * Create a data object migration task. + * + * @param path The data object path. + * @param userId The user Id requested the + * migration. + * @param configurationId The The data object configuration + * ID. + * @param fromS3ArchiveConfigurationId The migration source S3 archive + * configuration ID. + * @param toS3ArchiveConfigurationId The migration target S3 archive + * configuration ID. + * @param collectionMigrationTaskId (Optional) The collection migration + * task ID that is associated w/ this + * data object migration task + * @param alignArchivePath If true, the file is moved within + * its current archive to align w/ the + * iRODs path. + * @param size The data object size. + * @param retryTaskId The previous task ID if this is a + * retry request. + * @param retryUserId The user retrying the request if + * this is a retry request. + * @param metadataUpdateRequest True to indicate this is a metadata + * update request. + * @param metadataFromArchiveFileContainerId (Optional) The metadata migration + * from archive file container ID. + * @param metadataToArchiveFileContainerId (Optional) The metadata migration + * to archive file container ID. + * @param autoTieringRequest True to indicate this is an auto-tiering + * migration request for files from external + * archive to S3 Glacier Deep Archive. + * @param fromS3ArchiveLocation (Optional) The external archive file location + * for auto-tiering requests, containing the + * file container ID and file ID of the source + * file in the external archive. + * @return A migration task ID. + * @throws HpcException on service failure. + */ + public HpcDataMigrationTask createDataObjectMigrationTask(String path, String userId, String configurationId, + String fromS3ArchiveConfigurationId, String toS3ArchiveConfigurationId, String collectionMigrationTaskId, + boolean alignArchivePath, Long size, String retryTaskId, String retryUserId, boolean metadataUpdateRequest, + String metadataFromArchiveFileContainerId, String metadataToArchiveFileContainerId, + boolean autoTieringRequest, HpcFileLocation fromS3ArchiveLocation) throws HpcException; - /** - * Get a list of migration tasks in specific status and type. - * - * @param status The data migration status to query for. - * @param type The data migration type to query for. - * @return A List of data migration tasks - * @throws HpcException on service failure. - */ - public List getDataMigrationTasks(HpcDataMigrationStatus status, HpcDataMigrationType type) - throws HpcException; + /** + * Get a list of migration tasks in specific status and type. + * + * @param status The data migration status to query for. + * @param type The data migration type to query for. + * @return A List of data migration tasks + * @throws HpcException on service failure. + */ + public List getDataMigrationTasks(HpcDataMigrationStatus status, HpcDataMigrationType type) + throws HpcException; - /** - * Assign data migration tasks to servers - * - * @throws HpcException on service failure. - */ - public void assignDataMigrationTasks() throws HpcException; + /** + * Assign data migration tasks to servers + * + * @throws HpcException on service failure. + */ + public void assignDataMigrationTasks() throws HpcException; - /** - * Migrate a data object. - * - * @param dataObjectMigrationTask The data migration details. - * @throws HpcException on service failure. - */ - public void migrateDataObject(HpcDataMigrationTask dataObjectMigrationTask) throws HpcException; + /** + * Migrate a data object. + * + * @param dataObjectMigrationTask The data migration details. + * @throws HpcException on service failure. + */ + public void migrateDataObject(HpcDataMigrationTask dataObjectMigrationTask) throws HpcException; - /** - * Complete a data object migration task. - * - * @param dataObjectMigrationTask The data migration task. - * @param result The data migration result. - * @param message (Optional) An error message in case the - * migration failed. - * @param fromS3ArchiveAuthToken The from S3 archive token - * @param toS3ArchiveAuthToken The to S3 archive token - * @throws HpcException on service failure. - */ - public void completeDataObjectMigrationTask(HpcDataMigrationTask dataObjectMigrationTask, - HpcDataMigrationResult result, String message, Object fromS3ArchiveAuthToken, Object toS3ArchiveAuthToken) - throws HpcException; + /** + * Complete a data object migration task. + * + * @param dataObjectMigrationTask The data migration task. + * @param result The data migration result. + * @param message (Optional) An error message in case the + * migration failed. + * @param fromS3ArchiveAuthToken The from S3 archive token + * @param toS3ArchiveAuthToken The to S3 archive token + * @throws HpcException on service failure. + */ + public void completeDataObjectMigrationTask(HpcDataMigrationTask dataObjectMigrationTask, + HpcDataMigrationResult result, String message, Object fromS3ArchiveAuthToken, Object toS3ArchiveAuthToken) + throws HpcException; - /** - * Complete a data object metadata update task. - * - * @param dataObjectMetadataUpdateTask The data migration task. - * @param result The data migration result. - * @param message (Optional) An error message in case the - * migration failed. - * @throws HpcException on service failure. - */ - public void completeDataObjectMetadataUpdateTask(HpcDataMigrationTask dataObjectMetadataUpdateTask, - HpcDataMigrationResult result, String message) throws HpcException; + /** + * Complete a data object metadata update task. + * + * @param dataObjectMetadataUpdateTask The data migration task. + * @param result The data migration result. + * @param message (Optional) An error message in case the + * migration failed. + * @throws HpcException on service failure. + */ + public void completeDataObjectMetadataUpdateTask(HpcDataMigrationTask dataObjectMetadataUpdateTask, + HpcDataMigrationResult result, String message) throws HpcException; - /** - * Complete a bulk migration task (Collection, Data Object List or Collection - * List). - * - * @param bulkMigrationTask The bulk migration task. - * @param message (Optional) An error message in case the migration - * failed - * @throws HpcException on service failure. - */ - public void completeBulkMigrationTask(HpcDataMigrationTask bulkMigrationTask, String message) throws HpcException; + /** + * Complete a bulk migration task (Collection, Data Object List or Collection + * List). + * + * @param bulkMigrationTask The bulk migration task. + * @param message (Optional) An error message in case the migration + * failed + * @throws HpcException on service failure. + */ + public void completeBulkMigrationTask(HpcDataMigrationTask bulkMigrationTask, String message) throws HpcException; - /** - * Reset migration tasks that are in-progress - * - * @throws HpcException on service failure. - **/ - public void resetMigrationTasksInProcess() throws HpcException; + /** + * Reset migration tasks that are in-progress + * + * @throws HpcException on service failure. + **/ + public void resetMigrationTasksInProcess() throws HpcException; - /** - * Create a collection migration task. - * - * @param path The collection path. - * @param userId The user Id requested the migration. - * @param configurationId The The data object configuration ID. - * @param toS3ArchiveConfigurationId The migration target S3 archive - * configuration ID. - * @param alignArchivePath If true, the file is moved within its - * current archive to align w/ the iRODs path. - * @param retryTaskId The previous task ID if this is a retry - * request. - * @param retryUserId The user retrying the request if this is a - * retry request. - * @param retryFailedItemsOnly if set to true, only failed items of - * 'taskId' will be retried. Otherwise the - * collection will be re-scanned for a new - * migration to include any items added since - * the previous migration attempt. - * @return A migration task ID. - * @throws HpcException on service failure. - */ - public HpcDataMigrationTask createCollectionMigrationTask(String path, String userId, String configurationId, - String toS3ArchiveConfigurationId, boolean alignArchivePath, String retryTaskId, String retryUserId, - Boolean retryFailedItemsOnly) throws HpcException; + /** + * Create a collection migration task. + * + * @param path The collection path. + * @param userId The user Id requested the migration. + * @param configurationId The The data object configuration ID. + * @param toS3ArchiveConfigurationId The migration target S3 archive + * configuration ID. + * @param alignArchivePath If true, the file is moved within its + * current archive to align w/ the iRODs path. + * @param retryTaskId The previous task ID if this is a retry + * request. + * @param retryUserId The user retrying the request if this is a + * retry request. + * @param retryFailedItemsOnly if set to true, only failed items of + * 'taskId' will be retried. Otherwise the + * collection will be re-scanned for a new + * migration to include any items added since + * the previous migration attempt. + * @return A migration task ID. + * @throws HpcException on service failure. + */ + public HpcDataMigrationTask createCollectionMigrationTask(String path, String userId, String configurationId, + String toS3ArchiveConfigurationId, boolean alignArchivePath, String retryTaskId, String retryUserId, + Boolean retryFailedItemsOnly) throws HpcException; - /** - * Create a data object list migration task. - * - * @param dataObjectPaths The data object paths. - * @param userId The user Id requested the migration. - * @param configurationId The The data object configuration ID. - * @param toS3ArchiveConfigurationId The migration target S3 archive - * configuration ID. - * @param retryTaskId The previous task ID if this is a retry - * request. - * @param retryUserId The user retrying the request if this is a - * retry request. - * @param retryUserId The user retrying the request if this is a - * retry request. - * @param retryFailedItemsOnly if set to true, only failed items of - * 'taskId' will be retried. Otherwise the - * collection will be re-scanned for a new - * migration to include any items added since - * the previous migration attempt. - * @return A migration task ID. - * @throws HpcException on service failure. - */ - public HpcDataMigrationTask createDataObjectsMigrationTask(List dataObjectPaths, String userId, - String configurationId, String toS3ArchiveConfigurationId, String retryTaskId, String retryUserId, - Boolean retryFailedItemsOnly) throws HpcException; + /** + * Create a data object list migration task. + * + * @param dataObjectPaths The data object paths. + * @param userId The user Id requested the migration. + * @param configurationId The The data object configuration ID. + * @param toS3ArchiveConfigurationId The migration target S3 archive + * configuration ID. + * @param retryTaskId The previous task ID if this is a retry + * request. + * @param retryUserId The user retrying the request if this is a + * retry request. + * @param retryUserId The user retrying the request if this is a + * retry request. + * @param retryFailedItemsOnly if set to true, only failed items of + * 'taskId' will be retried. Otherwise the + * collection will be re-scanned for a new + * migration to include any items added since + * the previous migration attempt. + * @return A migration task ID. + * @throws HpcException on service failure. + */ + public HpcDataMigrationTask createDataObjectsMigrationTask(List dataObjectPaths, String userId, + String configurationId, String toS3ArchiveConfigurationId, String retryTaskId, String retryUserId, + Boolean retryFailedItemsOnly) throws HpcException; - /** - * Create a collection list migration task. - * - * @param dataObjectPaths The collection paths. - * @param userId The user Id requested the migration. - * @param configurationId The The data object configuration ID. - * @param toS3ArchiveConfigurationId The migration target S3 archive - * configuration ID. - * @param retryTaskId The previous task ID if this is a retry - * request. - * @param retryUserId The user retrying the request if this is a - * retry request. - * @param retryFailedItemsOnly if set to true, only failed items of - * 'taskId' will be retried. Otherwise the - * collection will be re-scanned for a new - * migration to include any items added since - * the previous migration attempt. - * @return A migration task ID. - * @throws HpcException on service failure. - */ - public HpcDataMigrationTask createCollectionsMigrationTask(List collectionPaths, String userId, - String configurationId, String toS3ArchiveConfigurationId, String retryTaskId, String retryUserId, - Boolean retryFailedItemsOnly) throws HpcException; + /** + * Create a collection list migration task. + * + * @param dataObjectPaths The collection paths. + * @param userId The user Id requested the migration. + * @param configurationId The The data object configuration ID. + * @param toS3ArchiveConfigurationId The migration target S3 archive + * configuration ID. + * @param retryTaskId The previous task ID if this is a retry + * request. + * @param retryUserId The user retrying the request if this is a + * retry request. + * @param retryFailedItemsOnly if set to true, only failed items of + * 'taskId' will be retried. Otherwise the + * collection will be re-scanned for a new + * migration to include any items added since + * the previous migration attempt. + * @return A migration task ID. + * @throws HpcException on service failure. + */ + public HpcDataMigrationTask createCollectionsMigrationTask(List collectionPaths, String userId, + String configurationId, String toS3ArchiveConfigurationId, String retryTaskId, String retryUserId, + Boolean retryFailedItemsOnly) throws HpcException; - /** - * Update a migration task. - * - * @param dataMigrationTask The migration task. - * @throws HpcException on service failure. - */ - public void updateDataMigrationTask(HpcDataMigrationTask dataMigrationTask) throws HpcException; + /** + * Update a migration task. + * + * @param dataMigrationTask The migration task. + * @throws HpcException on service failure. + */ + public void updateDataMigrationTask(HpcDataMigrationTask dataMigrationTask) throws HpcException; - /** - * Update a migration task progress (percent complete) - * - * @param dataMigrationTask The migration task. - * @param bytesTransferred The bytes transferred so far. - * @throws HpcException on service failure. - */ - public void updateDataMigrationTaskProgress(HpcDataMigrationTask dataMigrationTask, long bytesTransferred) - throws HpcException; + /** + * Update a migration task progress (percent complete) + * + * @param dataMigrationTask The migration task. + * @param bytesTransferred The bytes transferred so far. + * @throws HpcException on service failure. + */ + public void updateDataMigrationTaskProgress(HpcDataMigrationTask dataMigrationTask, long bytesTransferred) + throws HpcException; - /** - * Mark a migration task that is in a RECEIVED state 'in-process', so that the - * task can be started. This enables multiple threads to read off of the - * migration task - * - * @param dataObjectMigrationTask The data migration task to mark - * @param inProcess Indicator whether the task is being actively - * processed. - * @return true if the inProcess value was actually updated in the DB. - * @throws HpcException on service failure. - */ - public boolean markInProcess(HpcDataMigrationTask dataObjectMigrationTask, boolean inProcess) throws HpcException; + /** + * Mark a migration task that is in a RECEIVED state 'in-process', so that the + * task can be started. This enables multiple threads to read off of the + * migration task + * + * @param dataObjectMigrationTask The data migration task to mark + * @param inProcess Indicator whether the task is being actively + * processed. + * @return true if the inProcess value was actually updated in the DB. + * @throws HpcException on service failure. + */ + public boolean markInProcess(HpcDataMigrationTask dataObjectMigrationTask, boolean inProcess) throws HpcException; - /** - * Get migration task status. - * - * @param taskId The migration task ID. - * @param taskType The migration task type (data-object or collection). - * @return A migration status object, or null if the task can't be found. Note: - * The returned object is associated with a 'task' object if the - * migration is in-progress. If the migration completed or failed, the - * returned object is associated with a 'result' object. - * @throws HpcException on service failure. - */ - public HpcDataMigrationTaskStatus getMigrationTaskStatus(String taskId, HpcDataMigrationType taskType) - throws HpcException; + /** + * Get migration task status. + * + * @param taskId The migration task ID. + * @param taskType The migration task type (data-object or collection). + * @return A migration status object, or null if the task can't be found. Note: + * The returned object is associated with a 'task' object if the + * migration is in-progress. If the migration completed or failed, the + * returned object is associated with a 'result' object. + * @throws HpcException on service failure. + */ + public HpcDataMigrationTaskStatus getMigrationTaskStatus(String taskId, HpcDataMigrationType taskType) + throws HpcException; - /** - * Get a list of migration tasks in specific status and type. - * - * @param collectionMigrationTaskId A collection migration task id that this - * data object migration is part of. - * @param result The task result to query for. - * @return A List of data migration task results. - * @throws HpcException on service failure. - */ - public List getDataMigrationResults(String collectionMigrationTaskId, - HpcDataMigrationResult result) throws HpcException; + /** + * Get a list of migration tasks in specific status and type. + * + * @param collectionMigrationTaskId A collection migration task id that this + * data object migration is part of. + * @param result The task result to query for. + * @return A List of data migration task results. + * @throws HpcException on service failure. + */ + public List getDataMigrationResults(String collectionMigrationTaskId, + HpcDataMigrationResult result) throws HpcException; - /** - * Create a metadata migration task. - * - * @param fromS3ArchiveConfigurationId The metadata migration's from S3 config - * ID. - * @param toS3ArchiveConfigurationId The metadata migration's to S3 config ID. - * @param archiveFileContainerId The metadata migration's archive - * container ID. - * @param fromArchiveFileContainerId The metadata migration's from archive - * container ID. - * @param toArchiveFileIdPattern The metadata migration's to archive file - * ID pattern (% matches any) - * @param userId The user Id requested the migration. - * @return A migration task ID. - * @throws HpcException on service failure. - */ - public HpcDataMigrationTask createMetadataMigrationTask(String fromS3ArchiveConfigurationId, - String toS3ArchiveConfigurationId, String fromArchiveFileContainerId, String toArchiveFileContainerId, - String archiveFileIdPattern, String userId) throws HpcException; + /** + * Create a metadata migration task. + * + * @param fromS3ArchiveConfigurationId The metadata migration's from S3 config + * ID. + * @param toS3ArchiveConfigurationId The metadata migration's to S3 config ID. + * @param archiveFileContainerId The metadata migration's archive + * container ID. + * @param fromArchiveFileContainerId The metadata migration's from archive + * container ID. + * @param toArchiveFileIdPattern The metadata migration's to archive file + * ID pattern (% matches any) + * @param userId The user Id requested the migration. + * @return A migration task ID. + * @throws HpcException on service failure. + */ + public HpcDataMigrationTask createMetadataMigrationTask(String fromS3ArchiveConfigurationId, + String toS3ArchiveConfigurationId, String fromArchiveFileContainerId, String toArchiveFileContainerId, + String archiveFileIdPattern, String userId) throws HpcException; + /** + * Create a bulk auto-tiering task. + * + * @param configurationId The data management configuration ID. + * @param fromS3ArchiveConfigurationId The auto-tiering source S3 config ID. + * @param toS3ArchiveConfigurationId The auto-tiering destination S3 config ID. + * @param userId The user Id requested the bulk auto-tiering. + * @return A migration task ID. + * @throws HpcException on service failure. + */ + HpcDataMigrationTask createBulkAutoTieringTask(String configurationId, String fromS3ArchiveConfigurationId, + String toS3ArchiveConfigurationId, String userId) throws HpcException; + + /** + * Search for files in the external archive that are candidates for auto-tiering migration. + * Files that have not been accessed within the configured time period will be identified, + * and their DME path and external archive locations returned + * + * @param configurationId The data management configuration ID. + * @param s3ArchiveConfigurationId The S3 archive configuration ID. + * @return A map of data object paths to their corresponding file locations in the external archive. + * @throws HpcException on service failure. + */ + Map getDataObjectsForAutoTiering(String configurationId, String s3ArchiveConfigurationId) + throws HpcException; } + + diff --git a/src/hpc-server/hpc-app-service-impl/pom.xml b/src/hpc-server/hpc-app-service-impl/pom.xml index a7e7ef01bf..cacac279f1 100644 --- a/src/hpc-server/hpc-app-service-impl/pom.xml +++ b/src/hpc-server/hpc-app-service-impl/pom.xml @@ -11,7 +11,7 @@ gov.nih.nci.hpc hpc-server - 3.23.0 + 3.24.0 hpc-app-service-impl jar diff --git a/src/hpc-server/hpc-app-service-impl/src/main/java/gov/nih/nci/hpc/service/impl/HpcDataManagementServiceImpl.java b/src/hpc-server/hpc-app-service-impl/src/main/java/gov/nih/nci/hpc/service/impl/HpcDataManagementServiceImpl.java index e3c73ccec2..7ff549b4ea 100644 --- a/src/hpc-server/hpc-app-service-impl/src/main/java/gov/nih/nci/hpc/service/impl/HpcDataManagementServiceImpl.java +++ b/src/hpc-server/hpc-app-service-impl/src/main/java/gov/nih/nci/hpc/service/impl/HpcDataManagementServiceImpl.java @@ -28,6 +28,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -1205,6 +1206,13 @@ public HpcDataManagementConfiguration getDataManagementConfiguration(String id) return dataManagementConfigurationLocator.get(id); } + @Override + public List getAutoTieringDataManagementConfigurations() { + return dataManagementConfigurationLocator.values().stream() + .filter(config -> !StringUtils.isEmpty(config.getS3AutoTieringConfigurationId())) + .collect(Collectors.toList()); + } + @Override public void addDataObjectRegistrationResult(String path, HpcSystemGeneratedMetadata systemGeneratedMetadata, boolean result, String message) throws HpcException { diff --git a/src/hpc-server/hpc-app-service-impl/src/main/java/gov/nih/nci/hpc/service/impl/HpcDataMigrationServiceImpl.java b/src/hpc-server/hpc-app-service-impl/src/main/java/gov/nih/nci/hpc/service/impl/HpcDataMigrationServiceImpl.java index ad9763b7a5..36f7e59e1f 100644 --- a/src/hpc-server/hpc-app-service-impl/src/main/java/gov/nih/nci/hpc/service/impl/HpcDataMigrationServiceImpl.java +++ b/src/hpc-server/hpc-app-service-impl/src/main/java/gov/nih/nci/hpc/service/impl/HpcDataMigrationServiceImpl.java @@ -9,6 +9,7 @@ package gov.nih.nci.hpc.service.impl; import java.util.Calendar; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -24,17 +25,20 @@ import com.google.common.collect.Iterables; import gov.nih.nci.hpc.dao.HpcDataMigrationDAO; +import gov.nih.nci.hpc.dao.HpcExternalArchiveDAO; import gov.nih.nci.hpc.domain.datamigration.HpcDataMigrationResult; import gov.nih.nci.hpc.domain.datamigration.HpcDataMigrationStatus; import gov.nih.nci.hpc.domain.datamigration.HpcDataMigrationType; import gov.nih.nci.hpc.domain.datatransfer.HpcArchiveObjectMetadata; import gov.nih.nci.hpc.domain.datatransfer.HpcDataTransferType; import gov.nih.nci.hpc.domain.datatransfer.HpcDeepArchiveStatus; +import gov.nih.nci.hpc.domain.datatransfer.HpcFileLocation; import gov.nih.nci.hpc.domain.datatransfer.HpcStreamingUploadSource; import gov.nih.nci.hpc.domain.error.HpcErrorType; import gov.nih.nci.hpc.domain.model.HpcDataMigrationTask; import gov.nih.nci.hpc.domain.model.HpcDataMigrationTaskResult; import gov.nih.nci.hpc.domain.model.HpcDataMigrationTaskStatus; +import gov.nih.nci.hpc.domain.model.HpcDataManagementConfiguration; import gov.nih.nci.hpc.domain.model.HpcDataObjectUploadRequest; import gov.nih.nci.hpc.domain.model.HpcDataTransferConfiguration; import gov.nih.nci.hpc.domain.model.HpcSystemGeneratedMetadata; @@ -46,6 +50,8 @@ import gov.nih.nci.hpc.service.HpcMetadataService; import gov.nih.nci.hpc.service.HpcSecurityService; +import static gov.nih.nci.hpc.service.impl.HpcDomainValidator.isValidFileLocation; + /** * HPC Data Search Application Service Implementation. * @@ -85,6 +91,10 @@ public class HpcDataMigrationServiceImpl implements HpcDataMigrationService { @Autowired private HpcSecurityService securityService = null; + // External Archive DAO. + @Autowired + private HpcExternalArchiveDAO externalArchiveDAO = null; + // A configured ID representing the server performing a migration task. @Value("${hpc.service.serverId}") private String serverId = null; @@ -123,8 +133,9 @@ private HpcDataMigrationServiceImpl() throws HpcException { @Override public HpcDataMigrationTask createDataObjectMigrationTask(String path, String userId, String configurationId, String fromS3ArchiveConfigurationId, String toS3ArchiveConfigurationId, String collectionMigrationTaskId, - boolean alignArchivePath, long size, String retryTaskId, String retryUserId, boolean metadataUpdateRequest, - String metadataFromArchiveFileContainerId, String metadataToArchiveFileContainerId) throws HpcException { + boolean alignArchivePath, Long size, String retryTaskId, String retryUserId, boolean metadataUpdateRequest, + String metadataFromArchiveFileContainerId, String metadataToArchiveFileContainerId, + boolean autoTieringRequest, HpcFileLocation fromS3ArchiveLocation) throws HpcException { // Check if a task already exist. HpcDataMigrationTask migrationTask = dataMigrationDAO.getDataObjectMigrationTask(collectionMigrationTaskId, path); @@ -146,6 +157,12 @@ public HpcDataMigrationTask createDataObjectMigrationTask(String path, String us return migrationTask; } + // Validate auto tiering request. + if(autoTieringRequest && !isValidFileLocation(fromS3ArchiveLocation)) { + throw new HpcException("Auto tiering requested w/o an external file location", + HpcErrorType.INVALID_REQUEST_INPUT); + } + // Create and persist a migration task. migrationTask.setPath(path); migrationTask.setUserId(userId); @@ -153,7 +170,8 @@ public HpcDataMigrationTask createDataObjectMigrationTask(String path, String us migrationTask.setFromS3ArchiveConfigurationId(fromS3ArchiveConfigurationId); migrationTask.setToS3ArchiveConfigurationId(toS3ArchiveConfigurationId); migrationTask.setCreated(Calendar.getInstance()); - migrationTask.setStatus(HpcDataMigrationStatus.RECEIVED); + migrationTask.setStatus(autoTieringRequest ? HpcDataMigrationStatus.AUTO_TIERING_RECEIVED : + HpcDataMigrationStatus.RECEIVED); migrationTask.setType(metadataUpdateRequest ? HpcDataMigrationType.DATA_OBJECT_METADATA_UPDATE : HpcDataMigrationType.DATA_OBJECT); migrationTask.setParentId(collectionMigrationTaskId); @@ -164,6 +182,7 @@ public HpcDataMigrationTask createDataObjectMigrationTask(String path, String us migrationTask.setRetryUserId(retryUserId); migrationTask.setMetadataFromArchiveFileContainerId(metadataFromArchiveFileContainerId); migrationTask.setMetadataToArchiveFileContainerId(metadataToArchiveFileContainerId); + migrationTask.setFromS3ArchiveLocation(fromS3ArchiveLocation); // Persist the task. dataMigrationDAO.upsertDataMigrationTask(migrationTask); @@ -632,4 +651,87 @@ public HpcDataMigrationTask createMetadataMigrationTask(String fromS3ArchiveConf dataMigrationDAO.upsertDataMigrationTask(migrationTask); return migrationTask; } + + @Override + public HpcDataMigrationTask createBulkAutoTieringTask(String configurationId, String fromS3ArchiveConfigurationId, + String toS3ArchiveConfigurationId, String userId) throws HpcException { + // Create and persist a bulk auto-tiering task. + HpcDataMigrationTask migrationTask = new HpcDataMigrationTask(); + migrationTask.setConfigurationId(configurationId); + migrationTask.setUserId(userId); + migrationTask.setFromS3ArchiveConfigurationId(fromS3ArchiveConfigurationId); + migrationTask.setToS3ArchiveConfigurationId(toS3ArchiveConfigurationId); + migrationTask.setCreated(Calendar.getInstance()); + migrationTask.setStatus(HpcDataMigrationStatus.RECEIVED); + migrationTask.setType(HpcDataMigrationType.BULK_AUTO_TIERING); + migrationTask.setPath(null); + migrationTask.setAlignArchivePath(false); + migrationTask.setPercentComplete(0); + migrationTask.setSize(null); + migrationTask.setRetryTaskId(null); + migrationTask.setRetryUserId(null); + migrationTask.setRetryFailedItemsOnly(null); + migrationTask.setMetadataFromArchiveFileContainerId(null); + migrationTask.setMetadataToArchiveFileContainerId(null); + migrationTask.setMetadataArchiveFileIdPattern(null); + + // Persist the task. + dataMigrationDAO.upsertDataMigrationTask(migrationTask); + return migrationTask; + } + + @Override + public Map getDataObjectsForAutoTiering(String configurationId, + String s3ArchiveConfigurationId) throws HpcException { + logger.info("getDataObjectsForAutoTiering called with configurationId: {}, s3ArchiveConfigurationId: {}", + configurationId, s3ArchiveConfigurationId); + + // Get the data management configuration. + HpcDataManagementConfiguration dataManagementConfiguration = dataManagementConfigurationLocator.get(configurationId); + if (dataManagementConfiguration == null) { + throw new HpcException("Data management configuration not found for ID: " + configurationId, + HpcErrorType.INVALID_REQUEST_INPUT); + } + + // Get the S3 data transfer configuration + HpcDataTransferConfiguration s3Configuration = dataManagementConfigurationLocator + .getDataTransferConfiguration(configurationId, s3ArchiveConfigurationId, HpcDataTransferType.S_3); + if (s3Configuration == null) { + throw new HpcException("S3 archive configuration not found for ID: " + s3ArchiveConfigurationId, + HpcErrorType.INVALID_REQUEST_INPUT); + } + + // Validate search path and inactivity months are defined + if (StringUtils.isEmpty(s3Configuration.getAutoTieringSearchPath())) { + throw new HpcException("Auto-tiering search path is not defined for S3 configuration: " + s3ArchiveConfigurationId, + HpcErrorType.INVALID_REQUEST_INPUT); + } + if (s3Configuration.getAutoTieringInactivityMonths() == null) { + throw new HpcException("Auto-tiering inactivity months is not defined for S3 configuration: " + s3ArchiveConfigurationId, + HpcErrorType.INVALID_REQUEST_INPUT); + } + + // Build the map of file paths to HpcFileLocation - these are the data objects to be auto-tiered. + String basePath = dataManagementConfiguration.getBasePath(); + String bucket = s3Configuration.getBaseArchiveDestination().getFileLocation().getFileContainerId(); + String objectIdPrefix = s3Configuration.getBaseArchiveDestination().getFileLocation().getFileId(); + + Map autoTieringDataObjects = new HashMap<>(); + // Query for files not accessed within the specified period and create the map of data obejcts to return. + externalArchiveDAO.getFilesNotAccessed( + s3Configuration.getAutoTieringSearchPath(), + s3Configuration.getAutoTieringInactivityMonths()).forEach(searchRelativePath -> { + // Construct HpcFileLocation of the data object to be auto-tiered. + HpcFileLocation fileLocation = new HpcFileLocation(); + fileLocation.setFileContainerId(bucket); + fileLocation.setFileId(objectIdPrefix + searchRelativePath); + + // Add this data object to the returned map. + autoTieringDataObjects.put(basePath + searchRelativePath, fileLocation); + }); + + logger.info("Found {} files for auto-tiering for configuration: {}", autoTieringDataObjects.size(), configurationId); + return autoTieringDataObjects; + } } + diff --git a/src/hpc-server/hpc-bus-service-api/pom.xml b/src/hpc-server/hpc-bus-service-api/pom.xml index 149b534b83..9b570daa71 100644 --- a/src/hpc-server/hpc-bus-service-api/pom.xml +++ b/src/hpc-server/hpc-bus-service-api/pom.xml @@ -19,7 +19,7 @@ gov.nih.nci.hpc hpc-server - 3.23.0 + 3.24.0 hpc-bus-service-api jar diff --git a/src/hpc-server/hpc-bus-service-api/src/main/java/gov/nih/nci/hpc/bus/HpcDataMigrationBusService.java b/src/hpc-server/hpc-bus-service-api/src/main/java/gov/nih/nci/hpc/bus/HpcDataMigrationBusService.java index d88282031c..507cf5e860 100644 --- a/src/hpc-server/hpc-bus-service-api/src/main/java/gov/nih/nci/hpc/bus/HpcDataMigrationBusService.java +++ b/src/hpc-server/hpc-bus-service-api/src/main/java/gov/nih/nci/hpc/bus/HpcDataMigrationBusService.java @@ -164,7 +164,6 @@ public HpcMigrationResponseDTO migrateMetadata(HpcMetadataMigrationRequestDTO me * @throws HpcException on service failure. */ public void assignMigrationServer() throws HpcException; - /** * Restart data object and collection migration tasks that are in progress. @@ -172,4 +171,27 @@ public HpcMigrationResponseDTO migrateMetadata(HpcMetadataMigrationRequestDTO me * @throws HpcException on service failure. */ public void restartDataMigrationTasks() throws HpcException; + + /** + * Process auto-tiering for all data management configurations + * + * @throws HpcException on service failure. + */ + public void processAutoTiering() throws HpcException; + + /** + * Process auto-tiering for a given data management configuration ID + * + * @param configurationId The data management configuration ID. + * @return Migration Response DTO. + * @throws HpcException on service failure. + */ + public HpcMigrationResponseDTO processAutoTiering(String configurationId) throws HpcException; + + /** + * Process received bulk auto-tiering migration tasks. + * + * @throws HpcException on service failure. + */ + public void processBulkAutoTieringMigrationReceived() throws HpcException; } diff --git a/src/hpc-server/hpc-bus-service-impl/pom.xml b/src/hpc-server/hpc-bus-service-impl/pom.xml index 3bb54a4a2c..7f049dbdaa 100644 --- a/src/hpc-server/hpc-bus-service-impl/pom.xml +++ b/src/hpc-server/hpc-bus-service-impl/pom.xml @@ -18,7 +18,7 @@ gov.nih.nci.hpc hpc-server - 3.23.0 + 3.24.0 hpc-bus-service-impl jar diff --git a/src/hpc-server/hpc-bus-service-impl/src/main/java/gov/nih/nci/hpc/bus/impl/HpcDataMigrationBusServiceImpl.java b/src/hpc-server/hpc-bus-service-impl/src/main/java/gov/nih/nci/hpc/bus/impl/HpcDataMigrationBusServiceImpl.java index 23a85ed834..3d9e46bac1 100644 --- a/src/hpc-server/hpc-bus-service-impl/src/main/java/gov/nih/nci/hpc/bus/impl/HpcDataMigrationBusServiceImpl.java +++ b/src/hpc-server/hpc-bus-service-impl/src/main/java/gov/nih/nci/hpc/bus/impl/HpcDataMigrationBusServiceImpl.java @@ -10,18 +10,6 @@ */ package gov.nih.nci.hpc.bus.impl; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.Executor; - -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.beans.factory.annotation.Value; - import gov.nih.nci.hpc.bus.HpcDataMigrationBusService; import gov.nih.nci.hpc.bus.HpcDataSearchBusService; import gov.nih.nci.hpc.bus.aspect.HpcExecuteAsSystemAccount; @@ -36,11 +24,7 @@ import gov.nih.nci.hpc.domain.datatransfer.HpcFileLocation; import gov.nih.nci.hpc.domain.error.HpcErrorType; import gov.nih.nci.hpc.domain.error.HpcRequestRejectReason; -import gov.nih.nci.hpc.domain.metadata.HpcCompoundMetadataQuery; -import gov.nih.nci.hpc.domain.metadata.HpcCompoundMetadataQueryOperator; -import gov.nih.nci.hpc.domain.metadata.HpcMetadataQuery; -import gov.nih.nci.hpc.domain.metadata.HpcMetadataQueryLevelFilter; -import gov.nih.nci.hpc.domain.metadata.HpcMetadataQueryOperator; +import gov.nih.nci.hpc.domain.metadata.*; import gov.nih.nci.hpc.domain.model.HpcDataMigrationTask; import gov.nih.nci.hpc.domain.model.HpcDataMigrationTaskResult; import gov.nih.nci.hpc.domain.model.HpcDataMigrationTaskStatus; @@ -52,11 +36,19 @@ import gov.nih.nci.hpc.dto.datamigration.HpcMigrationResponseDTO; import gov.nih.nci.hpc.dto.datasearch.HpcCompoundMetadataQueryDTO; import gov.nih.nci.hpc.exception.HpcException; -import gov.nih.nci.hpc.service.HpcDataManagementService; -import gov.nih.nci.hpc.service.HpcDataMigrationService; -import gov.nih.nci.hpc.service.HpcDataTransferService; -import gov.nih.nci.hpc.service.HpcMetadataService; -import gov.nih.nci.hpc.service.HpcSecurityService; +import gov.nih.nci.hpc.service.*; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.Executor; /** * HPC Data Management Business Service Implementation. @@ -65,1184 +57,1303 @@ */ public class HpcDataMigrationBusServiceImpl implements HpcDataMigrationBusService { - // ---------------------------------------------------------------------// - // Constants - // ---------------------------------------------------------------------// - - // ---------------------------------------------------------------------// - // Instance members - // ---------------------------------------------------------------------// - - // The Data Migration Application Service Instance. - @Autowired - private HpcDataMigrationService dataMigrationService = null; - - // The Data Management Application Service Instance. - @Autowired - private HpcDataManagementService dataManagementService = null; - - // The Metadata Application Service Instance. - @Autowired - private HpcMetadataService metadataService = null; - - // The Security Application Service Instance. - @Autowired - private HpcSecurityService securityService = null; - - // The Data Transfer Application Service Instance. - @Autowired - private HpcDataTransferService dataTransferService = null; - - // The Data Search Business Service Instance. - @Autowired - private HpcDataSearchBusService dataSearchBusService = null; - - // The collection download task executor. - @Autowired - @Qualifier("hpcDataObjectMetadataUpdateTaskExecutor") - Executor dataObjectMetadataUpdateTaskExecutor = null; - - @Value("${hpc.bus.getMetadataUpdateObjectsDefaultPageSize}") - private int getMetadataUpdateObjectsDefaultPageSize = 0; - - // The logger instance. - private final Logger logger = LoggerFactory.getLogger(this.getClass().getName()); - - // ---------------------------------------------------------------------// - // Constructors - // ---------------------------------------------------------------------// - - /** Constructor for Spring Dependency Injection. */ - private HpcDataMigrationBusServiceImpl() { - } - - // ---------------------------------------------------------------------// - // Methods - // ---------------------------------------------------------------------// - - // ---------------------------------------------------------------------// - // HpcDataMigrationBusService Interface Implementation - // ---------------------------------------------------------------------// - - @Override - public HpcMigrationResponseDTO migrateDataObject(String path, HpcMigrationRequestDTO migrationRequest, - boolean alignArchivePath) throws HpcException { - return migrateDataObject(path, securityService.getRequestInvoker().getNciAccount().getUserId(), null, - migrationRequest, alignArchivePath, null, null); - } - - @Override - public HpcMigrationResponseDTO migrateCollection(String path, HpcMigrationRequestDTO migrationRequest, - boolean alignArchivePath) throws HpcException { - return migrateCollection(path, securityService.getRequestInvoker().getNciAccount().getUserId(), - migrationRequest, alignArchivePath, null, null, null); - } - - @Override - public HpcMigrationResponseDTO migrateDataObjectsOrCollections(HpcBulkMigrationRequestDTO migrationRequest) - throws HpcException { - return migrateDataObjectsOrCollections(migrationRequest, - securityService.getRequestInvoker().getNciAccount().getUserId(), null, null, null); - } - - @Override - public HpcMigrationResponseDTO retryDataObjectMigrationTask(String taskId) throws HpcException { - // Input validation. - HpcDataMigrationTaskStatus taskStatus = dataMigrationService.getMigrationTaskStatus(taskId, - HpcDataMigrationType.DATA_OBJECT); - if (taskStatus == null) { - throw new HpcException("Data object migration task not found: " + taskId, - HpcErrorType.INVALID_REQUEST_INPUT); - } - if (taskStatus.getInProgress() || taskStatus.getResult() == null) { - throw new HpcException("Data object migration task in-progress: " + taskId, - HpcErrorType.INVALID_REQUEST_INPUT); - } - if (taskStatus.getResult() != null - && taskStatus.getResult().getResult().equals(HpcDataMigrationResult.COMPLETED)) { - throw new HpcException("Data-object migration task already completed: " + taskId, - HpcErrorType.INVALID_REQUEST_INPUT); - } - HpcDataMigrationTaskResult migrationTask = taskStatus.getResult(); - HpcMigrationRequestDTO migrationRequest = new HpcMigrationRequestDTO(); - migrationRequest.setS3ArchiveConfigurationId(migrationTask.getToS3ArchiveConfigurationId()); - - return migrateDataObject(migrationTask.getPath(), migrationTask.getUserId(), null, migrationRequest, - migrationTask.getAlignArchivePath(), taskId, - securityService.getRequestInvoker().getNciAccount().getUserId()); - } - - @Override - public HpcMigrationResponseDTO retryCollectionMigrationTask(String taskId, Boolean failedItemsOnly) - throws HpcException { - // Input validation. - HpcDataMigrationTaskStatus taskStatus = dataMigrationService.getMigrationTaskStatus(taskId, - HpcDataMigrationType.COLLECTION); - if (taskStatus == null) { - throw new HpcException("Collection migration task not found: " + taskId, - HpcErrorType.INVALID_REQUEST_INPUT); - } - if (taskStatus.getInProgress() || taskStatus.getResult() == null) { - throw new HpcException("Collection migration task in-progress: " + taskId, - HpcErrorType.INVALID_REQUEST_INPUT); - } - if (taskStatus.getResult() != null - && taskStatus.getResult().getResult().equals(HpcDataMigrationResult.COMPLETED)) { - throw new HpcException("Collection migration task already completed: " + taskId, - HpcErrorType.INVALID_REQUEST_INPUT); - } - HpcDataMigrationTaskResult migrationTask = taskStatus.getResult(); - HpcMigrationRequestDTO migrationRequest = new HpcMigrationRequestDTO(); - migrationRequest.setS3ArchiveConfigurationId(migrationTask.getToS3ArchiveConfigurationId()); - - return migrateCollection(migrationTask.getPath(), migrationTask.getUserId(), migrationRequest, - migrationTask.getAlignArchivePath(), taskId, - securityService.getRequestInvoker().getNciAccount().getUserId(), - Optional.ofNullable(failedItemsOnly).orElse(true)); - } - - @Override - public HpcMigrationResponseDTO retryDataObjectsOrCollectionsMigrationTask(String taskId, Boolean failedItemsOnly) - throws HpcException { - // Input validation. - HpcDataMigrationTaskStatus taskStatus = dataMigrationService.getMigrationTaskStatus(taskId, - HpcDataMigrationType.DATA_OBJECT_LIST); - if (taskStatus == null) { - taskStatus = dataMigrationService.getMigrationTaskStatus(taskId, HpcDataMigrationType.COLLECTION_LIST); - } - if (taskStatus == null) { - throw new HpcException("Bulk migration task not found: " + taskId, HpcErrorType.INVALID_REQUEST_INPUT); - } - if (taskStatus.getInProgress() || taskStatus.getResult() == null) { - throw new HpcException("Bulk migration task in-progress: " + taskId, HpcErrorType.INVALID_REQUEST_INPUT); - } - if (taskStatus.getResult() != null - && taskStatus.getResult().getResult().equals(HpcDataMigrationResult.COMPLETED)) { - throw new HpcException("Bulk migration task already completed: " + taskId, - HpcErrorType.INVALID_REQUEST_INPUT); - } - HpcDataMigrationTaskResult migrationTask = taskStatus.getResult(); - HpcBulkMigrationRequestDTO migrationRequest = new HpcBulkMigrationRequestDTO(); - migrationRequest.setS3ArchiveConfigurationId(migrationTask.getToS3ArchiveConfigurationId()); - migrationRequest.getDataObjectPaths().addAll(migrationTask.getDataObjectPaths()); - migrationRequest.getCollectionPaths().addAll(migrationTask.getCollectionPaths()); - - return migrateDataObjectsOrCollections(migrationRequest, migrationTask.getUserId(), taskId, - securityService.getRequestInvoker().getNciAccount().getUserId(), - Optional.ofNullable(failedItemsOnly).orElse(true)); - } - - @Override - public HpcMigrationResponseDTO migrateMetadata(HpcMetadataMigrationRequestDTO metadataMigrationRequest) - throws HpcException { - // Input Validation - if (metadataMigrationRequest == null) { - throw new HpcException("Metadata migration request is null", HpcErrorType.INVALID_REQUEST_INPUT); - } - - if (StringUtils.isEmpty(metadataMigrationRequest.getFromS3ArchiveConfigurationId())) { - throw new HpcException("from S3 archive configuration ID is empty", HpcErrorType.INVALID_REQUEST_INPUT); - } else { - // Exception will be thrown if the S3 config ID can't be found. - dataManagementService.getS3ArchiveConfiguration(metadataMigrationRequest.getFromS3ArchiveConfigurationId()); - } - - if (StringUtils.isEmpty(metadataMigrationRequest.getToS3ArchiveConfigurationId())) { - throw new HpcException("To S3 archive configuration ID is empty", HpcErrorType.INVALID_REQUEST_INPUT); - } else { - // Exception will be thrown if the S3 config ID can't be found. - dataManagementService.getS3ArchiveConfiguration(metadataMigrationRequest.getToS3ArchiveConfigurationId()); - } - - if (metadataMigrationRequest.getFromS3ArchiveConfigurationId() - .equals(metadataMigrationRequest.getToS3ArchiveConfigurationId())) { - throw new HpcException("From/To S3 archive configuration ID is identical", - HpcErrorType.INVALID_REQUEST_INPUT); - } - - if (StringUtils.isEmpty(metadataMigrationRequest.getFromArchiveFileContainerId())) { - throw new HpcException("From Archive File Container ID is empty", HpcErrorType.INVALID_REQUEST_INPUT); - } - - if (StringUtils.isEmpty(metadataMigrationRequest.getToArchiveFileContainerId())) { - throw new HpcException("To Archive File Container ID is empty", HpcErrorType.INVALID_REQUEST_INPUT); - } - - // Create a migration task to perform bulk metadata update. - // Notes: - // 1. toArchiveFileContainerId is optional and defaulted to - // fromArchiveFileContainerId if not provided by the caller. - // 2. archiveFileIdPattern is optional and defaulted to '%' (match-all) if not - // provided by the caller - HpcMigrationResponseDTO migrationResponse = new HpcMigrationResponseDTO(); - migrationResponse - .setTaskId(dataMigrationService - .createMetadataMigrationTask(metadataMigrationRequest.getFromS3ArchiveConfigurationId(), - metadataMigrationRequest.getToS3ArchiveConfigurationId(), - metadataMigrationRequest.getFromArchiveFileContainerId(), - !StringUtils.isEmpty(metadataMigrationRequest.getToArchiveFileContainerId()) - ? metadataMigrationRequest.getToArchiveFileContainerId() - : metadataMigrationRequest.getFromArchiveFileContainerId(), - !StringUtils.isEmpty(metadataMigrationRequest.getArchiveFileIdPattern()) - ? metadataMigrationRequest.getArchiveFileIdPattern() - : "%", - securityService.getRequestInvoker().getNciAccount().getUserId()) - .getId()); - - return migrationResponse; - } - - @Override - @HpcExecuteAsSystemAccount - public void processDataObjectMigrationReceived() throws HpcException { - dataMigrationService.getDataMigrationTasks(HpcDataMigrationStatus.RECEIVED, HpcDataMigrationType.DATA_OBJECT) - .forEach(dataObjectMigrationTask -> { - if (markInProcess(dataObjectMigrationTask)) { - try { - logger.info("Migrating Data Object: task - {}, path - {}", dataObjectMigrationTask.getId(), - dataObjectMigrationTask.getPath()); - dataMigrationService.migrateDataObject(dataObjectMigrationTask); - - } catch (HpcException e) { - logger.error("Failed to migrate data object: task - {}, path - {}", - dataObjectMigrationTask.getId(), dataObjectMigrationTask.getPath(), e); - try { - dataMigrationService.completeDataObjectMigrationTask(dataObjectMigrationTask, - HpcDataMigrationResult.FAILED, e.getMessage(), null, null); - - } catch (HpcException ex) { - logger.error("Failed to complete data object migration: task - {}, path - {}", - dataObjectMigrationTask.getId(), dataObjectMigrationTask.getPath(), ex); - } - } finally { - doneProcessingDataMigrationTask(dataObjectMigrationTask); - } - } - }); - } - - @Override - @HpcExecuteAsSystemAccount - public void processCollectionMigrationReceived() throws HpcException { - dataMigrationService.getDataMigrationTasks(HpcDataMigrationStatus.RECEIVED, HpcDataMigrationType.COLLECTION) - .forEach(collectionMigrationTask -> { - if (markInProcess(collectionMigrationTask)) { - try { - logger.info("Migrating Collection: task - {}, path - {}", collectionMigrationTask.getId(), - collectionMigrationTask.getPath()); - - // Get the System generated metadata. - HpcSystemGeneratedMetadata metadata = metadataService - .getCollectionSystemGeneratedMetadata(collectionMigrationTask.getPath()); - - // Get the collection to be migrated. - HpcCollection collection = dataManagementService - .getFullCollection(collectionMigrationTask.getPath(), metadata.getLinkSourcePath()); - if (collection == null) { - throw new HpcException("Collection not found", HpcErrorType.INVALID_REQUEST_INPUT); - } - - // Create migration tasks for all data objects under this collection - migrateCollection(collection, collectionMigrationTask); - - // Mark the collection migration task - in-progress - collectionMigrationTask.setStatus(HpcDataMigrationStatus.IN_PROGRESS); - dataMigrationService.updateDataMigrationTask(collectionMigrationTask); - - } catch (HpcException e) { - logger.error("Failed to migrate collection: task - {}, path - {}", - collectionMigrationTask.getId(), collectionMigrationTask.getPath(), e); - try { - dataMigrationService.completeBulkMigrationTask(collectionMigrationTask, e.getMessage()); - - } catch (HpcException ex) { - logger.error("Failed to complete collection migration: task - {}, path - {}", - collectionMigrationTask.getId(), collectionMigrationTask.getPath(), ex); - } - } finally { - doneProcessingDataMigrationTask(collectionMigrationTask); - } - } - }); - } - - @Override - @HpcExecuteAsSystemAccount - public void processDataObjectListMigrationReceived() throws HpcException { - dataMigrationService - .getDataMigrationTasks(HpcDataMigrationStatus.RECEIVED, HpcDataMigrationType.DATA_OBJECT_LIST) - .forEach(dataObjectListMigrationTask -> { - if (markInProcess(dataObjectListMigrationTask)) { - try { - logger.info("Migrating Data Object List: task - {}, path - {}", - dataObjectListMigrationTask.getId(), - dataObjectListMigrationTask.getDataObjectPaths()); - - if (Optional.ofNullable(dataObjectListMigrationTask.getRetryFailedItemsOnly()) - .orElse(false)) { - // Retry failed items only. - migrateFailedItems(dataObjectListMigrationTask); - - } else { - // Iterate through the data objects in the list and migrate them. - for (String dataObjectPath : dataObjectListMigrationTask.getDataObjectPaths()) { - HpcMigrationRequestDTO migrationRequest = new HpcMigrationRequestDTO(); - migrationRequest.setS3ArchiveConfigurationId( - dataObjectListMigrationTask.getToS3ArchiveConfigurationId()); - migrateDataObject(dataObjectPath, dataObjectListMigrationTask.getUserId(), - dataObjectListMigrationTask.getId(), migrationRequest, - dataObjectListMigrationTask.getAlignArchivePath(), null, null); - } - } - - // Mark the collection migration task - in-progress - dataObjectListMigrationTask.setStatus(HpcDataMigrationStatus.IN_PROGRESS); - dataMigrationService.updateDataMigrationTask(dataObjectListMigrationTask); - - } catch (HpcException e) { - logger.error("Failed to migrate data object list: task - {}, path - {}", - dataObjectListMigrationTask.getId(), - dataObjectListMigrationTask.getDataObjectPaths(), e); - try { - dataMigrationService.completeBulkMigrationTask(dataObjectListMigrationTask, - e.getMessage()); - - } catch (HpcException ex) { - logger.error("Failed to complete data object list migration: task - {}, path - {}", - dataObjectListMigrationTask.getId(), - dataObjectListMigrationTask.getDataObjectPaths(), ex); - } - } finally { - doneProcessingDataMigrationTask(dataObjectListMigrationTask); - } - } - }); - } - - @Override - @HpcExecuteAsSystemAccount - public void processCollectionListMigrationReceived() throws HpcException { - dataMigrationService - .getDataMigrationTasks(HpcDataMigrationStatus.RECEIVED, HpcDataMigrationType.COLLECTION_LIST) - .forEach(collectionListMigrationTask -> { - if (markInProcess(collectionListMigrationTask)) { - try { - logger.info("Migrating Collection list: task - {}, path - {}", - collectionListMigrationTask.getId(), - collectionListMigrationTask.getCollectionPaths()); - - // Iterate through the collections in the list and migrate them. - for (String collectionPath : collectionListMigrationTask.getCollectionPaths()) { - // Get the System generated metadata. - HpcSystemGeneratedMetadata metadata = metadataService - .getCollectionSystemGeneratedMetadata(collectionPath); - - // Get the collection to be migrated. - HpcCollection collection = dataManagementService.getFullCollection(collectionPath, - metadata.getLinkSourcePath()); - if (collection == null) { - throw new HpcException("Collection not found", HpcErrorType.INVALID_REQUEST_INPUT); - } - - // Create migration tasks for all data objects under this collection - migrateCollection(collection, collectionListMigrationTask); - } - - // Mark the collection migration task - in-progress - collectionListMigrationTask.setStatus(HpcDataMigrationStatus.IN_PROGRESS); - dataMigrationService.updateDataMigrationTask(collectionListMigrationTask); - - } catch (HpcException e) { - logger.error("Failed to migrate collection list: task - {}, path - {}", - collectionListMigrationTask.getId(), - collectionListMigrationTask.getCollectionPaths(), e); - try { - dataMigrationService.completeBulkMigrationTask(collectionListMigrationTask, - e.getMessage()); - - } catch (HpcException ex) { - logger.error("Failed to complete collection list migration: task - {}, path - {}", - collectionListMigrationTask.getId(), collectionListMigrationTask.getPath(), ex); - } - } finally { - doneProcessingDataMigrationTask(collectionListMigrationTask); - } - } - }); - } - - @Override - @HpcExecuteAsSystemAccount - public void processBulkMetadataUpdateMigrationReceived() throws HpcException { - dataMigrationService - .getDataMigrationTasks(HpcDataMigrationStatus.RECEIVED, HpcDataMigrationType.BULK_METADATA_UPDATE) - .forEach(bulkMetadataUpdateTask -> { - if (markInProcess(bulkMetadataUpdateTask)) { - try { - logger.info("Processing bulk metadata update migration task - {}", - bulkMetadataUpdateTask.getId()); - - // Create metadata migration tasks for all data objects under this bulk-update - // task. - for (String dataObjectPath : getBulkMetadataUpdateDataObjectPaths(bulkMetadataUpdateTask)) { - updateDataObjectMetadata(dataObjectPath, bulkMetadataUpdateTask); - } - - // Mark the bulk metadata update task - in-progress - bulkMetadataUpdateTask.setStatus(HpcDataMigrationStatus.IN_PROGRESS); - dataMigrationService.updateDataMigrationTask(bulkMetadataUpdateTask); - - } catch (HpcException e) { - logger.error("Failed to process update-bulk-metadata: task - {}", - bulkMetadataUpdateTask.getId(), e); - try { - dataMigrationService.completeBulkMigrationTask(bulkMetadataUpdateTask, e.getMessage()); - - } catch (HpcException ex) { - logger.error("Failed to complete bulk metadata: task - {}", - bulkMetadataUpdateTask.getId(), ex); - } - } finally { - doneProcessingDataMigrationTask(bulkMetadataUpdateTask); - } - } - }); - } - - @Override - @HpcExecuteAsSystemAccount - public void processDataObjectMetadataUpdateMigrationReceived() throws HpcException { - dataMigrationService.getDataMigrationTasks(HpcDataMigrationStatus.RECEIVED, - HpcDataMigrationType.DATA_OBJECT_METADATA_UPDATE).forEach(dataObjectMetadataUpdateTask -> { - if (markInProcess(dataObjectMetadataUpdateTask)) { - try { - logger.info("Updating Data Object Metadata: task - {}, path - {}", - dataObjectMetadataUpdateTask.getId(), dataObjectMetadataUpdateTask.getPath()); - completeDataObjectMetadataUpdate(dataObjectMetadataUpdateTask); - - } catch (HpcException e) { - logger.error("Failed to update data object metadata: task - {}, path - {}", - dataObjectMetadataUpdateTask.getId(), dataObjectMetadataUpdateTask.getPath(), e); - try { - dataMigrationService.completeDataObjectMetadataUpdateTask(dataObjectMetadataUpdateTask, - HpcDataMigrationResult.FAILED, e.getMessage()); - - } catch (HpcException ex) { - logger.error("Failed to complete data object metadata update: task - {}, path - {}", - dataObjectMetadataUpdateTask.getId(), dataObjectMetadataUpdateTask.getPath(), - ex); - } - } finally { - doneProcessingDataMigrationTask(dataObjectMetadataUpdateTask); - } - } - }); - } - - @Override - @HpcExecuteAsSystemAccount - public void completeBulkMigrationInProgress() throws HpcException { - List bulkMigrationTasks = new ArrayList<>(); - bulkMigrationTasks.addAll(dataMigrationService.getDataMigrationTasks(HpcDataMigrationStatus.IN_PROGRESS, - HpcDataMigrationType.COLLECTION)); - bulkMigrationTasks.addAll(dataMigrationService.getDataMigrationTasks(HpcDataMigrationStatus.IN_PROGRESS, - HpcDataMigrationType.DATA_OBJECT_LIST)); - bulkMigrationTasks.addAll(dataMigrationService.getDataMigrationTasks(HpcDataMigrationStatus.IN_PROGRESS, - HpcDataMigrationType.COLLECTION_LIST)); - bulkMigrationTasks.addAll(dataMigrationService.getDataMigrationTasks(HpcDataMigrationStatus.IN_PROGRESS, - HpcDataMigrationType.BULK_METADATA_UPDATE)); - - bulkMigrationTasks.forEach(bulkMigrationTask -> { - try { - logger.info("Completing bulk migration: task - {}, type - {}", bulkMigrationTask.getId(), - bulkMigrationTask.getType()); - - dataMigrationService.completeBulkMigrationTask(bulkMigrationTask, null); - - } catch (HpcException e) { - logger.error("Failed to complete bulk migration: task - {}, type - {}", bulkMigrationTask.getId(), - bulkMigrationTask.getType(), e); - try { - dataMigrationService.completeBulkMigrationTask(bulkMigrationTask, e.getMessage()); - - } catch (HpcException ex) { - logger.error("Failed to complete bulk migration: task - {}, type - {}", bulkMigrationTask.getId(), - bulkMigrationTask.getType(), ex); - } - } - }); - } - - @Override - @HpcExecuteAsSystemAccount - public void assignMigrationServer() throws HpcException { - dataMigrationService.assignDataMigrationTasks(); - } - - @Override - @HpcExecuteAsSystemAccount - public void restartDataMigrationTasks() throws HpcException { - dataMigrationService.resetMigrationTasksInProcess(); - } - - // ---------------------------------------------------------------------// - // Helper Methods - // ---------------------------------------------------------------------// - - /** - * Validate a data object migration request - * - * @param path The data object path. - * @param migrationRequest The migration request. - * @param alignArchivePath If true, the file is moved within its current - * archive to align w/ the iRODs path. - * @param metadataUpdateRequest true if this is a metadata update request, false - * if it's data migration request. - * @return The data object system metadata - * @throws HpcException If the request is invalid. - */ - private HpcSystemGeneratedMetadata validateDataObjectMigrationRequest(String path, - HpcMigrationRequestDTO migrationRequest, boolean alignArchivePath, boolean metadataUpdateRequest) - throws HpcException { - - // Validate the following: - // 1. Path is not empty. - // 2. Migration request is not empty. - // 3. Data Object exists. - // 4. Migration is not supported for links. - // 5. Migration is supported only from S3 archive to S3 Archive. - // 6. Data Object is archived (i.e. registration completed). - // 7. The source-size system metadata is available. - - if (!metadataUpdateRequest) { - validateMigrationRequest(path, migrationRequest, alignArchivePath); - } - - // Get the System generated metadata. - HpcSystemGeneratedMetadata metadata = metadataService.getDataObjectSystemGeneratedMetadata(path); - - if (!StringUtils.isEmpty(metadata.getLinkSourcePath())) { - throw new HpcException("Migration request is not supported for soft-links", - HpcErrorType.INVALID_REQUEST_INPUT); - } - - // Migration supported only from S3 archive. - if (metadata.getDataTransferType() == null || !metadata.getDataTransferType().equals(HpcDataTransferType.S_3)) { - throw new HpcException("Migration request is not supported from POSIX based file system archive", - HpcErrorType.INVALID_REQUEST_INPUT); - } - - // Validate the file is archived. - HpcDataTransferUploadStatus dataTransferStatus = metadata.getDataTransferStatus(); - if (dataTransferStatus == null) { - throw new HpcException("Unknown upload data transfer status: " + path, HpcErrorType.UNEXPECTED_ERROR); - } - if (!dataTransferStatus.equals(HpcDataTransferUploadStatus.ARCHIVED) - && !dataTransferStatus.equals(HpcDataTransferUploadStatus.DELETE_REQUESTED)) { - throw new HpcException( - "Data Object [" + path + "] is not in archived or soft-deleted state. It is in " - + metadata.getDataTransferStatus().value() + " state", - HpcRequestRejectReason.FILE_NOT_ARCHIVED); - } - - // Validate a source-size system metadata is available. - if (metadata.getSourceSize() == null) { - throw new HpcException("Source size system-metadata is missing for data object to be migrated", - HpcErrorType.UNEXPECTED_ERROR); - } - - return metadata; - } - - /** - * Validate a collection migration request - * - * @param path The data object path. - * @param migrationRequest The migration request. - * @param alignArchivePath If true, the file is moved within its current archive - * to align w/ the iRODs path. - * @return The data object system metadata - * @throws HpcException If the request is invalid. - */ - private HpcSystemGeneratedMetadata validateCollectionMigrationRequest(String path, - HpcMigrationRequestDTO migrationRequest, boolean alignArchivePath) throws HpcException { - - // Validate the following: - // 1. Path is not empty. - // 2. Migration is not supported for links. - // 3. Migration is supported only from S3 archive to S3 Archive. - // 4. Collection exists. - // 5. Collection is not empty - // 6. Collection is not linked to another collection - - validateMigrationRequest(path, migrationRequest, alignArchivePath); - - // Get the System generated metadata. - HpcSystemGeneratedMetadata metadata = metadataService.getCollectionSystemGeneratedMetadata(path); - - logger.debug("Validating collection migration request for [{}] metadata returned - {}", path, (metadata == null ? "" : metadata.toString())); - - // Validate collection is not linked. - if (!StringUtils.isEmpty(metadata.getLinkSourcePath())) { - throw new HpcException("Collection is linked to: " + metadata.getLinkSourcePath(), - HpcErrorType.INVALID_REQUEST_INPUT); - } - - // Validate collection exists. - HpcCollection collection = dataManagementService.getCollection(path, true, metadata.getLinkSourcePath()); - if (collection == null) { - throw new HpcException("Collection doesn't exist: " + path, HpcErrorType.INVALID_REQUEST_INPUT); - } - - // Verify data objects found under this collection. - if (!dataManagementService.hasDataObjects(collection)) { - // No data objects found under this collection. - throw new HpcException("No data objects found under collection" + path, HpcErrorType.INVALID_REQUEST_INPUT); - } - - return metadata; - } - - /** - * Validate a bulk migration request - * - * @param migrationRequest The migration request. - * @return The data object system metadata - * @throws HpcException If the request is invalid. - */ - private HpcSystemGeneratedMetadata validateBulkMigrationRequest(HpcBulkMigrationRequestDTO bulkMigrationRequest) - throws HpcException { - if (bulkMigrationRequest == null) { - throw new HpcException("Null migration request", HpcErrorType.INVALID_REQUEST_INPUT); - } - - if (bulkMigrationRequest.getDataObjectPaths().isEmpty() - && bulkMigrationRequest.getCollectionPaths().isEmpty()) { - throw new HpcException("No data object or collection paths", HpcErrorType.INVALID_REQUEST_INPUT); - } - if (!bulkMigrationRequest.getDataObjectPaths().isEmpty() - && !bulkMigrationRequest.getCollectionPaths().isEmpty()) { - throw new HpcException("Both data object and collection paths provided", - HpcErrorType.INVALID_REQUEST_INPUT); - } - - HpcMigrationRequestDTO migrationRequest = new HpcMigrationRequestDTO(); - HpcSystemGeneratedMetadata systemGenerateMetadata = null; - - for (String path : bulkMigrationRequest.getDataObjectPaths()) { - migrationRequest.setS3ArchiveConfigurationId(bulkMigrationRequest.getS3ArchiveConfigurationId()); - systemGenerateMetadata = validateDataObjectMigrationRequest(path, migrationRequest, false, false); - } - - for (String path : bulkMigrationRequest.getCollectionPaths()) { - migrationRequest.setS3ArchiveConfigurationId(bulkMigrationRequest.getS3ArchiveConfigurationId()); - systemGenerateMetadata = validateCollectionMigrationRequest(path, migrationRequest, false); - } - - if (systemGenerateMetadata == null) { - throw new HpcException("Could not locate system metadata", HpcErrorType.UNEXPECTED_ERROR); - } - - // Get the System generated metadata. - return systemGenerateMetadata; - } - - /** - * Validate a migration request - * - * @param path The data object path. - * @param migrationRequest The migration request. - * @param alignArchivePath If true, the file is moved within its current archive - * to align w/ the iRODs path. - * @throws HpcException If the request is invalid. - */ - private void validateMigrationRequest(String path, HpcMigrationRequestDTO migrationRequest, - boolean alignArchivePath) throws HpcException { - - // Validate the following: - // 1. Path is not empty. - // 2. Migration request is not empty if alignArchivePath is false - // 3. Migration request is empty if alignArchivePath is true - // 3. Target S3 archive configuration exists. - - // Validate the path, - if (StringUtils.isEmpty(path)) { - throw new HpcException("Null / Empty path for migration", HpcErrorType.INVALID_REQUEST_INPUT); - } - - if (!alignArchivePath) { - // Validate the migration target S3 archive configuration. - if (migrationRequest == null || StringUtils.isEmpty(migrationRequest.getS3ArchiveConfigurationId())) { - throw new HpcException("Null / Empty migration request / s3ArchiveConfigurationId", - HpcErrorType.INVALID_REQUEST_INPUT); - } - - try { - // If target S3 archive configuration not found, an exception will be raised. - dataManagementService.getS3ArchiveConfiguration(migrationRequest.getS3ArchiveConfigurationId()); - } catch (HpcException e) { - throw new HpcException( - "S3 archive configuration ID not found: " + migrationRequest.getS3ArchiveConfigurationId(), - HpcErrorType.INVALID_REQUEST_INPUT); - } - } else { - if (migrationRequest != null) { - throw new HpcException("migration request provided w/ alignArchivePath set to true", - HpcErrorType.INVALID_REQUEST_INPUT); - } - } - } - - /** - * Migrate a data object. Validate the request and create a migration task. - * - * @param path The data object path. - * @param userId The user ID submitted the request. - * @param collectionMigrationTaskId (Optional) A collection migration task id - * that this data object migration is part of. - * @param migrationRequest The migration request. - * @param alignArchivePath If true, the file is moved within its - * current archive to align w/ the iRODs path. - * @param retryTaskId The previous task ID if this is a retry - * request. - * @param retryUserId The user retrying the request if this is a - * retry request. - * @return A data migration response DTO. - * @throws HpcException If failed to process the request. - */ - private HpcMigrationResponseDTO migrateDataObject(String path, String userId, String collectionMigrationTaskId, - HpcMigrationRequestDTO migrationRequest, boolean alignArchivePath, String retryTaskId, String retryUserId) - throws HpcException { - logger.info("Migrating Data Object: path - {}, align-archive-path - {}", path, alignArchivePath); - - // Input validation. - HpcSystemGeneratedMetadata metadata = null; - HpcMigrationResponseDTO migrationResponse = new HpcMigrationResponseDTO(); - try { - metadata = validateDataObjectMigrationRequest(path, migrationRequest, alignArchivePath, false); - - } catch (Exception e) { - if (!StringUtils.isEmpty(collectionMigrationTaskId)) { - // While processing a collection download, if a validation request invalid, we - // create a task and complete as failed. - HpcDataMigrationTask dataObjectMigrationTask = dataMigrationService.createDataObjectMigrationTask(path, - userId, null, null, - migrationRequest != null ? migrationRequest.getS3ArchiveConfigurationId() : null, - collectionMigrationTaskId, alignArchivePath, metadata != null ? metadata.getSourceSize() : null, - retryTaskId, retryUserId, false, null, null); - dataMigrationService.completeDataObjectMigrationTask(dataObjectMigrationTask, - HpcDataMigrationResult.IGNORED, "Invalid migration request: " + e.getMessage(), null, null); - migrationResponse.setTaskId(dataObjectMigrationTask.getId()); - return migrationResponse; - } - - throw e; - } - - // Create a migration task. - migrationResponse.setTaskId(dataMigrationService.createDataObjectMigrationTask(path, userId, - metadata.getConfigurationId(), metadata.getS3ArchiveConfigurationId(), - migrationRequest != null ? migrationRequest.getS3ArchiveConfigurationId() : null, - collectionMigrationTaskId, alignArchivePath, metadata.getSourceSize(), retryTaskId, retryUserId, false, - null, null).getId()); - - return migrationResponse; - } - - /** - * Migrate a collection to another archive. - * - * @param path The collection path. - * @param userId The user ID submitted the request. - * @param migrationRequest The migration request DTO. - * @param alignArchivePath If true, the file is moved within its current - * archive to align w/ the iRODs path. - * @param retryTaskId The previous task ID if this is a retry request. - * @param retryUserId The user retrying the request if this is a retry - * request. - * @param retryFailedItemsOnly if set to true, only failed items of 'taskId' - * will be retried. Otherwise the collection will be - * re-scanned for a new migration to include any - * items added since the previous migration attempt. - * @return Migration Response DTO. - * @throws HpcException on service failure. - */ - private HpcMigrationResponseDTO migrateCollection(String path, String userId, - HpcMigrationRequestDTO migrationRequest, boolean alignArchivePath, String retryTaskId, String retryUserId, - Boolean retryFailedItemsOnly) throws HpcException { - // Input validation. - HpcSystemGeneratedMetadata metadata = validateCollectionMigrationRequest(path, migrationRequest, - alignArchivePath); - - // Create a migration task. - HpcMigrationResponseDTO migrationResponse = new HpcMigrationResponseDTO(); - migrationResponse.setTaskId( - dataMigrationService.createCollectionMigrationTask(path, userId, metadata.getConfigurationId(), - migrationRequest != null ? migrationRequest.getS3ArchiveConfigurationId() : null, - alignArchivePath, retryTaskId, retryUserId, retryFailedItemsOnly).getId()); - - return migrationResponse; - } - - /** - * Migrate data objects or collections. Note: API doesn't support mixed, so user - * expected to provide a list of data objects or a list of collections, not - * both. - * - * @param migrationRequest The migration request DTO. - * @param userId The user ID submitted the request. - * @param retryTaskId The previous task ID if this is a retry request. - * @param retryUserId The user retrying the request if this is a retry - * request. - * @param retryFailedItemsOnly if set to true, only failed items of 'taskId' - * will be retried. Otherwise the collection will be - * re-scanned for a new migration to include any - * items added since the previous migration attempt. - * @return Migration Response DTO. - * @throws HpcException on service failure. - */ - private HpcMigrationResponseDTO migrateDataObjectsOrCollections(HpcBulkMigrationRequestDTO migrationRequest, - String userId, String retryTaskId, String retryUserId, Boolean retryFailedItemsOnly) throws HpcException { - // Input validation. - HpcSystemGeneratedMetadata metadata = validateBulkMigrationRequest(migrationRequest); - - HpcDataMigrationTask migrationTask = null; - if (!migrationRequest.getDataObjectPaths().isEmpty()) { - // Submit a request to migrate a list of data objects. - migrationTask = dataMigrationService.createDataObjectsMigrationTask(migrationRequest.getDataObjectPaths(), - userId, metadata.getConfigurationId(), migrationRequest.getS3ArchiveConfigurationId(), retryTaskId, - retryUserId, retryFailedItemsOnly); - - } else { - // Submit a request to migrate a list of collections. - migrationTask = dataMigrationService.createCollectionsMigrationTask(migrationRequest.getCollectionPaths(), - userId, metadata.getConfigurationId(), migrationRequest.getS3ArchiveConfigurationId(), retryTaskId, - retryUserId, retryFailedItemsOnly); - } - // Create and return a DTO with the request receipt. - HpcMigrationResponseDTO migrationResponse = new HpcMigrationResponseDTO(); - migrationResponse.setTaskId(migrationTask.getId()); - - return migrationResponse; - } - - /** - * Migrate a collection. Submit a migration task for each data object under the - * collection. - * - * @param collection The collection to be migrated - * @param collectionMigrationTask The migration task. - * @throws HpcException If failed to process the request. - */ - private void migrateCollection(HpcCollection collection, HpcDataMigrationTask collectionMigrationTask) - throws HpcException { - if (Optional.ofNullable(collectionMigrationTask.getRetryFailedItemsOnly()).orElse(false)) { - migrateFailedItems(collectionMigrationTask); - return; - } - - // Iterate through the data objects in the collection and migrate them. - for (HpcCollectionListingEntry dataObjectEntry : collection.getDataObjects()) { - // Iterate through the data objects directly under this collection and submit a - // migration task for each - HpcMigrationRequestDTO migrationRequest = null; - if (!collectionMigrationTask.getAlignArchivePath()) { - migrationRequest = new HpcMigrationRequestDTO(); - migrationRequest.setS3ArchiveConfigurationId(collectionMigrationTask.getToS3ArchiveConfigurationId()); - } - migrateDataObject(dataObjectEntry.getPath(), collectionMigrationTask.getUserId(), - collectionMigrationTask.getId(), migrationRequest, collectionMigrationTask.getAlignArchivePath(), - null, null); - } - - // Iterate through the sub-collections and migrate them. - for (HpcCollectionListingEntry subCollectionEntry : collection.getSubCollections()) { - String subCollectionPath = subCollectionEntry.getPath(); - HpcCollection subCollection = dataManagementService.getFullCollection(subCollectionPath, null); - if (subCollection != null) { - // Migrate this sub-collection. - migrateCollection(subCollection, collectionMigrationTask); - } - } - } - - /** - * Update data object metadata. Validate the request and create a migration - * task. - * - * @param path The data object path. - * @param bulkMetadataUpdateTask The bulk metadata update task that this data - * object migration is part of. - * @throws HpcException If failed to process the request. - */ - private void updateDataObjectMetadata(String path, HpcDataMigrationTask bulkMetadataUpdateTask) - throws HpcException { - logger.info("Processing data object in a bulk metadata migration task: path - {}, bulk-metadata-task-id - {}", - path, bulkMetadataUpdateTask.getId()); - - // Input validation. - HpcSystemGeneratedMetadata metadata = null; - try { - metadata = validateDataObjectMigrationRequest(path, null, false, true); - - } catch (Exception e) { - // On error / validation failure - we create a data object metadata update task - // and mark it ignored. - HpcDataMigrationTask dataObjectMetadataUpdateTask = dataMigrationService.createDataObjectMigrationTask(path, - bulkMetadataUpdateTask.getUserId(), null, bulkMetadataUpdateTask.getFromS3ArchiveConfigurationId(), - bulkMetadataUpdateTask.getToS3ArchiveConfigurationId(), bulkMetadataUpdateTask.getId(), false, - metadata != null ? Optional.ofNullable(metadata.getSourceSize()).orElse(0L) : 0, null, null, true, - bulkMetadataUpdateTask.getMetadataFromArchiveFileContainerId(), - bulkMetadataUpdateTask.getMetadataToArchiveFileContainerId()); - dataMigrationService.completeDataObjectMetadataUpdateTask(dataObjectMetadataUpdateTask, - HpcDataMigrationResult.IGNORED_METADATA_MIGRATION_TRANSFER_INCOMPLETE, - "Invalid metadata update request: " + e.getMessage()); - - logger.error("Failed to create Data Object Metadata update task: path - {}, bulk-metadata-task-id - {}", - path, bulkMetadataUpdateTask.getId(), e); - return; - } - - // Create a migration task. - dataMigrationService.createDataObjectMigrationTask(path, bulkMetadataUpdateTask.getUserId(), - metadata.getConfigurationId(), bulkMetadataUpdateTask.getFromS3ArchiveConfigurationId(), - bulkMetadataUpdateTask.getToS3ArchiveConfigurationId(), bulkMetadataUpdateTask.getId(), false, - metadata.getSourceSize(), null, null, true, - bulkMetadataUpdateTask.getMetadataFromArchiveFileContainerId(), - bulkMetadataUpdateTask.getMetadataToArchiveFileContainerId()); - } - - /** - * Complete data object metadata update migration task. Checks if the data - * object is present in the new S3 archive and update the system metadata - * accordingly. - * - * @param dataObjectMetadataUpdateTask The data object metadata update migration - * task. - * @throws HpcException If failed to update the metadata - */ - private void completeDataObjectMetadataUpdate(HpcDataMigrationTask dataObjectMetadataUpdateTask) - throws HpcException { - - // Update the task and persist. - dataObjectMetadataUpdateTask.setStatus(HpcDataMigrationStatus.IN_PROGRESS); - dataMigrationService.updateDataMigrationTask(dataObjectMetadataUpdateTask); - - // TODO: - insert thread pool exec here - - // Set the from/to archive locations on the task - HpcSystemGeneratedMetadata systemGeneratedMetadata = metadataService - .getDataObjectSystemGeneratedMetadata(dataObjectMetadataUpdateTask.getPath()); - dataObjectMetadataUpdateTask.setFromS3ArchiveLocation(systemGeneratedMetadata.getArchiveLocation()); - - HpcFileLocation toS3ArchiveLocation = new HpcFileLocation(); - toS3ArchiveLocation.setFileContainerId(dataObjectMetadataUpdateTask.getMetadataToArchiveFileContainerId()); - toS3ArchiveLocation.setFileId(systemGeneratedMetadata.getArchiveLocation().getFileId()); - dataObjectMetadataUpdateTask.setToS3ArchiveLocation(toS3ArchiveLocation); - - // Validate if metadata update is needed. - if (systemGeneratedMetadata.getS3ArchiveConfigurationId() - .equals(dataObjectMetadataUpdateTask.getToS3ArchiveConfigurationId()) - && systemGeneratedMetadata.getArchiveLocation().getFileContainerId() - .equals(dataObjectMetadataUpdateTask.getMetadataToArchiveFileContainerId())) { - // The data object is already in the new archive. - dataMigrationService.completeDataObjectMetadataUpdateTask(dataObjectMetadataUpdateTask, - HpcDataMigrationResult.IGNORED_METADATA_MIGRATION_TRANSFER_INCOMPLETE, - "data object is already in S3 archive ID: " - + systemGeneratedMetadata.getS3ArchiveConfigurationId()); - return; - } - - // Locate the file in the new archive. - HpcPathAttributes archivePathAttributes = dataTransferService.getPathAttributes(HpcDataTransferType.S_3, - dataObjectMetadataUpdateTask.getToS3ArchiveLocation(), true, - dataObjectMetadataUpdateTask.getConfigurationId(), - dataObjectMetadataUpdateTask.getToS3ArchiveConfigurationId()); - - // Validate the data object exists and accessible. - if (!archivePathAttributes.getExists() || !archivePathAttributes.getIsAccessible() - || !archivePathAttributes.getIsFile()) { - dataMigrationService.completeDataObjectMetadataUpdateTask(dataObjectMetadataUpdateTask, - HpcDataMigrationResult.IGNORED_METADATA_MIGRATION_TRANSFER_INCOMPLETE, - ": Data object not found in new archive location - " - + dataObjectMetadataUpdateTask.getToS3ArchiveLocation().getFileContainerId() + " : " - + dataObjectMetadataUpdateTask.getToS3ArchiveLocation().getFileId()); - return; - } - - // Validate the file size in the new archive identical to what is in the old - // archive - if (!dataObjectMetadataUpdateTask.getSize().equals(archivePathAttributes.getSize())) { - dataMigrationService.completeDataObjectMetadataUpdateTask(dataObjectMetadataUpdateTask, - HpcDataMigrationResult.FAILED, - ": Data object size in new archive location (" + archivePathAttributes.getSize() - + ") is different than old (" + dataObjectMetadataUpdateTask.getSize() + ")" - + dataObjectMetadataUpdateTask.getToS3ArchiveLocation().getFileContainerId() + " : " - + dataObjectMetadataUpdateTask.getToS3ArchiveLocation().getFileId()); - return; - } - - // Validate the metadata is set on the data object in the new archive. - if (!dataTransferService.validateDataObjectMetadata(dataObjectMetadataUpdateTask.getToS3ArchiveLocation(), - HpcDataTransferType.S_3, dataObjectMetadataUpdateTask.getConfigurationId(), - dataObjectMetadataUpdateTask.getToS3ArchiveConfigurationId(), systemGeneratedMetadata.getObjectId(), - systemGeneratedMetadata.getRegistrarId())) { - - dataMigrationService.completeDataObjectMetadataUpdateTask(dataObjectMetadataUpdateTask, - HpcDataMigrationResult.IGNORED_METADATA_MIGRATION_TRANSFER_INCOMPLETE, - ": Data object metadata is not set in new archive location -" - + dataObjectMetadataUpdateTask.getToS3ArchiveLocation().getFileContainerId() + " : " - + dataObjectMetadataUpdateTask.getToS3ArchiveLocation().getFileId()); - return; - } - - // Complete the task by updating the system generated metadata (iRODS). - dataMigrationService.completeDataObjectMetadataUpdateTask(dataObjectMetadataUpdateTask, - HpcDataMigrationResult.COMPLETED, null); - } - - /** - * Mark a data migration task in-process to be worked on. - * - * @param dataMigrationTask The migration task to mark in-process - * @return true if the task was claimed, or false otherwise - i.e. another - * process/thread already working on the task. - * - */ - private boolean markInProcess(HpcDataMigrationTask dataMigrationTask) { - if (Optional.ofNullable(dataMigrationTask.getInProcess()).orElse(false)) { - logger.info("Data migration: task - {} already in-process by {}", dataMigrationTask.getId(), - dataMigrationTask.getServerId()); - return false; - } - - // Try to claim the task - try { - logger.info("Data migration: task - {} attempting to mark in-process.", dataMigrationTask.getId()); - if (!dataMigrationService.markInProcess(dataMigrationTask, true)) { - logger.info("Data migration: task - {} failed to mark in-process. Already in-process by {}", - dataMigrationTask.getId(), dataMigrationTask.getServerId()); - return false; - } - } catch (HpcException e) { - logger.error("Data migration: task - {} failed to mark in-process", dataMigrationTask.getId(), e); - return false; - } - - logger.info("Data migration: task - {} marked in-process", dataMigrationTask.getId()); - return true; - } - - /** - * Done processing a migration task. Mark in-process to false - * - * @param dataMigrationTask The migration task to mark done processing. - * - */ - private void doneProcessingDataMigrationTask(HpcDataMigrationTask dataMigrationTask) { - try { - dataMigrationService.markInProcess(dataMigrationTask, false); - - } catch (HpcException e) { - logger.error("Data migration: task - {} failed to mark done processing", dataMigrationTask.getId(), e); - } - } - - /** - * Migrate the failed items (files) for retry of collection (bulk) migration - * task. - * - * @param collectionMigrationTask The collection(bulk) migration task. - * @throws HpcException If failed to process the request. - */ - private void migrateFailedItems(HpcDataMigrationTask collectionMigrationTask) throws HpcException { - if (StringUtils.isEmpty(collectionMigrationTask.getRetryTaskId())) { - throw new HpcException("Can't migrate failed items w/o the associated retry migration task", - HpcErrorType.UNEXPECTED_ERROR); - } - - for (HpcDataMigrationTaskResult failedTask : dataMigrationService - .getDataMigrationResults(collectionMigrationTask.getRetryTaskId(), HpcDataMigrationResult.FAILED)) { - // Iterate through the list of failed items for the bulk migration task that - // failed, and migrate them. - HpcMigrationRequestDTO migrationRequest = null; - if (!collectionMigrationTask.getAlignArchivePath()) { - migrationRequest = new HpcMigrationRequestDTO(); - migrationRequest.setS3ArchiveConfigurationId(collectionMigrationTask.getToS3ArchiveConfigurationId()); - } - migrateDataObject(failedTask.getPath(), collectionMigrationTask.getUserId(), - collectionMigrationTask.getId(), migrationRequest, collectionMigrationTask.getAlignArchivePath(), - null, null); - } - } - - /** - * Get a list of data object paths included in this bulk metadata update task. - * - * @param bulkMetadataUpdateTask The migration task. - * @return A list of data object paths - * @throws HpcException If failed to process the request. - */ - private List getBulkMetadataUpdateDataObjectPaths(HpcDataMigrationTask bulkMetadataUpdateTask) - throws HpcException { - // Build the compound query to identify all the data object paths to include in - // this bulk metadata update task. - HpcMetadataQueryLevelFilter levelFilter = new HpcMetadataQueryLevelFilter(); - levelFilter.setOperator(HpcMetadataQueryOperator.EQUAL); - levelFilter.setLevel(1); - - HpcMetadataQuery fromS3ConfigurationIdQuery = new HpcMetadataQuery(); - fromS3ConfigurationIdQuery.setAttribute("s3_archive_configuration_id"); - fromS3ConfigurationIdQuery.setOperator(HpcMetadataQueryOperator.EQUAL); - fromS3ConfigurationIdQuery.setValue(bulkMetadataUpdateTask.getFromS3ArchiveConfigurationId()); - fromS3ConfigurationIdQuery.setLevelFilter(levelFilter); - - HpcMetadataQuery archiveFileContainerIdQuery = new HpcMetadataQuery(); - archiveFileContainerIdQuery.setAttribute("archive_file_container_id"); - archiveFileContainerIdQuery.setOperator(HpcMetadataQueryOperator.EQUAL); - archiveFileContainerIdQuery.setValue(bulkMetadataUpdateTask.getMetadataFromArchiveFileContainerId()); - archiveFileContainerIdQuery.setLevelFilter(levelFilter); - - HpcMetadataQuery archiveFileIdPatternQuery = new HpcMetadataQuery(); - archiveFileIdPatternQuery.setAttribute("archive_file_id"); - archiveFileIdPatternQuery.setOperator(HpcMetadataQueryOperator.LIKE); - archiveFileIdPatternQuery.setValue(bulkMetadataUpdateTask.getMetadataArchiveFileIdPattern()); - archiveFileIdPatternQuery.setLevelFilter(levelFilter); - - HpcCompoundMetadataQuery bulkMetadataUpdateCompoundQuery = new HpcCompoundMetadataQuery(); - bulkMetadataUpdateCompoundQuery.setOperator(HpcCompoundMetadataQueryOperator.AND); - bulkMetadataUpdateCompoundQuery.getQueries().add(fromS3ConfigurationIdQuery); - bulkMetadataUpdateCompoundQuery.getQueries().add(archiveFileContainerIdQuery); - if (!archiveFileIdPatternQuery.getValue().equals("%")) { - bulkMetadataUpdateCompoundQuery.getQueries().add(archiveFileIdPatternQuery); - } - - HpcCompoundMetadataQueryDTO bulkMetadataUpdateCompoundQueryDTO = new HpcCompoundMetadataQueryDTO(); - bulkMetadataUpdateCompoundQueryDTO.setTotalCount(true); - bulkMetadataUpdateCompoundQueryDTO.setDetailedResponse(false); - bulkMetadataUpdateCompoundQueryDTO.setPage(1); - bulkMetadataUpdateCompoundQueryDTO.setPageSize(getMetadataUpdateObjectsDefaultPageSize); - bulkMetadataUpdateCompoundQueryDTO.setCompoundQuery(bulkMetadataUpdateCompoundQuery); - - // Perform the search for the data objects to include in this metadata update - // and use pagination to retrieve all the results. - List dataObjectPaths = new ArrayList<>(); - int totalCount = 0; - do { - HpcDataObjectListDTO searchResponseDTO = dataSearchBusService - .getDataObjects(bulkMetadataUpdateCompoundQueryDTO); - dataObjectPaths.addAll(searchResponseDTO.getDataObjectPaths()); - bulkMetadataUpdateCompoundQueryDTO.setPage(searchResponseDTO.getPage() + 1); - totalCount = searchResponseDTO.getTotalCount(); - } while (dataObjectPaths.size() < totalCount); - - if (dataObjectPaths.size() == 0) { - logger.info( - "Bulk metadata update migration task - {} : No data objects found. s3_archive_configuration_id - {}, archive_file_container_id - {}", - bulkMetadataUpdateTask.getId(), bulkMetadataUpdateTask.getFromS3ArchiveConfigurationId(), - bulkMetadataUpdateTask.getMetadataFromArchiveFileContainerId()); - } - - return dataObjectPaths; - } + // ---------------------------------------------------------------------// + // Constants + // ---------------------------------------------------------------------// + + // ---------------------------------------------------------------------// + // Instance members + // ---------------------------------------------------------------------// + + // The logger instance. + private final Logger logger = LoggerFactory.getLogger(this.getClass().getName()); + // The collection download task executor. + @Autowired + @Qualifier("hpcDataObjectMetadataUpdateTaskExecutor") + Executor dataObjectMetadataUpdateTaskExecutor = null; + // The Data Migration Application Service Instance. + @Autowired + private HpcDataMigrationService dataMigrationService = null; + // The Data Management Application Service Instance. + @Autowired + private HpcDataManagementService dataManagementService = null; + // The Metadata Application Service Instance. + @Autowired + private HpcMetadataService metadataService = null; + // The Security Application Service Instance. + @Autowired + private HpcSecurityService securityService = null; + // The Data Transfer Application Service Instance. + @Autowired + private HpcDataTransferService dataTransferService = null; + // The Data Search Business Service Instance. + @Autowired + private HpcDataSearchBusService dataSearchBusService = null; + @Value("${hpc.bus.getMetadataUpdateObjectsDefaultPageSize}") + private int getMetadataUpdateObjectsDefaultPageSize = 0; + + // ---------------------------------------------------------------------// + // Constructors + // ---------------------------------------------------------------------// + + /** + * Constructor for Spring Dependency Injection. + */ + private HpcDataMigrationBusServiceImpl() { + } + + // ---------------------------------------------------------------------// + // Methods + // ---------------------------------------------------------------------// + + // ---------------------------------------------------------------------// + // HpcDataMigrationBusService Interface Implementation + // ---------------------------------------------------------------------// + + @Override + public HpcMigrationResponseDTO migrateDataObject(String path, HpcMigrationRequestDTO migrationRequest, + boolean alignArchivePath) throws HpcException { + return migrateDataObject(path, securityService.getRequestInvoker().getNciAccount().getUserId(), null, + migrationRequest, alignArchivePath, null, null); + } + + @Override + public HpcMigrationResponseDTO migrateCollection(String path, HpcMigrationRequestDTO migrationRequest, + boolean alignArchivePath) throws HpcException { + return migrateCollection(path, securityService.getRequestInvoker().getNciAccount().getUserId(), + migrationRequest, alignArchivePath, null, null, null); + } + + @Override + public HpcMigrationResponseDTO migrateDataObjectsOrCollections(HpcBulkMigrationRequestDTO migrationRequest) + throws HpcException { + return migrateDataObjectsOrCollections(migrationRequest, + securityService.getRequestInvoker().getNciAccount().getUserId(), null, null, null); + } + + @Override + public HpcMigrationResponseDTO retryDataObjectMigrationTask(String taskId) throws HpcException { + // Input validation. + HpcDataMigrationTaskStatus taskStatus = dataMigrationService.getMigrationTaskStatus(taskId, + HpcDataMigrationType.DATA_OBJECT); + if (taskStatus == null) { + throw new HpcException("Data object migration task not found: " + taskId, + HpcErrorType.INVALID_REQUEST_INPUT); + } + if (taskStatus.getInProgress() || taskStatus.getResult() == null) { + throw new HpcException("Data object migration task in-progress: " + taskId, + HpcErrorType.INVALID_REQUEST_INPUT); + } + if (taskStatus.getResult() != null + && taskStatus.getResult().getResult().equals(HpcDataMigrationResult.COMPLETED)) { + throw new HpcException("Data-object migration task already completed: " + taskId, + HpcErrorType.INVALID_REQUEST_INPUT); + } + HpcDataMigrationTaskResult migrationTask = taskStatus.getResult(); + HpcMigrationRequestDTO migrationRequest = new HpcMigrationRequestDTO(); + migrationRequest.setS3ArchiveConfigurationId(migrationTask.getToS3ArchiveConfigurationId()); + + return migrateDataObject(migrationTask.getPath(), migrationTask.getUserId(), null, migrationRequest, + migrationTask.getAlignArchivePath(), taskId, + securityService.getRequestInvoker().getNciAccount().getUserId()); + } + + @Override + public HpcMigrationResponseDTO retryCollectionMigrationTask(String taskId, Boolean failedItemsOnly) + throws HpcException { + // Input validation. + HpcDataMigrationTaskStatus taskStatus = dataMigrationService.getMigrationTaskStatus(taskId, + HpcDataMigrationType.COLLECTION); + if (taskStatus == null) { + throw new HpcException("Collection migration task not found: " + taskId, + HpcErrorType.INVALID_REQUEST_INPUT); + } + if (taskStatus.getInProgress() || taskStatus.getResult() == null) { + throw new HpcException("Collection migration task in-progress: " + taskId, + HpcErrorType.INVALID_REQUEST_INPUT); + } + if (taskStatus.getResult() != null + && taskStatus.getResult().getResult().equals(HpcDataMigrationResult.COMPLETED)) { + throw new HpcException("Collection migration task already completed: " + taskId, + HpcErrorType.INVALID_REQUEST_INPUT); + } + HpcDataMigrationTaskResult migrationTask = taskStatus.getResult(); + HpcMigrationRequestDTO migrationRequest = new HpcMigrationRequestDTO(); + migrationRequest.setS3ArchiveConfigurationId(migrationTask.getToS3ArchiveConfigurationId()); + + return migrateCollection(migrationTask.getPath(), migrationTask.getUserId(), migrationRequest, + migrationTask.getAlignArchivePath(), taskId, + securityService.getRequestInvoker().getNciAccount().getUserId(), + Optional.ofNullable(failedItemsOnly).orElse(true)); + } + + @Override + public HpcMigrationResponseDTO retryDataObjectsOrCollectionsMigrationTask(String taskId, Boolean failedItemsOnly) + throws HpcException { + // Input validation. + HpcDataMigrationTaskStatus taskStatus = dataMigrationService.getMigrationTaskStatus(taskId, + HpcDataMigrationType.DATA_OBJECT_LIST); + if (taskStatus == null) { + taskStatus = dataMigrationService.getMigrationTaskStatus(taskId, HpcDataMigrationType.COLLECTION_LIST); + } + if (taskStatus == null) { + throw new HpcException("Bulk migration task not found: " + taskId, HpcErrorType.INVALID_REQUEST_INPUT); + } + if (taskStatus.getInProgress() || taskStatus.getResult() == null) { + throw new HpcException("Bulk migration task in-progress: " + taskId, HpcErrorType.INVALID_REQUEST_INPUT); + } + if (taskStatus.getResult() != null + && taskStatus.getResult().getResult().equals(HpcDataMigrationResult.COMPLETED)) { + throw new HpcException("Bulk migration task already completed: " + taskId, + HpcErrorType.INVALID_REQUEST_INPUT); + } + HpcDataMigrationTaskResult migrationTask = taskStatus.getResult(); + HpcBulkMigrationRequestDTO migrationRequest = new HpcBulkMigrationRequestDTO(); + migrationRequest.setS3ArchiveConfigurationId(migrationTask.getToS3ArchiveConfigurationId()); + migrationRequest.getDataObjectPaths().addAll(migrationTask.getDataObjectPaths()); + migrationRequest.getCollectionPaths().addAll(migrationTask.getCollectionPaths()); + + return migrateDataObjectsOrCollections(migrationRequest, migrationTask.getUserId(), taskId, + securityService.getRequestInvoker().getNciAccount().getUserId(), + Optional.ofNullable(failedItemsOnly).orElse(true)); + } + + @Override + public HpcMigrationResponseDTO migrateMetadata(HpcMetadataMigrationRequestDTO metadataMigrationRequest) + throws HpcException { + // Input Validation + if (metadataMigrationRequest == null) { + throw new HpcException("Metadata migration request is null", HpcErrorType.INVALID_REQUEST_INPUT); + } + + if (StringUtils.isEmpty(metadataMigrationRequest.getFromS3ArchiveConfigurationId())) { + throw new HpcException("from S3 archive configuration ID is empty", HpcErrorType.INVALID_REQUEST_INPUT); + } else { + // Exception will be thrown if the S3 config ID can't be found. + dataManagementService.getS3ArchiveConfiguration(metadataMigrationRequest.getFromS3ArchiveConfigurationId()); + } + + if (StringUtils.isEmpty(metadataMigrationRequest.getToS3ArchiveConfigurationId())) { + throw new HpcException("To S3 archive configuration ID is empty", HpcErrorType.INVALID_REQUEST_INPUT); + } else { + // Exception will be thrown if the S3 config ID can't be found. + dataManagementService.getS3ArchiveConfiguration(metadataMigrationRequest.getToS3ArchiveConfigurationId()); + } + + if (metadataMigrationRequest.getFromS3ArchiveConfigurationId() + .equals(metadataMigrationRequest.getToS3ArchiveConfigurationId())) { + throw new HpcException("From/To S3 archive configuration ID is identical", + HpcErrorType.INVALID_REQUEST_INPUT); + } + + if (StringUtils.isEmpty(metadataMigrationRequest.getFromArchiveFileContainerId())) { + throw new HpcException("From Archive File Container ID is empty", HpcErrorType.INVALID_REQUEST_INPUT); + } + + if (StringUtils.isEmpty(metadataMigrationRequest.getToArchiveFileContainerId())) { + throw new HpcException("To Archive File Container ID is empty", HpcErrorType.INVALID_REQUEST_INPUT); + } + + // Create a migration task to perform bulk metadata update. + // Notes: + // 1. toArchiveFileContainerId is optional and defaulted to + // fromArchiveFileContainerId if not provided by the caller. + // 2. archiveFileIdPattern is optional and defaulted to '%' (match-all) if not + // provided by the caller + HpcMigrationResponseDTO migrationResponse = new HpcMigrationResponseDTO(); + migrationResponse + .setTaskId(dataMigrationService + .createMetadataMigrationTask(metadataMigrationRequest.getFromS3ArchiveConfigurationId(), + metadataMigrationRequest.getToS3ArchiveConfigurationId(), + metadataMigrationRequest.getFromArchiveFileContainerId(), + !StringUtils.isEmpty(metadataMigrationRequest.getToArchiveFileContainerId()) + ? metadataMigrationRequest.getToArchiveFileContainerId() + : metadataMigrationRequest.getFromArchiveFileContainerId(), + !StringUtils.isEmpty(metadataMigrationRequest.getArchiveFileIdPattern()) + ? metadataMigrationRequest.getArchiveFileIdPattern() + : "%", + securityService.getRequestInvoker().getNciAccount().getUserId()) + .getId()); + + return migrationResponse; + } + + @Override + @HpcExecuteAsSystemAccount + public void processDataObjectMigrationReceived() throws HpcException { + dataMigrationService.getDataMigrationTasks(HpcDataMigrationStatus.RECEIVED, HpcDataMigrationType.DATA_OBJECT) + .forEach(dataObjectMigrationTask -> { + if (markInProcess(dataObjectMigrationTask)) { + try { + logger.info("Migrating Data Object: task - {}, path - {}", dataObjectMigrationTask.getId(), + dataObjectMigrationTask.getPath()); + dataMigrationService.migrateDataObject(dataObjectMigrationTask); + + } catch (HpcException e) { + logger.error("Failed to migrate data object: task - {}, path - {}", + dataObjectMigrationTask.getId(), dataObjectMigrationTask.getPath(), e); + try { + dataMigrationService.completeDataObjectMigrationTask(dataObjectMigrationTask, + HpcDataMigrationResult.FAILED, e.getMessage(), null, null); + + } catch (HpcException ex) { + logger.error("Failed to complete data object migration: task - {}, path - {}", + dataObjectMigrationTask.getId(), dataObjectMigrationTask.getPath(), ex); + } + } finally { + doneProcessingDataMigrationTask(dataObjectMigrationTask); + } + } + }); + } + + @Override + @HpcExecuteAsSystemAccount + public void processCollectionMigrationReceived() throws HpcException { + dataMigrationService.getDataMigrationTasks(HpcDataMigrationStatus.RECEIVED, HpcDataMigrationType.COLLECTION) + .forEach(collectionMigrationTask -> { + if (markInProcess(collectionMigrationTask)) { + try { + logger.info("Migrating Collection: task - {}, path - {}", collectionMigrationTask.getId(), + collectionMigrationTask.getPath()); + + // Get the System generated metadata. + HpcSystemGeneratedMetadata metadata = metadataService + .getCollectionSystemGeneratedMetadata(collectionMigrationTask.getPath()); + + // Get the collection to be migrated. + HpcCollection collection = dataManagementService + .getFullCollection(collectionMigrationTask.getPath(), metadata.getLinkSourcePath()); + if (collection == null) { + throw new HpcException("Collection not found", HpcErrorType.INVALID_REQUEST_INPUT); + } + + // Create migration tasks for all data objects under this collection + migrateCollection(collection, collectionMigrationTask); + + // Mark the collection migration task - in-progress + collectionMigrationTask.setStatus(HpcDataMigrationStatus.IN_PROGRESS); + dataMigrationService.updateDataMigrationTask(collectionMigrationTask); + + } catch (HpcException e) { + logger.error("Failed to migrate collection: task - {}, path - {}", + collectionMigrationTask.getId(), collectionMigrationTask.getPath(), e); + try { + dataMigrationService.completeBulkMigrationTask(collectionMigrationTask, e.getMessage()); + + } catch (HpcException ex) { + logger.error("Failed to complete collection migration: task - {}, path - {}", + collectionMigrationTask.getId(), collectionMigrationTask.getPath(), ex); + } + } finally { + doneProcessingDataMigrationTask(collectionMigrationTask); + } + } + }); + } + + @Override + @HpcExecuteAsSystemAccount + public void processDataObjectListMigrationReceived() throws HpcException { + dataMigrationService + .getDataMigrationTasks(HpcDataMigrationStatus.RECEIVED, HpcDataMigrationType.DATA_OBJECT_LIST) + .forEach(dataObjectListMigrationTask -> { + if (markInProcess(dataObjectListMigrationTask)) { + try { + logger.info("Migrating Data Object List: task - {}, path - {}", + dataObjectListMigrationTask.getId(), + dataObjectListMigrationTask.getDataObjectPaths()); + + if (Optional.ofNullable(dataObjectListMigrationTask.getRetryFailedItemsOnly()) + .orElse(false)) { + // Retry failed items only. + migrateFailedItems(dataObjectListMigrationTask); + + } else { + // Iterate through the data objects in the list and migrate them. + for (String dataObjectPath : dataObjectListMigrationTask.getDataObjectPaths()) { + HpcMigrationRequestDTO migrationRequest = new HpcMigrationRequestDTO(); + migrationRequest.setS3ArchiveConfigurationId( + dataObjectListMigrationTask.getToS3ArchiveConfigurationId()); + migrateDataObject(dataObjectPath, dataObjectListMigrationTask.getUserId(), + dataObjectListMigrationTask.getId(), migrationRequest, + dataObjectListMigrationTask.getAlignArchivePath(), null, null); + } + } + + // Mark the collection migration task - in-progress + dataObjectListMigrationTask.setStatus(HpcDataMigrationStatus.IN_PROGRESS); + dataMigrationService.updateDataMigrationTask(dataObjectListMigrationTask); + + } catch (HpcException e) { + logger.error("Failed to migrate data object list: task - {}, path - {}", + dataObjectListMigrationTask.getId(), + dataObjectListMigrationTask.getDataObjectPaths(), e); + try { + dataMigrationService.completeBulkMigrationTask(dataObjectListMigrationTask, + e.getMessage()); + + } catch (HpcException ex) { + logger.error("Failed to complete data object list migration: task - {}, path - {}", + dataObjectListMigrationTask.getId(), + dataObjectListMigrationTask.getDataObjectPaths(), ex); + } + } finally { + doneProcessingDataMigrationTask(dataObjectListMigrationTask); + } + } + }); + } + + @Override + @HpcExecuteAsSystemAccount + public void processCollectionListMigrationReceived() throws HpcException { + dataMigrationService + .getDataMigrationTasks(HpcDataMigrationStatus.RECEIVED, HpcDataMigrationType.COLLECTION_LIST) + .forEach(collectionListMigrationTask -> { + if (markInProcess(collectionListMigrationTask)) { + try { + logger.info("Migrating Collection list: task - {}, path - {}", + collectionListMigrationTask.getId(), + collectionListMigrationTask.getCollectionPaths()); + + // Iterate through the collections in the list and migrate them. + for (String collectionPath : collectionListMigrationTask.getCollectionPaths()) { + // Get the System generated metadata. + HpcSystemGeneratedMetadata metadata = metadataService + .getCollectionSystemGeneratedMetadata(collectionPath); + + // Get the collection to be migrated. + HpcCollection collection = dataManagementService.getFullCollection(collectionPath, + metadata.getLinkSourcePath()); + if (collection == null) { + throw new HpcException("Collection not found", HpcErrorType.INVALID_REQUEST_INPUT); + } + + // Create migration tasks for all data objects under this collection + migrateCollection(collection, collectionListMigrationTask); + } + + // Mark the collection migration task - in-progress + collectionListMigrationTask.setStatus(HpcDataMigrationStatus.IN_PROGRESS); + dataMigrationService.updateDataMigrationTask(collectionListMigrationTask); + + } catch (HpcException e) { + logger.error("Failed to migrate collection list: task - {}, path - {}", + collectionListMigrationTask.getId(), + collectionListMigrationTask.getCollectionPaths(), e); + try { + dataMigrationService.completeBulkMigrationTask(collectionListMigrationTask, + e.getMessage()); + + } catch (HpcException ex) { + logger.error("Failed to complete collection list migration: task - {}, path - {}", + collectionListMigrationTask.getId(), collectionListMigrationTask.getPath(), ex); + } + } finally { + doneProcessingDataMigrationTask(collectionListMigrationTask); + } + } + }); + } + + @Override + @HpcExecuteAsSystemAccount + public void processBulkMetadataUpdateMigrationReceived() throws HpcException { + dataMigrationService + .getDataMigrationTasks(HpcDataMigrationStatus.RECEIVED, HpcDataMigrationType.BULK_METADATA_UPDATE) + .forEach(bulkMetadataUpdateTask -> { + if (markInProcess(bulkMetadataUpdateTask)) { + try { + logger.info("Processing bulk metadata update migration task - {}", + bulkMetadataUpdateTask.getId()); + + // Create metadata migration tasks for all data objects under this bulk-update + // task. + for (String dataObjectPath : getBulkMetadataUpdateDataObjectPaths(bulkMetadataUpdateTask)) { + updateDataObjectMetadata(dataObjectPath, bulkMetadataUpdateTask); + } + + // Mark the bulk metadata update task - in-progress + bulkMetadataUpdateTask.setStatus(HpcDataMigrationStatus.IN_PROGRESS); + dataMigrationService.updateDataMigrationTask(bulkMetadataUpdateTask); + + } catch (HpcException e) { + logger.error("Failed to process update-bulk-metadata: task - {}", + bulkMetadataUpdateTask.getId(), e); + try { + dataMigrationService.completeBulkMigrationTask(bulkMetadataUpdateTask, e.getMessage()); + + } catch (HpcException ex) { + logger.error("Failed to complete bulk metadata: task - {}", + bulkMetadataUpdateTask.getId(), ex); + } + } finally { + doneProcessingDataMigrationTask(bulkMetadataUpdateTask); + } + } + }); + } + + @Override + @HpcExecuteAsSystemAccount + public void processDataObjectMetadataUpdateMigrationReceived() throws HpcException { + dataMigrationService.getDataMigrationTasks(HpcDataMigrationStatus.RECEIVED, + HpcDataMigrationType.DATA_OBJECT_METADATA_UPDATE).forEach(dataObjectMetadataUpdateTask -> { + if (markInProcess(dataObjectMetadataUpdateTask)) { + try { + logger.info("Updating Data Object Metadata: task - {}, path - {}", + dataObjectMetadataUpdateTask.getId(), dataObjectMetadataUpdateTask.getPath()); + completeDataObjectMetadataUpdate(dataObjectMetadataUpdateTask); + + } catch (HpcException e) { + logger.error("Failed to update data object metadata: task - {}, path - {}", + dataObjectMetadataUpdateTask.getId(), dataObjectMetadataUpdateTask.getPath(), e); + try { + dataMigrationService.completeDataObjectMetadataUpdateTask(dataObjectMetadataUpdateTask, + HpcDataMigrationResult.FAILED, e.getMessage()); + + } catch (HpcException ex) { + logger.error("Failed to complete data object metadata update: task - {}, path - {}", + dataObjectMetadataUpdateTask.getId(), dataObjectMetadataUpdateTask.getPath(), + ex); + } + } finally { + doneProcessingDataMigrationTask(dataObjectMetadataUpdateTask); + } + } + }); + } + + @Override + @HpcExecuteAsSystemAccount + public void completeBulkMigrationInProgress() throws HpcException { + List bulkMigrationTasks = new ArrayList<>(); + bulkMigrationTasks.addAll(dataMigrationService.getDataMigrationTasks(HpcDataMigrationStatus.IN_PROGRESS, + HpcDataMigrationType.COLLECTION)); + bulkMigrationTasks.addAll(dataMigrationService.getDataMigrationTasks(HpcDataMigrationStatus.IN_PROGRESS, + HpcDataMigrationType.DATA_OBJECT_LIST)); + bulkMigrationTasks.addAll(dataMigrationService.getDataMigrationTasks(HpcDataMigrationStatus.IN_PROGRESS, + HpcDataMigrationType.COLLECTION_LIST)); + bulkMigrationTasks.addAll(dataMigrationService.getDataMigrationTasks(HpcDataMigrationStatus.IN_PROGRESS, + HpcDataMigrationType.BULK_METADATA_UPDATE)); + + bulkMigrationTasks.forEach(bulkMigrationTask -> { + try { + logger.info("Completing bulk migration: task - {}, type - {}", bulkMigrationTask.getId(), + bulkMigrationTask.getType()); + + dataMigrationService.completeBulkMigrationTask(bulkMigrationTask, null); + + } catch (HpcException e) { + logger.error("Failed to complete bulk migration: task - {}, type - {}", bulkMigrationTask.getId(), + bulkMigrationTask.getType(), e); + try { + dataMigrationService.completeBulkMigrationTask(bulkMigrationTask, e.getMessage()); + + } catch (HpcException ex) { + logger.error("Failed to complete bulk migration: task - {}, type - {}", bulkMigrationTask.getId(), + bulkMigrationTask.getType(), ex); + } + } + }); + } + + @Override + @HpcExecuteAsSystemAccount + public void processAutoTiering() throws HpcException { + // obtain a list of HPC data management configurations which are to be included in auto-tiering scan and + // create a task for each. + dataManagementService.getAutoTieringDataManagementConfigurations().forEach(dataManagementConfiguration -> { + try { + logger.info("Create bulk auto-tiering task for data management configuration: {}", + dataManagementConfiguration.getId()); + dataMigrationService.createBulkAutoTieringTask(dataManagementConfiguration.getId(), + dataManagementConfiguration.getS3UploadConfigurationId(), + dataManagementConfiguration.getS3AutoTieringConfigurationId(), null); + + } catch (HpcException e) { + logger.error("Failed to process auto-tiering for data management configuration: {}", + dataManagementConfiguration.getId(), e); + } + }); + } + + @Override + public HpcMigrationResponseDTO processAutoTiering(String configurationId) throws HpcException { + // Validate configuration ID is not null or empty + if (StringUtils.isEmpty(configurationId)) { + throw new HpcException("Invalid configuration ID: null or empty", + HpcErrorType.INVALID_REQUEST_INPUT); + } + + // Validate the configuration ID is included in the IDs that are configured for auto-tiering + boolean isAutoTieringEnabled = dataManagementService.getAutoTieringDataManagementConfigurations() + .stream() + .anyMatch(config -> configurationId.equals(config.getId())); + + if (!isAutoTieringEnabled) { + throw new HpcException("Configuration ID " + configurationId + " doesn't exist or is not configured for auto-tiering", + HpcErrorType.INVALID_REQUEST_INPUT); + } + + // Get the specific data management configuration + var dataManagementConfiguration = dataManagementService.getDataManagementConfiguration(configurationId); + if (dataManagementConfiguration == null) { + throw new HpcException("Data management configuration not found for ID: " + configurationId, + HpcErrorType.UNEXPECTED_ERROR); + } + + // Process auto-tiering for the specific configuration + logger.info("Create bulk auto-tiering task for data management configuration ID: {}", configurationId); + HpcDataMigrationTask migrationTask = dataMigrationService.createBulkAutoTieringTask( + dataManagementConfiguration.getId(), + dataManagementConfiguration.getS3UploadConfigurationId(), + dataManagementConfiguration.getS3AutoTieringConfigurationId(), + securityService.getRequestInvoker().getNciAccount().getUserId()); + + // Create and return the migration response with task ID + HpcMigrationResponseDTO migrationResponse = new HpcMigrationResponseDTO(); + migrationResponse.setTaskId(migrationTask.getId()); + return migrationResponse; + } + + @Override + @HpcExecuteAsSystemAccount + public void processBulkAutoTieringMigrationReceived() throws HpcException { + dataMigrationService + .getDataMigrationTasks(HpcDataMigrationStatus.RECEIVED, HpcDataMigrationType.BULK_AUTO_TIERING) + .forEach(bulkAutoTieringTask -> { + if (markInProcess(bulkAutoTieringTask)) { + try { + logger.info("Processing bulk auto-tiering migration task - {}", + bulkAutoTieringTask.getId()); + + // Get data objects eligible for auto-tiering and create individual migration tasks + processBulkAutoTieringMigration(bulkAutoTieringTask); + + // Mark the bulk metadata update task - in-progress + bulkAutoTieringTask.setStatus(HpcDataMigrationStatus.IN_PROGRESS); + dataMigrationService.updateDataMigrationTask(bulkAutoTieringTask); + + } catch (HpcException e) { + logger.error("Failed to process bulk auto-tiering: task - {}", + bulkAutoTieringTask.getId(), e); + try { + dataMigrationService.completeBulkMigrationTask(bulkAutoTieringTask, e.getMessage()); + + } catch (HpcException ex) { + logger.error("Failed to complete bulk auto-tiering: task - {}", + bulkAutoTieringTask.getId(), ex); + } + } finally { + doneProcessingDataMigrationTask(bulkAutoTieringTask); + } + } + }); + } + + @Override + @HpcExecuteAsSystemAccount + public void assignMigrationServer() throws HpcException { + dataMigrationService.assignDataMigrationTasks(); + } + + @Override + @HpcExecuteAsSystemAccount + public void restartDataMigrationTasks() throws HpcException { + dataMigrationService.resetMigrationTasksInProcess(); + } + + // ---------------------------------------------------------------------// + // Helper Methods + // ---------------------------------------------------------------------// + + /** + * Validate a data object migration request + * + * @param path The data object path. + * @param migrationRequest The migration request. + * @param alignArchivePath If true, the file is moved within its current + * archive to align w/ the iRODs path. + * @param metadataUpdateRequest true if this is a metadata update request, false + * if it's data migration request. + * @return The data object system metadata + * @throws HpcException If the request is invalid. + */ + private HpcSystemGeneratedMetadata validateDataObjectMigrationRequest(String path, + HpcMigrationRequestDTO migrationRequest, boolean alignArchivePath, boolean metadataUpdateRequest) + throws HpcException { + + // Validate the following: + // 1. Path is not empty. + // 2. Migration request is not empty. + // 3. Data Object exists. + // 4. Migration is not supported for links. + // 5. Migration is supported only from S3 archive to S3 Archive. + // 6. Data Object is archived (i.e. registration completed). + // 7. The source-size system metadata is available. + + if (!metadataUpdateRequest) { + validateMigrationRequest(path, migrationRequest, alignArchivePath); + } + + // Get the System generated metadata. + HpcSystemGeneratedMetadata metadata = metadataService.getDataObjectSystemGeneratedMetadata(path); + + if (!StringUtils.isEmpty(metadata.getLinkSourcePath())) { + throw new HpcException("Migration request is not supported for soft-links", + HpcErrorType.INVALID_REQUEST_INPUT); + } + + // Migration supported only from S3 archive. + if (metadata.getDataTransferType() == null || !metadata.getDataTransferType().equals(HpcDataTransferType.S_3)) { + throw new HpcException("Migration request is not supported from POSIX based file system archive", + HpcErrorType.INVALID_REQUEST_INPUT); + } + + // Validate the file is archived. + HpcDataTransferUploadStatus dataTransferStatus = metadata.getDataTransferStatus(); + if (dataTransferStatus == null) { + throw new HpcException("Unknown upload data transfer status: " + path, HpcErrorType.UNEXPECTED_ERROR); + } + if (!dataTransferStatus.equals(HpcDataTransferUploadStatus.ARCHIVED) + && !dataTransferStatus.equals(HpcDataTransferUploadStatus.DELETE_REQUESTED)) { + throw new HpcException( + "Data Object [" + path + "] is not in archived or soft-deleted state. It is in " + + metadata.getDataTransferStatus().value() + " state", + HpcRequestRejectReason.FILE_NOT_ARCHIVED); + } + + // Validate a source-size system metadata is available. + if (metadata.getSourceSize() == null) { + throw new HpcException("Source size system-metadata is missing for data object to be migrated", + HpcErrorType.UNEXPECTED_ERROR); + } + + return metadata; + } + + /** + * Validate a collection migration request + * + * @param path The data object path. + * @param migrationRequest The migration request. + * @param alignArchivePath If true, the file is moved within its current archive + * to align w/ the iRODs path. + * @return The data object system metadata + * @throws HpcException If the request is invalid. + */ + private HpcSystemGeneratedMetadata validateCollectionMigrationRequest(String path, + HpcMigrationRequestDTO migrationRequest, boolean alignArchivePath) throws HpcException { + + // Validate the following: + // 1. Path is not empty. + // 2. Migration is not supported for links. + // 3. Migration is supported only from S3 archive to S3 Archive. + // 4. Collection exists. + // 5. Collection is not empty + // 6. Collection is not linked to another collection + + validateMigrationRequest(path, migrationRequest, alignArchivePath); + + // Get the System generated metadata. + HpcSystemGeneratedMetadata metadata = metadataService.getCollectionSystemGeneratedMetadata(path); + + logger.debug("Validating collection migration request for [{}] metadata returned - {}", path, (metadata == null ? "" : metadata.toString())); + + // Validate collection is not linked. + if (!StringUtils.isEmpty(metadata.getLinkSourcePath())) { + throw new HpcException("Collection is linked to: " + metadata.getLinkSourcePath(), + HpcErrorType.INVALID_REQUEST_INPUT); + } + + // Validate collection exists. + HpcCollection collection = dataManagementService.getCollection(path, true, metadata.getLinkSourcePath()); + if (collection == null) { + throw new HpcException("Collection doesn't exist: " + path, HpcErrorType.INVALID_REQUEST_INPUT); + } + + // Verify data objects found under this collection. + if (!dataManagementService.hasDataObjects(collection)) { + // No data objects found under this collection. + throw new HpcException("No data objects found under collection" + path, HpcErrorType.INVALID_REQUEST_INPUT); + } + + return metadata; + } + + /** + * Validate a bulk migration request + * + * @param bulkMigrationRequest The migration request. + * @return The data object system metadata + * @throws HpcException If the request is invalid. + */ + private HpcSystemGeneratedMetadata validateBulkMigrationRequest(HpcBulkMigrationRequestDTO bulkMigrationRequest) + throws HpcException { + if (bulkMigrationRequest == null) { + throw new HpcException("Null migration request", HpcErrorType.INVALID_REQUEST_INPUT); + } + + if (bulkMigrationRequest.getDataObjectPaths().isEmpty() + && bulkMigrationRequest.getCollectionPaths().isEmpty()) { + throw new HpcException("No data object or collection paths", HpcErrorType.INVALID_REQUEST_INPUT); + } + if (!bulkMigrationRequest.getDataObjectPaths().isEmpty() + && !bulkMigrationRequest.getCollectionPaths().isEmpty()) { + throw new HpcException("Both data object and collection paths provided", + HpcErrorType.INVALID_REQUEST_INPUT); + } + + HpcMigrationRequestDTO migrationRequest = new HpcMigrationRequestDTO(); + HpcSystemGeneratedMetadata systemGenerateMetadata = null; + + for (String path : bulkMigrationRequest.getDataObjectPaths()) { + migrationRequest.setS3ArchiveConfigurationId(bulkMigrationRequest.getS3ArchiveConfigurationId()); + systemGenerateMetadata = validateDataObjectMigrationRequest(path, migrationRequest, false, false); + } + + for (String path : bulkMigrationRequest.getCollectionPaths()) { + migrationRequest.setS3ArchiveConfigurationId(bulkMigrationRequest.getS3ArchiveConfigurationId()); + systemGenerateMetadata = validateCollectionMigrationRequest(path, migrationRequest, false); + } + + if (systemGenerateMetadata == null) { + throw new HpcException("Could not locate system metadata", HpcErrorType.UNEXPECTED_ERROR); + } + + // Get the System generated metadata. + return systemGenerateMetadata; + } + + /** + * Validate a migration request + * + * @param path The data object path. + * @param migrationRequest The migration request. + * @param alignArchivePath If true, the file is moved within its current archive + * to align w/ the iRODs path. + * @throws HpcException If the request is invalid. + */ + private void validateMigrationRequest(String path, HpcMigrationRequestDTO migrationRequest, + boolean alignArchivePath) throws HpcException { + + // Validate the following: + // 1. Path is not empty. + // 2. Migration request is not empty if alignArchivePath is false + // 3. Migration request is empty if alignArchivePath is true + // 3. Target S3 archive configuration exists. + + // Validate the path, + if (StringUtils.isEmpty(path)) { + throw new HpcException("Null / Empty path for migration", HpcErrorType.INVALID_REQUEST_INPUT); + } + + if (!alignArchivePath) { + // Validate the migration target S3 archive configuration. + if (migrationRequest == null || StringUtils.isEmpty(migrationRequest.getS3ArchiveConfigurationId())) { + throw new HpcException("Null / Empty migration request / s3ArchiveConfigurationId", + HpcErrorType.INVALID_REQUEST_INPUT); + } + + try { + // If target S3 archive configuration not found, an exception will be raised. + dataManagementService.getS3ArchiveConfiguration(migrationRequest.getS3ArchiveConfigurationId()); + } catch (HpcException e) { + throw new HpcException( + "S3 archive configuration ID not found: " + migrationRequest.getS3ArchiveConfigurationId(), + HpcErrorType.INVALID_REQUEST_INPUT); + } + } else { + if (migrationRequest != null) { + throw new HpcException("migration request provided w/ alignArchivePath set to true", + HpcErrorType.INVALID_REQUEST_INPUT); + } + } + } + + /** + * Migrate a data object. Validate the request and create a migration task. + * + * @param path The data object path. + * @param userId The user ID submitted the request. + * @param collectionMigrationTaskId (Optional) A collection migration task id + * that this data object migration is part of. + * @param migrationRequest The migration request. + * @param alignArchivePath If true, the file is moved within its + * current archive to align w/ the iRODs path. + * @param retryTaskId The previous task ID if this is a retry + * request. + * @param retryUserId The user retrying the request if this is a + * retry request. + * @return A data migration response DTO. + * @throws HpcException If failed to process the request. + */ + private HpcMigrationResponseDTO migrateDataObject(String path, String userId, String collectionMigrationTaskId, + HpcMigrationRequestDTO migrationRequest, boolean alignArchivePath, String retryTaskId, String retryUserId) + throws HpcException { + logger.info("Migrating Data Object: path - {}, align-archive-path - {}", path, alignArchivePath); + + // Input validation. + HpcSystemGeneratedMetadata metadata = null; + HpcMigrationResponseDTO migrationResponse = new HpcMigrationResponseDTO(); + try { + metadata = validateDataObjectMigrationRequest(path, migrationRequest, alignArchivePath, false); + + } catch (Exception e) { + if (!StringUtils.isEmpty(collectionMigrationTaskId)) { + // While processing a collection download, if a validation request invalid, we + // create a task and complete as failed. + HpcDataMigrationTask dataObjectMigrationTask = dataMigrationService.createDataObjectMigrationTask(path, + userId, null, null, + migrationRequest != null ? migrationRequest.getS3ArchiveConfigurationId() : null, + collectionMigrationTaskId, alignArchivePath, metadata != null ? metadata.getSourceSize() : null, + retryTaskId, retryUserId, false, null, null, + false, null); + dataMigrationService.completeDataObjectMigrationTask(dataObjectMigrationTask, + HpcDataMigrationResult.IGNORED, "Invalid migration request: " + e.getMessage(), null, null); + migrationResponse.setTaskId(dataObjectMigrationTask.getId()); + return migrationResponse; + } + + throw e; + } + + // Create a migration task. + migrationResponse.setTaskId(dataMigrationService.createDataObjectMigrationTask(path, userId, + metadata.getConfigurationId(), metadata.getS3ArchiveConfigurationId(), + migrationRequest != null ? migrationRequest.getS3ArchiveConfigurationId() : null, + collectionMigrationTaskId, alignArchivePath, metadata.getSourceSize(), retryTaskId, retryUserId, false, + null, null, false, null).getId()); + + return migrationResponse; + } + + /** + * Migrate a collection to another archive. + * + * @param path The collection path. + * @param userId The user ID submitted the request. + * @param migrationRequest The migration request DTO. + * @param alignArchivePath If true, the file is moved within its current + * archive to align w/ the iRODs path. + * @param retryTaskId The previous task ID if this is a retry request. + * @param retryUserId The user retrying the request if this is a retry + * request. + * @param retryFailedItemsOnly if set to true, only failed items of 'taskId' + * will be retried. Otherwise the collection will be + * re-scanned for a new migration to include any + * items added since the previous migration attempt. + * @return Migration Response DTO. + * @throws HpcException on service failure. + */ + private HpcMigrationResponseDTO migrateCollection(String path, String userId, + HpcMigrationRequestDTO migrationRequest, boolean alignArchivePath, String retryTaskId, String retryUserId, + Boolean retryFailedItemsOnly) throws HpcException { + // Input validation. + HpcSystemGeneratedMetadata metadata = validateCollectionMigrationRequest(path, migrationRequest, + alignArchivePath); + + // Create a migration task. + HpcMigrationResponseDTO migrationResponse = new HpcMigrationResponseDTO(); + migrationResponse.setTaskId( + dataMigrationService.createCollectionMigrationTask(path, userId, metadata.getConfigurationId(), + migrationRequest != null ? migrationRequest.getS3ArchiveConfigurationId() : null, + alignArchivePath, retryTaskId, retryUserId, retryFailedItemsOnly).getId()); + + return migrationResponse; + } + + /** + * Migrate data objects or collections. Note: API doesn't support mixed, so user + * expected to provide a list of data objects or a list of collections, not + * both. + * + * @param migrationRequest The migration request DTO. + * @param userId The user ID submitted the request. + * @param retryTaskId The previous task ID if this is a retry request. + * @param retryUserId The user retrying the request if this is a retry + * request. + * @param retryFailedItemsOnly if set to true, only failed items of 'taskId' + * will be retried. Otherwise the collection will be + * re-scanned for a new migration to include any + * items added since the previous migration attempt. + * @return Migration Response DTO. + * @throws HpcException on service failure. + */ + private HpcMigrationResponseDTO migrateDataObjectsOrCollections(HpcBulkMigrationRequestDTO migrationRequest, + String userId, String retryTaskId, String retryUserId, Boolean retryFailedItemsOnly) throws HpcException { + // Input validation. + HpcSystemGeneratedMetadata metadata = validateBulkMigrationRequest(migrationRequest); + + HpcDataMigrationTask migrationTask = null; + if (!migrationRequest.getDataObjectPaths().isEmpty()) { + // Submit a request to migrate a list of data objects. + migrationTask = dataMigrationService.createDataObjectsMigrationTask(migrationRequest.getDataObjectPaths(), + userId, metadata.getConfigurationId(), migrationRequest.getS3ArchiveConfigurationId(), retryTaskId, + retryUserId, retryFailedItemsOnly); + + } else { + // Submit a request to migrate a list of collections. + migrationTask = dataMigrationService.createCollectionsMigrationTask(migrationRequest.getCollectionPaths(), + userId, metadata.getConfigurationId(), migrationRequest.getS3ArchiveConfigurationId(), retryTaskId, + retryUserId, retryFailedItemsOnly); + } + // Create and return a DTO with the request receipt. + HpcMigrationResponseDTO migrationResponse = new HpcMigrationResponseDTO(); + migrationResponse.setTaskId(migrationTask.getId()); + + return migrationResponse; + } + + /** + * Migrate a collection. Submit a migration task for each data object under the + * collection. + * + * @param collection The collection to be migrated + * @param collectionMigrationTask The migration task. + * @throws HpcException If failed to process the request. + */ + private void migrateCollection(HpcCollection collection, HpcDataMigrationTask collectionMigrationTask) + throws HpcException { + if (Optional.ofNullable(collectionMigrationTask.getRetryFailedItemsOnly()).orElse(false)) { + migrateFailedItems(collectionMigrationTask); + return; + } + + // Iterate through the data objects in the collection and migrate them. + for (HpcCollectionListingEntry dataObjectEntry : collection.getDataObjects()) { + // Iterate through the data objects directly under this collection and submit a + // migration task for each + HpcMigrationRequestDTO migrationRequest = null; + if (!collectionMigrationTask.getAlignArchivePath()) { + migrationRequest = new HpcMigrationRequestDTO(); + migrationRequest.setS3ArchiveConfigurationId(collectionMigrationTask.getToS3ArchiveConfigurationId()); + } + migrateDataObject(dataObjectEntry.getPath(), collectionMigrationTask.getUserId(), + collectionMigrationTask.getId(), migrationRequest, collectionMigrationTask.getAlignArchivePath(), + null, null); + } + + // Iterate through the sub-collections and migrate them. + for (HpcCollectionListingEntry subCollectionEntry : collection.getSubCollections()) { + String subCollectionPath = subCollectionEntry.getPath(); + HpcCollection subCollection = dataManagementService.getFullCollection(subCollectionPath, null); + if (subCollection != null) { + // Migrate this sub-collection. + migrateCollection(subCollection, collectionMigrationTask); + } + } + } + + /** + * Update data object metadata. Validate the request and create a migration + * task. + * + * @param path The data object path. + * @param bulkMetadataUpdateTask The bulk metadata update task that this data + * object migration is part of. + * @throws HpcException If failed to process the request. + */ + private void updateDataObjectMetadata(String path, HpcDataMigrationTask bulkMetadataUpdateTask) + throws HpcException { + logger.info("Processing data object in a bulk metadata migration task: path - {}, bulk-metadata-task-id - {}", + path, bulkMetadataUpdateTask.getId()); + + // Input validation. + HpcSystemGeneratedMetadata metadata = null; + try { + metadata = validateDataObjectMigrationRequest(path, null, false, true); + + } catch (Exception e) { + // On error / validation failure - we create a data object metadata update task + // and mark it ignored. + HpcDataMigrationTask dataObjectMetadataUpdateTask = dataMigrationService.createDataObjectMigrationTask(path, + bulkMetadataUpdateTask.getUserId(), null, bulkMetadataUpdateTask.getFromS3ArchiveConfigurationId(), + bulkMetadataUpdateTask.getToS3ArchiveConfigurationId(), bulkMetadataUpdateTask.getId(), false, + metadata != null ? Optional.ofNullable(metadata.getSourceSize()).orElse(0L) : 0, null, null, true, + bulkMetadataUpdateTask.getMetadataFromArchiveFileContainerId(), + bulkMetadataUpdateTask.getMetadataToArchiveFileContainerId(), false, null); + dataMigrationService.completeDataObjectMetadataUpdateTask(dataObjectMetadataUpdateTask, + HpcDataMigrationResult.IGNORED_METADATA_MIGRATION_TRANSFER_INCOMPLETE, + "Invalid metadata update request: " + e.getMessage()); + + logger.error("Failed to create Data Object Metadata update task: path - {}, bulk-metadata-task-id - {}", + path, bulkMetadataUpdateTask.getId(), e); + return; + } + + // Create a migration task. + dataMigrationService.createDataObjectMigrationTask(path, bulkMetadataUpdateTask.getUserId(), + metadata.getConfigurationId(), bulkMetadataUpdateTask.getFromS3ArchiveConfigurationId(), + bulkMetadataUpdateTask.getToS3ArchiveConfigurationId(), bulkMetadataUpdateTask.getId(), false, + metadata.getSourceSize(), null, null, true, + bulkMetadataUpdateTask.getMetadataFromArchiveFileContainerId(), + bulkMetadataUpdateTask.getMetadataToArchiveFileContainerId(), false, null); + } + + /** + * Complete data object metadata update migration task. Checks if the data + * object is present in the new S3 archive and update the system metadata + * accordingly. + * + * @param dataObjectMetadataUpdateTask The data object metadata update migration + * task. + * @throws HpcException If failed to update the metadata + */ + private void completeDataObjectMetadataUpdate(HpcDataMigrationTask dataObjectMetadataUpdateTask) + throws HpcException { + + // Update the task and persist. + dataObjectMetadataUpdateTask.setStatus(HpcDataMigrationStatus.IN_PROGRESS); + dataMigrationService.updateDataMigrationTask(dataObjectMetadataUpdateTask); + + // TODO: - insert thread pool exec here + + // Set the from/to archive locations on the task + HpcSystemGeneratedMetadata systemGeneratedMetadata = metadataService + .getDataObjectSystemGeneratedMetadata(dataObjectMetadataUpdateTask.getPath()); + dataObjectMetadataUpdateTask.setFromS3ArchiveLocation(systemGeneratedMetadata.getArchiveLocation()); + + HpcFileLocation toS3ArchiveLocation = new HpcFileLocation(); + toS3ArchiveLocation.setFileContainerId(dataObjectMetadataUpdateTask.getMetadataToArchiveFileContainerId()); + toS3ArchiveLocation.setFileId(systemGeneratedMetadata.getArchiveLocation().getFileId()); + dataObjectMetadataUpdateTask.setToS3ArchiveLocation(toS3ArchiveLocation); + + // Validate if metadata update is needed. + if (systemGeneratedMetadata.getS3ArchiveConfigurationId() + .equals(dataObjectMetadataUpdateTask.getToS3ArchiveConfigurationId()) + && systemGeneratedMetadata.getArchiveLocation().getFileContainerId() + .equals(dataObjectMetadataUpdateTask.getMetadataToArchiveFileContainerId())) { + // The data object is already in the new archive. + dataMigrationService.completeDataObjectMetadataUpdateTask(dataObjectMetadataUpdateTask, + HpcDataMigrationResult.IGNORED_METADATA_MIGRATION_TRANSFER_INCOMPLETE, + "data object is already in S3 archive ID: " + + systemGeneratedMetadata.getS3ArchiveConfigurationId()); + return; + } + + // Locate the file in the new archive. + HpcPathAttributes archivePathAttributes = dataTransferService.getPathAttributes(HpcDataTransferType.S_3, + dataObjectMetadataUpdateTask.getToS3ArchiveLocation(), true, + dataObjectMetadataUpdateTask.getConfigurationId(), + dataObjectMetadataUpdateTask.getToS3ArchiveConfigurationId()); + + // Validate the data object exists and accessible. + if (!archivePathAttributes.getExists() || !archivePathAttributes.getIsAccessible() + || !archivePathAttributes.getIsFile()) { + dataMigrationService.completeDataObjectMetadataUpdateTask(dataObjectMetadataUpdateTask, + HpcDataMigrationResult.IGNORED_METADATA_MIGRATION_TRANSFER_INCOMPLETE, + ": Data object not found in new archive location - " + + dataObjectMetadataUpdateTask.getToS3ArchiveLocation().getFileContainerId() + " : " + + dataObjectMetadataUpdateTask.getToS3ArchiveLocation().getFileId()); + return; + } + + // Validate the file size in the new archive identical to what is in the old + // archive + if (!dataObjectMetadataUpdateTask.getSize().equals(archivePathAttributes.getSize())) { + dataMigrationService.completeDataObjectMetadataUpdateTask(dataObjectMetadataUpdateTask, + HpcDataMigrationResult.FAILED, + ": Data object size in new archive location (" + archivePathAttributes.getSize() + + ") is different than old (" + dataObjectMetadataUpdateTask.getSize() + ")" + + dataObjectMetadataUpdateTask.getToS3ArchiveLocation().getFileContainerId() + " : " + + dataObjectMetadataUpdateTask.getToS3ArchiveLocation().getFileId()); + return; + } + + // Validate the metadata is set on the data object in the new archive. + if (!dataTransferService.validateDataObjectMetadata(dataObjectMetadataUpdateTask.getToS3ArchiveLocation(), + HpcDataTransferType.S_3, dataObjectMetadataUpdateTask.getConfigurationId(), + dataObjectMetadataUpdateTask.getToS3ArchiveConfigurationId(), systemGeneratedMetadata.getObjectId(), + systemGeneratedMetadata.getRegistrarId())) { + + dataMigrationService.completeDataObjectMetadataUpdateTask(dataObjectMetadataUpdateTask, + HpcDataMigrationResult.IGNORED_METADATA_MIGRATION_TRANSFER_INCOMPLETE, + ": Data object metadata is not set in new archive location -" + + dataObjectMetadataUpdateTask.getToS3ArchiveLocation().getFileContainerId() + " : " + + dataObjectMetadataUpdateTask.getToS3ArchiveLocation().getFileId()); + return; + } + + // Complete the task by updating the system generated metadata (iRODS). + dataMigrationService.completeDataObjectMetadataUpdateTask(dataObjectMetadataUpdateTask, + HpcDataMigrationResult.COMPLETED, null); + } + + /** + * Mark a data migration task in-process to be worked on. + * + * @param dataMigrationTask The migration task to mark in-process + * @return true if the task was claimed, or false otherwise - i.e. another + * process/thread already working on the task. + * + */ + private boolean markInProcess(HpcDataMigrationTask dataMigrationTask) { + if (Optional.ofNullable(dataMigrationTask.getInProcess()).orElse(false)) { + logger.info("Data migration: task - {} already in-process by {}", dataMigrationTask.getId(), + dataMigrationTask.getServerId()); + return false; + } + + // Try to claim the task + try { + logger.info("Data migration: task - {} attempting to mark in-process.", dataMigrationTask.getId()); + if (!dataMigrationService.markInProcess(dataMigrationTask, true)) { + logger.info("Data migration: task - {} failed to mark in-process. Already in-process by {}", + dataMigrationTask.getId(), dataMigrationTask.getServerId()); + return false; + } + } catch (HpcException e) { + logger.error("Data migration: task - {} failed to mark in-process", dataMigrationTask.getId(), e); + return false; + } + + logger.info("Data migration: task - {} marked in-process", dataMigrationTask.getId()); + return true; + } + + /** + * Done processing a migration task. Mark in-process to false + * + * @param dataMigrationTask The migration task to mark done processing. + * + */ + private void doneProcessingDataMigrationTask(HpcDataMigrationTask dataMigrationTask) { + try { + dataMigrationService.markInProcess(dataMigrationTask, false); + + } catch (HpcException e) { + logger.error("Data migration: task - {} failed to mark done processing", dataMigrationTask.getId(), e); + } + } + + /** + * Migrate the failed items (files) for retry of collection (bulk) migration + * task. + * + * @param collectionMigrationTask The collection(bulk) migration task. + * @throws HpcException If failed to process the request. + */ + private void migrateFailedItems(HpcDataMigrationTask collectionMigrationTask) throws HpcException { + if (StringUtils.isEmpty(collectionMigrationTask.getRetryTaskId())) { + throw new HpcException("Can't migrate failed items w/o the associated retry migration task", + HpcErrorType.UNEXPECTED_ERROR); + } + + for (HpcDataMigrationTaskResult failedTask : dataMigrationService + .getDataMigrationResults(collectionMigrationTask.getRetryTaskId(), HpcDataMigrationResult.FAILED)) { + // Iterate through the list of failed items for the bulk migration task that + // failed, and migrate them. + HpcMigrationRequestDTO migrationRequest = null; + if (!collectionMigrationTask.getAlignArchivePath()) { + migrationRequest = new HpcMigrationRequestDTO(); + migrationRequest.setS3ArchiveConfigurationId(collectionMigrationTask.getToS3ArchiveConfigurationId()); + } + migrateDataObject(failedTask.getPath(), collectionMigrationTask.getUserId(), + collectionMigrationTask.getId(), migrationRequest, collectionMigrationTask.getAlignArchivePath(), + null, null); + } + } + + /** + * Get a list of data object paths included in this bulk metadata update task. + * + * @param bulkMetadataUpdateTask The migration task. + * @return A list of data object paths + * @throws HpcException If failed to process the request. + */ + private List getBulkMetadataUpdateDataObjectPaths(HpcDataMigrationTask bulkMetadataUpdateTask) + throws HpcException { + // Build the compound query to identify all the data object paths to include in + // this bulk metadata update task. + HpcMetadataQueryLevelFilter levelFilter = new HpcMetadataQueryLevelFilter(); + levelFilter.setOperator(HpcMetadataQueryOperator.EQUAL); + levelFilter.setLevel(1); + + HpcMetadataQuery fromS3ConfigurationIdQuery = new HpcMetadataQuery(); + fromS3ConfigurationIdQuery.setAttribute("s3_archive_configuration_id"); + fromS3ConfigurationIdQuery.setOperator(HpcMetadataQueryOperator.EQUAL); + fromS3ConfigurationIdQuery.setValue(bulkMetadataUpdateTask.getFromS3ArchiveConfigurationId()); + fromS3ConfigurationIdQuery.setLevelFilter(levelFilter); + + HpcMetadataQuery archiveFileContainerIdQuery = new HpcMetadataQuery(); + archiveFileContainerIdQuery.setAttribute("archive_file_container_id"); + archiveFileContainerIdQuery.setOperator(HpcMetadataQueryOperator.EQUAL); + archiveFileContainerIdQuery.setValue(bulkMetadataUpdateTask.getMetadataFromArchiveFileContainerId()); + archiveFileContainerIdQuery.setLevelFilter(levelFilter); + + HpcMetadataQuery archiveFileIdPatternQuery = new HpcMetadataQuery(); + archiveFileIdPatternQuery.setAttribute("archive_file_id"); + archiveFileIdPatternQuery.setOperator(HpcMetadataQueryOperator.LIKE); + archiveFileIdPatternQuery.setValue(bulkMetadataUpdateTask.getMetadataArchiveFileIdPattern()); + archiveFileIdPatternQuery.setLevelFilter(levelFilter); + + HpcCompoundMetadataQuery bulkMetadataUpdateCompoundQuery = new HpcCompoundMetadataQuery(); + bulkMetadataUpdateCompoundQuery.setOperator(HpcCompoundMetadataQueryOperator.AND); + bulkMetadataUpdateCompoundQuery.getQueries().add(fromS3ConfigurationIdQuery); + bulkMetadataUpdateCompoundQuery.getQueries().add(archiveFileContainerIdQuery); + if (!archiveFileIdPatternQuery.getValue().equals("%")) { + bulkMetadataUpdateCompoundQuery.getQueries().add(archiveFileIdPatternQuery); + } + + HpcCompoundMetadataQueryDTO bulkMetadataUpdateCompoundQueryDTO = new HpcCompoundMetadataQueryDTO(); + bulkMetadataUpdateCompoundQueryDTO.setTotalCount(true); + bulkMetadataUpdateCompoundQueryDTO.setDetailedResponse(false); + bulkMetadataUpdateCompoundQueryDTO.setPage(1); + bulkMetadataUpdateCompoundQueryDTO.setPageSize(getMetadataUpdateObjectsDefaultPageSize); + bulkMetadataUpdateCompoundQueryDTO.setCompoundQuery(bulkMetadataUpdateCompoundQuery); + + // Perform the search for the data objects to include in this metadata update + // and use pagination to retrieve all the results. + List dataObjectPaths = new ArrayList<>(); + int totalCount = 0; + do { + HpcDataObjectListDTO searchResponseDTO = dataSearchBusService + .getDataObjects(bulkMetadataUpdateCompoundQueryDTO); + dataObjectPaths.addAll(searchResponseDTO.getDataObjectPaths()); + bulkMetadataUpdateCompoundQueryDTO.setPage(searchResponseDTO.getPage() + 1); + totalCount = searchResponseDTO.getTotalCount(); + } while (dataObjectPaths.size() < totalCount); + + if (dataObjectPaths.size() == 0) { + logger.info( + "Bulk metadata update migration task - {} : No data objects found. s3_archive_configuration_id - {}, archive_file_container_id - {}", + bulkMetadataUpdateTask.getId(), bulkMetadataUpdateTask.getFromS3ArchiveConfigurationId(), + bulkMetadataUpdateTask.getMetadataFromArchiveFileContainerId()); + } + + return dataObjectPaths; + } + + /** + * Process a received bulk auto-tiering migration task. + * This method handles the migration of files from an external archive to S3 Glacier (auto-tiering). + * For a given DME config, It queries the files that have not been accessed + * within the specified time period and creates an individual migration task to get them registered with DME and + * then migrated to Glacier + * + * @param bulkAutoTieringTask The bulk auto-tiering migration task to process. + * @throws HpcException If failed to process the auto-tiering migration task. + */ + private void processBulkAutoTieringMigration(HpcDataMigrationTask bulkAutoTieringTask) throws HpcException { + + for(Map.Entry autoTieringEntry : + dataMigrationService.getDataObjectsForAutoTiering(bulkAutoTieringTask.getConfigurationId(), + bulkAutoTieringTask.getFromS3ArchiveConfigurationId()). + entrySet()) { + + HpcDataMigrationTask dataObjectMigrationTask = dataMigrationService.createDataObjectMigrationTask(autoTieringEntry.getKey(), + bulkAutoTieringTask.getUserId(), bulkAutoTieringTask.getConfigurationId(), + bulkAutoTieringTask.getFromS3ArchiveConfigurationId(), + bulkAutoTieringTask.getToS3ArchiveConfigurationId(), + bulkAutoTieringTask.getId(), bulkAutoTieringTask.getAlignArchivePath(), + null, null, bulkAutoTieringTask.getRetryUserId(), + false, null, null, + true, autoTieringEntry.getValue()); + + logger.info("Data object auto-tiering task - {}: created", dataObjectMigrationTask.getId()); + } + } } diff --git a/src/hpc-server/hpc-client/pom.xml b/src/hpc-server/hpc-client/pom.xml index 29085906dc..a3b3666521 100644 --- a/src/hpc-server/hpc-client/pom.xml +++ b/src/hpc-server/hpc-client/pom.xml @@ -19,7 +19,7 @@ gov.nih.nci.hpc hpc-server - 3.23.0 + 3.24.0 hpc-client jar diff --git a/src/hpc-server/hpc-common/pom.xml b/src/hpc-server/hpc-common/pom.xml index 0ef00fe75e..67bca3f75b 100644 --- a/src/hpc-server/hpc-common/pom.xml +++ b/src/hpc-server/hpc-common/pom.xml @@ -19,7 +19,7 @@ gov.nih.nci.hpc hpc-server - 3.23.0 + 3.24.0 hpc-common jar diff --git a/src/hpc-server/hpc-dao-api/pom.xml b/src/hpc-server/hpc-dao-api/pom.xml index edb252c449..33a8356bbe 100644 --- a/src/hpc-server/hpc-dao-api/pom.xml +++ b/src/hpc-server/hpc-dao-api/pom.xml @@ -19,7 +19,7 @@ gov.nih.nci.hpc hpc-server - 3.23.0 + 3.24.0 hpc-dao-api jar diff --git a/src/hpc-server/hpc-dao-api/src/main/java/gov/nih/nci/hpc/dao/HpcExternalArchiveDAO.java b/src/hpc-server/hpc-dao-api/src/main/java/gov/nih/nci/hpc/dao/HpcExternalArchiveDAO.java new file mode 100644 index 0000000000..a53f598bb5 --- /dev/null +++ b/src/hpc-server/hpc-dao-api/src/main/java/gov/nih/nci/hpc/dao/HpcExternalArchiveDAO.java @@ -0,0 +1,38 @@ +/** + * HpcExternalArchiveDAO.java + * + *

+ * Copyright SVG, Inc. Copyright Leidos Biomedical Research, Inc + * + *

+ * Distributed under the OSI-approved BSD 3-Clause License. See + * http://ncip.github.com/HPC/LICENSE.txt for details. + */ +package gov.nih.nci.hpc.dao; + +import java.util.List; + +import gov.nih.nci.hpc.exception.HpcException; + +/** + * HPC External Archive DAO Interface. + * + * This interface provides data access methods for querying files in external archives + * (VAST managed archives mounted via NFS on DME server) to support auto-tiering functionality. + * + * @author Eran Rosenberg + */ +public interface HpcExternalArchiveDAO { + /** + * Query for files in the external archive that have not been accessed within + * the specified time period. These files are candidates for auto-tiering migration + * to S3 Glacier Deep Archive. + * + * @param searchPath The external archive search path to scan for files. + * @param monthsNotAccessed The time period in months during which files were not accessed. + * Files with last access time older than this will be returned. + * @return A list of file paths that have not been accessed within the specified time period. + * @throws HpcException on service failure. + */ + List getFilesNotAccessed(String searchPath, Integer monthsNotAccessed) throws HpcException; +} diff --git a/src/hpc-server/hpc-dao-impl/pom.xml b/src/hpc-server/hpc-dao-impl/pom.xml index 0c7423741e..8917c10deb 100644 --- a/src/hpc-server/hpc-dao-impl/pom.xml +++ b/src/hpc-server/hpc-dao-impl/pom.xml @@ -18,7 +18,7 @@ gov.nih.nci.hpc hpc-server - 3.23.0 + 3.24.0 hpc-dao-impl jar @@ -60,6 +60,10 @@ commons-dbcp 1.4 + + io.trino + trino-jdbc + junit junit diff --git a/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcApiCallsAuditDAOImpl.java b/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcApiCallsAuditDAOImpl.java index 521f5165fb..5614d0f535 100644 --- a/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcApiCallsAuditDAOImpl.java +++ b/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcApiCallsAuditDAOImpl.java @@ -13,6 +13,7 @@ import java.util.Calendar; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.dao.DataAccessException; import org.springframework.jdbc.core.JdbcTemplate; @@ -44,8 +45,9 @@ public class HpcApiCallsAuditDAOImpl implements HpcApiCallsAuditDAO { // ---------------------------------------------------------------------// // The Spring JDBC Template instance. - @Autowired - private JdbcTemplate jdbcTemplate = null; + @Autowired + @Qualifier("hpcOracleJdbcTemplate") + private JdbcTemplate jdbcTemplate = null; // ---------------------------------------------------------------------// // Constructors diff --git a/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcBulkUpdateAuditDAOImpl.java b/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcBulkUpdateAuditDAOImpl.java index 070e74928f..ea7490d2ac 100644 --- a/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcBulkUpdateAuditDAOImpl.java +++ b/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcBulkUpdateAuditDAOImpl.java @@ -13,6 +13,7 @@ import java.util.Calendar; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.dao.DataAccessException; import org.springframework.jdbc.core.JdbcTemplate; @@ -48,9 +49,10 @@ public class HpcBulkUpdateAuditDAOImpl implements HpcBulkUpdateAuditDAO { // ---------------------------------------------------------------------// // The Spring JDBC Template instance. - @Autowired - private JdbcTemplate jdbcTemplate = null; - + @Autowired + @Qualifier("hpcOracleJdbcTemplate") + private JdbcTemplate jdbcTemplate = null; + private Gson gson = new Gson(); // ---------------------------------------------------------------------// diff --git a/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcCatalogDAOImpl.java b/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcCatalogDAOImpl.java index 86fad3bd99..02ccd53cca 100644 --- a/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcCatalogDAOImpl.java +++ b/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcCatalogDAOImpl.java @@ -16,6 +16,7 @@ import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.dao.DataAccessException; import org.springframework.dao.IncorrectResultSizeDataAccessException; import org.springframework.jdbc.core.JdbcTemplate; @@ -61,8 +62,9 @@ public class HpcCatalogDAOImpl implements HpcCatalogDAO { // ---------------------------------------------------------------------// // The Spring JDBC Template instance. - @Autowired - private JdbcTemplate jdbcTemplate = null; + @Autowired + @Qualifier("hpcOracleJdbcTemplate") + private JdbcTemplate jdbcTemplate = null; // Row mappers. private RowMapper catalogMetadataEntryRowMapper = (rs, rowNum) -> { diff --git a/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcDataDownloadDAOImpl.java b/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcDataDownloadDAOImpl.java index 4c443fc784..edba9c727a 100644 --- a/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcDataDownloadDAOImpl.java +++ b/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcDataDownloadDAOImpl.java @@ -35,6 +35,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.dao.DataAccessException; import org.springframework.dao.EmptyResultDataAccessException; import org.springframework.dao.IncorrectResultSizeDataAccessException; @@ -258,8 +259,9 @@ public class HpcDataDownloadDAOImpl implements HpcDataDownloadDAO { // ---------------------------------------------------------------------// // The Spring JDBC Template instance. - @Autowired - private JdbcTemplate jdbcTemplate = null; + @Autowired + @Qualifier("hpcOracleJdbcTemplate") + private JdbcTemplate jdbcTemplate = null; // The logger instance. private final Logger logger = LoggerFactory.getLogger(this.getClass().getName()); diff --git a/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcDataManagementAuditDAOImpl.java b/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcDataManagementAuditDAOImpl.java index e047e15386..b9c4342e93 100644 --- a/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcDataManagementAuditDAOImpl.java +++ b/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcDataManagementAuditDAOImpl.java @@ -15,6 +15,7 @@ import org.json.simple.JSONArray; import org.json.simple.JSONObject; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.dao.DataAccessException; import org.springframework.jdbc.core.JdbcTemplate; @@ -52,8 +53,9 @@ public class HpcDataManagementAuditDAOImpl implements HpcDataManagementAuditDAO // ---------------------------------------------------------------------// // The Spring JDBC Template instance. - @Autowired - private JdbcTemplate jdbcTemplate = null; + @Autowired + @Qualifier("hpcOracleJdbcTemplate") + private JdbcTemplate jdbcTemplate = null; // ---------------------------------------------------------------------// // Constructors diff --git a/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcDataManagementConfigurationDAOImpl.java b/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcDataManagementConfigurationDAOImpl.java index 94a3095ee4..66c67e3b67 100644 --- a/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcDataManagementConfigurationDAOImpl.java +++ b/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcDataManagementConfigurationDAOImpl.java @@ -19,6 +19,7 @@ import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.dao.DataAccessException; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.RowMapper; @@ -57,8 +58,9 @@ public class HpcDataManagementConfigurationDAOImpl implements HpcDataManagementC // ---------------------------------------------------------------------// // The Spring JDBC Template instance. - @Autowired - private JdbcTemplate jdbcTemplate = null; + @Autowired + @Qualifier("hpcOracleJdbcTemplate") + private JdbcTemplate jdbcTemplate = null; // Encryptor. @Autowired @@ -73,6 +75,8 @@ public class HpcDataManagementConfigurationDAOImpl implements HpcDataManagementC dataManagementConfiguration.setS3UploadConfigurationId(rs.getString("S3_UPLOAD_ARCHIVE_CONFIGURATION_ID")); dataManagementConfiguration .setS3DefaultDownloadConfigurationId(rs.getString("S3_DEFAULT_DOWNLOAD_ARCHIVE_CONFIGURATION_ID")); + dataManagementConfiguration + .setS3AutoTieringConfigurationId(rs.getString("S3_AUTO_TIERING_ARCHIVE_CONFIGURATION_ID")); dataManagementConfiguration.setCreateArchiveMetadata(rs.getBoolean("CREATE_ARCHIVE_METADATA")); dataManagementConfiguration.setRegistrationEventWithDownloadRequestURL( rs.getBoolean("REGISTRATION_EVENT_WITH_DOWNLOAD_REQUEST_URL")); @@ -144,6 +148,8 @@ public class HpcDataManagementConfigurationDAOImpl implements HpcDataManagementC s3Configuration.setDataManagementConfigurationId(rs.getString("DATA_MANAGEMENT_CONFIGURATION_ID")); s3Configuration.setExternalStorage(rs.getBoolean("EXTERNAL_STORAGE")); s3Configuration.setPosixPath(rs.getString("POSIX_PATH")); + s3Configuration.setAutoTieringSearchPath(rs.getString("AUTO_TIERING_SEARCH_PATH")); + s3Configuration.setAutoTieringInactivityMonths(rs.getInt("AUTO_TIERING_INACTIVITY_MONTHS")); return s3Configuration; }; diff --git a/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcDataMigrationDAOImpl.java b/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcDataMigrationDAOImpl.java index aacb7458d6..1f9731bbb4 100644 --- a/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcDataMigrationDAOImpl.java +++ b/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcDataMigrationDAOImpl.java @@ -10,22 +10,21 @@ */ package gov.nih.nci.hpc.dao.oracle.impl; -import static gov.nih.nci.hpc.util.HpcUtil.fromPathsString; -import static gov.nih.nci.hpc.util.HpcUtil.toPathsString; - -import java.sql.Types; -import java.util.AbstractMap; -import java.util.Calendar; -import java.util.EnumMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.UUID; - +import gov.nih.nci.hpc.dao.HpcDataMigrationDAO; +import gov.nih.nci.hpc.domain.datamigration.HpcDataMigrationResult; +import gov.nih.nci.hpc.domain.datamigration.HpcDataMigrationStatus; +import gov.nih.nci.hpc.domain.datamigration.HpcDataMigrationType; +import gov.nih.nci.hpc.domain.datatransfer.HpcFileLocation; +import gov.nih.nci.hpc.domain.error.HpcErrorType; +import gov.nih.nci.hpc.domain.model.HpcDataMigrationTask; +import gov.nih.nci.hpc.domain.model.HpcDataMigrationTaskResult; +import gov.nih.nci.hpc.domain.user.HpcIntegratedSystem; +import gov.nih.nci.hpc.exception.HpcException; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.dao.DataAccessException; import org.springframework.dao.IncorrectResultSizeDataAccessException; import org.springframework.jdbc.core.JdbcTemplate; @@ -34,16 +33,11 @@ import org.springframework.jdbc.support.lob.DefaultLobHandler; import org.springframework.jdbc.support.lob.LobHandler; -import gov.nih.nci.hpc.dao.HpcDataMigrationDAO; -import gov.nih.nci.hpc.domain.datamigration.HpcDataMigrationResult; -import gov.nih.nci.hpc.domain.datamigration.HpcDataMigrationStatus; -import gov.nih.nci.hpc.domain.datamigration.HpcDataMigrationType; -import gov.nih.nci.hpc.domain.datatransfer.HpcFileLocation; -import gov.nih.nci.hpc.domain.error.HpcErrorType; -import gov.nih.nci.hpc.domain.model.HpcDataMigrationTask; -import gov.nih.nci.hpc.domain.model.HpcDataMigrationTaskResult; -import gov.nih.nci.hpc.domain.user.HpcIntegratedSystem; -import gov.nih.nci.hpc.exception.HpcException; +import java.sql.Types; +import java.util.*; + +import static gov.nih.nci.hpc.util.HpcUtil.fromPathsString; +import static gov.nih.nci.hpc.util.HpcUtil.toPathsString; /** * HPC Data Migration DAO Implementation. @@ -60,11 +54,12 @@ public class HpcDataMigrationDAOImpl implements HpcDataMigrationDAO { + "when matched then update set PARENT_ID = ?, USER_ID = ?, PATH = ?, CONFIGURATION_ID = ?, FROM_S3_ARCHIVE_CONFIGURATION_ID = ?, " + "TO_S3_ARCHIVE_CONFIGURATION_ID = ?, TYPE = ?, STATUS = ?, CREATED = ?, ALIGN_ARCHIVE_PATH = ?, DATA_SIZE = ?, PERCENT_COMPLETE = ?, SERVER_ID = ?, " + "RETRY_TASK_ID = ?, RETRY_USER_ID = ?, RETRY_FAILED_ITEMS_ONLY = ?, METADATA_FROM_ARCHIVE_FILE_CONTAINER_ID = ?, " - + "METADATA_TO_ARCHIVE_FILE_CONTAINER_ID = ?, METADATA_ARCHIVE_FILE_ID_PATTERN = ? " + + "METADATA_TO_ARCHIVE_FILE_CONTAINER_ID = ?, METADATA_ARCHIVE_FILE_ID_PATTERN = ?, FROM_S3_ARCHIVE_LOCATION_FILE_CONTAINER_ID = ?, FROM_S3_ARCHIVE_LOCATION_FILE_ID = ? " + "when not matched then insert (ID, PARENT_ID, USER_ID, PATH, CONFIGURATION_ID, FROM_S3_ARCHIVE_CONFIGURATION_ID, " + "TO_S3_ARCHIVE_CONFIGURATION_ID, TYPE, STATUS, CREATED, ALIGN_ARCHIVE_PATH, DATA_SIZE, PERCENT_COMPLETE, SERVER_ID, RETRY_TASK_ID, RETRY_USER_ID, " - + "RETRY_FAILED_ITEMS_ONLY, METADATA_FROM_ARCHIVE_FILE_CONTAINER_ID, METADATA_TO_ARCHIVE_FILE_CONTAINER_ID, METADATA_ARCHIVE_FILE_ID_PATTERN) " - + "values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) "; + + "RETRY_FAILED_ITEMS_ONLY, METADATA_FROM_ARCHIVE_FILE_CONTAINER_ID, METADATA_TO_ARCHIVE_FILE_CONTAINER_ID, METADATA_ARCHIVE_FILE_ID_PATTERN, " + + "FROM_S3_ARCHIVE_LOCATION_FILE_CONTAINER_ID, FROM_S3_ARCHIVE_LOCATION_FILE_ID) " + + "values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) "; private static final String UPDATE_DATA_MIGRATION_TASK_CLOBS_SQL = "update HPC_DATA_MIGRATION_TASK set DATA_OBJECT_PATHS = ?, COLLECTION_PATHS = ? where ID = ?"; @@ -90,7 +85,7 @@ public class HpcDataMigrationDAOImpl implements HpcDataMigrationDAO { private static final String GET_DATA_MIGRATION_TASKS_SQL = "select * from HPC_DATA_MIGRATION_TASK where STATUS = ? and TYPE = ? and SERVER_ID = ?"; - private static final String GET_UNASSIGNED_DATA_MIGRATION_TASKS_SQL = "select * from HPC_DATA_MIGRATION_TASK where STATUS = 'RECEIVED' and SERVER_ID is null"; + private static final String GET_UNASSIGNED_DATA_MIGRATION_TASKS_SQL = "select * from HPC_DATA_MIGRATION_TASK where STATUS in ('RECEIVED', 'AUTO_TIERING_RECEIVED') and SERVER_ID is null"; private static final String GET_DATA_OBJECT_MIGRATION_TASKS_SQL = "select * from HPC_DATA_MIGRATION_TASK where PARENT_ID = ?"; @@ -123,8 +118,9 @@ public class HpcDataMigrationDAOImpl implements HpcDataMigrationDAO { // ---------------------------------------------------------------------// // The Spring JDBC Template instance. - @Autowired - private JdbcTemplate jdbcTemplate = null; + @Autowired + @Qualifier("hpcOracleJdbcTemplate") + private JdbcTemplate jdbcTemplate = null; // Lob handler private LobHandler lobHandler = new DefaultLobHandler(); @@ -155,6 +151,14 @@ public class HpcDataMigrationDAOImpl implements HpcDataMigrationDAO { dataMigrationTask.setMetadataToArchiveFileContainerId(rs.getString("METADATA_TO_ARCHIVE_FILE_CONTAINER_ID")); dataMigrationTask.setMetadataArchiveFileIdPattern(rs.getString("METADATA_ARCHIVE_FILE_ID_PATTERN")); + HpcFileLocation fromS3ArchiveLocation = new HpcFileLocation(); + fromS3ArchiveLocation.setFileContainerId(rs.getString("FROM_S3_ARCHIVE_LOCATION_FILE_CONTAINER_ID")); + fromS3ArchiveLocation.setFileId(rs.getString("FROM_S3_ARCHIVE_LOCATION_FILE_ID")); + if(!StringUtils.isEmpty(fromS3ArchiveLocation.getFileContainerId()) && + !StringUtils.isEmpty(fromS3ArchiveLocation.getFileId())) { + dataMigrationTask.setFromS3ArchiveLocation(fromS3ArchiveLocation); + } + if (rs.getObject("RETRY_FAILED_ITEMS_ONLY") != null) { dataMigrationTask.setRetryFailedItemsOnly(rs.getBoolean("RETRY_FAILED_ITEMS_ONLY")); } @@ -256,6 +260,12 @@ public void upsertDataMigrationTask(HpcDataMigrationTask dataMigrationTask) thro ? toPathsString(dataMigrationTask.getCollectionPaths()) : null; + String fromS3ArchiveLocationFileContainerId = null, fromS3ArchiveLocationFileId = null; + if(dataMigrationTask.getFromS3ArchiveLocation() != null) { + fromS3ArchiveLocationFileContainerId = dataMigrationTask.getFromS3ArchiveLocation().getFileContainerId(); + fromS3ArchiveLocationFileId = dataMigrationTask.getFromS3ArchiveLocation().getFileId(); + } + jdbcTemplate.update(UPSERT_DATA_MIGRATION_TASK_SQL, dataMigrationTask.getId(), dataMigrationTask.getParentId(), dataMigrationTask.getUserId(), dataMigrationTask.getPath(), dataMigrationTask.getConfigurationId(), dataMigrationTask.getFromS3ArchiveConfigurationId(), @@ -267,7 +277,9 @@ public void upsertDataMigrationTask(HpcDataMigrationTask dataMigrationTask) thro dataMigrationTask.getRetryFailedItemsOnly(), dataMigrationTask.getMetadataFromArchiveFileContainerId(), dataMigrationTask.getMetadataToArchiveFileContainerId(), - dataMigrationTask.getMetadataArchiveFileIdPattern(), dataMigrationTask.getId(), + dataMigrationTask.getMetadataArchiveFileIdPattern(), + fromS3ArchiveLocationFileContainerId, fromS3ArchiveLocationFileId, + dataMigrationTask.getId(), dataMigrationTask.getParentId(), dataMigrationTask.getUserId(), dataMigrationTask.getPath(), dataMigrationTask.getConfigurationId(), dataMigrationTask.getFromS3ArchiveConfigurationId(), dataMigrationTask.getToS3ArchiveConfigurationId(), dataMigrationTask.getType().value(), @@ -278,7 +290,8 @@ public void upsertDataMigrationTask(HpcDataMigrationTask dataMigrationTask) thro dataMigrationTask.getRetryFailedItemsOnly(), dataMigrationTask.getMetadataFromArchiveFileContainerId(), dataMigrationTask.getMetadataToArchiveFileContainerId(), - dataMigrationTask.getMetadataArchiveFileIdPattern()); + dataMigrationTask.getMetadataArchiveFileIdPattern(), + fromS3ArchiveLocationFileContainerId, fromS3ArchiveLocationFileId); jdbcTemplate.update(UPDATE_DATA_MIGRATION_TASK_CLOBS_SQL, new Object[] { new SqlLobValue(dataObjectPaths, lobHandler), @@ -440,7 +453,8 @@ public void upsertDataMigrationTaskResult(HpcDataMigrationTask dataMigrationTask dataMigrationTask.getSize(), dataMigrationTask.getRetryTaskId(), dataMigrationTask.getRetryUserId(), dataMigrationTask.getMetadataFromArchiveFileContainerId(), dataMigrationTask.getMetadataToArchiveFileContainerId(), - dataMigrationTask.getMetadataArchiveFileIdPattern(), dataMigrationTask.getId(), + dataMigrationTask.getMetadataArchiveFileIdPattern(), + dataMigrationTask.getId(), dataMigrationTask.getParentId(), dataMigrationTask.getUserId(), dataMigrationTask.getPath(), dataMigrationTask.getConfigurationId(), dataMigrationTask.getFromS3ArchiveConfigurationId(), dataMigrationTask.getToS3ArchiveConfigurationId(), dataMigrationTask.getType().value(), @@ -547,6 +561,5 @@ public int cleanupDataMigrationTasks() throws HpcException { throw new HpcException("Failed to cleanup a data migration tasks: " + e.getMessage(), HpcErrorType.DATABASE_ERROR, HpcIntegratedSystem.ORACLE, e); } - } } diff --git a/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcDataRegistrationDAOImpl.java b/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcDataRegistrationDAOImpl.java index aafb1da0b9..d87d061ecd 100644 --- a/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcDataRegistrationDAOImpl.java +++ b/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcDataRegistrationDAOImpl.java @@ -25,6 +25,7 @@ import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.dao.DataAccessException; import org.springframework.dao.IncorrectResultSizeDataAccessException; import org.springframework.jdbc.core.JdbcTemplate; @@ -131,8 +132,9 @@ public class HpcDataRegistrationDAOImpl implements HpcDataRegistrationDAO { HpcEncryptor encryptor = null; // The Spring JDBC Template instance. - @Autowired - private JdbcTemplate jdbcTemplate = null; + @Autowired + @Qualifier("hpcOracleJdbcTemplate") + private JdbcTemplate jdbcTemplate = null; // Lob handler private LobHandler lobHandler = new DefaultLobHandler(); diff --git a/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcDataTieringDAOImpl.java b/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcDataTieringDAOImpl.java index 35220b41e5..285cf36362 100644 --- a/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcDataTieringDAOImpl.java +++ b/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcDataTieringDAOImpl.java @@ -13,6 +13,7 @@ import java.util.Calendar; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.dao.DataAccessException; import org.springframework.jdbc.core.JdbcTemplate; @@ -46,8 +47,9 @@ public class HpcDataTieringDAOImpl implements HpcDataTieringDAO { // ---------------------------------------------------------------------// // The Spring JDBC Template instance. - @Autowired - private JdbcTemplate jdbcTemplate = null; + @Autowired + @Qualifier("hpcOracleJdbcTemplate") + private JdbcTemplate jdbcTemplate = null; // ---------------------------------------------------------------------// // Constructors diff --git a/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcEventDAOImpl.java b/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcEventDAOImpl.java index 88c5c18776..39ff355729 100644 --- a/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcEventDAOImpl.java +++ b/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcEventDAOImpl.java @@ -20,6 +20,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.dao.DataAccessException; import org.springframework.dao.IncorrectResultSizeDataAccessException; import org.springframework.jdbc.core.JdbcTemplate; @@ -64,8 +65,9 @@ public class HpcEventDAOImpl implements HpcEventDAO { // ---------------------------------------------------------------------// // The Spring JDBC Template instance. - @Autowired - private JdbcTemplate jdbcTemplate = null; + @Autowired + @Qualifier("hpcOracleJdbcTemplate") + private JdbcTemplate jdbcTemplate = null; // Encryptor. @Autowired diff --git a/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcGlobusTransferTaskDAOImpl.java b/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcGlobusTransferTaskDAOImpl.java index 51c019ff0a..5939736a9d 100644 --- a/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcGlobusTransferTaskDAOImpl.java +++ b/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcGlobusTransferTaskDAOImpl.java @@ -13,6 +13,7 @@ import java.util.List; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.dao.DataAccessException; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.RowMapper; @@ -50,8 +51,9 @@ public class HpcGlobusTransferTaskDAOImpl implements HpcGlobusTransferTaskDAO { // ---------------------------------------------------------------------// // The Spring JDBC Template instance. - @Autowired - private JdbcTemplate jdbcTemplate = null; + @Autowired + @Qualifier("hpcOracleJdbcTemplate") + private JdbcTemplate jdbcTemplate = null; // Row mapper. private RowMapper globusAccountsRowMapper = (rs, rowNum) -> { diff --git a/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcGroupDAOImpl.java b/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcGroupDAOImpl.java index bfe8e38d12..f3746808ef 100644 --- a/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcGroupDAOImpl.java +++ b/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcGroupDAOImpl.java @@ -20,6 +20,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.dao.DataAccessException; import org.springframework.dao.IncorrectResultSizeDataAccessException; import org.springframework.jdbc.core.JdbcTemplate; @@ -77,8 +78,9 @@ public class HpcGroupDAOImpl implements HpcGroupDAO { // ---------------------------------------------------------------------// // The Spring JDBC Template instance. - @Autowired - private JdbcTemplate jdbcTemplate = null; + @Autowired + @Qualifier("hpcOracleJdbcTemplate") + private JdbcTemplate jdbcTemplate = null; // Row mapper. private SingleColumnRowMapper rowMapper = new SingleColumnRowMapper<>(); diff --git a/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcInvestigatorDAOImpl.java b/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcInvestigatorDAOImpl.java index 4cf57b0bd8..983469042c 100644 --- a/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcInvestigatorDAOImpl.java +++ b/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcInvestigatorDAOImpl.java @@ -13,6 +13,7 @@ import java.util.List; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.dao.DataAccessException; import org.springframework.dao.IncorrectResultSizeDataAccessException; import org.springframework.jdbc.core.JdbcTemplate; @@ -48,8 +49,9 @@ public class HpcInvestigatorDAOImpl implements HpcInvestigatorDAO { // ---------------------------------------------------------------------// // The Spring JDBC Template instance. - @Autowired - private JdbcTemplate jdbcTemplate = null; + @Autowired + @Qualifier("hpcOracleJdbcTemplate") + private JdbcTemplate jdbcTemplate = null; // Row mapper. private SingleColumnRowMapper rowMapper = new SingleColumnRowMapper<>(); diff --git a/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcMetadataDAOImpl.java b/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcMetadataDAOImpl.java index 08592090cf..02e03e11b9 100644 --- a/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcMetadataDAOImpl.java +++ b/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcMetadataDAOImpl.java @@ -21,6 +21,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.dao.DataAccessException; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.RowMapper; @@ -293,8 +294,9 @@ public class HpcMetadataDAOImpl implements HpcMetadataDAO { // ---------------------------------------------------------------------// // The Spring JDBC Template instance. - @Autowired - private JdbcTemplate jdbcTemplate = null; + @Autowired + @Qualifier("hpcOracleJdbcTemplate") + private JdbcTemplate jdbcTemplate = null; private int fetchSize = 1000; diff --git a/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcNotificationDAOImpl.java b/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcNotificationDAOImpl.java index f1a991b562..437c6d73f9 100644 --- a/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcNotificationDAOImpl.java +++ b/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcNotificationDAOImpl.java @@ -19,6 +19,7 @@ import java.util.Map; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.dao.DataAccessException; import org.springframework.dao.IncorrectResultSizeDataAccessException; import org.springframework.jdbc.core.JdbcTemplate; @@ -93,8 +94,9 @@ public class HpcNotificationDAOImpl implements HpcNotificationDAO { // ---------------------------------------------------------------------// // The Spring JDBC Template instance. - @Autowired - private JdbcTemplate jdbcTemplate = null; + @Autowired + @Qualifier("hpcOracleJdbcTemplate") + private JdbcTemplate jdbcTemplate = null; // Row mappers. private RowMapper notificationSubscriptionRowMapper = (rs, rowNum) -> { diff --git a/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcQueryConfigDAOImpl.java b/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcQueryConfigDAOImpl.java index 38eb9e6a73..03dbbc6615 100644 --- a/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcQueryConfigDAOImpl.java +++ b/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcQueryConfigDAOImpl.java @@ -12,6 +12,7 @@ import java.util.List; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.dao.DataAccessException; import org.springframework.dao.IncorrectResultSizeDataAccessException; import org.springframework.jdbc.core.JdbcTemplate; @@ -47,6 +48,7 @@ public class HpcQueryConfigDAOImpl implements HpcQueryConfigDAO { // The Spring JDBC Template instance. @Autowired + @Qualifier("hpcOracleJdbcTemplate") private JdbcTemplate jdbcTemplate = null; // Encryptor. diff --git a/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcReportsDAOImpl.java b/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcReportsDAOImpl.java index 195c3d334a..9c32a042df 100644 --- a/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcReportsDAOImpl.java +++ b/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcReportsDAOImpl.java @@ -20,6 +20,7 @@ import java.util.Map; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.dao.DataAccessException; import org.springframework.dao.EmptyResultDataAccessException; import org.springframework.jdbc.core.JdbcTemplate; @@ -412,8 +413,9 @@ public class HpcReportsDAOImpl implements HpcReportsDAO { // ---------------------------------------------------------------------// // The Spring JDBC Template instance. - @Autowired - private JdbcTemplate jdbcTemplate = null; + @Autowired + @Qualifier("hpcOracleJdbcTemplate") + private JdbcTemplate jdbcTemplate = null; private String iRodsBasePath = ""; diff --git a/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcReviewDAOImpl.java b/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcReviewDAOImpl.java index b221a43845..899e1460a5 100644 --- a/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcReviewDAOImpl.java +++ b/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcReviewDAOImpl.java @@ -16,6 +16,7 @@ import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.dao.DataAccessException; import org.springframework.dao.IncorrectResultSizeDataAccessException; import org.springframework.jdbc.core.JdbcTemplate; @@ -75,8 +76,9 @@ public class HpcReviewDAOImpl implements HpcReviewDAO { // ---------------------------------------------------------------------// // The Spring JDBC Template instance. - @Autowired - private JdbcTemplate jdbcTemplate = null; + @Autowired + @Qualifier("hpcOracleJdbcTemplate") + private JdbcTemplate jdbcTemplate = null; // Row mappers. private SingleColumnRowMapper curatorRowMapper = new SingleColumnRowMapper<>(); diff --git a/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcSystemAccountDAOImpl.java b/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcSystemAccountDAOImpl.java index 665d9b4bfd..1b9cd1afdc 100644 --- a/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcSystemAccountDAOImpl.java +++ b/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcSystemAccountDAOImpl.java @@ -11,6 +11,7 @@ import java.util.List; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.dao.DataAccessException; import org.springframework.dao.IncorrectResultSizeDataAccessException; import org.springframework.jdbc.core.JdbcTemplate; @@ -48,8 +49,9 @@ public class HpcSystemAccountDAOImpl implements HpcSystemAccountDAO { // ---------------------------------------------------------------------// // The Spring JDBC Template instance. - @Autowired - private JdbcTemplate jdbcTemplate = null; + @Autowired + @Qualifier("hpcOracleJdbcTemplate") + private JdbcTemplate jdbcTemplate = null; // Encryptor. @Autowired diff --git a/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcUserBookmarkDAOImpl.java b/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcUserBookmarkDAOImpl.java index bb8b90020f..8798ee13e3 100644 --- a/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcUserBookmarkDAOImpl.java +++ b/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcUserBookmarkDAOImpl.java @@ -16,6 +16,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.dao.DataAccessException; import org.springframework.dao.IncorrectResultSizeDataAccessException; import org.springframework.jdbc.core.JdbcTemplate; @@ -59,8 +60,9 @@ public class HpcUserBookmarkDAOImpl implements HpcUserBookmarkDAO { // ---------------------------------------------------------------------// // The Spring JDBC Template instance. - @Autowired - private JdbcTemplate jdbcTemplate = null; + @Autowired + @Qualifier("hpcOracleJdbcTemplate") + private JdbcTemplate jdbcTemplate = null; // Row mappers. private RowMapper userBookmarkRowMapper = (rs, rowNum) -> { diff --git a/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcUserDAOImpl.java b/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcUserDAOImpl.java index 1251154853..4e3ea919e5 100644 --- a/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcUserDAOImpl.java +++ b/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcUserDAOImpl.java @@ -17,6 +17,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.dao.DataAccessException; import org.springframework.dao.EmptyResultDataAccessException; import org.springframework.dao.IncorrectResultSizeDataAccessException; @@ -86,6 +87,7 @@ public class HpcUserDAOImpl implements HpcUserDAO { // The Spring JDBC Template instance. @Autowired + @Qualifier("hpcOracleJdbcTemplate") private JdbcTemplate jdbcTemplate = null; // Encryptor. diff --git a/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcUserNamedQueryDAOImpl.java b/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcUserNamedQueryDAOImpl.java index 8ee3217171..6a8853d560 100644 --- a/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcUserNamedQueryDAOImpl.java +++ b/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/oracle/impl/HpcUserNamedQueryDAOImpl.java @@ -22,6 +22,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.dao.DataAccessException; import org.springframework.dao.IncorrectResultSizeDataAccessException; import org.springframework.jdbc.core.JdbcTemplate; @@ -74,6 +75,7 @@ public class HpcUserNamedQueryDAOImpl implements HpcUserNamedQueryDAO { // The Spring JDBC Template instance. @Autowired + @Qualifier("hpcOracleJdbcTemplate") private JdbcTemplate jdbcTemplate = null; // Encryptor. diff --git a/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/trino/impl/HpcExternalArchiveDAOImpl.java b/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/trino/impl/HpcExternalArchiveDAOImpl.java new file mode 100644 index 0000000000..632028c644 --- /dev/null +++ b/src/hpc-server/hpc-dao-impl/src/main/java/gov/nih/nci/hpc/dao/trino/impl/HpcExternalArchiveDAOImpl.java @@ -0,0 +1,110 @@ +/** + * HpcExternalArchiveDAOImpl.java + * + *

+ * Copyright SVG, Inc. Copyright Leidos Biomedical Research, Inc + * + *

+ * Distributed under the OSI-approved BSD 3-Clause License. See + * http://ncip.github.com/HPC/LICENSE.txt for details. + */ +package gov.nih.nci.hpc.dao.trino.impl; + +import gov.nih.nci.hpc.dao.HpcExternalArchiveDAO; +import gov.nih.nci.hpc.domain.error.HpcErrorType; +import gov.nih.nci.hpc.domain.user.HpcIntegratedSystem; +import gov.nih.nci.hpc.exception.HpcException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.dao.DataAccessException; +import org.springframework.jdbc.core.JdbcTemplate; + +import java.util.List; + +/** + * HPC External Archive DAO Implementation. + * + *

This implementation queries external archives (VAST managed archives mounted via NFS on the + * DME server) to identify files that have not been accessed within a specified time period, for + * auto-tiering migration to S3 Glacier Deep Archive. + * + * @author Eran Rosenberg + */ +public class HpcExternalArchiveDAOImpl implements HpcExternalArchiveDAO { + // ---------------------------------------------------------------------// + // Constants + // ---------------------------------------------------------------------// + + // SQL Queries. + private static final String GET_FILES_NOT_ACCESSED_SQL = + "select replace(search_path, ?, '') || name as path " + + "from \"vast-big-catalog-bucket|vast_big_catalog_schema\".\"vast_big_catalog_table\" " + + "where element_type = 'FILE' " + + "and atime < current_timestamp - INTERVAL '{months}' MONTH " + + "and search_path like ? " + + "and name NOT LIKE '%/'"; + + // ---------------------------------------------------------------------// + // Instance members + // ---------------------------------------------------------------------// + + // The Spring JDBC Template instance. + @Autowired + @Qualifier("hpcTrinoJdbcTemplate") + private JdbcTemplate jdbcTemplate = null; + + // The logger instance. + private static final Logger logger = LoggerFactory.getLogger(HpcExternalArchiveDAOImpl.class.getName()); + + // ---------------------------------------------------------------------// + // Constructors + // ---------------------------------------------------------------------// + + /** Constructor for Spring Dependency Injection. */ + private HpcExternalArchiveDAOImpl() {} + + // ---------------------------------------------------------------------// + // Methods + // ---------------------------------------------------------------------// + + // ---------------------------------------------------------------------// + // HpcExternalArchiveDAO Interface Implementation + // ---------------------------------------------------------------------// + + @Override + public List getFilesNotAccessed(String searchPath, Integer monthsNotAccessed) throws HpcException { + try { + return jdbcTemplate.queryForList( + GET_FILES_NOT_ACCESSED_SQL.replace("{months}", monthsNotAccessed.toString()), + String.class, searchPath, searchPath); + + } catch (DataAccessException e) { + throw new HpcException( + "Failed to query files not accessed in external archive [searchPath=" + searchPath + + ", monthsNotAccessed=" + monthsNotAccessed + "]: " + e.getMessage(), + HpcErrorType.DATABASE_ERROR, HpcIntegratedSystem.VAST, e); + } + } + + // ---------------------------------------------------------------------// + // Helper Methods + // ---------------------------------------------------------------------// + + /** + * Verify connection to Trino DB. Called by Spring as init-method. + * + * @throws HpcException If it failed to connect to the database. + */ + @SuppressWarnings("unused") + private void dbConnect() throws HpcException { + try { + jdbcTemplate.getDataSource().getConnection(); + + } catch (Exception e) { + throw new HpcException("Failed to connect to Trino DB. Check connection & credentials config", + HpcErrorType.DATABASE_ERROR, HpcIntegratedSystem.VAST, e); + } + } +} diff --git a/src/hpc-server/hpc-dao-impl/src/main/resources/META-INF/spring/hpc-dao-impl-beans-configuration.xml b/src/hpc-server/hpc-dao-impl/src/main/resources/META-INF/spring/hpc-dao-impl-beans-configuration.xml index 4310f996e5..18777cc78b 100644 --- a/src/hpc-server/hpc-dao-impl/src/main/resources/META-INF/spring/hpc-dao-impl-beans-configuration.xml +++ b/src/hpc-server/hpc-dao-impl/src/main/resources/META-INF/spring/hpc-dao-impl-beans-configuration.xml @@ -35,11 +35,25 @@ - + - + + + + + + + + + + + + + + + @@ -70,5 +84,6 @@ - + \ No newline at end of file diff --git a/src/hpc-server/hpc-dao-impl/src/main/scripts/migration/release-3.23.0/hpc_auto_tiering.sql b/src/hpc-server/hpc-dao-impl/src/main/scripts/migration/release-3.23.0/hpc_auto_tiering.sql new file mode 100644 index 0000000000..a835ddb048 --- /dev/null +++ b/src/hpc-server/hpc-dao-impl/src/main/scripts/migration/release-3.23.0/hpc_auto_tiering.sql @@ -0,0 +1,42 @@ +-- +-- hpc_auto_tiering.sql +-- +-- Copyright SVG, Inc. +-- Copyright Leidos Biomedical Research, Inc +-- +-- Distributed under the OSI-approved BSD 3-Clause License. +-- See http://ncip.github.com/HPC/LICENSE.txt for details. +-- +-- +-- @author Eran Rosenberg +-- + +-- HPC_S3_ARCHIVE_CONFIGURATION +ALTER TABLE HPC_S3_ARCHIVE_CONFIGURATION add ( + AUTO_TIERING_SEARCH_PATH VARCHAR2(2700) + ); +COMMENT ON COLUMN HPC_S3_ARCHIVE_CONFIGURATION.AUTO_TIERING_SEARCH_PATH IS 'The search path for auto-tiering external archive files'; + +ALTER TABLE HPC_S3_ARCHIVE_CONFIGURATION add ( + AUTO_TIERING_INACTIVITY_MONTHS NUMBER(10) + ); +COMMENT ON COLUMN HPC_S3_ARCHIVE_CONFIGURATION.AUTO_TIERING_INACTIVITY_MONTHS IS 'The inactivity period in months before files are auto-tiered to Glacier Deep Archive'; + +-- HPC_DATA_MANAGEMENT_CONFIGURATION +ALTER TABLE HPC_DATA_MANAGEMENT_CONFIGURATION add ( + S3_AUTO_TIERING_ARCHIVE_CONFIGURATION_ID VARCHAR2(50) + ); +COMMENT ON COLUMN HPC_DATA_MANAGEMENT_CONFIGURATION.S3_AUTO_TIERING_ARCHIVE_CONFIGURATION_ID IS 'The S3 archive configuration ID for auto-tiering external archive into'; + +-- HPC_DATA_MIGRATION_TASK +ALTER TABLE HPC_DATA_MIGRATION_TASK add ( + FROM_S3_ARCHIVE_LOCATION_FILE_CONTAINER_ID VARCHAR2(50) + ); +COMMENT ON COLUMN HPC_DATA_MIGRATION_TASK.FROM_S3_ARCHIVE_LOCATION_FILE_CONTAINER_ID IS 'The archive file container ID (bucket) migrating from'; + +ALTER TABLE HPC_DATA_MIGRATION_TASK add ( + FROM_S3_ARCHIVE_LOCATION_FILE_ID VARCHAR2(2700) + ); +COMMENT ON COLUMN HPC_DATA_MIGRATION_TASK.FROM_S3_ARCHIVE_LOCATION_FILE_ID IS 'The archive file ID (object ID) migrating from'; + + diff --git a/src/hpc-server/hpc-dao-impl/src/main/scripts/migration/release-3.24.0/hpc_auto_tiering.sql b/src/hpc-server/hpc-dao-impl/src/main/scripts/migration/release-3.24.0/hpc_auto_tiering.sql new file mode 100644 index 0000000000..a835ddb048 --- /dev/null +++ b/src/hpc-server/hpc-dao-impl/src/main/scripts/migration/release-3.24.0/hpc_auto_tiering.sql @@ -0,0 +1,42 @@ +-- +-- hpc_auto_tiering.sql +-- +-- Copyright SVG, Inc. +-- Copyright Leidos Biomedical Research, Inc +-- +-- Distributed under the OSI-approved BSD 3-Clause License. +-- See http://ncip.github.com/HPC/LICENSE.txt for details. +-- +-- +-- @author Eran Rosenberg +-- + +-- HPC_S3_ARCHIVE_CONFIGURATION +ALTER TABLE HPC_S3_ARCHIVE_CONFIGURATION add ( + AUTO_TIERING_SEARCH_PATH VARCHAR2(2700) + ); +COMMENT ON COLUMN HPC_S3_ARCHIVE_CONFIGURATION.AUTO_TIERING_SEARCH_PATH IS 'The search path for auto-tiering external archive files'; + +ALTER TABLE HPC_S3_ARCHIVE_CONFIGURATION add ( + AUTO_TIERING_INACTIVITY_MONTHS NUMBER(10) + ); +COMMENT ON COLUMN HPC_S3_ARCHIVE_CONFIGURATION.AUTO_TIERING_INACTIVITY_MONTHS IS 'The inactivity period in months before files are auto-tiered to Glacier Deep Archive'; + +-- HPC_DATA_MANAGEMENT_CONFIGURATION +ALTER TABLE HPC_DATA_MANAGEMENT_CONFIGURATION add ( + S3_AUTO_TIERING_ARCHIVE_CONFIGURATION_ID VARCHAR2(50) + ); +COMMENT ON COLUMN HPC_DATA_MANAGEMENT_CONFIGURATION.S3_AUTO_TIERING_ARCHIVE_CONFIGURATION_ID IS 'The S3 archive configuration ID for auto-tiering external archive into'; + +-- HPC_DATA_MIGRATION_TASK +ALTER TABLE HPC_DATA_MIGRATION_TASK add ( + FROM_S3_ARCHIVE_LOCATION_FILE_CONTAINER_ID VARCHAR2(50) + ); +COMMENT ON COLUMN HPC_DATA_MIGRATION_TASK.FROM_S3_ARCHIVE_LOCATION_FILE_CONTAINER_ID IS 'The archive file container ID (bucket) migrating from'; + +ALTER TABLE HPC_DATA_MIGRATION_TASK add ( + FROM_S3_ARCHIVE_LOCATION_FILE_ID VARCHAR2(2700) + ); +COMMENT ON COLUMN HPC_DATA_MIGRATION_TASK.FROM_S3_ARCHIVE_LOCATION_FILE_ID IS 'The archive file ID (object ID) migrating from'; + + diff --git a/src/hpc-server/hpc-domain-model/pom.xml b/src/hpc-server/hpc-domain-model/pom.xml index 3f14a0e9b6..b35d8a3772 100644 --- a/src/hpc-server/hpc-domain-model/pom.xml +++ b/src/hpc-server/hpc-domain-model/pom.xml @@ -18,7 +18,7 @@ gov.nih.nci.hpc hpc-server - 3.23.0 + 3.24.0 hpc-domain-model jar diff --git a/src/hpc-server/hpc-domain-model/src/main/resources/schema/HpcDataManagement.xsd b/src/hpc-server/hpc-domain-model/src/main/resources/schema/HpcDataManagement.xsd index 6d8124cb98..ca35f5d94d 100644 --- a/src/hpc-server/hpc-domain-model/src/main/resources/schema/HpcDataManagement.xsd +++ b/src/hpc-server/hpc-domain-model/src/main/resources/schema/HpcDataManagement.xsd @@ -54,7 +54,9 @@ - + + + @@ -68,6 +70,7 @@ type="hpc-domain-datatransfer:HpcDataTransferType" /> + - diff --git a/src/hpc-server/hpc-domain-types/pom.xml b/src/hpc-server/hpc-domain-types/pom.xml index 647270c654..82b563923f 100644 --- a/src/hpc-server/hpc-domain-types/pom.xml +++ b/src/hpc-server/hpc-domain-types/pom.xml @@ -18,7 +18,7 @@ gov.nih.nci.hpc hpc-server - 3.23.0 + 3.24.0 hpc-domain-types jar diff --git a/src/hpc-server/hpc-domain-types/src/main/resources/schema/HpcDataMigrationTypes.xsd b/src/hpc-server/hpc-domain-types/src/main/resources/schema/HpcDataMigrationTypes.xsd index e7e95d9231..6b1d93e1b9 100644 --- a/src/hpc-server/hpc-domain-types/src/main/resources/schema/HpcDataMigrationTypes.xsd +++ b/src/hpc-server/hpc-domain-types/src/main/resources/schema/HpcDataMigrationTypes.xsd @@ -20,6 +20,7 @@ + @@ -31,6 +32,7 @@ + diff --git a/src/hpc-server/hpc-dto/pom.xml b/src/hpc-server/hpc-dto/pom.xml index eb7032e951..1a37f9075b 100644 --- a/src/hpc-server/hpc-dto/pom.xml +++ b/src/hpc-server/hpc-dto/pom.xml @@ -18,7 +18,7 @@ gov.nih.nci.hpc hpc-server - 3.23.0 + 3.24.0 hpc-dto jar diff --git a/src/hpc-server/hpc-integration-api/pom.xml b/src/hpc-server/hpc-integration-api/pom.xml index dc80ca67df..b19386af0e 100644 --- a/src/hpc-server/hpc-integration-api/pom.xml +++ b/src/hpc-server/hpc-integration-api/pom.xml @@ -19,7 +19,7 @@ gov.nih.nci.hpc hpc-server - 3.23.0 + 3.24.0 hpc-integration-api jar diff --git a/src/hpc-server/hpc-integration-impl/pom.xml b/src/hpc-server/hpc-integration-impl/pom.xml index 818778c737..7e467535bf 100644 --- a/src/hpc-server/hpc-integration-impl/pom.xml +++ b/src/hpc-server/hpc-integration-impl/pom.xml @@ -18,7 +18,7 @@ gov.nih.nci.hpc hpc-server - 3.23.0 + 3.24.0 hpc-integration-impl jar diff --git a/src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/irods/impl/HpcDataManagementProxyImpl.java b/src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/irods/impl/HpcDataManagementProxyImpl.java index 7f29b20cb1..c7852ca536 100644 --- a/src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/irods/impl/HpcDataManagementProxyImpl.java +++ b/src/hpc-server/hpc-integration-impl/src/main/java/gov/nih/nci/hpc/integration/irods/impl/HpcDataManagementProxyImpl.java @@ -227,8 +227,16 @@ public void addMetadataToCollection(Object authenticatedToken, String path, List } if (!avuDatas.isEmpty()) { - irodsConnection.getCollectionAO(authenticatedToken) - .addBulkAVUMetadataToCollection(getAbsolutePath(path), avuDatas); + // Add bulk metadata to iRODS, and validate the result. + for (BulkAVUOperationResponse addAvuResponse : irodsConnection.getCollectionAO(authenticatedToken) + .addBulkAVUMetadataToCollection(getAbsolutePath(path), avuDatas)) { + if (addAvuResponse.getResultStatus() != ResultStatus.OK) { + // Add metadata failed. + String message = "Failed to add metadata to a collection [" + path + "]: " + + addAvuResponse.getResultStatus() + " - " + addAvuResponse.getMessage(); + throw new HpcException(message, HpcErrorType.DATA_MANAGEMENT_ERROR, HpcIntegratedSystem.IRODS); + } + } } } catch (JargonException e) { diff --git a/src/hpc-server/hpc-scheduler-migration/pom.xml b/src/hpc-server/hpc-scheduler-migration/pom.xml index 3ce7fae7b9..8a26f450a3 100644 --- a/src/hpc-server/hpc-scheduler-migration/pom.xml +++ b/src/hpc-server/hpc-scheduler-migration/pom.xml @@ -18,7 +18,7 @@ gov.nih.nci.hpc hpc-server - 3.23.0 + 3.24.0 hpc-scheduler-migration war diff --git a/src/hpc-server/hpc-scheduler-migration/src/main/java/gov/nih/nci/hpc/scheduler/migration/impl/HpcScheduledTasksImpl.java b/src/hpc-server/hpc-scheduler-migration/src/main/java/gov/nih/nci/hpc/scheduler/migration/impl/HpcScheduledTasksImpl.java index 5ebe7aa5ef..e21ecb0377 100644 --- a/src/hpc-server/hpc-scheduler-migration/src/main/java/gov/nih/nci/hpc/scheduler/migration/impl/HpcScheduledTasksImpl.java +++ b/src/hpc-server/hpc-scheduler-migration/src/main/java/gov/nih/nci/hpc/scheduler/migration/impl/HpcScheduledTasksImpl.java @@ -102,6 +102,22 @@ private void processDataObjectMetadataUpdatetMigrationReceivedTask() { dataMigrationBusService::processDataObjectMetadataUpdateMigrationReceived, logger); } + /** + * Process auto-tiering task. Retrieves all data management configurations with auto-tiering enabled + * and creates a bulk auto-tiering migration task for each configuration. + */ + @Scheduled(cron = "${hpc.scheduler.migration.cron.processAutoTiering.delay}") + private void processAutoTieringTask() { + execute("processAutoTiering()", dataMigrationBusService::processAutoTiering, logger); + } + + /** process bulk auto-tiering migration task **/ + @Scheduled(cron = "${hpc.scheduler.migration.cron.processBulkAutoTieringMigrationReceived.delay}") + private void processBulkAutoTieringMigrationReceivedTask() { + execute("processBulkAutoTieringMigrationReceived()", + dataMigrationBusService::processBulkAutoTieringMigrationReceived, logger); + } + /** * Called by Spring dependency injection. Reset all active S3 upload/download in * progress tasks, so they are restarted following a server restart. diff --git a/src/hpc-server/hpc-scheduler-migration/src/main/resources/WEB-INF/spring/hpc-scheduler-migration.properties b/src/hpc-server/hpc-scheduler-migration/src/main/resources/WEB-INF/spring/hpc-scheduler-migration.properties index 2edc3db681..68a86c0c5e 100644 --- a/src/hpc-server/hpc-scheduler-migration/src/main/resources/WEB-INF/spring/hpc-scheduler-migration.properties +++ b/src/hpc-server/hpc-scheduler-migration/src/main/resources/WEB-INF/spring/hpc-scheduler-migration.properties @@ -186,7 +186,7 @@ hpc.service.dataManagement.registrationResultsPageSize=100 hpc.service.dataManagement.deletedBasePath=/DME_Deleted_Archive hpc.service.dataManagement.deletedDataObjectRetentionDays=730 hpc.service.event.invokerCollectionUpdateNotification=false -hpc.service.dataMigration.serverIds= +hpc.service.dataMigration.serverIds=server-id hpc.service.serverId=server-id ############################################################################# @@ -231,7 +231,7 @@ hpc.integration.ldap.userIdDomainName=nih.gov # S3 hpc.integration.s3.executorThreadPoolSize=8 hpc.integration.s3.awsTransferManagerThreadPoolSize=8 -hpc.integration.s3.pathStyleAccessEnabledProviders=CLOUDIAN,VAST +hpc.integration.s3.pathStyleAccessEnabledProviders=CLOUDIAN,VAST,VAST_NCIVS hpc.integration.s3.minimumUploadPartSize=52428800 hpc.integration.s3.multipartUploadThreshold=52428800 hpc.integration.s3.restoreNumDays=2 @@ -260,12 +260,13 @@ hpc.integration.sps.URL=https://authproxyint.nih.gov/authazws/AuthRestService/au hpc.integration.sps.resource=authproxyint.nih.gov/nihuser hpc.integration.sps.domain=NIH - +hpc.integration.irods.basePath=/tempZone/home ############################################################################# # DAO Properties. ############################################################################# +# Oracle hpc.dao.oracle.dbName=icatdev hpc.dao.oracle.host=fsitgl-oradb02t.ncifcrf.gov hpc.dao.oracle.port=1521 @@ -275,8 +276,15 @@ hpc.dao.oracle.url=jdbc:oracle:thin:@${hpc.dao.oracle.host}:${hpc.dao.oracle.por hpc.dao.oracle.maxActive=50 hpc.dao.oracle.encryptor.key=***128-bit-key** hpc.dao.jdbc.template.fetchSize=1000 -hpc.integration.irods.basePath=/tempZone/home - + +# Trino +hpc.dao.trino.host=trino.fairway.nci.nih.gov +hpc.dao.trino.port=443 +hpc.dao.trino.username=ncif-hpcdm-svc +hpc.dao.trino.password=< Configure Me > +hpc.dao.trino.url=jdbc:trino://${hpc.dao.trino.host}:${hpc.dao.trino.port}/vast +hpc.dao.trino.maxActive=10 + ############################################################################# # Scheduler Properties. ############################################################################# @@ -326,4 +334,5 @@ hpc.scheduler.migration.cron.processDataObjectListMigrationReceived.delay=25 0/1 hpc.scheduler.migration.cron.processCollectionListMigrationReceived.delay=35 0/1 * * * ? hpc.scheduler.migration.cron.processBulkMetadataUpdatetMigrationReceived.delay=32 0/1 * * * ? hpc.scheduler.migration.cron.processDataObjectMetadataUpdatetMigrationReceived.delay=38 0/1 * * * ? - +hpc.scheduler.migration.cron.processAutoTiering.delay=0 0 7 * * ? +hpc.scheduler.migration.cron.processBulkAutoTieringMigrationReceived.delay=56 0/1 * * * ? diff --git a/src/hpc-server/hpc-scheduler-migration/src/main/resources/logback.xml b/src/hpc-server/hpc-scheduler-migration/src/main/resources/logback.xml index c5acd035ff..a22625e296 100644 --- a/src/hpc-server/hpc-scheduler-migration/src/main/resources/logback.xml +++ b/src/hpc-server/hpc-scheduler-migration/src/main/resources/logback.xml @@ -1,7 +1,7 @@ - + ${SCHEDULER_MIGRATION_SERVER_LOG} diff --git a/src/hpc-server/hpc-scheduler/pom.xml b/src/hpc-server/hpc-scheduler/pom.xml index 9ae56f7294..84e90eceb5 100644 --- a/src/hpc-server/hpc-scheduler/pom.xml +++ b/src/hpc-server/hpc-scheduler/pom.xml @@ -18,7 +18,7 @@ gov.nih.nci.hpc hpc-server - 3.23.0 + 3.24.0 hpc-scheduler war diff --git a/src/hpc-server/hpc-scheduler/src/main/resources/WEB-INF/spring/hpc-scheduler.properties b/src/hpc-server/hpc-scheduler/src/main/resources/WEB-INF/spring/hpc-scheduler.properties index 8f8ec1e238..87ba681602 100644 --- a/src/hpc-server/hpc-scheduler/src/main/resources/WEB-INF/spring/hpc-scheduler.properties +++ b/src/hpc-server/hpc-scheduler/src/main/resources/WEB-INF/spring/hpc-scheduler.properties @@ -221,7 +221,7 @@ hpc.integration.ldap.userIdDomainName=nih.gov # S3 hpc.integration.s3.executorThreadPoolSize=8 hpc.integration.s3.awsTransferManagerThreadPoolSize=8 -hpc.integration.s3.pathStyleAccessEnabledProviders=CLOUDIAN,VAST +hpc.integration.s3.pathStyleAccessEnabledProviders=CLOUDIAN,VAST,VAST_NCIVS hpc.integration.s3.minimumUploadPartSize=52428800 hpc.integration.s3.multipartUploadThreshold=52428800 hpc.integration.s3.restoreNumDays=2 diff --git a/src/hpc-server/hpc-scheduler/src/main/resources/logback.xml b/src/hpc-server/hpc-scheduler/src/main/resources/logback.xml index 3f46786803..2ba098af6d 100644 --- a/src/hpc-server/hpc-scheduler/src/main/resources/logback.xml +++ b/src/hpc-server/hpc-scheduler/src/main/resources/logback.xml @@ -1,7 +1,7 @@ - + ${SCHEDULER_SERVER_LOG} diff --git a/src/hpc-server/hpc-ws-rs-api/pom.xml b/src/hpc-server/hpc-ws-rs-api/pom.xml index 2a1d99da46..84e4d1709b 100644 --- a/src/hpc-server/hpc-ws-rs-api/pom.xml +++ b/src/hpc-server/hpc-ws-rs-api/pom.xml @@ -19,7 +19,7 @@ gov.nih.nci.hpc hpc-server - 3.23.0 + 3.24.0 hpc-ws-rs-api jar diff --git a/src/hpc-server/hpc-ws-rs-api/src/main/java/gov/nih/nci/hpc/ws/rs/HpcDataMigrationRestService.java b/src/hpc-server/hpc-ws-rs-api/src/main/java/gov/nih/nci/hpc/ws/rs/HpcDataMigrationRestService.java index 0c510faeff..7391ae31e1 100644 --- a/src/hpc-server/hpc-ws-rs-api/src/main/java/gov/nih/nci/hpc/ws/rs/HpcDataMigrationRestService.java +++ b/src/hpc-server/hpc-ws-rs-api/src/main/java/gov/nih/nci/hpc/ws/rs/HpcDataMigrationRestService.java @@ -126,4 +126,16 @@ public Response retryDataObjectsOrCollectionsMigrationTask(@PathParam("taskId") @Consumes("application/json; charset=UTF-8, application/xml; charset=UTF-8") @Produces("application/json; charset=UTF-8, application/xml; charset=UTF-8") public Response migrateMetadata(HpcMetadataMigrationRequestDTO metadataMigrationRequest); + + /** + * Process auto-tiering for a specific data management configuration. + * + * @param configurationId The data management configuration ID. + * @return The REST service response. + */ + @POST + @Path("/autoTiering/{configurationId}") + @Consumes("application/json; charset=UTF-8, application/xml; charset=UTF-8") + @Produces("application/json; charset=UTF-8, application/xml; charset=UTF-8") + public Response processAutoTiering(@PathParam("configurationId") String configurationId); } diff --git a/src/hpc-server/hpc-ws-rs-impl/pom.xml b/src/hpc-server/hpc-ws-rs-impl/pom.xml index 7a285d72e0..62bb52b1cb 100644 --- a/src/hpc-server/hpc-ws-rs-impl/pom.xml +++ b/src/hpc-server/hpc-ws-rs-impl/pom.xml @@ -17,7 +17,7 @@ gov.nih.nci.hpc hpc-server - 3.23.0 + 3.24.0 hpc-ws-rs-impl war diff --git a/src/hpc-server/hpc-ws-rs-impl/src/main/java/gov/nih/nci/hpc/ws/rs/impl/HpcDataMigrationRestServiceImpl.java b/src/hpc-server/hpc-ws-rs-impl/src/main/java/gov/nih/nci/hpc/ws/rs/impl/HpcDataMigrationRestServiceImpl.java index 74b494850a..0d0162f8d7 100644 --- a/src/hpc-server/hpc-ws-rs-impl/src/main/java/gov/nih/nci/hpc/ws/rs/impl/HpcDataMigrationRestServiceImpl.java +++ b/src/hpc-server/hpc-ws-rs-impl/src/main/java/gov/nih/nci/hpc/ws/rs/impl/HpcDataMigrationRestServiceImpl.java @@ -147,5 +147,17 @@ public Response migrateMetadata(HpcMetadataMigrationRequestDTO metadataMigration return okResponse(migrationResponse, false); } - + + @Override + public Response processAutoTiering(String configurationId) { + HpcMigrationResponseDTO migrationResponse = null; + try { + migrationResponse = dataMigrationBusService.processAutoTiering(configurationId); + + } catch (HpcException e) { + return errorResponse(e); + } + + return okResponse(migrationResponse, false); + } } diff --git a/src/hpc-server/hpc-ws-rs-impl/src/main/resources/WEB-INF/spring/hpc-server.properties b/src/hpc-server/hpc-ws-rs-impl/src/main/resources/WEB-INF/spring/hpc-server.properties index e86e6bbe81..1b764aed5e 100644 --- a/src/hpc-server/hpc-ws-rs-impl/src/main/resources/WEB-INF/spring/hpc-server.properties +++ b/src/hpc-server/hpc-ws-rs-impl/src/main/resources/WEB-INF/spring/hpc-server.properties @@ -236,7 +236,7 @@ hpc.integration.ldap.userIdDomainName=nih.gov # S3 hpc.integration.s3.executorThreadPoolSize=8 hpc.integration.s3.awsTransferManagerThreadPoolSize=8 -hpc.integration.s3.pathStyleAccessEnabledProviders=CLOUDIAN,VAST +hpc.integration.s3.pathStyleAccessEnabledProviders=CLOUDIAN,VAST,VAST_NCIVS hpc.integration.s3.minimumUploadPartSize=52428800 hpc.integration.s3.multipartUploadThreshold=52428800 hpc.integration.s3.restoreNumDays=2 diff --git a/src/hpc-server/hpc-ws-rs-test/pom.xml b/src/hpc-server/hpc-ws-rs-test/pom.xml index 0ea3106248..1f81933589 100644 --- a/src/hpc-server/hpc-ws-rs-test/pom.xml +++ b/src/hpc-server/hpc-ws-rs-test/pom.xml @@ -18,7 +18,7 @@ gov.nih.nci.hpc hpc-server - 3.23.0 + 3.24.0 hpc-ws-rs-test diff --git a/src/hpc-server/pom.xml b/src/hpc-server/pom.xml index 364abdbfa2..bde6a2b3ce 100644 --- a/src/hpc-server/pom.xml +++ b/src/hpc-server/pom.xml @@ -18,7 +18,7 @@ gov.nih.nci.hpc hpc - 3.23.0 + 3.24.0 hpc-server pom @@ -41,7 +41,7 @@ 1.12.420 2.32.31 0.38.11 - 2.18.1 + 2.18.6 4.3.3.0-RELEASE 0.12.5 1.79 @@ -55,6 +55,7 @@ 5.4.0 4.8.0 8.14.0 + 476 @@ -492,6 +493,11 @@ bucket4j_jdk17-core ${bucket4j.version} + + io.trino + trino-jdbc + ${trino.version} + diff --git a/src/hpc-web/pom.xml b/src/hpc-web/pom.xml index 304ec38952..b5247f6881 100644 --- a/src/hpc-web/pom.xml +++ b/src/hpc-web/pom.xml @@ -5,7 +5,7 @@ 4.0.0 gov.nih.nci.hpc hpc-web - 3.23.0 + 3.24.0 war HPC-DME Web Application HPC Data Management @@ -17,7 +17,7 @@ - 3.23.0 + 3.24.0 UTF-8 gov.nih.nci.hpc.web.HpcWebApplication 21 diff --git a/src/pom.xml b/src/pom.xml index 2b54c892c6..74bf16bdbb 100644 --- a/src/pom.xml +++ b/src/pom.xml @@ -17,7 +17,7 @@ HPC gov.nih.nci.hpc hpc - 3.23.0 + 3.24.0 pom diff --git a/utils/hpc-client/hpc-cli-3.23.0.jar b/utils/hpc-client/hpc-cli-3.24.0.jar similarity index 91% rename from utils/hpc-client/hpc-cli-3.23.0.jar rename to utils/hpc-client/hpc-cli-3.24.0.jar index b7c4f53a9d..1f43f50185 100644 Binary files a/utils/hpc-client/hpc-cli-3.23.0.jar and b/utils/hpc-client/hpc-cli-3.24.0.jar differ diff --git a/utils/hpc-client/lib/hpc-common-3.23.0.jar b/utils/hpc-client/lib/hpc-common-3.24.0.jar similarity index 64% rename from utils/hpc-client/lib/hpc-common-3.23.0.jar rename to utils/hpc-client/lib/hpc-common-3.24.0.jar index 8c487925f5..342e13a8b7 100644 Binary files a/utils/hpc-client/lib/hpc-common-3.23.0.jar and b/utils/hpc-client/lib/hpc-common-3.24.0.jar differ diff --git a/utils/hpc-client/lib/hpc-domain-types-3.23.0.jar b/utils/hpc-client/lib/hpc-domain-types-3.24.0.jar similarity index 57% rename from utils/hpc-client/lib/hpc-domain-types-3.23.0.jar rename to utils/hpc-client/lib/hpc-domain-types-3.24.0.jar index 132e111dcb..6008f89fcb 100644 Binary files a/utils/hpc-client/lib/hpc-domain-types-3.23.0.jar and b/utils/hpc-client/lib/hpc-domain-types-3.24.0.jar differ diff --git a/utils/hpc-client/lib/hpc-dto-3.23.0.jar b/utils/hpc-client/lib/hpc-dto-3.24.0.jar similarity index 52% rename from utils/hpc-client/lib/hpc-dto-3.23.0.jar rename to utils/hpc-client/lib/hpc-dto-3.24.0.jar index 96b1a2854f..05c14477b5 100644 Binary files a/utils/hpc-client/lib/hpc-dto-3.23.0.jar and b/utils/hpc-client/lib/hpc-dto-3.24.0.jar differ diff --git a/utils/version b/utils/version index a28d6e83d5..555b7545c5 100644 --- a/utils/version +++ b/utils/version @@ -1,2 +1,2 @@ -hpc.jar.version=hpc-cli-3.23.0.jar +hpc.jar.version=hpc-cli-3.24.0.jar