Features/hpcdatamgm 2088 - DON'T MERGE

src/hpc-server/hpc-app-service-api/src/main/java/gov/nih/nci/hpc/service/HpcDataManagementService.java

@@ -594,6 +594,15 @@ public List<HpcBulkDataObjectRegistrationResult> getRegistrationResults(String u
 	 */
 	public HpcDataManagementConfiguration getDataManagementConfiguration(String id);
 
+	/**
+	 * Get all Data Management Configurations that have auto-tiering enabled.
+	 * A configuration is considered to have auto-tiering enabled if it has a
+	 * non-null s3AutoTieringArchiveConfigurationId.
+	 *
+	 * @return List of data management configurations with auto-tiering enabled.
+	 */
+	public List<HpcDataManagementConfiguration> getAutoTieringDataManagementConfigurations();
+
 	/**
 	 * Add a data object registration request result to the DB.
 	 *

src/hpc-server/hpc-app-service-impl/src/main/java/gov/nih/nci/hpc/service/impl/HpcDataManagementServiceImpl.java

@@ -28,6 +28,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.stream.Collectors;
 
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
@@ -1205,6 +1206,13 @@ public HpcDataManagementConfiguration getDataManagementConfiguration(String id)
 		return dataManagementConfigurationLocator.get(id);
 	}
 
+	@Override
+	public List<HpcDataManagementConfiguration> getAutoTieringDataManagementConfigurations() {
+		return dataManagementConfigurationLocator.values().stream()
+				.filter(config -> !StringUtils.isEmpty(config.getS3AutoTieringConfigurationId()))
+				.collect(Collectors.toList());
+	}
+
 	@Override
 	public void addDataObjectRegistrationResult(String path, HpcSystemGeneratedMetadata systemGeneratedMetadata,
 			boolean result, String message) throws HpcException {

src/hpc-server/hpc-app-service-impl/src/main/java/gov/nih/nci/hpc/service/impl/HpcDataMigrationServiceImpl.java

@@ -9,6 +9,7 @@
 package gov.nih.nci.hpc.service.impl;
 
 import java.util.Calendar;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -24,17 +25,20 @@
 import com.google.common.collect.Iterables;
 
 import gov.nih.nci.hpc.dao.HpcDataMigrationDAO;
+import gov.nih.nci.hpc.dao.HpcExternalArchiveDAO;
 import gov.nih.nci.hpc.domain.datamigration.HpcDataMigrationResult;
 import gov.nih.nci.hpc.domain.datamigration.HpcDataMigrationStatus;
 import gov.nih.nci.hpc.domain.datamigration.HpcDataMigrationType;
 import gov.nih.nci.hpc.domain.datatransfer.HpcArchiveObjectMetadata;
 import gov.nih.nci.hpc.domain.datatransfer.HpcDataTransferType;
 import gov.nih.nci.hpc.domain.datatransfer.HpcDeepArchiveStatus;
+import gov.nih.nci.hpc.domain.datatransfer.HpcFileLocation;
 import gov.nih.nci.hpc.domain.datatransfer.HpcStreamingUploadSource;
 import gov.nih.nci.hpc.domain.error.HpcErrorType;
 import gov.nih.nci.hpc.domain.model.HpcDataMigrationTask;
 import gov.nih.nci.hpc.domain.model.HpcDataMigrationTaskResult;
 import gov.nih.nci.hpc.domain.model.HpcDataMigrationTaskStatus;
+import gov.nih.nci.hpc.domain.model.HpcDataManagementConfiguration;
 import gov.nih.nci.hpc.domain.model.HpcDataObjectUploadRequest;
 import gov.nih.nci.hpc.domain.model.HpcDataTransferConfiguration;
 import gov.nih.nci.hpc.domain.model.HpcSystemGeneratedMetadata;
@@ -46,6 +50,8 @@
 import gov.nih.nci.hpc.service.HpcMetadataService;
 import gov.nih.nci.hpc.service.HpcSecurityService;
 
+import static gov.nih.nci.hpc.service.impl.HpcDomainValidator.isValidFileLocation;
+
 /**
  * HPC Data Search Application Service Implementation.
  *
@@ -85,6 +91,10 @@ public class HpcDataMigrationServiceImpl implements HpcDataMigrationService {
 	@Autowired
 	private HpcSecurityService securityService = null;
 
+	// External Archive DAO.
+	@Autowired
+	private HpcExternalArchiveDAO externalArchiveDAO = null;
+
 	// A configured ID representing the server performing a migration task.
 	@Value("${hpc.service.serverId}")
 	private String serverId = null;
@@ -123,8 +133,9 @@ private HpcDataMigrationServiceImpl() throws HpcException {
 	@Override
 	public HpcDataMigrationTask createDataObjectMigrationTask(String path, String userId, String configurationId,
 			String fromS3ArchiveConfigurationId, String toS3ArchiveConfigurationId, String collectionMigrationTaskId,
-			boolean alignArchivePath, long size, String retryTaskId, String retryUserId, boolean metadataUpdateRequest,
-			String metadataFromArchiveFileContainerId, String metadataToArchiveFileContainerId) throws HpcException {
+			boolean alignArchivePath, Long size, String retryTaskId, String retryUserId, boolean metadataUpdateRequest,
+			String metadataFromArchiveFileContainerId, String metadataToArchiveFileContainerId,
+			boolean autoTieringRequest, HpcFileLocation fromS3ArchiveLocation) throws HpcException {
 		// Check if a task already exist.
 		HpcDataMigrationTask migrationTask = dataMigrationDAO.getDataObjectMigrationTask(collectionMigrationTaskId,
 				path);
@@ -146,14 +157,21 @@ public HpcDataMigrationTask createDataObjectMigrationTask(String path, String us
 			return migrationTask;
 		}
 
+		// Validate auto tiering request.
+		if(autoTieringRequest && !isValidFileLocation(fromS3ArchiveLocation)) {
+			throw new HpcException("Auto tiering requested w/o an external file location",
+			                       HpcErrorType.INVALID_REQUEST_INPUT);
+		}
+
 		// Create and persist a migration task.
 		migrationTask.setPath(path);
 		migrationTask.setUserId(userId);
 		migrationTask.setConfigurationId(configurationId);
 		migrationTask.setFromS3ArchiveConfigurationId(fromS3ArchiveConfigurationId);
 		migrationTask.setToS3ArchiveConfigurationId(toS3ArchiveConfigurationId);
 		migrationTask.setCreated(Calendar.getInstance());
-		migrationTask.setStatus(HpcDataMigrationStatus.RECEIVED);
+		migrationTask.setStatus(autoTieringRequest ? HpcDataMigrationStatus.AUTO_TIERING_RECEIVED :
+		                                             HpcDataMigrationStatus.RECEIVED);
 		migrationTask.setType(metadataUpdateRequest ? HpcDataMigrationType.DATA_OBJECT_METADATA_UPDATE
 				: HpcDataMigrationType.DATA_OBJECT);
 		migrationTask.setParentId(collectionMigrationTaskId);
@@ -164,6 +182,7 @@ public HpcDataMigrationTask createDataObjectMigrationTask(String path, String us
 		migrationTask.setRetryUserId(retryUserId);
 		migrationTask.setMetadataFromArchiveFileContainerId(metadataFromArchiveFileContainerId);
 		migrationTask.setMetadataToArchiveFileContainerId(metadataToArchiveFileContainerId);
+		migrationTask.setFromS3ArchiveLocation(fromS3ArchiveLocation);
 
 		// Persist the task.
 		dataMigrationDAO.upsertDataMigrationTask(migrationTask);
@@ -632,4 +651,87 @@ public HpcDataMigrationTask createMetadataMigrationTask(String fromS3ArchiveConf
 		dataMigrationDAO.upsertDataMigrationTask(migrationTask);
 		return migrationTask;
 	}
+
+	@Override
+	public HpcDataMigrationTask createBulkAutoTieringTask(String configurationId, String fromS3ArchiveConfigurationId,
+			String toS3ArchiveConfigurationId, String userId) throws HpcException {
+		// Create and persist a bulk auto-tiering task.
+		HpcDataMigrationTask migrationTask = new HpcDataMigrationTask();
+		migrationTask.setConfigurationId(configurationId);
+		migrationTask.setUserId(userId);
+		migrationTask.setFromS3ArchiveConfigurationId(fromS3ArchiveConfigurationId);
+		migrationTask.setToS3ArchiveConfigurationId(toS3ArchiveConfigurationId);
+		migrationTask.setCreated(Calendar.getInstance());
+		migrationTask.setStatus(HpcDataMigrationStatus.RECEIVED);
+		migrationTask.setType(HpcDataMigrationType.BULK_AUTO_TIERING);
+		migrationTask.setPath(null);
+		migrationTask.setAlignArchivePath(false);
+		migrationTask.setPercentComplete(0);
+		migrationTask.setSize(null);
+		migrationTask.setRetryTaskId(null);
+		migrationTask.setRetryUserId(null);
+		migrationTask.setRetryFailedItemsOnly(null);
+		migrationTask.setMetadataFromArchiveFileContainerId(null);
+		migrationTask.setMetadataToArchiveFileContainerId(null);
+		migrationTask.setMetadataArchiveFileIdPattern(null);
+
+		// Persist the task.
+		dataMigrationDAO.upsertDataMigrationTask(migrationTask);
+		return migrationTask;
+	}
+
+	@Override
+	public Map<String, HpcFileLocation> getDataObjectsForAutoTiering(String configurationId,
+			String s3ArchiveConfigurationId) throws HpcException {
+		logger.info("getDataObjectsForAutoTiering called with configurationId: {}, s3ArchiveConfigurationId: {}",
+				configurationId, s3ArchiveConfigurationId);
+
+		// Get the data management configuration.
+		HpcDataManagementConfiguration dataManagementConfiguration = dataManagementConfigurationLocator.get(configurationId);
+		if (dataManagementConfiguration == null) {
+			throw new HpcException("Data management configuration not found for ID: " + configurationId,
+					HpcErrorType.INVALID_REQUEST_INPUT);
+		}
+
+		// Get the S3 data transfer configuration
+		HpcDataTransferConfiguration s3Configuration = dataManagementConfigurationLocator
+				.getDataTransferConfiguration(configurationId, s3ArchiveConfigurationId, HpcDataTransferType.S_3);
+		if (s3Configuration == null) {
+			throw new HpcException("S3 archive configuration not found for ID: " + s3ArchiveConfigurationId,
+					HpcErrorType.INVALID_REQUEST_INPUT);
+		}
+
+		// Validate search path and inactivity months are defined
+		if (StringUtils.isEmpty(s3Configuration.getAutoTieringSearchPath())) {
+			throw new HpcException("Auto-tiering search path is not defined for S3 configuration: " + s3ArchiveConfigurationId,
+					HpcErrorType.INVALID_REQUEST_INPUT);
+		}
+		if (s3Configuration.getAutoTieringInactivityMonths() == null) {
+			throw new HpcException("Auto-tiering inactivity months is not defined for S3 configuration: " + s3ArchiveConfigurationId,
+					HpcErrorType.INVALID_REQUEST_INPUT);
+		}
+
+		// Build the map of file paths to HpcFileLocation - these are the data objects to be auto-tiered.
+		String basePath = dataManagementConfiguration.getBasePath();
+		String bucket = s3Configuration.getBaseArchiveDestination().getFileLocation().getFileContainerId();
+		String objectIdPrefix = s3Configuration.getBaseArchiveDestination().getFileLocation().getFileId();
+
+		Map<String, HpcFileLocation> autoTieringDataObjects = new HashMap<>();
+		// Query for files not accessed within the specified period and create the map of data obejcts to return.
+		externalArchiveDAO.getFilesNotAccessed(
+				s3Configuration.getAutoTieringSearchPath(),
+				s3Configuration.getAutoTieringInactivityMonths()).forEach(searchRelativePath -> {
+					// Construct HpcFileLocation of the data object to be auto-tiered.
+					HpcFileLocation fileLocation = new HpcFileLocation();
+					fileLocation.setFileContainerId(bucket);
+			        fileLocation.setFileId(objectIdPrefix + searchRelativePath);
+
+					// Add this data object to the returned map.
+					autoTieringDataObjects.put(basePath + searchRelativePath, fileLocation);
+				});
+
+		logger.info("Found {} files for auto-tiering for configuration: {}", autoTieringDataObjects.size(), configurationId);
+		return autoTieringDataObjects;
+	}
 }
+

src/hpc-server/hpc-bus-service-api/src/main/java/gov/nih/nci/hpc/bus/HpcDataMigrationBusService.java

@@ -164,12 +164,34 @@ public HpcMigrationResponseDTO migrateMetadata(HpcMetadataMigrationRequestDTO me
 	 * @throws HpcException on service failure.
 	 */
 	public void assignMigrationServer() throws HpcException;
-
 
 	/**
 	 * Restart data object and collection migration tasks that are in progress.
 	 *
 	 * @throws HpcException on service failure.
 	 */
 	public void restartDataMigrationTasks() throws HpcException;
+
+	/**
+	 * Process auto-tiering for all data management configurations
+	 *
+	 * @throws HpcException on service failure.
+	 */
+	public void processAutoTiering() throws HpcException;
+
+	/**
+	 * Process auto-tiering for a given data management configuration ID
+	 *
+	 * @param configurationId The data management configuration ID.
+	 * @return Migration Response DTO.
+	 * @throws HpcException on service failure.
+	 */
+	public HpcMigrationResponseDTO processAutoTiering(String configurationId) throws HpcException;
+
+	/**
+	 * Process received bulk auto-tiering migration tasks.
+	 *
+	 * @throws HpcException on service failure.
+	 */
+	public void processBulkAutoTieringMigrationReceived() throws HpcException;
 }