Features/hpcdatamgm 2097 version2

src/hpc-server/hpc-app-service-api/src/main/java/gov/nih/nci/hpc/service/HpcDataManagementService.java

@@ -28,6 +28,7 @@
 import gov.nih.nci.hpc.domain.model.HpcBulkDataObjectRegistrationStatus;
 import gov.nih.nci.hpc.domain.model.HpcBulkDataObjectRegistrationTask;
 import gov.nih.nci.hpc.domain.model.HpcDataManagementConfiguration;
+import gov.nih.nci.hpc.domain.model.HpcDataTransferConfiguration;
 import gov.nih.nci.hpc.domain.model.HpcDataObjectRegistrationRequest;
 import gov.nih.nci.hpc.domain.model.HpcDataObjectRegistrationResult;
 import gov.nih.nci.hpc.domain.model.HpcDataTransferConfiguration;
@@ -594,6 +595,14 @@ public List<HpcBulkDataObjectRegistrationResult> getRegistrationResults(String u
 	 */
 	public HpcDataManagementConfiguration getDataManagementConfiguration(String id);
 
+	/**
+	 * Returns HpcDataTransferConfiguration for the given external path
+	 *
+	 * @return HpcDataTransferConfiguration
+	 */
+
+	public HpcDataTransferConfiguration findDataTransferConfigurationForExternalPath(String path)  throws HpcException;
+
 	/**
 	 * Add a data object registration request result to the DB.
 	 *

src/hpc-server/hpc-app-service-api/src/main/java/gov/nih/nci/hpc/service/HpcDataTransferService.java

@@ -202,7 +202,7 @@ public HpcDataObjectDownloadResponse downloadDataObject(String path, HpcFileLoca
 			HpcSynchronousDownloadFilter synchronousDownloadFilter, HpcDataTransferType dataTransferType,
 			String configurationId, String s3ArchiveConfigurationId, String retryTaskId, String userId,
 			String retryUserId, boolean completionEvent, String collectionDownloadTaskId, long size,
-			HpcDataTransferUploadStatus downloadDataObject, HpcDeepArchiveStatus deepArchiveStatus) throws HpcException;
+			HpcDataTransferUploadStatus downloadDataObject, HpcDeepArchiveStatus deepArchiveStatus, boolean externalArchiveFlag) throws HpcException;
 
 	/**
 	 * Generate a (pre-signed) download URL for a data object file.
@@ -601,6 +601,15 @@ public void completeSynchronousDataObjectDownloadTask(HpcDownloadTaskResult task
 	public boolean markProcessedDataObjectDownloadTask(HpcDataObjectDownloadTask downloadTask,
 			HpcDataTransferType dataTransferType, boolean inProcess) throws HpcException;
 
+	/**
+	 * Update a data object download task.
+	 *
+	 * @param downloadTask     The download task to update
+	 * @throws HpcException on service failure.
+	 */
+	public void changeDataObjectDownloadTaskExternalStatus(HpcDataObjectDownloadTask downloadTask)
+			throws HpcException;
+
 	/**
 	 * Update a data object download task. % Complete is calculated and any change
 	 * on the task object will be persisted.
@@ -644,6 +653,8 @@ public boolean updateDataObjectDownloadTask(HpcDataObjectDownloadTask downloadTa
 	 *                                              used in the destination path,
 	 *                                              otherwise just the object name
 	 *                                              will be used.
+	 * @param externalArchiveFlag                  	Indicates if the collection contains data objects in an external archive.
+	 *
 	 * @return The submitted collection download task.
 	 * @throws HpcException on service failure.
 	 */
@@ -653,7 +664,7 @@ public HpcCollectionDownloadTask downloadCollection(String path,
 			HpcGoogleDownloadDestination googleCloudStorageDownloadDestination,
 			HpcAsperaDownloadDestination asperaDownloadDestination, HpcBoxDownloadDestination boxDownloadDestination,
 			String userId, String configurationId, boolean appendPathToDownloadDestination,
-			boolean appendCollectionNameToDownloadDestination) throws HpcException;
+			boolean appendCollectionNameToDownloadDestination, boolean externalArchiveFlag) throws HpcException;
 
 	/**
 	 * Submit a request to download collections.
@@ -701,7 +712,7 @@ public HpcCollectionDownloadTask downloadCollections(List<String> collectionPath
 			HpcGoogleDownloadDestination googleCloudStorageDownloadDestination,
 			HpcAsperaDownloadDestination asperaDownloadDestination, HpcBoxDownloadDestination boxDownloadDestination,
 			String userId, String configurationId, boolean appendPathToDownloadDestination,
-			boolean appendCollectionNameToDownloadDestination) throws HpcException;
+			boolean appendCollectionNameToDownloadDestination, boolean externalArchiveFlag) throws HpcException;
 
 	/**
 	 * Submit a request to download data objects.

src/hpc-server/hpc-app-service-impl/src/main/java/gov/nih/nci/hpc/service/impl/HpcDataManagementConfigurationLocator.java

@@ -15,6 +15,7 @@
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
@@ -100,6 +101,15 @@ public Set<String> getDocs() {
 		return docs;
 	}
 
+	/**
+	 * Get all supported S3 Archive configurations.
+	 *
+	 * @return A collection of all supported S3 Archive configurations.
+	 */
+	public Collection<HpcDataTransferConfiguration>  getS3ArchiveConfigurations() {
+		return s3ArchiveConfigurations.values();
+	}
+
 	/**
 	 * Get configuration ID by base path.
 	 *

src/hpc-server/hpc-app-service-impl/src/main/java/gov/nih/nci/hpc/service/impl/HpcDataManagementServiceImpl.java

@@ -25,9 +25,11 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Calendar;
+import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
 
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
@@ -70,6 +72,7 @@
 import gov.nih.nci.hpc.domain.model.HpcBulkDataObjectRegistrationStatus;
 import gov.nih.nci.hpc.domain.model.HpcBulkDataObjectRegistrationTask;
 import gov.nih.nci.hpc.domain.model.HpcDataManagementConfiguration;
+import gov.nih.nci.hpc.domain.model.HpcDataTransferConfiguration;
 import gov.nih.nci.hpc.domain.model.HpcDataObjectRegistrationRequest;
 import gov.nih.nci.hpc.domain.model.HpcDataObjectRegistrationResult;
 import gov.nih.nci.hpc.domain.model.HpcDataTransferConfiguration;
@@ -1179,6 +1182,22 @@ public List<HpcDataManagementConfiguration> getDataManagementConfigurations() {
 		return new ArrayList<>(dataManagementConfigurationLocator.values());
 	}
 
+	@Override
+	public HpcDataTransferConfiguration findDataTransferConfigurationForExternalPath(String path) throws HpcException  {
+		HpcDataTransferConfiguration dataTransferConfiguration = null;
+		for (HpcDataManagementConfiguration dataManagementConfiguration : dataManagementConfigurationLocator.values()) {
+			String dataTransferConfigurationId = dataManagementConfiguration.getS3UploadConfigurationId();
+			if (dataTransferConfigurationId != null) {
+				HpcDataTransferConfiguration dataTransferConfigurationCandidate = dataManagementConfigurationLocator.getS3ArchiveConfiguration(dataTransferConfigurationId);
+				if (dataTransferConfigurationCandidate != null && dataTransferConfigurationCandidate.getPosixPath() != null && path.startsWith(dataTransferConfigurationCandidate.getPosixPath())) {
+					dataTransferConfiguration = dataTransferConfigurationCandidate;
+					break;
+				}
+			}
+		}
+		return dataTransferConfiguration;
+	}
+
 	@Override
 	public String findDataManagementConfigurationId(String path) {
 		if (StringUtils.isEmpty(path)) {

src/hpc-server/hpc-app-service-impl/src/main/java/gov/nih/nci/hpc/service/impl/HpcDataTransferServiceImpl.java

@@ -52,6 +52,7 @@
 import gov.nih.nci.hpc.dao.HpcDataDownloadDAO;
 import gov.nih.nci.hpc.dao.HpcDataRegistrationDAO;
 import gov.nih.nci.hpc.dao.HpcGlobusTransferTaskDAO;
+import gov.nih.nci.hpc.domain.datamanagement.HpcDataObject;
 import gov.nih.nci.hpc.domain.datamanagement.HpcPathAttributes;
 import gov.nih.nci.hpc.domain.datamanagement.HpcPathPermissions;
 import gov.nih.nci.hpc.domain.datatransfer.HpcAddArchiveObjectMetadataResponse;
@@ -96,6 +97,7 @@
 import gov.nih.nci.hpc.domain.error.HpcDomainValidationResult;
 import gov.nih.nci.hpc.domain.error.HpcErrorType;
 import gov.nih.nci.hpc.domain.error.HpcRequestRejectReason;
+import gov.nih.nci.hpc.domain.metadata.HpcMetadataEntries;
 import gov.nih.nci.hpc.domain.metadata.HpcMetadataEntry;
 import gov.nih.nci.hpc.domain.model.HpcDataObjectUploadRequest;
 import gov.nih.nci.hpc.domain.model.HpcDataObjectUploadResponse;
@@ -572,7 +574,7 @@ public HpcDataObjectDownloadResponse downloadDataObject(String path, HpcFileLoca
 			HpcSynchronousDownloadFilter synchronousDownloadFilter, HpcDataTransferType dataTransferType,
 			String configurationId, String s3ArchiveConfigurationId, String retryTaskId, String userId,
 			String retryUserId, boolean completionEvent, String collectionDownloadTaskId, long size,
-			HpcDataTransferUploadStatus dataTransferStatus, HpcDeepArchiveStatus deepArchiveStatus)
+			HpcDataTransferUploadStatus dataTransferStatus, HpcDeepArchiveStatus deepArchiveStatus, boolean externalArchiveFlag)
 			throws HpcException {
 		// Input Validation.
 		if (dataTransferType == null || !isValidFileLocation(archiveLocation)) {
@@ -603,6 +605,7 @@ public HpcDataObjectDownloadResponse downloadDataObject(String path, HpcFileLoca
 		downloadRequest.setCompletionEvent(completionEvent);
 		downloadRequest.setCollectionDownloadTaskId(collectionDownloadTaskId);
 		downloadRequest.setSize(size);
+		downloadRequest.setExternalArchiveFlag(externalArchiveFlag);
 
 		// Create a download response.
 		HpcDataObjectDownloadResponse response = new HpcDataObjectDownloadResponse();
@@ -1130,7 +1133,6 @@ public boolean getCollectionDownloadTaskCancellationRequested(String taskId) thr
 	@Override
 	public HpcDownloadTaskResult completeDataObjectDownloadTask(HpcDataObjectDownloadTask downloadTask,
 			HpcDownloadResult result, String message, Calendar completed, long bytesTransferred) throws HpcException {
-
 		// Input validation
 		if (downloadTask == null) {
 			throw new HpcException("Invalid data object download task", HpcErrorType.INVALID_REQUEST_INPUT);
@@ -1488,13 +1490,16 @@ public void stageHyperfileDataObjectDownloadTask(HpcDataObjectDownloadTask downl
 
 	@Override
 	public void resetDataObjectDownloadTask(HpcDataObjectDownloadTask downloadTask) throws HpcException {
-
 		logger.debug(
 				"download task: [taskId={}] - resetDataObjectDownloadTask called. Setting in-process=false [transfer-type={}, server-id={}]",
 				downloadTask.getId(), downloadTask.getDataTransferType(),
 				HpcDataTransferType.S_3.equals(downloadTask.getDataTransferType()) ? s3DownloadTaskServerId : null);
 
-		downloadTask.setDataTransferStatus(HpcDataTransferDownloadStatus.RECEIVED);
+		if(downloadTask.getExternalArchiveFlag()) {
+			downloadTask.setDataTransferStatus(HpcDataTransferDownloadStatus.RECEIVED_EXTERNAL);
+		} else {
+			downloadTask.setDataTransferStatus(HpcDataTransferDownloadStatus.RECEIVED);
+		}
 		downloadTask.setPercentComplete(0);
 		downloadTask.setInProcess(false);
 		downloadTask.setS3DownloadTaskServerId(null);
@@ -1515,11 +1520,24 @@ public void resetDataObjectDownloadTasksInProcess() throws HpcException {
 		dataDownloadDAO.resetDataObjectDownloadTaskInProcess(s3DownloadTaskServerId);
 	}
 
+	@Override
+	public void changeDataObjectDownloadTaskExternalStatus(HpcDataObjectDownloadTask downloadTask) throws HpcException {
+		if(downloadTask.getDataTransferStatus().equals(HpcDataTransferDownloadStatus.RECEIVED_EXTERNAL)) {
+			downloadTask.setDataTransferStatus(HpcDataTransferDownloadStatus.RECEIVED);
+			downloadTask.setExternalArchiveFlag(true);
+		}
+		dataDownloadDAO.updateDataObjectDownloadTask(downloadTask);
+		//return downloadTask;
+	}
+
 	@Override
 	public boolean markProcessedDataObjectDownloadTask(HpcDataObjectDownloadTask downloadTask,
 			HpcDataTransferType dataTransferType, boolean inProcess) throws HpcException {
 		// Only set in-process to true if this task in a RECEIVED status, and the
 		// in-process not already true.
+		if(downloadTask.getDataTransferStatus().equals(HpcDataTransferDownloadStatus.RECEIVED_EXTERNAL)) {
+			return false;
+		}
 		boolean updated = true;
 		String serverId = HpcDataTransferType.S_3.equals(dataTransferType) ? s3DownloadTaskServerId : null;
 
@@ -1593,7 +1611,6 @@ public boolean updateDataObjectDownloadTask(HpcDataObjectDownloadTask downloadTa
 		logger.info("download task: [taskId={}] - % complete - {} [transfer-type={}, destination-type={}]",
 				downloadTask.getId(), percentComplete, downloadTask.getDataTransferType(),
 				downloadTask.getDestinationType());
-
 		return dataDownloadDAO.updateDataObjectDownloadTask(downloadTask);
 	}
 
@@ -1604,7 +1621,7 @@ public HpcCollectionDownloadTask downloadCollection(String path,
 			HpcGoogleDownloadDestination googleCloudStorageDownloadDestination,
 			HpcAsperaDownloadDestination asperaDownloadDestination, HpcBoxDownloadDestination boxDownloadDestination,
 			String userId, String configurationId, boolean appendPathToDownloadDestination,
-			boolean appendCollectionNameToDownloadDestination) throws HpcException {
+			boolean appendCollectionNameToDownloadDestination, boolean externalArchiveFlag) throws HpcException {
 
 		// Validate the download destination.
 		validateDownloadDestination(globusDownloadDestination, s3DownloadDestination, googleDriveDownloadDestination,
@@ -1632,6 +1649,11 @@ public HpcCollectionDownloadTask downloadCollection(String path,
 		downloadTask.setDoc(dataManagementService.getDataManagementConfiguration(configurationId).getDoc());
 		downloadTask.setAppendPathToDownloadDestination(appendPathToDownloadDestination);
 		downloadTask.setAppendCollectionNameToDownloadDestination(appendCollectionNameToDownloadDestination);
+		if(externalArchiveFlag) {
+			downloadTask.setStatus(HpcCollectionDownloadTaskStatus.RECEIVED_EXTERNAL);
+		} else {
+			downloadTask.setStatus(HpcCollectionDownloadTaskStatus.RECEIVED);
+		}
 
 		// Persist the request.
 		dataDownloadDAO.upsertCollectionDownloadTask(downloadTask);
@@ -1646,7 +1668,7 @@ public HpcCollectionDownloadTask downloadCollections(List<String> collectionPath
 			HpcGoogleDownloadDestination googleCloudStorageDownloadDestination,
 			HpcAsperaDownloadDestination asperaDownloadDestination, HpcBoxDownloadDestination boxDownloadDestination,
 			String userId, String configurationId, boolean appendPathToDownloadDestination,
-			boolean appendCollectionNameToDownloadDestination) throws HpcException {
+			boolean appendCollectionNameToDownloadDestination, boolean externalArchiveFlag) throws HpcException {
 		// Validate the download destination.
 		validateDownloadDestination(globusDownloadDestination, s3DownloadDestination, googleDriveDownloadDestination,
 				googleCloudStorageDownloadDestination, asperaDownloadDestination, boxDownloadDestination, null,
@@ -1669,6 +1691,11 @@ public HpcCollectionDownloadTask downloadCollections(List<String> collectionPath
 		downloadTask.setAppendPathToDownloadDestination(appendPathToDownloadDestination);
 		downloadTask.setAppendCollectionNameToDownloadDestination(appendCollectionNameToDownloadDestination);
 		downloadTask.setDoc(dataManagementService.getDataManagementConfiguration(configurationId).getDoc());
+		if(externalArchiveFlag) {
+			downloadTask.setStatus(HpcCollectionDownloadTaskStatus.RECEIVED_EXTERNAL);
+		} else {
+			downloadTask.setStatus(HpcCollectionDownloadTaskStatus.RECEIVED);
+		}
 
 		// Persist the request.
 		dataDownloadDAO.upsertCollectionDownloadTask(downloadTask);
@@ -4310,7 +4337,6 @@ private void createDownloadTask(HpcDataObjectDownloadRequest firstHopDownloadReq
 				HpcFileLocation secondHopArchiveLocation, HpcGlobusDownloadDestination secondHopGlobusDestination,
 				HpcDataTransferDownloadStatus dataTransferDownloadStatus, HpcDataTransferType destinationType)
 				throws HpcException {
-
 			downloadTask.setDataTransferType(HpcDataTransferType.S_3);
 			downloadTask.setDataTransferStatus(dataTransferDownloadStatus);
 			downloadTask.setDownloadFilePath(sourceFile.getAbsolutePath());
@@ -4330,6 +4356,7 @@ private void createDownloadTask(HpcDataObjectDownloadRequest firstHopDownloadReq
 			downloadTask.setPercentComplete(0);
 			downloadTask.setSize(firstHopDownloadRequest.getSize());
 			downloadTask.setFirstHopRetried(false);
+			downloadTask.setExternalArchiveFlag(firstHopDownloadRequest.getExternalArchiveFlag());
 			downloadTask.setS3DownloadTaskServerId(
 					dataTransferDownloadStatus.equals(HpcDataTransferDownloadStatus.IN_PROGRESS)
 							? s3DownloadTaskServerId
@@ -4374,7 +4401,7 @@ private void updateDownloadTask(HpcDataObjectDownloadTask downloadTask,
 			this.downloadTask.setFirstHopRetried(downloadTask.getFirstHopRetried());
 			this.downloadTask.setRetryTaskId(downloadTask.getRetryTaskId());
 			this.downloadTask.setRetryUserId(downloadTask.getRetryUserId());
-
+			this.downloadTask.setExternalArchiveFlag(downloadTask.getExternalArchiveFlag());
 			dataDownloadDAO.updateDataObjectDownloadTask(this.downloadTask);
 		}
 

src/hpc-server/hpc-app-service-impl/src/test/java/gov/nih/nci/hpc/service/impl/HpcDataTransferServiceImplTest.java

@@ -245,7 +245,7 @@ public void testDownloadDataObjectNullDataTransferType() throws HpcException {
 		expectedException.expect(HpcException.class);
 		expectedException.expectMessage("Invalid data transfer request");
 		dataTransferService.downloadDataObject("", null, null, null, null, null, null, null, null, null, null, null, "",
-				"", "", false, null, 0L, HpcDataTransferUploadStatus.ARCHIVED, null);
+				"", "", false, null, 0L, HpcDataTransferUploadStatus.ARCHIVED, null, false);
 	}
 
 	/**
@@ -260,7 +260,7 @@ public void testDownloadDataObjectInvalidArchiveLocation() throws HpcException {
 		expectedException.expect(HpcException.class);
 		expectedException.expectMessage("Invalid data transfer request");
 		dataTransferService.downloadDataObject("", new HpcFileLocation(), null, null, null, null, null, null, null,
-				null, null, null, "", "", "", false, null, 0L, HpcDataTransferUploadStatus.ARCHIVED, null);
+				null, null, null, "", "", "", false, null, 0L, HpcDataTransferUploadStatus.ARCHIVED, null, false);
 	}
 
 	/**
@@ -319,7 +319,7 @@ public void testS3DownloadDataObject() throws HpcException {
 		HpcDataObjectDownloadResponse downloadResponse = dataTransferService.downloadDataObject("/test/path",
 				archiveLocation, null, s3loadDestination, null, null, null, null, null, HpcDataTransferType.S_3,
 				"testConfigId", "", null, "testUserId", null, false, null, 0L, HpcDataTransferUploadStatus.ARCHIVED,
-				null);
+				null, false);
 
 		// Assert expected result.
 		assertNull(downloadResponse.getDownloadTaskId());

src/hpc-server/hpc-bus-service-api/src/main/java/gov/nih/nci/hpc/bus/HpcDataManagementBusService.java

@@ -141,6 +141,17 @@ public HpcCollectionDTO getCollectionChildrenWithPaging(String path, Integer off
 	public HpcCollectionDownloadResponseDTO downloadCollection(String path, HpcDownloadRequestDTO downloadRequest)
 			throws HpcException;
 
+	/**
+	 * Download a collection tree.
+	 *
+	 * @param path            The collection path.
+	 * @param downloadRequest The download request DTO.
+	 * @return Download Response DTO.
+	 * @throws HpcException on service failure.
+	 */
+	public HpcCollectionDownloadResponseDTO downloadCollectionFromExternalSource(String path, HpcDownloadRequestDTO downloadRequest)
+			throws HpcException;
+
 	/**
 	 * Download data objects or collections. Note: API doesn't support mixed, so
 	 * user expected to provide a list of data objects or a list of collections, not
@@ -153,6 +164,18 @@ public HpcCollectionDownloadResponseDTO downloadCollection(String path, HpcDownl
 	public HpcBulkDataObjectDownloadResponseDTO downloadDataObjectsOrCollections(
 			HpcBulkDataObjectDownloadRequestDTO downloadRequest) throws HpcException;
 
+	/**
+	 * Download data objects or collections from external source. Note: API doesn't support mixed, so
+	 * user expected to provide a list of data objects or a list of collections, not
+	 * both.
+	 *
+	 * @param downloadRequest The download request DTO.
+	 * @return Download Response DTO.
+	 * @throws HpcException on service failure.
+	 */
+	public HpcBulkDataObjectDownloadResponseDTO downloadDataObjectsOrCollectionsFromExternalSource(
+			HpcBulkDataObjectDownloadRequestDTO downloadRequest) throws HpcException;
+
 	/**
 	 * Get collection download task status.
 	 *
@@ -455,6 +478,19 @@ public gov.nih.nci.hpc.dto.datamanagement.v2.HpcDataObjectDTO getDataObject(Stri
 	public HpcDataObjectDownloadResponseDTO downloadDataObject(String path, HpcDownloadRequestDTO downloadRequest)
 			throws HpcException;
 
+	/**
+	 * Download Data Object from External Source. In this overloaded method, the request invoker will be
+	 * notified (if subscribed) when the download is complete. To specify a
+	 * different user-id and turn off notification, use the other overloaded method.
+	 *
+	 * @param path            The data object path.
+	 * @param downloadRequest The download request DTO.
+	 * @return Download ResponseDTO
+	 * @throws HpcException on service failure.
+	 */
+	public HpcDataObjectDownloadResponseDTO downloadDataObjectFromExternalSource(String path, HpcDownloadRequestDTO downloadRequest)
+			throws HpcException;
+
 	/**
 	 * Download Data Object.
 	 *