HPCDATAMGM-2165: Create a task to populate system or user metadata

src/hpc-server/hpc-app-service-api/src/main/java/gov/nih/nci/hpc/service/HpcDataMigrationService.java

@@ -16,6 +16,7 @@
 import gov.nih.nci.hpc.domain.model.HpcDataMigrationTask;
 import gov.nih.nci.hpc.domain.model.HpcDataMigrationTaskResult;
 import gov.nih.nci.hpc.domain.model.HpcDataMigrationTaskStatus;
+import gov.nih.nci.hpc.domain.model.HpcStagedMetadataAttribute;
 import gov.nih.nci.hpc.exception.HpcException;
 
 /**
@@ -286,4 +287,43 @@ public HpcDataMigrationTask createMetadataMigrationTask(String fromS3ArchiveConf
 			String toS3ArchiveConfigurationId, String fromArchiveFileContainerId, String toArchiveFileContainerId,
 			String archiveFileIdPattern, String userId) throws HpcException;
 
+	/**
+	 * Get a list of staged metadata attributes to be processed.
+	 *
+	 * @return A List of staged metadata entries.
+	 * @throws HpcException on service failure.
+	 */
+	public List<HpcStagedMetadataAttribute> getStagedMetadataAttributes() throws HpcException;
+
+
+	/**
+	 * Add staged metadata attribute to path.
+	 * 
+	 * @param stagedMetadataAttribute staged metadata entry to add
+	 * @param isCollection true if path is a collection path
+	 * @param configurationId The configuration ID.
+	 * @param collectionType The collection type.
+	 * @throws HpcException on service failure.
+	 */
+	public void addStagedMetadataAttribute(HpcStagedMetadataAttribute stagedMetadataAttribute, boolean isCollection,
+			String configurationId, String collectionType) throws HpcException;
+
+	/**
+	 * Claim a staged metadata attribute for processing, so that in a multi-node
+	 * deployment only one server processes each row.
+	 *
+	 * @param stagedMetadataAttribute staged metadata entry to claim
+	 * @return true if successfully claimed, false if another node already claimed it.
+	 * @throws HpcException on service failure.
+	 */
+	public boolean claimStagedMetadataAttribute(HpcStagedMetadataAttribute stagedMetadataAttribute)
+			throws HpcException;
+
+	/**
+	 * Clean processed staged metadata attribute.
+	 * 
+	 * @param stagedMetadataAttribute staged metadata entry to clean up
+	 * @throws HpcException on service failure.
+	 */
+	public void cleanupStagedMetadataAttribute(HpcStagedMetadataAttribute stagedMetadataAttribute) throws HpcException;
 }

src/hpc-server/hpc-app-service-api/src/main/java/gov/nih/nci/hpc/service/HpcMetadataService.java

@@ -55,9 +55,10 @@ public void addMetadataToCollection(String path, List<HpcMetadataEntry> metadata
 	 * @param metadataEntries The metadata entries to update.
 	 * @param configurationId The configuration to apply validation rules. Metadata
 	 *                        validation rules are configuration specific.
+	 * @param allowSystemMetadata True if system metadata update is allowed.
 	 * @throws HpcException on service failure.
 	 */
-	public void updateCollectionMetadata(String path, List<HpcMetadataEntry> metadataEntries, String configurationId)
+	public void updateCollectionMetadata(String path, List<HpcMetadataEntry> metadataEntries, String configurationId, boolean allowSystemMetadata)
 			throws HpcException;
 
 	/**
@@ -310,10 +311,11 @@ public void updateDataObjectSystemGeneratedMetadata(String path, HpcFileLocation
 	 * @param extractedMetadata True if the data object is updated w/ extracted
 	 *                          metadata (this is optionally performed during data
 	 *                          object registration)
+	 * @param allowSystemMetadata True if system metadata update is allowed.
 	 * @throws HpcException on service failure.
 	 */
 	public void updateDataObjectMetadata(String path, List<HpcMetadataEntry> metadataEntries, String configurationId,
-			String collectionType, boolean extractedMetadata) throws HpcException;
+			String collectionType, boolean extractedMetadata, boolean allowSystemMetadata) throws HpcException;
 
 	/**
 	 * Get metadata of a data object.

src/hpc-server/hpc-app-service-impl/src/main/java/gov/nih/nci/hpc/service/impl/HpcDataMigrationServiceImpl.java

@@ -8,6 +8,7 @@
  */
 package gov.nih.nci.hpc.service.impl;
 
+import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.Iterator;
 import java.util.List;
@@ -24,6 +25,7 @@
 import com.google.common.collect.Iterables;
 
 import gov.nih.nci.hpc.dao.HpcDataMigrationDAO;
+import gov.nih.nci.hpc.domain.datamanagement.HpcAuditRequestType;
 import gov.nih.nci.hpc.domain.datamigration.HpcDataMigrationResult;
 import gov.nih.nci.hpc.domain.datamigration.HpcDataMigrationStatus;
 import gov.nih.nci.hpc.domain.datamigration.HpcDataMigrationType;
@@ -32,15 +34,19 @@
 import gov.nih.nci.hpc.domain.datatransfer.HpcDeepArchiveStatus;
 import gov.nih.nci.hpc.domain.datatransfer.HpcStreamingUploadSource;
 import gov.nih.nci.hpc.domain.error.HpcErrorType;
+import gov.nih.nci.hpc.domain.metadata.HpcMetadataEntries;
+import gov.nih.nci.hpc.domain.metadata.HpcMetadataEntry;
 import gov.nih.nci.hpc.domain.model.HpcDataMigrationTask;
 import gov.nih.nci.hpc.domain.model.HpcDataMigrationTaskResult;
 import gov.nih.nci.hpc.domain.model.HpcDataMigrationTaskStatus;
 import gov.nih.nci.hpc.domain.model.HpcDataObjectUploadRequest;
 import gov.nih.nci.hpc.domain.model.HpcDataTransferConfiguration;
+import gov.nih.nci.hpc.domain.model.HpcStagedMetadataAttribute;
 import gov.nih.nci.hpc.domain.model.HpcSystemGeneratedMetadata;
 import gov.nih.nci.hpc.domain.user.HpcIntegratedSystemAccount;
 import gov.nih.nci.hpc.exception.HpcException;
 import gov.nih.nci.hpc.integration.HpcDataTransferProxy;
+import gov.nih.nci.hpc.service.HpcDataManagementService;
 import gov.nih.nci.hpc.service.HpcDataMigrationService;
 import gov.nih.nci.hpc.service.HpcDataTransferService;
 import gov.nih.nci.hpc.service.HpcMetadataService;
@@ -85,6 +91,10 @@ public class HpcDataMigrationServiceImpl implements HpcDataMigrationService {
 	@Autowired
 	private HpcSecurityService securityService = null;
 
+	// The Data Management Application Service Instance.
+	@Autowired
+	private HpcDataManagementService dataManagementService = null;
+
 	// A configured ID representing the server performing a migration task.
 	@Value("${hpc.service.serverId}")
 	private String serverId = null;
@@ -441,7 +451,9 @@ public void resetMigrationTasksInProcess() throws HpcException {
 
 		int taskCount = dataMigrationDAO.cleanupDataMigrationTasks();
 		logger.info("Cleaned up {} data migration tasks", taskCount);
-
+
+		// Reset any in-process metadata attribute migration
+		dataMigrationDAO.resetStagedMetadataAttribute();
 	}
 
 	@Override
@@ -632,4 +644,87 @@ public HpcDataMigrationTask createMetadataMigrationTask(String fromS3ArchiveConf
 		dataMigrationDAO.upsertDataMigrationTask(migrationTask);
 		return migrationTask;
 	}
+
+	@Override
+	public List<HpcStagedMetadataAttribute> getStagedMetadataAttributes()
+			throws HpcException {
+
+		List<HpcStagedMetadataAttribute> stagedMetadataEntries = dataMigrationDAO.getStagedMetadataAttributes();
+		logger.info("{} staged metadata attributes retrieved.", stagedMetadataEntries.size());
+
+		return stagedMetadataEntries;
+	}
+
+	@Override
+	public void addStagedMetadataAttribute(HpcStagedMetadataAttribute stagedMetadataAttribute, boolean isCollection,
+			String configurationId, String collectionType) throws HpcException {
+
+		// Construct metadata entry to add.
+		List<HpcMetadataEntry> metadataEntries = new ArrayList<>();
+		HpcMetadataEntry entry = new HpcMetadataEntry();
+		entry.setAttribute(stagedMetadataAttribute.getAttribute());
+		entry.setValue(stagedMetadataAttribute.getValue());
+		metadataEntries.add(entry);
+
+		// Call addMetadata method to add the staged metadata entry to the relevant data object or collection
+		// with allowSystemMetadata set to true to allow adding metadata with reserved attributes
+		if (isCollection) {
+			// Get the metadata for this collection.
+			HpcMetadataEntries metadataBefore = metadataService.getCollectionMetadataEntries(stagedMetadataAttribute.getPath());
+
+			// Update the metadata.
+			boolean updated = true;
+			String message = null;
+			try {
+				metadataService.updateCollectionMetadata(stagedMetadataAttribute.getPath(), metadataEntries,
+						configurationId, true);
+			} catch (HpcException e) {
+				// Collection metadata update failed. Capture this in the audit record.
+				updated = false;
+				message = e.getMessage();
+			} finally {
+				// Add an audit record of this update collection attempt.
+				dataManagementService.addAuditRecord(stagedMetadataAttribute.getPath(), HpcAuditRequestType.UPDATE_COLLECTION, metadataBefore,
+						metadataService.getCollectionMetadataEntries(stagedMetadataAttribute.getPath()), null, updated, null, message, "add-staged-metadata-task", null,
+						null);
+			}
+
+		} else {
+			// Get the metadata for this data object.
+			HpcMetadataEntries metadataBefore = metadataService.getDataObjectMetadataEntries(stagedMetadataAttribute.getPath(), true);
+
+			// Update the metadata.
+			boolean updated = true;
+			String message = null;
+			try {
+				metadataService.updateDataObjectMetadata(stagedMetadataAttribute.getPath(), metadataEntries,
+						configurationId, collectionType, false, true);
+			} catch (HpcException e) {
+				// Data object metadata update failed. Capture this in the audit record.
+				updated = false;
+				message = e.getMessage();
+			} finally {
+				// Add an audit record of this update collection attempt.
+				dataManagementService.addAuditRecord(stagedMetadataAttribute.getPath(), HpcAuditRequestType.UPDATE_DATA_OBJECT, metadataBefore,
+						metadataService.getDataObjectMetadataEntries(stagedMetadataAttribute.getPath(), true), null, updated, null, message, "add-staged-metadata-task", null,
+						null);
+			}
+		}
+
+	}
+
+	@Override
+	public boolean claimStagedMetadataAttribute(HpcStagedMetadataAttribute stagedMetadataAttribute)
+			throws HpcException {
+
+		return dataMigrationDAO.claimStagedMetadataAttribute(stagedMetadataAttribute);
+	}
+
+	@Override
+	public void cleanupStagedMetadataAttribute(HpcStagedMetadataAttribute stagedMetadataAttribute) throws HpcException {
+
+		int count = dataMigrationDAO.cleanupStagedMetadataAttribute(stagedMetadataAttribute);
+		logger.info("{} staged metadata attribute cleaned up.", count);
+
+	}
 }

src/hpc-server/hpc-app-service-impl/src/main/java/gov/nih/nci/hpc/service/impl/HpcMetadataServiceImpl.java

@@ -205,15 +205,15 @@ public void addMetadataToCollection(String path, List<HpcMetadataEntry> metadata
 		}
 
 		// Validate Metadata.
-		metadataValidator.validateCollectionMetadata(configurationId, null, metadataEntries);
+		metadataValidator.validateCollectionMetadata(configurationId, null, metadataEntries, false);
 
 		// Add Metadata to the DM system.
 		dataManagementProxy.addMetadataToCollection(dataManagementAuthenticator.getAuthenticatedToken(), path,
 				metadataEntries);
 	}
 
 	@Override
-	public void updateCollectionMetadata(String path, List<HpcMetadataEntry> metadataEntries, String configurationId)
+	public void updateCollectionMetadata(String path, List<HpcMetadataEntry> metadataEntries, String configurationId, boolean allowSystemMetadata)
 			throws HpcException {
 		// Input validation.
 		if (path == null) {
@@ -234,7 +234,7 @@ public void updateCollectionMetadata(String path, List<HpcMetadataEntry> metadat
 		validateCollectionTypeUpdate(existingMetadataEntries, metadataEntries);
 
 		// Validate the metadata.
-		metadataValidator.validateCollectionMetadata(configurationId, existingMetadataEntries, metadataEntries);
+		metadataValidator.validateCollectionMetadata(configurationId, existingMetadataEntries, metadataEntries, allowSystemMetadata);
 
 		// Update the 'metadata updated' system-metadata to record the time of this
 		// metadata update.
@@ -541,7 +541,7 @@ public HpcMetadataEntry addMetadataToDataObject(String path, List<HpcMetadataEnt
 		}
 
 		// Validate Metadata.
-		metadataValidator.validateDataObjectMetadata(configurationId, null, metadataEntries, collectionType);
+		metadataValidator.validateDataObjectMetadata(configurationId, null, metadataEntries, collectionType, false);
 
 		// Add Metadata to the DM system.
 		dataManagementProxy.addMetadataToDataObject(dataManagementAuthenticator.getAuthenticatedToken(), path,
@@ -608,7 +608,7 @@ public void addExtractedMetadataToDataObject(String path, List<HpcMetadataEntry>
 			String configurationId, String collectionType) throws HpcException {
 		// Update the data object's metadata.
 		updateDataObjectMetadata(path, new ArrayList<HpcMetadataEntry>(extractedMetadataEntries), configurationId,
-				collectionType, true);
+				collectionType, true, false);
 
 		// Set the extracted-metadata-attributes system generated metadata to have all
 		// the attributes
@@ -884,7 +884,7 @@ public void updateDataObjectSystemGeneratedMetadata(String path, HpcFileLocation
 
 	@Override
 	public void updateDataObjectMetadata(String path, List<HpcMetadataEntry> metadataEntries, String configurationId,
-			String collectionType, boolean extractedMetadata) throws HpcException {
+			String collectionType, boolean extractedMetadata, boolean allowSystemMetadata) throws HpcException {
 		// Input validation.
 		if (path == null) {
 			throw new HpcException(INVALID_PATH_MSG, HpcErrorType.INVALID_REQUEST_INPUT);
@@ -901,7 +901,7 @@ public void updateDataObjectMetadata(String path, List<HpcMetadataEntry> metadat
 
 		// Validate the metadata.
 		metadataValidator.validateDataObjectMetadata(configurationId, metadataRetriever.getDataObjectMetadata(path),
-				metadataEntries, collectionType);
+				metadataEntries, collectionType, allowSystemMetadata);
 
 		// Update the 'metadata updated' system-metadata to record the time of this
 		// metadata update. This is skipped for updated the data object w/ extracted

src/hpc-server/hpc-app-service-impl/src/main/java/gov/nih/nci/hpc/service/impl/HpcMetadataValidator.java

@@ -156,7 +156,7 @@ private HpcMetadataValidator() {
 	 * @throws HpcException If the metadata is invalid.
 	 */
 	public void validateCollectionMetadata(String configurationId, List<HpcMetadataEntry> existingMetadataEntries,
-			List<HpcMetadataEntry> addUpdateMetadataEntries) throws HpcException {
+			List<HpcMetadataEntry> addUpdateMetadataEntries, boolean allowSystemMetadata) throws HpcException {
 		HpcDataManagementConfiguration dataManagementConfiguration = dataManagementConfigurationLocator
 				.get(configurationId);
 		if (dataManagementConfiguration == null) {
@@ -165,7 +165,7 @@ public void validateCollectionMetadata(String configurationId, List<HpcMetadataE
 
 		validateMetadata(existingMetadataEntries, addUpdateMetadataEntries,
 				dataManagementConfiguration.getCollectionMetadataValidationRules(), null,
-				dataManagementConfiguration.getRestrictMetadata());
+				dataManagementConfiguration.getRestrictMetadata(), allowSystemMetadata);
 	}
 
 	/**
@@ -185,7 +185,7 @@ public void validateCollectionMetadata(String configurationId, List<HpcMetadataE
 	 * @throws HpcException If the metadata is invalid.
 	 */
 	public void validateDataObjectMetadata(String configurationId, List<HpcMetadataEntry> existingMetadataEntries,
-			List<HpcMetadataEntry> addUpdateMetadataEntries, String collectionType) throws HpcException {
+			List<HpcMetadataEntry> addUpdateMetadataEntries, String collectionType, boolean allowSystemMetadata) throws HpcException {
 		HpcDataManagementConfiguration dataManagementConfiguration = dataManagementConfigurationLocator
 				.get(configurationId);
 		if (dataManagementConfiguration == null) {
@@ -194,7 +194,8 @@ public void validateDataObjectMetadata(String configurationId, List<HpcMetadataE
 
 		validateMetadata(existingMetadataEntries, addUpdateMetadataEntries,
 				dataManagementConfiguration.getDataObjectMetadataValidationRules(), collectionType,
-				dataManagementConfiguration.getRestrictMetadata());
+				dataManagementConfiguration.getRestrictMetadata(),
+				allowSystemMetadata);
 	}
 
 	/**
@@ -245,7 +246,7 @@ public List<String> getDataObjectSystemGeneratedMetadataAttributeNames() {
 	 */
 	private void validateMetadata(List<HpcMetadataEntry> existingMetadataEntries,
 			List<HpcMetadataEntry> addUpdateMetadataEntries, List<HpcMetadataValidationRule> metadataValidationRules,
-			String collectionType, Boolean restrictMetadata) throws HpcException {
+			String collectionType, Boolean restrictMetadata, boolean allowSystemMetadata) throws HpcException {
 		// Crate a metadata <attribute, value> map. Put existing entries first.
 		Map<String, String> metadataEntriesMap = new HashMap<>();
 		if (existingMetadataEntries != null) {
@@ -298,7 +299,8 @@ private void validateMetadata(List<HpcMetadataEntry> existingMetadataEntries,
 			// If the restrict_metadata flag is set in the database, then ensure
 			// that the add/update metadata entry defined in the validation rules
 			// i.e. it is a mandatory or optional metadata for the applicable DOC
-			if (restrictMetadata) {
+			if (restrictMetadata && !(allowSystemMetadata
+					&& systemGeneratedMetadataAttributes.contains(metadataEntry.getAttribute()))) {
 				// Do this check only for new files or collections so that fixing
 				// of existing ones is not necessitated
 				if (CollectionUtils.isNullOrEmpty(existingMetadataEntries)) {
@@ -318,12 +320,15 @@ private void validateMetadata(List<HpcMetadataEntry> existingMetadataEntries,
 			}
 		}
 
-		// Validate the add/update metadata entries don't include reserved system
-		// generated metadata.
-		for (String metadataAttribue : systemGeneratedMetadataAttributes) {
-			if (addUpdateMetadataEntriesMap.containsKey(metadataAttribue)) {
-				throw new HpcException("System generated metadata cannot be set/changed: " + metadataAttribue,
-						HpcErrorType.INVALID_REQUEST_INPUT);
+		// If the allowSystemMetadata flag is not set, validate that add/update metadata entries don't include reserved system generated metadata.
+		if (!allowSystemMetadata) {
+			// Validate the add/update metadata entries don't include reserved system
+			// generated metadata.
+			for (String metadataAttribue : systemGeneratedMetadataAttributes) {
+				if (addUpdateMetadataEntriesMap.containsKey(metadataAttribue)) {
+					throw new HpcException("System generated metadata cannot be set/changed: " + metadataAttribue,
+							HpcErrorType.INVALID_REQUEST_INPUT);
+				}
 			}
 		}
 

src/hpc-server/hpc-bus-service-api/src/main/java/gov/nih/nci/hpc/bus/HpcDataMigrationBusService.java

@@ -172,4 +172,12 @@ public HpcMigrationResponseDTO migrateMetadata(HpcMetadataMigrationRequestDTO me
 	 * @throws HpcException on service failure.
 	 */
 	public void restartDataMigrationTasks() throws HpcException;
+
+	/**
+	 * Process staged metadata attributes.
+	 *
+	 * @throws HpcException on service failure.
+	 */
+	public void processStagedMetadataAttributes() throws HpcException;
+
 }

src/hpc-server/hpc-bus-service-impl/src/main/java/gov/nih/nci/hpc/bus/impl/HpcDataManagementBusServiceImpl.java

@@ -3676,7 +3676,7 @@ private void updateCollection(String path, List<HpcMetadataEntry> metadataEntrie
 			if (!metadataContained(metadataEntries, metadataBefore.getSelfMetadataEntries())) {
 				synchronized (this) {
 					metadataService.updateCollectionMetadata(path, metadataEntries,
-							systemGeneratedMetadata.getConfigurationId());
+							systemGeneratedMetadata.getConfigurationId(), false);
 				}
 			} else {
 				logger.info(
@@ -3740,7 +3740,7 @@ private HpcDataObjectUploadResponse updateDataObject(String path, List<HpcMetada
 		String message = null;
 		try {
 			metadataService.updateDataObjectMetadata(path, metadataEntries,
-					systemGeneratedMetadata.getConfigurationId(), collectionType, false);
+					systemGeneratedMetadata.getConfigurationId(), collectionType, false, false);
 
 		} catch (HpcException e) {
 			// Data object metadata update failed. Capture this in the audit record.