checksums for upload path (#3625)

Author: pulimsr

src/aws-cpp-sdk-transfer/source/transfer/TransferManager.cpp

@@ -6,6 +6,11 @@
 #include <aws/core/platform/FileSystem.h>
 #include <aws/core/utils/HashingUtils.h>
 #include <aws/core/utils/logging/LogMacros.h>
+#include <aws/core/utils/crypto/Hash.h>
+#include <aws/core/utils/crypto/CRC32.h>
+#include <aws/core/utils/crypto/CRC64.h>
+#include <aws/core/utils/crypto/Sha1.h>
+#include <aws/core/utils/crypto/Sha256.h>
 #include <aws/core/utils/memory/AWSMemory.h>
 #include <aws/core/utils/memory/stl/AWSStreamFwd.h>
 #include <aws/core/utils/memory/stl/AWSStringStream.h>
@@ -399,6 +404,25 @@ namespace Aws
             bool isRetry = !handle->GetMultiPartId().empty();
             uint64_t sentBytes = 0;
 
+            const auto fullObjectHashCalculator = [](const std::shared_ptr<TransferHandle>& handle, bool isRetry, S3::Model::ChecksumAlgorithm algorithm) -> std::shared_ptr<Aws::Utils::Crypto::Hash> {
+                if (handle->GetChecksum().empty() && !isRetry) {
+                    if (algorithm == S3::Model::ChecksumAlgorithm::CRC32) {
+                        return Aws::MakeShared<Aws::Utils::Crypto::CRC32>("TransferManager");
+                    }
+                    if (algorithm == S3::Model::ChecksumAlgorithm::CRC32C) {
+                        return Aws::MakeShared<Aws::Utils::Crypto::CRC32C>("TransferManager");
+                    }
+                    if (algorithm == S3::Model::ChecksumAlgorithm::SHA1) {
+                        return Aws::MakeShared<Aws::Utils::Crypto::Sha1>("TransferManager");
+                    }
+                    if (algorithm == S3::Model::ChecksumAlgorithm::SHA256) {
+                        return Aws::MakeShared<Aws::Utils::Crypto::Sha256>("TransferManager");
+                    }
+                    return Aws::MakeShared<Aws::Utils::Crypto::CRC64>("TransferManager");
+                }
+                return nullptr;
+            }(handle, isRetry, m_transferConfig.checksumAlgorithm);
+
             if (!isRetry) {
               Aws::S3::Model::CreateMultipartUploadRequest createMultipartRequest = m_transferConfig.createMultipartUploadTemplate;
               createMultipartRequest.SetChecksumAlgorithm(m_transferConfig.checksumAlgorithm);
@@ -466,6 +490,10 @@ namespace Aws
                 streamToPut->seekg((partsIter->first - 1) * m_transferConfig.bufferSize);
                 streamToPut->read(reinterpret_cast<char*>(buffer), lengthToWrite);
 
+                if (fullObjectHashCalculator) {
+                    fullObjectHashCalculator->Update(buffer, static_cast<size_t>(lengthToWrite));
+                }
+
                 auto streamBuf = Aws::New<Aws::Utils::Stream::PreallocatedStreamBuf>(CLASS_TAG, buffer, static_cast<size_t>(lengthToWrite));
                 auto preallocatedStreamReader = Aws::MakeShared<Aws::IOStream>(CLASS_TAG, streamBuf);
 
@@ -525,6 +553,13 @@ namespace Aws
                 handle->UpdateStatus(DetermineIfFailedOrCanceled(*handle));
                 TriggerTransferStatusUpdatedCallback(handle);
             }
+            else if (fullObjectHashCalculator && handle->GetChecksum().empty()) {
+                // Finalize checksum calculation and set on handle
+                auto hashResult = fullObjectHashCalculator->GetHash();
+                if (hashResult.IsSuccess()) {
+                    handle->SetChecksum(Aws::Utils::HashingUtils::Base64Encode(hashResult.GetResult()));
+                }
+            }
         }
 
         void TransferManager::DoSinglePartUpload(const std::shared_ptr<TransferHandle>& handle)
@@ -1560,4 +1595,4 @@ namespace Aws
           }
         }
     }
-}
+}

tests/aws-cpp-sdk-transfer-tests/TransferTests.cpp

@@ -9,6 +9,7 @@
 #include <aws/s3/S3Client.h>
 #include <aws/s3/model/DeleteObjectsRequest.h>
 #include <aws/s3/model/AbortMultipartUploadRequest.h>
+#include <aws/s3/model/CompleteMultipartUploadRequest.h>
 #include <aws/s3/model/ListObjectsRequest.h>
 #include <aws/s3/model/ListMultipartUploadsRequest.h>
 #include <aws/s3/model/GetObjectRequest.h>
@@ -160,6 +161,21 @@ class MockS3Client : public S3Client
         return S3Client::ListObjectsV2(request);
     }
 
+    // Override to verify checksum is being sent
+    Model::CompleteMultipartUploadOutcome CompleteMultipartUpload(const Model::CompleteMultipartUploadRequest& request) const override
+    {
+        AWS_LOGSTREAM_INFO("TransferTests", "=== CompleteMultipartUpload Request ===");
+        AWS_LOGSTREAM_INFO("TransferTests", "Available ChecksumAlgorithm enum values:");
+        AWS_LOGSTREAM_INFO("TransferTests", "ChecksumType: " << (int)request.GetChecksumType());
+        AWS_LOGSTREAM_INFO("TransferTests", "ChecksumCRC32: " << request.GetChecksumCRC32());
+        AWS_LOGSTREAM_INFO("TransferTests", "ChecksumCRC32C: " << request.GetChecksumCRC32C());
+        AWS_LOGSTREAM_INFO("TransferTests", "ChecksumCRC64NVME: " << request.GetChecksumCRC64NVME());
+        AWS_LOGSTREAM_INFO("TransferTests", "ChecksumSHA1: " << request.GetChecksumSHA1());
+        AWS_LOGSTREAM_INFO("TransferTests", "ChecksumSHA256: " << request.GetChecksumSHA256());
+        AWS_LOGSTREAM_INFO("TransferTests", "=======================================");
+        return S3Client::CompleteMultipartUpload(request);
+    }
+
     // m_executor in Base class is private, we need our own one.
     std::shared_ptr<Aws::Utils::Threading::Executor> executor;