diff --git a/README.md b/README.md index a448c314c4c3..4b8a8b27149b 100644 --- a/README.md +++ b/README.md @@ -665,7 +665,6 @@ bindings to be included as a dependency with your application. This functionalit - [Amazon S3 Multi-Region Access Points](https://docs.aws.amazon.com/AmazonS3/latest/userguide/MultiRegionAccessPoints.html) - [Amazon S3 Object Integrity](https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html) - [Amazon CloudFront-KeyValueStore](https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/kvs-with-functions-kvp.html) -- [Amazon S3 CRC64-NVME checksums](https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html) If the required AWS Common Runtime components are not installed, you will receive an error like: @@ -682,6 +681,10 @@ https://github.com/aws/aws-sdk-js-v3#functionality-requiring-aws-common-runtime- indicating that the required dependency is missing to use the associated functionality. To install this dependency, follow the provided [instructions](#installing-the-aws-common-runtime-crt-dependency). +For [Amazon S3 CRC64-NVME checksums](https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html), +we include JavaScript implementation by default, but you can optionally install and include `"@aws-sdk/crc64-nvme-crt"` +on Node.js runtime for improved performance. + #### Installing the AWS Common Runtime (CRT) Dependency You can install the CRT dependency with different commands depending on the package management tool you are using. diff --git a/packages/middleware-flexible-checksums/src/middleware-flexible-checksums.crt.integ.spec.ts b/packages/middleware-flexible-checksums/src/middleware-flexible-checksums.crt.integ.spec.ts new file mode 100644 index 000000000000..7c7333307e8f --- /dev/null +++ b/packages/middleware-flexible-checksums/src/middleware-flexible-checksums.crt.integ.spec.ts @@ -0,0 +1,145 @@ +// Testing CRT implementation of CRC64NVME +import "@aws-sdk/crc64-nvme-crt"; + +import { requireRequestsFrom } from "@aws-sdk/aws-util-test/src"; +import { ChecksumAlgorithm, S3 } from "@aws-sdk/client-s3"; +import { HttpHandler, HttpRequest, HttpResponse } from "@smithy/protocol-http"; +import { Readable, Transform } from "stream"; +import { describe, expect, test as it } from "vitest"; + +import { RequestChecksumCalculation, ResponseChecksumValidation } from "./constants"; + +describe("middleware-flexible-checksums", () => { + const logger = { + trace() {}, + debug() {}, + info() {}, + warn() {}, + error() {}, + }; + + const testCases: [string, string][] = [ + ["", "AAAAAAAAAAA="], + ["abc", "BeXKuz/B+us="], + ["Hello world", "OOJZ0D8xKts="], + ]; + + describe(S3.name, () => { + describe("putObject", () => { + describe.each([undefined, RequestChecksumCalculation.WHEN_SUPPORTED, RequestChecksumCalculation.WHEN_REQUIRED])( + `when requestChecksumCalculation='%s'`, + (requestChecksumCalculation) => { + testCases.forEach(([body, checksumValue]) => { + const client = new S3({ region: "us-west-2", logger, requestChecksumCalculation }); + const checksumHeader = `x-amz-checksum-${ChecksumAlgorithm.CRC64NVME.toLowerCase()}`; + + it(`tests ${checksumHeader}="${checksumValue}"" for checksum="${ChecksumAlgorithm.CRC64NVME}"`, async () => { + requireRequestsFrom(client).toMatch({ + method: "PUT", + hostname: "s3.us-west-2.amazonaws.com", + protocol: "https:", + path: "/b/k", + headers: { + "content-type": "application/octet-stream", + ...(body.length + ? { + "content-length": body.length.toString(), + } + : {}), + "x-amz-sdk-checksum-algorithm": ChecksumAlgorithm.CRC64NVME, + [checksumHeader]: checksumValue, + host: "s3.us-west-2.amazonaws.com", + "x-amz-user-agent": /./, + "user-agent": /./, + "amz-sdk-invocation-id": /./, + "amz-sdk-request": /./, + "x-amz-date": /./, + "x-amz-content-sha256": /./, + authorization: /./, + }, + query: { + "x-id": "PutObject", + }, + }); + + await client.putObject({ + Bucket: "b", + Key: "k", + Body: body, + ChecksumAlgorithm: ChecksumAlgorithm.CRC64NVME, + }); + + expect.hasAssertions(); + }); + }); + } + ); + }); + + describe("getObject", () => { + describe.each([undefined, ResponseChecksumValidation.WHEN_SUPPORTED, ResponseChecksumValidation.WHEN_REQUIRED])( + `when responseChecksumValidation='%s'`, + (responseChecksumValidation) => { + testCases.forEach(([body, checksumValue]) => { + const checksumHeader = `x-amz-checksum-${ChecksumAlgorithm.CRC64NVME.toLowerCase()}`; + + it(`validates ${checksumHeader}="${checksumValue}"" for checksum="${ChecksumAlgorithm.CRC64NVME}"`, async () => { + const client = new S3({ + region: "us-west-2", + logger, + requestHandler: new (class implements HttpHandler { + async handle(request: HttpRequest): Promise { + expect(request).toMatchObject({ + method: "GET", + hostname: "s3.us-west-2.amazonaws.com", + protocol: "https:", + path: "/b/k", + headers: { + "x-amz-checksum-mode": "ENABLED", + host: "s3.us-west-2.amazonaws.com", + "x-amz-user-agent": /./, + "user-agent": /./, + "amz-sdk-invocation-id": /./, + "amz-sdk-request": /./, + "x-amz-date": /./, + "x-amz-content-sha256": /./, + authorization: /./, + }, + query: { + "x-id": "GetObject", + }, + }); + return { + response: new HttpResponse({ + statusCode: 200, + headers: { + "content-type": "application/octet-stream", + "content-length": body.length.toString(), + [checksumHeader]: checksumValue, + }, + body: Readable.from([body]), + }), + }; + } + updateHttpClientConfig(key: never, value: never): void {} + httpHandlerConfigs() { + return {}; + } + })(), + responseChecksumValidation, + }); + + const response = await client.getObject({ + Bucket: "b", + Key: "k", + ChecksumMode: "ENABLED", + }); + + await expect(response.Body?.transformToString()).resolves.toEqual(body); + }); + }); + } + ); + }); + }); +}); diff --git a/packages/middleware-flexible-checksums/src/middleware-flexible-checksums.e2e.spec.ts b/packages/middleware-flexible-checksums/src/middleware-flexible-checksums.e2e.spec.ts index 9e52d72e225a..d87faa92458c 100644 --- a/packages/middleware-flexible-checksums/src/middleware-flexible-checksums.e2e.spec.ts +++ b/packages/middleware-flexible-checksums/src/middleware-flexible-checksums.e2e.spec.ts @@ -1,5 +1,5 @@ import { getE2eTestResources } from "@aws-sdk/aws-util-test/src"; -import { S3, UploadPartCommandOutput } from "@aws-sdk/client-s3"; +import { ChecksumAlgorithm, S3, UploadPartCommandOutput } from "@aws-sdk/client-s3"; import { Upload } from "@aws-sdk/lib-storage"; import { FetchHttpHandler } from "@smithy/fetch-http-handler"; import type { HttpRequest, HttpResponse } from "@smithy/types"; @@ -108,6 +108,7 @@ describe("S3 checksums", () => { }); expect.hasAssertions(); }); + it("should assist user input streams by buffering to the minimum 8kb required by S3", async () => { await s3.putObject({ Bucket, @@ -124,6 +125,7 @@ describe("S3 checksums", () => { }); expect((await get.Body?.transformToByteArray())?.byteLength).toEqual(24 * 1024); }); + it("should be able to write an object with a webstream body (using fetch handler without checksum)", async () => { const handler = s3_noChecksum.config.requestHandler; s3_noChecksum.config.requestHandler = new FetchHttpHandler(); @@ -140,6 +142,7 @@ describe("S3 checksums", () => { }); expect((await get.Body?.transformToByteArray())?.byteLength).toEqual(24 * 1024); }); + it("@aws-sdk/lib-storage Upload should allow webstreams to be used", async () => { await new Upload({ client: s3, @@ -155,6 +158,7 @@ describe("S3 checksums", () => { }); expect((await get.Body?.transformToByteArray())?.byteLength).toEqual(6 * 1024 * 1024); }); + it("should allow streams to be used in a manually orchestrated MPU", async () => { const cmpu = await s3.createMultipartUpload({ Bucket, @@ -252,4 +256,40 @@ describe("S3 checksums", () => { await expect(checksumStream.getReader().closed).resolves.toBe(undefined); }); }); + + describe("Checksum computation", () => { + it.each([ + ["Hello world", ChecksumAlgorithm.CRC32, "i9aeUg=="], + ["Hello world", ChecksumAlgorithm.CRC32C, "crUfeA=="], + ["Hello world", ChecksumAlgorithm.CRC64NVME, "OOJZ0D8xKts="], + ["Hello world", ChecksumAlgorithm.SHA1, "e1AsOh9IyGCa4hLN+2Od7jlnP14="], + ["Hello world", ChecksumAlgorithm.SHA256, "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw="], + ])( + "when body is '%s' with checksum algorithm '%s' the checksum is '%s'", + async (body, checksumAlgorithm, checksumValue) => { + const client = new S3(); + client.middlewareStack.addRelativeTo( + (next: any) => async (args: any) => { + const request = args.request as HttpRequest; + expect(request.headers["x-amz-sdk-checksum-algorithm"]).toEqual(checksumAlgorithm); + expect(request.headers[`x-amz-checksum-${checksumAlgorithm.toLowerCase()}`]).toEqual(checksumValue); + return next(args); + }, + { + relation: "after", + toMiddleware: "flexibleChecksumsMiddleware", + } + ); + + await client.putObject({ + Bucket, + Key: Key + "-checksum-" + checksumAlgorithm, + Body: body, + ChecksumAlgorithm: checksumAlgorithm, + }); + + expect.assertions(2); + } + ); + }); }, 60_000); diff --git a/packages/middleware-flexible-checksums/src/middleware-flexible-checksums.integ.spec.ts b/packages/middleware-flexible-checksums/src/middleware-flexible-checksums.integ.spec.ts index 50d5f73226a8..f59ec852c56c 100644 --- a/packages/middleware-flexible-checksums/src/middleware-flexible-checksums.integ.spec.ts +++ b/packages/middleware-flexible-checksums/src/middleware-flexible-checksums.integ.spec.ts @@ -1,6 +1,3 @@ -// Required for testing CRC64NVME -import "@aws-sdk/crc64-nvme-crt"; - import { requireRequestsFrom } from "@aws-sdk/aws-util-test/src"; import { ChecksumAlgorithm, S3 } from "@aws-sdk/client-s3"; import { HttpHandler, HttpRequest, HttpResponse } from "@smithy/protocol-http"; diff --git a/packages/middleware-flexible-checksums/src/selectChecksumAlgorithmFunction.spec.ts b/packages/middleware-flexible-checksums/src/selectChecksumAlgorithmFunction.spec.ts index b181cba9d601..aa639a67874e 100644 --- a/packages/middleware-flexible-checksums/src/selectChecksumAlgorithmFunction.spec.ts +++ b/packages/middleware-flexible-checksums/src/selectChecksumAlgorithmFunction.spec.ts @@ -1,5 +1,5 @@ import { AwsCrc32c } from "@aws-crypto/crc32c"; -import { crc64NvmeCrtContainer } from "@aws-sdk/crc64-nvme"; +import { Crc64Nvme, crc64NvmeCrtContainer } from "@aws-sdk/crc64-nvme"; import { describe, expect, test as it, vi } from "vitest"; import { ChecksumAlgorithm } from "./constants"; @@ -27,8 +27,9 @@ describe(selectChecksumAlgorithmFunction.name, () => { expect(() => selectChecksumAlgorithmFunction("UNSUPPORTED" as any, mockConfig as any)).toThrow(); }); - it("throws error if crc64NvmeCrtContainer.CrtCrc64Nvme is not a function", () => { - expect(() => selectChecksumAlgorithmFunction(ChecksumAlgorithm.CRC64NVME, mockConfig as any)).toThrow(); + it("returns Crc64Nvme if crc64NvmeCrtContainer.CrtCrc64Nvme is not a function", () => { + crc64NvmeCrtContainer.CrtCrc64Nvme = null; + expect(selectChecksumAlgorithmFunction(ChecksumAlgorithm.CRC64NVME, mockConfig as any)).toBe(Crc64Nvme); }); it("returns crc64NvmeCrtContainer.CrtCrc64Nvme if available from container", () => { diff --git a/packages/middleware-flexible-checksums/src/selectChecksumAlgorithmFunction.ts b/packages/middleware-flexible-checksums/src/selectChecksumAlgorithmFunction.ts index 6762cc663a5a..5ceb445f2d20 100644 --- a/packages/middleware-flexible-checksums/src/selectChecksumAlgorithmFunction.ts +++ b/packages/middleware-flexible-checksums/src/selectChecksumAlgorithmFunction.ts @@ -1,5 +1,5 @@ import { AwsCrc32c } from "@aws-crypto/crc32c"; -import { crc64NvmeCrtContainer } from "@aws-sdk/crc64-nvme"; +import { Crc64Nvme, crc64NvmeCrtContainer } from "@aws-sdk/crc64-nvme"; import { ChecksumConstructor, HashConstructor } from "@smithy/types"; import { PreviouslyResolved } from "./configuration"; @@ -22,13 +22,7 @@ export const selectChecksumAlgorithmFunction = ( return AwsCrc32c; case ChecksumAlgorithm.CRC64NVME: if (typeof crc64NvmeCrtContainer.CrtCrc64Nvme !== "function") { - throw new Error( - `Please check whether you have installed the "@aws-sdk/crc64-nvme-crt" package explicitly. \n` + - `You must also register the package by calling [require("@aws-sdk/crc64-nvme-crt");] ` + - `or an ESM equivalent such as [import "@aws-sdk/crc64-nvme-crt";]. \n` + - "For more information please go to " + - "https://github.com/aws/aws-sdk-js-v3#functionality-requiring-aws-common-runtime-crt" - ); + return Crc64Nvme; } return crc64NvmeCrtContainer.CrtCrc64Nvme; case ChecksumAlgorithm.SHA1: