From 6ae6ccddd6667cb650e83e60b3e74442276057c3 Mon Sep 17 00:00:00 2001 From: Trivikram Kamat <16024985+trivikr@users.noreply.github.com> Date: Mon, 22 Dec 2025 04:12:21 +0000 Subject: [PATCH 1/6] chore: move crt dependency to peerDependencies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit yarn build is throwing error in current setup × Invalid package dependency graph: ╰─▶ Cyclic dependency detected: @aws-sdk/crc64-nvme, @aws-sdk/middleware-flexible-checksums, @aws-sdk/crc64-nvme-crt The cycle can be broken by removing any of these sets of dependencies: { @aws-sdk/middleware-flexible-checksums -> @aws-sdk/crc64-nvme } { @aws-sdk/crc64-nvme -> @aws-sdk/crc64-nvme-crt } { @aws-sdk/crc64-nvme-crt -> @aws-sdk/middleware-flexible-checksums } --- packages/crc64-nvme/package.json | 3 +++ yarn.lock | 2 ++ 2 files changed, 5 insertions(+) diff --git a/packages/crc64-nvme/package.json b/packages/crc64-nvme/package.json index 63280695ce1a..14607961bcbc 100644 --- a/packages/crc64-nvme/package.json +++ b/packages/crc64-nvme/package.json @@ -35,6 +35,9 @@ "rimraf": "3.0.2", "typescript": "~5.8.3" }, + "peerDependencies": { + "@aws-sdk/crc64-nvme-crt": "*" + }, "engines": { "node": ">=18.0.0" }, diff --git a/yarn.lock b/yarn.lock index 0ba411438896..43ec3531661b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -23451,6 +23451,8 @@ __metadata: rimraf: "npm:3.0.2" tslib: "npm:^2.6.2" typescript: "npm:~5.8.3" + peerDependencies: + "@aws-sdk/crc64-nvme-crt": "*" languageName: unknown linkType: soft From 452cda4dae6a78a5e11c7cce6dfe1f30e27ed4db Mon Sep 17 00:00:00 2001 From: Trivikram Kamat <16024985+trivikr@users.noreply.github.com> Date: Mon, 22 Dec 2025 04:48:45 +0000 Subject: [PATCH 2/6] chore: use CRC64NVME JS implementation if CRT is not available --- .../src/selectChecksumAlgorithmFunction.spec.ts | 7 ++++--- .../src/selectChecksumAlgorithmFunction.ts | 10 ++-------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/packages/middleware-flexible-checksums/src/selectChecksumAlgorithmFunction.spec.ts b/packages/middleware-flexible-checksums/src/selectChecksumAlgorithmFunction.spec.ts index b181cba9d601..aa639a67874e 100644 --- a/packages/middleware-flexible-checksums/src/selectChecksumAlgorithmFunction.spec.ts +++ b/packages/middleware-flexible-checksums/src/selectChecksumAlgorithmFunction.spec.ts @@ -1,5 +1,5 @@ import { AwsCrc32c } from "@aws-crypto/crc32c"; -import { crc64NvmeCrtContainer } from "@aws-sdk/crc64-nvme"; +import { Crc64Nvme, crc64NvmeCrtContainer } from "@aws-sdk/crc64-nvme"; import { describe, expect, test as it, vi } from "vitest"; import { ChecksumAlgorithm } from "./constants"; @@ -27,8 +27,9 @@ describe(selectChecksumAlgorithmFunction.name, () => { expect(() => selectChecksumAlgorithmFunction("UNSUPPORTED" as any, mockConfig as any)).toThrow(); }); - it("throws error if crc64NvmeCrtContainer.CrtCrc64Nvme is not a function", () => { - expect(() => selectChecksumAlgorithmFunction(ChecksumAlgorithm.CRC64NVME, mockConfig as any)).toThrow(); + it("returns Crc64Nvme if crc64NvmeCrtContainer.CrtCrc64Nvme is not a function", () => { + crc64NvmeCrtContainer.CrtCrc64Nvme = null; + expect(selectChecksumAlgorithmFunction(ChecksumAlgorithm.CRC64NVME, mockConfig as any)).toBe(Crc64Nvme); }); it("returns crc64NvmeCrtContainer.CrtCrc64Nvme if available from container", () => { diff --git a/packages/middleware-flexible-checksums/src/selectChecksumAlgorithmFunction.ts b/packages/middleware-flexible-checksums/src/selectChecksumAlgorithmFunction.ts index 6762cc663a5a..5ceb445f2d20 100644 --- a/packages/middleware-flexible-checksums/src/selectChecksumAlgorithmFunction.ts +++ b/packages/middleware-flexible-checksums/src/selectChecksumAlgorithmFunction.ts @@ -1,5 +1,5 @@ import { AwsCrc32c } from "@aws-crypto/crc32c"; -import { crc64NvmeCrtContainer } from "@aws-sdk/crc64-nvme"; +import { Crc64Nvme, crc64NvmeCrtContainer } from "@aws-sdk/crc64-nvme"; import { ChecksumConstructor, HashConstructor } from "@smithy/types"; import { PreviouslyResolved } from "./configuration"; @@ -22,13 +22,7 @@ export const selectChecksumAlgorithmFunction = ( return AwsCrc32c; case ChecksumAlgorithm.CRC64NVME: if (typeof crc64NvmeCrtContainer.CrtCrc64Nvme !== "function") { - throw new Error( - `Please check whether you have installed the "@aws-sdk/crc64-nvme-crt" package explicitly. \n` + - `You must also register the package by calling [require("@aws-sdk/crc64-nvme-crt");] ` + - `or an ESM equivalent such as [import "@aws-sdk/crc64-nvme-crt";]. \n` + - "For more information please go to " + - "https://github.com/aws/aws-sdk-js-v3#functionality-requiring-aws-common-runtime-crt" - ); + return Crc64Nvme; } return crc64NvmeCrtContainer.CrtCrc64Nvme; case ChecksumAlgorithm.SHA1: From 0883b0c12d2bd93e1e6d78cee3e8382975c67c68 Mon Sep 17 00:00:00 2001 From: Trivikram Kamat <16024985+trivikr@users.noreply.github.com> Date: Mon, 22 Dec 2025 05:14:08 +0000 Subject: [PATCH 3/6] test(middleware-flexible-checksums): move crt crcr64nvme to different file --- ...eware-flexible-checksums.crt.integ.spec.ts | 145 ++++++++++++++++++ ...iddleware-flexible-checksums.integ.spec.ts | 3 - 2 files changed, 145 insertions(+), 3 deletions(-) create mode 100644 packages/middleware-flexible-checksums/src/middleware-flexible-checksums.crt.integ.spec.ts diff --git a/packages/middleware-flexible-checksums/src/middleware-flexible-checksums.crt.integ.spec.ts b/packages/middleware-flexible-checksums/src/middleware-flexible-checksums.crt.integ.spec.ts new file mode 100644 index 000000000000..7c7333307e8f --- /dev/null +++ b/packages/middleware-flexible-checksums/src/middleware-flexible-checksums.crt.integ.spec.ts @@ -0,0 +1,145 @@ +// Testing CRT implementation of CRC64NVME +import "@aws-sdk/crc64-nvme-crt"; + +import { requireRequestsFrom } from "@aws-sdk/aws-util-test/src"; +import { ChecksumAlgorithm, S3 } from "@aws-sdk/client-s3"; +import { HttpHandler, HttpRequest, HttpResponse } from "@smithy/protocol-http"; +import { Readable, Transform } from "stream"; +import { describe, expect, test as it } from "vitest"; + +import { RequestChecksumCalculation, ResponseChecksumValidation } from "./constants"; + +describe("middleware-flexible-checksums", () => { + const logger = { + trace() {}, + debug() {}, + info() {}, + warn() {}, + error() {}, + }; + + const testCases: [string, string][] = [ + ["", "AAAAAAAAAAA="], + ["abc", "BeXKuz/B+us="], + ["Hello world", "OOJZ0D8xKts="], + ]; + + describe(S3.name, () => { + describe("putObject", () => { + describe.each([undefined, RequestChecksumCalculation.WHEN_SUPPORTED, RequestChecksumCalculation.WHEN_REQUIRED])( + `when requestChecksumCalculation='%s'`, + (requestChecksumCalculation) => { + testCases.forEach(([body, checksumValue]) => { + const client = new S3({ region: "us-west-2", logger, requestChecksumCalculation }); + const checksumHeader = `x-amz-checksum-${ChecksumAlgorithm.CRC64NVME.toLowerCase()}`; + + it(`tests ${checksumHeader}="${checksumValue}"" for checksum="${ChecksumAlgorithm.CRC64NVME}"`, async () => { + requireRequestsFrom(client).toMatch({ + method: "PUT", + hostname: "s3.us-west-2.amazonaws.com", + protocol: "https:", + path: "/b/k", + headers: { + "content-type": "application/octet-stream", + ...(body.length + ? { + "content-length": body.length.toString(), + } + : {}), + "x-amz-sdk-checksum-algorithm": ChecksumAlgorithm.CRC64NVME, + [checksumHeader]: checksumValue, + host: "s3.us-west-2.amazonaws.com", + "x-amz-user-agent": /./, + "user-agent": /./, + "amz-sdk-invocation-id": /./, + "amz-sdk-request": /./, + "x-amz-date": /./, + "x-amz-content-sha256": /./, + authorization: /./, + }, + query: { + "x-id": "PutObject", + }, + }); + + await client.putObject({ + Bucket: "b", + Key: "k", + Body: body, + ChecksumAlgorithm: ChecksumAlgorithm.CRC64NVME, + }); + + expect.hasAssertions(); + }); + }); + } + ); + }); + + describe("getObject", () => { + describe.each([undefined, ResponseChecksumValidation.WHEN_SUPPORTED, ResponseChecksumValidation.WHEN_REQUIRED])( + `when responseChecksumValidation='%s'`, + (responseChecksumValidation) => { + testCases.forEach(([body, checksumValue]) => { + const checksumHeader = `x-amz-checksum-${ChecksumAlgorithm.CRC64NVME.toLowerCase()}`; + + it(`validates ${checksumHeader}="${checksumValue}"" for checksum="${ChecksumAlgorithm.CRC64NVME}"`, async () => { + const client = new S3({ + region: "us-west-2", + logger, + requestHandler: new (class implements HttpHandler { + async handle(request: HttpRequest): Promise { + expect(request).toMatchObject({ + method: "GET", + hostname: "s3.us-west-2.amazonaws.com", + protocol: "https:", + path: "/b/k", + headers: { + "x-amz-checksum-mode": "ENABLED", + host: "s3.us-west-2.amazonaws.com", + "x-amz-user-agent": /./, + "user-agent": /./, + "amz-sdk-invocation-id": /./, + "amz-sdk-request": /./, + "x-amz-date": /./, + "x-amz-content-sha256": /./, + authorization: /./, + }, + query: { + "x-id": "GetObject", + }, + }); + return { + response: new HttpResponse({ + statusCode: 200, + headers: { + "content-type": "application/octet-stream", + "content-length": body.length.toString(), + [checksumHeader]: checksumValue, + }, + body: Readable.from([body]), + }), + }; + } + updateHttpClientConfig(key: never, value: never): void {} + httpHandlerConfigs() { + return {}; + } + })(), + responseChecksumValidation, + }); + + const response = await client.getObject({ + Bucket: "b", + Key: "k", + ChecksumMode: "ENABLED", + }); + + await expect(response.Body?.transformToString()).resolves.toEqual(body); + }); + }); + } + ); + }); + }); +}); diff --git a/packages/middleware-flexible-checksums/src/middleware-flexible-checksums.integ.spec.ts b/packages/middleware-flexible-checksums/src/middleware-flexible-checksums.integ.spec.ts index 50d5f73226a8..f59ec852c56c 100644 --- a/packages/middleware-flexible-checksums/src/middleware-flexible-checksums.integ.spec.ts +++ b/packages/middleware-flexible-checksums/src/middleware-flexible-checksums.integ.spec.ts @@ -1,6 +1,3 @@ -// Required for testing CRC64NVME -import "@aws-sdk/crc64-nvme-crt"; - import { requireRequestsFrom } from "@aws-sdk/aws-util-test/src"; import { ChecksumAlgorithm, S3 } from "@aws-sdk/client-s3"; import { HttpHandler, HttpRequest, HttpResponse } from "@smithy/protocol-http"; From a8f15bbd214327a41802a650dcd5846be3e310fc Mon Sep 17 00:00:00 2001 From: Trivikram Kamat <16024985+trivikr@users.noreply.github.com> Date: Mon, 22 Dec 2025 07:16:13 +0000 Subject: [PATCH 4/6] test: add E2E tests for default cchecksums computation --- .../middleware-flexible-checksums.e2e.spec.ts | 42 ++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/packages/middleware-flexible-checksums/src/middleware-flexible-checksums.e2e.spec.ts b/packages/middleware-flexible-checksums/src/middleware-flexible-checksums.e2e.spec.ts index 9e52d72e225a..d87faa92458c 100644 --- a/packages/middleware-flexible-checksums/src/middleware-flexible-checksums.e2e.spec.ts +++ b/packages/middleware-flexible-checksums/src/middleware-flexible-checksums.e2e.spec.ts @@ -1,5 +1,5 @@ import { getE2eTestResources } from "@aws-sdk/aws-util-test/src"; -import { S3, UploadPartCommandOutput } from "@aws-sdk/client-s3"; +import { ChecksumAlgorithm, S3, UploadPartCommandOutput } from "@aws-sdk/client-s3"; import { Upload } from "@aws-sdk/lib-storage"; import { FetchHttpHandler } from "@smithy/fetch-http-handler"; import type { HttpRequest, HttpResponse } from "@smithy/types"; @@ -108,6 +108,7 @@ describe("S3 checksums", () => { }); expect.hasAssertions(); }); + it("should assist user input streams by buffering to the minimum 8kb required by S3", async () => { await s3.putObject({ Bucket, @@ -124,6 +125,7 @@ describe("S3 checksums", () => { }); expect((await get.Body?.transformToByteArray())?.byteLength).toEqual(24 * 1024); }); + it("should be able to write an object with a webstream body (using fetch handler without checksum)", async () => { const handler = s3_noChecksum.config.requestHandler; s3_noChecksum.config.requestHandler = new FetchHttpHandler(); @@ -140,6 +142,7 @@ describe("S3 checksums", () => { }); expect((await get.Body?.transformToByteArray())?.byteLength).toEqual(24 * 1024); }); + it("@aws-sdk/lib-storage Upload should allow webstreams to be used", async () => { await new Upload({ client: s3, @@ -155,6 +158,7 @@ describe("S3 checksums", () => { }); expect((await get.Body?.transformToByteArray())?.byteLength).toEqual(6 * 1024 * 1024); }); + it("should allow streams to be used in a manually orchestrated MPU", async () => { const cmpu = await s3.createMultipartUpload({ Bucket, @@ -252,4 +256,40 @@ describe("S3 checksums", () => { await expect(checksumStream.getReader().closed).resolves.toBe(undefined); }); }); + + describe("Checksum computation", () => { + it.each([ + ["Hello world", ChecksumAlgorithm.CRC32, "i9aeUg=="], + ["Hello world", ChecksumAlgorithm.CRC32C, "crUfeA=="], + ["Hello world", ChecksumAlgorithm.CRC64NVME, "OOJZ0D8xKts="], + ["Hello world", ChecksumAlgorithm.SHA1, "e1AsOh9IyGCa4hLN+2Od7jlnP14="], + ["Hello world", ChecksumAlgorithm.SHA256, "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw="], + ])( + "when body is '%s' with checksum algorithm '%s' the checksum is '%s'", + async (body, checksumAlgorithm, checksumValue) => { + const client = new S3(); + client.middlewareStack.addRelativeTo( + (next: any) => async (args: any) => { + const request = args.request as HttpRequest; + expect(request.headers["x-amz-sdk-checksum-algorithm"]).toEqual(checksumAlgorithm); + expect(request.headers[`x-amz-checksum-${checksumAlgorithm.toLowerCase()}`]).toEqual(checksumValue); + return next(args); + }, + { + relation: "after", + toMiddleware: "flexibleChecksumsMiddleware", + } + ); + + await client.putObject({ + Bucket, + Key: Key + "-checksum-" + checksumAlgorithm, + Body: body, + ChecksumAlgorithm: checksumAlgorithm, + }); + + expect.assertions(2); + } + ); + }); }, 60_000); From c0bdcf7f74d01e44c99cae492ecba42e29932890 Mon Sep 17 00:00:00 2001 From: Trivikram Kamat <16024985+trivikr@users.noreply.github.com> Date: Mon, 22 Dec 2025 16:22:03 +0000 Subject: [PATCH 5/6] docs: remove need of CRT for CRC64NVME --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a448c314c4c3..4b8a8b27149b 100644 --- a/README.md +++ b/README.md @@ -665,7 +665,6 @@ bindings to be included as a dependency with your application. This functionalit - [Amazon S3 Multi-Region Access Points](https://docs.aws.amazon.com/AmazonS3/latest/userguide/MultiRegionAccessPoints.html) - [Amazon S3 Object Integrity](https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html) - [Amazon CloudFront-KeyValueStore](https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/kvs-with-functions-kvp.html) -- [Amazon S3 CRC64-NVME checksums](https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html) If the required AWS Common Runtime components are not installed, you will receive an error like: @@ -682,6 +681,10 @@ https://github.com/aws/aws-sdk-js-v3#functionality-requiring-aws-common-runtime- indicating that the required dependency is missing to use the associated functionality. To install this dependency, follow the provided [instructions](#installing-the-aws-common-runtime-crt-dependency). +For [Amazon S3 CRC64-NVME checksums](https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html), +we include JavaScript implementation by default, but you can optionally install and include `"@aws-sdk/crc64-nvme-crt"` +on Node.js runtime for improved performance. + #### Installing the AWS Common Runtime (CRT) Dependency You can install the CRT dependency with different commands depending on the package management tool you are using. From 85fa1e9151da95c1e5ddb865a86d34ac0ec2f6ab Mon Sep 17 00:00:00 2001 From: Trivikram Kamat <16024985+trivikr@users.noreply.github.com> Date: Mon, 22 Dec 2025 17:39:19 +0000 Subject: [PATCH 6/6] chore: remove crt dependency from nvme --- packages/crc64-nvme/package.json | 3 --- yarn.lock | 2 -- 2 files changed, 5 deletions(-) diff --git a/packages/crc64-nvme/package.json b/packages/crc64-nvme/package.json index 14607961bcbc..63280695ce1a 100644 --- a/packages/crc64-nvme/package.json +++ b/packages/crc64-nvme/package.json @@ -35,9 +35,6 @@ "rimraf": "3.0.2", "typescript": "~5.8.3" }, - "peerDependencies": { - "@aws-sdk/crc64-nvme-crt": "*" - }, "engines": { "node": ">=18.0.0" }, diff --git a/yarn.lock b/yarn.lock index 43ec3531661b..0ba411438896 100644 --- a/yarn.lock +++ b/yarn.lock @@ -23451,8 +23451,6 @@ __metadata: rimraf: "npm:3.0.2" tslib: "npm:^2.6.2" typescript: "npm:~5.8.3" - peerDependencies: - "@aws-sdk/crc64-nvme-crt": "*" languageName: unknown linkType: soft