feat(middleware-flexible-checksums): use CRC64NVME JS implementation if CRT is not available

README.md

@@ -665,7 +665,6 @@ bindings to be included as a dependency with your application. This functionalit
 - [Amazon S3 Multi-Region Access Points](https://docs.aws.amazon.com/AmazonS3/latest/userguide/MultiRegionAccessPoints.html)
 - [Amazon S3 Object Integrity](https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html)
 - [Amazon CloudFront-KeyValueStore](https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/kvs-with-functions-kvp.html)
-- [Amazon S3 CRC64-NVME checksums](https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html)
 
 If the required AWS Common Runtime components are not installed, you will receive an error like:
 
@@ -682,6 +681,10 @@ https://github.com/aws/aws-sdk-js-v3#functionality-requiring-aws-common-runtime-
 indicating that the required dependency is missing to use the associated functionality. To install this dependency, follow
 the provided [instructions](#installing-the-aws-common-runtime-crt-dependency).
 
+For [Amazon S3 CRC64-NVME checksums](https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html),
+we include JavaScript implementation by default, but you can optionally install and include `"@aws-sdk/crc64-nvme-crt"`
+on Node.js runtime for improved performance.
+
 #### Installing the AWS Common Runtime (CRT) Dependency
 
 You can install the CRT dependency with different commands depending on the package management tool you are using.

packages/middleware-flexible-checksums/src/middleware-flexible-checksums.crt.integ.spec.ts

@@ -0,0 +1,145 @@
+// Testing CRT implementation of CRC64NVME
+import "@aws-sdk/crc64-nvme-crt";
+
+import { requireRequestsFrom } from "@aws-sdk/aws-util-test/src";
+import { ChecksumAlgorithm, S3 } from "@aws-sdk/client-s3";
+import { HttpHandler, HttpRequest, HttpResponse } from "@smithy/protocol-http";
+import { Readable, Transform } from "stream";
+import { describe, expect, test as it } from "vitest";
+
+import { RequestChecksumCalculation, ResponseChecksumValidation } from "./constants";
+
+describe("middleware-flexible-checksums", () => {
+  const logger = {
+    trace() {},
+    debug() {},
+    info() {},
+    warn() {},
+    error() {},
+  };
+
+  const testCases: [string, string][] = [
+    ["", "AAAAAAAAAAA="],
+    ["abc", "BeXKuz/B+us="],
+    ["Hello world", "OOJZ0D8xKts="],
+  ];
+
+  describe(S3.name, () => {
+    describe("putObject", () => {
+      describe.each([undefined, RequestChecksumCalculation.WHEN_SUPPORTED, RequestChecksumCalculation.WHEN_REQUIRED])(
+        `when requestChecksumCalculation='%s'`,
+        (requestChecksumCalculation) => {
+          testCases.forEach(([body, checksumValue]) => {
+            const client = new S3({ region: "us-west-2", logger, requestChecksumCalculation });
+            const checksumHeader = `x-amz-checksum-${ChecksumAlgorithm.CRC64NVME.toLowerCase()}`;
+
+            it(`tests ${checksumHeader}="${checksumValue}"" for checksum="${ChecksumAlgorithm.CRC64NVME}"`, async () => {
+              requireRequestsFrom(client).toMatch({
+                method: "PUT",
+                hostname: "s3.us-west-2.amazonaws.com",
+                protocol: "https:",
+                path: "/b/k",
+                headers: {
+                  "content-type": "application/octet-stream",
+                  ...(body.length
+                    ? {
+                        "content-length": body.length.toString(),
+                      }
+                    : {}),
+                  "x-amz-sdk-checksum-algorithm": ChecksumAlgorithm.CRC64NVME,
+                  [checksumHeader]: checksumValue,
+                  host: "s3.us-west-2.amazonaws.com",
+                  "x-amz-user-agent": /./,
+                  "user-agent": /./,
+                  "amz-sdk-invocation-id": /./,
+                  "amz-sdk-request": /./,
+                  "x-amz-date": /./,
+                  "x-amz-content-sha256": /./,
+                  authorization: /./,
+                },
+                query: {
+                  "x-id": "PutObject",
+                },
+              });
+
+              await client.putObject({
+                Bucket: "b",
+                Key: "k",
+                Body: body,
+                ChecksumAlgorithm: ChecksumAlgorithm.CRC64NVME,
+              });
+
+              expect.hasAssertions();
+            });
+          });
+        }
+      );
+    });
+
+    describe("getObject", () => {
+      describe.each([undefined, ResponseChecksumValidation.WHEN_SUPPORTED, ResponseChecksumValidation.WHEN_REQUIRED])(
+        `when responseChecksumValidation='%s'`,
+        (responseChecksumValidation) => {
+          testCases.forEach(([body, checksumValue]) => {
+            const checksumHeader = `x-amz-checksum-${ChecksumAlgorithm.CRC64NVME.toLowerCase()}`;
+
+            it(`validates ${checksumHeader}="${checksumValue}"" for checksum="${ChecksumAlgorithm.CRC64NVME}"`, async () => {
+              const client = new S3({
+                region: "us-west-2",
+                logger,
+                requestHandler: new (class implements HttpHandler {
+                  async handle(request: HttpRequest): Promise<any> {
+                    expect(request).toMatchObject({
+                      method: "GET",
+                      hostname: "s3.us-west-2.amazonaws.com",
+                      protocol: "https:",
+                      path: "/b/k",
+                      headers: {
+                        "x-amz-checksum-mode": "ENABLED",
+                        host: "s3.us-west-2.amazonaws.com",
+                        "x-amz-user-agent": /./,
+                        "user-agent": /./,
+                        "amz-sdk-invocation-id": /./,
+                        "amz-sdk-request": /./,
+                        "x-amz-date": /./,
+                        "x-amz-content-sha256": /./,
+                        authorization: /./,
+                      },
+                      query: {
+                        "x-id": "GetObject",
+                      },
+                    });
+                    return {
+                      response: new HttpResponse({
+                        statusCode: 200,
+                        headers: {
+                          "content-type": "application/octet-stream",
+                          "content-length": body.length.toString(),
+                          [checksumHeader]: checksumValue,
+                        },
+                        body: Readable.from([body]),
+                      }),
+                    };
+                  }
+                  updateHttpClientConfig(key: never, value: never): void {}
+                  httpHandlerConfigs() {
+                    return {};
+                  }
+                })(),
+                responseChecksumValidation,
+              });
+
+              const response = await client.getObject({
+                Bucket: "b",
+                Key: "k",
+                ChecksumMode: "ENABLED",
+              });
+
+              await expect(response.Body?.transformToString()).resolves.toEqual(body);
+            });
+          });
+        }
+      );
+    });
+  });
+});

packages/middleware-flexible-checksums/src/middleware-flexible-checksums.e2e.spec.ts

@@ -1,5 +1,5 @@
 import { getE2eTestResources } from "@aws-sdk/aws-util-test/src";
-import { S3, UploadPartCommandOutput } from "@aws-sdk/client-s3";
+import { ChecksumAlgorithm, S3, UploadPartCommandOutput } from "@aws-sdk/client-s3";
 import { Upload } from "@aws-sdk/lib-storage";
 import { FetchHttpHandler } from "@smithy/fetch-http-handler";
 import type { HttpRequest, HttpResponse } from "@smithy/types";
@@ -108,6 +108,7 @@ describe("S3 checksums", () => {
         });
       expect.hasAssertions();
     });
+
     it("should assist user input streams by buffering to the minimum 8kb required by S3", async () => {
       await s3.putObject({
         Bucket,
@@ -124,6 +125,7 @@ describe("S3 checksums", () => {
       });
       expect((await get.Body?.transformToByteArray())?.byteLength).toEqual(24 * 1024);
     });
+
     it("should be able to write an object with a webstream body (using fetch handler without checksum)", async () => {
       const handler = s3_noChecksum.config.requestHandler;
       s3_noChecksum.config.requestHandler = new FetchHttpHandler();
@@ -140,6 +142,7 @@ describe("S3 checksums", () => {
       });
       expect((await get.Body?.transformToByteArray())?.byteLength).toEqual(24 * 1024);
     });
+
     it("@aws-sdk/lib-storage Upload should allow webstreams to be used", async () => {
       await new Upload({
         client: s3,
@@ -155,6 +158,7 @@ describe("S3 checksums", () => {
       });
       expect((await get.Body?.transformToByteArray())?.byteLength).toEqual(6 * 1024 * 1024);
     });
+
     it("should allow streams to be used in a manually orchestrated MPU", async () => {
       const cmpu = await s3.createMultipartUpload({
         Bucket,
@@ -252,4 +256,40 @@ describe("S3 checksums", () => {
       await expect(checksumStream.getReader().closed).resolves.toBe(undefined);
     });
   });
+
+  describe("Checksum computation", () => {
+    it.each([
+      ["Hello world", ChecksumAlgorithm.CRC32, "i9aeUg=="],
+      ["Hello world", ChecksumAlgorithm.CRC32C, "crUfeA=="],
+      ["Hello world", ChecksumAlgorithm.CRC64NVME, "OOJZ0D8xKts="],
+      ["Hello world", ChecksumAlgorithm.SHA1, "e1AsOh9IyGCa4hLN+2Od7jlnP14="],
+      ["Hello world", ChecksumAlgorithm.SHA256, "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw="],
+    ])(
+      "when body is '%s' with checksum algorithm '%s' the checksum is '%s'",
+      async (body, checksumAlgorithm, checksumValue) => {
+        const client = new S3();
+        client.middlewareStack.addRelativeTo(
+          (next: any) => async (args: any) => {
+            const request = args.request as HttpRequest;
+            expect(request.headers["x-amz-sdk-checksum-algorithm"]).toEqual(checksumAlgorithm);
+            expect(request.headers[`x-amz-checksum-${checksumAlgorithm.toLowerCase()}`]).toEqual(checksumValue);
+            return next(args);
+          },
+          {
+            relation: "after",
+            toMiddleware: "flexibleChecksumsMiddleware",
+          }
+        );
+
+        await client.putObject({
+          Bucket,
+          Key: Key + "-checksum-" + checksumAlgorithm,
+          Body: body,
+          ChecksumAlgorithm: checksumAlgorithm,
+        });
+
+        expect.assertions(2);
+      }
+    );
+  });
 }, 60_000);

packages/middleware-flexible-checksums/src/middleware-flexible-checksums.integ.spec.ts

@@ -1,6 +1,3 @@
-// Required for testing CRC64NVME
-import "@aws-sdk/crc64-nvme-crt";
-
 import { requireRequestsFrom } from "@aws-sdk/aws-util-test/src";
 import { ChecksumAlgorithm, S3 } from "@aws-sdk/client-s3";
 import { HttpHandler, HttpRequest, HttpResponse } from "@smithy/protocol-http";

packages/middleware-flexible-checksums/src/selectChecksumAlgorithmFunction.spec.ts

@@ -1,5 +1,5 @@
 import { AwsCrc32c } from "@aws-crypto/crc32c";
-import { crc64NvmeCrtContainer } from "@aws-sdk/crc64-nvme";
+import { Crc64Nvme, crc64NvmeCrtContainer } from "@aws-sdk/crc64-nvme";
 import { describe, expect, test as it, vi } from "vitest";
 
 import { ChecksumAlgorithm } from "./constants";
@@ -27,8 +27,9 @@ describe(selectChecksumAlgorithmFunction.name, () => {
     expect(() => selectChecksumAlgorithmFunction("UNSUPPORTED" as any, mockConfig as any)).toThrow();
   });
 
-  it("throws error if crc64NvmeCrtContainer.CrtCrc64Nvme is not a function", () => {
-    expect(() => selectChecksumAlgorithmFunction(ChecksumAlgorithm.CRC64NVME, mockConfig as any)).toThrow();
+  it("returns Crc64Nvme if crc64NvmeCrtContainer.CrtCrc64Nvme is not a function", () => {
+    crc64NvmeCrtContainer.CrtCrc64Nvme = null;
+    expect(selectChecksumAlgorithmFunction(ChecksumAlgorithm.CRC64NVME, mockConfig as any)).toBe(Crc64Nvme);
   });
 
   it("returns crc64NvmeCrtContainer.CrtCrc64Nvme if available from container", () => {

packages/middleware-flexible-checksums/src/selectChecksumAlgorithmFunction.ts

@@ -1,5 +1,5 @@
 import { AwsCrc32c } from "@aws-crypto/crc32c";
-import { crc64NvmeCrtContainer } from "@aws-sdk/crc64-nvme";
+import { Crc64Nvme, crc64NvmeCrtContainer } from "@aws-sdk/crc64-nvme";
 import { ChecksumConstructor, HashConstructor } from "@smithy/types";
 
 import { PreviouslyResolved } from "./configuration";
@@ -22,13 +22,7 @@ export const selectChecksumAlgorithmFunction = (
       return AwsCrc32c;
     case ChecksumAlgorithm.CRC64NVME:
       if (typeof crc64NvmeCrtContainer.CrtCrc64Nvme !== "function") {
-        throw new Error(
-          `Please check whether you have installed the "@aws-sdk/crc64-nvme-crt" package explicitly. \n` +
-            `You must also register the package by calling [require("@aws-sdk/crc64-nvme-crt");] ` +
-            `or an ESM equivalent such as [import "@aws-sdk/crc64-nvme-crt";]. \n` +
-            "For more information please go to " +
-            "https://github.com/aws/aws-sdk-js-v3#functionality-requiring-aws-common-runtime-crt"
-        );
+        return Crc64Nvme;
       }
       return crc64NvmeCrtContainer.CrtCrc64Nvme;
     case ChecksumAlgorithm.SHA1: