Support for compressed base64 wasm files

Upgrade to use WebAssembly.instantiateStreaming

Author: mainnet-pat

.eslintrc

@@ -7,7 +7,8 @@
     "node_modules",
     "build",
     "coverage",
-    "src/lib/schema/ajv/*.js"
+    "src/lib/schema/ajv/*.js",
+    "src/lib/bin/secp256k1/secp256k1.js"
   ],
   "extends": ["bitauth"],
   // "globals": { "BigInt": true, "console": true, "WebAssembly": true },

src/lib/bin/hashes.ts

@@ -1,10 +1,47 @@
+import { binsAreEqual } from '../format/hex.js';
+
 export type HashFunction = {
   final: (rawState: Uint8Array) => Uint8Array;
   hash: (input: Uint8Array) => Uint8Array;
   init: () => Uint8Array;
   update: (rawState: Uint8Array, input: Uint8Array) => Uint8Array;
 };
 
+/* eslint-disable @typescript-eslint/require-await, functional/no-return-void, functional/no-expression-statements, @typescript-eslint/no-magic-numbers, @typescript-eslint/naming-convention */
+/**
+ * Reads in a wasm binary as an ArrayBuffer, checks if it was compressed and
+ * decompresses it if necessary. Returns a Response object with the wasm binary compatible with WebAssembly.instantiateStreaming.
+ */
+export const streamWasmArrayBuffer = async (
+  wasmArrayBuffer: ArrayBuffer,
+): Promise<Response> => {
+  // currently, we consume the data in a single chunk, but when ReadableStream.from() becomes widely available, we can switch to it
+  const wasmStream = new ReadableStream({
+    start(controller): void {
+      controller.enqueue(new Uint8Array(wasmArrayBuffer));
+      controller.close();
+    },
+  });
+
+  // if source is uncompressed, then return as is
+  if (
+    binsAreEqual(
+      new Uint8Array(wasmArrayBuffer, 0, 4),
+      new Uint8Array([0x00, 0x61, 0x73, 0x6d]),
+    )
+  ) {
+    return new Response(wasmStream, {
+      headers: new Headers({ 'content-type': 'application/wasm' }),
+    });
+  }
+
+  // otherwise, decompress the source
+  return new Response(wasmStream.pipeThrough(new DecompressionStream('gzip')), {
+    headers: new Headers({ 'content-type': 'application/wasm' }),
+  });
+};
+/* eslint-enable @typescript-eslint/require-await, functional/no-return-void, functional/no-expression-statements, @typescript-eslint/no-magic-numbers, @typescript-eslint/naming-convention */
+
 /* eslint-disable functional/no-conditional-statements, functional/no-let, functional/no-expression-statements, no-underscore-dangle, functional/no-try-statements, @typescript-eslint/no-magic-numbers, @typescript-eslint/max-params, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-non-null-assertion */
 /**
  * Note, most of this method is translated and boiled-down from the wasm-pack
@@ -19,8 +56,9 @@ export const instantiateRustWasm = async (
   updateExportName: string,
   finalExportName: string,
 ): Promise<HashFunction> => {
+  const webassemblyByteStream = await streamWasmArrayBuffer(webassemblyBytes);
   const wasm = (
-    await WebAssembly.instantiate(webassemblyBytes, {
+    await WebAssembly.instantiateStreaming(webassemblyByteStream, {
       [expectedImportModuleName]: {
         /**
          * This would only be called in cases where a `__wbindgen_malloc` failed.

src/lib/bin/ripemd160/ripemd160.base64.ts

@@ -12,6 +12,7 @@ import {
   getEmbeddedSecp256k1Binary,
   instantiateSecp256k1Wasm,
   instantiateSecp256k1WasmBytes,
+  streamWasmArrayBuffer,
 } from '../../lib.js';
 
 // test vectors from `zoo zoo zoo zoo zoo zoo zoo zoo zoo zoo zoo wrong` (`xprv9s21ZrQH143K2PfMvkNViFc1fgumGqBew45JD8SxA59Jc5M66n3diqb92JjvaR61zT9P89Grys12kdtV4EFVo6tMwER7U2hcUmZ9VfMYPLC`), m/0 and m/1:
@@ -491,10 +492,35 @@ const testSecp256k1Wasm = (
 
 const binary = getEmbeddedSecp256k1Binary();
 
-test('[crypto] getEmbeddedSecp256k1Binary returns the proper binary', (t) => {
+test('[crypto] getEmbeddedSecp256k1Binary returns the proper binary', async (t) => {
   const path = join(new URL('.', import.meta.url).pathname, 'secp256k1.wasm');
   const binaryFromDisk = readFileSync(path).buffer;
-  t.deepEqual(binary, binaryFromDisk);
+  const eventuallyDecompressedBinary = await (
+    await streamWasmArrayBuffer(binary)
+  ).arrayBuffer();
+  t.deepEqual(eventuallyDecompressedBinary, binaryFromDisk);
+
+  // compress the wasm binary
+  const stream = new ReadableStream({
+    start(controller): void {
+      controller.enqueue(new Uint8Array(binaryFromDisk));
+      controller.close();
+    },
+  }).pipeThrough(new CompressionStream('gzip'));
+
+  const compressedBinary = await new Response(stream).arrayBuffer();
+
+  // decompress the compressed wasm binary
+  const decompressedBinary = await (
+    await streamWasmArrayBuffer(compressedBinary)
+  ).arrayBuffer();
+  t.deepEqual(decompressedBinary, binaryFromDisk);
+
+  // check that the uncompressed binary produced by `streamWasmArrayBuffer` is the same as the binary from disk
+  const uncompressedBinary = await (
+    await streamWasmArrayBuffer(binaryFromDisk)
+  ).arrayBuffer();
+  t.deepEqual(uncompressedBinary, binaryFromDisk);
 });
 
 test('[crypto] Secp256k1Wasm instantiated with embedded binary', async (t) => {

src/lib/bin/secp256k1/secp256k1-wasm.spec.ts

@@ -1,6 +1,7 @@
 /* eslint-disable no-underscore-dangle, @typescript-eslint/max-params, @typescript-eslint/naming-convention */
 // cSpell:ignore memcpy, anyfunc
 import { base64ToBin } from '../../format/format.js';
+import { streamWasmArrayBuffer } from '../hashes.js';
 
 import type { Secp256k1Wasm } from './secp256k1-wasm-types.js';
 import { CompressionFlag, ContextFlag } from './secp256k1-wasm-types.js';
@@ -352,11 +353,15 @@ export const instantiateSecp256k1WasmBytes = async (
     global: { Infinity, NaN },
   };
 
-  return WebAssembly.instantiate(webassemblyBytes, info).then((result) => {
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-assignment
-    getErrNoLocation = result.instance.exports['___errno_location'] as any;
-    return wrapSecp256k1Wasm(result.instance, heapU8, heapU32);
-  });
+  const webassemblyByteStream = await streamWasmArrayBuffer(webassemblyBytes);
+
+  return WebAssembly.instantiateStreaming(webassemblyByteStream, info).then(
+    (result) => {
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-assignment
+      getErrNoLocation = result.instance.exports['___errno_location'] as any;
+      return wrapSecp256k1Wasm(result.instance, heapU8, heapU32);
+    },
+  );
 };
 /* eslint-enable functional/immutable-data, functional/no-expression-statements, @typescript-eslint/no-magic-numbers, functional/no-conditional-statements, no-bitwise, functional/no-throw-statements */
 

src/lib/bin/secp256k1/secp256k1-wasm.ts

@@ -7,7 +7,7 @@ import { join } from 'path';
 import test from 'ava';
 
 import type { HashFunction } from '../lib.js';
-import { utf8ToBin } from '../lib.js';
+import { streamWasmArrayBuffer, utf8ToBin } from '../lib.js';
 
 import fc from 'fast-check';
 import hashJs from 'hash.js';
@@ -43,7 +43,7 @@ export const testHashFunction = <T extends HashFunction>({
   nodeJsAlgorithm: 'ripemd160' | 'sha1' | 'sha256' | 'sha512';
 }) => {
   const binary = getEmbeddedBinary();
-  test(`[crypto] ${hashFunctionName} getEmbeddedBinary returns the proper binary`, (t) => {
+  test(`[crypto] ${hashFunctionName} getEmbeddedBinary returns the proper binary`, async (t) => {
     const path = join(
       new URL('.', import.meta.url).pathname,
       '..',
@@ -52,7 +52,10 @@ export const testHashFunction = <T extends HashFunction>({
       `${hashFunctionName}.wasm`,
     );
     const binaryFromDisk = readFileSync(path).buffer;
-    t.deepEqual(binary, binaryFromDisk);
+    const eventuallyDecompressedBinary = await (
+      await streamWasmArrayBuffer(binary)
+    ).arrayBuffer();
+    t.deepEqual(eventuallyDecompressedBinary, binaryFromDisk);
   });
 
   test(`[crypto] ${hashFunctionName} instantiated with embedded binary`, async (t) => {

src/lib/bin/secp256k1/secp256k1.base64.ts

@@ -22,7 +22,7 @@ RUN /binaryen/bin/wasm-opt -O3 pkg/ripemd160_bg.wasm -o pkg/ripemd160.wasm
 RUN cp pkg/ripemd160.wasm out/ripemd160
 RUN cp pkg/ripemd160.d.ts out/ripemd160
 RUN cp pkg/ripemd160.js out/ripemd160
-RUN OUTPUT_TS_FILE=out/ripemd160/ripemd160.base64.ts; printf "/**\n * @hidden\n */\n// prettier-ignore\nexport const ripemd160Base64Bytes =\n  '" > $OUTPUT_TS_FILE && base64 -w 0 pkg/ripemd160.wasm >> $OUTPUT_TS_FILE && printf "';\n" >> $OUTPUT_TS_FILE
+RUN OUTPUT_TS_FILE=out/ripemd160/ripemd160.base64.ts; printf "/**\n * @hidden\n */\n// prettier-ignore\nexport const ripemd160Base64Bytes =\n  '" > $OUTPUT_TS_FILE && cat pkg/ripemd160.wasm | gzip | base64 -w 0 >> $OUTPUT_TS_FILE && printf "';\n" >> $OUTPUT_TS_FILE
 RUN cp -r /libauth/wasm/hashes/ripemd160/out/ripemd160 /libauth/bin
 
 # sha256
@@ -34,7 +34,7 @@ RUN /binaryen/bin/wasm-opt -O3 pkg/sha256_bg.wasm -o pkg/sha256.wasm
 RUN cp pkg/sha256.wasm out/sha256
 RUN cp pkg/sha256.d.ts out/sha256
 RUN cp pkg/sha256.js out/sha256
-RUN OUTPUT_TS_FILE=out/sha256/sha256.base64.ts; printf "/**\n * @hidden\n */\n// prettier-ignore\nexport const sha256Base64Bytes =\n  '" > $OUTPUT_TS_FILE && base64 -w 0 pkg/sha256.wasm >> $OUTPUT_TS_FILE && printf "';\n" >> $OUTPUT_TS_FILE
+RUN OUTPUT_TS_FILE=out/sha256/sha256.base64.ts; printf "/**\n * @hidden\n */\n// prettier-ignore\nexport const sha256Base64Bytes =\n  '" > $OUTPUT_TS_FILE && cat pkg/sha256.wasm | gzip | base64 -w 0 >> $OUTPUT_TS_FILE && printf "';\n" >> $OUTPUT_TS_FILE
 RUN cp -r /libauth/wasm/hashes/sha256/out/sha256 /libauth/bin
 
 # sha512
@@ -46,7 +46,7 @@ RUN /binaryen/bin/wasm-opt -O3 pkg/sha512_bg.wasm -o pkg/sha512.wasm
 RUN cp pkg/sha512.wasm out/sha512
 RUN cp pkg/sha512.d.ts out/sha512
 RUN cp pkg/sha512.js out/sha512
-RUN OUTPUT_TS_FILE=out/sha512/sha512.base64.ts; printf "/**\n * @hidden\n */\n// prettier-ignore\nexport const sha512Base64Bytes =\n  '" > $OUTPUT_TS_FILE && base64 -w 0 pkg/sha512.wasm >> $OUTPUT_TS_FILE && printf "';\n" >> $OUTPUT_TS_FILE
+RUN OUTPUT_TS_FILE=out/sha512/sha512.base64.ts; printf "/**\n * @hidden\n */\n// prettier-ignore\nexport const sha512Base64Bytes =\n  '" > $OUTPUT_TS_FILE && cat pkg/sha512.wasm | gzip | base64 -w 0 >> $OUTPUT_TS_FILE && printf "';\n" >> $OUTPUT_TS_FILE
 RUN cp -r /libauth/wasm/hashes/sha512/out/sha512 /libauth/bin
 
 # sha1
@@ -58,7 +58,7 @@ RUN /binaryen/bin/wasm-opt -O3 pkg/sha1_bg.wasm -o pkg/sha1.wasm
 RUN cp pkg/sha1.wasm out/sha1
 RUN cp pkg/sha1.d.ts out/sha1
 RUN cp pkg/sha1.js out/sha1
-RUN OUTPUT_TS_FILE=out/sha1/sha1.base64.ts; printf "/**\n * @hidden\n */\n// prettier-ignore\nexport const sha1Base64Bytes =\n  '" > $OUTPUT_TS_FILE && base64 -w 0 pkg/sha1.wasm >> $OUTPUT_TS_FILE && printf "';\n" >> $OUTPUT_TS_FILE
+RUN OUTPUT_TS_FILE=out/sha1/sha1.base64.ts; printf "/**\n * @hidden\n */\n// prettier-ignore\nexport const sha1Base64Bytes =\n  '" > $OUTPUT_TS_FILE && cat pkg/sha1.wasm | gzip | base64 -w 0 >> $OUTPUT_TS_FILE && printf "';\n" >> $OUTPUT_TS_FILE
 RUN cp -r /libauth/wasm/hashes/sha1/out/sha1 /libauth/bin
 
 WORKDIR /libauth/wasm/hashes/

src/lib/bin/sha1/sha1.base64.ts

@@ -59,7 +59,7 @@ RUN emcc src/libsecp256k1_la-secp256k1.o \
   ]' \
   -o out/secp256k1/secp256k1.js
 
-RUN OUTPUT_TS_FILE=out/secp256k1/secp256k1.base64.ts; printf "/**\n * @hidden\n */\n// prettier-ignore\nexport const secp256k1Base64Bytes =\n  '" > $OUTPUT_TS_FILE && base64 -w 0 out/secp256k1/secp256k1.wasm >> $OUTPUT_TS_FILE && printf "';\n" >> $OUTPUT_TS_FILE
+RUN OUTPUT_TS_FILE=out/secp256k1/secp256k1.base64.ts; printf "/**\n * @hidden\n */\n// prettier-ignore\nexport const secp256k1Base64Bytes =\n  '" > $OUTPUT_TS_FILE && cat out/secp256k1/secp256k1.wasm | gzip | base64 -w 0 >> $OUTPUT_TS_FILE && printf "';\n" >> $OUTPUT_TS_FILE
 
 RUN cp -r /libauth/wasm/secp256k1/out /libauth/bin