feat: streaming sha256 CAR hash (#162)

Author: Alan Shaw

cmd/hash.js

@@ -1,28 +1,26 @@
 import fs from 'fs'
+import crypto from 'crypto'
 import { pipeline } from 'stream/promises'
 import { CID } from 'multiformats/cid'
+import * as Digest from 'multiformats/hashes/digest'
 import { sha256 } from 'multiformats/hashes/sha2'
 
 /** CAR CID code */
 const carCode = 0x0202
 
 /** @param {string} carPath */
 export default async function hash (carPath) {
-  let bytes
-  if (carPath) {
-    bytes = await fs.promises.readFile(carPath)
-  } else {
-    bytes = await pipeline(
-      process.stdin,
-      async (source) => {
-        const chunks = []
-        for await (const chunk of source) {
-          chunks.push(chunk)
-        }
-        return Buffer.concat(chunks)
+  const hasher = crypto.createHash('sha256')
+
+  await pipeline(
+    carPath ? fs.createReadStream(carPath) : process.stdin,
+    async (source) => {
+      for await (const chunk of source) {
+        hasher.update(chunk)
       }
-    )
-  }
+    }
+  )
 
-  console.log(CID.createV1(carCode, await sha256.digest(bytes)).toString())
+  const digest = Digest.create(sha256.code, hasher.digest())
+  console.log(CID.createV1(carCode, digest).toString())
 }

package-lock.json

@@ -36,11 +36,11 @@
     "@ipld/dag-cbor": "^9.0.0",
     "@ipld/dag-json": "^10.0.1",
     "@ipld/dag-pb": "^4.0.2",
-    "@ipld/unixfs": "^2.1.1",
+    "@ipld/unixfs": "^3.0.0",
     "@web3-storage/car-block-validator": "^1.0.1",
     "files-from-path": "^1.0.0",
     "ipfs-unixfs-exporter": "^13.0.1",
-    "multiformats": "^11.0.2",
+    "multiformats": "^13.0.1",
     "sade": "^1.8.1",
     "varint": "^6.0.0"
   },